From 8dca1822556879c84482c6c31a22de9eb840d6df Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 13 Sep 2021 11:59:31 +0200 Subject: [PATCH 01/30] CI scripted deployment of OAI CN5G --- ci-scripts/epc.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index 1c202438d2..69861b5202 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -220,7 +220,7 @@ class EPCManagement(): mySSH = SSH.SSHConnection() mySSH.open(self.IPAddress, self.UserName, self.Password) if re.match('ltebox', self.Type, re.IGNORECASE): - logging.debug('Using the sabox simulated HSS') + logging.debug('Using the SABOX simulated HSS') mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5) mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('cd /opt/hss_sim0609', '\$', 5) @@ -229,6 +229,13 @@ class EPCManagement(): logging.debug('Using the sabox') mySSH.command('cd /opt/ltebox/tools', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S ./start_sabox', '\$', 5) + elif re.match('OAI', self.Type, re.IGNORECASE): + logging.debug('Starting OAI CN5G') + mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5) + mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) + mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) + mySSH.command('echo ' + self.Password + ' | sudo -S ./core-network.sh start nrf spgwu', '\$', 5) + time.sleep(20) #leave some time for deployment and health check else: logging.error('This option should not occur!') mySSH.close() @@ -433,6 +440,7 @@ class EPCManagement(): mySSH = SSH.SSHConnection() mySSH.open(self.IPAddress, self.UserName, self.Password) if re.match('ltebox', self.Type, re.IGNORECASE): + logging.debug('Terminating SA BOX') mySSH.command('cd /opt/ltebox/tools', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S ./stop_sabox', '\$', 5) time.sleep(1) @@ -440,6 +448,10 @@ class EPCManagement(): mySSH.command('cd scripts', '\$', 5) time.sleep(1) mySSH.command('echo ' + self.Password + ' | sudo -S screen -S simulated_5g_hss -X quit', '\$', 5) + elif re.match('OAI', self.Type, re.IGNORECASE): + logging.debug('Terminating OAI CN5G') + mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) + mySSH.command('echo ' + self.Password + ' | sudo -S ./core-network.sh stop nrf spgwu', '\$', 5) else: logging.error('This should not happen!') mySSH.close() From 7a505f75d163604e884210c535078a7023a1900d Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 14 Sep 2021 17:28:28 +0200 Subject: [PATCH 02/30] adding cn logs to artifacts --- ci-scripts/Jenkinsfile-tmp-multi-enb-nsa | 15 ++++++++++++ ci-scripts/epc.py | 31 ++++++++++++++++++++---- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa index 1e4933838b..07dc1e73f6 100644 --- a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa +++ b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa @@ -262,6 +262,21 @@ pipeline { if(fileExists("enb.log.${env.BUILD_ID}.zip")) { archiveArtifacts "enb.log.${env.BUILD_ID}.zip" } + } + } + } + stage('Log Collection (CN)') { + steps { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'] + ]) { + echo '\u2705 \u001B[32mLog Transfer (CN)\u001B[0m' + sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/logs/oai-cn5g.log.zip ./oai-cn5g.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("oai-cn5g.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "oai-cn5g.log.${env.BUILD_ID}.zip" + } if(fileExists("ci-scripts/test_results.html")) { sh "mv ci-scripts/test_results.html test_results-${JOB_NAME}.html" sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: ${BUILD_ID}#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results-${JOB_NAME}.html" diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index 69861b5202..dead126fcd 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -229,13 +229,12 @@ class EPCManagement(): logging.debug('Using the sabox') mySSH.command('cd /opt/ltebox/tools', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S ./start_sabox', '\$', 5) - elif re.match('OAI', self.Type, re.IGNORECASE): + elif re.match('OAICN5G', self.Type, re.IGNORECASE): logging.debug('Starting OAI CN5G') mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5) mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) - mySSH.command('echo ' + self.Password + ' | sudo -S ./core-network.sh start nrf spgwu', '\$', 5) - time.sleep(20) #leave some time for deployment and health check + mySSH.command('./core-network.sh start nrf spgwu', '\$', 30) else: logging.error('This option should not occur!') mySSH.close() @@ -249,6 +248,14 @@ class EPCManagement(): return if re.match('ltebox', self.Type, re.IGNORECASE): self.MmeIPAddress = self.IPAddress + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + mySSH = SSH.SSHConnection() + mySSH.open(self.IPAddress, self.UserName, self.Password) + mySSH.command2('docker inspect --format=\'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' oai-amf', 5) + self.MmeIPAddress = str(mySSH.cmd2Results[0],'utf-8') + self.MmeIPAddress.rstrip() + logging.debug('AMF IP Address ' + self.MmeIPAddress) + mySSH.close() def CheckHSSProcess(self, status_queue): try: @@ -448,10 +455,11 @@ class EPCManagement(): mySSH.command('cd scripts', '\$', 5) time.sleep(1) mySSH.command('echo ' + self.Password + ' | sudo -S screen -S simulated_5g_hss -X quit', '\$', 5) - elif re.match('OAI', self.Type, re.IGNORECASE): + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + self.LogCollectOAICN5G() logging.debug('Terminating OAI CN5G') mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) - mySSH.command('echo ' + self.Password + ' | sudo -S ./core-network.sh stop nrf spgwu', '\$', 5) + mySSH.command('./core-network.sh stop nrf spgwu', '\$', 5) else: logging.error('This should not happen!') mySSH.close() @@ -691,3 +699,16 @@ class EPCManagement(): logging.error('This option should not occur!') mySSH.close() + def LogCollectOAICN5G(self): + mySSH = SSH.SSHConnection() + mySSH.open(self.IPAddress, self.UserName, self.Password) + logging.debug('OAI CN5G Collecting Log files to workspace') + mySSH.command('echo ' + self.Password + ' | sudo rm -rf ' + self.SourceCodePath + '/logs', '\$', 5) + mySSH.command('mkdir ' + self.SourceCodePath + '/logs','\$', 5) + containers_list=['oai-smf','oai-spgwu','oai-amf','oai-nrf'] + for c in containers_list: + mySSH.command('docker logs ' + c + ' > ' + self.SourceCodePath + '/logs/' + c + '.log', '\$', 5) + mySSH.command('cd ' + self.SourceCodePath + '/logs', '\$', 5) + mySSH.command('zip oai-cn5g.log.zip *.log', '\$', 60) + mySSH.close() + From 5a24210b45b5120f33c0f796766a59af792af37a Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 14 Sep 2021 22:29:33 +0200 Subject: [PATCH 03/30] check if container is running --- ci-scripts/epc.py | 17 ++++++++++++----- ci-scripts/sshconnection.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index dead126fcd..24712cc427 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -235,6 +235,11 @@ class EPCManagement(): mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) mySSH.command('./core-network.sh start nrf spgwu', '\$', 30) + #time.sleep(10) + #response = mySSH.command3('docker inspect --format=\'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' oai-amf', 5) + #self.MmeIPAddress = str(response[0],'utf-8') + #self.MmeIPAddress.rstrip() + #logging.debug('AMF IP Address ' + self.MmeIPAddress) else: logging.error('This option should not occur!') mySSH.close() @@ -251,10 +256,12 @@ class EPCManagement(): elif re.match('OAICN5G', self.Type, re.IGNORECASE): mySSH = SSH.SSHConnection() mySSH.open(self.IPAddress, self.UserName, self.Password) - mySSH.command2('docker inspect --format=\'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' oai-amf', 5) - self.MmeIPAddress = str(mySSH.cmd2Results[0],'utf-8') - self.MmeIPAddress.rstrip() - logging.debug('AMF IP Address ' + self.MmeIPAddress) + response=mySSH.command3('docker container ls -f name=oai-amf', 10) + if len(response)>1: + response=mySSH.command3('docker inspect --format=\'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' oai-amf', 10) + tmp = str(response[0],'utf-8') + self.MmeIPAddress = tmp.rstrip() + logging.debug('AMF IP Address ' + self.MmeIPAddress) mySSH.close() def CheckHSSProcess(self, status_queue): @@ -459,7 +466,7 @@ class EPCManagement(): self.LogCollectOAICN5G() logging.debug('Terminating OAI CN5G') mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) - mySSH.command('./core-network.sh stop nrf spgwu', '\$', 5) + mySSH.command('./core-network.sh stop nrf spgwu', '\$', 30) else: logging.error('This should not happen!') mySSH.close() diff --git a/ci-scripts/sshconnection.py b/ci-scripts/sshconnection.py index b4087c9695..b85c40a0be 100644 --- a/ci-scripts/sshconnection.py +++ b/ci-scripts/sshconnection.py @@ -163,6 +163,18 @@ class SSHConnection(): lSsh = subprocess.Popen(["ssh", "%s" % myHost, commandline],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) self.cmd2Results = str(lSsh.stdout.readlines()) + def command3(self, commandline, timeout, silent=False): + if not silent: + logging.debug(commandline) + self.cmd2Results = '' + myHost = self.username + '@' + self.ipaddress + # CAUTION: THIS METHOD IMPLIES THAT THERE ARE VALID SSH KEYS + # BETWEEN THE PYTHON EXECUTOR NODE AND THE REMOTE HOST + # OTHERWISE IT WON'T WORK + lSsh = subprocess.Popen(["ssh", "%s" % myHost, commandline],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + return lSsh.stdout.readlines() + + def close(self): self.ssh.timeout = 5 self.ssh.sendline('exit') From 18d66873f3b4afe65b1139449418c214a5aa2a8d Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 15 Sep 2021 09:54:29 +0200 Subject: [PATCH 04/30] fixing dnn to use with OAICN5G in infra yaml --- ci-scripts/ci_ueinfra.yaml | 4 ++-- ci-scripts/epc.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-scripts/ci_ueinfra.yaml b/ci-scripts/ci_ueinfra.yaml index 619e8fba15..aafe046e1f 100644 --- a/ci-scripts/ci_ueinfra.yaml +++ b/ci-scripts/ci_ueinfra.yaml @@ -20,11 +20,11 @@ nrmodule2_quectel: Kind : quectel Process : Name : quectel-CM - Cmd : /home/nrmodule2/quectel-CM/quectel-CM -s oai.ipv4 -4 + Cmd : /home/nrmodule2/quectel-CM/quectel-CM -s oai -4 #oai is the dnn for OAI CN5G (hardcoded in CN) WakeupScript : ci_ctl_qtel.py /dev/ttyUSB7 wup DetachScript : ci_ctl_qtel.py /dev/ttyUSB7 detach LogStore : /media/ci_qlogs - PLMN : 20899 + PLMN : 20897 UENetwork : wwan1 HostIPAddress : 192.168.18.189 HostUsername : nrmodule2 diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index 24712cc427..3fae85af52 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -466,7 +466,7 @@ class EPCManagement(): self.LogCollectOAICN5G() logging.debug('Terminating OAI CN5G') mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) - mySSH.command('./core-network.sh stop nrf spgwu', '\$', 30) + mySSH.command('./core-network.sh stop nrf spgwu', '\$', 60) else: logging.error('This should not happen!') mySSH.close() From 9c2785a72c17e3a5420f737eb027583daa4e4e75 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 15 Sep 2021 10:00:42 +0200 Subject: [PATCH 05/30] fix delay to start OAI5GCN --- ci-scripts/epc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index 3fae85af52..b1648d6b66 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -234,7 +234,7 @@ class EPCManagement(): mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5) mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) - mySSH.command('./core-network.sh start nrf spgwu', '\$', 30) + mySSH.command('./core-network.sh start nrf spgwu', '\$', 60) #time.sleep(10) #response = mySSH.command3('docker inspect --format=\'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' oai-amf', 5) #self.MmeIPAddress = str(response[0],'utf-8') From 41131bf994b1888ae7a54ab8c8295a19341926a3 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 15 Sep 2021 14:59:56 +0200 Subject: [PATCH 06/30] updating testcase for ping --- ci-scripts/cls_oaicitest.py | 2 ++ ci-scripts/epc.py | 5 ----- ci-scripts/xml_files/fr1_sa_quectel.xml | 6 ++++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index f1ab8c106f..f6ef7e4826 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -1539,6 +1539,8 @@ class OaiCiTest(): #target address is different depending on EPC type if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): Target = EPC.MmeIPAddress + elif re.match('OAICN5G', EPC.Type, re.IGNORECASE): + Target = '8.8.8.8' else: Target = EPC.IPAddress #ping from module NIC rather than IP address to make sure round trip is over the air diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index b1648d6b66..90b4f03165 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -235,11 +235,6 @@ class EPCManagement(): mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) mySSH.command('./core-network.sh start nrf spgwu', '\$', 60) - #time.sleep(10) - #response = mySSH.command3('docker inspect --format=\'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' oai-amf', 5) - #self.MmeIPAddress = str(response[0],'utf-8') - #self.MmeIPAddress.rstrip() - #logging.debug('AMF IP Address ' + self.MmeIPAddress) else: logging.error('This option should not occur!') mySSH.close() diff --git a/ci-scripts/xml_files/fr1_sa_quectel.xml b/ci-scripts/xml_files/fr1_sa_quectel.xml index 4c8fba0611..989e626886 100644 --- a/ci-scripts/xml_files/fr1_sa_quectel.xml +++ b/ci-scripts/xml_files/fr1_sa_quectel.xml @@ -31,6 +31,8 @@ 010000 000001 050000 + 000002 + 050001 000001 010002 080000 @@ -81,7 +83,7 @@ Ping: 20pings in 20sec nrmodule2_quectel -c 20 - 50 + 5 @@ -89,7 +91,7 @@ Ping: 100pings in 20sec nrmodule2_quectel -c 100 -i 0.2 - 50 + 5 From e3a1c21a3d2b34a27383d1542696b8f7322fc527 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 20 Sep 2021 11:25:09 +0200 Subject: [PATCH 07/30] update of the NSA/LTE 2x2 bench pipeline --- ci-scripts/ci_ueinfra.yaml | 2 + ci-scripts/cls_oaicitest.py | 14 +- .../enb.band38.nsa_2x2.100PRB.usrpn310.conf | 2 +- .../gnb.band78.nsa_2x2.106PRB.usrpn310.conf | 2 +- ci-scripts/xml_files/fr1_enb_build.xml | 50 +++++ ci-scripts/xml_files/fr1_lte_2x2_quectel.xml | 145 +++++++++++++ ci-scripts/xml_files/fr1_oai_cn_deploy.xml | 2 +- .../docker-compose.yml | 18 +- .../fr1_epc_tim/docker-compose.yml.orig | 203 ------------------ 9 files changed, 217 insertions(+), 221 deletions(-) create mode 100644 ci-scripts/xml_files/fr1_enb_build.xml create mode 100644 ci-scripts/xml_files/fr1_lte_2x2_quectel.xml rename ci-scripts/yaml_files/{fr1_epc_tim => fr1_epc_20897}/docker-compose.yml (96%) delete mode 100644 ci-scripts/yaml_files/fr1_epc_tim/docker-compose.yml.orig diff --git a/ci-scripts/ci_ueinfra.yaml b/ci-scripts/ci_ueinfra.yaml index 619e8fba15..ccb03299c5 100644 --- a/ci-scripts/ci_ueinfra.yaml +++ b/ci-scripts/ci_ueinfra.yaml @@ -30,6 +30,8 @@ nrmodule2_quectel: HostUsername : nrmodule2 HostPassword : linux HostSourceCodePath : none + StartCommands : + - sudo -S ip link set dev wwan1 mtu 1500 dummy: ID: '' State : '' diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index f1ab8c106f..0b6ed88f85 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -400,10 +400,18 @@ class OaiCiTest(): if status==0: HTML.CreateHtmlTestRow(Module_UE.UEIPAddress, 'OK', CONST.ALL_PROCESSES_OK) logging.debug('UE IP addresss : '+ Module_UE.UEIPAddress) + #execute additional commands from yaml file after UE attach + SSH = sshconnection.SSHConnection() + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + for startcommand in Module_UE.StartCommands: + cmd = 'echo ' + EPC.Password + ' | ' + startcommand + SSH.command(cmd,'\$',5) + SSH.close() else: #status==-1 failed to retrieve IP address HTML.CreateHtmlTestRow('N/A', 'KO', CONST.UE_IP_ADDRESS_ISSUE) self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return + def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): @@ -2231,12 +2239,6 @@ class OaiCiTest(): def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE): SSH = sshconnection.SSHConnection() - #RH temporary quick n dirty for test - SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - cmd = 'echo ' + EPC.Password + ' | sudo -S ip link set dev tun5 mtu 1358' - SSH.command(cmd,'\$',5) - SSH.close() - #kill iperf processes before (in case there are still some remaining) SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) diff --git a/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf b/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf index db0802fdd7..3c355c48fa 100644 --- a/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf +++ b/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf @@ -14,7 +14,7 @@ eNBs = // Tracking area code, 0x0000 and 0xfffe are reserved values tracking_area_code = 1; - plmn_list = ( { mcc = 208; mnc = 99; mnc_length = 2; } ); + plmn_list = ( { mcc = 208; mnc = 97; mnc_length = 2; } ); tr_s_preference = "local_mac" diff --git a/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf b/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf index 24dbdbd74b..e7dbe4dd5f 100644 --- a/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf +++ b/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf @@ -13,7 +13,7 @@ gNBs = // Tracking area code, 0x0000 and 0xfffe are reserved values tracking_area_code = 1; - plmn_list = ({mcc = 208; mnc = 99; mnc_length = 2;}); + plmn_list = ({mcc = 208; mnc = 97; mnc_length = 2;}); tr_s_preference = "local_mac" diff --git a/ci-scripts/xml_files/fr1_enb_build.xml b/ci-scripts/xml_files/fr1_enb_build.xml new file mode 100644 index 0000000000..084f58a591 --- /dev/null +++ b/ci-scripts/xml_files/fr1_enb_build.xml @@ -0,0 +1,50 @@ + + + build-tab + Build + wrench + + 000001 + 000002 + + + + + Build_eNB + Build eNB + -w USRP -c --eNB --ninja + 0 + 0 + True + True + + + + WaitEndBuild_eNB + Wait for end of Build eNB + 0 + 0 + + + diff --git a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml new file mode 100644 index 0000000000..6afd92ef43 --- /dev/null +++ b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml @@ -0,0 +1,145 @@ + + + TEST-LTE-TM2 + LTE 2x2 Ping DL UL with QUECTEL + tasks + 1 + + 030000 + 000002 + 010000 + 000001 + 050000 + 050001 + 000001 + 010002 + 080000 + + + + + Initialize_UE + Initialize Quectel + nrmodule2_quectel + yes + + + + + Terminate_UE + Terminate Quectel + nrmodule2_quectel + + + + + Initialize_eNB + Initialize eNB + -O ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf + 0 + 0 + lte + yes + 192.168.18.241 + + + + + Initialize_eNB + Initialize gNB + -O ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf -q + 1 + 1 + nr + 192.168.18.240 + + + + IdleSleep + Sleep + 5 + + + + IdleSleep + Sleep + 20 + + + + + Ping + Ping: 20 pings in 20sec + nrmodule2_quectel + -c 20 -s 2048 + 50 + + + + Ping + Ping: 100 pings in 20sec + nrmodule2_quectel + -c 100 -i 0.2 -s 2048 + 50 + + + + Iperf + iperf (DL/20Mbps/UDP)(60 sec)(single-ue profile) + -u -b 20M -t 60 + DL + nrmodule2_quectel + 50 + single-ue + + + + Iperf + iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile) + -u -b 3M -t 60 + UL + nrmodule2_quectel + 50 + single-ue + + + + + Terminate_eNB + Terminate eNB + 0 + 0 + lte + + + + Terminate_eNB + Terminate gNB + 1 + 1 + nr + + + + diff --git a/ci-scripts/xml_files/fr1_oai_cn_deploy.xml b/ci-scripts/xml_files/fr1_oai_cn_deploy.xml index 919def7aa3..1ccc952c56 100644 --- a/ci-scripts/xml_files/fr1_oai_cn_deploy.xml +++ b/ci-scripts/xml_files/fr1_oai_cn_deploy.xml @@ -33,7 +33,7 @@ Deploy_EPC Deploy all EPC containers - yaml_files/fr1_epc_tim + yaml_files/fr1_epc_20897 diff --git a/ci-scripts/yaml_files/fr1_epc_tim/docker-compose.yml b/ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml similarity index 96% rename from ci-scripts/yaml_files/fr1_epc_tim/docker-compose.yml rename to ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml index 73a6b3a1ff..31eee2ca3d 100644 --- a/ci-scripts/yaml_files/fr1_epc_tim/docker-compose.yml +++ b/ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml @@ -51,7 +51,7 @@ services: LTE_K: FEC86BA6EB707ED08905757B1BB44B8F APN1: oai.ipv4 APN2: oai2.ipv4 - FIRST_IMSI: 208990100001127 + FIRST_IMSI: 208970100001127 NB_USERS: 5 healthcheck: test: /bin/bash -c "pgrep oai_hss" @@ -78,7 +78,7 @@ services: HSS_FQDN: hss.openairinterface.org HSS_REALM: openairinterface.org MCC: '208' - MNC: '99' + MNC: '97' MME_GID: 32768 MME_CODE: 3 TAC_0: 1 @@ -97,15 +97,15 @@ services: PEER_MME_IPV4_ADDRESS_FOR_S10_0: 0.0.0.0 PEER_MME_IPV4_ADDRESS_FOR_S10_1: 0.0.0.0 MCC_SGW_0: '208' - MNC3_SGW_0: '099' + MNC3_SGW_0: '097' TAC_LB_SGW_0: '01' TAC_HB_SGW_0: '00' MCC_MME_0: '208' - MNC3_MME_0: '099' + MNC3_MME_0: '097' TAC_LB_MME_0: '02' TAC_HB_MME_0: '00' MCC_MME_1: '208' - MNC3_MME_1: '099' + MNC3_MME_1: '097' TAC_LB_MME_1: '03' TAC_HB_MME_1: '00' TAC_LB_SGW_TEST_0: '03' @@ -138,8 +138,8 @@ services: UE_IP_ADDRESS_POOL_1: '12.1.1.2 - 12.1.1.254' UE_IP_ADDRESS_POOL_2: '12.0.0.2 - 12.0.0.254' MCC: '208' - MNC: '99' - MNC03: '099' + MNC: '97' + MNC03: '097' TAC: 1 GW_ID: 1 REALM: openairinterface.org @@ -168,8 +168,8 @@ services: NETWORK_UE_IP: '12.1.1.0/24' NETWORK_UE_NAT_OPTION: 'yes' MCC: '208' - MNC: '99' - MNC03: '099' + MNC: '97' + MNC03: '097' TAC: 1 GW_ID: 1 REALM: openairinterface.org diff --git a/ci-scripts/yaml_files/fr1_epc_tim/docker-compose.yml.orig b/ci-scripts/yaml_files/fr1_epc_tim/docker-compose.yml.orig deleted file mode 100644 index 28ae154838..0000000000 --- a/ci-scripts/yaml_files/fr1_epc_tim/docker-compose.yml.orig +++ /dev/null @@ -1,203 +0,0 @@ -version: '3.8' - -services: - cassandra: - image: cassandra:2.1 - container_name: prod-cassandra - networks: - private_net: - ipv4_address: 192.168.68.2 - environment: - CASSANDRA_CLUSTER_NAME: "OAI HSS Cluster" - CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch - healthcheck: - test: /bin/bash -c "nodetool status" - interval: 10s - timeout: 5s - retries: 5 - - db_init: - image: cassandra:2.1 - container_name: prod-db-init - depends_on: [cassandra] - deploy: - restart_policy: - condition: on-failure - max_attempts: 10 - networks: - private_net: - ipv4_address: 192.168.68.4 - volumes: - - ./oai_db.cql:/home/oai_db.cql - entrypoint: /bin/bash -c "cqlsh --file /home/oai_db.cql 192.168.68.2 && echo 'OK'" - - oai_hss: - image: oai-hss:production - container_name: prod-oai-hss - privileged: true - depends_on: [cassandra] - networks: - private_net: - ipv4_address: 192.168.68.3 - public_net: - ipv4_address: 192.168.61.2 - environment: - REALM: openairinterface.org - HSS_FQDN: hss.openairinterface.org - PREFIX: /openair-hss/etc - cassandra_Server_IP: 192.168.68.2 - OP_KEY: 1006020f0a478bf6b699f15c062e42b3 - LTE_K: fec86ba6eb707ed08905757b1bb44b8f - APN1: oai.ipv4 - APN2: internet - FIRST_IMSI: 222010100001120 - NB_USERS: 10 - healthcheck: - test: /bin/bash -c "pgrep oai_hss" - interval: 10s - timeout: 5s - retries: 5 - - oai_mme: - image: oai-mme:production - container_name: prod-oai-mme - privileged: true - depends_on: [oai_hss] - networks: - public_net: - ipv4_address: 192.168.61.3 - environment: - REALM: openairinterface.org - PREFIX: /openair-mme/etc - INSTANCE: 1 - PID_DIRECTORY: /var/run - HSS_IP_ADDR: 192.168.61.2 - HSS_HOSTNAME: hss - HSS_FQDN: hss.openairinterface.org - HSS_REALM: openairinterface.org - MCC: '222' - MNC: '01' - MME_GID: 32768 - MME_CODE: 3 - TAC_0: 1 - TAC_1: 2 - TAC_2: 3 - MME_FQDN: mme.openairinterface.org - MME_S6A_IP_ADDR: 192.168.61.3 - MME_INTERFACE_NAME_FOR_S1_MME: eth0 - MME_IPV4_ADDRESS_FOR_S1_MME: 192.168.61.3 - MME_INTERFACE_NAME_FOR_S11: eth0 - MME_IPV4_ADDRESS_FOR_S11: 192.168.61.3 - MME_INTERFACE_NAME_FOR_S10: lo - MME_IPV4_ADDRESS_FOR_S10: 127.0.0.10 - OUTPUT: CONSOLE - SGW_IPV4_ADDRESS_FOR_S11_0: 192.168.61.4 - PEER_MME_IPV4_ADDRESS_FOR_S10_0: 0.0.0.0 - PEER_MME_IPV4_ADDRESS_FOR_S10_1: 0.0.0.0 - MCC_SGW_0: '222' - MNC3_SGW_0: '001' - TAC_LB_SGW_0: '01' - TAC_HB_SGW_0: '00' - MCC_MME_0: '222' - MNC3_MME_0: '001' - TAC_LB_MME_0: '02' - TAC_HB_MME_0: '00' - MCC_MME_1: '222' - MNC3_MME_1: '001' - TAC_LB_MME_1: '03' - TAC_HB_MME_1: '00' - TAC_LB_SGW_TEST_0: '03' - TAC_HB_SGW_TEST_0: '00' - SGW_IPV4_ADDRESS_FOR_S11_TEST_0: 0.0.0.0 - healthcheck: - test: /bin/bash -c "pgrep oai_mme" - interval: 10s - timeout: 5s - retries: 5 - - oai_spgwc: - image: oai-spgwc:production - privileged: true - depends_on: [oai_mme] - container_name: prod-oai-spgwc - networks: - public_net: - ipv4_address: 192.168.61.4 - environment: - PID_DIRECTORY: /var/run - SGW_INTERFACE_NAME_FOR_S11: eth0 - SGW_IP_FOR_S5_S8_CP: 127.0.0.11/8 - PGW_IP_FOR_S5_S8_CP: 127.0.0.12/8 - PGW_INTERFACE_NAME_FOR_SX: eth0 - DEFAULT_APN: oai.ipv4 - DEFAULT_DNS_IPV4_ADDRESS: 192.168.18.129 - DEFAULT_DNS_SEC_IPV4_ADDRESS: 8.8.4.4 - UE_IP_ADDRESS_POOL: '12.1.1.2 - 12.1.1.254' - PUSH_PROTOCOL_OPTION: 'yes' - healthcheck: - test: /bin/bash -c "pgrep oai_spgwc" - interval: 10s - timeout: 5s - retries: 5 - - oai_spgwu: - image: oai-spgwu-tiny:production - privileged: true - container_name: prod-oai-spgwu-tiny - depends_on: [oai_spgwc] - networks: - public_net: - ipv4_address: 192.168.61.5 - environment: - PID_DIRECTORY: /var/run - INSTANCE: 1 - SGW_INTERFACE_NAME_FOR_S1U_S12_S4_UP: eth0 - PGW_INTERFACE_NAME_FOR_SGI: eth0 - SGW_INTERFACE_NAME_FOR_SX: eth0 - SPGWC0_IP_ADDRESS: 192.168.61.4 - NETWORK_UE_IP: '12.1.1.0/24' - NETWORK_UE_NAT_OPTION: 'yes' - healthcheck: - test: /bin/bash -c "pgrep oai_spgwu" - interval: 10s - timeout: 5s - retries: 5 - - flexran_rtc: - image: flexran-rtc:production - privileged: true - container_name: prod-flexran-rtc - networks: - public_net: - ipv4_address: 192.168.61.10 - healthcheck: - test: /bin/bash -c "pgrep rt_controller" - interval: 10s - timeout: 5s - retries: 5 - - trf_gen: - image: trf-gen:production - privileged: true - container_name: prod-trf-gen - networks: - public_net: - ipv4_address: 192.168.61.11 - entrypoint: /bin/bash -c "ip route add 12.1.1.0/24 via 192.168.61.5 dev eth0; sleep infinity" - healthcheck: - test: /bin/bash -c "ping -c 2 192.168.61.5" - interval: 10s - timeout: 5s - retries: 5 - -networks: - private_net: - name: prod-oai-private-net - ipam: - config: - - subnet: 192.168.68.0/26 - public_net: - name: prod-oai-public-net - ipam: - config: - - subnet: 192.168.61.0/26 From 2d46faaf13637401361eeb5ec15d90b66bd6bc59 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 29 Sep 2021 21:28:53 +0200 Subject: [PATCH 08/30] check mtu is as expected --- ci-scripts/ci_ueinfra.yaml | 3 ++- ci-scripts/cls_module_ue.py | 27 +++++++++++++++++++++++++++ ci-scripts/cls_oaicitest.py | 2 ++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/ci-scripts/ci_ueinfra.yaml b/ci-scripts/ci_ueinfra.yaml index ccb03299c5..c8ba1fd042 100644 --- a/ci-scripts/ci_ueinfra.yaml +++ b/ci-scripts/ci_ueinfra.yaml @@ -24,7 +24,7 @@ nrmodule2_quectel: WakeupScript : ci_ctl_qtel.py /dev/ttyUSB7 wup DetachScript : ci_ctl_qtel.py /dev/ttyUSB7 detach LogStore : /media/ci_qlogs - PLMN : 20899 + PLMN : 20897 UENetwork : wwan1 HostIPAddress : 192.168.18.189 HostUsername : nrmodule2 @@ -32,6 +32,7 @@ nrmodule2_quectel: HostSourceCodePath : none StartCommands : - sudo -S ip link set dev wwan1 mtu 1500 + MTU : 1500 dummy: ID: '' State : '' diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index 96d4e0310e..3363a5fff5 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -131,6 +131,33 @@ class Module_UE: logging.debug('\u001B[1;37;41m Module IP Address Not Found! \u001B[0m') return -1 + def CheckModuleMTU(self): + HOST=self.HostUsername+'@'+self.HostIPAddress + response= [] + tentative = 3 + while (len(response)==0) and (tentative>0): + COMMAND="ip a show dev " + self.UENetwork + " | grep mtu" + logging.debug(COMMAND) + ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + response = ssh.stdout.readlines() + tentative-=1 + time.sleep(10) + if (tentative==0) and (len(response)==0): + logging.debug('\u001B[1;37;41m Module NIC MTU Not Found! Time expired \u001B[0m') + return -1 + else: #check response + result = re.search('mtu (?P[0-9]+)', response[0].decode("utf-8") ) + if result is not None: + if (result.group('mtu') is not None) and (result.group('mtu')==self.MTU) : + logging.debug('\u001B[1mUE Module NIC MTU is ' + self.MTU + ' as requested\u001B[0m') + return 0 + else: + logging.debug('\u001B[1;37;41m Incorrect Module NIC MTU ' + str(result.group('mtu')) + '! Expected : ' + str(self.MTU) + '\u001B[0m') + return -1 + else: + logging.debug('\u001B[1;37;41m Module NIC MTU Not Found! \u001B[0m') + return -1 + def EnableTrace(self): if self.ue_trace=="yes": mySSH = sshconnection.SSHConnection() diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 0b6ed88f85..bd5fd5bf0b 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -407,6 +407,8 @@ class OaiCiTest(): cmd = 'echo ' + EPC.Password + ' | ' + startcommand SSH.command(cmd,'\$',5) SSH.close() + #check that the MTU is as expected / requested + Module_UE.CheckModuleMTU() else: #status==-1 failed to retrieve IP address HTML.CreateHtmlTestRow('N/A', 'KO', CONST.UE_IP_ADDRESS_ISSUE) self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) From ea4a60d9261ae5dcbf17d65c87b2199a4462bad4 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 29 Sep 2021 23:03:37 +0200 Subject: [PATCH 09/30] DL support + testcase fix --- ci-scripts/cls_module_ue.py | 4 +- ci-scripts/cls_oaicitest.py | 181 ++++++++++++------- ci-scripts/xml_files/fr1_lte_2x2_quectel.xml | 22 +-- 3 files changed, 126 insertions(+), 81 deletions(-) diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index 3363a5fff5..758e7866f0 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -148,8 +148,8 @@ class Module_UE: else: #check response result = re.search('mtu (?P[0-9]+)', response[0].decode("utf-8") ) if result is not None: - if (result.group('mtu') is not None) and (result.group('mtu')==self.MTU) : - logging.debug('\u001B[1mUE Module NIC MTU is ' + self.MTU + ' as requested\u001B[0m') + if (result.group('mtu') is not None) and (str(result.group('mtu'))==str(self.MTU)) : + logging.debug('\u001B[1mUE Module NIC MTU is ' + str(self.MTU) + ' as expected\u001B[0m') return 0 else: logging.debug('\u001B[1;37;41m Incorrect Module NIC MTU ' + str(result.group('mtu')) + '! Expected : ' + str(self.MTU) + '\u001B[0m') diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index bd5fd5bf0b..6d773614d1 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -402,9 +402,9 @@ class OaiCiTest(): logging.debug('UE IP addresss : '+ Module_UE.UEIPAddress) #execute additional commands from yaml file after UE attach SSH = sshconnection.SSHConnection() - SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) for startcommand in Module_UE.StartCommands: - cmd = 'echo ' + EPC.Password + ' | ' + startcommand + cmd = 'echo ' + Module_UE.HostPassword + ' | ' + startcommand SSH.command(cmd,'\$',5) SSH.close() #check that the MTU is as expected / requested @@ -2240,79 +2240,124 @@ class OaiCiTest(): def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE): - SSH = sshconnection.SSHConnection() - - #kill iperf processes before (in case there are still some remaining) - SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) - cmd = 'killall --signal=SIGKILL iperf' - SSH.command(cmd,'\$',5) - SSH.close() - SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - cmd = 'killall --signal=SIGKILL iperf' - SSH.command(cmd,'\$',5) - SSH.close() - - - iperf_time = self.Iperf_ComputeTime() - if self.iperf_direction=="DL": - logging.debug("Iperf for Module in DL mode detected") - #server side UE - SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) - cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' - SSH.command(cmd,'\$',5) - cmd = 'echo $USER; nohup /opt/iperf-2.0.10/iperf -s -B ' + UE_IPAddress + ' -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' - SSH.command(cmd,'\$',5) - SSH.close() - #client side EPC + if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): + #retrieve trf-gen container IP address + SSH = sshconnection.SSHConnection() SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' - SSH.command(cmd,'\$',5) - cmd = 'iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' - SSH.command(cmd,'\$',int(iperf_time)*5.0) + SSH.command('docker inspect --format="TRF_IP_ADDR = {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" prod-trf-gen', '\$', 5) + result = re.search('TRF_IP_ADDR = (?P[0-9\.]+)', SSH.getBefore()) + if result is not None: + EPC_Iperf_UE_IPAddress = result.group('trf_ip_addr') SSH.close() - #copy the 2 resulting files locally - SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log', '.') - SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '.') - #send for analysis - filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' - self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1) - - elif self.iperf_direction=="UL":#does not work at the moment - logging.debug("Iperf for Module in UL mode detected") - #server side EPC - SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' - SSH.command(cmd,'\$',5) - cmd = 'echo $USER; nohup iperf -s -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' - SSH.command(cmd,'\$',5) - SSH.close() - - #client side UE + #kill iperf processes on UE side before (in case there are still some remaining) SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) - cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + cmd = 'killall --signal=SIGKILL iperf' SSH.command(cmd,'\$',5) - SSH.command('/opt/iperf-2.0.10/iperf -c 192.172.0.1 ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', int(iperf_time)*5.0) SSH.close() - #copy the 2 resulting files locally - SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '.') - SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log', '.') - #send for analysis - filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' - self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1) - else : - logging.debug("Incorrect or missing IPERF direction in XML") + iperf_time = self.Iperf_ComputeTime() + if self.iperf_direction=="DL": + logging.debug("Iperf for Module in DL mode detected") + #server side UE + server_filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + cmd = 'rm ' + server_filename + SSH.command(cmd,'\$',5) + cmd = 'echo $USER; nohup iperf -s -B ' + UE_IPAddress + ' -u 2>&1 > ' + server_filename + SSH.command(cmd,'\$',5) + SSH.close() + #client side EPC + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + iperf_cmd = 'bin/iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename + cmd = 'docker exec -it prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' + SSH.command(cmd,'\$',int(iperf_time)*5.0) + SSH.command('docker cp prod-trf-gen:' + client_filename + ' ' + EPC.SourceCodePath, '\$', 5) + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/' + client_filename, '.') + SSH.close() - #kill iperf processes after to be clean - SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) - cmd = 'killall --signal=SIGKILL iperf' - SSH.command(cmd,'\$',5) - SSH.close() - SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - cmd = 'killall --signal=SIGKILL iperf' - SSH.command(cmd,'\$',5) - SSH.close() - return + + #copy the 2 resulting files locally + SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, server_filename, '.') + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, client_filename, '.') + #send for analysis + self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,server_filename,1) + + else: + #default is ltebox + SSH = sshconnection.SSHConnection() + + #kill iperf processes before (in case there are still some remaining) + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + cmd = 'killall --signal=SIGKILL iperf' + SSH.command(cmd,'\$',5) + SSH.close() + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + cmd = 'killall --signal=SIGKILL iperf' + SSH.command(cmd,'\$',5) + SSH.close() + + + iperf_time = self.Iperf_ComputeTime() + if self.iperf_direction=="DL": + logging.debug("Iperf for Module in DL mode detected") + #server side UE + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + cmd = 'echo $USER; nohup /opt/iperf-2.0.10/iperf -s -B ' + UE_IPAddress + ' -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + SSH.close() + #client side EPC + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + cmd = 'iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',int(iperf_time)*5.0) + SSH.close() + #copy the 2 resulting files locally + SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log', '.') + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '.') + #send for analysis + filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1) + + elif self.iperf_direction=="UL":#does not work at the moment + logging.debug("Iperf for Module in UL mode detected") + #server side EPC + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + cmd = 'rm iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + cmd = 'echo $USER; nohup iperf -s -u 2>&1 > iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + SSH.close() + + #client side UE + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + cmd = 'rm iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command(cmd,'\$',5) + SSH.command('/opt/iperf-2.0.10/iperf -c 192.172.0.1 ' + self.iperf_args + ' 2>&1 > iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '\$', int(iperf_time)*5.0) + SSH.close() + + #copy the 2 resulting files locally + SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log', '.') + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log', '.') + #send for analysis + filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1) + else : + logging.debug("Incorrect or missing IPERF direction in XML") + + #kill iperf processes after to be clean + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + cmd = 'killall --signal=SIGKILL iperf' + SSH.command(cmd,'\$',5) + SSH.close() + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + cmd = 'killall --signal=SIGKILL iperf' + SSH.command(cmd,'\$',5) + SSH.close() + return def Iperf_common(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC): try: diff --git a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml index 6afd92ef43..6e04d65522 100644 --- a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml @@ -56,7 +56,7 @@ Initialize_eNB Initialize eNB - -O ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf + -O ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 0 0 lte @@ -90,24 +90,24 @@ Ping - Ping: 20 pings in 20sec + Ping: 20 pings nrmodule2_quectel - -c 20 -s 2048 + -c 20 + 50 + + + Ping + Ping: 100 pings, size 2048 + nrmodule2_quectel + -c 100 -s 2048 50 - - Ping - Ping: 100 pings in 20sec - nrmodule2_quectel - -c 100 -i 0.2 -s 2048 - 50 - Iperf iperf (DL/20Mbps/UDP)(60 sec)(single-ue profile) - -u -b 20M -t 60 + -u -b 26M -t 60 DL nrmodule2_quectel 50 From c16e71b0221252f59a6a9b0cc58548ca3883574b Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 30 Sep 2021 07:57:46 +0200 Subject: [PATCH 10/30] add DL stage to NSA 2x2 test --- ci-scripts/xml_files/fr1_lte_2x2_quectel.xml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml index 6e04d65522..8f8a400cbf 100644 --- a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml @@ -32,6 +32,7 @@ 000001 050000 050001 + 070000 000001 010002 080000 @@ -93,20 +94,20 @@ Ping: 20 pings nrmodule2_quectel -c 20 - 50 + 5 Ping Ping: 100 pings, size 2048 nrmodule2_quectel -c 100 -s 2048 - 50 + 5 Iperf - iperf (DL/20Mbps/UDP)(60 sec)(single-ue profile) + iperf (DL/26Mbps/UDP)(60 sec)(single-ue profile) -u -b 26M -t 60 DL nrmodule2_quectel From 893cf7c2e013a7a96a2827710258d141e2d76064 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 30 Sep 2021 08:29:39 +0200 Subject: [PATCH 11/30] temp remove ping to debug DL --- ci-scripts/xml_files/fr1_lte_2x2_quectel.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml index 8f8a400cbf..c28fb65804 100644 --- a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml @@ -31,7 +31,6 @@ 010000 000001 050000 - 050001 070000 000001 010002 From afb748ecd4ecd4dad84315b3335e6d349e610bce Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 30 Sep 2021 15:33:45 +0200 Subject: [PATCH 12/30] update for radio conf and testcase --- .../enb.band38.nsa_2x2.100PRB.usrpn310.conf | 4 ++-- ci-scripts/xml_files/fr1_lte_2x2_quectel.xml | 17 +++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf b/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf index 3c355c48fa..7abe460215 100644 --- a/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf +++ b/ci-scripts/conf_files/enb.band38.nsa_2x2.100PRB.usrpn310.conf @@ -209,8 +209,8 @@ MACRLCs = ( scheduler_mode = "fairRR"; bler_target_upper = 20.0; bler_target_lower = 10.0; - max_ul_rb_index = 24; - puSch10xSnr = 100; + max_ul_rb_index = 27; + puSch10xSnr = 200; puCch10xSnr = 150; } ); diff --git a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml index c28fb65804..a62449f1a3 100644 --- a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml @@ -31,7 +31,10 @@ 010000 000001 050000 + 050001 + 000002 070000 + 070001 000001 010002 080000 @@ -97,9 +100,9 @@ Ping - Ping: 100 pings, size 2048 + Ping: 100 pings, size 1024 nrmodule2_quectel - -c 100 -s 2048 + -c 100 -s 1024 -i 0,2 5 @@ -110,21 +113,23 @@ -u -b 26M -t 60 DL nrmodule2_quectel - 50 + 5 single-ue Iperf - iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile) - -u -b 3M -t 60 + iperf (UL/7Mbps/UDP)(60 sec)(single-ue profile) + -u -b 7M -t 60 UL nrmodule2_quectel - 50 + 5 single-ue + + Terminate_eNB Terminate eNB From 66219a9a763a86e586c650c7743f1006db297248 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 30 Sep 2021 21:58:25 +0200 Subject: [PATCH 13/30] add UL implementation --- ci-scripts/cls_oaicitest.py | 45 ++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 6d773614d1..37e6d2bbab 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -2247,7 +2247,7 @@ class OaiCiTest(): SSH.command('docker inspect --format="TRF_IP_ADDR = {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" prod-trf-gen', '\$', 5) result = re.search('TRF_IP_ADDR = (?P[0-9\.]+)', SSH.getBefore()) if result is not None: - EPC_Iperf_UE_IPAddress = result.group('trf_ip_addr') + trf_gen_IP = result.group('trf_ip_addr') SSH.close() #kill iperf processes on UE side before (in case there are still some remaining) SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) @@ -2269,22 +2269,55 @@ class OaiCiTest(): #client side EPC SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command('docker exec -it prod-trf-gen /bin/bash -c "killall --signal SIGKILL iperf"', '\$', 5) iperf_cmd = 'bin/iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename cmd = 'docker exec -it prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' SSH.command(cmd,'\$',int(iperf_time)*5.0) - SSH.command('docker cp prod-trf-gen:' + client_filename + ' ' + EPC.SourceCodePath, '\$', 5) + SSH.command('docker cp prod-trf-gen:bin/' + client_filename + ' ' + EPC.SourceCodePath, '\$', 5) SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/' + client_filename, '.') SSH.close() - #copy the 2 resulting files locally SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, server_filename, '.') SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, client_filename, '.') #send for analysis self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,server_filename,1) - else: - #default is ltebox + elif self.iperf_direction=="UL": + logging.debug("Iperf for Module in UL mode detected") + #server side EPC + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' + SSH.command('docker exec -it prod-trf-gen /bin/bash -c "killall --signal SIGKILL iperf"', '\$', 5) + iperf_cmd = 'echo $USER; nohup bin/iperf -s -u 2>&1 > ' + server_filename + cmd = 'docker exec -it prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' + SSH.command(cmd,'\$',5) + SSH.close() + + #client side UE + SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) + client_filename = 'iperf_client_' + self.testCase_id + '_' + self.ue_id + '.log' + cmd = 'rm '+ client_filename + SSH.command(cmd,'\$',5) + SSH.command('iperf -B ' + UE_IPAddress + ' -c ' + trf_gen_IP + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename, '\$', int(iperf_time)*5.0) + SSH.close() + + #once client is done, retrieve the server file from container to EPC Host + SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) + SSH.command('docker cp prod-trf-gen:bin/' + server_filename + ' ' + EPC.SourceCodePath, '\$', 5) + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/' + server_filename, '.') + SSH.close() + + #copy the 2 resulting files locally + SSH.copyin(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword, client_filename, '.') + SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, server_filename, '.') + #send for analysis + self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,server_filename,1) + else : + logging.debug("Incorrect or missing IPERF direction in XML") + + else: #default is ltebox + SSH = sshconnection.SSHConnection() #kill iperf processes before (in case there are still some remaining) @@ -2322,7 +2355,7 @@ class OaiCiTest(): filename='iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log' self.Iperf_analyzeV2Server(lock, UE_IPAddress, device_id, statusQueue, self.iperf_args,filename,1) - elif self.iperf_direction=="UL":#does not work at the moment + elif self.iperf_direction=="UL": logging.debug("Iperf for Module in UL mode detected") #server side EPC SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) From a7a8a6cb1c64910ee0a7e20d4081c3e24127a418 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 30 Sep 2021 23:32:55 +0200 Subject: [PATCH 14/30] fix --- ci-scripts/cls_oaicitest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 37e6d2bbab..8cbeaeac1b 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -2273,7 +2273,7 @@ class OaiCiTest(): iperf_cmd = 'bin/iperf -c ' + UE_IPAddress + ' ' + self.iperf_args + ' 2>&1 > ' + client_filename cmd = 'docker exec -it prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' SSH.command(cmd,'\$',int(iperf_time)*5.0) - SSH.command('docker cp prod-trf-gen:bin/' + client_filename + ' ' + EPC.SourceCodePath, '\$', 5) + SSH.command('docker cp prod-trf-gen:/iperf-2.0.13/'+ client_filename + ' ' + EPC.SourceCodePath, '\$', 5) SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/' + client_filename, '.') SSH.close() @@ -2304,7 +2304,7 @@ class OaiCiTest(): #once client is done, retrieve the server file from container to EPC Host SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - SSH.command('docker cp prod-trf-gen:bin/' + server_filename + ' ' + EPC.SourceCodePath, '\$', 5) + SSH.command('docker cp prod-trf-gen:/iperf-2.0.13/' + server_filename + ' ' + EPC.SourceCodePath, '\$', 5) SSH.copyin(EPC.IPAddress, EPC.UserName, EPC.Password, EPC.SourceCodePath + '/' + server_filename, '.') SSH.close() From 76be9fe5bac507e192992dfdba87ba50099f4b72 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 3 Oct 2021 13:18:12 +0200 Subject: [PATCH 15/30] poc : add radio monitor script for test runtime --- ci-scripts/radio_monitor.py | 89 +++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100755 ci-scripts/radio_monitor.py diff --git a/ci-scripts/radio_monitor.py b/ci-scripts/radio_monitor.py new file mode 100755 index 0000000000..24aebaec6b --- /dev/null +++ b/ci-scripts/radio_monitor.py @@ -0,0 +1,89 @@ +import subprocess +import time +import shlex +import re +import sys +import matplotlib.pyplot as plt +import pickle +import numpy as np + +def collect(d): + cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' + process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + output = process.stdout.readlines() + for l in output: + tmp=l.decode("utf-8") + result=re.match(rf'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp) + if result is not None: + d['PHR'].append(int(result.group(1))) + d['bler'].append(float(result.group(2))) + d['mcsoff'].append(int(result.group(3))) + d['mcs'].append(int(result.group(4))) + + +def graph(d): + + + figure, axis = plt.subplots(4, 1,figsize=(10, 10)) + + major_ticks = np.arange(0, len(d['PHR'])+1, 1) + axis[0].set_xticks(major_ticks) + axis[0].set_xticklabels([]) + axis[0].plot(d['PHR'],marker='o') + axis[0].set_xlabel('time') + axis[0].set_ylabel('PHR') + axis[0].set_title("PHR") + + major_ticks = np.arange(0, len(d['bler'])+1, 1) + axis[1].set_xticks(major_ticks) + axis[1].set_xticklabels([]) + axis[1].plot(d['bler'],marker='o') + axis[1].set_xlabel('time') + axis[1].set_ylabel('bler') + axis[1].set_title("bler") + + major_ticks = np.arange(0, len(d['mcsoff'])+1, 1) + axis[2].set_xticks(major_ticks) + axis[2].set_xticklabels([]) + axis[2].plot(d['mcsoff'],marker='o') + axis[2].set_xlabel('time') + axis[2].set_ylabel('mcsoff') + axis[2].set_title("mcsoff") + + major_ticks = np.arange(0, len(d['mcs'])+1, 1) + axis[3].set_xticks(major_ticks) + axis[3].set_xticklabels([]) + axis[3].plot(d['mcs'],marker='o') + axis[3].set_xlabel('time') + axis[3].set_ylabel('mcs') + axis[3].set_title("mcs") + + plt.tight_layout() + # Combine all the operations and display + plt.savefig('/tmp/radio_monitor_dump.png') + plt.show() + +if __name__ == "__main__": + + + d={} + d['PHR']=[] + d['bler']=[] + d['mcsoff']=[] + d['mcs']=[] + + + cmd='ps aux | grep mode | grep -v grep' + process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + output = process.stdout.readlines() + while len(output)!=0 : + collect(d) + process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + output = process.stdout.readlines() + time.sleep(1) + print('process stopped') + with open('/tmp/radio_monitor_dump.pickle', 'wb') as handle: + pickle.dump(d, handle, protocol=pickle.HIGHEST_PROTOCOL) + graph(d) + + From 8ce6e4f6e09e0a99d72db0b8cb0a22e6808e2c89 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 3 Oct 2021 20:38:43 +0200 Subject: [PATCH 16/30] poc : stats monitoring --- ci-scripts/main.py | 1 + ci-scripts/ran.py | 22 ++++++++++++++- .../{radio_monitor.py => stats_monitor.py} | 27 ++++++++++--------- ci-scripts/xml_files/fr1_lte_2x2_quectel.xml | 11 +------- 4 files changed, 37 insertions(+), 24 deletions(-) rename ci-scripts/{radio_monitor.py => stats_monitor.py} (71%) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index d66c0c6305..9923d515a4 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -154,6 +154,7 @@ def GetParametersFromXML(action): elif action == 'Initialize_eNB': RAN.eNB_Trace=test.findtext('eNB_Trace') + RAN.eNB_Stats=test.findtext('eNB_Stats') RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args') eNB_instance=test.findtext('eNB_instance') USRPIPAddress=test.findtext('USRP_IPAddress') diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 5e3d8e388f..a1492b062a 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -94,6 +94,7 @@ class RANManagement(): self.runtime_stats= '' self.datalog_rt_stats={} self.eNB_Trace = '' #if 'yes', Tshark will be launched at initialization + self.eNB_Stats = '' #if 'yes', Statistics Monitor will be launched at initialization self.USRPIPAddress = '' @@ -539,6 +540,23 @@ class RANManagement(): self.eNBstatuses[int(self.eNB_instance)] = int(self.eNB_serverId[self.eNB_instance]) mySSH.close() + + + #if enabled in the xml, + #copyout the monitor script and launch it + if self.eNB_Stats=='yes': + + mySSH.open(lIpAddr, lUserName, lPassWord) + + monitor_file='stats_monitor.py' + mySSH.command('echo ' + lPassWord + ' | sudo -S cp ' + self.eNBSourceCodePath + '/ci-scripts/'+ monitor_file + ' ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/.','\$', 5) + + #launch, it will stop when the softmodem stops + mySSH.command('echo $USER; nohup python3 ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/' + monitor_file + ' 2>&1 &', '\$', 5) + + mySSH.close() + + HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'OK', CONST.ALL_PROCESSES_OK) logging.debug('\u001B[1m Initialize eNB/gNB/ocp-eNB Completed\u001B[0m') @@ -690,8 +708,10 @@ class RANManagement(): mySSH.command('cd cmake_targets', '\$', 5) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/*monitor.pickle .','\$',20) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/*monitor.png .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 60) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor.png', '\$', 60) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 5) mySSH.close() diff --git a/ci-scripts/radio_monitor.py b/ci-scripts/stats_monitor.py similarity index 71% rename from ci-scripts/radio_monitor.py rename to ci-scripts/stats_monitor.py index 24aebaec6b..112d8ff799 100755 --- a/ci-scripts/radio_monitor.py +++ b/ci-scripts/stats_monitor.py @@ -6,19 +6,20 @@ import sys import matplotlib.pyplot as plt import pickle import numpy as np +import os def collect(d): - cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' - process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - output = process.stdout.readlines() - for l in output: - tmp=l.decode("utf-8") - result=re.match(rf'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp) - if result is not None: - d['PHR'].append(int(result.group(1))) - d['bler'].append(float(result.group(2))) - d['mcsoff'].append(int(result.group(3))) - d['mcs'].append(int(result.group(4))) + cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' + process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + output = process.stdout.readlines() + for l in output: + tmp=l.decode("utf-8") + result=re.match(rf'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp) + if result is not None: + d['PHR'].append(int(result.group(1))) + d['bler'].append(float(result.group(2))) + d['mcsoff'].append(int(result.group(3))) + d['mcs'].append(int(result.group(4))) def graph(d): @@ -60,7 +61,7 @@ def graph(d): plt.tight_layout() # Combine all the operations and display - plt.savefig('/tmp/radio_monitor_dump.png') + plt.savefig('/tmp/stats_monitor.png') plt.show() if __name__ == "__main__": @@ -82,7 +83,7 @@ if __name__ == "__main__": output = process.stdout.readlines() time.sleep(1) print('process stopped') - with open('/tmp/radio_monitor_dump.pickle', 'wb') as handle: + with open('/tmp/stats_monitor.pickle', 'wb') as handle: pickle.dump(d, handle, protocol=pickle.HIGHEST_PROTOCOL) graph(d) diff --git a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml index a62449f1a3..86d9e5d5f0 100644 --- a/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_lte_2x2_quectel.xml @@ -64,20 +64,11 @@ 0 lte yes + yes 192.168.18.241 - - Initialize_eNB - Initialize gNB - -O ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf -q - 1 - 1 - nr - 192.168.18.240 - - IdleSleep Sleep From 21adc587d6b453c4dc75dc7ebbb7ca060ce83283 Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 4 Oct 2021 07:48:01 +0200 Subject: [PATCH 17/30] tentative to fix capture of stat files --- ci-scripts/ran.py | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index a1492b062a..1b9a75f77f 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -466,6 +466,18 @@ class RANManagement(): mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5) mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10) + + + #tentative + time.sleep(20) + if self.eNB_Stats=='yes': + monitor_file='stats_monitor.py' + mySSH.command('echo ' + lPassWord + ' | sudo -S cp ' + self.eNBSourceCodePath + '/ci-scripts/'+ monitor_file + ' ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/.','\$', 5) + mySSH.command('echo $USER; nohup python3 ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/' + monitor_file + ' 2>&1 &', '\$', 5) + + + + self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log' if extra_options != '': self.eNBOptions[int(self.eNB_instance)] = extra_options @@ -543,18 +555,14 @@ class RANManagement(): #if enabled in the xml, - #copyout the monitor script and launch it - if self.eNB_Stats=='yes': - - mySSH.open(lIpAddr, lUserName, lPassWord) - - monitor_file='stats_monitor.py' - mySSH.command('echo ' + lPassWord + ' | sudo -S cp ' + self.eNBSourceCodePath + '/ci-scripts/'+ monitor_file + ' ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/.','\$', 5) - - #launch, it will stop when the softmodem stops - mySSH.command('echo $USER; nohup python3 ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/' + monitor_file + ' 2>&1 &', '\$', 5) - - mySSH.close() + #copyout the monitor script and launch itg +# if self.eNB_Stats=='yes': +# mySSH.open(lIpAddr, lUserName, lPassWord) +# monitor_file='stats_monitor.py' +# mySSH.command('echo ' + lPassWord + ' | sudo -S cp ' + self.eNBSourceCodePath + '/ci-scripts/'+ monitor_file + ' ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/.','\$', 5) +# #launch, it will stop when the softmodem stops +# mySSH.command('echo $USER; nohup python3 ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/' + monitor_file + ' 2>&1 &', '\$', 5) +# mySSH.close() HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'OK', CONST.ALL_PROCESSES_OK) From f1134557d46191d554ca70a93cf64b2f098cfdde Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 4 Oct 2021 18:46:28 +0200 Subject: [PATCH 18/30] tentative change of monitor script launch --- ci-scripts/ran.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 1b9a75f77f..5009b3be98 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -472,9 +472,9 @@ class RANManagement(): time.sleep(20) if self.eNB_Stats=='yes': monitor_file='stats_monitor.py' - mySSH.command('echo ' + lPassWord + ' | sudo -S cp ' + self.eNBSourceCodePath + '/ci-scripts/'+ monitor_file + ' ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/.','\$', 5) - mySSH.command('echo $USER; nohup python3 ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/' + monitor_file + ' 2>&1 &', '\$', 5) - + #mySSH.command('echo ' + lPassWord + ' | sudo -S cp ' + self.eNBSourceCodePath + '/ci-scripts/'+ monitor_file + ' ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/.','\$', 5) + #mySSH.command('echo $USER; nohup python3 ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/' + monitor_file + ' 2>&1 &', '\$', 5) + mySSH.command('echo $USER; nohup python3 ../ci-scripts/' + monitor_file + ' 2>&1 &', '\$', 5) @@ -554,17 +554,6 @@ class RANManagement(): mySSH.close() - #if enabled in the xml, - #copyout the monitor script and launch itg -# if self.eNB_Stats=='yes': -# mySSH.open(lIpAddr, lUserName, lPassWord) -# monitor_file='stats_monitor.py' -# mySSH.command('echo ' + lPassWord + ' | sudo -S cp ' + self.eNBSourceCodePath + '/ci-scripts/'+ monitor_file + ' ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/.','\$', 5) -# #launch, it will stop when the softmodem stops -# mySSH.command('echo $USER; nohup python3 ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/' + monitor_file + ' 2>&1 &', '\$', 5) -# mySSH.close() - - HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'OK', CONST.ALL_PROCESSES_OK) logging.debug('\u001B[1m Initialize eNB/gNB/ocp-eNB Completed\u001B[0m') From 0206cd687fe43bbb144d1781fc77fcd3f917916e Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 4 Oct 2021 18:07:42 +0200 Subject: [PATCH 19/30] do not build in bg --- ci-scripts/xml_files/fr1_gnb_build.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci-scripts/xml_files/fr1_gnb_build.xml b/ci-scripts/xml_files/fr1_gnb_build.xml index d0c91f407d..43cff75620 100644 --- a/ci-scripts/xml_files/fr1_gnb_build.xml +++ b/ci-scripts/xml_files/fr1_gnb_build.xml @@ -26,7 +26,6 @@ wrench 000001 - 000002 @@ -36,7 +35,6 @@ -w USRP -c --gNB --ninja 0 0 - True True From 72d3be655c2eb5bc8604b2e1cbed57c787ed79c8 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 5 Oct 2021 08:25:47 +0200 Subject: [PATCH 20/30] UHD_RFNOC_DIR hack + testcase fix for testing --- ci-scripts/ran.py | 14 +++++++++++++- ci-scripts/xml_files/fr1_sa_quectel.xml | 4 +--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 5e3d8e388f..e91f48b5ab 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -461,7 +461,19 @@ class RANManagement(): mySSH.command('if [ -e rbconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm rbconfig.raw; fi', '\$', 5) mySSH.command('if [ -e reconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm reconfig.raw; fi', '\$', 5) # NOTE: WE SHALL do a check if the executable is present (in case build went wrong) - mySSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + + #hack UHD_RFNOC_DIR variable for gNB / N310 on RHEL8 server: + #if the USRP address is in the xml then we are using an eth USRP (N3xx) + if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'): + gNB = False + else: + gNB = True + if ((self.USRPIPAddress!='') and (gNB==True)): + mySSH.command('echo ' + lPassWord + ' | echo "ulimit -c unlimited && sudo UHD_RFNOC_DIR=/usr/local/share/uhd/rfnoc ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + #otherwise the regular command is ok + else: + mySSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5) mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10) diff --git a/ci-scripts/xml_files/fr1_sa_quectel.xml b/ci-scripts/xml_files/fr1_sa_quectel.xml index 989e626886..379f61a361 100644 --- a/ci-scripts/xml_files/fr1_sa_quectel.xml +++ b/ci-scripts/xml_files/fr1_sa_quectel.xml @@ -31,8 +31,6 @@ 010000 000001 050000 - 000002 - 050001 000001 010002 080000 @@ -57,7 +55,7 @@ Initialize_eNB Initialize gNB - -O ci-scripts/conf_files/gnb.band78.sa.fr1.106PRB.usrpn310.conf --sa -q + -O ci-scripts/conf_files/gnb.band78.sa.fr1.106PRB.usrpn310.conf --sa -q --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 0 0 nr From 700a149d20b7520e4c1d0cb352f96121ffac3eec Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 Oct 2021 15:51:08 +0200 Subject: [PATCH 21/30] update for CI SA test --- ci-scripts/cls_oaicitest.py | 8 +------- ci-scripts/epc.py | 1 + ci-scripts/xml_files/fr1_sa_quectel.xml | 23 +++++++++-------------- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index f6ef7e4826..178f78e1fe 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -2232,13 +2232,7 @@ class OaiCiTest(): def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE): - SSH = sshconnection.SSHConnection() - #RH temporary quick n dirty for test - SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) - cmd = 'echo ' + EPC.Password + ' | sudo -S ip link set dev tun5 mtu 1358' - SSH.command(cmd,'\$',5) - SSH.close() - + SSH = sshconnection.SSHConnection() #kill iperf processes before (in case there are still some remaining) SSH.open(Module_UE.HostIPAddress, Module_UE.HostUsername, Module_UE.HostPassword) diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index 90b4f03165..9fa335e463 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -235,6 +235,7 @@ class EPCManagement(): mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) mySSH.command('./core-network.sh start nrf spgwu', '\$', 60) + mySSH.command('docker-compose up -d trf_gen', '\$', 5) else: logging.error('This option should not occur!') mySSH.close() diff --git a/ci-scripts/xml_files/fr1_sa_quectel.xml b/ci-scripts/xml_files/fr1_sa_quectel.xml index 379f61a361..7cb405ff74 100644 --- a/ci-scripts/xml_files/fr1_sa_quectel.xml +++ b/ci-scripts/xml_files/fr1_sa_quectel.xml @@ -31,6 +31,9 @@ 010000 000001 050000 + 050001 + 070000 + 070001 000001 010002 080000 @@ -84,31 +87,23 @@ 5 - - Ping - Ping: 100pings in 20sec - nrmodule2_quectel - -c 100 -i 0.2 - 5 - - Iperf - iperf (DL/20Mbps/UDP)(60 sec)(single-ue profile) - -u -b 20M -t 60 + iperf (DL/5Mbps/UDP)(60 sec)(single-ue profile) + -u -b 5M -t 60 DL nrmodule2_quectel - 50 + 5 single-ue Iperf - iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile) - -u -b 3M -t 60 + iperf (UL/1Mbps/UDP)(60 sec)(single-ue profile) + -u -b 1M -t 60 UL nrmodule2_quectel - 50 + 5 single-ue From 7442aab5c93a9bca8fa1c56943f176cdab94817b Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 Oct 2021 16:01:30 +0200 Subject: [PATCH 22/30] completing SA test for iperf and stats mon --- ci-scripts/cls_oaicitest.py | 2 +- ci-scripts/xml_files/fr1_sa_quectel.xml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index d75e6f36ad..73e7698ce3 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -2242,7 +2242,7 @@ class OaiCiTest(): def Iperf_Module(self, lock, UE_IPAddress, device_id, idx, ue_num, statusQueue,EPC, Module_UE): - if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): + if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)): #retrieve trf-gen container IP address SSH = sshconnection.SSHConnection() SSH.open(EPC.IPAddress, EPC.UserName, EPC.Password) diff --git a/ci-scripts/xml_files/fr1_sa_quectel.xml b/ci-scripts/xml_files/fr1_sa_quectel.xml index 7cb405ff74..9b658b5483 100644 --- a/ci-scripts/xml_files/fr1_sa_quectel.xml +++ b/ci-scripts/xml_files/fr1_sa_quectel.xml @@ -63,6 +63,7 @@ 0 nr yes + yes 192.168.18.240 From b80d8ae5c56ec689f5cdbee8576ec2f8508391f5 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 Oct 2021 16:11:08 +0200 Subject: [PATCH 23/30] terminate trf_gen when closing CN5G --- ci-scripts/epc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index 9fa335e463..c53d49a04d 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -462,6 +462,7 @@ class EPCManagement(): self.LogCollectOAICN5G() logging.debug('Terminating OAI CN5G') mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) + mySSH.command('docker-compose down', '\$', 5) mySSH.command('./core-network.sh stop nrf spgwu', '\$', 60) else: logging.error('This should not happen!') From 504fb9df7fc66f8cd443001788833ba272dd11cf Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 Oct 2021 16:49:44 +0200 Subject: [PATCH 24/30] fix regex for iperf server analysis --- ci-scripts/cls_oaicitest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 73e7698ce3..5a775c7f8c 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -1998,7 +1998,7 @@ class OaiCiTest(): if type==0: result = re.search('(?P[0-9\.]+ [KMG]bits\/sec) +(?P[0-9\.]+ ms) +(?P[0-9]+)/ +(?P[0-9]+)', str(line)) else: - result = re.search('^\[ \d\].+ +(?P[0-9\.]+ [KMG]bits\/sec) +(?P[0-9\.]+ ms) +(?P[0-9]+)\/(?P[0-9]+)', str(line)) + result = re.search('^\[\s+\d\].+ (?P[0-9\.]+ [KMG]bits\/sec) +(?P[0-9\.]+ ms) +(?P[0-9]+)\/.*(?P[0-9]+)', str(line)) if result is not None: bitrate = result.group('bitrate') From b8ea886efda1b886ae1e4e99cd137f8e429a6c20 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 Oct 2021 16:51:47 +0200 Subject: [PATCH 25/30] add NR stats files to monitoring --- ci-scripts/stats_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/stats_monitor.py b/ci-scripts/stats_monitor.py index 112d8ff799..7ce00dab46 100755 --- a/ci-scripts/stats_monitor.py +++ b/ci-scripts/stats_monitor.py @@ -9,7 +9,7 @@ import numpy as np import os def collect(d): - cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' + cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log' process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) output = process.stdout.readlines() for l in output: From 28ce54a83114ea7a7bd5ce68a7208b7ec02d4cb4 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 6 Oct 2021 17:24:09 +0200 Subject: [PATCH 26/30] fix regex --- ci-scripts/cls_oaicitest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 5a775c7f8c..ada51aae45 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -1998,7 +1998,7 @@ class OaiCiTest(): if type==0: result = re.search('(?P[0-9\.]+ [KMG]bits\/sec) +(?P[0-9\.]+ ms) +(?P[0-9]+)/ +(?P[0-9]+)', str(line)) else: - result = re.search('^\[\s+\d\].+ (?P[0-9\.]+ [KMG]bits\/sec) +(?P[0-9\.]+ ms) +(?P[0-9]+)\/.*(?P[0-9]+)', str(line)) + result = re.search('^\[\s+\d\].+ (?P[0-9\.]+ [KMG]bits\/sec) +(?P[0-9\.]+ ms) +(?P[0-9]+)\/\s*(?P[0-9]+)', str(line)) if result is not None: bitrate = result.group('bitrate') From 5f2f1b5eabb1a662d42dfc2f2d473437b6750605 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 7 Oct 2021 10:32:04 +0200 Subject: [PATCH 27/30] adding the support of different apn depending on CN in ue infra yaml --- ci-scripts/ci_ueinfra.yaml | 9 +++++++-- ci-scripts/cls_module_ue.py | 4 ++-- ci-scripts/cls_oaicitest.py | 2 +- ci-scripts/ran.py | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ci-scripts/ci_ueinfra.yaml b/ci-scripts/ci_ueinfra.yaml index 2fb3f79a2e..0957a1e465 100644 --- a/ci-scripts/ci_ueinfra.yaml +++ b/ci-scripts/ci_ueinfra.yaml @@ -4,7 +4,9 @@ idefix: Kind : quectel Process : Name : quectel-CM - Cmd : /home/oaicicd/quectel-CM/quectel-CM -s oai.ipv4 -4 + Cmd : /home/oaicicd/quectel-CM/quectel-CM -4 -s + Apn : + ltebox : oai.ipv4 WakeupScript : ci_ctl_qtel.py /dev/ttyUSB2 wup DetachScript : ci_ctl_qtel.py /dev/ttyUSB2 detach LogStore : /media/usb-drive/ci_qlogs @@ -20,7 +22,10 @@ nrmodule2_quectel: Kind : quectel Process : Name : quectel-CM - Cmd : /home/nrmodule2/quectel-CM/quectel-CM -s oai -4 #oai is the dnn for OAI CN5G (hardcoded in CN) + Cmd : /home/nrmodule2/quectel-CM/quectel-CM -4 -s + Apn : + OAICN5G : oai + OAI-Rel14-Docker : oai.ipv4 WakeupScript : ci_ctl_qtel.py /dev/ttyUSB7 wup DetachScript : ci_ctl_qtel.py /dev/ttyUSB7 detach LogStore : /media/ci_qlogs diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index 758e7866f0..74ae9c207e 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -61,7 +61,7 @@ class Module_UE: #this method checks if the specified Process is running on the server hosting the module #if not it will be started - def CheckCMProcess(self): + def CheckCMProcess(self,CNType): HOST=self.HostUsername+'@'+self.HostIPAddress COMMAND="ps aux | grep " + self.Process['Name'] + " | grep -v grep " logging.debug(COMMAND) @@ -76,7 +76,7 @@ class Module_UE: logging.debug('Starting ' + self.Process['Name']) mySSH = sshconnection.SSHConnection() mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) - mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S ' + self.Process['Cmd'] + ' &','\$',5) + mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S ' + self.Process['Cmd'] + ' ' + self.Process['Apn'][CNType] + ' &','\$',5) mySSH.close() #checking the process time.sleep(5) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index ada51aae45..8bda1fa202 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -388,7 +388,7 @@ class OaiCiTest(): #RH Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) Module_UE.ue_trace=ue_trace - is_module=Module_UE.CheckCMProcess() + is_module=Module_UE.CheckCMProcess(EPC.Type) if is_module: Module_UE.EnableTrace() time.sleep(5) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 1eb76e505f..807fa1dd07 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -486,7 +486,7 @@ class RANManagement(): monitor_file='stats_monitor.py' #mySSH.command('echo ' + lPassWord + ' | sudo -S cp ' + self.eNBSourceCodePath + '/ci-scripts/'+ monitor_file + ' ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/.','\$', 5) #mySSH.command('echo $USER; nohup python3 ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/' + monitor_file + ' 2>&1 &', '\$', 5) - mySSH.command('echo $USER; nohup python3 ../ci-scripts/' + monitor_file + ' 2>&1 &', '\$', 5) + mySSH.command('echo $USER; nohup python3 ../ci-scripts/' + monitor_file + ' 2>&1 > stats_monitor_execution.log &', '\$', 5) From 5a4dd071fea82b3f79df9425302733de7cd65a65 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 7 Oct 2021 15:55:19 +0200 Subject: [PATCH 28/30] copyout active_net_interfaces.awk to xNB tmp --- ci-scripts/ran.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 807fa1dd07..d447883316 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -340,6 +340,8 @@ class RANManagement(): sys.exit('Insufficient Parameter') logging.debug('Starting eNB/gNB on server: ' + lIpAddr) + SSH.copyout(lIpAddr,lUserName,lPassWord, cwd + "/active_net_interfaces.awk", "/tmp") + self.testCase_id = HTML.testCase_id mySSH = SSH.SSHConnection() From 5f12399c7a3fcf77fb4492abb76c4f434bfe67ce Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 11 Oct 2021 11:40:02 +0200 Subject: [PATCH 29/30] complete nsa 2x2 test --- ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml b/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml index a8e385b854..09ba970977 100644 --- a/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml @@ -21,8 +21,8 @@ --> - TEST-NSA-FR1-TM1 - NSA Ping DL UL with QUECTEL + TEST-NSA-FR1-TM2 + NSA 2x2 Ping DL UL with QUECTEL tasks 1 @@ -32,6 +32,9 @@ 010000 000001 050000 + 000002 + 070000 + 070001 000001 010002 080001 @@ -111,7 +114,7 @@ -u -b 20M -t 60 DL nrmodule2_quectel - 50 + 5 single-ue @@ -121,7 +124,7 @@ -u -b 3M -t 60 UL nrmodule2_quectel - 50 + 5 single-ue From ace498824be1b3b38cf769ddc7e7b944f1cfd75f Mon Sep 17 00:00:00 2001 From: hardy Date: Mon, 11 Oct 2021 11:47:04 +0200 Subject: [PATCH 30/30] remove deploy of trf_gen (redundant) --- ci-scripts/epc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index c53d49a04d..baae5644a1 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -235,7 +235,6 @@ class EPCManagement(): mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) mySSH.command('./core-network.sh start nrf spgwu', '\$', 60) - mySSH.command('docker-compose up -d trf_gen', '\$', 5) else: logging.error('This option should not occur!') mySSH.close()