diff --git a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa index 1e4933838b..07dc1e73f6 100644 --- a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa +++ b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa @@ -262,6 +262,21 @@ pipeline { if(fileExists("enb.log.${env.BUILD_ID}.zip")) { archiveArtifacts "enb.log.${env.BUILD_ID}.zip" } + } + } + } + stage('Log Collection (CN)') { + steps { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'] + ]) { + echo '\u2705 \u001B[32mLog Transfer (CN)\u001B[0m' + sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/logs/oai-cn5g.log.zip ./oai-cn5g.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("oai-cn5g.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "oai-cn5g.log.${env.BUILD_ID}.zip" + } if(fileExists("ci-scripts/test_results.html")) { sh "mv ci-scripts/test_results.html test_results-${JOB_NAME}.html" sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: ${BUILD_ID}#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results-${JOB_NAME}.html" diff --git a/ci-scripts/ci_ueinfra.yaml b/ci-scripts/ci_ueinfra.yaml index c8ba1fd042..2fb3f79a2e 100644 --- a/ci-scripts/ci_ueinfra.yaml +++ b/ci-scripts/ci_ueinfra.yaml @@ -20,7 +20,7 @@ nrmodule2_quectel: Kind : quectel Process : Name : quectel-CM - Cmd : /home/nrmodule2/quectel-CM/quectel-CM -s oai.ipv4 -4 + Cmd : /home/nrmodule2/quectel-CM/quectel-CM -s oai -4 #oai is the dnn for OAI CN5G (hardcoded in CN) WakeupScript : ci_ctl_qtel.py /dev/ttyUSB7 wup DetachScript : ci_ctl_qtel.py /dev/ttyUSB7 detach LogStore : /media/ci_qlogs diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 8cbeaeac1b..d75e6f36ad 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -1549,6 +1549,8 @@ class OaiCiTest(): #target address is different depending on EPC type if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): Target = EPC.MmeIPAddress + elif re.match('OAICN5G', EPC.Type, re.IGNORECASE): + Target = '8.8.8.8' else: Target = EPC.IPAddress #ping from module NIC rather than IP address to make sure round trip is over the air diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index 1c202438d2..9fa335e463 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -220,7 +220,7 @@ class EPCManagement(): mySSH = SSH.SSHConnection() mySSH.open(self.IPAddress, self.UserName, self.Password) if re.match('ltebox', self.Type, re.IGNORECASE): - logging.debug('Using the sabox simulated HSS') + logging.debug('Using the SABOX simulated HSS') mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5) mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('cd /opt/hss_sim0609', '\$', 5) @@ -229,6 +229,13 @@ class EPCManagement(): logging.debug('Using the sabox') mySSH.command('cd /opt/ltebox/tools', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S ./start_sabox', '\$', 5) + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + logging.debug('Starting OAI CN5G') + mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5) + mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) + mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) + mySSH.command('./core-network.sh start nrf spgwu', '\$', 60) + mySSH.command('docker-compose up -d trf_gen', '\$', 5) else: logging.error('This option should not occur!') mySSH.close() @@ -242,6 +249,16 @@ class EPCManagement(): return if re.match('ltebox', self.Type, re.IGNORECASE): self.MmeIPAddress = self.IPAddress + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + mySSH = SSH.SSHConnection() + mySSH.open(self.IPAddress, self.UserName, self.Password) + response=mySSH.command3('docker container ls -f name=oai-amf', 10) + if len(response)>1: + response=mySSH.command3('docker inspect --format=\'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}\' oai-amf', 10) + tmp = str(response[0],'utf-8') + self.MmeIPAddress = tmp.rstrip() + logging.debug('AMF IP Address ' + self.MmeIPAddress) + mySSH.close() def CheckHSSProcess(self, status_queue): try: @@ -433,6 +450,7 @@ class EPCManagement(): mySSH = SSH.SSHConnection() mySSH.open(self.IPAddress, self.UserName, self.Password) if re.match('ltebox', self.Type, re.IGNORECASE): + logging.debug('Terminating SA BOX') mySSH.command('cd /opt/ltebox/tools', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S ./stop_sabox', '\$', 5) time.sleep(1) @@ -440,6 +458,11 @@ class EPCManagement(): mySSH.command('cd scripts', '\$', 5) time.sleep(1) mySSH.command('echo ' + self.Password + ' | sudo -S screen -S simulated_5g_hss -X quit', '\$', 5) + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + self.LogCollectOAICN5G() + logging.debug('Terminating OAI CN5G') + mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) + mySSH.command('./core-network.sh stop nrf spgwu', '\$', 60) else: logging.error('This should not happen!') mySSH.close() @@ -679,3 +702,16 @@ class EPCManagement(): logging.error('This option should not occur!') mySSH.close() + def LogCollectOAICN5G(self): + mySSH = SSH.SSHConnection() + mySSH.open(self.IPAddress, self.UserName, self.Password) + logging.debug('OAI CN5G Collecting Log files to workspace') + mySSH.command('echo ' + self.Password + ' | sudo rm -rf ' + self.SourceCodePath + '/logs', '\$', 5) + mySSH.command('mkdir ' + self.SourceCodePath + '/logs','\$', 5) + containers_list=['oai-smf','oai-spgwu','oai-amf','oai-nrf'] + for c in containers_list: + mySSH.command('docker logs ' + c + ' > ' + self.SourceCodePath + '/logs/' + c + '.log', '\$', 5) + mySSH.command('cd ' + self.SourceCodePath + '/logs', '\$', 5) + mySSH.command('zip oai-cn5g.log.zip *.log', '\$', 60) + mySSH.close() + diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 5009b3be98..1eb76e505f 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -462,7 +462,19 @@ class RANManagement(): mySSH.command('if [ -e rbconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm rbconfig.raw; fi', '\$', 5) mySSH.command('if [ -e reconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm reconfig.raw; fi', '\$', 5) # NOTE: WE SHALL do a check if the executable is present (in case build went wrong) - mySSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + + #hack UHD_RFNOC_DIR variable for gNB / N310 on RHEL8 server: + #if the USRP address is in the xml then we are using an eth USRP (N3xx) + if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'): + gNB = False + else: + gNB = True + if ((self.USRPIPAddress!='') and (gNB==True)): + mySSH.command('echo ' + lPassWord + ' | echo "ulimit -c unlimited && sudo UHD_RFNOC_DIR=/usr/local/share/uhd/rfnoc ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + #otherwise the regular command is ok + else: + mySSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5) mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10) diff --git a/ci-scripts/sshconnection.py b/ci-scripts/sshconnection.py index b4087c9695..b85c40a0be 100644 --- a/ci-scripts/sshconnection.py +++ b/ci-scripts/sshconnection.py @@ -163,6 +163,18 @@ class SSHConnection(): lSsh = subprocess.Popen(["ssh", "%s" % myHost, commandline],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) self.cmd2Results = str(lSsh.stdout.readlines()) + def command3(self, commandline, timeout, silent=False): + if not silent: + logging.debug(commandline) + self.cmd2Results = '' + myHost = self.username + '@' + self.ipaddress + # CAUTION: THIS METHOD IMPLIES THAT THERE ARE VALID SSH KEYS + # BETWEEN THE PYTHON EXECUTOR NODE AND THE REMOTE HOST + # OTHERWISE IT WON'T WORK + lSsh = subprocess.Popen(["ssh", "%s" % myHost, commandline],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + return lSsh.stdout.readlines() + + def close(self): self.ssh.timeout = 5 self.ssh.sendline('exit') diff --git a/ci-scripts/xml_files/fr1_gnb_build.xml b/ci-scripts/xml_files/fr1_gnb_build.xml index d0c91f407d..43cff75620 100644 --- a/ci-scripts/xml_files/fr1_gnb_build.xml +++ b/ci-scripts/xml_files/fr1_gnb_build.xml @@ -26,7 +26,6 @@ wrench 000001 - 000002 @@ -36,7 +35,6 @@ -w USRP -c --gNB --ninja 0 0 - True True diff --git a/ci-scripts/xml_files/fr1_sa_quectel.xml b/ci-scripts/xml_files/fr1_sa_quectel.xml index 4c8fba0611..7cb405ff74 100644 --- a/ci-scripts/xml_files/fr1_sa_quectel.xml +++ b/ci-scripts/xml_files/fr1_sa_quectel.xml @@ -31,6 +31,9 @@ 010000 000001 050000 + 050001 + 070000 + 070001 000001 010002 080000 @@ -55,7 +58,7 @@ Initialize_eNB Initialize gNB - -O ci-scripts/conf_files/gnb.band78.sa.fr1.106PRB.usrpn310.conf --sa -q + -O ci-scripts/conf_files/gnb.band78.sa.fr1.106PRB.usrpn310.conf --sa -q --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 0 0 nr @@ -81,34 +84,26 @@ Ping: 20pings in 20sec nrmodule2_quectel -c 20 - 50 - - - - Ping - Ping: 100pings in 20sec - nrmodule2_quectel - -c 100 -i 0.2 - 50 + 5 Iperf - iperf (DL/20Mbps/UDP)(60 sec)(single-ue profile) - -u -b 20M -t 60 + iperf (DL/5Mbps/UDP)(60 sec)(single-ue profile) + -u -b 5M -t 60 DL nrmodule2_quectel - 50 + 5 single-ue Iperf - iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile) - -u -b 3M -t 60 + iperf (UL/1Mbps/UDP)(60 sec)(single-ue profile) + -u -b 1M -t 60 UL nrmodule2_quectel - 50 + 5 single-ue