diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab index 5fa46ddcc0..cd2d783638 100644 --- a/ci-scripts/Jenkinsfile-gitlab +++ b/ci-scripts/Jenkinsfile-gitlab @@ -133,6 +133,7 @@ pipeline { script { def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): Merge Conflicts -- Cannot perform CI" addGitLabMRComment comment: message + currentBuild.result = 'FAILURE' } } } @@ -283,6 +284,11 @@ pipeline { } } post { + failure { + script { + currentBuild.result = 'FAILURE' + } + } always { script { dir ('archives') { @@ -392,7 +398,7 @@ pipeline { } } } - stage ("Test FDD - Band 7 - B210") { + stage ("Test MONOLITHIC - FDD - Band 7 - B210") { steps { script { if ("MERGE".equals(env.gitlabActionType)) { @@ -443,7 +449,7 @@ pipeline { } } } - stage ("Test TDD - Band 40 - B210") { + stage ("Test MONOLITHIC - TDD - Band 40 - B210") { steps { script { if ("MERGE".equals(env.gitlabActionType)) { @@ -647,6 +653,57 @@ pipeline { } } } + stage ("Test OAI UE Sniffing - FDD - Band 20 - B200") { + steps { + script { + if ("MERGE".equals(env.gitlabActionType)) { + //gitlabCommitStatus(name: "Test-OAI-UE-FDD-Band20") { + build job: 'UE-CI-FDD-Band20-B200', + parameters: [ + string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), + string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)), + string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)), + booleanParam(name: 'eNB_mergeRequest', value: true), + string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch)) + ] + //} + } else { + //gitlabCommitStatus(name: "Test-OAI-UE-FDD-Band20") { + build job: 'UE-CI-FDD-Band20-B200', + parameters: [ + string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)), + string(name: 'eNB_Branch', value: String.valueOf(GIT_BRANCH)), + string(name: 'eNB_CommitID', value: String.valueOf(GIT_COMMIT)), + booleanParam(name: 'eNB_mergeRequest', value: false) + ] + //} + } + } + } + post { + // In case of any non-success, we are retrieving the HTML report of the last completed + // slave job. + // The only drop-back is that we may retrieve the HTML report of a previous build + always { + script { + if (!fileExists('test_results-UE-CI-FDD-Band20-B200.html')) { + copyArtifacts(projectName: 'UE-CI-FDD-Band20-B200', + filter: 'test_results*.html', + selector: lastCompleted()) + if (fileExists('test_results-UE-CI-FDD-Band20-B200.html')) { + sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_results-UE-CI-FDD-Band20-B200.html" + archiveArtifacts artifacts: 'test_results-UE-CI-FDD-Band20-B200.html' + } + } + } + } + failure { + script { + currentBuild.result = 'FAILURE' + } + } + } + } } post { always { diff --git a/ci-scripts/Jenkinsfile-tmp-ue b/ci-scripts/Jenkinsfile-tmp-ue new file mode 100644 index 0000000000..70fe29879f --- /dev/null +++ b/ci-scripts/Jenkinsfile-tmp-ue @@ -0,0 +1,300 @@ +#!/bin/groovy +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +// Template Jenkins Declarative Pipeline script to run Test w/ RF HW + +// Location of the python executor node shall be in the same subnet as the others servers +def pythonExecutor = params.pythonExecutor + +// Location of the test XML file to be run +def testXMLFile = params.pythonTestXmlFile +def mainPythonAllXmlFiles = "" +def buildStageStatus = true + +// Name of the test stage +def testStageName = params.pipelineTestStageName + +// Name of the phone resource +def ciSmartPhoneResource = params.smartphonesResource + +// Terminate Status +def termUE = 0 +def termENB = 1 +def termSPGW = 2 +def termMME = 3 +def termHSS = 4 +def termStatusArray = new Boolean[termHSS + 1] +termStatusArray[termUE] = false +termStatusArray[termENB] = false +termStatusArray[termSPGW] = false +termStatusArray[termMME] = false +termStatusArray[termHSS] = false + +// Global Parameters. Normally they should be populated when the master job +// triggers the slave job with parameters +def eNB_Repository +def eNB_Branch +def eNB_CommitID +def eNB_AllowMergeRequestProcess = false +def eNB_TargetBranch + +pipeline { + agent { + label pythonExecutor + } + options { + disableConcurrentBuilds() + ansiColor('xterm') + lock (ciSmartPhoneResource) + } + // the following parameter options are commented out so it shows the ones + // that you SHALL have to run the job. + // You can use them as template + /* + parameters { + //node-test parameters + string(name: 'pythonExecutor', defaultValue: 'nodea', description: 'Node where the pipeline - python scripts will be executed') + string(name: 'pythonTestXmlFile', defaultValue: 'enb_usrpB210_band7_50PRB.xml', description: 'Location of the Test XML to be run') + string(name: 'pipelineTestStageName', defaultValue: 'Test COTS-UE - OAI eNB - LTEBOX EPC', description: 'Naming of the Test Stage') + booleanParam(name: 'pipelineZipsConsoleLog', defaultValue: 'True', description: 'If true, the pipeline script retrieves the job console log, zips it and archives it as artifact') + string(name: 'smartphonesResource', defaultValue: 'CI-Bench-1-Phones', description: 'Lockeable Resource to prevent multiple jobs to run simultaneously with the same resource') + + //eNB parameters + string(name: 'eNB_IPAddress', defaultValue: '192.168.XX.XX', description: 'IP Address of eNB') + credentials(name: 'eNB_Credentials', defaultValue: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', credentialType: "Username with password", required: true, description: 'Credentials for eNB') + string(name: 'eNB_SourceCodePath', defaultValue: '/tmp/CI-enb', description: 'Full path of eNB source code') + + //EPC parameters + string(name: 'EPC_IPAddress', defaultValue: '192.168.XX.XX', description: 'IP Address of EPC') + string(name: 'EPC_Type', defaultValue: 'ltebox', description: 'EPC type: OAI or ltebox') + credentials(name: 'EPC_Credentials', defaultValue: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', credentialType: "Username with password", required: true, description: 'Credentials for EPC') + string(name: 'EPC_SourceCodePath', defaultValue: '/tmp/CI-enb', description: 'Full path of EPC source code') + + //ADB server parameters + string(name: 'ADB_IPAddress', defaultValue: '192.168.XX.XX', description: 'IP Address of ADB server') + credentials(name: 'ADB_Credentials', defaultValue: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', credentialType: "Username with password", required: true, description: 'Credentials for ADB') + } + */ + + stages { + stage ("Verify Parameters") { + steps { + script { + echo '\u2705 \u001B[32mVerify Parameters\u001B[0m' + def allParametersPresent = true + + // It is already to late to check it + if (params.pythonExecutor != null) { + echo "eNB CI executor node : ${pythonExecutor}" + } + // If not present picking a default XML file + if (params.pythonTestXmlFile == null) { + // picking default + testXMLFile = 'xml_files/enb_usrpB210_band7_50PRB.xml' + echo "Test XML file(default): ${testXMLFile}" + mainPythonAllXmlFiles += "--XMLTestFile=" + testXMLFile + " " + } else { + String[] myXmlTestSuite = testXMLFile.split("\\r?\\n") + for (xmlFile in myXmlTestSuite) { + if (fileExists("ci-scripts/" + xmlFile)) { + mainPythonAllXmlFiles += "--XMLTestFile=" + xmlFile + " " + echo "Test XML file : ${xmlFile}" + } + } + } + // If not present picking a default Stage Name + if (params.pipelineTestStageName == null) { + // picking default + testStageName = 'Template Test Stage' + } + + if (params.smartphonesResource == null) { + allParametersPresent = false + } + if (params.UE_IPAddress == null) { + allParametersPresent = false + } + if (params.UE_SourceCodePath == null) { + allParametersPresent = false + } + if (params.UE_Credentials == null) { + allParametersPresent = false + } + // the following 4 parameters should be pushed by the master trigger + // if not present, take the job GIT variables (used for developing) + if (params.eNB_Repository == null) { + eNB_Repository = env.GIT_URL + } else { + eNB_Repository = params.eNB_Repository + } + echo "eNB_Repository : ${eNB_Repository}" + if (params.eNB_Branch == null) { + eNB_Branch = env.GIT_BRANCH + } else { + eNB_Branch = params.eNB_Branch + } + echo "eNB_Branch : ${eNB_Branch}" + if (params.eNB_CommitID == null) { + eNB_CommitID = env.GIT_COMMIT + } else { + eNB_CommitID = params.eNB_CommitID + } + echo "eNB_CommitID : ${eNB_CommitID}" + if (params.eNB_mergeRequest != null) { + eNB_AllowMergeRequestProcess = params.eNB_mergeRequest + if (eNB_AllowMergeRequestProcess) { + if (params.eNB_TargetBranch != null) { + eNB_TargetBranch = params.eNB_TargetBranch + } else { + eNB_TargetBranch = 'develop' + } + echo "eNB_TargetBranch : ${eNB_TargetBranch}" + } + } +/* + if (params.EPC_IPAddress == null) { + allParametersPresent = false + } + if (params.EPC_Type == null) { + allParametersPresent = false + } + if (params.EPC_SourceCodePath == null) { + allParametersPresent = false + } + if (params.EPC_Credentials == null) { + allParametersPresent = false + } +*/ + if (params.ADB_IPAddress == null) { + allParametersPresent = false + } + if (params.ADB_Credentials == null) { + allParametersPresent = false + } + + if (allParametersPresent) { + echo "All parameters are present" + if (eNB_AllowMergeRequestProcess) { + sh "git fetch" + sh "./ci-scripts/doGitLabMerge.sh --src-branch ${eNB_Branch} --src-commit ${eNB_CommitID} --target-branch ${eNB_TargetBranch} --target-commit latest" + } else { + sh "git fetch" + sh "git checkout -f ${eNB_CommitID}" + } + } else { + echo "Some parameters are missing" + sh "./ci-scripts/fail.sh" + } + } + } + } + stage ("Build and Test") { + steps { + script { + dir ('ci-scripts') { + echo "\u2705 \u001B[32m${testStageName}\u001B[0m" + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'], + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.ADB_Credentials}", usernameVariable: 'ADB_Username', passwordVariable: 'ADB_Password'] + ]) { + sh "python3 main.py --mode=InitiateHtml --eNBRepository=${eNB_Repository} --eNBBranch=${eNB_Branch} --eNBCommitID=${eNB_CommitID} --eNB_AllowMerge=${eNB_AllowMergeRequestProcess} --eNBTargetBranch=${eNB_TargetBranch} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} ${mainPythonAllXmlFiles}" + String[] myXmlTestSuite = testXMLFile.split("\\r?\\n") + for (xmlFile in myXmlTestSuite) { + if (fileExists(xmlFile)) { + try { + sh "python3 main.py --mode=TestUE --UEIPAddress=${params.UE_IPAddress} --eNBRepository=${eNB_Repository} --eNBBranch=${eNB_Branch} --eNBCommitID=${eNB_CommitID} --eNB_AllowMerge=${eNB_AllowMergeRequestProcess} --eNBTargetBranch=${eNB_TargetBranch} --UEUserName=${UE_Username} --UEPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} --XMLTestFile=${xmlFile}" + } catch (Exception e) { + currentBuild.result = 'FAILURE' + buildStageStatus = false + } + } + } + sh "python3 main.py --mode=FinalizeHtml --finalStatus=${buildStageStatus} --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password}" + } + } + } + } + } + stage('Log Collection') { + parallel { + stage('Log Collection (OAI UE - Build)') { + steps { + echo '\u2705 \u001B[32mLog Collection (OAI UE - Build)\u001B[0m' + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'] + ]) { + sh "python3 ci-scripts/main.py --mode=LogCollectBuild --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath}" + + echo '\u2705 \u001B[32mLog Transfer (UE - Build)\u001B[0m' + sh "sshpass -p \'${UE_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${UE_Username}@${params.UE_IPAddress}:${UE_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("build.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "build.log.${env.BUILD_ID}.zip" + } + } + } + } + stage('Log Collection (OAI UE - Run)') { + steps { + echo '\u2705 \u001B[32mLog Collection (OAI UE - Run)\u001B[0m' + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'] + ]) { + sh "python3 ci-scripts/main.py --mode=LogCollectOAIUE --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath}" + + echo '\u2705 \u001B[32mLog Transfer (UE - Run)\u001B[0m' + sh "sshpass -p \'${UE_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${UE_Username}@${params.UE_IPAddress}:${UE_SourceCodePath}/cmake_targets/ue.log.zip ./ue.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("ue.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "ue.log.${env.BUILD_ID}.zip" + } + if(fileExists("ci-scripts/test_results.html")) { + sh "mv ci-scripts/test_results.html test_results-${JOB_NAME}.html" + sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: ${BUILD_ID}#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results-${JOB_NAME}.html" + archiveArtifacts "test_results-${JOB_NAME}.html" + } + } + } + } + } + } + } + + post { + always { + script { + if (params.pipelineZipsConsoleLog != null) { + if (params.pipelineZipsConsoleLog) { + echo "Archiving Jenkins console log" + sh "wget --no-check-certificate --no-proxy ${env.JENKINS_URL}/job/${env.JOB_NAME}/${env.BUILD_ID}/consoleText -O consoleText.log || true" + sh "zip -m consoleText.log.${env.BUILD_ID}.zip consoleText.log || true" + if(fileExists("consoleText.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "consoleText.log.${env.BUILD_ID}.zip" + } + } + } + } + } + } +} diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 5b1dfd8901..96d7397e46 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -1,4 +1,3 @@ -#/* # * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more # * contributor license agreements. See the NOTICE file distributed with # * this work for additional information regarding copyright ownership. @@ -43,6 +42,9 @@ ENB_PROCESS_SEG_FAULT = -11 ENB_PROCESS_ASSERTION = -12 ENB_PROCESS_REALTIME_ISSUE = -13 ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14 +UE_PROCESS_NOLOGFILE_TO_ANALYZE = -20 +UE_PROCESS_COULD_NOT_SYNC = -21 +UE_PROCESS_ASSERTION = -22 HSS_PROCESS_FAILED = -2 HSS_PROCESS_OK = +2 MME_PROCESS_FAILED = -3 @@ -122,13 +124,21 @@ class SSHConnection(): self.htmlTabNames = [] self.htmlTabIcons = [] self.finalStatus = False + self.OsVersion = '' + self.KernelVersion = '' + self.UhdVersion = '' + self.UsrpBoard = '' + self.CpuNb = '' + self.CpuModel = '' + self.CpuMHz = '' + self.UEIPAddress = '' + self.UEUserName = '' + self.UEPassword = '' + self.UE_instance = '' + self.UESourceCodePath = '' + self.Build_OAI_UE_args = '' + self.Initialize_OAI_UE_args = '' self.eNBOsVersion = '' - self.eNBKernelVersion = '' - self.eNBUhdVersion = '' - self.eNBUsrpBoard = '' - self.eNBCpuNb = '' - self.eNBCpuModel = '' - self.eNBCpuMHz = '' def open(self, ipaddress, username, password): count = 0 @@ -324,11 +334,46 @@ class SSHConnection(): self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) self.command('mv compile_oai_enb.log ' + 'build_log_' + self.testCase_id, '\$', 5) - # Workaround to run with develop-nr - self.command('if [ -e ran_build ]; then cp -rf ran_build lte_build_oai; fi', '\$', 30) self.close() self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK) + def BuildOAIUE(self): + if self.UEIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': + Usage() + sys.exit('Insufficient Parameter') + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('mkdir -p ' + self.UESourceCodePath, '\$', 5) + self.command('cd ' + self.UESourceCodePath, '\$', 5) + self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.eNBRepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600) + # here add a check if git clone or git fetch went smoothly + self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) + self.command('git config user.name "OAI Jenkins"', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S git clean -x -d -ff', '\$', 30) + # if the commit ID is provided use it to point to it + if self.eNBCommitID != '': + self.command('git checkout -f ' + self.eNBCommitID, '\$', 5) + # if the branch is not develop, then it is a merge request and we need to do + # the potential merge. Note that merge conflicts should already been checked earlier + if (self.eNB_AllowMerge): + if self.eNBTargetBranch == '': + if (self.eNBBranch != 'develop') and (self.eNBBranch != 'origin/develop'): + self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + else: + logging.debug('Merging with the target branch: ' + self.eNBTargetBranch) + self.command('git merge --ff origin/' + self.eNBTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + self.command('source oaienv', '\$', 5) + self.command('cd cmake_targets', '\$', 5) + self.command('mkdir -p log', '\$', 5) + self.command('chmod 777 log', '\$', 5) + # no need to remove in log (git clean did the trick) + self.command('stdbuf -o0 ./build_oai ' + self.Build_OAI_UE_args + ' 2>&1 | stdbuf -o0 tee -a compile_oai_ue.log', 'Bypassing the Tests', 600) + self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) + self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) + self.command('mv compile_oai_ue.log ' + 'build_log_' + self.testCase_id, '\$', 5) + self.close() + self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE') + + def InitializeHSS(self): if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '': Usage() @@ -452,8 +497,8 @@ class SSHConnection(): # Launch eNB with the modified config file self.command('source oaienv', '\$', 5) self.command('cd cmake_targets', '\$', 5) - self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.eNBSourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh ', '\$', 5) - self.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh ', '\$', 5) + self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.eNBSourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + self.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) self.command('echo ' + self.eNBPassword + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5) self.command('echo ' + self.eNBPassword + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + self.eNBSourceCodePath + '/cmake_targets -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) if not rruCheck: @@ -535,6 +580,99 @@ class SSHConnection(): job.join() self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) + def InitializeOAIUE(self): + if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': + Usage() + sys.exit('Insufficient Parameter') + #initialize_OAI_UE_flag = True + #pStatus = self.CheckOAIUEProcessExist(initialize_OAI_UE_flag) + #if (pStatus < 0): + # self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', pStatus) + # self.CreateHtmlTabFooter(False) + # sys.exit(1) + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + # b2xx_fx3_utils reset procedure + self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 5) + result = re.search('type: b200', str(self.ssh.before)) + if result is not None: + logging.debug('Found a B2xx device --> resetting it') + self.command('echo ' + self.UEPassword + ' | sudo -S sudo b2xx_fx3_utils --reset-device', '\$', 5) + # Reloading FGPA bin firmware + self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 5) + else: + logging.debug('Did not find any B2xx device') + self.command('cd ' + self.UESourceCodePath, '\$', 5) + # Initialize_OAI_UE_args usually start with -C and followed by the location in repository + #full_config_file = self.Initialize_OAI_UE_args.replace('-O ','') + #extIdx = full_config_file.find('.conf') + #if (extIdx > 0): + # extra_options = full_config_file[extIdx + 5:] + # # if tracer options is on, compiling and running T Tracer + # result = re.search('T_stdout', str(extra_options)) + ## if result is not None: + # logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m') + # self.command('cd common/utils/T/tracer', '\$', 5) + # self.command('make', '\$', 10) + # self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.UESourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', self.UEUserName, 5) + # self.command('cd ' + self.UESourceCodePath, '\$', 5) + # full_config_file = full_config_file[:extIdx + 5] + # config_path, config_file = os.path.split(full_config_file) + #ci_full_config_file = config_path + '/ci-' + config_file + #rruCheck = False + #result = re.search('rru', str(config_file)) + #if result is not None: + # rruCheck = True + ## Make a copy and adapt to EPC / UE IP addresses + #self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5) + #self.command('sed -i -e \'s/CI_UE_IP_ADDR/' + self.UEIPAddress + '/\' ' + ci_full_config_file, '\$', 2); + # Launch UE with the modified config file + self.command('source oaienv', '\$', 5) + self.command('cd cmake_targets/lte_build_oai/build', '\$', 5) + self.command('echo "ulimit -c unlimited && ./lte-uesoftmodem ' + self.Initialize_OAI_UE_args + '" > ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) + self.command('chmod 775 ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S -E daemon --inherit --unsafe --name=ue' + str(self.UE_instance) + '_daemon --chdir=' + self.UESourceCodePath + '/cmake_targets/lte_build_oai/build -o ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) + self.UELogFile = 'ue_' + self.testCase_id + '.log' + time.sleep(6) + self.command('cd ../..', '\$', 5) + doLoop = True + loopCounter = 10 + while (doLoop): + loopCounter = loopCounter - 1 + if (loopCounter == 0): + # In case of T tracer recording, we may need to kill it + #result = re.search('T_stdout', str(self.Initialize_OAI_UE_args)) + #if result is not None: + # self.command('killall --signal SIGKILL record', '\$', 5) + self.close() + doLoop = False + logging.error('\u001B[1;37;41m UE logging system did not show got sync! \u001B[0m') + self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', ALL_PROCESSES_OK, 'OAI UE') + self.CreateHtmlTabFooter(False) + ## In case of T tracer recording, we need to kill tshark on EPC side + #result = re.search('T_stdout', str(self.Initialize_OAI_UE_args)) + #if result is not None: + # self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) + # logging.debug('\u001B[1m Stopping tshark \u001B[0m') + # self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) + # self.close() + # time.sleep(1) + # pcap_log_file = 'enb_' + self.testCase_id + '_s1log.pcap' + # copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + pcap_log_file, '.') + # if (copyin_res == 0): + # self.copyout(self.UEIPAddress, self.UEUserName, self.UEPassword, pcap_log_file, self.UESourceCodePath + '/cmake_targets/.') + sys.exit(1) + else: + self.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4) + result = re.search('got sync', str(self.ssh.before)) + if result is None: + time.sleep(6) + else: + doLoop = False + self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE') + logging.debug('\u001B[1m Initialize OAI UE Completed\u001B[0m') + self.close() + def checkDevTTYisUnlocked(self): self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword) count = 0 @@ -1674,6 +1812,48 @@ class SSHConnection(): result = logStatus return result + def CheckOAIUEProcessExist(self, initialize_OAI_UE_flag): + multi_jobs = [] + status_queue = SimpleQueue() + if initialize_OAI_UE_flag == False: + p = Process(target = SSH.CheckOAIUEProcess, args = (status_queue,)) + p.daemon = True + p.start() + multi_jobs.append(p) + for job in multi_jobs: + job.join() + + if (status_queue.empty()): + return -15 + else: + result = 0 + while (not status_queue.empty()): + status = status_queue.get() + if (status < 0): + result = status + if result == OAI_UE_PROCESS_FAILED: + fileCheck = re.search('enb_', str(self.UELogFile)) + if fileCheck is not None: + self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.') + logStatus = self.AnalyzeLogFile_UE(self.UELogFile) + if logStatus < 0: + result = logStatus + return result + + def CheckOAIUEProcess(self, status_queue): + try: + self.open(self.OAIUEIPAddress, self.OAIUEUserName, self.OAIUEPassword) + self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never lte-uesoftmodem', '\$', 5) + result = re.search('lte-uesoftmodem', str(self.ssh.before)) + if result is None: + logging.debug('\u001B[1;37;41m OAI UE Process Not Found! \u001B[0m') + status_queue.put(OAI_UE_PROCESS_FAILED) + else: + status_queue.put(OAI_UE_PROCESS_OK) + self.close() + except: + os.kill(os.getppid(),signal.SIGUSR1) + def CheckeNBProcess(self, status_queue): try: self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) @@ -1891,6 +2071,156 @@ class SSHConnection(): return ENB_PROCESS_REALTIME_ISSUE return 0 + def AnalyzeLogFile_UE(self, UElogFile): + if (not os.path.isfile('./' + UElogFile)): + return -1 + ue_log_file = open('./' + UElogFile, 'r') + foundAssertion = False + msgAssertion = '' + msgLine = 0 + foundSegFault = False + foundRealTimeIssue = False + rlcDiscardBuffer = 0 + rachCanceledProcedure = 0 + uciStatMsgCount = 0 + pdcpFailure = 0 + ulschFailure = 0 + no_cell_sync_found = False + mib_found = False + frequency_found = False + self.htmlUEFailureMsg = '' + for line in ue_log_file.readlines(): + result = re.search('[Ss]egmentation [Ff]ault', str(line)) + if result is not None: + foundSegFault = True + result = re.search('[Cc]ore [dD]ump', str(line)) + if result is not None: + foundSegFault = True + result = re.search('[Aa]ssertion', str(line)) + if result is not None: + foundAssertion = True + result = re.search('LLL', str(line)) + if result is not None: + foundRealTimeIssue = True + if foundAssertion and (msgLine < 3): + msgLine += 1 + msgAssertion += str(line) + result = re.search('uci->stat', str(line)) + if result is not None: + uciStatMsgCount += 1 + # No cell synchronization found, abandoning + result = re.search('No cell synchronization found, abandoning', str(line)) + if result is not None: + no_cell_sync_found = True + result = re.search("MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P\d{1,3}), N_RB_DL (?P\d{1,3}), PHICH DURATION (?P\d), PHICH RESOURCE (?P.{1,4}), TX_ANT (?P\d)", str(line)) + if result is not None and (not mib_found): + try: + mibMsg = "MIB Information: " + result.group(1) + ', ' + result.group(2) + self.htmlUEFailureMsg += mibMsg + '\n' + logging.debug('\033[94m' + mibMsg + '\033[0m') + mibMsg = " nidcell = " + result.group('nidcell') + self.htmlUEFailureMsg += mibMsg + logging.debug('\033[94m' + mibMsg + '\033[0m') + mibMsg = " n_rb_dl = " + result.group('n_rb_dl') + self.htmlUEFailureMsg += mibMsg + '\n' + logging.debug('\033[94m' + mibMsg + '\033[0m') + mibMsg = " phich_duration = " + result.group('phich_duration') + self.htmlUEFailureMsg += mibMsg + logging.debug('\033[94m' + mibMsg + '\033[0m') + mibMsg = " phich_resource = " + result.group('phich_resource') + self.htmlUEFailureMsg += mibMsg + '\n' + logging.debug('\033[94m' + mibMsg + '\033[0m') + mibMsg = " tx_ant = " + result.group('tx_ant') + self.htmlUEFailureMsg += mibMsg + '\n' + logging.debug('\033[94m' + mibMsg + '\033[0m') + mib_found = True + except Exception as e: + logging.error('\033[91m' + "MIB marker was not found" + '\033[0m') + result = re.search("Measured Carrier Frequency (?P\d{1,15}) Hz", str(line)) + if result is not None and (not frequency_found): + try: + mibMsg = "Measured Carrier Frequency = " + result.group('measured_carrier_frequency') + ' Hz' + self.htmlUEFailureMsg += mibMsg + '\n' + logging.debug('\033[94m' + mibMsg + '\033[0m') + frequency_found = True + except Exception as e: + logging.error('\033[91m' + "Measured Carrier Frequency not found" + '\033[0m') + result = re.search("Found (?P[\w,\s]{1,15}) \(name from internal table\)", str(line)) + if result is not None: + try: + mibMsg = "The operator is: " + result.group('operator') + self.htmlUEFailureMsg += mibMsg + '\n' + logging.debug('\033[94m' + mibMsg + '\033[0m') + except Exception as e: + logging.error('\033[91m' + "Operator name not found" + '\033[0m') + result = re.search("SIB5 InterFreqCarrierFreq element (.{1,4})/(.{1,4})", str(line)) + if result is not None: + try: + mibMsg = "SIB5 InterFreqCarrierFreq element " + result.group(1) + '/' + result.group(2) + self.htmlUEFailureMsg += mibMsg + ' -> ' + logging.debug('\033[94m' + mibMsg + '\033[0m') + except Exception as e: + logging.error('\033[91m' + "SIB5 InterFreqCarrierFreq element not found" + '\033[0m') + result = re.search("DL Carrier Frequency/ARFCN : (?P\d{1,15}/\d{1,4})", str(line)) + if result is not None: + try: + freq = result.group('carrier_frequency') + new_freq = re.sub('/[0-9]+','',freq) + float_freq = float(new_freq) / 1000000 + self.htmlUEFailureMsg += 'DL Freq: ' + ('%.1f' % float_freq) + ' MHz' + logging.debug('\033[94m' + " DL Carrier Frequency is: " + freq + '\033[0m') + except Exception as e: + logging.error('\033[91m' + " DL Carrier Frequency not found" + '\033[0m') + result = re.search("AllowedMeasBandwidth : (?P\d{1,7})", str(line)) + if result is not None: + try: + prb = result.group('allowed_bandwidth') + self.htmlUEFailureMsg += ' -- PRB: ' + prb + '\n' + logging.debug('\033[94m' + " AllowedMeasBandwidth: " + prb + '\033[0m') + except Exception as e: + logging.error('\033[91m' + " AllowedMeasBandwidth not found" + '\033[0m') + ue_log_file.close() + if uciStatMsgCount > 0: + statMsg = 'UE showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)' + logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') + self.htmlUEFailureMsg += statMsg + '\n' + if pdcpFailure > 0: + statMsg = 'UE showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)' + logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') + self.htmlUEFailureMsg += statMsg + '\n' + if ulschFailure > 0: + statMsg = 'UE showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)' + logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') + self.htmlUEFailureMsg += statMsg + '\n' + if rachCanceledProcedure > 0: + rachMsg = 'UE cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)' + logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rachMsg + '\n' + if foundSegFault: + logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m') + return ENB_PROCESS_SEG_FAULT + if foundAssertion: + logging.debug('\u001B[1;37;43m UE ended with an assertion! \u001B[0m') + # removed for esthetics + #self.htmlUEFailureMsg += msgAssertion + self.htmlUEFailureMsg += 'UE ended with an assertion!\n' + if not mib_found or not frequency_found: + return UE_PROCESS_ASSERTION + if foundRealTimeIssue: + logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m') + self.htmlUEFailureMsg += 'UE faced real time issues!\n' + #return ENB_PROCESS_REALTIME_ISSUE + if no_cell_sync_found and not mib_found: + logging.debug('\u001B[1;37;41m UE could not synchronize ! \u001B[0m') + self.htmlUEFailureMsg += 'UE could not synchronize!\n' + return UE_PROCESS_COULD_NOT_SYNC + if rlcDiscardBuffer > 0: + rlcMsg = 'UE RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)' + logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rlcMsg + '\n' + return ENB_PROCESS_REALTIME_ISSUE + return 0 + def TerminateeNB(self): self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) self.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5) @@ -2031,6 +2361,75 @@ class SSHConnection(): job.join() self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) + def TerminateOAIUE(self): + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S daemon --name=ue' + str(self.UE_instance) + '_daemon --stop', '\$', 5) + self.command('rm -f my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGINT lte-uesoftmodem || true', '\$', 5) + time.sleep(5) + self.command('stdbuf -o0 ps -aux | grep -v grep | grep lte-uesoftmodem', '\$', 5) + result = re.search('lte-uesoftmodem', str(self.ssh.before)) + if result is not None: + self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGKILL lte-uesoftmodem || true', '\$', 5) + time.sleep(5) + self.close() + # If tracer options is on, stopping tshark on EPC side + result = re.search('T_stdout', str(self.Initialize_OAI_UE_args)) + if result is not None: + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + logging.debug('\u001B[1m Stopping tshark \u001B[0m') + self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) + time.sleep(1) + pcap_log_file = self.UELogFile.replace('.log', '_s1log.pcap') + self.command('echo ' + self.UEPassword + ' | sudo -S chmod 666 /tmp/' + pcap_log_file, '\$', 5) + self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, '/tmp/' + pcap_log_file, '.') + self.copyout(self.UEIPAddress, self.UEUserName, self.UEPassword, pcap_log_file, self.UESourceCodePath + '/cmake_targets/.') + self.close() + logging.debug('\u001B[1m Replaying RAW record file\u001B[0m') + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('cd ' + self.UESourceCodePath + '/common/utils/T/tracer/', '\$', 5) + raw_record_file = self.UELogFile.replace('.log', '_record.raw') + replay_log_file = self.UELogFile.replace('.log', '_replay.log') + extracted_txt_file = self.UELogFile.replace('.log', '_extracted_messages.txt') + extracted_log_file = self.UELogFile.replace('.log', '_extracted_messages.log') + self.command('./extract_config -i ' + self.UESourceCodePath + '/cmake_targets/' + raw_record_file + ' > ' + self.UESourceCodePath + '/cmake_targets/' + extracted_txt_file, '\$', 5) + self.command('echo $USER; nohup ./replay -i ' + self.UESourceCodePath + '/cmake_targets/' + raw_record_file + ' > ' + self.UESourceCodePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', self.UEUserName, 5) + self.command('./textlog -d ' + self.UESourceCodePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + self.UESourceCodePath + '/cmake_targets/' + extracted_log_file, '\$', 5) + self.close() + self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + extracted_log_file, '.') + logging.debug('\u001B[1m Analyzing UE replay logfile \u001B[0m') + logStatus = self.AnalyzeLogFile_UE(extracted_log_file) + self.CreateHtmlTestRow(html_queue, 'OK', ALL_PROCESSES_OK) + self.UELogFile = '' + else: + result = re.search('ue_', str(self.UELogFile)) + if result is not None: + copyin_res = self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.') + if (copyin_res == -1): + logging.debug('\u001B[1;37;41m Could not copy UE logfile to analyze it! \u001B[0m') + optionsMsg = '
Could not copy UE logfile to analyze it!
' + self.CreateHtmlTestRow(optionsMsg, 'KO', UE_PROCESS_NOLOGFILE_TO_ANALYZE, 'UE') + self.UELogFile = '' + return + logging.debug('\u001B[1m Analyzing UE logfile \u001B[0m') + logStatus = self.AnalyzeLogFile_UE(self.UELogFile) + if (logStatus < 0): + optionsMsg = '
Sniffing Unsuccessful\n'
+					optionsMsg += self.htmlUEFailureMsg
+					optionsMsg += '
' + self.CreateHtmlTestRow(optionsMsg, 'KO', logStatus, 'UE') + self.CreateHtmlTabFooter(False) + sys.exit(1) + else: + optionsMsg = '
Sniffing Successful\n'
+					optionsMsg += self.htmlUEFailureMsg
+					optionsMsg += '
' + self.CreateHtmlTestRow(optionsMsg, 'OK', ALL_PROCESSES_OK) + self.UELogFile = '' + else: + self.CreateHtmlTestRow('
No Log File to analyze
', 'OK', ALL_PROCESSES_OK) + def AutoTerminateUEandeNB(self): self.testCase_id = 'AUTO-KILL-UE' self.desc = 'Automatic Termination of UE' @@ -2047,12 +2446,24 @@ class SSHConnection(): self.CreateHtmlTestRow(str(self.idle_sleep_time) + ' sec', 'OK', ALL_PROCESSES_OK) def LogCollectBuild(self): - self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) - self.command('cd ' + self.eNBSourceCodePath, '\$', 5) + if (self.eNBIPAddress != '' and self.eNBUserName != '' and self.eNBPassword != ''): + IPAddress = self.eNBIPAddress + UserName = self.eNBUserName + Password = self.eNBPassword + SourceCodePath = self.eNBSourceCodePath + elif (self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != ''): + IPAddress = self.UEIPAddress + UserName = self.UEUserName + Password = self.UEPassword + SourceCodePath = self.UESourceCodePath + else: + sys.exit('Insufficient Parameter') + self.open(IPAddress, UserName, Password) + self.command('cd ' + SourceCodePath, '\$', 5) self.command('cd cmake_targets', '\$', 5) self.command('rm -f build.log.zip', '\$', 5) self.command('zip build.log.zip build_log_*/*', '\$', 60) - self.command('echo ' + self.eNBPassword + ' | sudo -S rm -rf build_log_*', '\$', 5) + self.command('echo ' + Password + ' | sudo -S rm -rf build_log_*', '\$', 5) self.close() def LogCollecteNB(self): @@ -2121,49 +2532,69 @@ class SSHConnection(): self.command('zip spgw.log.zip xGwLog.0', '\$', 60) self.close() + def LogCollectOAIUE(self): + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('cd ' + self.UESourceCodePath, '\$', 5) + self.command('cd cmake_targets', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S rm -f ue.log.zip', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S zip ue.log.zip ue*.log core* ue_*record.raw ue_*.pcap ue_*txt', '\$', 60) + self.command('echo ' + self.UEPassword + ' | sudo -S rm ue*.log core* ue_*record.raw ue_*.pcap ue_*txt', '\$', 5) + self.close() + def RetrieveSystemVersion(self): if self.eNBIPAddress == 'none': - self.eNBOsVersion = 'Ubuntu 16.04.5 LTS' - self.eNBKernelVersion = '4.15.0-45-generic' - self.eNBUhdVersion = '3.13.0.1-0' - self.eNBUsrpBoard = 'B210' - self.eNBCpuNb = '4' - self.eNBCpuModel = 'Intel(R) Core(TM) i5-6200U' - self.eNBCpuMHz = '2399.996 MHz' + self.OsVersion = 'Ubuntu 16.04.5 LTS' + self.KernelVersion = '4.15.0-45-generic' + self.UhdVersion = '3.13.0.1-0' + self.UsrpBoard = 'B210' + self.CpuNb = '4' + self.CpuModel = 'Intel(R) Core(TM) i5-6200U' + self.CpuMHz = '2399.996 MHz' return - if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '': + machine = None + if self.eNBIPAddress != '' and self.eNBUserName != '' and self.eNBPassword != '': + machine = 'eNB' + IPAddress = self.eNBIPAddress + UserName = self.eNBUserName + Password = self.eNBPassword + elif self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != '': + machine = 'UE' + IPAddress = self.UEIPAddress + UserName = self.UEUserName + Password = self.UEPassword + if machine is None: Usage() sys.exit('Insufficient Parameter') - self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) + self.open(IPAddress, UserName, Password) self.command('lsb_release -a', '\$', 5) result = re.search('Description:\\\\t(?P[a-zA-Z0-9\-\_\.\ ]+)', str(self.ssh.before)) if result is not None: - self.eNBOsVersion = result.group('os_type') - logging.debug('OS is: ' + self.eNBOsVersion) + self.OsVersion = result.group('os_type') + logging.debug('OS is: ' + self.OsVersion) self.command('uname -r', '\$', 5) result = re.search('uname -r\\\\r\\\\n(?P[a-zA-Z0-9\-\_\.]+)', str(self.ssh.before)) if result is not None: - self.eNBKernelVersion = result.group('kernel_version') - logging.debug('Kernel Version is: ' + self.eNBKernelVersion) + self.KernelVersion = result.group('kernel_version') + logging.debug('Kernel Version is: ' + self.KernelVersion) self.command('dpkg --list | egrep --color=never libuhd003', '\$', 5) result = re.search('libuhd003:amd64 *(?P[0-9\.]+)', str(self.ssh.before)) if result is not None: - self.eNBUhdVersion = result.group('uhd_version') - logging.debug('UHD Version is: ' + self.eNBUhdVersion) - self.command('echo ' + self.eNBPassword + ' | sudo -S uhd_find_devices', '\$', 5) + self.UhdVersion = result.group('uhd_version') + logging.debug('UHD Version is: ' + self.UhdVersion) + self.command('echo ' + Password + ' | sudo -S uhd_find_devices', '\$', 5) result = re.search('product: (?P[0-9A-Za-z]+)\\\\r\\\\n', str(self.ssh.before)) if result is not None: - self.eNBUsrpBoard = result.group('usrp_board') - logging.debug('USRP Board is: ' + self.eNBUsrpBoard) + self.UsrpBoard = result.group('usrp_board') + logging.debug('USRP Board is: ' + self.UsrpBoard) self.command('lscpu', '\$', 5) result = re.search('CPU\(s\): *(?P[0-9]+).*Model name: *(?P[a-zA-Z0-9\-\_\.\ \(\)]+).*CPU MHz: *(?P[0-9\.]+)', str(self.ssh.before)) if result is not None: - self.eNBCpuNb = result.group('nb_cpus') - logging.debug('nb_cpus: ' + self.eNBCpuNb) - self.eNBCpuModel = result.group('model') - logging.debug('model: ' + self.eNBCpuModel) - self.eNBCpuMHz = result.group('cpu_mhz') + ' MHz' - logging.debug('cpu_mhz: ' + self.eNBCpuMHz) + self.CpuNb = result.group('nb_cpus') + logging.debug('nb_cpus: ' + self.CpuNb) + self.CpuModel = result.group('model') + logging.debug('model: ' + self.CpuModel) + self.CpuMHz = result.group('cpu_mhz') + ' MHz' + logging.debug('cpu_mhz: ' + self.CpuMHz) self.close() #----------------------------------------------------------- @@ -2171,6 +2602,9 @@ class SSHConnection(): #----------------------------------------------------------- def CreateHtmlHeader(self): if (not self.htmlHeaderCreated): + logging.debug('\u001B[1m----------------------------------------\u001B[0m') + logging.debug('\u001B[1m Creating HTML header \u001B[0m') + logging.debug('\u001B[1m----------------------------------------\u001B[0m') self.htmlFile = open('test_results.html', 'w') self.htmlFile.write('\n') self.htmlFile.write('\n') @@ -2248,18 +2682,13 @@ class SSHConnection(): self.htmlFile.write(' \n') self.htmlFile.write(' \n') - terminate_ue_flag = True if (self.ADBIPAddress != 'none'): + terminate_ue_flag = True self.GetAllUEDevices(terminate_ue_flag) self.GetAllCatMDevices(terminate_ue_flag) - else: - self.UEDevices.append('doughq9rehg') - self.UEDevices.append('dnsgiuahgia') - self.UEDevices.append('uehgieng9') - self.htmlUEConnected = len(self.UEDevices) - - self.htmlFile.write('

' + str(len(self.UEDevices)) + ' UE(s) is(are) connected to ADB bench server

\n') - self.htmlFile.write('

' + str(len(self.CatMDevices)) + ' CAT-M UE(s) is(are) connected to bench server

\n') + self.htmlUEConnected = len(self.UEDevices) + self.htmlFile.write('

' + str(len(self.UEDevices)) + ' UE(s) is(are) connected to ADB bench server

\n') + self.htmlFile.write('

' + str(len(self.CatMDevices)) + ' CAT-M UE(s) is(are) connected to bench server

\n') self.htmlFile.write('
\n') self.htmlFile.write('
    \n') count = 0 @@ -2298,10 +2727,6 @@ class SSHConnection(): if (self.ADBIPAddress != 'none'): self.GetAllUEDevices(terminate_ue_flag) self.GetAllCatMDevices(terminate_ue_flag) - else: - self.UEDevices.append('doughq9rehg') - self.UEDevices.append('dnsgiuahgia') - self.UEDevices.append('uehgieng9') self.htmlUEConnected = len(self.UEDevices) i = 0 @@ -2326,6 +2751,9 @@ class SSHConnection(): def CreateHtmlFooter(self, passStatus): if (os.path.isfile('test_results.html')): + logging.debug('\u001B[1m----------------------------------------\u001B[0m') + logging.debug('\u001B[1m Creating HTML footer \u001B[0m') + logging.debug('\u001B[1m----------------------------------------\u001B[0m') self.RetrieveSystemVersion() self.htmlFile = open('test_results.html', 'a') self.htmlFile.write('\n') @@ -2336,21 +2764,21 @@ class SSHConnection(): self.htmlFile.write(' \n') self.htmlFile.write(' \n') self.htmlFile.write(' OS Version\n') - self.htmlFile.write(' ' + self.eNBOsVersion + '\n') + self.htmlFile.write(' ' + self.OsVersion + '\n') self.htmlFile.write(' Kernel Version\n') - self.htmlFile.write(' ' + self.eNBKernelVersion + '\n') + self.htmlFile.write(' ' + self.KernelVersion + '\n') self.htmlFile.write(' UHD Version\n') - self.htmlFile.write(' ' + self.eNBUhdVersion + '\n') + self.htmlFile.write(' ' + self.UhdVersion + '\n') self.htmlFile.write(' USRP Board\n') - self.htmlFile.write(' ' + self.eNBUsrpBoard + '\n') + self.htmlFile.write(' ' + self.UsrpBoard + '\n') self.htmlFile.write(' \n') self.htmlFile.write(' \n') self.htmlFile.write(' Nb CPUs\n') - self.htmlFile.write(' ' + self.eNBCpuNb + '\n') + self.htmlFile.write(' ' + self.CpuNb + '\n') self.htmlFile.write(' CPU Model Name\n') - self.htmlFile.write(' ' + self.eNBCpuModel + '\n') + self.htmlFile.write(' ' + self.CpuModel + '\n') self.htmlFile.write(' CPU Frequency\n') - self.htmlFile.write(' ' + self.eNBCpuMHz + '\n') + self.htmlFile.write(' ' + self.CpuMHz + '\n') self.htmlFile.write(' \n') self.htmlFile.write(' \n') self.htmlFile.write(' \n') @@ -2368,7 +2796,7 @@ class SSHConnection(): self.htmlFile.write('\n') self.htmlFile.close() - def CreateHtmlTestRow(self, options, status, processesStatus): + def CreateHtmlTestRow(self, options, status, processesStatus, machine='eNB'): if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)): self.htmlFile.write(' \n') self.htmlFile.write(' ' + self.testCase_id + '\n') @@ -2380,15 +2808,17 @@ class SSHConnection(): if (processesStatus == 0): self.htmlFile.write(' ' + str(status) + '\n') elif (processesStatus == ENB_PROCESS_FAILED): - self.htmlFile.write(' KO - eNB process not found\n') + self.htmlFile.write(' KO - ' + machine + ' process not found\n') elif (processesStatus == ENB_PROCESS_SEG_FAULT): - self.htmlFile.write(' KO - eNB process ended in Segmentation Fault\n') - elif (processesStatus == ENB_PROCESS_ASSERTION): - self.htmlFile.write(' KO - eNB process ended in Assertion\n') + self.htmlFile.write(' KO - ' + machine + ' process ended in Segmentation Fault\n') + elif (processesStatus == ENB_PROCESS_ASSERTION) or (processesStatus == UE_PROCESS_ASSERTION): + self.htmlFile.write(' KO - ' + machine + ' process ended in Assertion\n') elif (processesStatus == ENB_PROCESS_REALTIME_ISSUE): - self.htmlFile.write(' KO - eNB process faced Real Time issue(s)\n') - elif (processesStatus == ENB_PROCESS_NOLOGFILE_TO_ANALYZE): - self.htmlFile.write(' OK\n') + self.htmlFile.write(' KO - ' + machine + ' process faced Real Time issue(s)\n') + elif (processesStatus == ENB_PROCESS_NOLOGFILE_TO_ANALYZE) or (processesStatus == UE_PROCESS_NOLOGFILE_TO_ANALYZE): + self.htmlFile.write(' OK?\n') + elif (processesStatus == UE_PROCESS_COULD_NOT_SYNC): + self.htmlFile.write(' KO - UE could not sync\n') elif (processesStatus == HSS_PROCESS_FAILED): self.htmlFile.write(' KO - HSS process not found\n') elif (processesStatus == MME_PROCESS_FAILED): @@ -2493,7 +2923,7 @@ def Usage(): print('------------------------------------------------------------') def CheckClassValidity(action,id): - if action != 'Build_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW' and action != 'Initialize_CatM_module' and action != 'Terminate_CatM_module' and action != 'Attach_CatM_module' and action != 'Detach_CatM_module' and action != 'Ping_CatM_module' and action != 'IdleSleep': + if action != 'Build_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Build_OAI_UE' and action != 'Initialize_OAI_UE' and action != 'Terminate_OAI_UE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW' and action != 'Initialize_CatM_module' and action != 'Terminate_CatM_module' and action != 'Attach_CatM_module' and action != 'Detach_CatM_module' and action != 'Ping_CatM_module' and action != 'IdleSleep': logging.debug('ERROR: test-case ' + id + ' has wrong class ' + action) return False return True @@ -2520,6 +2950,20 @@ def GetParametersFromXML(action): else: SSH.nbMaxUEtoAttach = int(nbMaxUEtoAttach) + if action == 'Build_OAI_UE': + SSH.Build_OAI_UE_args = test.findtext('Build_OAI_UE_args') + + if action == 'Initialize_OAI_UE': + SSH.Initialize_OAI_UE_args = test.findtext('Initialize_OAI_UE_args') + SSH.UE_instance = test.findtext('UE_instance') + if (SSH.UE_instance is None): + SSH.UE_instance = '0' + + if action == 'Terminate_OAI_UE': + SSH.eNB_instance = test.findtext('UE_instance') + if (SSH.UE_instance is None): + SSH.UE_instance = '0' + if action == 'Ping' or action == 'Ping_CatM_module': SSH.ping_args = test.findtext('ping_args') SSH.ping_packetloss_threshold = test.findtext('ping_packetloss_threshold') @@ -2562,6 +3006,7 @@ SSH = SSHConnection() argvs = sys.argv argc = len(argvs) +cwd = os.getcwd() while len(argvs) > 1: myArgv = argvs.pop(1) # 0th is this file's name @@ -2634,6 +3079,18 @@ while len(argvs) > 1: matchReg = re.match('^\-\-XMLTestFile=(.+)$', myArgv, re.IGNORECASE) SSH.testXMLfiles.append(matchReg.group(1)) SSH.nbTestXMLfiles += 1 + elif re.match('^\-\-UEIPAddress=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UEIPAddress=(.+)$', myArgv, re.IGNORECASE) + SSH.UEIPAddress = matchReg.group(1) + elif re.match('^\-\-UEUserName=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UEUserName=(.+)$', myArgv, re.IGNORECASE) + SSH.UEUserName = matchReg.group(1) + elif re.match('^\-\-UEPassword=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UEPassword=(.+)$', myArgv, re.IGNORECASE) + SSH.UEPassword = matchReg.group(1) + elif re.match('^\-\-UESourceCodePath=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UESourceCodePath=(.+)$', myArgv, re.IGNORECASE) + SSH.UESourceCodePath = matchReg.group(1) elif re.match('^\-\-finalStatus=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-finalStatus=(.+)$', myArgv, re.IGNORECASE) finalStatus = matchReg.group(1) @@ -2649,7 +3106,13 @@ if re.match('^TerminateeNB$', mode, re.IGNORECASE): sys.exit('Insufficient Parameter') SSH.TerminateeNB() elif re.match('^TerminateUE$', mode, re.IGNORECASE): - if SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '': + if (SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == ''): + Usage() + sys.exit('Insufficient Parameter') + signal.signal(signal.SIGUSR1, receive_signal) + SSH.TerminateUE() +elif re.match('^TerminateOAIUE$', mode, re.IGNORECASE): + if SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '': Usage() sys.exit('Insufficient Parameter') signal.signal(signal.SIGUSR1, receive_signal) @@ -2670,7 +3133,7 @@ elif re.match('^TerminateSPGW$', mode, re.IGNORECASE): sys.exit('Insufficient Parameter') SSH.TerminateSPGW() elif re.match('^LogCollectBuild$', mode, re.IGNORECASE): - if SSH.eNBIPAddress == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '': + if (SSH.eNBIPAddress == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '') and (SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == ''): Usage() sys.exit('Insufficient Parameter') SSH.LogCollectBuild() @@ -2704,36 +3167,53 @@ elif re.match('^LogCollectIperf$', mode, re.IGNORECASE): Usage() sys.exit('Insufficient Parameter') SSH.LogCollectIperf() +elif re.match('^LogCollectOAIUE$', mode, re.IGNORECASE): + if SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '': + Usage() + sys.exit('Insufficient Parameter') + SSH.LogCollectOAIUE() elif re.match('^InitiateHtml$', mode, re.IGNORECASE): - if SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '': + if (SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == ''): Usage() sys.exit('Insufficient Parameter') count = 0 + foundCount = 0 while (count < SSH.nbTestXMLfiles): + xml_test_file = cwd + "/" + SSH.testXMLfiles[count] xml_test_file = sys.path[0] + "/" + SSH.testXMLfiles[count] - xmlTree = ET.parse(xml_test_file) - xmlRoot = xmlTree.getroot() - SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-' + str(count))) - SSH.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-' + str(count))) - SSH.htmlTabIcons.append(xmlRoot.findtext('htmlTabIcon',default='info-sign')) + if (os.path.isfile(xml_test_file)): + xmlTree = ET.parse(xml_test_file) + xmlRoot = xmlTree.getroot() + SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-' + str(count))) + SSH.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-' + str(count))) + SSH.htmlTabIcons.append(xmlRoot.findtext('htmlTabIcon',default='info-sign')) + foundCount += 1 count += 1 + if foundCount != SSH.nbTestXMLfiles: + SSH.nbTestXMLfiles = foundCount SSH.CreateHtmlHeader() elif re.match('^FinalizeHtml$', mode, re.IGNORECASE): SSH.CreateHtmlFooter(SSH.finalStatus) -elif re.match('^TesteNB$', mode, re.IGNORECASE): - if SSH.eNBIPAddress == '' or SSH.eNBRepository == '' or SSH.eNBBranch == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '' or SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '' or SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '': - Usage() - sys.exit('Insufficient Parameter') +elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re.IGNORECASE): + if re.match('^TesteNB$', mode, re.IGNORECASE): + if SSH.eNBIPAddress == '' or SSH.eNBRepository == '' or SSH.eNBBranch == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '' or SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '' or SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '': + Usage() + sys.exit('Insufficient Parameter') + + if (SSH.EPCIPAddress != ''): + SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, cwd + "/tcp_iperf_stats.awk", "/tmp") + SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, cwd + "/active_net_interfaces.awk", "/tmp") + else: + if SSH.UEIPAddress == '' or SSH.eNBRepository == '' or SSH.eNBBranch == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '': + Usage() + sys.exit('UE: Insufficient Parameter') - if (SSH.EPCIPAddress != 'none'): - SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/tcp_iperf_stats.awk", "/tmp") - SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/active_net_interfaces.awk", "/tmp") #read test_case_list.xml file # if no parameters for XML file, use default value if (SSH.nbTestXMLfiles != 1): - xml_test_file = sys.path[0] + "/test_case_list.xml" + xml_test_file = cwd + "/test_case_list.xml" else: - xml_test_file = sys.path[0] + "/" + SSH.testXMLfiles[0] + xml_test_file = cwd + "/" + SSH.testXMLfiles[0] xmlTree = ET.parse(xml_test_file) xmlRoot = xmlTree.getroot() @@ -2810,6 +3290,12 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE): SSH.AttachUE() elif action == 'Detach_UE': SSH.DetachUE() + elif action == 'Build_OAI_UE': + SSH.BuildOAIUE() + elif action == 'Initialize_OAI_UE': + SSH.InitializeOAIUE() + elif action == 'Terminate_OAI_UE': + SSH.TerminateOAIUE() elif action == 'Initialize_CatM_module': SSH.InitializeCatM() elif action == 'Terminate_CatM_module': diff --git a/ci-scripts/xml_files/ue_band20_build.xml b/ci-scripts/xml_files/ue_band20_build.xml new file mode 100644 index 0000000000..9ca1a2b234 --- /dev/null +++ b/ci-scripts/xml_files/ue_band20_build.xml @@ -0,0 +1,38 @@ + + + build-tab + Build + wrench + +090101 + + + + + Build_OAI_UE + Build OAI UE + -w USRP --UE + + + diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml new file mode 100644 index 0000000000..0e61e75d56 --- /dev/null +++ b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml @@ -0,0 +1,49 @@ + + + test-10mhz-orange + Test-10Mhz-Orange + tasks + +090102 000001 090109 + + + + + Initialize_OAI_UE + Initialize OAI UE -- sniffing Orange frequency + -C 816000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect + + + + IdleSleep + Sleep + 30 + + + + Terminate_OAI_UE + Terminate OAI UE + + + diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml new file mode 100644 index 0000000000..ae8aa0e668 --- /dev/null +++ b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml @@ -0,0 +1,49 @@ + + + test-10mHz-sfr + Test-10MHz-SFR + tasks + +090104 000001 090109 + + + + + Initialize_OAI_UE + Initialize OAI UE -- sniffing SFR frequency + -C 806000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect + + + + IdleSleep + Sleep + 30 + + + + Terminate_OAI_UE + Terminate OAI UE + + +