From 864df49dc90bf013fa01bb690ebdf2f3722dea48 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Fri, 5 Nov 2021 16:13:31 +0100 Subject: [PATCH] fix(ci): proper collection of logs for Core Network Signed-off-by: Raphael Defosseux --- ci-scripts/Jenkinsfile-tmp-multi-enb-nsa | 535 ++++++++++++----------- ci-scripts/epc.py | 36 +- 2 files changed, 305 insertions(+), 266 deletions(-) diff --git a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa index 3243d1645d..5a1fbdb96b 100644 --- a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa +++ b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa @@ -51,270 +51,307 @@ def StatusForDb = "" pipeline { - agent {label pythonExecutor} - options { - disableConcurrentBuilds() - ansiColor('xterm') - lock(extra: [[resource: ciSmartPhonesResource2]], resource: ciSmartPhonesResource1) + agent {label pythonExecutor} + options { + disableConcurrentBuilds() + ansiColor('xterm') + lock(extra: [[resource: ciSmartPhonesResource2]], resource: ciSmartPhonesResource1) + } + stages { + stage("Build Init") { + steps { + // update the build name and description + buildName "${params.eNB_MR}" + buildDescription "Branch : ${params.eNB_Branch}" + } } - stages { - stage("Build Init") { - steps { - // update the build name and description - buildName "${params.eNB_MR}" - buildDescription "Branch : ${params.eNB_Branch}" + stage ("Verify Parameters") { + steps { + script { + echo '\u2705 \u001B[32mVerify Parameters\u001B[0m' + def allParametersPresent = true + + // It is already to late to check it + if (params.pythonExecutor != null) { + echo "eNB CI executor node : ${pythonExecutor}" + } + // If not present picking a default Stage Name + if (params.pipelineTestStageName == null) { + // picking default + testStageName = 'Template Test Stage' + } + + if (params.SmartPhonesResource1 == null) { + allParametersPresent = false + } + if (params.SmartPhonesResource2 == null) { + allParametersPresent = false + } + // 1st eNB parameters + if (params.eNB_IPAddress == null) { + allParametersPresent = false + } + if (params.eNB_SourceCodePath == null) { + allParametersPresent = false + } + if (params.eNB_Credentials == null) { + allParametersPresent = false + } + // 2nd eNB parameters + if (params.eNB1_IPAddress == null) { + allParametersPresent = false + } + if (params.eNB1_SourceCodePath == null) { + allParametersPresent = false + } + if (params.eNB1_Credentials == null) { + allParametersPresent = false + } + // 3rd eNB parameters + if (params.eNB2_IPAddress == null) { + allParametersPresent = false + } + if (params.eNB2_SourceCodePath == null) { + allParametersPresent = false + } + if (params.eNB2_Credentials == null) { + allParametersPresent = false + } + // the following 4 parameters should be pushed by the master trigger + // if not present, take the job GIT variables (used for developing) + if (params.eNB_Repository == null) { + eNB_Repository = env.GIT_URL + } else { + eNB_Repository = params.eNB_Repository + } + echo "eNB_Repository : ${eNB_Repository}" + if (params.eNB_Branch == null) { + eNB_Branch = env.GIT_BRANCH + } else { + eNB_Branch = params.eNB_Branch + } + echo "eNB_Branch : ${eNB_Branch}" + if (params.eNB_CommitID == null) { + eNB_CommitID = env.GIT_COMMIT + } else { + eNB_CommitID = params.eNB_CommitID + } + echo "eNB_CommitID : ${eNB_CommitID}" + if (params.eNB_AllowMergeRequestProcess!= null) { + eNB_AllowMergeRequestProcess = params.eNB_AllowMergeRequestProcess + if (eNB_AllowMergeRequestProcess) { + if (params.eNB_TargetBranch != null) { + eNB_TargetBranch = params.eNB_TargetBranch + } else { + eNB_TargetBranch = 'develop' + } + echo "eNB_TargetBranch : ${eNB_TargetBranch}" } - } - stage ("Verify Parameters") { - steps { - script { - echo '\u2705 \u001B[32mVerify Parameters\u001B[0m' - def allParametersPresent = true + } - // It is already to late to check it - if (params.pythonExecutor != null) { - echo "eNB CI executor node : ${pythonExecutor}" - } - // If not present picking a default Stage Name - if (params.pipelineTestStageName == null) { - // picking default - testStageName = 'Template Test Stage' - } + if (params.EPC_IPAddress == null) { + allParametersPresent = false + } + if (params.EPC_Type == null) { + allParametersPresent = false + } + if (params.EPC_SourceCodePath == null) { + allParametersPresent = false + } + if (params.EPC_Credentials == null) { + allParametersPresent = false + } - if (params.SmartPhonesResource1 == null) { - allParametersPresent = false - } - if (params.SmartPhonesResource2 == null) { - allParametersPresent = false - } - // 1st eNB parameters - if (params.eNB_IPAddress == null) { - allParametersPresent = false - } - if (params.eNB_SourceCodePath == null) { - allParametersPresent = false - } - if (params.eNB_Credentials == null) { - allParametersPresent = false - } - // 2nd eNB parameters - if (params.eNB1_IPAddress == null) { - allParametersPresent = false - } - if (params.eNB1_SourceCodePath == null) { - allParametersPresent = false - } - if (params.eNB1_Credentials == null) { - allParametersPresent = false - } - // 3rd eNB parameters - if (params.eNB2_IPAddress == null) { - allParametersPresent = false - } - if (params.eNB2_SourceCodePath == null) { - allParametersPresent = false - } - if (params.eNB2_Credentials == null) { - allParametersPresent = false - } - // the following 4 parameters should be pushed by the master trigger - // if not present, take the job GIT variables (used for developing) - if (params.eNB_Repository == null) { - eNB_Repository = env.GIT_URL - } else { - eNB_Repository = params.eNB_Repository - } - echo "eNB_Repository : ${eNB_Repository}" - if (params.eNB_Branch == null) { - eNB_Branch = env.GIT_BRANCH - } else { - eNB_Branch = params.eNB_Branch - } - echo "eNB_Branch : ${eNB_Branch}" - if (params.eNB_CommitID == null) { - eNB_CommitID = env.GIT_COMMIT - } else { - eNB_CommitID = params.eNB_CommitID - } - echo "eNB_CommitID : ${eNB_CommitID}" - if (params.eNB_AllowMergeRequestProcess!= null) { - eNB_AllowMergeRequestProcess = params.eNB_AllowMergeRequestProcess - if (eNB_AllowMergeRequestProcess) { - if (params.eNB_TargetBranch != null) { - eNB_TargetBranch = params.eNB_TargetBranch - } else { - eNB_TargetBranch = 'develop' - } - echo "eNB_TargetBranch : ${eNB_TargetBranch}" - } - } + if (params.ADB_IPAddress == null) { + allParametersPresent = false + } + if (params.ADB_Credentials == null) { + allParametersPresent = false + } - if (params.EPC_IPAddress == null) { - allParametersPresent = false - } - if (params.EPC_Type == null) { - allParametersPresent = false - } - if (params.EPC_SourceCodePath == null) { - allParametersPresent = false - } - if (params.EPC_Credentials == null) { - allParametersPresent = false - } - - if (params.ADB_IPAddress == null) { - allParametersPresent = false - } - if (params.ADB_Credentials == null) { - allParametersPresent = false - } - - if (allParametersPresent) { - echo "All parameters are present" - if (eNB_AllowMergeRequestProcess) { - sh "git fetch" - sh "./ci-scripts/doGitLabMerge.sh --src-branch ${eNB_Branch} --src-commit ${eNB_CommitID} --target-branch ${eNB_TargetBranch} --target-commit latest" - } else { - sh "git fetch" - sh "git checkout -f ${eNB_CommitID}" - } - } else { - echo "Some parameters are missing" - sh "./ci-scripts/fail.sh" - } - } + if (allParametersPresent) { + echo "All parameters are present" + if (eNB_AllowMergeRequestProcess) { + sh "git fetch" + sh "./ci-scripts/doGitLabMerge.sh --src-branch ${eNB_Branch} --src-commit ${eNB_CommitID} --target-branch ${eNB_TargetBranch} --target-commit latest" + } else { + sh "git fetch" + sh "git checkout -f ${eNB_CommitID}" } + } else { + echo "Some parameters are missing" + sh "./ci-scripts/fail.sh" + } } - stage ("Build and Test") { - steps { - script { - dir ('ci-scripts') { - echo "\u2705 \u001B[32m${testStageName}\u001B[0m" - // If not present picking a default XML file - if (params.pythonTestXmlFile == null) { - // picking default - testXMLFile = 'xml_files/enb_usrpB210_band7_50PRB.xml' - echo "Test XML file(default): ${testXMLFile}" - mainPythonAllXmlFiles += "--XMLTestFile=" + testXMLFile + " " - } else { - String[] myXmlTestSuite = testXMLFile.split("\\r?\\n") - for (xmlFile in myXmlTestSuite) { - if (fileExists(xmlFile)) { - mainPythonAllXmlFiles += "--XMLTestFile=" + xmlFile + " " - echo "Test XML file : ${xmlFile}" - } - } - } - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'], - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB1_Credentials}", usernameVariable: 'eNB1_Username', passwordVariable: 'eNB1_Password'], - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB2_Credentials}", usernameVariable: 'eNB2_Username', passwordVariable: 'eNB2_Password'], - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'], - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.ADB_Credentials}", usernameVariable: 'ADB_Username', passwordVariable: 'ADB_Password'] - ]) { - sh "python3 main.py --mode=InitiateHtml --ranRepository=${eNB_Repository} --ranBranch=${eNB_Branch} --ranCommitID=${eNB_CommitID} --ranAllowMerge=${eNB_AllowMergeRequestProcess} --ranTargetBranch=${eNB_TargetBranch} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} ${mainPythonAllXmlFiles}" - String[] myXmlTestSuite = testXMLFile.split("\\r?\\n") - for (xmlFile in myXmlTestSuite) { - if (fileExists(xmlFile)) { - try { - sh "python3 main.py --mode=TesteNB --ranRepository=${eNB_Repository} --ranBranch=${eNB_Branch} --ranCommitID=${eNB_CommitID} --ranAllowMerge=${eNB_AllowMergeRequestProcess} --ranTargetBranch=${eNB_TargetBranch} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath} --eNB1IPAddress=${params.eNB1_IPAddress} --eNB1UserName=${eNB1_Username} --eNB1Password=${eNB1_Password} --eNB1SourceCodePath=${params.eNB1_SourceCodePath} --eNB2IPAddress=${params.eNB2_IPAddress} --eNB2UserName=${eNB2_Username} --eNB2Password=${eNB2_Password} --eNB2SourceCodePath=${params.eNB2_SourceCodePath} --EPCIPAddress=${params.EPC_IPAddress} --EPCType=${params.EPC_Type} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} --XMLTestFile=${xmlFile}" - } catch (Exception e) { - currentBuild.result = 'FAILURE' - buildStageStatus = false - } - } - } - sh "python3 main.py --mode=FinalizeHtml --finalStatus=${buildStageStatus} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password}" - } - } + } + } + stage ("Build and Test") { + steps { + script { + dir ('ci-scripts') { + echo "\u2705 \u001B[32m${testStageName}\u001B[0m" + // If not present picking a default XML file + if (params.pythonTestXmlFile == null) { + // picking default + testXMLFile = 'xml_files/enb_usrpB210_band7_50PRB.xml' + echo "Test XML file(default): ${testXMLFile}" + mainPythonAllXmlFiles += "--XMLTestFile=" + testXMLFile + " " + } else { + String[] myXmlTestSuite = testXMLFile.split("\\r?\\n") + for (xmlFile in myXmlTestSuite) { + if (fileExists(xmlFile)) { + mainPythonAllXmlFiles += "--XMLTestFile=" + xmlFile + " " + echo "Test XML file : ${xmlFile}" } + } } - } - stage('Log Collection') { - parallel { - stage('Log Collection (eNB - Build)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' - sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("build.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "build.log.${env.BUILD_ID}.zip" - } - } - } - } - stage('Log Collection (eNB - Run)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - echo '\u2705 \u001B[32mLog Collection (eNB - Run)\u001B[0m' - sh "python3 ci-scripts/main.py --mode=LogCollecteNB --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Run)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/enb.log.zip ./enb.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("enb.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "enb.log.${env.BUILD_ID}.zip" - } - } - } - } - stage('Log Collection (CN)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'] - ]) { - echo '\u2705 \u001B[32mLog Transfer (CN)\u001B[0m' - sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/logs/oai-cn5g.log.zip ./oai-cn5g.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("oai-cn5g.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "oai-cn5g.log.${env.BUILD_ID}.zip" - } - if(fileExists("ci-scripts/test_results.html")) { - sh "mv ci-scripts/test_results.html test_results-${JOB_NAME}.html" - sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: ${BUILD_ID}#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results-${JOB_NAME}.html" - archiveArtifacts "test_results-${JOB_NAME}.html" - } - } - } - } - stage ("SQL Collect"){ - agent {label DataBaseHost} - steps { - script { - if (currentBuild.result=='FAILURE') {StatusForDb = 'FAIL'} else {StatusForDb = 'PASS'} - sh "python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} ''" - } - } + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'], + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB1_Credentials}", usernameVariable: 'eNB1_Username', passwordVariable: 'eNB1_Password'], + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB2_Credentials}", usernameVariable: 'eNB2_Username', passwordVariable: 'eNB2_Password'], + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'], + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.ADB_Credentials}", usernameVariable: 'ADB_Username', passwordVariable: 'ADB_Password'] + ]) { + sh "python3 main.py --mode=InitiateHtml --ranRepository=${eNB_Repository} --ranBranch=${eNB_Branch} --ranCommitID=${eNB_CommitID} --ranAllowMerge=${eNB_AllowMergeRequestProcess} --ranTargetBranch=${eNB_TargetBranch} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} ${mainPythonAllXmlFiles}" + String[] myXmlTestSuite = testXMLFile.split("\\r?\\n") + for (xmlFile in myXmlTestSuite) { + if (fileExists(xmlFile)) { + try { + sh "python3 main.py --mode=TesteNB --ranRepository=${eNB_Repository} --ranBranch=${eNB_Branch} --ranCommitID=${eNB_CommitID} --ranAllowMerge=${eNB_AllowMergeRequestProcess} --ranTargetBranch=${eNB_TargetBranch} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath} --eNB1IPAddress=${params.eNB1_IPAddress} --eNB1UserName=${eNB1_Username} --eNB1Password=${eNB1_Password} --eNB1SourceCodePath=${params.eNB1_SourceCodePath} --eNB2IPAddress=${params.eNB2_IPAddress} --eNB2UserName=${eNB2_Username} --eNB2Password=${eNB2_Password} --eNB2SourceCodePath=${params.eNB2_SourceCodePath} --EPCIPAddress=${params.EPC_IPAddress} --EPCType=${params.EPC_Type} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} --XMLTestFile=${xmlFile}" + } catch (Exception e) { + currentBuild.result = 'FAILURE' + buildStageStatus = false + } } + } + sh "python3 main.py --mode=FinalizeHtml --finalStatus=${buildStageStatus} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password}" } + } } - } + } + } + stage('Log Collection') { + parallel { + stage('Log Collection (eNB - Build)') { + steps { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] + ]) { + echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' + sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - post { - always { + echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' + sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" + } script { - if (params.pipelineZipsConsoleLog != null) { - if (params.pipelineZipsConsoleLog) { - echo "Archiving Jenkins console log" - sh "wget --no-check-certificate --no-proxy ${env.JENKINS_URL}/job/${env.JOB_NAME}/${env.BUILD_ID}/consoleText -O consoleText.log || true" - sh "zip -m consoleText.log.${env.BUILD_ID}.zip consoleText.log || true" - if(fileExists("consoleText.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "consoleText.log.${env.BUILD_ID}.zip" - } - } - } + if(fileExists("build.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "build.log.${env.BUILD_ID}.zip" + } } + } } + stage('Log Collection (eNB - Run)') { + steps { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] + ]) { + echo '\u2705 \u001B[32mLog Collection (eNB - Run)\u001B[0m' + sh "python3 ci-scripts/main.py --mode=LogCollecteNB --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" + + echo '\u2705 \u001B[32mLog Transfer (eNB - Run)\u001B[0m' + sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/enb.log.zip ./enb.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("enb.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "enb.log.${env.BUILD_ID}.zip" + } + } + } + } + stage('Log Collection (CN)') { + // Bypassing this stage if EPC server is not defined + when { + expression { params.EPC_IPAddress != "none" } + } + steps { + script { + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'] + ]) { + echo '\u2705 \u001B[32mLog Collection (HSS)\u001B[0m' + sh "python3 ci-scripts/main.py --mode=LogCollectHSS --EPCIPAddress=${params.EPC_IPAddress} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --EPCType=${params.EPC_Type}" + if (params.EPC_Type != 'OAICN5G') { + sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/hss.log.zip ./hss.log.${env.BUILD_ID}.zip || true" + } + echo '\u2705 \u001B[32mLog Collection (MME or AMF)\u001B[0m' + sh "python3 ci-scripts/main.py --mode=LogCollectMME --EPCIPAddress=${params.EPC_IPAddress} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --EPCType=${params.EPC_Type}" + if (params.EPC_Type == 'OAICN5G') { + sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/mme.log.zip ./amf.log.${env.BUILD_ID}.zip || true" + } else { + sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/mme.log.zip ./mme.log.${env.BUILD_ID}.zip || true" + } + echo '\u2705 \u001B[32mLog Collection (SPGW or SMF/UPF)\u001B[0m' + sh "python3 ci-scripts/main.py --mode=LogCollectSPGW --EPCIPAddress=${params.EPC_IPAddress} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --EPCType=${params.EPC_Type}" + if (params.EPC_Type == 'OAICN5G') { + sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/smf-upf.log.zip ./spgw.log.${env.BUILD_ID}.zip || true" + } else { + sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/spgw.log.zip ./spgw.log.${env.BUILD_ID}.zip || true" + } + } + if(fileExists("hss.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "hss.log.${env.BUILD_ID}.zip" + } + if(fileExists("mme.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "mme.log.${env.BUILD_ID}.zip" + } + if(fileExists("spgw.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "spgw.log.${env.BUILD_ID}.zip" + } + if(fileExists("amf.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "amf.log.${env.BUILD_ID}.zip" + } + if(fileExists("smf-upf.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "smf-upf.log.${env.BUILD_ID}.zip" + } + echo '\u2705 \u001B[32mLog Collection for CoreNetwork Done!\u001B[0m' + } + } + } + stage ("SQL Collect"){ + when { + expression { DataBaseHost != "none" } + } + agent {label DataBaseHost} + steps { + script { + if (currentBuild.result=='FAILURE') {StatusForDb = 'FAIL'} else {StatusForDb = 'PASS'} + sh "python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} ''" + } + } + } + } } + } + + post { + always { + script { + if(fileExists("ci-scripts/test_results.html")) { + sh "mv ci-scripts/test_results.html test_results-${JOB_NAME}.html" + sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: ${BUILD_ID}#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results-${JOB_NAME}.html" + archiveArtifacts "test_results-${JOB_NAME}.html" + } + if (params.pipelineZipsConsoleLog != null) { + if (params.pipelineZipsConsoleLog) { + echo "Archiving Jenkins console log" + sh "wget --no-check-certificate --no-proxy ${env.JENKINS_URL}/job/${env.JOB_NAME}/${env.BUILD_ID}/consoleText -O consoleText.log || true" + sh "zip -m consoleText.log.${env.BUILD_ID}.zip consoleText.log || true" + if(fileExists("consoleText.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "consoleText.log.${env.BUILD_ID}.zip" + } + } + } + } + } + } } diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index 1867091d93..8b295fabb5 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -53,7 +53,7 @@ import constants as CONST class EPCManagement(): def __init__(self): - + self.IPAddress = '' self.UserName = '' self.Password = '' @@ -475,10 +475,15 @@ class EPCManagement(): time.sleep(1) mySSH.command('echo ' + self.Password + ' | sudo -S screen -S simulated_5g_hss -X quit', '\$', 5) elif re.match('OAICN5G', self.Type, re.IGNORECASE): - self.LogCollectOAICN5G() + logging.debug('OAI CN5G Collecting Log files to workspace') + mySSH.command('echo ' + self.Password + ' | sudo rm -rf ' + self.SourceCodePath + '/logs', '\$', 5) + mySSH.command('mkdir ' + self.SourceCodePath + '/logs','\$', 5) + containers_list=['oai-smf','oai-spgwu','oai-amf','oai-nrf'] + for c in containers_list: + mySSH.command('docker logs ' + c + ' > ' + self.SourceCodePath + '/logs/' + c + '.log', '\$', 5) + logging.debug('Terminating OAI CN5G') mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) - mySSH.command('docker-compose down', '\$', 5) mySSH.command('./core-network.sh stop nrf spgwu', '\$', 60) else: logging.error('This should not happen!') @@ -693,6 +698,8 @@ class EPCManagement(): mySSH.command('docker cp ' + self.containerPrefix + '-oai-hss:/openair-hss/hss_check_run.log .', '\$', 60) mySSH.command('docker cp ' + self.containerPrefix + '-oai-hss:/tmp/hss_check_run.pcap .', '\$', 60) mySSH.command('zip hss.log.zip hss_check_run.*', '\$', 60) + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + logging.debug('LogCollect is bypassed for that variant') elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): mySSH.command('zip hss.log.zip hss*.log', '\$', 60) mySSH.command('echo ' + self.Password + ' | sudo -S rm hss*.log', '\$', 5) @@ -723,6 +730,11 @@ class EPCManagement(): mySSH.command('docker cp ' + self.containerPrefix + '-oai-mme:/openair-mme/mme_check_run.log .', '\$', 60) mySSH.command('docker cp ' + self.containerPrefix + '-oai-mme:/tmp/mme_check_run.pcap .', '\$', 60) mySSH.command('zip mme.log.zip mme_check_run.*', '\$', 60) + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + mySSH.command('cd ' + self.SourceCodePath + '/logs','\$', 5) + mySSH.command('cp -f /tmp/oai-cn5g.pcap .','\$', 30) + mySSH.command('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g.pcap','\$', 30) + mySSH.command('mv mme.log.zip ' + self.SourceCodePath + '/scripts','\$', 30) elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): mySSH.command('zip mme.log.zip mme*.log', '\$', 60) mySSH.command('echo ' + self.Password + ' | sudo -S rm mme*.log', '\$', 5) @@ -752,6 +764,10 @@ class EPCManagement(): mySSH.command('docker cp ' + self.containerPrefix + '-oai-spgwc:/tmp/spgwc_check_run.pcap .', '\$', 60) mySSH.command('docker cp ' + self.containerPrefix + '-oai-spgwu-tiny:/tmp/spgwu_check_run.pcap .', '\$', 60) mySSH.command('zip spgw.log.zip spgw*_check_run.*', '\$', 60) + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + mySSH.command('cd ' + self.SourceCodePath + '/logs','\$', 5) + mySSH.command('zip spgw.log.zip oai-smf.log oai-spgwu.log','\$', 30) + mySSH.command('mv spgw.log.zip ' + self.SourceCodePath + '/scripts','\$', 30) elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): mySSH.command('zip spgw.log.zip spgw*.log', '\$', 60) mySSH.command('echo ' + self.Password + ' | sudo -S rm spgw*.log', '\$', 5) @@ -761,17 +777,3 @@ class EPCManagement(): else: logging.error('This option should not occur!') mySSH.close() - - def LogCollectOAICN5G(self): - mySSH = SSH.SSHConnection() - mySSH.open(self.IPAddress, self.UserName, self.Password) - logging.debug('OAI CN5G Collecting Log files to workspace') - mySSH.command('echo ' + self.Password + ' | sudo rm -rf ' + self.SourceCodePath + '/logs', '\$', 5) - mySSH.command('mkdir ' + self.SourceCodePath + '/logs','\$', 5) - containers_list=['oai-smf','oai-spgwu','oai-amf','oai-nrf'] - for c in containers_list: - mySSH.command('docker logs ' + c + ' > ' + self.SourceCodePath + '/logs/' + c + '.log', '\$', 5) - mySSH.command('cd ' + self.SourceCodePath + '/logs', '\$', 5) - mySSH.command('zip oai-cn5g.log.zip *.log', '\$', 60) - mySSH.close() -