Merge remote-tracking branch 'origin/ci-build-cluster' into integration_2022_wk36b
This commit is contained in:
Robert Schmidt
@@ -136,16 +136,16 @@ pipeline {
|
||||
}
|
||||
}
|
||||
}
|
||||
stage ("RHEL8 Build") {
|
||||
stage ("RHEL8-Cluster-Image-Builder") {
|
||||
steps {
|
||||
script {
|
||||
triggerSlaveJob ('RAN-RHEL8-Image-Builder', 'RHEL8-Images-Build')
|
||||
triggerSlaveJob ('RAN-RHEL8-Cluster-Image-Builder', 'RHEL8-Cluster-Image-Builder')
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
script {
|
||||
finalizeSlaveJob('RAN-RHEL8-Image-Builder')
|
||||
finalizeSlaveJob('RAN-RHEL8-Cluster-Image-Builder')
|
||||
}
|
||||
}
|
||||
failure {
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
#!/bin/groovy
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
// Location of the executor node
|
||||
def pythonExecutor = params.pythonExecutor
|
||||
|
||||
// Location of the test XML file to be run
|
||||
def testXMLFile = params.pythonTestXmlFile
|
||||
def mainPythonAllXmlFiles = ""
|
||||
def buildStageStatus = true
|
||||
|
||||
// Name of the test stage
|
||||
def testStageName = params.pipelineTestStageName
|
||||
|
||||
// Name of the resource
|
||||
def lockResources = []
|
||||
if (params.LockResources != null && params.LockResources.trim().length() > 0)
|
||||
params.LockResources.trim().split(",").each{lockResources += [resource: it.trim()]}
|
||||
|
||||
// Global Parameters. Normally they should be populated when the master job
|
||||
// triggers the slave job with parameters
|
||||
def eNB_Repository
|
||||
def eNB_Branch
|
||||
def eNB_CommitID
|
||||
def eNB_AllowMergeRequestProcess = false
|
||||
def eNB_TargetBranch
|
||||
|
||||
def directoryExistsGlob(fileNameWithGlob) {
|
||||
/* if multiple directories match, will join their names and call fileExists() on that */
|
||||
def dir = sh returnStdout: true, script: 'find . -name \'' + fileNameWithGlob + '\' -type d'
|
||||
dir = dir.replaceAll("\n", "").trim()
|
||||
echo "matching '" + fileNameWithGlob + "' found: '" + dir + "'"
|
||||
if (dir == "")
|
||||
return false
|
||||
return fileExists(dir)
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------------------
|
||||
// Pipeline start
|
||||
pipeline {
|
||||
agent {
|
||||
label pythonExecutor
|
||||
}
|
||||
options {
|
||||
disableConcurrentBuilds()
|
||||
timestamps()
|
||||
ansiColor('xterm')
|
||||
lock(extra: lockResources)
|
||||
}
|
||||
|
||||
stages {
|
||||
stage ('Verify Parameters') {
|
||||
steps {
|
||||
script {
|
||||
echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'
|
||||
|
||||
JOB_TIMESTAMP = sh returnStdout: true, script: 'date --utc --rfc-3339=seconds | sed -e "s#+00:00##"'
|
||||
JOB_TIMESTAMP = JOB_TIMESTAMP.trim()
|
||||
|
||||
def allParametersPresent = true
|
||||
|
||||
// It is already to late to check it
|
||||
if (params.pythonExecutor != null) {
|
||||
echo "eNB CI executor node : ${pythonExecutor}"
|
||||
}
|
||||
// If not present picking a default Stage Name
|
||||
if (params.pipelineTestStageName == null) {
|
||||
// picking default
|
||||
testStageName = 'Template Test Stage'
|
||||
}
|
||||
|
||||
if (params.LockResources == null) {
|
||||
echo "no LockResources given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (params.eNB_IPAddress == null) {
|
||||
echo "no eNB_IPAddress given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (params.eNB_SourceCodePath == null) {
|
||||
echo "no eNB_SourceCodePath given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (params.eNB_Credentials == null) {
|
||||
echo "no eNB_Credentials given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
// the following 4 parameters should be pushed by the master trigger
|
||||
// if not present, take the job GIT variables (used for developing)
|
||||
if (params.eNB_Repository == null) {
|
||||
eNB_Repository = env.GIT_URL
|
||||
} else {
|
||||
eNB_Repository = params.eNB_Repository
|
||||
}
|
||||
echo "eNB_Repository : ${eNB_Repository}"
|
||||
if (params.eNB_Branch == null) {
|
||||
eNB_Branch = env.GIT_BRANCH
|
||||
} else {
|
||||
eNB_Branch = params.eNB_Branch
|
||||
}
|
||||
echo "eNB_Branch : ${eNB_Branch}"
|
||||
if (params.eNB_CommitID == null) {
|
||||
eNB_CommitID = env.GIT_COMMIT
|
||||
} else {
|
||||
eNB_CommitID = params.eNB_CommitID
|
||||
}
|
||||
echo "eNB_CommitID : ${eNB_CommitID}"
|
||||
if (params.eNB_mergeRequest != null) {
|
||||
eNB_AllowMergeRequestProcess = params.eNB_mergeRequest
|
||||
if (eNB_AllowMergeRequestProcess) {
|
||||
if (params.eNB_TargetBranch != null) {
|
||||
eNB_TargetBranch = params.eNB_TargetBranch
|
||||
} else {
|
||||
eNB_TargetBranch = 'develop'
|
||||
}
|
||||
echo "eNB_TargetBranch : ${eNB_TargetBranch}"
|
||||
}
|
||||
} else {
|
||||
echo "no eNB_mergeRequest given - not merging develop"
|
||||
}
|
||||
|
||||
if (params.EPC_IPAddress == null) {
|
||||
echo "no EPC_IPAddress given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (params.EPC_Type == null) {
|
||||
echo "no EPC_Type given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (params.EPC_SourceCodePath == null) {
|
||||
echo "no EPC_SourceCodePath given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (params.EPC_Credentials == null) {
|
||||
echo "no EPC_Credentials given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
|
||||
if (params.ADB_IPAddress == null) {
|
||||
echo "no ADB_IPAddress given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (params.ADB_Credentials == null) {
|
||||
echo "no ADB_IPAddress given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (params.OC_Credentials == null) {
|
||||
echo "no OC_Credentials given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (params.OC_ProjectName == null) {
|
||||
echo "no OC_ProjectName given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (params.pythonTestXmlFile == null) {
|
||||
echo "no pythonTestXmlFile given"
|
||||
allParametersPresent = false
|
||||
}
|
||||
if (allParametersPresent) {
|
||||
echo "All parameters are present"
|
||||
if (eNB_AllowMergeRequestProcess) {
|
||||
sh "git fetch"
|
||||
sh "./ci-scripts/doGitLabMerge.sh --src-branch ${eNB_Branch} --src-commit ${eNB_CommitID} --target-branch ${eNB_TargetBranch} --target-commit latest"
|
||||
} else {
|
||||
sh "git fetch"
|
||||
sh "git checkout -f ${eNB_CommitID}"
|
||||
}
|
||||
} else {
|
||||
echo "Some parameters are missing"
|
||||
sh "./ci-scripts/fail.sh"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage ("Deploy and Test") {
|
||||
steps {
|
||||
script {
|
||||
dir ('ci-scripts') {
|
||||
echo "\u2705 \u001B[32m${testStageName}\u001B[0m"
|
||||
String[] myXmlTestSuite = testXMLFile.split("\\r?\\n")
|
||||
for (xmlFile in myXmlTestSuite) {
|
||||
if (fileExists(xmlFile)) {
|
||||
mainPythonAllXmlFiles += "--XMLTestFile=" + xmlFile + " "
|
||||
echo "Test XML file : ${xmlFile}"
|
||||
}
|
||||
}
|
||||
withCredentials([
|
||||
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'],
|
||||
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'],
|
||||
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.ADB_Credentials}", usernameVariable: 'ADB_Username', passwordVariable: 'ADB_Password'],
|
||||
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.OC_Credentials}", usernameVariable: 'OC_Username', passwordVariable: 'OC_Password']
|
||||
]) {
|
||||
sh "python3 main.py --mode=InitiateHtml --ranRepository=${eNB_Repository} --ranBranch=${eNB_Branch} --ranCommitID=${eNB_CommitID} --ranAllowMerge=${eNB_AllowMergeRequestProcess} --ranTargetBranch=${eNB_TargetBranch} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} ${mainPythonAllXmlFiles}"
|
||||
for (xmlFile in myXmlTestSuite) {
|
||||
if (fileExists(xmlFile)) {
|
||||
try {
|
||||
sh "python3 main.py --mode=TesteNB --eNBIPAddress=${params.eNB_IPAddress} --ranRepository=${eNB_Repository} --ranBranch=${eNB_Branch} --ranCommitID=${eNB_CommitID} --ranAllowMerge=${eNB_AllowMergeRequestProcess} --ranTargetBranch=${eNB_TargetBranch} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath} --EPCIPAddress=${params.EPC_IPAddress} --EPCType=${params.EPC_Type} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} --XMLTestFile=${xmlFile} --OCUserName=${OC_Username} --OCPassword=${OC_Password} --OCProjectName=${OC_ProjectName}"
|
||||
} catch (Exception e) {
|
||||
currentBuild.result = 'FAILURE'
|
||||
buildStageStatus = false
|
||||
}
|
||||
}
|
||||
}
|
||||
sh "python3 main.py --mode=FinalizeHtml --finalStatus=${buildStageStatus} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage ("Log Collection") {
|
||||
steps {
|
||||
script {
|
||||
dir ('ci-scripts') {
|
||||
if (directoryExistsGlob("build_log_*")) {
|
||||
sh "zip -r -qq build_log_${env.BUILD_ID}.zip build_log_*/*"
|
||||
sh "rm -rf build_log_*/"
|
||||
archiveArtifacts artifacts: "build_log_${env.BUILD_ID}.zip"
|
||||
}
|
||||
if (directoryExistsGlob("test_log_*")) {
|
||||
sh "zip -r -qq test_log_${env.BUILD_ID}.zip test_log_*/*"
|
||||
sh "rm -rf test_log_*/"
|
||||
archiveArtifacts artifacts: "test_log_${env.BUILD_ID}.zip"
|
||||
}
|
||||
if (fileExists("test_results.html")) {
|
||||
sh "mv test_results.html test_results-${env.JOB_NAME}.html"
|
||||
sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: <a href=\"${BUILD_URL}\">${BUILD_ID}</a>#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results-${JOB_NAME}.html"
|
||||
archiveArtifacts artifacts: "test_results-${env.JOB_NAME}.html"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -41,7 +41,7 @@ import constants as CONST
|
||||
#-----------------------------------------------------------
|
||||
|
||||
|
||||
def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
|
||||
|
||||
|
||||
py_param_file_present = False
|
||||
@@ -82,6 +82,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
CONTAINERS.ranRepository=matchReg.group(1)
|
||||
SCA.ranRepository=matchReg.group(1)
|
||||
PHYSIM.ranRepository=matchReg.group(1)
|
||||
CLUSTER.ranRepository=matchReg.group(1)
|
||||
elif re.match('^\-\-eNB_AllowMerge=(.+)$|^\-\-ranAllowMerge=(.+)$', myArgv, re.IGNORECASE):
|
||||
if re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE)
|
||||
@@ -96,6 +97,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
CONTAINERS.ranAllowMerge=True
|
||||
SCA.ranAllowMerge=True
|
||||
PHYSIM.ranAllowMerge=True
|
||||
CLUSTER.ranAllowMerge=True
|
||||
elif re.match('^\-\-eNBBranch=(.+)$|^\-\-ranBranch=(.+)$', myArgv, re.IGNORECASE):
|
||||
if re.match('^\-\-eNBBranch=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-eNBBranch=(.+)$', myArgv, re.IGNORECASE)
|
||||
@@ -108,6 +110,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
CONTAINERS.ranBranch=matchReg.group(1)
|
||||
SCA.ranBranch=matchReg.group(1)
|
||||
PHYSIM.ranBranch=matchReg.group(1)
|
||||
CLUSTER.ranBranch=matchReg.group(1)
|
||||
elif re.match('^\-\-eNBCommitID=(.*)$|^\-\-ranCommitID=(.*)$', myArgv, re.IGNORECASE):
|
||||
if re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE)
|
||||
@@ -120,6 +123,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
CONTAINERS.ranCommitID=matchReg.group(1)
|
||||
SCA.ranCommitID=matchReg.group(1)
|
||||
PHYSIM.ranCommitID=matchReg.group(1)
|
||||
CLUSTER.ranCommitID=matchReg.group(1)
|
||||
elif re.match('^\-\-eNBTargetBranch=(.*)$|^\-\-ranTargetBranch=(.*)$', myArgv, re.IGNORECASE):
|
||||
if re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE)
|
||||
@@ -132,6 +136,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
CONTAINERS.ranTargetBranch=matchReg.group(1)
|
||||
SCA.ranTargetBranch=matchReg.group(1)
|
||||
PHYSIM.ranTargetBranch=matchReg.group(1)
|
||||
CLUSTER.ranTargetBranch=matchReg.group(1)
|
||||
elif re.match('^\-\-eNBIPAddress=(.+)$|^\-\-eNB[1-2]IPAddress=(.+)$', myArgv, re.IGNORECASE):
|
||||
if re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE)
|
||||
@@ -140,6 +145,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
CONTAINERS.eNBIPAddress=matchReg.group(1)
|
||||
SCA.eNBIPAddress=matchReg.group(1)
|
||||
PHYSIM.eNBIPAddress=matchReg.group(1)
|
||||
CLUSTER.eNBIPAddress=matchReg.group(1)
|
||||
elif re.match('^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE)
|
||||
RAN.eNB1IPAddress=matchReg.group(1)
|
||||
@@ -156,6 +162,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
CONTAINERS.eNBUserName=matchReg.group(1)
|
||||
SCA.eNBUserName=matchReg.group(1)
|
||||
PHYSIM.eNBUserName=matchReg.group(1)
|
||||
CLUSTER.eNBUserName=matchReg.group(1)
|
||||
elif re.match('^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE)
|
||||
RAN.eNB1UserName=matchReg.group(1)
|
||||
@@ -172,6 +179,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
CONTAINERS.eNBPassword=matchReg.group(1)
|
||||
SCA.eNBPassword=matchReg.group(1)
|
||||
PHYSIM.eNBPassword=matchReg.group(1)
|
||||
CLUSTER.eNBPassword=matchReg.group(1)
|
||||
elif re.match('^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE)
|
||||
RAN.eNB1Password=matchReg.group(1)
|
||||
@@ -188,6 +196,7 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
CONTAINERS.eNBSourceCodePath=matchReg.group(1)
|
||||
SCA.eNBSourceCodePath=matchReg.group(1)
|
||||
PHYSIM.eNBSourceCodePath=matchReg.group(1)
|
||||
CLUSTER.eNBSourceCodePath=matchReg.group(1)
|
||||
elif re.match('^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE)
|
||||
RAN.eNB1SourceCodePath=matchReg.group(1)
|
||||
@@ -260,12 +269,21 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM):
|
||||
elif re.match('^\-\-OCUserName=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-OCUserName=(.+)$', myArgv, re.IGNORECASE)
|
||||
PHYSIM.OCUserName = matchReg.group(1)
|
||||
CLUSTER.OCUserName = matchReg.group(1)
|
||||
elif re.match('^\-\-OCPassword=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-OCPassword=(.+)$', myArgv, re.IGNORECASE)
|
||||
PHYSIM.OCPassword = matchReg.group(1)
|
||||
CLUSTER.OCPassword = matchReg.group(1)
|
||||
elif re.match('^\-\-OCProjectName=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-OCProjectName=(.+)$', myArgv, re.IGNORECASE)
|
||||
PHYSIM.OCProjectName = matchReg.group(1)
|
||||
CLUSTER.OCProjectName = matchReg.group(1)
|
||||
elif re.match('^\-\-OCUrl=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-OCUrl=(.+)$', myArgv, re.IGNORECASE)
|
||||
CLUSTER.OCUrl = matchReg.group(1)
|
||||
elif re.match('^\-\-OCRegistry=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-OCRegistry=(.+)$', myArgv, re.IGNORECASE)
|
||||
CLUSTER.OCRegistry = matchReg.group(1)
|
||||
elif re.match('^\-\-BuildId=(.+)$', myArgv, re.IGNORECASE):
|
||||
matchReg = re.match('^\-\-BuildId=(.+)$', myArgv, re.IGNORECASE)
|
||||
RAN.BuildId = matchReg.group(1)
|
||||
|
||||
@@ -0,0 +1,377 @@
|
||||
#/*
|
||||
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
# * contributor license agreements. See the NOTICE file distributed with
|
||||
# * this work for additional information regarding copyright ownership.
|
||||
# * The OpenAirInterface Software Alliance licenses this file to You under
|
||||
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
# * except in compliance with the License.
|
||||
# * You may obtain a copy of the License at
|
||||
# *
|
||||
# * http://www.openairinterface.org/?page_id=698
|
||||
# *
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# * See the License for the specific language governing permissions and
|
||||
# * limitations under the License.
|
||||
# *-------------------------------------------------------------------------------
|
||||
# * For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
# * contact@openairinterface.org
|
||||
# */
|
||||
#---------------------------------------------------------------------
|
||||
# Python for CI of OAI-eNB + COTS-UE
|
||||
#
|
||||
# Required Python Version
|
||||
# Python 3.x
|
||||
#
|
||||
# Required Python Package
|
||||
# pexpect
|
||||
#---------------------------------------------------------------------
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Import
|
||||
#-----------------------------------------------------------
|
||||
import logging
|
||||
import sshconnection as SSH
|
||||
import cls_oai_html
|
||||
#import os
|
||||
import re
|
||||
import time
|
||||
#import subprocess
|
||||
import sys
|
||||
import constants as CONST
|
||||
import helpreadme as HELP
|
||||
import cls_containerize
|
||||
|
||||
class Cluster:
|
||||
def __init__(self):
|
||||
self.eNBIPAddress = ""
|
||||
self.eNBSourceCodePath = ""
|
||||
self.forcedWorkspaceCleanup = False
|
||||
self.OCUserName = ""
|
||||
self.OCPassword = ""
|
||||
self.OCProjectName = ""
|
||||
self.OCUrl = "https://api.oai.cs.eurecom.fr:6443"
|
||||
self.OCRegistry = "default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/"
|
||||
self.ranRepository = ""
|
||||
self.ranBranch = ""
|
||||
self.ranCommitID = ""
|
||||
self.ranAllowMerge = False
|
||||
self.ranTargetBranch = ""
|
||||
|
||||
def _recreate_entitlements(self, sshSession):
|
||||
# recreating entitlements, don't care if deletion fails
|
||||
sshSession.command('oc delete secret etc-pki-entitlement', '\$', 5)
|
||||
if re.search(r"not found", sshSession.getBefore()):
|
||||
logging.warning("no secrets etc-pki-entitlement found, recreating")
|
||||
sshSession.command('ls /etc/pki/entitlement/???????????????????.pem | tail -1', '\$', 5, silent=True)
|
||||
regres1 = re.search(r"/etc/pki/entitlement/[0-9]+.pem", sshSession.getBefore())
|
||||
sshSession.command('ls /etc/pki/entitlement/???????????????????-key.pem | tail -1', '\$', 5, silent=True)
|
||||
regres2 = re.search(r"/etc/pki/entitlement/[0-9]+-key.pem", sshSession.getBefore())
|
||||
if regres1 is None or regres2 is None:
|
||||
logging.error("could not find entitlements")
|
||||
return false
|
||||
file1 = regres1.group(0)
|
||||
file2 = regres2.group(0)
|
||||
sshSession.command(f'oc create secret generic etc-pki-entitlement --from-file {file1} --from-file {file2}', '\$', 5)
|
||||
regres = re.search(r"secret/etc-pki-entitlement created", sshSession.getBefore())
|
||||
if regres is None:
|
||||
logging.error("could not create secret/etc-pki-entitlement")
|
||||
return False
|
||||
return True
|
||||
|
||||
def _recreate_bc(self, sshSession, name, newTag, filename):
|
||||
self._retag_image_statement(sshSession, name, name, newTag, filename)
|
||||
sshSession.command(f'oc delete -f {filename}', '\$', 5)
|
||||
sshSession.command(f'oc create -f {filename}', '\$', 5)
|
||||
before = sshSession.getBefore()
|
||||
if re.search('buildconfig.build.openshift.io/[a-zA-Z\-0-9]+ created', before) is not None:
|
||||
return True
|
||||
logging.error('error while creating buildconfig: ' + sshSession.getBefore())
|
||||
return False
|
||||
|
||||
def _recreate_is(self, sshSession, name, newTag, filename):
|
||||
sshSession.command(f'sed -i -e "s#tag: *latest#tag: {newTag}#" {filename}', '\$', 5)
|
||||
sshSession.command(f'oc delete -f {filename}', '\$', 5)
|
||||
sshSession.command(f'oc create -f {filename}', '\$', 5)
|
||||
before = sshSession.getBefore()
|
||||
if re.search('imagestream.image.openshift.io/[a-zA-Z\-0-9]+ created', before) is not None:
|
||||
return True
|
||||
logging.error('error while creating imagestream: ' + sshSession.getBefore())
|
||||
return False
|
||||
|
||||
def _start_build(self, sshSession, name):
|
||||
# will return "immediately" but build runs in background
|
||||
# if multiple builds are started at the same time, this can take some time, however
|
||||
ret = sshSession.command(f'oc start-build {name} --from-file={self.eNBSourceCodePath}', '\$', 300)
|
||||
before = sshSession.getBefore()
|
||||
regres = re.search(r'build.build.openshift.io/(?P<jobname>[a-zA-Z0-9\-]+) started', str(before))
|
||||
if ret != 0 or before.count('Uploading finished') != 1 or regres is None:
|
||||
logging.error("error during oc start-build: " + sshSession.getBefore())
|
||||
self._delete_pod(sshSession, name)
|
||||
return None
|
||||
return regres.group('jobname') + '-build'
|
||||
|
||||
def _delete_pod(self, sshSession, shortName):
|
||||
sshSession.command(f"oc get pods | grep {shortName}", '\$', 5)
|
||||
regres = re.search(rf'{shortName}-[0-9]+-build', sshSession.getBefore())
|
||||
if regres is not None:
|
||||
sshSession.command(f"oc delete pod {regres.group(0)}", '\$', 5)
|
||||
else:
|
||||
logging.warning(f"no pod found with name {shortName}")
|
||||
|
||||
def _wait_build_end(self, sshSession, jobs, timeout_sec, check_interval_sec = 5):
|
||||
logging.debug(f"waiting for jobs {jobs} to finish building")
|
||||
while timeout_sec > 0:
|
||||
# check status
|
||||
for j in jobs:
|
||||
sshSession.command(f'oc get pods | grep {j}', '\$', 5, silent = True)
|
||||
if sshSession.getBefore().count('Completed') > 0: jobs.remove(j)
|
||||
if sshSession.getBefore().count('Error') > 0:
|
||||
logging.error(f'error for job {j}: ' + sshSession.getBefore())
|
||||
return False
|
||||
if jobs == []:
|
||||
logging.debug('all jobs completed')
|
||||
return True
|
||||
time.sleep(check_interval_sec)
|
||||
timeout_sec -= check_interval_sec
|
||||
logging.error(f"timeout while waiting for end of build of {jobs}")
|
||||
return False
|
||||
|
||||
def _retag_image_statement(self, sshSession, oldImage, newImage, newTag, filename):
|
||||
sshSession.command(f'sed -i -e "s#{oldImage}:latest#{newImage}:{newTag}#" {filename}', '\$', 5)
|
||||
|
||||
def _pull_image(self, sshSession, image, tag):
|
||||
sshSession.command(f'oc whoami -t | sudo podman login -u oaicicd --password-stdin https://{self.OCRegistry} --tls-verify=false', '\$', 5, silent=True)
|
||||
if sshSession.getBefore().count('Login Succeeded!') == 0:
|
||||
return None
|
||||
imageName = f'{self.OCRegistry}{self.OCProjectName}/{image}:{tag}'
|
||||
sshSession.command(f'sudo podman pull {imageName} --tls-verify=false', '\$', 300)
|
||||
pullResult = sshSession.getBefore()
|
||||
sshSession.command(f'sudo podman logout https://{self.OCRegistry}', '\$', 10, silent=True)
|
||||
if pullResult.count('Storing signatures') == 0:
|
||||
return None
|
||||
return imageName
|
||||
|
||||
def _get_image_size(self, sshSession, image, tag):
|
||||
# get the SHA of the image we built using the image name and its tag
|
||||
sshSession.command(f'oc describe is {image} | grep -A4 {tag}', '\$', 5)
|
||||
result = re.search(f'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/(?P<imageSha>{image}@sha256:[a-f0-9]+)', sshSession.getBefore())
|
||||
if result is None:
|
||||
return -1
|
||||
imageSha = result.group("imageSha")
|
||||
|
||||
# retrieve the size
|
||||
sshSession.command(f'oc get -o json isimage {imageSha} | jq -Mc "{{dockerImageSize: .image.dockerImageMetadata.Size}}"', '\$', 5)
|
||||
result = re.search('{"dockerImageSize":(?P<size>[0-9]+)}', str(sshSession.getBefore()))
|
||||
if result is None:
|
||||
return -1
|
||||
return int(result.group("size"))
|
||||
|
||||
def _deploy_pod(self, sshSession, filename, timeout = 30):
|
||||
sshSession.command(f'oc create -f {filename}', '\$', 10)
|
||||
result = re.search(f'pod/(?P<pod>[a-zA-Z0-9_\-]+) created', sshSession.getBefore())
|
||||
if result is None:
|
||||
logging.error(f'could not deploy pod: {sshSession.getBefore()}')
|
||||
return None
|
||||
pod = result.group("pod")
|
||||
logging.debug(f'checking if pod {pod} is in Running state')
|
||||
while timeout > 0:
|
||||
sshSession.command(f'oc get pod {pod} -o json | jq -Mc .status.phase', '\$', 5, silent=True)
|
||||
if re.search('"Running"', sshSession.getBefore()) is not None: return pod
|
||||
timeout -= 1
|
||||
time.sleep(1)
|
||||
logging.error(f'pod {pod} did not reach Running state')
|
||||
self._undeploy_pod(sshSession, filename)
|
||||
return None
|
||||
|
||||
def _undeploy_pod(self, sshSession, filename):
|
||||
# to save time we start this in the background and trust that oc stops correctly
|
||||
sshSession.command(f'oc delete -f {filename} &', '\$', 5)
|
||||
|
||||
def BuildClusterImage(self, HTML):
|
||||
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
sys.exit(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {ranBranch} ranCommitID {self.ranCommitID}')
|
||||
|
||||
lIpAddr = self.eNBIPAddress
|
||||
lSourcePath = self.eNBSourceCodePath
|
||||
if lIpAddr == '' or lSourcePath == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
sys.exit('Insufficient Parameter: no SSH Credentials')
|
||||
ocUserName = self.OCUserName
|
||||
ocPassword = self.OCPassword
|
||||
ocProjectName = self.OCProjectName
|
||||
if ocUserName == '' or ocPassword == '' or ocProjectName == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
sys.exit('Insufficient Parameter: no OC Credentials')
|
||||
if self.OCRegistry.startswith("http") and not self.OCRegistry.endswith("/"):
|
||||
sys.exit(f'ocRegistry {self.OCRegistry} should not start with http:// or https:// and end on a slash /')
|
||||
|
||||
logging.debug(f'Building on cluster triggered from server: {lIpAddr}')
|
||||
mySSH = SSH.SSHConnection()
|
||||
mySSH.open(lIpAddr, "oaicicd", "SHOULD NOT BE NECESSARY")
|
||||
|
||||
self.testCase_id = HTML.testCase_id
|
||||
|
||||
# Workaround for some servers, we need to erase completely the workspace
|
||||
if self.forcedWorkspaceCleanup:
|
||||
mySSH.command(f'sudo rm -Rf {lSourcePath}', '\$', 15)
|
||||
cls_containerize.CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
||||
|
||||
# we don't necessarily need a forced workspace cleanup, but in
|
||||
# order to reduce the amount of data send to OpenShift, we
|
||||
# manually delete all generated files in the workspace
|
||||
mySSH.command(f'rm -rf {lSourcePath}/cmake_targets/ran_build', '\$', 30);
|
||||
|
||||
baseTag = 'develop'
|
||||
forceBaseImageBuild = False
|
||||
imageTag = 'develop'
|
||||
if self.ranAllowMerge: # merging MR branch into develop -> temporary image
|
||||
imageTag = 'ci-temp'
|
||||
if self.ranTargetBranch == 'develop':
|
||||
mySSH.command(f'git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base.rhel8.2 | grep --colour=never -i INDEX', '\$', 5)
|
||||
result = re.search('index', mySSH.getBefore())
|
||||
if result is not None:
|
||||
forceBaseImageBuild = True
|
||||
baseTag = 'ci-temp'
|
||||
else:
|
||||
forceBaseImageBuild = True
|
||||
|
||||
# logging to OC Cluster and then switch to corresponding project
|
||||
mySSH.command(f'oc login -u {ocUserName} -p {ocPassword} --server {self.OCUrl}', '\$', 31)
|
||||
if mySSH.getBefore().count('Login successful.') == 0:
|
||||
logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m')
|
||||
mySSH.close()
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_LOGIN_FAIL)
|
||||
return False
|
||||
|
||||
mySSH.command(f'oc project {ocProjectName}', '\$', 30)
|
||||
if mySSH.getBefore().count(f'Already on project "{ocProjectName}"') == 0 and mySSH.getBefore().count(f'Now using project "{self.OCProjectName}"') == 0:
|
||||
logging.error(f'\u001B[1mUnable to access OC project {ocProjectName}\u001B[0m')
|
||||
mySSH.command('oc logout', '\$', 30)
|
||||
mySSH.close()
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_PROJECT_FAIL)
|
||||
return False
|
||||
|
||||
self._recreate_entitlements(mySSH)
|
||||
|
||||
status = True # flag to abandon compiling if any image fails
|
||||
attemptedImages = []
|
||||
if forceBaseImageBuild:
|
||||
self._recreate_is(mySSH, 'ran-base', baseTag, 'openshift/ran-base-is.yaml')
|
||||
self._recreate_bc(mySSH, 'ran-base', baseTag, 'openshift/ran-base-bc.yaml')
|
||||
ranbase_job = self._start_build(mySSH, 'ran-base')
|
||||
attemptedImages += ['ran-base']
|
||||
status = ranbase_job is not None and self._wait_build_end(mySSH, [ranbase_job], 600)
|
||||
if not status: logging.error('failure during build of ran-base')
|
||||
mySSH.command(f'oc logs {ranbase_job} > cmake_targets/log/ran-base.log', '\$', 10)
|
||||
# recover logs by mounting image
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml')
|
||||
pod = self._deploy_pod(mySSH, 'openshift/ran-base-log-retrieval.yaml')
|
||||
if pod is not None:
|
||||
mySSH.command(f'mkdir -p cmake_targets/log/ran-base', '\$', 5)
|
||||
mySSH.command(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base', '\$', 5)
|
||||
self._undeploy_pod(mySSH, 'openshift/ran-base-log-retrieval.yaml')
|
||||
else:
|
||||
status = False
|
||||
|
||||
if status:
|
||||
self._recreate_is(mySSH, 'oai-physim', imageTag, 'openshift/oai-physim-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-physim', imageTag, 'openshift/oai-physim-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel8.2')
|
||||
physim_job = self._start_build(mySSH, 'oai-physim')
|
||||
attemptedImages += ['oai-physim']
|
||||
|
||||
self._recreate_is(mySSH, 'ran-build', imageTag, 'openshift/ran-build-is.yaml')
|
||||
self._recreate_bc(mySSH, 'ran-build', imageTag, 'openshift/ran-build-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel8.2')
|
||||
ranbuild_job = self._start_build(mySSH, 'ran-build')
|
||||
attemptedImages += ['ran-build']
|
||||
|
||||
wait = ranbuild_job is not None and physim_job is not None and self._wait_build_end(mySSH, [ranbuild_job, physim_job], 1200)
|
||||
if not wait: logging.error('error during build of ranbuild_job or physim_job')
|
||||
status = status and wait
|
||||
mySSH.command(f'oc logs {ranbuild_job} > cmake_targets/log/ran-build.log', '\$', 10)
|
||||
mySSH.command(f'oc logs {physim_job} > cmake_targets/log/oai-physim.log', '\$', 10)
|
||||
|
||||
if status:
|
||||
self._recreate_is(mySSH, 'oai-enb', imageTag, 'openshift/oai-enb-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-enb', imageTag, 'openshift/oai-enb-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel8.2')
|
||||
self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel8.2')
|
||||
enb_job = self._start_build(mySSH, 'oai-enb')
|
||||
attemptedImages += ['oai-enb']
|
||||
|
||||
self._recreate_is(mySSH, 'oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel8.2')
|
||||
self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel8.2')
|
||||
gnb_job = self._start_build(mySSH, 'oai-gnb')
|
||||
attemptedImages += ['oai-gnb']
|
||||
|
||||
self._recreate_is(mySSH, 'oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel8.2')
|
||||
self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel8.2')
|
||||
lteue_job = self._start_build(mySSH, 'oai-lte-ue')
|
||||
attemptedImages += ['oai-lte-ue']
|
||||
|
||||
self._recreate_is(mySSH, 'oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml')
|
||||
self._recreate_bc(mySSH, 'oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml')
|
||||
self._retag_image_statement(mySSH, 'ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel8.2')
|
||||
self._retag_image_statement(mySSH, 'ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel8.2')
|
||||
nrue_job = self._start_build(mySSH, 'oai-nr-ue')
|
||||
attemptedImages += ['oai-nr-ue']
|
||||
|
||||
wait = enb_job is not None and gnb_job is not None and lteue_job is not None and nrue_job is not None and self._wait_build_end(mySSH, [enb_job, gnb_job, lteue_job, nrue_job], 600)
|
||||
if not wait: logging.error('error during build of eNB/gNB/lteUE/nrUE')
|
||||
status = status and wait
|
||||
# recover logs
|
||||
mySSH.command(f'oc logs {enb_job} > cmake_targets/log/oai-enb.log', '\$', 10)
|
||||
mySSH.command(f'oc logs {gnb_job} > cmake_targets/log/oai-gnb.log', '\$', 10)
|
||||
mySSH.command(f'oc logs {lteue_job} > cmake_targets/log/oai-lte-ue.log', '\$', 10)
|
||||
mySSH.command(f'oc logs {nrue_job} > cmake_targets/log/oai-nr-ue.log', '\$', 10)
|
||||
|
||||
# split and analyze logs
|
||||
imageSize = {}
|
||||
for image in attemptedImages:
|
||||
mySSH.command(f'mkdir -p cmake_targets/log/{image}', '\$', 5)
|
||||
mySSH.command(f'python3 ci-scripts/docker_log_split.py --logfilename=cmake_targets/log/{image}.log', '\$', 5)
|
||||
tag = imageTag if image != 'ran-base' else baseTag
|
||||
size = self._get_image_size(mySSH, image, tag)
|
||||
if size <= 0:
|
||||
imageSize[image] = 'unknown -- BUILD FAILED'
|
||||
status = False
|
||||
else:
|
||||
sizeMb = float(size) / 1000000
|
||||
imageSize[image] = f'{sizeMb:.1f} Mbytes (uncompressed: ~{sizeMb*2.5:.1f} Mbytes)'
|
||||
logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m')
|
||||
|
||||
build_log_name = f'build_log_{self.testCase_id}'
|
||||
cls_containerize.CopyLogsToExecutor(mySSH, lSourcePath, build_log_name, lIpAddr, 'oaicicd', CONST.CI_NO_PASSWORD)
|
||||
|
||||
mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done', '\$', 60)
|
||||
|
||||
# logout will return eventually, but we don't care when -> start in background
|
||||
mySSH.command('oc logout &', '\$', 5)
|
||||
mySSH.close()
|
||||
|
||||
# Analyze the logs
|
||||
collectInfo = cls_containerize.AnalyzeBuildLogs(build_log_name, attemptedImages, status)
|
||||
for img in collectInfo:
|
||||
for f in collectInfo[img]:
|
||||
status = status and collectInfo[img][f]['status']
|
||||
if not status:
|
||||
logging.debug(collectInfo)
|
||||
|
||||
if status:
|
||||
logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
|
||||
HTML.CreateHtmlTestRow('all', 'OK', CONST.ALL_PROCESSES_OK)
|
||||
else:
|
||||
logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
|
||||
HTML.CreateHtmlTestRow('all', 'KO', CONST.ALL_PROCESSES_OK)
|
||||
|
||||
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize)
|
||||
|
||||
return status
|
||||
+156
-167
@@ -48,6 +48,107 @@ import sshconnection as SSH
|
||||
import helpreadme as HELP
|
||||
import constants as CONST
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Helper functions used here and in other classes
|
||||
# (e.g., cls_cluster.py)
|
||||
#-----------------------------------------------------------
|
||||
def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTargetBranch, ranAllowMerge):
|
||||
# on RedHat/CentOS .git extension is mandatory
|
||||
result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', ranRepository)
|
||||
if result is not None:
|
||||
full_ran_repo_name = ranRepository.replace('git/', 'git')
|
||||
else:
|
||||
full_ran_repo_name = ranRepository + '.git'
|
||||
sshSession.command('mkdir -p ' + sourcePath, '\$', 5)
|
||||
sshSession.command('cd ' + sourcePath, '\$', 5)
|
||||
sshSession.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
|
||||
if sshSession.getBefore().count('done.') == 0:
|
||||
logging.warning('did not find \'done.\' in git output while cloning/fetching, was not successful?')
|
||||
sshSession.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
|
||||
sshSession.command('git config user.name "OAI Jenkins"', '\$', 5)
|
||||
|
||||
sshSession.command('sudo git clean -x -d -ff', '\$', 30)
|
||||
sshSession.command('mkdir -p cmake_targets/log', '\$', 5)
|
||||
# if the commit ID is provided use it to point to it
|
||||
if ranCommitID != '':
|
||||
sshSession.command('git checkout -f ' + ranCommitID, '\$', 30)
|
||||
# if the branch is not develop, then it is a merge request and we need to do
|
||||
# the potential merge. Note that merge conflicts should already been checked earlier
|
||||
if ranAllowMerge:
|
||||
if ranTargetBranch == '':
|
||||
sshSession.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
|
||||
else:
|
||||
logging.debug('Merging with the target branch: ' + ranTargetBranch)
|
||||
sshSession.command('git merge --ff origin/' + ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
|
||||
|
||||
def CopyLogsToExecutor(sshSession, sourcePath, log_name, scpIp, scpUser, scpPw):
|
||||
sshSession.command(f'cd {sourcePath}/cmake_targets', '\$', 5)
|
||||
sshSession.command(f'rm -f {log_name}.zip', '\$', 5)
|
||||
sshSession.command(f'mkdir -p {log_name}', '\$', 5)
|
||||
sshSession.command(f'mv log/* {log_name}', '\$', 5)
|
||||
sshSession.command(f'zip -r -qq {log_name}.zip {log_name}', '\$', 5)
|
||||
|
||||
# copy zip to executor for analysis
|
||||
if (os.path.isfile(f'./{log_name}.zip')):
|
||||
os.remove(f'./{log_name}.zip')
|
||||
if (os.path.isdir(f'./{log_name}')):
|
||||
shutil.rmtree(f'./{log_name}')
|
||||
sshSession.copyin(scpIp, scpUser, scpPw, f'{sourcePath}/cmake_targets/{log_name}.zip', '.')
|
||||
sshSession.command(f'rm -f {log_name}.zip','\$', 5)
|
||||
ZipFile(f'{log_name}.zip').extractall('.')
|
||||
|
||||
def AnalyzeBuildLogs(buildRoot, images, globalStatus):
|
||||
collectInfo = {}
|
||||
for image in images:
|
||||
files = {}
|
||||
file_list = [f for f in os.listdir(f'{buildRoot}/{image}') if os.path.isfile(os.path.join(f'{buildRoot}/{image}', f)) and f.endswith('.txt')]
|
||||
# Analyze the "sub-logs" of every target image
|
||||
for fil in file_list:
|
||||
errorandwarnings = {}
|
||||
warningsNo = 0
|
||||
errorsNo = 0
|
||||
with open(f'{buildRoot}/{image}/{fil}', mode='r') as inputfile:
|
||||
for line in inputfile:
|
||||
result = re.search(' ERROR ', str(line))
|
||||
if result is not None:
|
||||
errorsNo += 1
|
||||
result = re.search(' error:', str(line))
|
||||
if result is not None:
|
||||
errorsNo += 1
|
||||
result = re.search(' WARNING ', str(line))
|
||||
if result is not None:
|
||||
warningsNo += 1
|
||||
result = re.search(' warning:', str(line))
|
||||
if result is not None:
|
||||
warningsNo += 1
|
||||
errorandwarnings['errors'] = errorsNo
|
||||
errorandwarnings['warnings'] = warningsNo
|
||||
errorandwarnings['status'] = globalStatus
|
||||
files[fil] = errorandwarnings
|
||||
# Analyze the target image
|
||||
if os.path.isfile(f'{buildRoot}/{image}.log'):
|
||||
errorandwarnings = {}
|
||||
committed = False
|
||||
tagged = False
|
||||
with open(f'{buildRoot}/{image}.log', mode='r') as inputfile:
|
||||
startOfTargetImageCreation = False # check for tagged/committed only after image created
|
||||
for line in inputfile:
|
||||
result = re.search(f'FROM .* [aA][sS] {image}$', str(line))
|
||||
if result is not None:
|
||||
startOfTargetImageCreation = True
|
||||
if startOfTargetImageCreation:
|
||||
lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None
|
||||
tagged = tagged or lineHasTag
|
||||
# the OpenShift Cluster builder prepends image registry URL
|
||||
lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None
|
||||
committed = committed or lineHasCommit
|
||||
errorandwarnings['errors'] = 0 if committed or tagged else 1
|
||||
errorandwarnings['warnings'] = 0
|
||||
errorandwarnings['status'] = committed or tagged
|
||||
files['Target Image Creation'] = errorandwarnings
|
||||
collectInfo[image] = files
|
||||
return collectInfo
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Class Declaration
|
||||
#-----------------------------------------------------------
|
||||
@@ -91,8 +192,6 @@ class Containerize():
|
||||
self.cliBuildOptions = ''
|
||||
self.dockerfileprefix = ''
|
||||
self.host = ''
|
||||
self.allImagesSize = {}
|
||||
self.collectInfo = {}
|
||||
|
||||
self.deployedContainers = []
|
||||
self.tsharkStarted = False
|
||||
@@ -115,35 +214,6 @@ class Containerize():
|
||||
# Container management functions
|
||||
#-----------------------------------------------------------
|
||||
|
||||
def _createWorkspace(self, sshSession, password, sourcePath):
|
||||
# on RedHat/CentOS .git extension is mandatory
|
||||
result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
|
||||
if result is not None:
|
||||
full_ran_repo_name = self.ranRepository.replace('git/', 'git')
|
||||
else:
|
||||
full_ran_repo_name = self.ranRepository + '.git'
|
||||
sshSession.command('mkdir -p ' + sourcePath, '\$', 5)
|
||||
sshSession.command('cd ' + sourcePath, '\$', 5)
|
||||
sshSession.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
|
||||
# Raphael: here add a check if git clone or git fetch went smoothly
|
||||
sshSession.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
|
||||
sshSession.command('git config user.name "OAI Jenkins"', '\$', 5)
|
||||
|
||||
sshSession.command('echo ' + password + ' | sudo -S git clean -x -d -ff', '\$', 30)
|
||||
sshSession.command('mkdir -p cmake_targets/log', '\$', 5)
|
||||
# if the commit ID is provided use it to point to it
|
||||
if self.ranCommitID != '':
|
||||
sshSession.command('git checkout -f ' + self.ranCommitID, '\$', 30)
|
||||
# if the branch is not develop, then it is a merge request and we need to do
|
||||
# the potential merge. Note that merge conflicts should already been checked earlier
|
||||
if (self.ranAllowMerge):
|
||||
if self.ranTargetBranch == '':
|
||||
if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
|
||||
sshSession.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
|
||||
else:
|
||||
logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
|
||||
sshSession.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
|
||||
|
||||
def BuildImage(self, HTML):
|
||||
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
@@ -211,13 +281,14 @@ class Containerize():
|
||||
|
||||
self.testCase_id = HTML.testCase_id
|
||||
|
||||
self._createWorkspace(mySSH, lPassWord, lSourcePath)
|
||||
CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
||||
|
||||
# if asterix, copy the entitlement and subscription manager configurations
|
||||
if self.host == 'Red Hat':
|
||||
mySSH.command('mkdir -p tmp/ca/ tmp/entitlement/', '\$', 5)
|
||||
mySSH.command('sudo cp /etc/rhsm/ca/redhat-uep.pem tmp/ca/', '\$', 5)
|
||||
mySSH.command('sudo cp /etc/pki/entitlement/*.pem tmp/entitlement/', '\$', 5)
|
||||
mySSH.command('mkdir -p ./etc-pki-entitlement ./rhsm-conf ./rhsm-ca', '\$', 5)
|
||||
mySSH.command('sudo cp /etc/rhsm/rhsm.conf ./rhsm-conf/', '\$', 5)
|
||||
mySSH.command('sudo cp /etc/rhsm/ca/redhat-uep.pem ./rhsm-ca/', '\$', 5)
|
||||
mySSH.command('sudo cp /etc/pki/entitlement/*.pem ./etc-pki-entitlement/', '\$', 5)
|
||||
|
||||
baseImage = 'ran-base'
|
||||
baseTag = 'develop'
|
||||
@@ -246,32 +317,10 @@ class Containerize():
|
||||
if forceBaseImageBuild:
|
||||
mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + baseImage + ' --tag ' + baseImage + ':' + baseTag + ' --file docker/Dockerfile.base' + self.dockerfileprefix + ' . > cmake_targets/log/ran-base.log 2>&1', '\$', 1600)
|
||||
# First verify if the base image was properly created.
|
||||
status = True
|
||||
mySSH.command(self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + baseImage + ':' + baseTag, '\$', 5)
|
||||
allImagesSize = {}
|
||||
if mySSH.getBefore().count('o such image') != 0:
|
||||
logging.error('\u001B[1m Could not build properly ran-base\u001B[0m')
|
||||
status = False
|
||||
else:
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
|
||||
if result is not None:
|
||||
imageSize = float(result.group('size'))
|
||||
imageSize = imageSize / 1000
|
||||
if imageSize < 1000:
|
||||
logging.debug('\u001B[1m ran-base size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
|
||||
self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' kbytes'
|
||||
else:
|
||||
imageSize = imageSize / 1000
|
||||
if imageSize < 1000:
|
||||
logging.debug('\u001B[1m ran-base size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
|
||||
self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' Mbytes'
|
||||
else:
|
||||
imageSize = imageSize / 1000
|
||||
logging.debug('\u001B[1m ran-base size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
|
||||
self.allImagesSize['ran-base'] = str(round(imageSize,1)) + ' Gbytes'
|
||||
else:
|
||||
logging.debug('ran-base size is unknown')
|
||||
# If the base image failed, no need to continue
|
||||
if not status:
|
||||
# Recover the name of the failed container?
|
||||
mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5)
|
||||
mySSH.command(self.cli + ' image prune --force', '\$', 30)
|
||||
@@ -281,20 +330,32 @@ class Containerize():
|
||||
HTML.CreateHtmlTabFooter(False)
|
||||
sys.exit(1)
|
||||
else:
|
||||
# Recover build logs, for the moment only possible when build is successful
|
||||
mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5)
|
||||
mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5)
|
||||
mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5)
|
||||
mySSH.command(self.cli + ' rm -f test', '\$', 5)
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
|
||||
if result is not None:
|
||||
size = float(result.group("size")) / 1000000
|
||||
imageSizeStr = f'{size:.1f}'
|
||||
logging.debug(f'\u001B[1m ran-base size is {imageSizeStr} Mbytes\u001B[0m')
|
||||
allImagesSize['ran-base'] = f'{imageSizeStr} Mbytes'
|
||||
else:
|
||||
logging.debug('ran-base size is unknown')
|
||||
|
||||
# Recover build logs, for the moment only possible when build is successful
|
||||
mySSH.command(self.cli + ' create --name test ' + baseImage + ':' + baseTag, '\$', 5)
|
||||
mySSH.command('mkdir -p cmake_targets/log/ran-base', '\$', 5)
|
||||
mySSH.command(self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base', '\$', 5)
|
||||
mySSH.command(self.cli + ' rm -f test', '\$', 5)
|
||||
|
||||
# Build the target image(s)
|
||||
status = True
|
||||
attemptedImages = ['ran-base']
|
||||
for image,pattern in imageNames:
|
||||
attemptedImages += [image]
|
||||
# the archived Dockerfiles have "ran-base:latest" as base image
|
||||
# we need to update them with proper tag
|
||||
mySSH.command('sed -i -e "s#' + baseImage + ':latest#' + baseImage + ':' + baseTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5)
|
||||
mySSH.command(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5)
|
||||
if image != 'ran-build':
|
||||
mySSH.command('sed -i -e "s#' + "ran-build" + ':latest#' + "ran-build" + ':' + imageTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5)
|
||||
mySSH.command(self.cli + ' build ' + self.cliBuildOptions + ' --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + self.dockerfileprefix + ' . > cmake_targets/log/' + image + '.log 2>&1', '\$', 1200)
|
||||
mySSH.command(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}', '\$', 5)
|
||||
mySSH.command(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {image}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} . > cmake_targets/log/{image}.log 2>&1', '\$', 1200)
|
||||
# Flatten Image
|
||||
if image != 'ran-build':
|
||||
mySSH.command('python3 ./ci-scripts/flatten_image.py --tag ' + image + ':' + imageTag, '\$', 300)
|
||||
@@ -308,117 +369,43 @@ class Containerize():
|
||||
status = False
|
||||
# Here we should check if the last container corresponds to a failed command and destroy it
|
||||
mySSH.command(self.cli + ' ps --quiet --filter "status=exited" -n1 | xargs ' + self.cli + ' rm -f', '\$', 5)
|
||||
self.allImagesSize[image] = 'N/A -- Build Failed'
|
||||
allImagesSize[image] = 'N/A -- Build Failed'
|
||||
break
|
||||
else:
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
|
||||
if result is not None:
|
||||
imageSize = float(result.group('size'))
|
||||
imageSize = imageSize / 1000
|
||||
if imageSize < 1000:
|
||||
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
|
||||
self.allImagesSize[image] = str(round(imageSize,1)) + ' kbytes'
|
||||
else:
|
||||
imageSize = imageSize / 1000
|
||||
if imageSize < 1000:
|
||||
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
|
||||
self.allImagesSize[image] = str(round(imageSize,1)) + ' Mbytes'
|
||||
else:
|
||||
imageSize = imageSize / 1000
|
||||
logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
|
||||
self.allImagesSize[image] = str(round(imageSize,1)) + ' Gbytes'
|
||||
size = float(result.group("size")) / 1000000
|
||||
imageSizeStr = f'{size:.1f}'
|
||||
logging.debug(f'\u001B[1m {image} size is {imageSizeStr} Mbytes\u001B[0m')
|
||||
allImagesSize[image] = f'{imageSizeStr} Mbytes'
|
||||
else:
|
||||
logging.debug('ran-base size is unknown')
|
||||
self.allImagesSize[image] = 'unknown'
|
||||
logging.debug(f'{image} size is unknown')
|
||||
allImagesSize[image] = 'unknown'
|
||||
# Now pruning dangling images in between target builds
|
||||
mySSH.command(self.cli + ' image prune --force', '\$', 30)
|
||||
|
||||
# Analyzing the logs
|
||||
mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
|
||||
mySSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
|
||||
mySSH.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
|
||||
|
||||
mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
|
||||
mySSH.command('rm -f build_log_' + self.testCase_id + '.zip || true', '\$', 5)
|
||||
if (os.path.isfile('./build_log_' + self.testCase_id + '.zip')):
|
||||
os.remove('./build_log_' + self.testCase_id + '.zip')
|
||||
if (os.path.isdir('./build_log_' + self.testCase_id)):
|
||||
shutil.rmtree('./build_log_' + self.testCase_id)
|
||||
mySSH.command('zip -r -qq build_log_' + self.testCase_id + '.zip build_log_' + self.testCase_id, '\$', 5)
|
||||
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/build_log_' + self.testCase_id + '.zip', '.')
|
||||
mySSH.command('rm -f build_log_' + self.testCase_id + '.zip','\$', 5)
|
||||
# Remove all intermediate build images
|
||||
# Remove all intermediate build images and clean up
|
||||
if self.ranAllowMerge and forceBaseImageBuild:
|
||||
mySSH.command(self.cli + ' image rm ' + baseImage + ':' + baseTag + ' || true', '\$', 30)
|
||||
mySSH.command(self.cli + ' image rm ran-build:' + imageTag + ' || true','\$', 30)
|
||||
# Cleaning any created tmp volume
|
||||
mySSH.command(self.cli + ' volume prune --force || true','\$', 15)
|
||||
mySSH.close()
|
||||
ZipFile('build_log_' + self.testCase_id + '.zip').extractall('.')
|
||||
mySSH.command(f'{self.cli} image rm {baseImage}:{baseTag} || true', '\$', 30)
|
||||
mySSH.command(f'{self.cli} image rm ran-build:{imageTag}','\$', 30)
|
||||
mySSH.command(f'{self.cli} volume prune --force','\$', 15)
|
||||
|
||||
#Trying to identify the errors and warnings for each built images
|
||||
imageNames1 = imageNames
|
||||
base = ('ran-base','ran')
|
||||
imageNames1.insert(0, base)
|
||||
for image,pattern in imageNames1:
|
||||
files = {}
|
||||
file_list = [f for f in os.listdir('build_log_' + self.testCase_id + '/' + image) if os.path.isfile(os.path.join('build_log_' + self.testCase_id + '/' + image, f)) and f.endswith('.txt')]
|
||||
for fil in file_list:
|
||||
errorandwarnings = {}
|
||||
warningsNo = 0
|
||||
errorsNo = 0
|
||||
with open('build_log_{}/{}/{}'.format(self.testCase_id,image,fil), mode='r') as inputfile:
|
||||
for line in inputfile:
|
||||
result = re.search(' ERROR ', str(line))
|
||||
if result is not None:
|
||||
errorsNo += 1
|
||||
result = re.search(' error:', str(line))
|
||||
if result is not None:
|
||||
errorsNo += 1
|
||||
result = re.search(' WARNING ', str(line))
|
||||
if result is not None:
|
||||
warningsNo += 1
|
||||
result = re.search(' warning:', str(line))
|
||||
if result is not None:
|
||||
warningsNo += 1
|
||||
errorandwarnings['errors'] = errorsNo
|
||||
errorandwarnings['warnings'] = warningsNo
|
||||
errorandwarnings['status'] = status
|
||||
files[fil] = errorandwarnings
|
||||
# Let analyze the target image creation part
|
||||
if os.path.isfile('build_log_{}/{}.log'.format(self.testCase_id,image)):
|
||||
errorandwarnings = {}
|
||||
with open('build_log_{}/{}.log'.format(self.testCase_id,image), mode='r') as inputfile:
|
||||
startOfTargetImageCreation = False
|
||||
buildStatus = False
|
||||
for line in inputfile:
|
||||
result = re.search('FROM .* [aA][sS] ' + image + '$', str(line))
|
||||
if result is not None:
|
||||
startOfTargetImageCreation = True
|
||||
if startOfTargetImageCreation:
|
||||
result = re.search('Successfully tagged ' + image + ':', str(line))
|
||||
if result is not None:
|
||||
buildStatus = True
|
||||
result = re.search('COMMIT ' + image + ':', str(line))
|
||||
if result is not None:
|
||||
buildStatus = True
|
||||
inputfile.close()
|
||||
if buildStatus:
|
||||
errorandwarnings['errors'] = 0
|
||||
else:
|
||||
errorandwarnings['errors'] = 1
|
||||
errorandwarnings['warnings'] = 0
|
||||
errorandwarnings['status'] = buildStatus
|
||||
files['Target Image Creation'] = errorandwarnings
|
||||
self.collectInfo[image] = files
|
||||
# create a zip with all logs
|
||||
build_log_name = f'build_log_{self.testCase_id}'
|
||||
CopyLogsToExecutor(mySSH, lSourcePath, build_log_name, lIpAddr, lUserName, lPassWord)
|
||||
mySSH.close()
|
||||
|
||||
# Analyze the logs
|
||||
collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status)
|
||||
|
||||
if status:
|
||||
logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
|
||||
HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK)
|
||||
HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize)
|
||||
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
|
||||
else:
|
||||
logging.error('\u001B[1m Building OAI Images Failed\u001B[0m')
|
||||
HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK)
|
||||
HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize)
|
||||
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
|
||||
HTML.CreateHtmlTabFooter(False)
|
||||
sys.exit(1)
|
||||
|
||||
@@ -473,7 +460,7 @@ class Containerize():
|
||||
self.ranCommitID = self.proxyCommit
|
||||
self.ranRepository = 'https://github.com/EpiSci/oai-lte-5g-multi-ue-proxy.git'
|
||||
self.ranAllowMerge = False
|
||||
self._createWorkspace(mySSH, lPassWord, lSourcePath)
|
||||
CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
||||
# to prevent accidentally overwriting data that might be used later
|
||||
self.ranCommitID = oldRanCommidID
|
||||
self.ranRepository = oldRanRepository
|
||||
@@ -540,16 +527,18 @@ class Containerize():
|
||||
errorandwarnings['warnings'] = 0
|
||||
errorandwarnings['status'] = True
|
||||
files['Target Image Creation'] = errorandwarnings
|
||||
self.collectInfo['proxy'] = files
|
||||
collectInfo = {}
|
||||
collectInfo['proxy'] = files
|
||||
mySSH.command('docker image inspect --format=\'Size = {{.Size}} bytes\' proxy:' + tag, '\$', 5)
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
|
||||
allImagesSize = {}
|
||||
if result is not None:
|
||||
imageSize = float(result.group('size')) / 1000000
|
||||
logging.debug('\u001B[1m proxy size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
|
||||
self.allImagesSize['proxy'] = str(round(imageSize,1)) + ' Mbytes'
|
||||
allImagesSize['proxy'] = str(round(imageSize,1)) + ' Mbytes'
|
||||
else:
|
||||
logging.debug('proxy size is unknown')
|
||||
self.allImagesSize['proxy'] = 'unknown'
|
||||
allImagesSize['proxy'] = 'unknown'
|
||||
|
||||
# Cleaning any created tmp volume
|
||||
mySSH.command(self.cli + ' volume prune --force || true','\$', 15)
|
||||
@@ -557,7 +546,7 @@ class Containerize():
|
||||
|
||||
logging.info('\u001B[1m Building L2sim Proxy Image Pass\u001B[0m')
|
||||
HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK)
|
||||
HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize)
|
||||
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize)
|
||||
|
||||
def Copy_Image_to_Test_Server(self, HTML):
|
||||
imageTag = 'develop'
|
||||
@@ -652,7 +641,7 @@ class Containerize():
|
||||
mySSH = SSH.SSHConnection()
|
||||
mySSH.open(lIpAddr, lUserName, lPassWord)
|
||||
|
||||
self._createWorkspace(mySSH, lPassWord, lSourcePath)
|
||||
CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge)
|
||||
|
||||
mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
|
||||
mySSH.command('cp docker-compose.yml ci-docker-compose.yml', '\$', 5)
|
||||
|
||||
@@ -453,13 +453,11 @@ class HTMLManagement():
|
||||
self.htmlFile.write(' <td bgcolor = "red" >' + str(parameters['warnings']) + '</td>\n')
|
||||
elif (parameters['warnings'] == 0):
|
||||
self.htmlFile.write(' <td bgcolor = "green" >' + str(parameters['warnings']) + '</td>\n')
|
||||
elif ((parameters['warnings'] > 0) and (parameters['warnings'] <= 20)):
|
||||
self.htmlFile.write(' <td bgcolor = "orange" >' + str(parameters['warnings']) + '</td>\n')
|
||||
else:
|
||||
self.htmlFile.write(' <td bgcolor = "red" >' + str(parameters['warnings']) + '</td>\n')
|
||||
self.htmlFile.write(' <td bgcolor = "orange" >' + str(parameters['warnings']) + '</td>\n')
|
||||
if (parameters['errors'] == 0) and (parameters['warnings'] == 0):
|
||||
self.htmlFile.write(' <th colspan=' + str(1+self.htmlUEConnected) + ' bgcolor = "green" ><font color="white">OK </font></th>\n')
|
||||
elif (parameters['errors'] == 0) and ((parameters['warnings'] > 0) and (parameters['warnings'] <= 20)):
|
||||
elif (parameters['errors'] == 0):
|
||||
self.htmlFile.write(' <th colspan=' + str(1+self.htmlUEConnected) + ' bgcolor = "orange" ><font color="white">OK </font></th>\n')
|
||||
else:
|
||||
self.htmlFile.write(' <th colspan=' + str(1+self.htmlUEConnected) + ' bgcolor = "red" > NOT OK </th>\n')
|
||||
|
||||
@@ -115,30 +115,6 @@ class PhySim:
|
||||
mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
|
||||
else:
|
||||
imageTag = "develop"
|
||||
# Check if image is exist on the Red Hat server, before pushing it to OC cluster
|
||||
mySSH.command('sudo podman image inspect --format="Size = {{.Size}} bytes" oai-physim:' + imageTag, '\$', 60)
|
||||
if mySSH.getBefore().count('no such image') != 0:
|
||||
logging.error('\u001B[1m No such image oai-physim\u001B[0m')
|
||||
mySSH.close()
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.PHYSIM_IMAGE_ABSENT)
|
||||
RAN.prematureExit = True
|
||||
return
|
||||
else:
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore())
|
||||
if result is not None:
|
||||
imageSize = float(result.group('size'))
|
||||
imageSize = imageSize / 1000
|
||||
if imageSize < 1000:
|
||||
logging.debug('\u001B[1m oai-physim size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m')
|
||||
else:
|
||||
imageSize = imageSize / 1000
|
||||
if imageSize < 1000:
|
||||
logging.debug('\u001B[1m oai-physim size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m')
|
||||
else:
|
||||
imageSize = imageSize / 1000
|
||||
logging.debug('\u001B[1m oai-physim is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m')
|
||||
else:
|
||||
logging.debug('oai-physim size is unknown')
|
||||
|
||||
# logging to OC Cluster and then switch to corresponding project
|
||||
mySSH.command(f'oc login -u {ocUserName} -p {ocPassword} --server https://api.oai.cs.eurecom.fr:6443', '\$', 30)
|
||||
@@ -161,46 +137,7 @@ class PhySim:
|
||||
else:
|
||||
logging.debug(f'\u001B[1m Now using project {ocProjectName}\u001B[0m')
|
||||
|
||||
# Tag the image and push to the OC cluster
|
||||
mySSH.command('oc whoami -t | sudo podman login -u ' + ocUserName + ' --password-stdin https://default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/ --tls-verify=false', '\$', 30)
|
||||
if mySSH.getBefore().count('Login Succeeded!') == 0:
|
||||
logging.error('\u001B[1m Podman Login to OC Cluster Registry Failed\u001B[0m')
|
||||
mySSH.command('oc logout', '\$', 30)
|
||||
mySSH.close()
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_LOGIN_FAIL)
|
||||
RAN.prematureExit = True
|
||||
return
|
||||
else:
|
||||
logging.debug('\u001B[1m Podman Login to OC Cluster Registry Successfully\u001B[0m')
|
||||
time.sleep(2)
|
||||
mySSH.command('oc create -f openshift/oai-physim-image-stream.yml || true', '\$', 30)
|
||||
if mySSH.getBefore().count('(AlreadyExists):') == 0 and mySSH.getBefore().count('created') == 0:
|
||||
logging.error(f'\u001B[1m Image Stream "oai-physim" Creation Failed on OC Cluster {ocProjectName}\u001B[0m')
|
||||
mySSH.command('sudo podman logout https://default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/', '\$', 6)
|
||||
mySSH.command('oc logout', '\$', 30)
|
||||
mySSH.close()
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_IS_FAIL)
|
||||
RAN.prematureExit = True
|
||||
return
|
||||
else:
|
||||
logging.debug(f'\u001B[1m Image Stream "oai-physim" created on OC project {ocProjectName}\u001B[0m')
|
||||
time.sleep(2)
|
||||
mySSH.command(f'sudo podman tag oai-physim:{imageTag} default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/{self.OCProjectName}/oai-physim:{imageTag}', '\$', 30)
|
||||
time.sleep(2)
|
||||
mySSH.command(f'sudo podman push default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/{self.OCProjectName}/oai-physim:{imageTag} --tls-verify=false', '\$', 180)
|
||||
if mySSH.getBefore().count('Storing signatures') == 0:
|
||||
logging.error('\u001B[1m Image "oai-physim" push to OC Cluster Registry Failed\u001B[0m')
|
||||
mySSH.command('sudo podman logout https://default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/', '\$', 6)
|
||||
mySSH.command('oc logout', '\$', 30)
|
||||
mySSH.close()
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_IS_FAIL)
|
||||
RAN.prematureExit = True
|
||||
return
|
||||
else:
|
||||
logging.debug('\u001B[1m Image "oai-physim" push to OC Cluster Registry Successfully\u001B[0m')
|
||||
|
||||
# Using helm charts deployment
|
||||
time.sleep(5)
|
||||
mySSH.command(f'grep -rl OAICICD_PROJECT ./charts/ | xargs sed -i -e "s#OAICICD_PROJECT#{ocProjectName}#"', '\$', 30)
|
||||
mySSH.command(f'sed -i -e "s#TAG#{imageTag}#g" ./charts/physims/values.yaml', '\$', 6)
|
||||
mySSH.command('helm install physim ./charts/physims/ 2>&1 | tee -a cmake_targets/log/physim_helm_summary.txt', '\$', 30)
|
||||
@@ -213,9 +150,6 @@ class PhySim:
|
||||
mySSH.command('oc get pods -l app.kubernetes.io/instance=physim', '\$', 6, resync=True)
|
||||
if re.search('No resources found', mySSH.getBefore()):
|
||||
isFinished1 = True
|
||||
mySSH.command(f'sudo podman rmi default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/{self.OCProjectName}/oai-physim:{imageTag}', '\$', 30)
|
||||
mySSH.command('oc delete is oai-physim', '\$', 30)
|
||||
mySSH.command('sudo podman logout https://default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/', '\$', 6)
|
||||
mySSH.command('oc logout', '\$', 30)
|
||||
mySSH.close()
|
||||
self.AnalyzeLogFile_phySim(HTML)
|
||||
@@ -244,9 +178,6 @@ class PhySim:
|
||||
mySSH.command('oc get pods -l app.kubernetes.io/instance=physim', '\$', 6, resync=True)
|
||||
if re.search('No resources found', mySSH.getBefore()):
|
||||
isFinished1 = True
|
||||
mySSH.command(f'sudo podman rmi default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/{self.OCProjectName}/oai-physim:{imageTag}', '\$', 6)
|
||||
mySSH.command('oc delete is oai-physim', '\$', 6)
|
||||
mySSH.command('sudo podman logout https://default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/', '\$', 6)
|
||||
mySSH.command('oc logout', '\$', 30)
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_PHYSIM_DEPLOY_FAIL)
|
||||
HTML.CreateHtmlTestRowPhySimTestResult(self.testSummary,self.testResult)
|
||||
@@ -304,10 +235,6 @@ class PhySim:
|
||||
isFinished1 = True
|
||||
if isFinished1 == True:
|
||||
logging.debug('\u001B[1m UnDeployed PhySim Successfully on OC Cluster\u001B[0m')
|
||||
mySSH.command(f'sudo podman rmi default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/{self.OCProjectName}/oai-physim:{imageTag}', '\$', 6)
|
||||
mySSH.command('oc delete is oai-physim', '\$', 6)
|
||||
logging.debug('\u001B[1m Deleted the Image and ImageStream\u001B[0m')
|
||||
mySSH.command('sudo podman logout https://default-route-openshift-image-registry.apps.oai.cs.eurecom.fr/', '\$', 6)
|
||||
mySSH.command('oc logout', '\$', 6)
|
||||
mySSH.close()
|
||||
self.AnalyzeLogFile_phySim(HTML)
|
||||
|
||||
@@ -79,3 +79,7 @@ X2_HO_REQ_STATE__TARGET_RECEIVES_REQ = 1
|
||||
X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE = 2
|
||||
X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ = 3
|
||||
X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK = 10
|
||||
|
||||
# placeholder for a "don't care" password to gradually transition to
|
||||
# passwordless CI
|
||||
CI_NO_PASSWORD = "CIPASSWORDDONTCARE"
|
||||
|
||||
+14
-12
@@ -45,6 +45,7 @@ import cls_containerize #class Containerize for all container-based oper
|
||||
import cls_static_code_analysis #class for static code analysis
|
||||
import cls_ci_ueinfra #class defining the multi Ue infrastrucure
|
||||
import cls_physim1 #class PhySim for physical simulators deploy and run
|
||||
import cls_cluster # class for building/deploying on cluster
|
||||
|
||||
import sshconnection
|
||||
import epc
|
||||
@@ -102,20 +103,17 @@ def AssignParams(params_dict):
|
||||
|
||||
|
||||
def GetParametersFromXML(action):
|
||||
if action == 'Build_eNB' or action == 'Build_Image' or action == 'Build_Proxy':
|
||||
if action == 'Build_eNB' or action == 'Build_Image' or action == 'Build_Proxy' or action == "Build_Cluster_Image":
|
||||
RAN.Build_eNB_args=test.findtext('Build_eNB_args')
|
||||
CONTAINERS.imageKind=test.findtext('kind')
|
||||
forced_workspace_cleanup = test.findtext('forced_workspace_cleanup')
|
||||
if (forced_workspace_cleanup is None):
|
||||
RAN.Build_eNB_forced_workspace_cleanup=False
|
||||
CONTAINERS.forcedWorkspaceCleanup=False
|
||||
else:
|
||||
if re.match('true', forced_workspace_cleanup, re.IGNORECASE):
|
||||
RAN.Build_eNB_forced_workspace_cleanup=True
|
||||
CONTAINERS.forcedWorkspaceCleanup=True
|
||||
else:
|
||||
RAN.Build_eNB_forced_workspace_cleanup=True
|
||||
CONTAINERS.forcedWorkspaceCleanup=False
|
||||
RAN.Build_eNB_forced_workspace_cleanup=False
|
||||
CONTAINERS.forcedWorkspaceCleanup=False
|
||||
CLUSTER.forcedWorkspaceCleanup = False
|
||||
if forced_workspace_cleanup is not None and re.match('true', forced_workspace_cleanup, re.IGNORECASE):
|
||||
RAN.Build_eNB_forced_workspace_cleanup = True
|
||||
CONTAINERS.forcedWorkspaceCleanup = True
|
||||
CLUSTER.forcedWorkspaceCleanup = True
|
||||
eNB_instance=test.findtext('eNB_instance')
|
||||
if (eNB_instance is None):
|
||||
RAN.eNB_instance=0
|
||||
@@ -534,6 +532,7 @@ HTML = cls_oai_html.HTMLManagement()
|
||||
CONTAINERS = cls_containerize.Containerize()
|
||||
SCA = cls_static_code_analysis.StaticCodeAnalysis()
|
||||
PHYSIM = cls_physim1.PhySim()
|
||||
CLUSTER = cls_cluster.Cluster()
|
||||
|
||||
ldpc=cls_physim.PhySim() #create an instance for LDPC test using GPU or CPU build
|
||||
|
||||
@@ -543,7 +542,7 @@ ldpc=cls_physim.PhySim() #create an instance for LDPC test using GPU or CPU b
|
||||
#-----------------------------------------------------------
|
||||
|
||||
import args_parse
|
||||
py_param_file_present, py_params, mode = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM)
|
||||
py_param_file_present, py_params, mode = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,EPC,ldpc,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER)
|
||||
|
||||
|
||||
|
||||
@@ -931,6 +930,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
|
||||
HTML=ldpc.Run_NRulsimTest(HTML,CONST,id)
|
||||
if ldpc.exitStatus==1:
|
||||
RAN.prematureExit = True
|
||||
elif action == 'Build_Cluster_Image':
|
||||
if not CLUSTER.BuildClusterImage(HTML):
|
||||
RAN.prematureExit = True
|
||||
elif action == 'Build_Image':
|
||||
CONTAINERS.BuildImage(HTML)
|
||||
elif action == 'Build_Proxy':
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
- Build_Proxy
|
||||
- Build_Cluster_Image
|
||||
- Build_PhySim
|
||||
- Run_LDPCTest
|
||||
- Run_NRulsimTest
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
<!--
|
||||
|
||||
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The OpenAirInterface Software Alliance licenses this file to You under
|
||||
the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.openairinterface.org/?page_id=698
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
contact@openairinterface.org
|
||||
-->
|
||||
|
||||
<testCaseList>
|
||||
<htmlTabRef>build-tab</htmlTabRef>
|
||||
<htmlTabName>Build Images on Cluster</htmlTabName>
|
||||
<htmlTabIcon>wrench</htmlTabIcon>
|
||||
<TestCaseRequestedList>000001</TestCaseRequestedList>
|
||||
<TestCaseExclusionList></TestCaseExclusionList>
|
||||
|
||||
<testCase id="000001">
|
||||
<class>Build_Cluster_Image</class>
|
||||
<desc>Build Images on OpenShift Cluster</desc>
|
||||
</testCase>
|
||||
|
||||
</testCaseList>
|
||||
Reference in New Issue
Block a user