Merge remote-tracking branch 'origin/phy_simulators_ctest' into integration_2025_w20 (!3379)

Add physim tests into ctest framework

Convert physim testcases listed in
cmake_targets/autotests/test_case_list.xml into ctest. Note: Tests for
dlsim_tm4 are excluded, due to compilation issues.

CTest evaluates success based on return code: '0' = pass, non-zero =
fail. For some simulators, the return code is non-zero even when the
test is successful (as indicated by success messages in the test logs).
For some simulators, the return code is always '0'. This inconsistency
has been address in the first two commits of this MR.

To build and execute these tests:

- using cmake

    cd openairinterface5g
    mkdir build && cd build # you can also do that in cmake_targets/ran_build/build
    cmake .. -GNinja -DENABLE_PHYSIM_TESTS=ON && ninja tests
    ctest

- using build_oai: physim-related ctests are activated when PHY
  simulators are built (use build_oai with '-P' or '--phy_simulators'
  option)

    cd openairinterface5g/cmake_targets
    ./build_oai --ninja --phy_simulators
    cd ran_build/build
    ctest

Link to newly created 4g and 5g RAN-Physim-Cluster pipelines:

- RAN-Physim-Cluster-4G: https://jenkins-oai.eurecom.fr/job/RAN-Physim-Cluster-4G/
- RAN-Physim-Cluster-5G: https://jenkins-oai.eurecom.fr/job/RAN-Physim-Cluster-5G/
This commit is contained in:
Robert Schmidt
2025-05-17 12:05:27 +02:00
126 changed files with 782 additions and 5539 deletions
+28 -5
View File
@@ -246,11 +246,11 @@ pipeline {
stage ("Image Test Processes") {
when { expression {doBuild} }
parallel {
stage ("PhySim-Cluster") {
when { expression {do4Gtest || do5Gtest || do5GUeTest} }
stage ("PhySim-Cluster-5G") {
when { expression {do5Gtest || do5GUeTest} }
steps {
script {
triggerSlaveJob ('RAN-PhySim-Cluster', 'PhySim-Cluster')
triggerSlaveJob ('RAN-PhySim-Cluster-5G', 'PhySim-Cluster-5G')
}
}
post {
@@ -258,13 +258,36 @@ pipeline {
script {
// Using a unique variable name for each test stage to avoid overwriting on a global variable
// due to parallel-time concurrency
physimStatus = finalizeSlaveJob('RAN-PhySim-Cluster')
physim5GStatus = finalizeSlaveJob('RAN-PhySim-Cluster-5G')
}
}
failure {
script {
currentBuild.result = 'FAILURE'
failingStages += physimStatus
failingStages += physim5GStatus
}
}
}
}
stage ("PhySim-Cluster-4G") {
when { expression {do4Gtest} }
steps {
script {
triggerSlaveJob ('RAN-PhySim-Cluster-4G', 'PhySim-Cluster-4G')
}
}
post {
always {
script {
// Using a unique variable name for each test stage to avoid overwriting on a global variable
// due to parallel-time concurrency
physim4GStatus = finalizeSlaveJob('RAN-PhySim-Cluster-4G')
}
}
failure {
script {
currentBuild.result = 'FAILURE'
failingStages += physim4GStatus
}
}
}
+1 -13
View File
@@ -41,7 +41,7 @@ import constants as CONST
#-----------------------------------------------------------
def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER):
py_param_file_present = False
@@ -84,7 +84,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
HTML.ranRepository=matchReg.group(1)
CONTAINERS.ranRepository=matchReg.group(1)
SCA.ranRepository=matchReg.group(1)
PHYSIM.ranRepository=matchReg.group(1)
CLUSTER.ranRepository=matchReg.group(1)
elif re.match(r'^\-\-eNB_AllowMerge=(.+)$|^\-\-ranAllowMerge=(.+)$', myArgv, re.IGNORECASE):
if re.match(r'^\-\-eNB_AllowMerge=(.+)$', myArgv, re.IGNORECASE):
@@ -98,7 +97,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
HTML.ranAllowMerge=True
CONTAINERS.ranAllowMerge=True
SCA.ranAllowMerge=True
PHYSIM.ranAllowMerge=True
CLUSTER.ranAllowMerge=True
elif re.match(r'^\-\-eNBBranch=(.+)$|^\-\-ranBranch=(.+)$', myArgv, re.IGNORECASE):
if re.match(r'^\-\-eNBBranch=(.+)$', myArgv, re.IGNORECASE):
@@ -110,7 +108,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
HTML.ranBranch=matchReg.group(1)
CONTAINERS.ranBranch=matchReg.group(1)
SCA.ranBranch=matchReg.group(1)
PHYSIM.ranBranch=matchReg.group(1)
CLUSTER.ranBranch=matchReg.group(1)
elif re.match(r'^\-\-eNBCommitID=(.*)$|^\-\-ranCommitID=(.*)$', myArgv, re.IGNORECASE):
if re.match(r'^\-\-eNBCommitID=(.*)$', myArgv, re.IGNORECASE):
@@ -122,7 +119,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
HTML.ranCommitID=matchReg.group(1)
CONTAINERS.ranCommitID=matchReg.group(1)
SCA.ranCommitID=matchReg.group(1)
PHYSIM.ranCommitID=matchReg.group(1)
CLUSTER.ranCommitID=matchReg.group(1)
elif re.match(r'^\-\-eNBTargetBranch=(.*)$|^\-\-ranTargetBranch=(.*)$', myArgv, re.IGNORECASE):
if re.match(r'^\-\-eNBTargetBranch=(.*)$', myArgv, re.IGNORECASE):
@@ -134,7 +130,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
HTML.ranTargetBranch=matchReg.group(1)
CONTAINERS.ranTargetBranch=matchReg.group(1)
SCA.ranTargetBranch=matchReg.group(1)
PHYSIM.ranTargetBranch=matchReg.group(1)
CLUSTER.ranTargetBranch=matchReg.group(1)
elif re.match(r'^\-\-eNBIPAddress=(.+)$|^\-\-eNB[1-2]IPAddress=(.+)$', myArgv, re.IGNORECASE):
if re.match(r'^\-\-eNBIPAddress=(.+)$', myArgv, re.IGNORECASE):
@@ -142,7 +137,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
RAN.eNBIPAddress=matchReg.group(1)
CONTAINERS.eNBIPAddress=matchReg.group(1)
SCA.eNBIPAddress=matchReg.group(1)
PHYSIM.eNBIPAddress=matchReg.group(1)
CLUSTER.eNBIPAddress=matchReg.group(1)
elif re.match(r'^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match(r'^\-\-eNB1IPAddress=(.+)$', myArgv, re.IGNORECASE)
@@ -158,7 +152,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
RAN.eNBUserName=matchReg.group(1)
CONTAINERS.eNBUserName=matchReg.group(1)
SCA.eNBUserName=matchReg.group(1)
PHYSIM.eNBUserName=matchReg.group(1)
CLUSTER.eNBUserName=matchReg.group(1)
elif re.match(r'^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match(r'^\-\-eNB1UserName=(.+)$', myArgv, re.IGNORECASE)
@@ -174,7 +167,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
RAN.eNBPassword=matchReg.group(1)
CONTAINERS.eNBPassword=matchReg.group(1)
SCA.eNBPassword=matchReg.group(1)
PHYSIM.eNBPassword=matchReg.group(1)
CLUSTER.eNBPassword=matchReg.group(1)
elif re.match(r'^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match(r'^\-\-eNB1Password=(.+)$', myArgv, re.IGNORECASE)
@@ -190,7 +182,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
RAN.eNBSourceCodePath=matchReg.group(1)
CONTAINERS.eNBSourceCodePath=matchReg.group(1)
SCA.eNBSourceCodePath=matchReg.group(1)
PHYSIM.eNBSourceCodePath=matchReg.group(1)
CLUSTER.eNBSourceCodePath=matchReg.group(1)
elif re.match(r'^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match(r'^\-\-eNB1SourceCodePath=(.+)$', myArgv, re.IGNORECASE)
@@ -236,15 +227,12 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER):
CiTestObj.finalStatus = True
elif re.match(r'^\-\-OCUserName=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match(r'^\-\-OCUserName=(.+)$', myArgv, re.IGNORECASE)
PHYSIM.OCUserName = matchReg.group(1)
CLUSTER.OCUserName = matchReg.group(1)
elif re.match(r'^\-\-OCPassword=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match(r'^\-\-OCPassword=(.+)$', myArgv, re.IGNORECASE)
PHYSIM.OCPassword = matchReg.group(1)
CLUSTER.OCPassword = matchReg.group(1)
elif re.match(r'^\-\-OCProjectName=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match(r'^\-\-OCProjectName=(.+)$', myArgv, re.IGNORECASE)
PHYSIM.OCProjectName = matchReg.group(1)
CLUSTER.OCProjectName = matchReg.group(1)
elif re.match(r'^\-\-OCUrl=(.+)$', myArgv, re.IGNORECASE):
matchReg = re.match(r'^\-\-OCUrl=(.+)$', myArgv, re.IGNORECASE)
+66 -1
View File
@@ -20,8 +20,11 @@
# */
#---------------------------------------------------------------------
import logging
import re
import os
import xml.etree.ElementTree as ET
import json
# Define the mapping of physim_test values to search patterns
PHYSIM_PATTERN_MAPPING = {
@@ -80,3 +83,65 @@ class Analysis():
msg = f'{time1_match.group(1)}: {time1_match.group(2)} us\n{time2_match.group(1)}: {time2_match.group(2)} us exceeds a limit of {threshold} us'
return success,msg
def analyze_oc_physim(result_junit, details_json):
try:
tree = ET.parse(result_junit)
root = tree.getroot()
nb_tests = int(root.attrib["tests"])
nb_failed = int(root.attrib["failures"])
except ET.ParseError as e:
return False, False, f'Could not parse XML log file {result_junit}: {e}'
except FileNotFoundError as e:
return False, False, f'JUnit XML log file {result_junit} not found: {e}'
except Exception as e:
return False, False, f'While parsing JUnit XML log file: exception: {e}'
try:
with open(details_json) as f:
j = json.load(f)
# prepare JSON for easier access of strings
json_test_desc = {}
for e in j["tests"]:
json_test_desc[e["name"]] = e
except json.JSONDecodeError as e:
return False, False, f'Could not decode JSON log file {details_json}: {e}'
except FileNotFoundError as e:
return False, False, f'Physim JSON log file {details_json} not found: {e}'
except Exception as e:
return False, False, f'While parsing physim JSON log file: exception: {e}'
test_result = {}
for test in root: # for each test
test_name = test.attrib["name"]
test_exec = json_test_desc[test_name]["properties"][1]["value"][0]
desc = json_test_desc[test_name]["properties"][1]["value"][1]
# get runtime and checks
test_check = test.attrib["status"] == "run"
time = round(float(test.attrib["time"]), 1)
time_check = time < 150
output = test.findtext("system-out")
output_check = "exceeds the threshold" not in output
# collect logs
log_dir = f'../cmake_targets/log/{test_exec}'
os.makedirs(log_dir, exist_ok=True)
with open(f'{log_dir}/{test_name}.log', 'w') as f:
f.write(output)
# prepare result and info
info = f"Runtime: {f'{time:.3f}'[:5]} s"
resultstr = 'PASS' if (test_check and time_check and output_check) else 'FAIL'
if test_check:
if not output_check:
info += " Test log exceeds maximal allowed length 100 kB"
if not time_check:
info += " Test exceeds 150s"
if not (time_check and output_check):
nb_failed += 1 # time threshold/output length error, not counted for by ctest as of now
test_result[test_name] = [desc, info, resultstr]
test_summary = {}
test_summary['Nbtests'] = nb_tests
test_summary['Nbpass'] = nb_tests - nb_failed
test_summary['Nbfail'] = nb_failed
return nb_failed == 0, test_summary, test_result
+37
View File
@@ -31,8 +31,10 @@
import logging
import re
import time
import os
import cls_oai_html
import cls_analysis
import constants as CONST
import helpreadme as HELP
import cls_containerize
@@ -457,3 +459,38 @@ class Cluster:
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize)
return status
def deploy_oc_physim(self, HTML, oc_release, svr_id):
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
HELP.GenericHelp(CONST.Version)
raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {self.ranBranch} ranCommitID {self.ranCommitID}')
image_tag = cls_containerize.CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
logging.debug(f'Running physims from server: {svr_id}')
script = "scripts/oc-deploy-physims.sh"
options = f"oaicicd-core-for-ci-ran {oc_release} {image_tag} {self.eNBSourceCodePath}"
ret = cls_cmd.runScript(svr_id, script, 600, options)
logging.debug(f'"{script}" finished with code {ret.returncode}, output:\n{ret.stdout}')
log_dir = f'{os.getcwd()}/../cmake_targets/log'
os.makedirs(log_dir, exist_ok=True)
result_junit = f'{oc_release}-run.xml'
details_json = f'{oc_release}-tests.json'
with cls_cmd.getConnection(svr_id) as ssh:
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{details_json}', tgt=f'{log_dir}/{details_json}')
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{result_junit}', tgt=f'{log_dir}/{result_junit}')
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt', tgt=f'{log_dir}/physim_log.txt')
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt', tgt=f'{log_dir}/physim_pods_summary.txt')
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log', tgt=f'{log_dir}/LastTestsFailed.log')
test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(f'{log_dir}/{result_junit}', f'{log_dir}/{details_json}')
if test_summary:
if test_status:
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
HTML.CreateHtmlTestRowPhySimTestResult(test_summary, test_result)
logging.info('\u001B[1m Physical Simulator Pass\u001B[0m')
else:
HTML.CreateHtmlTestRow('Some test(s) failed!', 'KO', CONST.OC_PHYSIM_DEPLOY_FAIL)
HTML.CreateHtmlTestRowPhySimTestResult(test_summary, test_result)
logging.error('\u001B[1m Physical Simulator Fail\u001B[0m')
else:
HTML.CreateHtmlTestRowQueue('Physical simulator failed', 'KO', [test_result])
logging.error('\u001B[1m Physical Simulator Fail\u001B[0m')
return test_status
+12 -10
View File
@@ -564,13 +564,13 @@ class HTMLManagement():
self.htmlFile.write(' <th>Nb Pass</th>\n')
self.htmlFile.write(' </tr>\n')
self.htmlFile.write(' <tr>\n')
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" > physim_test.txt </td>\n')
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" > physim_log.txt </td>\n')
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" >' + str(testSummary['Nbtests']) + ' </td>\n')
if testSummary['Nbfail'] == 0:
self.htmlFile.write(' <td bgcolor = "lightcyan" >' + str(testSummary['Nbfail']) + ' </td>\n')
self.htmlFile.write(' <td bgcolor = "lightcyan" >' + str(testSummary['Nbfail']) + '</td>\n')
else:
self.htmlFile.write(' <td bgcolor = "red" >' + str(testSummary['Nbfail']) + ' </td>\n')
self.htmlFile.write(' <td gcolor = "lightcyan" >' + str(testSummary['Nbpass']) + ' </td>\n')
self.htmlFile.write(' <td bgcolor = "red" ><font color="white">' + str(testSummary['Nbfail']) + '</font></td>\n')
self.htmlFile.write(' <td bgcolor = "lightcyan" >' + str(testSummary['Nbpass']) + ' </td>\n')
self.htmlFile.write(' </tr>\n')
self.htmlFile.write(' <tr bgcolor = "#F0F0F0" >\n')
self.htmlFile.write(' <td colspan="6"><b> ---- PHYSIM TEST DETAIL INFO---- </b></td>\n')
@@ -578,22 +578,24 @@ class HTMLManagement():
self.htmlFile.write(' <tr bgcolor = "#33CCFF" >\n')
self.htmlFile.write(' <th colspan="2">Test Name</th>\n')
self.htmlFile.write(' <th colspan="2">Test Description</th>\n')
self.htmlFile.write(' <th colspan="2">Result</th>\n')
self.htmlFile.write(' <th>Test Status</th>\n')
self.htmlFile.write(' <th>Info</th>\n')
self.htmlFile.write(' </tr>\n')
y = ''
for key, value in testResult.items():
x = key.split(".")
if x[0] != y:
if x[2] != y:
self.htmlFile.write(' <tr bgcolor = "lightgreen" >\n')
self.htmlFile.write(' <td style="text-align: center;" colspan="6"><b>"' + x[0] + '" series </b></td>\n')
self.htmlFile.write(' <td style="text-align: center;" colspan="6"><b>"' + x[2] + '" series </b></td>\n')
self.htmlFile.write(' </tr>\n')
y = x[0]
y = x[2]
self.htmlFile.write(' <tr>\n')
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" >' + key + ' </td>\n')
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" >' + value[0] + '</td>\n')
if 'PASS' in value:
self.htmlFile.write(' <td colspan="2" bgcolor = "green" >' + value[1] + '</td>\n')
self.htmlFile.write(' <td bgcolor = "green" ><font color="white"><b>' + value[2] + '</b></font></td>\n')
else:
self.htmlFile.write(' <td colspan="2" bgcolor = "red" >' + value[1] + '</td>\n')
self.htmlFile.write(' <td bgcolor = "red" ><font color="white"><b>' + value[2] + '</b></font></td>\n')
self.htmlFile.write(' <td bgcolor = "lightcyan">' + value[1] + '</td>\n')
self.htmlFile.close()
-286
View File
@@ -1,286 +0,0 @@
#/*
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
# * contributor license agreements. See the NOTICE file distributed with
# * this work for additional information regarding copyright ownership.
# * The OpenAirInterface Software Alliance licenses this file to You under
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
# * except in compliance with the License.
# * You may obtain a copy of the License at
# *
# * http://www.openairinterface.org/?page_id=698
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# *-------------------------------------------------------------------------------
# * For more information about the OpenAirInterface (OAI) Software Alliance:
# * contact@openairinterface.org
# */
#---------------------------------------------------------------------
# Python for CI of OAI-eNB + COTS-UE
#
# Required Python Version
# Python 3.x
#
# Required Python Package
# pexpect
#---------------------------------------------------------------------
#-----------------------------------------------------------
# Import
#-----------------------------------------------------------
import logging
import os
import re
import time
import subprocess
import sys
import sshconnection as SSH
import cls_oai_html
import constants as CONST
import helpreadme as HELP
class PhySim:
def __init__(self):
self.eNBIpAddr = ""
self.eNBUserName = ""
self.eNBPassword = ""
self.OCUserName = ""
self.OCPassword = ""
self.OCProjectName = ""
self.eNBSourceCodePath = ""
self.ranRepository = ""
self.ranBranch = ""
self.ranCommitID= ""
self.ranAllowMerge= False
self.ranTargetBranch= ""
self.testResult = {}
self.testSummary = {}
self.testStatus = False
#-----------------$
#PUBLIC Methods$
#-----------------$
def Deploy_PhySim(self, HTML):
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
lIpAddr = self.eNBIPAddress
lUserName = self.eNBUserName
lPassWord = self.eNBPassword
lSourcePath = self.eNBSourceCodePath
ocUserName = self.OCUserName
ocPassword = self.OCPassword
ocProjectName = self.OCProjectName
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '' or ocUserName == '' or ocPassword == '' or ocProjectName == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
logging.debug('Building on server: ' + lIpAddr)
mySSH = SSH.SSHConnection()
mySSH.open(lIpAddr, lUserName, lPassWord)
self.testCase_id = HTML.testCase_id
# on RedHat/CentOS .git extension is mandatory
result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
if result is not None:
full_ran_repo_name = self.ranRepository.replace('git/', 'git')
else:
full_ran_repo_name = self.ranRepository + '.git'
mySSH.command('rm -Rf ' + lSourcePath, '\$', 30)
mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
mySSH.command('cd ' + lSourcePath, '\$', 5)
mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
# Raphael: here add a check if git clone or git fetch went smoothly
mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
mySSH.command('git clean -x -d -ff', '\$', 30)
mySSH.command('mkdir -p cmake_targets/log', '\$', 5)
# if the commit ID is provided use it to point to it
if self.ranCommitID != '':
mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
if self.ranAllowMerge:
branchName = self.ranBranch.replace('/','-')
imageTag = f'{branchName}-{self.ranCommitID[0:8]}'
if self.ranTargetBranch == '':
if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30)
else:
logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 30)
else:
imageTag = f'develop-{self.ranCommitID[0:8]}'
# logging to OC Cluster and then switch to corresponding project
mySSH.command(f'oc login -u {ocUserName} -p {ocPassword} --server https://api.oai.cs.eurecom.fr:6443', '\$', 30)
if mySSH.getBefore().count('Login successful.') == 0:
logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m')
mySSH.close()
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_LOGIN_FAIL)
return False
else:
logging.debug('\u001B[1m Login to OC Cluster Successfully\u001B[0m')
mySSH.command(f'oc project {ocProjectName}', '\$', 30)
if mySSH.getBefore().count(f'Already on project "{ocProjectName}"') == 0 and mySSH.getBefore().count(f'Now using project "{self.OCProjectName}"') == 0:
logging.error(f'\u001B[1m Unable to access OC project {ocProjectName}\u001B[0m')
mySSH.command('oc logout', '\$', 30)
mySSH.close()
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_PROJECT_FAIL)
return False
else:
logging.debug(f'\u001B[1m Now using project {ocProjectName}\u001B[0m')
# Using helm charts deployment
mySSH.command(f'helm install physim ./charts/physims/ --set global.image.version={imageTag} --wait 2>&1 | tee -a cmake_targets/log/physim_helm_summary.txt', '\$', 30)
if mySSH.getBefore().count('STATUS: deployed') == 0:
logging.error('\u001B[1m Deploying PhySim Failed using helm chart on OC Cluster\u001B[0m')
mySSH.command('helm uninstall physim | tee -a cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 30)
isFinished1 = False
while(isFinished1 == False):
time.sleep(20)
mySSH.command('oc get pods -l app=physim', '\$', 6, resync=True)
if re.search('No resources found', mySSH.getBefore()):
isFinished1 = True
mySSH.command('oc logout', '\$', 30)
mySSH.close()
self.AnalyzeLogFile_phySim()
return False
else:
logging.debug('\u001B[1m Deployed PhySim Successfully using helm chart\u001B[0m')
isRunning = False
count = 0
# Check whether the containers are in Running state or not under 2 mins
while(count < 5 and isRunning == False):
time.sleep(60)
mySSH.command('oc get pods -o wide -l app=physim | tee -a cmake_targets/log/physim_pods_summary.txt', '\$', 30, resync=True)
running_count = mySSH.getBefore().count('Running')
completed_count = mySSH.getBefore().count('Completed')
if (running_count + completed_count) == 23:
logging.debug('\u001B[1m Running the physim test Scenarios\u001B[0m')
isRunning = True
podNames = re.findall('oai-[\S\d\w]+', mySSH.getBefore())
count +=1
if isRunning == False:
logging.error('\u001B[1m Some pods are in Error state \u001B[0m')
mySSH.command('oc get pods -l app=physim 2>&1 | tee -a cmake_targets/log/physim_pods_summary.txt', '\$', 6)
mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc describe pod $pod >> cmake_targets/log/physim_pods_summary.txt; done', '\$', 10)
mySSH.command('helm uninstall physim | tee -a cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6)
self.AnalyzeLogFile_phySim()
isFinished1 = False
while(isFinished1 == False):
time.sleep(20)
mySSH.command('oc get pods -l app=physim', '\$', 6, resync=True)
if re.search('No resources found', mySSH.getBefore()):
isFinished1 = True
mySSH.command('oc logout', '\$', 30)
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.OC_PHYSIM_DEPLOY_FAIL)
HTML.CreateHtmlTestRowPhySimTestResult(self.testSummary,self.testResult)
return False
# Waiting to complete the running test
count = 0
isFinished = False
# doing a deep copy!
tmpPodNames = podNames.copy()
while(count < 9 and isFinished == False):
time.sleep(60)
for podName in tmpPodNames:
mySSH.command2(f'oc logs --tail=1 {podName} 2>&1', 6, silent=True)
if mySSH.cmd2Results.count('FINISHED') != 0:
logging.debug(podName + ' is finished')
tmpPodNames.remove(podName)
if not tmpPodNames:
isFinished = True
count += 1
if isFinished:
logging.debug('\u001B[1m PhySim test is Complete\u001B[0m')
else:
logging.error('\u001B[1m PhySim test Timed-out!\u001B[0m')
# Getting the logs of each executables running in individual pods
for podName in podNames:
mySSH.command(f'oc logs {podName} >> cmake_targets/log/physim_test.txt 2>&1', '\$', 15, resync=True)
mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc describe pod $pod >> cmake_targets/log/physim_pods_summary.txt; done', '\$', 10)
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/log/physim_test.txt', '.')
try:
listLogFiles = subprocess.check_output('grep -E --colour=never "Execution Log file|Linux oai-" physim_test.txt', shell=True, universal_newlines=True)
for line in listLogFiles.split('\n'):
res1 = re.search('Linux (?P<pod>oai-[a-zA-Z0-9\-]+) ', str(line))
res2 = re.search('Execution Log file = (?P<name>[a-zA-Z0-9\-\/\.\_\+]+)', str(line))
if res1 is not None:
podName = res1.group('pod')
if res2 is not None:
logFileInPod = res2.group('name')
folderName = logFileInPod.replace('/opt/oai-physim/cmake_targets/autotests/log/', '')
folderName = re.sub('/test.*', '', folderName)
fileName = logFileInPod.replace('/opt/oai-physim/cmake_targets/autotests/log/' + folderName + '/', '')
mySSH.command('mkdir -p cmake_targets/log/' + folderName, '\$', 5, silent=True)
mySSH.command('oc cp ' + podName + ':' + logFileInPod + ' cmake_targets/log/' + folderName + '/' + fileName, '\$', 20, silent=True)
except Exception as e:
pass
# UnDeploy the physical simulator pods
mySSH.command('helm uninstall physim | tee -a cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6)
isFinished1 = False
while(isFinished1 == False):
time.sleep(20)
mySSH.command('oc get pods -l app=physim', '\$', 6, resync=True)
if re.search('No resources found', mySSH.getBefore()):
isFinished1 = True
if isFinished1 == True:
logging.debug('\u001B[1m UnDeployed PhySim Successfully on OC Cluster\u001B[0m')
mySSH.command('oc logout', '\$', 6)
mySSH.close()
self.AnalyzeLogFile_phySim()
if self.testStatus and isFinished:
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
HTML.CreateHtmlTestRowPhySimTestResult(self.testSummary,self.testResult)
logging.info('\u001B[1m Physical Simulator Pass\u001B[0m')
else:
if isFinished:
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK)
else:
HTML.CreateHtmlTestRow('Some test(s) timed-out!', 'KO', CONST.ALL_PROCESSES_OK)
HTML.CreateHtmlTestRowPhySimTestResult(self.testSummary,self.testResult)
logging.error('\u001B[1m Physical Simulator Fail\u001B[0m')
return self.testStatus
def AnalyzeLogFile_phySim(self):
mySSH = SSH.SSHConnection()
mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
dirToCopy = f'{self.eNBSourceCodePath}/cmake_targets/log/'
mySSH.copyin(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, dirToCopy, f'./physim_test_logs_{self.testCase_id}/')
mySSH.command(f'rm -rf {dirToCopy}', '\$', 5)
mySSH.close()
# physim test log analysis
nextt = 0
nbTests = 0
nbFailed = 0
if (os.path.isfile(f'./physim_test_logs_{self.testCase_id}/physim_test.txt')):
with open(f'./physim_test_logs_{self.testCase_id}/physim_test.txt', 'r') as logfile:
for line in logfile:
# the following regex would match the following line:
# execution nr_pbchsim.106rb.test1 {Test1: PBCH-only, 106 PRB} Run_Result = " Run_1 =PASS Run_2 =PASS Run_3 =PASS" Result = PASS
# ^testname ^testdescription ^test1 ^test2 ^test3 ^status
ret = re.search('execution\s+(?P<name>[a-zA-Z0-9\._\-\+]+)\s+{(?P<desc>[A-Za-z0-9\.\_\-\+:,;\/\%\=\(\)\s]+)}\s+Run_Result\s*=\s*\"\s*Run_1\s*=\s*(?P<run1>[A-Za-z]+)\s*Run_2\s*=\s*(?P<run2>[A-Za-z]+)\s*Run_3\s*=\s*(?P<run3>[A-Za-z]+)\s*\"\s+Result\s*=\s*(?P<status>[A-Za-z]+)', line)
if ret is None:
continue
nbTests += 1
r = [ret.group('run1'), ret.group('run2'), ret.group('run3')]
nbPass = sum([x == 'PASS' for x in r])
resultstr = 'PASS'
if nbPass < 3 or ret.group('status') != 'PASS':
resultstr = f'FAIL ({3-nbPass}/3)'
nbFailed += 1
self.testResult[ret.group('name')] = [ret.group('desc'), resultstr]
self.testSummary['Nbtests'] = nbTests
self.testSummary['Nbpass'] = nbTests - nbFailed
self.testSummary['Nbfail'] = nbFailed
if self.testSummary['Nbfail'] == 0:
self.testStatus = True
return 0
+4 -5
View File
@@ -41,7 +41,6 @@ import constants as CONST
import cls_oaicitest #main class for OAI CI test framework
import cls_containerize #class Containerize for all container-based operations on RAN/UE objects
import cls_static_code_analysis #class for static code analysis
import cls_physim1 #class PhySim for physical simulators deploy and run
import cls_cluster # class for building/deploying on cluster
import cls_native # class for all native/source-based operations
@@ -97,7 +96,6 @@ def ExecuteActionWithParam(action):
global HTML
global CONTAINERS
global SCA
global PHYSIM
global CLUSTER
if action == 'Build_eNB' or action == 'Build_Image' or action == 'Build_Proxy' or action == "Build_Cluster_Image" or action == "Build_Run_Tests":
RAN.Build_eNB_args=test.findtext('Build_eNB_args')
@@ -295,7 +293,9 @@ def ExecuteActionWithParam(action):
success = cls_oaicitest.IdleSleep(HTML, int(st))
elif action == 'Deploy_Run_PhySim':
success = PHYSIM.Deploy_PhySim(HTML)
oc_release = test.findtext('oc_release')
svr_id = test.findtext('svr_id') or None
success = CLUSTER.deploy_oc_physim(HTML, oc_release, svr_id)
elif action == 'DeployCoreNetwork' or action == 'UndeployCoreNetwork':
cn_id = test.findtext('cn_id')
@@ -442,7 +442,6 @@ RAN = ran.RANManagement()
HTML = cls_oai_html.HTMLManagement()
CONTAINERS = cls_containerize.Containerize()
SCA = cls_static_code_analysis.StaticCodeAnalysis()
PHYSIM = cls_physim1.PhySim()
CLUSTER = cls_cluster.Cluster()
#-----------------------------------------------------------
@@ -452,7 +451,7 @@ CLUSTER = cls_cluster.Cluster()
import args_parse
# Force local execution, move all execution targets to localhost
force_local = False
py_param_file_present, py_params, mode, force_local = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,PHYSIM,CLUSTER)
py_param_file_present, py_params, mode, force_local = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER)
+33
View File
@@ -0,0 +1,33 @@
#!/bin/bash
set -e
function die() { echo $@; exit 1; }
[ $# -eq 4 ] || die "usage: $0 <namespace> <release> <image tag> <oai directory>"
OC_NS=${1}
OC_RELEASE=${2}
IMG_TAG=${3}
OAI_DIR=${4}
cat /opt/oc-password | oc login -u oaicicd --server https://api.oai.cs.eurecom.fr:6443 > /dev/null
oc project ${OC_NS} > /dev/null
oc tag oaicicd-ran/oai-physim:${IMG_TAG} ${OC_NS}/oai-physim:${IMG_TAG}
helm install ${OC_RELEASE} ${OAI_DIR}/charts/${OC_RELEASE} --set global.image.version=${IMG_TAG} --wait
POD_ID=$(oc get pods | grep oai-${OC_RELEASE} | awk '{print $1}')
sleep 10
echo "Monitoring logs for 'FINISHED' in pod '$POD_ID'"
oc logs -f -n ${OC_NS} "$POD_ID" | while read -r line; do
if [[ "$line" == *"FINISHED"* ]]; then
echo "'FINISHED' detected in logs. Copying logs..."
oc logs -n ${OC_NS} "$POD_ID" >> ${OAI_DIR}/ci-scripts/physim_log.txt
oc describe pod $POD_ID >> ${OAI_DIR}/ci-scripts/physim_pods_summary.txt
oc cp "$POD_ID":/opt/oai-physim/Testing/Temporary/LastTestsFailed.log ${OAI_DIR}/ci-scripts/LastTestsFailed.log
oc cp "$POD_ID":/opt/oai-physim/${OC_RELEASE}-tests.json ${OAI_DIR}/ci-scripts/${OC_RELEASE}-tests.json
oc cp "$POD_ID":/opt/oai-physim/${OC_RELEASE}-run.xml ${OAI_DIR}/ci-scripts/${OC_RELEASE}-run.xml
break
fi
done
helm uninstall ${OC_RELEASE} --wait
oc delete istag oai-physim:${IMG_TAG} -n ${OC_NS}
oc logout > /dev/null
@@ -25,13 +25,23 @@
<htmlTabName>Physical Simulators Check</htmlTabName>
<htmlTabIcon>wrench</htmlTabIcon>
<TestCaseRequestedList>
010201
800813
010201
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="010201">
<testCase id="800813">
<class>Create_Workspace</class>
<desc>Create new Workspace</desc>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
</testCase>
<testCase id="010201">
<class>Deploy_Run_PhySim</class>
<desc>Deploy and run physical simulator on openshift</desc>
<desc>Deploy and run physical simulator on openshift</desc>
<oc_release>physims-4g</oc_release>
<svr_id>avra</svr_id>
</testCase>
</testCaseList>
@@ -0,0 +1,47 @@
<!--
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The OpenAirInterface Software Alliance licenses this file to You under
the OAI Public License, Version 1.1 (the "License"); you may not use this file
except in compliance with the License.
You may obtain a copy of the License at
http://www.openairinterface.org/?page_id=698
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
For more information about the OpenAirInterface (OAI) Software Alliance:
contact@openairinterface.org
-->
<testCaseList>
<htmlTabRef>build-tab</htmlTabRef>
<htmlTabName>Physical Simulators Check</htmlTabName>
<htmlTabIcon>wrench</htmlTabIcon>
<TestCaseRequestedList>
800813
010201
</TestCaseRequestedList>
<TestCaseExclusionList></TestCaseExclusionList>
<testCase id="800813">
<class>Create_Workspace</class>
<desc>Create new Workspace</desc>
<eNB_instance>0</eNB_instance>
<eNB_serverId>0</eNB_serverId>
</testCase>
<testCase id="010201">
<class>Deploy_Run_PhySim</class>
<desc>Deploy and run physical simulator on openshift</desc>
<oc_release>physims-5g</oc_release>
<svr_id>avra</svr_id>
</testCase>
</testCaseList>