CI: Add functions for OC Physim deployment and analysis
This commit is contained in:
Jaroslava Fiedlerova
@@ -20,8 +20,11 @@
|
||||
# */
|
||||
#---------------------------------------------------------------------
|
||||
|
||||
import logging
|
||||
import re
|
||||
import os
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
import json
|
||||
|
||||
# Define the mapping of physim_test values to search patterns
|
||||
PHYSIM_PATTERN_MAPPING = {
|
||||
@@ -80,3 +83,65 @@ class Analysis():
|
||||
msg = f'{time1_match.group(1)}: {time1_match.group(2)} us\n{time2_match.group(1)}: {time2_match.group(2)} us exceeds a limit of {threshold} us'
|
||||
|
||||
return success,msg
|
||||
|
||||
def analyze_oc_physim(result_junit, details_json):
|
||||
try:
|
||||
tree = ET.parse(result_junit)
|
||||
root = tree.getroot()
|
||||
nb_tests = int(root.attrib["tests"])
|
||||
nb_failed = int(root.attrib["failures"])
|
||||
except ET.ParseError as e:
|
||||
return False, False, f'Could not parse XML log file {result_junit}: {e}'
|
||||
except FileNotFoundError as e:
|
||||
return False, False, f'JUnit XML log file {result_junit} not found: {e}'
|
||||
except Exception as e:
|
||||
return False, False, f'While parsing JUnit XML log file: exception: {e}'
|
||||
|
||||
try:
|
||||
with open(details_json) as f:
|
||||
j = json.load(f)
|
||||
# prepare JSON for easier access of strings
|
||||
json_test_desc = {}
|
||||
for e in j["tests"]:
|
||||
json_test_desc[e["name"]] = e
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
return False, False, f'Could not decode JSON log file {details_json}: {e}'
|
||||
except FileNotFoundError as e:
|
||||
return False, False, f'Physim JSON log file {details_json} not found: {e}'
|
||||
except Exception as e:
|
||||
return False, False, f'While parsing physim JSON log file: exception: {e}'
|
||||
|
||||
test_result = {}
|
||||
for test in root: # for each test
|
||||
test_name = test.attrib["name"]
|
||||
test_exec = json_test_desc[test_name]["properties"][1]["value"][0]
|
||||
desc = json_test_desc[test_name]["properties"][1]["value"][1]
|
||||
# get runtime and checks
|
||||
test_check = test.attrib["status"] == "run"
|
||||
time = round(float(test.attrib["time"]), 1)
|
||||
time_check = time < 150
|
||||
output = test.findtext("system-out")
|
||||
output_check = "exceeds the threshold" not in output
|
||||
# collect logs
|
||||
log_dir = f'../cmake_targets/log/{test_exec}'
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
with open(f'{log_dir}/{test_name}.log', 'w') as f:
|
||||
f.write(output)
|
||||
# prepare result and info
|
||||
info = f"Runtime: {f'{time:.3f}'[:5]} s"
|
||||
resultstr = 'PASS' if (test_check and time_check and output_check) else 'FAIL'
|
||||
if test_check:
|
||||
if not output_check:
|
||||
info += " Test log exceeds maximal allowed length 100 kB"
|
||||
if not time_check:
|
||||
info += " Test exceeds 150s"
|
||||
if not (time_check and output_check):
|
||||
nb_failed += 1 # time threshold/output length error, not counted for by ctest as of now
|
||||
test_result[test_name] = [desc, info, resultstr]
|
||||
|
||||
test_summary = {}
|
||||
test_summary['Nbtests'] = nb_tests
|
||||
test_summary['Nbpass'] = nb_tests - nb_failed
|
||||
test_summary['Nbfail'] = nb_failed
|
||||
return nb_failed == 0, test_summary, test_result
|
||||
|
||||
@@ -31,8 +31,10 @@
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import os
|
||||
|
||||
import cls_oai_html
|
||||
import cls_analysis
|
||||
import constants as CONST
|
||||
import helpreadme as HELP
|
||||
import cls_containerize
|
||||
@@ -457,3 +459,38 @@ class Cluster:
|
||||
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize)
|
||||
|
||||
return status
|
||||
|
||||
def deploy_oc_physim(self, HTML, oc_release, svr_id):
|
||||
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {self.ranBranch} ranCommitID {self.ranCommitID}')
|
||||
image_tag = cls_containerize.CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge)
|
||||
logging.debug(f'Running physims from server: {svr_id}')
|
||||
script = "scripts/oc-deploy-physims.sh"
|
||||
options = f"oaicicd-core-for-ci-ran {oc_release} {image_tag} {self.eNBSourceCodePath}"
|
||||
ret = cls_cmd.runScript(svr_id, script, 600, options)
|
||||
logging.debug(f'"{script}" finished with code {ret.returncode}, output:\n{ret.stdout}')
|
||||
log_dir = f'{os.getcwd()}/../cmake_targets/log'
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
result_junit = f'{oc_release}-run.xml'
|
||||
details_json = f'{oc_release}-tests.json'
|
||||
with cls_cmd.getConnection(svr_id) as ssh:
|
||||
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{details_json}', tgt=f'{log_dir}/{details_json}')
|
||||
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{result_junit}', tgt=f'{log_dir}/{result_junit}')
|
||||
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt', tgt=f'{log_dir}/physim_log.txt')
|
||||
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt', tgt=f'{log_dir}/physim_pods_summary.txt')
|
||||
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log', tgt=f'{log_dir}/LastTestsFailed.log')
|
||||
test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(f'{log_dir}/{result_junit}', f'{log_dir}/{details_json}')
|
||||
if test_summary:
|
||||
if test_status:
|
||||
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
|
||||
HTML.CreateHtmlTestRowPhySimTestResult(test_summary, test_result)
|
||||
logging.info('\u001B[1m Physical Simulator Pass\u001B[0m')
|
||||
else:
|
||||
HTML.CreateHtmlTestRow('Some test(s) failed!', 'KO', CONST.OC_PHYSIM_DEPLOY_FAIL)
|
||||
HTML.CreateHtmlTestRowPhySimTestResult(test_summary, test_result)
|
||||
logging.error('\u001B[1m Physical Simulator Fail\u001B[0m')
|
||||
else:
|
||||
HTML.CreateHtmlTestRowQueue('Physical simulator failed', 'KO', [test_result])
|
||||
logging.error('\u001B[1m Physical Simulator Fail\u001B[0m')
|
||||
return test_status
|
||||
|
||||
+12
-10
@@ -564,13 +564,13 @@ class HTMLManagement():
|
||||
self.htmlFile.write(' <th>Nb Pass</th>\n')
|
||||
self.htmlFile.write(' </tr>\n')
|
||||
self.htmlFile.write(' <tr>\n')
|
||||
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" > physim_test.txt </td>\n')
|
||||
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" > physim_log.txt </td>\n')
|
||||
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" >' + str(testSummary['Nbtests']) + ' </td>\n')
|
||||
if testSummary['Nbfail'] == 0:
|
||||
self.htmlFile.write(' <td bgcolor = "lightcyan" >' + str(testSummary['Nbfail']) + ' </td>\n')
|
||||
self.htmlFile.write(' <td bgcolor = "lightcyan" >' + str(testSummary['Nbfail']) + '</td>\n')
|
||||
else:
|
||||
self.htmlFile.write(' <td bgcolor = "red" >' + str(testSummary['Nbfail']) + ' </td>\n')
|
||||
self.htmlFile.write(' <td gcolor = "lightcyan" >' + str(testSummary['Nbpass']) + ' </td>\n')
|
||||
self.htmlFile.write(' <td bgcolor = "red" ><font color="white">' + str(testSummary['Nbfail']) + '</font></td>\n')
|
||||
self.htmlFile.write(' <td bgcolor = "lightcyan" >' + str(testSummary['Nbpass']) + ' </td>\n')
|
||||
self.htmlFile.write(' </tr>\n')
|
||||
self.htmlFile.write(' <tr bgcolor = "#F0F0F0" >\n')
|
||||
self.htmlFile.write(' <td colspan="6"><b> ---- PHYSIM TEST DETAIL INFO---- </b></td>\n')
|
||||
@@ -578,22 +578,24 @@ class HTMLManagement():
|
||||
self.htmlFile.write(' <tr bgcolor = "#33CCFF" >\n')
|
||||
self.htmlFile.write(' <th colspan="2">Test Name</th>\n')
|
||||
self.htmlFile.write(' <th colspan="2">Test Description</th>\n')
|
||||
self.htmlFile.write(' <th colspan="2">Result</th>\n')
|
||||
self.htmlFile.write(' <th>Test Status</th>\n')
|
||||
self.htmlFile.write(' <th>Info</th>\n')
|
||||
self.htmlFile.write(' </tr>\n')
|
||||
y = ''
|
||||
for key, value in testResult.items():
|
||||
x = key.split(".")
|
||||
if x[0] != y:
|
||||
if x[2] != y:
|
||||
self.htmlFile.write(' <tr bgcolor = "lightgreen" >\n')
|
||||
self.htmlFile.write(' <td style="text-align: center;" colspan="6"><b>"' + x[0] + '" series </b></td>\n')
|
||||
self.htmlFile.write(' <td style="text-align: center;" colspan="6"><b>"' + x[2] + '" series </b></td>\n')
|
||||
self.htmlFile.write(' </tr>\n')
|
||||
y = x[0]
|
||||
y = x[2]
|
||||
self.htmlFile.write(' <tr>\n')
|
||||
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" >' + key + ' </td>\n')
|
||||
self.htmlFile.write(' <td colspan="2" bgcolor = "lightcyan" >' + value[0] + '</td>\n')
|
||||
if 'PASS' in value:
|
||||
self.htmlFile.write(' <td colspan="2" bgcolor = "green" >' + value[1] + '</td>\n')
|
||||
self.htmlFile.write(' <td bgcolor = "green" ><font color="white"><b>' + value[2] + '</b></font></td>\n')
|
||||
else:
|
||||
self.htmlFile.write(' <td colspan="2" bgcolor = "red" >' + value[1] + '</td>\n')
|
||||
self.htmlFile.write(' <td bgcolor = "red" ><font color="white"><b>' + value[2] + '</b></font></td>\n')
|
||||
self.htmlFile.write(' <td bgcolor = "lightcyan">' + value[1] + '</td>\n')
|
||||
|
||||
self.htmlFile.close()
|
||||
|
||||
Executable
+33
@@ -0,0 +1,33 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
function die() { echo $@; exit 1; }
|
||||
[ $# -eq 4 ] || die "usage: $0 <namespace> <release> <image tag> <oai directory>"
|
||||
|
||||
OC_NS=${1}
|
||||
OC_RELEASE=${2}
|
||||
IMG_TAG=${3}
|
||||
OAI_DIR=${4}
|
||||
|
||||
cat /opt/oc-password | oc login -u oaicicd --server https://api.oai.cs.eurecom.fr:6443 > /dev/null
|
||||
oc project ${OC_NS} > /dev/null
|
||||
oc tag oaicicd-ran/oai-physim:${IMG_TAG} ${OC_NS}/oai-physim:${IMG_TAG}
|
||||
helm install ${OC_RELEASE} ${OAI_DIR}/charts/${OC_RELEASE} --set global.image.version=${IMG_TAG} --wait
|
||||
POD_ID=$(oc get pods | grep oai-${OC_RELEASE} | awk '{print $1}')
|
||||
sleep 10
|
||||
echo "Monitoring logs for 'FINISHED' in pod '$POD_ID'"
|
||||
oc logs -f -n ${OC_NS} "$POD_ID" | while read -r line; do
|
||||
if [[ "$line" == *"FINISHED"* ]]; then
|
||||
echo "'FINISHED' detected in logs. Copying logs..."
|
||||
oc logs -n ${OC_NS} "$POD_ID" >> ${OAI_DIR}/ci-scripts/physim_log.txt
|
||||
oc describe pod $POD_ID >> ${OAI_DIR}/ci-scripts/physim_pods_summary.txt
|
||||
oc cp "$POD_ID":/opt/oai-physim/Testing/Temporary/LastTestsFailed.log ${OAI_DIR}/ci-scripts/LastTestsFailed.log
|
||||
oc cp "$POD_ID":/opt/oai-physim/${OC_RELEASE}-tests.json ${OAI_DIR}/ci-scripts/${OC_RELEASE}-tests.json
|
||||
oc cp "$POD_ID":/opt/oai-physim/${OC_RELEASE}-run.xml ${OAI_DIR}/ci-scripts/${OC_RELEASE}-run.xml
|
||||
break
|
||||
fi
|
||||
done
|
||||
helm uninstall ${OC_RELEASE} --wait
|
||||
oc delete istag oai-physim:${IMG_TAG} -n ${OC_NS}
|
||||
oc logout > /dev/null
|
||||
Reference in New Issue
Block a user