diff --git a/ci-scripts/cls_analysis.py b/ci-scripts/cls_analysis.py index 25d9b0c659..ac591b52fb 100644 --- a/ci-scripts/cls_analysis.py +++ b/ci-scripts/cls_analysis.py @@ -20,8 +20,11 @@ # */ #--------------------------------------------------------------------- -import logging import re +import os + +import xml.etree.ElementTree as ET +import json # Define the mapping of physim_test values to search patterns PHYSIM_PATTERN_MAPPING = { @@ -80,3 +83,65 @@ class Analysis(): msg = f'{time1_match.group(1)}: {time1_match.group(2)} us\n{time2_match.group(1)}: {time2_match.group(2)} us exceeds a limit of {threshold} us' return success,msg + + def analyze_oc_physim(result_junit, details_json): + try: + tree = ET.parse(result_junit) + root = tree.getroot() + nb_tests = int(root.attrib["tests"]) + nb_failed = int(root.attrib["failures"]) + except ET.ParseError as e: + return False, False, f'Could not parse XML log file {result_junit}: {e}' + except FileNotFoundError as e: + return False, False, f'JUnit XML log file {result_junit} not found: {e}' + except Exception as e: + return False, False, f'While parsing JUnit XML log file: exception: {e}' + + try: + with open(details_json) as f: + j = json.load(f) + # prepare JSON for easier access of strings + json_test_desc = {} + for e in j["tests"]: + json_test_desc[e["name"]] = e + + except json.JSONDecodeError as e: + return False, False, f'Could not decode JSON log file {details_json}: {e}' + except FileNotFoundError as e: + return False, False, f'Physim JSON log file {details_json} not found: {e}' + except Exception as e: + return False, False, f'While parsing physim JSON log file: exception: {e}' + + test_result = {} + for test in root: # for each test + test_name = test.attrib["name"] + test_exec = json_test_desc[test_name]["properties"][1]["value"][0] + desc = json_test_desc[test_name]["properties"][1]["value"][1] + # get runtime and checks + test_check = test.attrib["status"] == "run" + time = round(float(test.attrib["time"]), 1) + time_check = time < 150 + output = test.findtext("system-out") + output_check = "exceeds the threshold" not in output + # collect logs + log_dir = f'../cmake_targets/log/{test_exec}' + os.makedirs(log_dir, exist_ok=True) + with open(f'{log_dir}/{test_name}.log', 'w') as f: + f.write(output) + # prepare result and info + info = f"Runtime: {f'{time:.3f}'[:5]} s" + resultstr = 'PASS' if (test_check and time_check and output_check) else 'FAIL' + if test_check: + if not output_check: + info += " Test log exceeds maximal allowed length 100 kB" + if not time_check: + info += " Test exceeds 150s" + if not (time_check and output_check): + nb_failed += 1 # time threshold/output length error, not counted for by ctest as of now + test_result[test_name] = [desc, info, resultstr] + + test_summary = {} + test_summary['Nbtests'] = nb_tests + test_summary['Nbpass'] = nb_tests - nb_failed + test_summary['Nbfail'] = nb_failed + return nb_failed == 0, test_summary, test_result diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index ed7c638195..7e88c6b07b 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -31,8 +31,10 @@ import logging import re import time +import os import cls_oai_html +import cls_analysis import constants as CONST import helpreadme as HELP import cls_containerize @@ -457,3 +459,38 @@ class Cluster: HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize) return status + + def deploy_oc_physim(self, HTML, oc_release, svr_id): + if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': + HELP.GenericHelp(CONST.Version) + raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {self.ranBranch} ranCommitID {self.ranCommitID}') + image_tag = cls_containerize.CreateTag(self.ranCommitID, self.ranBranch, self.ranAllowMerge) + logging.debug(f'Running physims from server: {svr_id}') + script = "scripts/oc-deploy-physims.sh" + options = f"oaicicd-core-for-ci-ran {oc_release} {image_tag} {self.eNBSourceCodePath}" + ret = cls_cmd.runScript(svr_id, script, 600, options) + logging.debug(f'"{script}" finished with code {ret.returncode}, output:\n{ret.stdout}') + log_dir = f'{os.getcwd()}/../cmake_targets/log' + os.makedirs(log_dir, exist_ok=True) + result_junit = f'{oc_release}-run.xml' + details_json = f'{oc_release}-tests.json' + with cls_cmd.getConnection(svr_id) as ssh: + ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{details_json}', tgt=f'{log_dir}/{details_json}') + ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{result_junit}', tgt=f'{log_dir}/{result_junit}') + ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt', tgt=f'{log_dir}/physim_log.txt') + ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt', tgt=f'{log_dir}/physim_pods_summary.txt') + ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log', tgt=f'{log_dir}/LastTestsFailed.log') + test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(f'{log_dir}/{result_junit}', f'{log_dir}/{details_json}') + if test_summary: + if test_status: + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlTestRowPhySimTestResult(test_summary, test_result) + logging.info('\u001B[1m Physical Simulator Pass\u001B[0m') + else: + HTML.CreateHtmlTestRow('Some test(s) failed!', 'KO', CONST.OC_PHYSIM_DEPLOY_FAIL) + HTML.CreateHtmlTestRowPhySimTestResult(test_summary, test_result) + logging.error('\u001B[1m Physical Simulator Fail\u001B[0m') + else: + HTML.CreateHtmlTestRowQueue('Physical simulator failed', 'KO', [test_result]) + logging.error('\u001B[1m Physical Simulator Fail\u001B[0m') + return test_status diff --git a/ci-scripts/cls_oai_html.py b/ci-scripts/cls_oai_html.py index 8e99b20de7..478cbd1860 100644 --- a/ci-scripts/cls_oai_html.py +++ b/ci-scripts/cls_oai_html.py @@ -564,13 +564,13 @@ class HTMLManagement(): self.htmlFile.write(' Nb Pass\n') self.htmlFile.write(' \n') self.htmlFile.write(' \n') - self.htmlFile.write(' physim_test.txt \n') + self.htmlFile.write(' physim_log.txt \n') self.htmlFile.write(' ' + str(testSummary['Nbtests']) + ' \n') if testSummary['Nbfail'] == 0: - self.htmlFile.write(' ' + str(testSummary['Nbfail']) + ' \n') + self.htmlFile.write(' ' + str(testSummary['Nbfail']) + '\n') else: - self.htmlFile.write(' ' + str(testSummary['Nbfail']) + ' \n') - self.htmlFile.write(' ' + str(testSummary['Nbpass']) + ' \n') + self.htmlFile.write(' ' + str(testSummary['Nbfail']) + '\n') + self.htmlFile.write(' ' + str(testSummary['Nbpass']) + ' \n') self.htmlFile.write(' \n') self.htmlFile.write(' \n') self.htmlFile.write(' ---- PHYSIM TEST DETAIL INFO---- \n') @@ -578,22 +578,24 @@ class HTMLManagement(): self.htmlFile.write(' \n') self.htmlFile.write(' Test Name\n') self.htmlFile.write(' Test Description\n') - self.htmlFile.write(' Result\n') + self.htmlFile.write(' Test Status\n') + self.htmlFile.write(' Info\n') self.htmlFile.write(' \n') y = '' for key, value in testResult.items(): x = key.split(".") - if x[0] != y: + if x[2] != y: self.htmlFile.write(' \n') - self.htmlFile.write(' "' + x[0] + '" series \n') + self.htmlFile.write(' "' + x[2] + '" series \n') self.htmlFile.write(' \n') - y = x[0] + y = x[2] self.htmlFile.write(' \n') self.htmlFile.write(' ' + key + ' \n') self.htmlFile.write(' ' + value[0] + '\n') if 'PASS' in value: - self.htmlFile.write(' ' + value[1] + '\n') + self.htmlFile.write(' ' + value[2] + '\n') else: - self.htmlFile.write(' ' + value[1] + '\n') + self.htmlFile.write(' ' + value[2] + '\n') + self.htmlFile.write(' ' + value[1] + '\n') self.htmlFile.close() diff --git a/ci-scripts/scripts/oc-deploy-physims.sh b/ci-scripts/scripts/oc-deploy-physims.sh new file mode 100755 index 0000000000..ddb2e964ef --- /dev/null +++ b/ci-scripts/scripts/oc-deploy-physims.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e + +function die() { echo $@; exit 1; } +[ $# -eq 4 ] || die "usage: $0 " + +OC_NS=${1} +OC_RELEASE=${2} +IMG_TAG=${3} +OAI_DIR=${4} + +cat /opt/oc-password | oc login -u oaicicd --server https://api.oai.cs.eurecom.fr:6443 > /dev/null +oc project ${OC_NS} > /dev/null +oc tag oaicicd-ran/oai-physim:${IMG_TAG} ${OC_NS}/oai-physim:${IMG_TAG} +helm install ${OC_RELEASE} ${OAI_DIR}/charts/${OC_RELEASE} --set global.image.version=${IMG_TAG} --wait +POD_ID=$(oc get pods | grep oai-${OC_RELEASE} | awk '{print $1}') +sleep 10 +echo "Monitoring logs for 'FINISHED' in pod '$POD_ID'" +oc logs -f -n ${OC_NS} "$POD_ID" | while read -r line; do + if [[ "$line" == *"FINISHED"* ]]; then + echo "'FINISHED' detected in logs. Copying logs..." + oc logs -n ${OC_NS} "$POD_ID" >> ${OAI_DIR}/ci-scripts/physim_log.txt + oc describe pod $POD_ID >> ${OAI_DIR}/ci-scripts/physim_pods_summary.txt + oc cp "$POD_ID":/opt/oai-physim/Testing/Temporary/LastTestsFailed.log ${OAI_DIR}/ci-scripts/LastTestsFailed.log + oc cp "$POD_ID":/opt/oai-physim/${OC_RELEASE}-tests.json ${OAI_DIR}/ci-scripts/${OC_RELEASE}-tests.json + oc cp "$POD_ID":/opt/oai-physim/${OC_RELEASE}-run.xml ${OAI_DIR}/ci-scripts/${OC_RELEASE}-run.xml + break + fi +done +helm uninstall ${OC_RELEASE} --wait +oc delete istag oai-physim:${IMG_TAG} -n ${OC_NS} +oc logout > /dev/null