diff --git a/ci-scripts/args_parse.py b/ci-scripts/args_parse.py index 3a3ea52fa0..9f181ae68e 100644 --- a/ci-scripts/args_parse.py +++ b/ci-scripts/args_parse.py @@ -33,7 +33,6 @@ #----------------------------------------------------------- import sys # arg import re # reg -import yaml import constants as CONST #----------------------------------------------------------- @@ -44,6 +43,7 @@ import constants as CONST def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER): force_local = False + date_fmt = None while len(argvs) > 1: myArgv = argvs.pop(1) # 0th is this file's name @@ -56,6 +56,9 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER): #consider inline parameters + elif re.match(r'^\-\-datefmt=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match(r'^\-\-datefmt=(.+)$', myArgv, re.IGNORECASE) + date_fmt = matchReg.group(1) elif re.match(r'^\-\-mode=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match(r'^\-\-mode=(.+)$', myArgv, re.IGNORECASE) mode = matchReg.group(1) @@ -158,9 +161,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER): elif re.match(r'^\-\-OCRegistry=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match(r'^\-\-OCRegistry=(.+)$', myArgv, re.IGNORECASE) CLUSTER.OCRegistry = matchReg.group(1) - elif re.match(r'^\-\-BuildId=(.+)$', myArgv, re.IGNORECASE): - matchReg = re.match(r'^\-\-BuildId=(.+)$', myArgv, re.IGNORECASE) - RAN.BuildId = matchReg.group(1) elif re.match(r'^\-\-FlexRicTag=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match(r'^\-\-FlexRicTag=(.+)$', myArgv, re.IGNORECASE) CONTAINERS.flexricTag = matchReg.group(1) @@ -168,4 +168,4 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER): HELP.GenericHelp(CONST.Version) sys.exit('Invalid Parameter: ' + myArgv) - return mode, force_local + return mode, force_local, date_fmt diff --git a/ci-scripts/cls_analysis.py b/ci-scripts/cls_analysis.py index 588490da62..866a261352 100644 --- a/ci-scripts/cls_analysis.py +++ b/ci-scripts/cls_analysis.py @@ -22,10 +22,15 @@ import re import os +import logging +import yaml +import signal import xml.etree.ElementTree as ET import json +import importlib, inspect + class Analysis(): def _get_test_description(properties): @@ -109,3 +114,134 @@ class Analysis(): test_summary['Nbpass'] = nb_tests - nb_failed test_summary['Nbfail'] = nb_failed return nb_failed == 0, test_summary, test_result + + def analyze_rt_stats(thresholds, L1_stats, MAC_stats): + with open(thresholds, 'r') as f: + datalog_rt_stats = yaml.load(f, Loader=yaml.FullLoader) + + rt_keys = datalog_rt_stats['Ref'] + real_time_stats = {} + for sf in [L1_stats, MAC_stats]: + with open(sf, 'r') as f: + for line in f.readlines(): + for k in rt_keys: + result = re.search(k, line) + if result is not None: + tmp = re.match(rf'^.*?(\b{k}\b.*)', line.rstrip()) + if tmp is not None: + real_time_stats[k] = tmp.group(1) + + # datalog_rt_stats format must align with HTML.CreateHtmlDataLogTable() + datalog_rt_stats['Data']={} + for k in real_time_stats: + tmp = re.match(r'^(?P.*):\s+(?P\d+\.\d+) us;\s+(?P\d+);\s+(?P\d+\.\d+) us;', real_time_stats[k]) + if tmp is not None: + metric = tmp.group('metric') + avg = float(tmp.group('avg')) + max = float(tmp.group('max')) + count = int(tmp.group('count')) + datalog_rt_stats['Data'][metric] = ["{:.0f}".format(avg),"{:.0f}".format(max),"{:d}".format(count),"{:.2f}".format(avg / datalog_rt_stats['Ref'][metric])] + + success = True + for k in datalog_rt_stats['Data']: + valnorm = float(datalog_rt_stats['Data'][k][3]) + dev = datalog_rt_stats['DeviationThreshold'][k] + if valnorm > 1.0 + dev or valnorm < 1.0 - dev: # condition for fail: avg/ref deviates by more than "deviation threshold" + logging.debug(f'\u001B[1;30;43m normalized metric {k}={valnorm} deviates by more than {dev}\u001B[0m') + success = False + return success, datalog_rt_stats + + +# returns tuple of (service, analyzer, option string) +def _lookupServiceAnalyzerOpt(s, analyzers): + res = s.split("=", 2) + name = res[0] + if len(res) == 1: + return res[0], analyzers["Default"], None + opt = res[2] if len(res) > 2 else None + func = res[1] + a = analyzers[func] if func in analyzers else None + return name, a, opt + +# groups requested service analysis (service=func[=option]) on per service +# basis and looks up analyzer func. If log analysis is requested, will always +# run Default log analysis +def _groupServices(to_analyze): + req_analysis = {} + # get content of cls_loganalysis module, then get all analyzers (classes) in this module + mod = importlib.import_module("cls_loganalysis") + analyzers = {name:cl for name, cl in inspect.getmembers(mod, inspect.isclass)} + for req in to_analyze: + s, func, opt = _lookupServiceAnalyzerOpt(req, analyzers) + logging.debug(f"requested check '{req}' => service {s}, function {func.__name__}, options '{opt}'") + # always put default analyzer first + l = req_analysis[s] if s in req_analysis else [(analyzers["Default"], None, "default")] + if func is not analyzers["Default"]: + l.append((func, opt, req)) + req_analysis[s] = l + return req_analysis + +def _describe_exit_code(code): + if code > 128: + sig = code - 128 + try: + return f"terminated by signal {sig} ({signal.Signals(sig).name})" + except ValueError: + return f"terminated by unknown signal {sig}" + else: + return f"custom exit code" + +def AnalyzeServices(HTML, service_desc, to_analyze): + success = True + # hack: we want to give as a description "log analysis", but the description + # is set outside, so retain what was set before + orig_html_desc = HTML.desc + # group analysis on a per-service basis, then iterate + for serv, list_analysis in _groupServices(to_analyze).items(): + HTML.desc = f"Log analysis for service {serv}" + if serv not in service_desc: + success = False + logging.error(f"requested service {serv} not in list of services") + HTML.CreateHtmlTestRowQueue("N/A", 'KO', ["service not detected"]) + continue + # pre-initialize with return code + rc = service_desc[serv]["returncode"] + logging.info(f"analyze service {serv}: return code {rc}") + service_success = True # TODO rc == 0: too many functions (eNB, lteUE, RIC, nrUE) fail non-zero + logs = [] + if rc != 0: + logs.append(f"=> return code {rc}, likely " + _describe_exit_code(rc) + " [ignored by CI]") + # skip if the file is too big + logfile = service_desc[serv]["logfile"] + filename = os.path.basename(logfile) + b = os.path.getsize(logfile) + logging.debug(f"using logfile {logfile} of size {b} bytes") + if b > 10 * 1024 * 1024: + success = False + logs.append(f"logfile too big (>10MB)") + logging.error(logs) + HTML.CreateHtmlTestRowQueue(filename, 'KO', ["\n".join(logs)]) + continue + # run each analyzer with its options on the logfile + for (func, opt, desc) in list_analysis: + if func is None: + service_success = False + logging.error(f"request analysis function for desc {desc} not found") + HTML.CreateHtmlTestRowQueue(serv, 'KO', [f"no analysis function for {desc}"]) + continue + result, l = func.run(logfile, opt) + logging.info(f"service {serv}: analysis with func {func}, result {result}, logs '{l}'") + service_success = service_success and result + if not result: + logs.append(f"=> {func.__name__} check (options '{opt}') FAILED") + logs.append(l) + logs = '\n'.join(logs) + if not service_success: + logging.error(l) + else: + logging.info(l) + all_funcs = ", ".join([f.__name__ for (f, _, _) in list_analysis]) + HTML.CreateHtmlTestRowQueue(f"Check {all_funcs} on {filename}", 'OK' if service_success else 'KO', [logs]) + success = success and service_success + HTML.desc = orig_html_desc + return success diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index ac76f3083b..f1e49bc0a7 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -31,23 +31,17 @@ #----------------------------------------------------------- # Import #----------------------------------------------------------- -import sys # arg import re # reg import logging import os -import shutil -import time -from zipfile import ZipFile #----------------------------------------------------------- # OAI Testing modules #----------------------------------------------------------- import cls_cmd -import helpreadme as HELP import constants as CONST -import cls_oaicitest +import cls_analysis from cls_ci_helper import archiveArtifact -from collections import deque #----------------------------------------------------------- # Helper functions used here and in other classes @@ -186,62 +180,9 @@ def GetDeployedServices(ssh, file): else: c = ret.stdout logging.info(f'service {s} with container id {c}') - deployed_services.append(s) + deployed_services.append((s, c)) return deployed_services -def CheckLogs(self, filename, HTML, RAN): - success = True - name = os.path.basename(filename) - if (any(sub in name for sub in ['oai_ue','oai-nr-ue','lte_ue'])): - logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m') - logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(filename, HTML, RAN) - opt = f"UE log analysis ({name})" - # usage of htmlUEFailureMsg/htmleNBFailureMsg is because Analyze log files - # abuse HTML to store their reports, and we here want to put custom options, - # which is not possible with CreateHtmlTestRow - # solution: use HTML templates, where we don't need different HTML write funcs - if (logStatus < 0): - HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmlUEFailureMsg]) - success = False - else: - HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmlUEFailureMsg]) - HTML.htmlUEFailureMsg = "" - elif 'nv-cubb' in name: - msg = 'Undeploy PNF/Nvidia CUBB' - HTML.CreateHtmlTestRow(msg, 'OK', CONST.ALL_PROCESSES_OK) - elif (any(sub in name for sub in ['enb','rru','rcc','cu','du','gnb','vnf'])): - logging.debug(f'\u001B[1m Analyzing XnB logfile {filename}\u001B[0m') - logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers) - opt = f"xNB log analysis ({name})" - if (logStatus < 0): - HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmleNBFailureMsg]) - success = False - else: - HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg]) - HTML.htmleNBFailureMsg = "" - elif 'xapp' in name: - opt = f"Undeploy {name}" - with open(f'{filename}', "r") as f: - last_line = deque(f, maxlen=1).pop() - if ('Test xApp run SUCCESSFULLY' in last_line): - HTML.CreateHtmlTestRowQueue(opt, 'OK', ["xApp run successfully"]) - else: - HTML.CreateHtmlTestRowQueue(opt, 'KO', ["xApp didn't run successfully"]) - success = False - elif 'RIC' in name: - opt = f"Undeploy {name}" - with open(f'{filename}', 'r') as f: - last_line = deque(f, maxlen=1).pop() - if ('Removing E2 Node' in last_line): - HTML.CreateHtmlTestRowQueue(opt, 'OK', ["nearRT-RIC run successfully"]) - else: - HTML.CreateHtmlTestRowQueue(opt, 'KO', ["nearRT-RIC didn't run successfully"]) - success = False - else: - logging.info(f"Skipping analysis of log '{filename}': no submatch for xNB/UE") - logging.debug(f"log check: file {filename} passed analysis {success}") - return success - #----------------------------------------------------------- # Class Declaration #----------------------------------------------------------- @@ -266,8 +207,6 @@ class Containerize(): self.dockerfileprefix = '' self.host = '' - #checkers from xml - self.ran_checkers={} self.num_attempts = 1 self.flexricTag = '' @@ -796,7 +735,7 @@ class Containerize(): wd_yaml = f'{wd}/docker-compose.y*ml' with cls_cmd.getConnection(node) as ssh: ExistEnvFilePrint(ssh, wd) - services = GetDeployedServices(ssh, wd_yaml) + services = [s for s, _ in GetDeployedServices(ssh, wd_yaml)] success = [] fail = [] for s in reqServices: @@ -814,7 +753,7 @@ class Containerize(): HTML.CreateHtmlTestRowQueue(self.services, 'KO', [f'Failed stopping {" ".join(fail)}, succeeded {" ".join(success)}']) return success - def UndeployObject(self, ctx, node, HTML, RAN): + def UndeployObject(self, ctx, node, HTML, to_analyze): lSourcePath = self.eNBSourceCodePath logging.info(f'\u001B[1m Undeploying all objects from server {node}\u001B[0m') yaml = self.yamlPath.strip('/') @@ -823,23 +762,49 @@ class Containerize(): with cls_cmd.getConnection(node) as ssh: ExistEnvFilePrint(ssh, wd) services = GetDeployedServices(ssh, wd_yaml) - copyin_res = None + all_logs = True ssh.run(f'docker compose -f {wd_yaml} stop') if services is not None: - copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, wd_yaml, ctx) for s in services] + service_desc = {} + for s, c in services: + ret = ssh.run(f'docker inspect {c} --format="{{{{.State.ExitCode}}}}"') + rc = int(ret.stdout.strip()) + f = CopyinServiceLog(ssh, lSourcePath, s, wd_yaml, ctx) + all_logs = all_logs and f is not None + service_desc[s] = {'returncode': rc, 'logfile': f} else: logging.warning('could not identify services to stop => no log file') ssh.run(f'docker compose -f {wd_yaml} down -v') + HTML.CreateHtmlTestRowQueue(node, 'OK', ['Undeployment successful']) ssh.run(f'rm {wd}/.env') - if not copyin_res: + if not all_logs: HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)']) - logging.error(f"could not copy all files: {copyin_res=} {services=}") + logging.error(f"could not copy all files: {all_logs=} {services=}") success = False else: - log_results = [CheckLogs(self, f, HTML, RAN) for f in copyin_res] - success = all(log_results) + success = cls_analysis.AnalyzeServices(HTML, service_desc, to_analyze) if success: logging.info('\u001B[1m Undeploying objects Pass\u001B[0m') else: logging.error('\u001B[1m Undeploying objects Failed\u001B[0m') return success + + def AnalyzeRTStatsObject(self, HTML, node, ctx, thresholds): + logging.info(f'Analyzing realtime stats from server: {node}') + yaml = self.yamlPath.strip('/') + wd = f'{self.eNBSourceCodePath}/{yaml}' + wd_yaml = f'{wd}/docker-compose.y*ml' + + with cls_cmd.getConnection(node) as cmd: + services = GetDeployedServices(cmd, wd_yaml) + s, _ = services[0] # we simply assume something is deployed, if not Exception will report + # similar to BuildRunTests(), use docker cp to avoid problems with permissions + cmd.run(f'docker compose -f {wd_yaml} cp {s}:/opt/oai-gnb/nrL1_stats.log {wd}/') + l1_file = archiveArtifact(cmd, ctx, f"{wd}/nrL1_stats.log") + cmd.run(f'docker compose -f {wd_yaml} cp {s}:/opt/oai-gnb/nrMAC_stats.log {wd}/') + mac_file = archiveArtifact(cmd, ctx, f"{wd}/nrMAC_stats.log") + + logging.info(f"check against thresholds from {thresholds}") + success, datalog_rt_stats = cls_analysis.Analysis.analyze_rt_stats(thresholds, l1_file, mac_file) + HTML.CreateHtmlDataLogTable(datalog_rt_stats) + return success diff --git a/ci-scripts/cls_loganalysis.py b/ci-scripts/cls_loganalysis.py new file mode 100644 index 0000000000..b28e217707 --- /dev/null +++ b/ci-scripts/cls_loganalysis.py @@ -0,0 +1,161 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- + +import re +import os +import logging +from collections import deque + +class Default: + def run(file, opt=None): + success = True + logs = [] + with open(file, "r") as f: + for line in f.readlines(): + result = re.search('[Aa]ssertion', line) + if result: + logs.append(line) + success = False + return success, "\n".join(logs) + +class ContainsString: + def run(file, needle): + with open(file, "r") as f: + for line in f.readlines(): + if re.search(needle, line): + return True, "" + return False, f"could not find '{needle}' in logs" + +class LastLineContains: + def run(file, string): + with open(file, "r") as f: + last_line = deque(f, maxlen=1).pop() + success = string in last_line + log = "" if success else f"could not find '{string}' in last line" + return success, log + +class EndsWithBye: + def run(file, option): + # no option: we ignore any + with open(file, "r") as f: + n = 0 + for line in reversed(f.readlines()): + if re.search(r'^Bye.\n', line, re.MULTILINE) is not None: + return True, "" + n = n + 1 + if n > 50: # if not found in last 50 lines, assume it does not exist + break + return False, "No Bye. message found, did not stop properly" + +class RetxCheck(): + def _parseList(s): + if not s: + return [] + return [int(n) for n in s.split(",")] + + def _get_ul_dl(s): + dl, ul = (None, None) + for o in s.split(";", 1): + if o.startswith("dl="): + dl = o[3:] + elif o.startswith("ul="): + ul = o[3:] + else: + raise ValueError(f"unrecognized option {s}: needs format '[ul=,dl=]X,Y,...[;ul=Z,V,...]'") + return dl, ul + + def _parseRetxCheckers(options): + if not "=" in options: + dlul = RetxCheck._parseList(options) + return dlul, dlul + dl, ul = RetxCheck._get_ul_dl(options) + return RetxCheck._parseList(dl), RetxCheck._parseList(ul) + + def _analyzeUeRetx(rounds, checkers, regex): + perc = list(0 for i in checkers) # results in % + stats = list(False for i in checkers) # status if succeeded + tmp = re.match(regex, rounds) + if tmp is None: + logging.error('_analyzeUeRetx: did not match regex for DL retx analysis') + return stats + retx_data = [float(x) for x in tmp.groups()] + for i in range(0, len(perc)): + #case where numerator > denumerator with denum ==0 is disregarded, cannot hapen in principle, will lead to 0% + perc[i] = 0 if (retx_data[i] == 0) else 100 * retx_data[i + 1] / retx_data[i] + #treating % > 100 , % > requirement + stats[i] = perc[i] <= 100 and perc[i] <= checkers[i] + return stats + + def _filter_retx_rounds(l): + # map to round number if False, else None + l = [i+1 if x is False else None for i, x in enumerate(l)] + # filter out None + return [x for x in l if x is not None] + + def run(file, options): + # aggregate statistics per UE + dlsch_ulsch_stats = {} + status = True + with open(file, "r") as f: + for line in f.readlines(): + keys = {'dlsch_rounds','ulsch_rounds'} + for k in keys: + result = re.search(k, line) + if result is None: + continue + result = re.search('UE (?:RNTI )?([0-9a-f]{4})', line) + if result is None: + logging.error(f'did not find RNTI while matching key {k}') + status = False + continue + rnti = result.group(1) + if not rnti in dlsch_ulsch_stats: + dlsch_ulsch_stats[rnti] = {} + dlsch_ulsch_stats[rnti][k]=re.sub(r'^.*\]\s+', r'' , line.rstrip()) + + dl, ul = RetxCheck._parseRetxCheckers(options) + msgs = [] + for ue in dlsch_ulsch_stats: + dlulstat = dlsch_ulsch_stats[ue] + + for key in dlulstat: + msgs.append(dlulstat[key]) + + retx_dl = RetxCheck._analyzeUeRetx(dlulstat['dlsch_rounds'], dl, r'^.*dlsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+dlsch_errors\s+(\d+)') + retx_ul = RetxCheck._analyzeUeRetx(dlulstat['ulsch_rounds'], ul, r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+ulsch_errors\s+(\d+)') + + if False in retx_dl or False in retx_ul: + dlstr = RetxCheck._filter_retx_rounds(retx_dl) + ulstr = RetxCheck._filter_retx_rounds(retx_ul) + msg = f"UE {ue}: retx rounds" + if dlstr: + msg = f"{msg} DL {dlstr}" + if ulstr: + msg = f"{msg} UL {ulstr}" + msg = f"{msg} exceeded threshold" + msgs.append('!!! Failure: ' + msg) + logging.error(f'\u001B[1;37;41m {msg}\u001B[0m') + status = False + else: + msg = f"UE {ue} retransmissions within bounds" + logging.debug(msg) + return status, "\n".join(msgs) diff --git a/ci-scripts/cls_oai_html.py b/ci-scripts/cls_oai_html.py index 6e29063192..444fc68b5c 100644 --- a/ci-scripts/cls_oai_html.py +++ b/ci-scripts/cls_oai_html.py @@ -218,7 +218,7 @@ class HTMLManagement(): cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__//' test_results.html" subprocess.run(cmd, shell=True) else: - cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<\/span>/' test_results.html" + cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + r"__/<\/span>/' test_results.html" subprocess.run(cmd, shell=True) self.htmlFooterCreated = False diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 237e795937..26ac91e668 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -32,7 +32,6 @@ #----------------------------------------------------------- # Import Libs #----------------------------------------------------------- -import sys # arg import re # reg import time # sleep import os @@ -41,7 +40,6 @@ import concurrent.futures import json #import our libs -import helpreadme as HELP import constants as CONST import cls_module @@ -301,7 +299,6 @@ class OaiCiTest(): self.iperf_profile = '' self.iperf_tcp_rate_target = '' self.finalStatus = False - self.air_interface='' self.ue_ids = [] self.svr_node = None self.svr_id = None @@ -579,277 +576,6 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', [f'{ue_header}\n{msg}']) return success - def AnalyzeLogFile_UE(self, UElogFile,HTML,RAN): - if (not os.path.isfile(f'{UElogFile}')): - return -1 - ue_log_file = open(f'{UElogFile}', 'r') - exitSignalReceived = False - foundAssertion = False - msgAssertion = '' - msgLine = 0 - foundSegFault = False - foundRealTimeIssue = False - uciStatMsgCount = 0 - pdcpDataReqFailedCount = 0 - badDciCount = 0 - f1aRetransmissionCount = 0 - fatalErrorCount = 0 - macBsrTimerExpiredCount = 0 - rrcConnectionRecfgComplete = 0 - no_cell_sync_found = False - mib_found = False - frequency_found = False - plmn_found = False - nrUEFlag = False - nrDecodeMib = 0 - nrFoundDCI = 0 - nrCRCOK = 0 - mbms_messages = 0 - nbPduSessAccept = 0 - nbPduDiscard = 0 - HTML.htmlUEFailureMsg='' - global_status = CONST.ALL_PROCESSES_OK - for line in ue_log_file.readlines(): - result = re.search('nr_synchro_time|Starting NR UE soft modem', str(line)) - sidelink = re.search('sl-mode', str(line)) - if result is not None: - nrUEFlag = True - if sidelink is not None: - nrUEFlag = False - if nrUEFlag: - result = re.search('decode mib', str(line)) - if result is not None: - nrDecodeMib += 1 - result = re.search('found 1 DCIs', str(line)) - if result is not None: - nrFoundDCI += 1 - result = re.search('CRC OK', str(line)) - if result is not None: - nrCRCOK += 1 - result = re.search('Received PDU Session Establishment Accept', str(line)) - if result is not None: - nbPduSessAccept += 1 - result = re.search('warning: discard PDU, sn out of window', str(line)) - if result is not None: - nbPduDiscard += 1 - result = re.search('--nfapi STANDALONE_PNF --node-number 2', str(line)) - if result is not None: - frequency_found = True - result = re.search('Exiting OAI softmodem', str(line)) - if result is not None: - exitSignalReceived = True - result = re.search('System error|[Ss]egmentation [Ff]ault|======= Backtrace: =========|======= Memory map: ========', str(line)) - if result is not None and not exitSignalReceived: - foundSegFault = True - result = re.search('[Cc]ore [dD]ump', str(line)) - if result is not None and not exitSignalReceived: - foundSegFault = True - result = re.search('[Aa]ssertion', str(line)) - if result is not None and not exitSignalReceived: - foundAssertion = True - result = re.search('LLL', str(line)) - if result is not None and not exitSignalReceived: - foundRealTimeIssue = True - if foundAssertion and (msgLine < 3): - msgLine += 1 - msgAssertion += str(line) - result = re.search('uci->stat', str(line)) - if result is not None and not exitSignalReceived: - uciStatMsgCount += 1 - result = re.search('PDCP data request failed', str(line)) - if result is not None and not exitSignalReceived: - pdcpDataReqFailedCount += 1 - result = re.search('bad DCI 1', str(line)) - if result is not None and not exitSignalReceived: - badDciCount += 1 - result = re.search('Format1A Retransmission but TBS are different', str(line)) - if result is not None and not exitSignalReceived: - f1aRetransmissionCount += 1 - result = re.search('FATAL ERROR', str(line)) - if result is not None and not exitSignalReceived: - fatalErrorCount += 1 - result = re.search('MAC BSR Triggered ReTxBSR Timer expiry', str(line)) - if result is not None and not exitSignalReceived: - macBsrTimerExpiredCount += 1 - result = re.search('Generating RRCConnectionReconfigurationComplete', str(line)) - if result is not None: - rrcConnectionRecfgComplete += 1 - # No cell synchronization found, abandoning - result = re.search('No cell synchronization found, abandoning', str(line)) - if result is not None: - no_cell_sync_found = True - if RAN.eNBmbmsEnables[0]: - result = re.search('TRIED TO PUSH MBMS DATA', str(line)) - if result is not None: - mbms_messages += 1 - result = re.search(r"MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P\d{1,3}), N_RB_DL (?P\d{1,3}), PHICH DURATION (?P\d), PHICH RESOURCE (?P.{1,4}), TX_ANT (?P\d)", str(line)) - if result is not None and (not mib_found): - try: - mibMsg = "MIB Information: " + result.group(1) + ', ' + result.group(2) - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - mibMsg = " nidcell = " + result.group('nidcell') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg - logging.debug(f'\033[94m{mibMsg}\033[0m') - mibMsg = " n_rb_dl = " + result.group('n_rb_dl') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - mibMsg = " phich_duration = " + result.group('phich_duration') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg - logging.debug(f'\033[94m{mibMsg}\033[0m') - mibMsg = " phich_resource = " + result.group('phich_resource') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - mibMsg = " tx_ant = " + result.group('tx_ant') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - mib_found = True - except Exception as e: - logging.error(f'\033[91m MIB marker was not found \033[0m') - result = re.search("Initial sync: pbch decoded sucessfully", str(line)) - if result is not None and (not frequency_found): - try: - mibMsg = f"UE decoded PBCH successfully" - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - frequency_found = True - except Exception as e: - logging.error(f'\033[91m UE did not find PBCH\033[0m') - result = re.search(r"PLMN MCC (?P\d{1,3}), MNC (?P\d{1,3}), TAC", str(line)) - if result is not None and (not plmn_found): - try: - mibMsg = f"PLMN MCC = {result.group('mcc')} MNC = {result.group('mnc')}" - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - plmn_found = True - except Exception as e: - logging.error(f'\033[91m PLMN not found \033[0m') - result = re.search(r"Found (?P[\w,\s]{1,15}) \(name from internal table\)", str(line)) - if result is not None: - try: - mibMsg = f"The operator is: {result.group('operator')}" - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - except Exception as e: - logging.error(f'\033[91m Operator name not found \033[0m') - result = re.search("SIB5 InterFreqCarrierFreq element (.{1,4})/(.{1,4})", str(line)) - if result is not None: - try: - mibMsg = f'SIB5 InterFreqCarrierFreq element {result.group(1)}/{result.group(2)}' - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + ' -> ' - logging.debug(f'\033[94m{mibMsg}\033[0m') - except Exception as e: - logging.error(f'\033[91m SIB5 InterFreqCarrierFreq element not found \033[0m') - result = re.search(r"DL Carrier Frequency/ARFCN : \-*(?P\d{1,15}/\d{1,4})", str(line)) - if result is not None: - try: - freq = result.group('carrier_frequency') - new_freq = re.sub('/[0-9]+','',freq) - float_freq = float(new_freq) / 1000000 - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'DL Freq: ' + ('%.1f' % float_freq) + ' MHz' - logging.debug(f'\033[94m DL Carrier Frequency is: {freq}\033[0m') - except Exception as e: - logging.error(f'\033[91m DL Carrier Frequency not found \033[0m') - result = re.search(r"AllowedMeasBandwidth : (?P\d{1,7})", str(line)) - if result is not None: - try: - prb = result.group('allowed_bandwidth') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + ' -- PRB: ' + prb + '\n' - logging.debug(f'\033[94m AllowedMeasBandwidth: {prb}\033[0m') - except Exception as e: - logging.error(f'\033[91m AllowedMeasBandwidth not found \033[0m') - ue_log_file.close() - if rrcConnectionRecfgComplete > 0: - statMsg = f'UE connected to eNB ({rrcConnectionRecfgComplete}) RRCConnectionReconfigurationComplete message(s) generated)' - logging.debug(f'\033[94m{statMsg}\033[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if nrUEFlag: - if nrDecodeMib > 0: - statMsg = f'UE showed {nrDecodeMib} "MIB decode" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if nrFoundDCI > 0: - statMsg = f'UE showed {nrFoundDCI} "DCI found" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if nrCRCOK > 0: - statMsg = f'UE showed {nrCRCOK} "PDSCH decoding" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if not frequency_found: - statMsg = 'NR-UE could NOT synch!' - logging.error(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if nbPduSessAccept > 0: - statMsg = f'UE showed {nbPduSessAccept} "Received PDU Session Establishment Accept" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if nbPduDiscard > 0: - statMsg = f'UE showed {nbPduDiscard} "warning: discard PDU, sn out of window" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if uciStatMsgCount > 0: - statMsg = f'UE showed {uciStatMsgCount} "uci->stat" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if pdcpDataReqFailedCount > 0: - statMsg = f'UE showed {pdcpDataReqFailedCount} "PDCP data request failed" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if badDciCount > 0: - statMsg = f'UE showed {badDciCount} "bad DCI 1(A)" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if f1aRetransmissionCount > 0: - statMsg = f'UE showed {f1aRetransmissionCount} "Format1A Retransmission but TBS are different" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if fatalErrorCount > 0: - statMsg = f'UE showed {fatalErrorCount} "FATAL ERROR:" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if macBsrTimerExpiredCount > 0: - statMsg = f'UE showed {fatalErrorCount} "MAC BSR Triggered ReTxBSR Timer expiry" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if RAN.eNBmbmsEnables[0]: - if mbms_messages > 0: - statMsg = f'UE showed {mbms_messages} "TRIED TO PUSH MBMS DATA" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - else: - statMsg = 'UE did NOT SHOW "TRIED TO PUSH MBMS DATA" message(s)' - logging.debug(f'\u001B[1;30;41m{statMsg}\u001B[0m') - global_status = CONST.OAI_UE_PROCESS_NO_MBMS_MSGS - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if foundSegFault: - logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m') - if not nrUEFlag: - global_status = CONST.OAI_UE_PROCESS_SEG_FAULT - else: - if not frequency_found: - global_status = CONST.OAI_UE_PROCESS_SEG_FAULT - if foundAssertion: - logging.debug('\u001B[1;30;43m UE showed an assertion! \u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE showed an assertion!\n' - if not nrUEFlag: - if not mib_found or not frequency_found: - global_status = CONST.OAI_UE_PROCESS_ASSERTION - else: - if not frequency_found: - global_status = CONST.OAI_UE_PROCESS_ASSERTION - if foundRealTimeIssue: - logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE faced real time issues!\n' - if nrUEFlag: - if not frequency_found: - global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC - else: - if no_cell_sync_found and not mib_found: - logging.debug('\u001B[1;37;41m UE could not synchronize ! \u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE could not synchronize!\n' - global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC - return global_status - def TerminateUE(self, ctx, node, HTML): ues = [cls_module.Module_UE(n.strip(), node) for n in self.ue_ids] with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor: diff --git a/ci-scripts/cls_static_code_analysis.py b/ci-scripts/cls_static_code_analysis.py index 5ed52e4027..cc9abedf23 100644 --- a/ci-scripts/cls_static_code_analysis.py +++ b/ci-scripts/cls_static_code_analysis.py @@ -89,7 +89,7 @@ class StaticCodeAnalysis(): logging.debug('Building on server: ' + node) cmd = cls_cmd.getConnection(node) # on RedHat/CentOS .git extension is mandatory - result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) + result = re.search(r'([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) if result is not None: full_ran_repo_name = self.ranRepository.replace('git/', 'git') else: @@ -121,7 +121,7 @@ class StaticCodeAnalysis(): xmlStart = False with open(filename, 'r') as logfile: for line in logfile: - ret = re.search('cppcheck version="(?P[0-9\.]+)"', str(line)) + ret = re.search(r'cppcheck version="(?P[0-9\.]+)"', str(line)) if ret is not None: CCR.versions[vId] = ret.group('version') if re.search('RUN cat cmake_targets/log/cppcheck.xml', str(line)) is not None: diff --git a/ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf b/ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf new file mode 100644 index 0000000000..0b087e7406 --- /dev/null +++ b/ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf @@ -0,0 +1,266 @@ +Active_gNBs = ( "gNB-OAI"); +# Asn1_verbosity, choice in: none, info, annoying +Asn1_verbosity = "none"; + +gNBs = +( + { + ////////// Identification parameters: + gNB_ID = 0xe00; + gNB_name = "gNB-OAI"; + + // Tracking area code, 0x0000 and 0xfffe are reserved values + tracking_area_code = 1; + plmn_list = ({ mcc = 208; mnc = 99; mnc_length = 2; snssaiList = ( { sst = 1; }); }); + + nr_cellid = 1; + + ////////// Physical parameters: + + pdsch_AntennaPorts_XP = 2; + pdsch_AntennaPorts_N1 = 2; + maxMIMO_layers = 4; + pusch_AntennaPorts = 4; + do_CSIRS = 0; + do_SRS = 0; + + servingCellConfigCommon = ( + { + #spCellConfigCommon + physCellId = 0; +# downlinkConfigCommon + #frequencyInfoDL + # center frequency = 4049.76 MHz + # selected SSB frequency = 4049.76 MHz + absoluteFrequencySSB = 668640; #669984; + dl_frequencyBand = 77; + # frequency point A = 4000.62 MHz + dl_absoluteFrequencyPointA = 666708; + #scs-SpecificCarrierList + dl_offstToCarrier = 0; +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + dl_subcarrierSpacing = 1; + dl_carrierBandwidth = 273; + #initialDownlinkBWP + #genericParameters + initialDLBWPlocationAndBandwidth = 1099; #38.101-1 Table 5.3.2-1 + # +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + initialDLBWPsubcarrierSpacing = 1; + #pdcch-ConfigCommon + initialDLBWPcontrolResourceSetZero = 11; + initialDLBWPsearchSpaceZero = 0; + + #uplinkConfigCommon + #frequencyInfoUL + ul_frequencyBand = 77; + #scs-SpecificCarrierList + ul_offstToCarrier = 0; +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + ul_subcarrierSpacing = 1; + ul_carrierBandwidth = 273; + pMax = 23; + #initialUplinkBWP + #genericParameters + initialULBWPlocationAndBandwidth = 1099; +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + initialULBWPsubcarrierSpacing = 1; + #rach-ConfigCommon + #rach-ConfigGeneric + prach_ConfigurationIndex = 159; +#prach_msg1_FDM +#0 = one, 1=two, 2=four, 3=eight + prach_msg1_FDM = 0; + prach_msg1_FrequencyStart = 0; + zeroCorrelationZoneConfig = 0; + preambleReceivedTargetPower = -100; +#preamblTransMax (0...10) = (3,4,5,6,7,8,10,20,50,100,200) + preambleTransMax = 7; +#powerRampingStep +# 0=dB0,1=dB2,2=dB4,3=dB6 + powerRampingStep = 2; +#ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR +#1=oneeighth,2=onefourth,3=half,4=one,5=two,6=four,7=eight,8=sixteen + ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR = 4; +#one (0..15) 4,8,12,16,...60,64 + ssb_perRACH_OccasionAndCB_PreamblesPerSSB = 15; +#ra_ContentionResolutionTimer +#(0..7) 8,16,24,32,40,48,56,64 + ra_ContentionResolutionTimer = 7; + rsrp_ThresholdSSB = 19; +#prach-RootSequenceIndex_PR +#1 = 839, 2 = 139 + prach_RootSequenceIndex_PR = 2; + prach_RootSequenceIndex = 1; + # SCS for msg1, can only be 15 for 30 kHz < 6 GHz, takes precendence over the one derived from prach-ConfigIndex + # + msg1_SubcarrierSpacing = 1, +# restrictedSetConfig +# 0=unrestricted, 1=restricted type A, 2=restricted type B + restrictedSetConfig = 0, + +# this is the offset between the last PRACH preamble power and the Msg3 PUSCH, 2 times the field value in dB + msg3_DeltaPreamble = 2; + p0_NominalWithGrant = -100; + +# pucch-ConfigCommon setup : +# pucchGroupHopping +# 0 = neither, 1= group hopping, 2=sequence hopping + pucchGroupHopping = 0; + hoppingId = 0; + p0_nominal = -96; + + ssb_PositionsInBurst_Bitmap = 0x1; + +# ssb_periodicityServingCell +# 0 = ms5, 1=ms10, 2=ms20, 3=ms40, 4=ms80, 5=ms160, 6=spare2, 7=spare1 + ssb_periodicityServingCell = 2; + +# dmrs_TypeA_position +# 0 = pos2, 1 = pos3 + dmrs_TypeA_Position = 0; + +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + subcarrierSpacing = 1; + + + #tdd-UL-DL-ConfigurationCommon +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + referenceSubcarrierSpacing = 1; + # pattern1 + # dl_UL_TransmissionPeriodicity + # 0=ms0p5, 1=ms0p625, 2=ms1, 3=ms1p25, 4=ms2, 5=ms2p5, 6=ms5, 7=ms10 + dl_UL_TransmissionPeriodicity = 5; + nrofDownlinkSlots = 3; + nrofDownlinkSymbols = 6; + nrofUplinkSlots = 1; + nrofUplinkSymbols = 4; + + ssPBCH_BlockPower = -11; + } + + ); + + + # ------- SCTP definitions + SCTP : + { + # Number of streams to use in input/output + SCTP_INSTREAMS = 2; + SCTP_OUTSTREAMS = 2; + }; + + + ////////// AMF parameters: + amf_ip_address = ({ ipv4 = "172.21.6.5"; }); + + NETWORK_INTERFACES : + { + GNB_IPV4_ADDRESS_FOR_NG_AMF = "172.21.16.209"; + GNB_IPV4_ADDRESS_FOR_NGU = "172.21.16.209"; + GNB_PORT_FOR_S1U = 2152; # Spec 2152 + }; + } +); + +MACRLCs = ( +{ + num_cc = 1; + tr_s_preference = "local_L1"; + tr_n_preference = "local_RRC"; + pusch_TargetSNRx10 = 220; + pucch_TargetSNRx10 = 220; + pusch_FailureThres = 100; +} +); + +L1s = ( +{ + num_cc = 1; + tr_n_preference = "local_mac"; + prach_dtx_threshold = 120 + pucch0_dtx_threshold = 80; + pusch_dtx_threshold = -100; + tx_amp_backoff_dB = 3; + L1_rx_thread_core = 8; + L1_tx_thread_core = 9; + max_ldpc_iterations = 20; + phase_compensation = 0; # needs to match O-RU configuration +} +); + +RUs = ( +{ + local_rf = "no"; + nb_tx = 4; + nb_rx = 4; + att_tx = 0; + att_rx = 0; + bands = [77]; + max_pdschReferenceSignalPower = -27; + max_rxgain = 75; + sf_extension = 0; + eNB_instances = [0]; + ru_thread_core = 10; + sl_ahead = 5; + tr_preference = "raw_if4p5"; # important: activate FHI7.2 + do_precoding = 0; # needs to match O-RU configuration +} +); + +security = { + # preferred ciphering algorithms + # the first one of the list that an UE supports in chosen + # valid values: nea0, nea1, nea2, nea3 + ciphering_algorithms = ( "nea0" ); + + # preferred integrity algorithms + # the first one of the list that an UE supports in chosen + # valid values: nia0, nia1, nia2, nia3 + integrity_algorithms = ( "nia2", "nia0" ); + + # setting 'drb_ciphering' to "no" disables ciphering for DRBs, no matter + # what 'ciphering_algorithms' configures; same thing for 'drb_integrity' + drb_ciphering = "yes"; + drb_integrity = "no"; +}; + +log_config : +{ + global_log_level = "info"; + hw_log_level = "info"; + phy_log_level = "info"; + mac_log_level = "info"; + rlc_log_level = "info"; + pdcp_log_level = "info"; + rrc_log_level = "info"; + ngap_log_level = "info"; + f1ap_log_level = "info"; +}; + +fhi_72 = { + dpdk_devices = ("0000:01:01.0"); + dpdk_iova_mode = "VA"; + system_core = 11; + io_core = 12; + worker_cores = (13); + ru_addr = ("98:ae:71:04:83:e3"); + mtu = 9600; + fh_config = ({ + T1a_cp_dl = (285, 470); + T1a_cp_ul = (285, 429); + T1a_up = (125, 350); + Ta4 = (110, 180); + ru_config = { + iq_width = 9; + iq_width_prach = 9; + }; + }); +}; + diff --git a/ci-scripts/conf_files/enb.band38.nsa_2x2.100prb.usrpn310.conf b/ci-scripts/conf_files/untested/enb.band38.nsa_2x2.100prb.usrpn310.conf similarity index 100% rename from ci-scripts/conf_files/enb.band38.nsa_2x2.100prb.usrpn310.conf rename to ci-scripts/conf_files/untested/enb.band38.nsa_2x2.100prb.usrpn310.conf diff --git a/ci-scripts/conf_files/enb.band7.tm1.50prb.usrpb210.conf b/ci-scripts/conf_files/untested/enb.band7.tm1.50prb.usrpb210.conf similarity index 100% rename from ci-scripts/conf_files/enb.band7.tm1.50prb.usrpb210.conf rename to ci-scripts/conf_files/untested/enb.band7.tm1.50prb.usrpb210.conf diff --git a/ci-scripts/conf_files/gnb.nsa.band78.106prb.usrpn310.2x2.conf b/ci-scripts/conf_files/untested/gnb.nsa.band78.106prb.usrpn310.2x2.conf similarity index 100% rename from ci-scripts/conf_files/gnb.nsa.band78.106prb.usrpn310.2x2.conf rename to ci-scripts/conf_files/untested/gnb.nsa.band78.106prb.usrpn310.2x2.conf diff --git a/ci-scripts/conf_files/gnb.sa.band78.106prb.usrp310.2x2.conf b/ci-scripts/conf_files/untested/gnb.sa.band78.106prb.usrp310.2x2.conf similarity index 100% rename from ci-scripts/conf_files/gnb.sa.band78.106prb.usrp310.2x2.conf rename to ci-scripts/conf_files/untested/gnb.sa.band78.106prb.usrp310.2x2.conf diff --git a/ci-scripts/conf_files/gnb.sa.band78.106prb.usrpn310.ddsuu-2x2.conf b/ci-scripts/conf_files/untested/gnb.sa.band78.106prb.usrpn310.ddsuu-2x2.conf similarity index 100% rename from ci-scripts/conf_files/gnb.sa.band78.106prb.usrpn310.ddsuu-2x2.conf rename to ci-scripts/conf_files/untested/gnb.sa.band78.106prb.usrpn310.ddsuu-2x2.conf diff --git a/ci-scripts/conf_files/gnb.sa.band78.162prb.usrpn310.2x2.conf b/ci-scripts/conf_files/untested/gnb.sa.band78.162prb.usrpn310.2x2.conf similarity index 100% rename from ci-scripts/conf_files/gnb.sa.band78.162prb.usrpn310.2x2.conf rename to ci-scripts/conf_files/untested/gnb.sa.band78.162prb.usrpn310.2x2.conf diff --git a/ci-scripts/datalog_rt_stats.100.4x4.fhi72.yaml b/ci-scripts/datalog_rt_stats.100.4x4.fhi72.yaml new file mode 100644 index 0000000000..b907b081c1 --- /dev/null +++ b/ci-scripts/datalog_rt_stats.100.4x4.fhi72.yaml @@ -0,0 +1,28 @@ +#this is a configuration file +#used to build real time processing statistics +#for 5G NR phy test (gNB terminate) +Title : Processing Time (us) from datalog_rt_stats.4x4.fhi72.yaml +ColNames : + - Metric + - Average; Max; Count + - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold) +Ref : + tx_fhaul : 40.2 + feptx_prec (per port, half_slot) : 12.3 + L1 Tx processing : 221.0 + DLSCH encoding : 141.0 + L1 Rx processing : 332.5 + UL segments decoding : 55.2 + UL Indication : 0.4 + Slot Indication : 14.3 + PUSCH inner-receiver : 421.4 +DeviationThreshold : + tx_fhaul : 0.25 + feptx_prec (per port, half_slot) : 0.25 + L1 Tx processing : 0.25 + DLSCH encoding : 0.25 + L1 Rx processing : 0.25 + UL segments decoding : 0.25 + UL Indication : 1.00 + Slot Indication : 0.50 + PUSCH inner-receiver : 0.25 diff --git a/ci-scripts/main.py b/ci-scripts/main.py index d86af05871..9e1401fcf0 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -62,13 +62,6 @@ import lxml.etree as ET import logging import signal import traceback -logging.basicConfig( - level=logging.DEBUG, - stream=sys.stdout, - format="[%(asctime)s] %(levelname)8s: %(message)s" -) - - #----------------------------------------------------------- @@ -109,41 +102,20 @@ def ExecuteActionWithParam(action, ctx, node): success = CONTAINERS.BuildRunTests(ctx, node, HTML) elif action == 'Initialize_eNB': - datalog_rt_stats_file=test.findtext('rt_stats_cfg') - if datalog_rt_stats_file is None: - RAN.datalog_rt_stats_file='datalog_rt_stats.default.yaml' - else: - RAN.datalog_rt_stats_file=datalog_rt_stats_file RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args') - USRPIPAddress = test.findtext('USRP_IPAddress') or '' - - #local variable air_interface - air_interface = test.findtext('air_interface') - if (air_interface is None) or (air_interface.lower() not in ['nr','lte']): - RAN.air_interface = 'lte-softmodem' - else: - RAN.air_interface = air_interface.lower() +'-softmodem' - cmd_prefix = test.findtext('cmd_prefix') if cmd_prefix is not None: RAN.cmd_prefix = cmd_prefix success = RAN.InitializeeNB(ctx, node, HTML) elif action == 'Terminate_eNB': - #retx checkers - string_field = test.findtext('d_retx_th') - if (string_field is not None): - RAN.ran_checkers['d_retx_th'] = [float(x) for x in string_field.split(',')] - string_field=test.findtext('u_retx_th') - if (string_field is not None): - RAN.ran_checkers['u_retx_th'] = [float(x) for x in string_field.split(',')] - - #local variable air_interface - air_interface = test.findtext('air_interface') - if (air_interface is None) or (air_interface.lower() not in ['nr','lte']): - RAN.air_interface = 'lte-softmodem' - else: - RAN.air_interface = air_interface.lower() +'-softmodem' - success = RAN.TerminateeNB(ctx, node, HTML) + services = [] + analysis = test.find("analysis") + if analysis is not None: + # services: multiple services to analyse, separated by whitespace + services = analysis.findtext("services", default="").split() + # service: individual services to analyze, in case they have whitespace + services = services + [s.text for s in analysis.findall("service")] + success = RAN.TerminateeNB(ctx, node, HTML, services) elif action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Terminate_UE' or action == 'CheckStatusUE' or action == 'DataEnable_UE' or action == 'DataDisable_UE': CiTestObj.ue_ids = test.findtext('id').split(' ') @@ -216,12 +188,6 @@ def ExecuteActionWithParam(action, ctx, node): elif action == 'Deploy_Object' or action == 'Undeploy_Object' or action == "Create_Workspace" or action == "Stop_Object": CONTAINERS.yamlPath = test.findtext('yaml_path') - string_field=test.findtext('d_retx_th') - if (string_field is not None): - CONTAINERS.ran_checkers['d_retx_th'] = [float(x) for x in string_field.split(',')] - string_field=test.findtext('u_retx_th') - if (string_field is not None): - CONTAINERS.ran_checkers['u_retx_th'] = [float(x) for x in string_field.split(',')] CONTAINERS.services = test.findtext('services') CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1) CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge) @@ -230,7 +196,14 @@ def ExecuteActionWithParam(action, ctx, node): elif action == 'Stop_Object': success = CONTAINERS.StopObject(ctx, node, HTML) elif action == 'Undeploy_Object': - success = CONTAINERS.UndeployObject(ctx, node, HTML, RAN) + analysis = test.find("analysis") + services = [] + if analysis is not None: + # services: multiple services to analyse, separated by whitespace + services = analysis.findtext("services", default="").split() + # service: individual services to analyze, in case they have whitespace + services = services + [s.text for s in analysis.findall("service")] + success = CONTAINERS.UndeployObject(ctx, node, HTML, services) elif action == 'Create_Workspace': if force_local: # Do not create a working directory when running locally. Current repo directory will be used @@ -280,21 +253,20 @@ def ExecuteActionWithParam(action, ctx, node): images = test.findtext('images').split() success = CLUSTER.PullClusterImage(HTML, node, images, tag_prefix=tag_prefix) + elif action == 'AnalyzeRTStats': + yaml = test.findtext('stats_cfg') + success = RAN.AnalyzeRTStats(HTML, node, ctx, yaml) + + elif action == 'AnalyzeRTStats_Object': + yaml = test.findtext('stats_cfg') + success = CONTAINERS.AnalyzeRTStatsObject(HTML, node, ctx, yaml) + else: logging.warning(f"unknown action {action}, skip step") success = True # by default, we skip the step and print a warning return success -#check if given test is in list -#it is in list if one of the strings in 'list' is at the beginning of 'test' -def test_in_list(test, list): - for check in list: - check=check.replace('+','') - if (test.startswith(check)): - return True - return False - test_runner_abort = False def receive_signal(signum, frame): global test_runner_abort @@ -349,7 +321,12 @@ CLUSTER = cls_cluster.Cluster() import args_parse # Force local execution, move all execution targets to localhost force_local = False -mode, force_local = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER) +mode, force_local, date_fmt = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER) +fmt = "%(levelname)8s: %(message)s" +if date_fmt: + fmt = "[%(asctime)s] %(levelname)s %(message)s" +logging.basicConfig(level=logging.DEBUG, stream=sys.stdout, format=fmt, datefmt=date_fmt,) + #----------------------------------------------------------- # mode amd XML class (action) analysis @@ -367,18 +344,7 @@ elif re.match('^TerminateSPGW$', mode, re.IGNORECASE): elif re.match('^LogCollectBuild$', mode, re.IGNORECASE): logging.warning("Option LogCollectBuild ignored") elif re.match('^LogCollecteNB$', mode, re.IGNORECASE): - if RAN.eNBSourceCodePath == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - if os.path.isdir('cmake_targets/log'): - cmd = 'zip -r enb.log.' + RAN.BuildId + '.zip cmake_targets/log' - logging.info(cmd) - try: - zipStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=60) - except subprocess.CalledProcessError as e: - logging.error("Command '{}' returned non-zero exit status {}.".format(e.cmd, e.returncode)) - logging.error("Error output:\n{}".format(e.output)) - sys.exit(0) + logging.warning("Option LogCollecteNB ignored") elif re.match('^LogCollectHSS$', mode, re.IGNORECASE): logging.warning("Option LogCollectHSS ignored") elif re.match('^LogCollectMME$', mode, re.IGNORECASE): diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 04384ad4f4..950b868ad3 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -31,21 +31,16 @@ #----------------------------------------------------------- # Import #----------------------------------------------------------- -import sys # arg import re # reg import logging import os import time -import yaml -import cls_cmd - #----------------------------------------------------------- # OAI Testing modules #----------------------------------------------------------- import cls_cmd -import helpreadme as HELP -import constants as CONST +import cls_analysis from cls_ci_helper import archiveArtifact #----------------------------------------------------------- @@ -63,16 +58,9 @@ class RANManagement(): self.eNBSourceCodePath = '' self.Initialize_eNB_args = '' self.imageKind = '' - self.air_interface = '' self.eNBOptions = ['', '', ''] - self.eNBmbmsEnables = [False, False, False] self.eNBstatuses = [-1, -1, -1] self.runtime_stats= '' - self.datalog_rt_stats={} - self.datalog_rt_stats_file='datalog_rt_stats.default.yaml' - self.USRPIPAddress = '' - #checkers from xml - self.ran_checkers={} self.cmd_prefix = '' # prefix before {lte,nr}-softmodem self.node = '' self.command = '' @@ -102,7 +90,7 @@ class RANManagement(): logfile = f'{lSourcePath}/cmake_targets/enb.log' cmd.cd(f"{lSourcePath}/cmake_targets/") # important: set wd so nrL1_stats.log etc are logged here - cmd.run(f'sudo -E stdbuf -o0 {self.cmd_prefix} {lSourcePath}/cmake_targets/ran_build/build/{self.air_interface} -O {lSourcePath}/{full_config_file} {extra_options} > {logfile} 2>&1 &') + cmd.run(f'sudo -E stdbuf -o0 {self.cmd_prefix} {lSourcePath}/cmake_targets/ran_build/build/nr-softmodem -O {lSourcePath}/{full_config_file} {extra_options} > {logfile} 2>&1 &') if extra_options != '': self.eNBOptions = extra_options @@ -116,12 +104,12 @@ class RANManagement(): enbDidSync = True break if not enbDidSync: - cmd.run(f'sudo killall -9 {self.air_interface}') # in case it did not stop automatically + cmd.run(f'sudo killall -9 nr-softmodem') # in case it did not stop automatically archiveArtifact(cmd, ctx, logfile) cmd.close() - msg = f'{self.cmd_prefix} {self.air_interface} -O {config_file} {extra_options}' + msg = f'{self.cmd_prefix} nr-softmodem -O {config_file} {extra_options}' if enbDidSync: logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m') HTML.CreateHtmlTestRowQueue(msg, 'OK', []) @@ -131,7 +119,7 @@ class RANManagement(): return enbDidSync - def TerminateeNB(self, ctx, node, HTML): + def TerminateeNB(self, ctx, node, HTML, to_analyze): logging.debug('Stopping eNB/gNB on server: ' + node) lSourcePath = self.eNBSourceCodePath cmd = cls_cmd.getConnection(node) @@ -146,14 +134,13 @@ class RANManagement(): if result is not None: cmd.run('sudo -S killall --signal SIGKILL -r .*-softmodem') time.sleep(5) + HTML.CreateHtmlTestRowQueue(node, 'OK', ['Undeployment successful']) # see InitializeeNB() logfile = f'{lSourcePath}/cmake_targets/enb.log' logdir = os.path.dirname(logfile) file = archiveArtifact(cmd, ctx, logfile) - archiveArtifact(cmd, ctx, f"{logdir}/nrL1_stats.log") - archiveArtifact(cmd, ctx, f"{logdir}/nrMAC_stats.log") cmd.close() if file is None: logging.debug('\u001B[1;37;41m Could not copy xNB logfile to analyze it! \u001B[0m') @@ -162,633 +149,21 @@ class RANManagement(): return False logging.debug('\u001B[1m Analyzing xNB logfile \u001B[0m ' + file) - logStatus = self.AnalyzeLogFile_eNB(file, HTML, self.ran_checkers) - if logStatus < 0: - HTML.CreateHtmlTestRow('N/A', 'KO', logStatus) - else: - HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) + service_desc = {} + service_desc["nr-softmodem"] = {'returncode': 0, 'logfile': file} + success = cls_analysis.AnalyzeServices(HTML, service_desc, to_analyze) + return success - #display rt stats for gNB only - if len(self.datalog_rt_stats) != 0: - HTML.CreateHtmlDataLogTable(self.datalog_rt_stats) + def AnalyzeRTStats(self, HTML, node, ctx, thresholds): + logging.info(f'Analyzing realtime stats from server: {node}') + lSourcePath = self.eNBSourceCodePath - return logStatus >= 0 + logdir = f'{lSourcePath}/cmake_targets' + with cls_cmd.getConnection(node) as cmd: + l1_file = archiveArtifact(cmd, ctx, f"{logdir}/nrL1_stats.log") + mac_file = archiveArtifact(cmd, ctx, f"{logdir}/nrMAC_stats.log") - def _analyzeUeRetx(self, rounds, checkers, regex): - if len(rounds) == 0 or len(checkers) == 0: - logging.warning(f'warning: rounds={rounds} checkers={checkers}') - return [] - - perc = list(0 for i in checkers) # results in % - stats = list(False for i in checkers) # status if succeeded - tmp = re.match(regex, rounds) - if tmp is None: - logging.error('_analyzeUeRetx: did not match regex for DL retx analysis') - return stats - retx_data = [float(x) for x in tmp.groups()] - for i in range(0, len(perc)): - #case where numerator > denumerator with denum ==0 is disregarded, cannot hapen in principle, will lead to 0% - perc[i] = 0 if (retx_data[i] == 0) else 100 * retx_data[i + 1] / retx_data[i] - #treating % > 100 , % > requirement - stats[i] = perc[i] <= 100 and perc[i] <= checkers[i] - return stats - - def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}): - if (not os.path.isfile(eNBlogFile)): - return -1 - enb_log_file = open(eNBlogFile, 'r') - exitSignalReceived = False - foundAssertion = False - msgAssertion = '' - msgLine = 0 - foundSegFault = False - foundRealTimeIssue = False - foundRealTimeIssue_cnt = 0 - rrcSetupComplete = 0 - rrcReleaseRequest = 0 - rrcReconfigRequest = 0 - rrcReconfigComplete = 0 - rrcReestablishRequest = 0 - rrcReestablishComplete = 0 - rrcReestablishReject = 0 - rlcDiscardBuffer = 0 - rachCanceledProcedure = 0 - uciStatMsgCount = 0 - pdcpFailure = 0 - ulschFailure = 0 - ulschAllocateCCEerror = 0 - uplinkSegmentsAborted = 0 - ulschReceiveOK = 0 - gnbRxTxWakeUpFailure = 0 - gnbTxWriteThreadEnabled = False - cdrxActivationMessageCount = 0 - dropNotEnoughRBs = 0 - mbmsRequestMsg = 0 - htmleNBFailureMsg = '' - isRRU = False - isSlave = False - slaveReceivesFrameResyncCmd = False - X2HO_state = CONST.X2_HO_REQ_STATE__IDLE - X2HO_inNbProcedures = 0 - X2HO_outNbProcedures = 0 - global_status = CONST.ALL_PROCESSES_OK - # Runtime statistics - runTime = '' - userTime = '' - systemTime = '' - maxPhyMemUsage = '' - nbContextSwitches = '' - #NSA FR1 check - NSA_RAPROC_PUSCH_check = 0 - #dlsch and ulsch statistics (dictionary) - dlsch_ulsch_stats = {} - #real time statistics (dictionary) - real_time_stats = {} - #count "problem receiving samples" msg - pb_receiving_samples_cnt = 0 - #count "removing UE" msg - removing_ue = 0 - #count"X2AP-PDU" - x2ap_pdu = 0 - #gnb specific log markers - gnb_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': [], 'Detected UL Failure on PUSCH':[]} - nodeB_prefix_found = False - RealTimeProcessingIssue = False - retx_status = {} - nrRrcRcfgComplete = 0 - harqFeedbackPast = 0 - showedByeMsg = False # last line is Bye. -> stopped properly - - line_cnt=0 #log file line counter - for line in enb_log_file.readlines(): - line_cnt+=1 - # Detection of eNB/gNB from a container log - result = re.search('Starting eNB soft modem', str(line)) - if result is not None: - nodeB_prefix_found = True - nodeB_prefix = 'e' - result = re.search('Starting gNB soft modem', str(line)) - if result is not None: - nodeB_prefix_found = True - nodeB_prefix = 'g' - result = re.search('Run time:' ,str(line)) - # Runtime statistics - result = re.search('Run time:' ,str(line)) - if result is not None: - runTime = str(line).strip() - if runTime != '': - result = re.search('Time executing user inst', str(line)) - if result is not None: - fields=line.split(':') - userTime = 'userTime : ' + fields[1].replace('\n','') - result = re.search('Time executing system inst', str(line)) - if result is not None: - fields=line.split(':') - systemTime = 'systemTime : ' + fields[1].replace('\n','') - result = re.search('Max. Phy. memory usage:', str(line)) - if result is not None: - fields=line.split(':') - maxPhyMemUsage = 'maxPhyMemUsage : ' + fields[1].replace('\n','') - result = re.search('Number of context switch.*process origin', str(line)) - if result is not None: - fields=line.split(':') - nbContextSwitches = 'nbContextSwitches : ' + fields[1].replace('\n','') - if X2HO_state == CONST.X2_HO_REQ_STATE__IDLE: - result = re.search('target eNB Receives X2 HO Req X2AP_HANDOVER_REQ', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ - result = re.search('source eNB receives the X2 HO ACK X2AP_HANDOVER_REQ_ACK', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK - if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ: - result = re.search('Received LTE_RRCConnectionReconfigurationComplete from UE', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE - if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE: - result = re.search('issue rrc_eNB_send_PATH_SWITCH_REQ', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ - if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ: - result = re.search('received path switch ack S1AP_PATH_SWITCH_REQ_ACK', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__IDLE - X2HO_inNbProcedures += 1 - if X2HO_state == CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK: - result = re.search('source eNB receives the X2 UE CONTEXT RELEASE X2AP_UE_CONTEXT_RELEASE', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__IDLE - X2HO_outNbProcedures += 1 - - if self.eNBOptions[0] != '': - res1 = re.search('max_rxgain (?P[0-9]+)', self.eNBOptions[0]) - res2 = re.search('max_rxgain (?P[0-9]+)', str(line)) - if res1 is not None and res2 is not None: - requested_option = int(res1.group('requested_option')) - applied_option = int(res2.group('applied_option')) - if requested_option == applied_option: - htmleNBFailureMsg += ' Command line option(s) correctly applied ' + self.eNBOptions[0] + '\n\n' - else: - htmleNBFailureMsg += ' Command line option(s) NOT applied ' + self.eNBOptions[0] + '\n\n' - result = re.search('Exiting OAI softmodem|Caught SIGTERM, shutting down', str(line)) - if result is not None: - exitSignalReceived = True - result = re.search('[Ss]egmentation [Ff]ault', str(line)) - if result is not None and not exitSignalReceived: - foundSegFault = True - result = re.search('[Cc]ore [dD]ump', str(line)) - if result is not None and not exitSignalReceived: - foundSegFault = True - result = re.search('[Aa]ssertion', str(line)) - if result is not None and not exitSignalReceived: - foundAssertion = True - result = re.search('LLL', str(line)) - if result is not None and not exitSignalReceived: - foundRealTimeIssue = True - foundRealTimeIssue_cnt += 1 - if foundAssertion and (msgLine < 3): - msgLine += 1 - msgAssertion += str(line) - result = re.search('Setting function for RU', str(line)) - if result is not None: - isRRU = True - if isRRU: - result = re.search('RU 0 is_slave=yes', str(line)) - if result is not None: - isSlave = True - if isSlave: - result = re.search('Received RRU_frame_resynch command', str(line)) - if result is not None: - slaveReceivesFrameResyncCmd = True - result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line)) - if result is not None: - rrcSetupComplete += 1 - result = re.search('Generate LTE_RRCConnectionRelease|Generate RRCConnectionRelease', str(line)) - if result is not None: rrcReleaseRequest += 1 - result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line)) - if result is not None: - rrcReconfigRequest += 1 - result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line)) - if result is not None: - rrcReconfigComplete += 1 - result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line)) - if result is not None: - rrcReestablishRequest += 1 - result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line)) - if result is not None: - rrcReestablishComplete += 1 - result = re.search('LTE_RRCConnectionReestablishmentReject', str(line)) - if result is not None: - rrcReestablishReject += 1 - result = re.search('CDRX configuration activated after RRC Connection', str(line)) - if result is not None: - cdrxActivationMessageCount += 1 - result = re.search('uci->stat', str(line)) - if result is not None: - uciStatMsgCount += 1 - result = re.search('PDCP.*Out of Resources.*reason', str(line)) - if result is not None: - pdcpFailure += 1 - result = re.search('could not wakeup gNB rxtx process', str(line)) - if result is not None: - gnbRxTxWakeUpFailure += 1 - result = re.search('tx write thread ready', str(line)) - if result is not None: - gnbTxWriteThreadEnabled = True - result = re.search('ULSCH in error in round|ULSCH 0 in error', str(line)) - if result is not None: - ulschFailure += 1 - result = re.search('ERROR ALLOCATING CCEs', str(line)) - if result is not None: - ulschAllocateCCEerror += 1 - result = re.search('uplink segment error.*aborted [1-9] segments', str(line)) - if result is not None: - uplinkSegmentsAborted += 1 - result = re.search('ULSCH received ok', str(line)) - if result is not None: - ulschReceiveOK += 1 - result = re.search('BAD all_segments_received', str(line)) - if result is not None: - rlcDiscardBuffer += 1 - result = re.search('Canceled RA procedure for UE rnti', str(line)) - if result is not None: - rachCanceledProcedure += 1 - result = re.search('dropping, not enough RBs', str(line)) - if result is not None: - dropNotEnoughRBs += 1 - if self.eNBmbmsEnables[0]: - result = re.search('MBMS USER-PLANE.*Requesting.*bytes from RLC', str(line)) - if result is not None: - mbmsRequestMsg += 1 - #FR1 NSA test : add new markers to make sure gNB is used - result = re.search('\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI 0x[0-9a-fA-F]+ received correctly, adding UE MAC Context RNTI 0x[0-9a-fA-F]+', str(line)) - if result is not None: - NSA_RAPROC_PUSCH_check = 1 - - # Collect information on UE DLSCH and ULSCH statistics - keys = {'dlsch_rounds','ulsch_rounds'} - for k in keys: - result = re.search(k, line) - if result is None: - continue - result = re.search('UE (?:RNTI )?([0-9a-f]{4})', line) - if result is None: - logging.error(f'did not find RNTI while matching key {k}') - continue - rnti = result.group(1) - - #remove 1- all useless char before relevant info (ulsch or dlsch) 2- trailing char - if not rnti in dlsch_ulsch_stats: dlsch_ulsch_stats[rnti] = {} - dlsch_ulsch_stats[rnti][k]=re.sub(r'^.*\]\s+', r'' , line.rstrip()) - - result = re.search('Received NR_RRCReconfigurationComplete from UE', str(line)) - if result is not None: - nrRrcRcfgComplete += 1 - result = re.search('HARQ feedback is in the past', str(line)) - if result is not None: - harqFeedbackPast += 1 - - - #count "problem receiving samples" msg - result = re.search('\[PHY\]\s+problem receiving samples', str(line)) - if result is not None: - pb_receiving_samples_cnt += 1 - #count "Removing UE" msg - result = re.search('\[MAC\]\s+Removing UE', str(line)) - if result is not None: - removing_ue += 1 - #count "X2AP-PDU" - result = re.search('X2AP-PDU', str(line)) - if result is not None: - x2ap_pdu += 1 - #gnb markers logging - for k in gnb_markers: - result = re.search(k, line) - if result is not None: - gnb_markers[k].append(line_cnt) - - # check whether e/gNB log finishes with "Bye." message - showedByeMsg |= re.search(r'^Bye.\n', str(line), re.MULTILINE) is not None - - enb_log_file.close() - - - #the following part takes the *_stats.log files as source (not the stdout log file) - - #the datalog config file has to be loaded - datalog_rt_stats_file=self.datalog_rt_stats_file - if (os.path.isfile(datalog_rt_stats_file)): - yaml_file=datalog_rt_stats_file - elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)): - yaml_file='ci-scripts/'+datalog_rt_stats_file - else: - logging.error("Datalog RT stats yaml file cannot be found") - sys.exit("Datalog RT stats yaml file cannot be found") - - with open(yaml_file,'r') as f: - datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader) - rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field - - # nrL1_stats.log/nrMAC_stats.log should be in the same directory as main log file - # currently the link is only implicit as below based on pattern matching - # I will rework this to give the file explicitly - l1_stats_fn = re.sub(r'-enb.log$', '-nrL1_stats.log', eNBlogFile) - mac_stats_fn = re.sub(r'-enb.log$', '-nrMAC_stats.log', eNBlogFile) - if os.path.isfile(l1_stats_fn) and os.path.isfile(mac_stats_fn): - # don't use CI-nrL1_stats.log, as this will increase the processing time for - # no reason, we just need the last occurence - nrL1_stats = open(l1_stats_fn, 'r') - nrMAC_stats = open(mac_stats_fn, 'r') - for line in nrL1_stats.readlines(): - for k in rt_keys: - result = re.search(k, line) - if result is not None: - #remove 1- all useless char before relevant info 2- trailing char - tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex - if tmp!=None: - real_time_stats[k]=tmp.group(1) - for line in nrMAC_stats.readlines(): - for k in rt_keys: - result = re.search(k, line) - if result is not None: - #remove 1- all useless char before relevant info 2- trailing char - tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex - if tmp!=None: - real_time_stats[k]=tmp.group(1) - nrL1_stats.close() - nrMAC_stats.close() - else: - logging.debug(f"NR Stats files for RT analysis not found: {l1_stats_fn}, {mac_stats_fn}") - - #stdout log file and stat log files analysis completed - logging.debug(' File analysis (stdout, stats) completed') - - #post processing depending on the node type - if not nodeB_prefix_found: - if self.air_interface == 'lte-softmodem': - nodeB_prefix = 'e' - else: - nodeB_prefix = 'g' - - if nodeB_prefix == 'g': - if ulschReceiveOK > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if gnbRxTxWakeUpFailure > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(gnbRxTxWakeUpFailure) + ' "could not wakeup gNB rxtx process" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if gnbTxWriteThreadEnabled: - statMsg = nodeB_prefix + 'NB ran with TX Write thread enabled' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if nrRrcRcfgComplete > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(nrRrcRcfgComplete) + ' "Received NR_RRCReconfigurationComplete from UE" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if harqFeedbackPast > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(harqFeedbackPast) + ' "HARQ feedback is in the past" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - #FR1 NSA test : add new markers to make sure gNB is used - if NSA_RAPROC_PUSCH_check: - statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : PASS ' - htmlMsg = statMsg+'\n' - else: - statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : FAIL or not relevant' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - #problem receiving samples log - statMsg = '[PHY] problem receiving samples msg count = '+str(pb_receiving_samples_cnt) - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - #gnb markers - statMsg = 'logfile line count = ' + str(line_cnt) - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - if len(gnb_markers['SgNBReleaseRequestAcknowledge'])!=0: - statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequestAcknowledge'][0]) - else: - statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - statMsg = 'FAILURE = ' + str(len(gnb_markers['FAILURE'])) + ' occurences' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - statMsg = 'Detected UL Failure on PUSCH = ' + str(len(gnb_markers['Detected UL Failure on PUSCH'])) + ' occurences' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - - #ulsch and dlsch statistics and checkers - for ue in dlsch_ulsch_stats: - dlulstat = dlsch_ulsch_stats[ue] - #print statistics into html - statMsg='' - for key in dlulstat: - statMsg += dlulstat[key] + '\n' - logging.debug(dlulstat[key]) - htmleNBFailureMsg += statMsg - - retx_status[ue] = {} - dlcheckers = [] if 'd_retx_th' not in checkers else checkers['d_retx_th'] - retx_status[ue]['dl'] = self._analyzeUeRetx(dlulstat['dlsch_rounds'], dlcheckers, r'^.*dlsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+dlsch_errors\s+(\d+)') - ulcheckers = [] if 'u_retx_th' not in checkers else checkers['u_retx_th'] - retx_status[ue]['ul'] = self._analyzeUeRetx(dlulstat['ulsch_rounds'], ulcheckers, r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+ulsch_errors\s+(\d+)') - - - #real time statistics - datalog_rt_stats['Data']={} - if len(real_time_stats)!=0: #check if dictionary is not empty - for k in real_time_stats: - tmp=re.match(r'^(?P.*):\s+(?P\d+\.\d+) us;\s+(?P\d+);\s+(?P\d+\.\d+) us;',real_time_stats[k]) - if tmp is not None: - metric=tmp.group('metric') - avg=float(tmp.group('avg')) - max=float(tmp.group('max')) - count=int(tmp.group('count')) - datalog_rt_stats['Data'][metric]=["{:.0f}".format(avg),"{:.0f}".format(max),"{:d}".format(count),"{:.2f}".format(avg/datalog_rt_stats['Ref'][metric])] - #once all metrics are collected, store the data as a class attribute to build a dedicated HTML table afterward - self.datalog_rt_stats=datalog_rt_stats - #check if there is a fail => will render the test as failed - for k in datalog_rt_stats['Data']: - valnorm = float(datalog_rt_stats['Data'][k][3]) - dev = datalog_rt_stats['DeviationThreshold'][k] - if valnorm > 1.0 + dev or valnorm < 1.0 - dev: # condition for fail : avg/ref deviates by more than "deviation threshold" - logging.debug(f'\u001B[1;30;43m normalized datalog_rt_stats metric {k}={valnorm} deviates by more than {dev}\u001B[0m') - RealTimeProcessingIssue = True - else: - statMsg = 'No real time stats found in the log file\n' - logging.debug('No real time stats found in the log file') - htmleNBFailureMsg += statMsg - - if not showedByeMsg: - logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB did not show "Bye." message at end, it likely did not stop properly! \u001B[0m') - htmleNBFailureMsg += 'No Bye. message found, did not stop properly\n' - global_status = CONST.ENB_SHUTDOWN_NO_BYE - else: - logging.debug('"Bye." message found at end.') - - else: - #Removing UE log - statMsg = '[MAC] Removing UE msg count = '+str(removing_ue) - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - #X2AP-PDU log - statMsg = 'X2AP-PDU msg count = '+str(x2ap_pdu) - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - #nsa markers - statMsg = 'logfile line count = ' + str(line_cnt) - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - if len(gnb_markers['SgNBReleaseRequest'])!=0: - statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequest'][0]) - else: - statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - statMsg = 'scgFailureInformationNR-r15 = ' + str(len(gnb_markers['scgFailureInformationNR-r15'])) + ' occurences' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - - for ue in retx_status: - msg = f"retransmissions for UE {ue}: DL {retx_status[ue]['dl']} UL {retx_status[ue]['ul']}" - if False in retx_status[ue]['dl'] or False in retx_status[ue]['ul']: - msg = 'Failure: ' + msg - logging.error(f'\u001B[1;37;41m {msg}\u001B[0m') - htmleNBFailureMsg += f'{msg}\n' - global_status = CONST.ENB_RETX_ISSUE - else: - logging.debug(msg) - - if RealTimeProcessingIssue: - logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with real time processing issue! \u001B[0m') - htmleNBFailureMsg += 'Fail due to real time processing issue\n' - global_status = CONST.ENB_REAL_TIME_PROCESSING_ISSUE - if uciStatMsgCount > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if pdcpFailure > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if ulschFailure > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if ulschAllocateCCEerror > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(ulschAllocateCCEerror) + ' "eNB_dlsch_ulsch_scheduler(); ERROR ALLOCATING CCEs" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if uplinkSegmentsAborted > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(uplinkSegmentsAborted) + ' "uplink segment error 0/2, aborted * segments" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if dropNotEnoughRBs > 0: - statMsg = 'eNB showed ' + str(dropNotEnoughRBs) + ' "dropping, not enough RBs" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if rrcSetupComplete > 0: - rrcMsg = nodeB_prefix + 'NB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if rrcReleaseRequest > 0: - rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if rrcReconfigRequest > 0 or rrcReconfigComplete > 0: - rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0: - rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if self.eNBmbmsEnables[0]: - if mbmsRequestMsg > 0: - rrcMsg = 'eNB requested ' + str(mbmsRequestMsg) + ' times the RLC for MBMS USER-PLANE' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if X2HO_inNbProcedures > 0: - rrcMsg = 'eNB completed ' + str(X2HO_inNbProcedures) + ' X2 Handover Connection procedure(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if X2HO_outNbProcedures > 0: - rrcMsg = 'eNB completed ' + str(X2HO_outNbProcedures) + ' X2 Handover Release procedure(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if self.eNBOptions[0] != '': - res1 = re.search('drx_Config_present prSetup', self.eNBOptions[0]) - if res1 is not None: - if cdrxActivationMessageCount > 0: - rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - else: - rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration' - logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if rachCanceledProcedure > 0: - rachMsg = nodeB_prefix + 'NB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)' - logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m') - htmleNBFailureMsg += rachMsg + '\n' - if isRRU: - if isSlave: - if slaveReceivesFrameResyncCmd: - rruMsg = 'Slave RRU received the RRU_frame_resynch command from RAU' - logging.debug('\u001B[1;30;43m ' + rruMsg + ' \u001B[0m') - htmleNBFailureMsg += rruMsg + '\n' - else: - rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU' - logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m') - htmleNBFailureMsg += rruMsg + '\n' - global_status = CONST.ENB_PROCESS_SLAVE_RRU_NOT_SYNCED - if foundSegFault: - logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with a Segmentation Fault! \u001B[0m') - global_status = CONST.ENB_PROCESS_SEG_FAULT - if foundAssertion: - logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with an assertion! \u001B[0m') - htmleNBFailureMsg += msgAssertion - global_status = CONST.ENB_PROCESS_ASSERTION - if foundRealTimeIssue: - logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB faced real time issues! \u001B[0m') - htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues! COUNT = '+ str(foundRealTimeIssue_cnt) +' lines\n' - if rlcDiscardBuffer > 0: - rlcMsg = nodeB_prefix + 'NB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)' - logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m') - htmleNBFailureMsg += rlcMsg + '\n' - global_status = CONST.ENB_PROCESS_REALTIME_ISSUE - HTML.htmleNBFailureMsg=htmleNBFailureMsg - # Runtime statistics for console output and HTML - if runTime != '': - logging.debug(runTime) - logging.debug(userTime) - logging.debug(systemTime) - logging.debug(maxPhyMemUsage) - logging.debug(nbContextSwitches) - self.runtime_stats='
'+runTime + '\n'+ userTime + '\n' + systemTime + '\n' + maxPhyMemUsage + '\n' + nbContextSwitches+'
' - return global_status + logging.info(f"check against thresholds from {thresholds}") + success, datalog_rt_stats = cls_analysis.Analysis.analyze_rt_stats(thresholds, l1_file, mac_file) + HTML.CreateHtmlDataLogTable(datalog_rt_stats) + return success diff --git a/ci-scripts/run_locally.sh b/ci-scripts/run_locally.sh index 6423378e9c..f01d263433 100755 --- a/ci-scripts/run_locally.sh +++ b/ci-scripts/run_locally.sh @@ -1,6 +1,5 @@ #!/bin/bash -set -e SHORT_COMMIT_SHA=$(git rev-parse --short=8 HEAD) COMMIT_SHA=$(git rev-parse HEAD) CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) @@ -13,6 +12,8 @@ if [ $# -eq 0 ] exit 1 fi +set -x + # The script assumes you've build the following images: # # docker build . -f docker/Dockerfile.gNB.ubuntu -t oai-gnb @@ -31,15 +32,15 @@ docker tag oai-nr-cuup oai-ci/oai-nr-cuup:develop-${SHORT_COMMIT_SHA} python3 main.py --mode=InitiateHtml --ranRepository=NONE --ranBranch=${CURRENT_BRANCH} \ --ranCommitID=${COMMIT_SHA} --ranAllowMerge=false \ --ranTargetBranch=NONE \ - --XMLTestFile=xml_files/${TESTCASE} --local + --XMLTestFile=xml_files/${TESTCASE} --local --datefmt="%H:%M:%S" python3 main.py --mode=TesteNB --ranRepository=NONE --ranBranch=${CURRENT_BRANCH} \ --ranCommitID=${COMMIT_SHA} --ranAllowMerge=false \ --ranTargetBranch=NONE \ --eNBSourceCodePath=${REPO_PATH} \ - --XMLTestFile=${TESTCASE} --local + --XMLTestFile=${TESTCASE} --local --datefmt="%H:%M:%S" RET=$? -python3 main.py --mode=FinalizeHtml --local +python3 main.py --mode=FinalizeHtml --local --datefmt="%H:%M:%S" exit ${RET} diff --git a/ci-scripts/tests/README.md b/ci-scripts/tests/README.md index f583c51c08..ac3aa2115f 100644 --- a/ci-scripts/tests/README.md +++ b/ci-scripts/tests/README.md @@ -10,12 +10,15 @@ directory, i.e., `ci-scripts/`, and run To run individual unit tests, start them like so: + python tests/analysis.py -v python tests/build.py -v python tests/cmd.py -v python tests/corenetwork.py -v python tests/deployment.py -v python tests/iperf-analysis.py -v + python tests/log-analysis.py -v python tests/ping-iperf.py -v + python tests/pull-clean-int-registry.py -v The logs will indicate if all tests passed. `tests/deployment.py` requires these images to be present: diff --git a/ci-scripts/tests/analysis.py b/ci-scripts/tests/analysis.py new file mode 100644 index 0000000000..e77bf14603 --- /dev/null +++ b/ci-scripts/tests/analysis.py @@ -0,0 +1,71 @@ +import sys +import logging +logging.basicConfig( + level=logging.DEBUG, + stream=sys.stdout, + format="[%(asctime)s] %(levelname)8s: %(message)s" +) +import os +import unittest +import yaml + +sys.path.append('./') # to find OAI imports below +import cls_analysis +import cls_oai_html +import constants + +class TestAnalysis(unittest.TestCase): + def setUp(self): + self.html = cls_oai_html.HTMLManagement() + self.html.testCaseId = "000000" + self.maxDiff = 10000 # to compare long files below + + def test_gnb_phytest(self): + rtsf = "datalog_rt_stats.100.2x2.yaml" + l1f = "tests/analysis/gnb_phytest.success.nrL1.log" + macf = "tests/analysis/gnb_phytest.success.nrMAC.log" + status, s = cls_analysis.Analysis.analyze_rt_stats(rtsf, l1f, macf) + self.assertTrue(status) + self.assertEqual(s['Title'], "Processing Time (us) from datalog_rt_stats.100.2x2.yaml") + self.assertEqual(s['ColNames'], ["Metric", "Average; Max; Count", "Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold)"]) + with open(rtsf, 'r') as f: + rt_stats = yaml.load(f, Loader=yaml.FullLoader) + # check that analyze_rt_stats() picked up all Ref/DeviationThreshold values from yaml file, e.g. + # s['Ref']['L1 Tx processing'] == corresponding value in yaml + # s['DeviationThreshold']['L1 Tx processing'] == corresponding value in yaml + for k, v in rt_stats['Ref'].items(): + self.assertEqual(s['Ref'][k], v) + for k, v in rt_stats['DeviationThreshold'].items(): + self.assertEqual(s['DeviationThreshold'][k], v) + for k, v in s['Ref'].items(): + self.assertEqual(v, rt_stats['Ref'][k]) + for k, v in s['DeviationThreshold'].items(): + self.assertEqual(v, rt_stats['DeviationThreshold'][k]) + self.assertEqual(s['Data']['L1 Tx processing'], ['476', '872', '102933', '1.00']) + self.assertEqual(s['Data']['DLSCH encoding'], ['216', '544', '77201', '1.08']) + self.assertEqual(s['Data']['L1 Rx processing'], ['488', '868', '38598', '0.92']) + self.assertEqual(s['Data']['UL Indication'], ['1', '10', '38598', '0.76']) + self.assertEqual(s['Data']['Slot Indication'], ['18', '94', '128667', '1.38']) + self.assertEqual(s['Data']['PUSCH inner-receiver'], ['307', '530', '25730', '0.85']) + self.assertEqual(s['Data']['feprx'], ['169', '310', '38600', '1.12']) + self.assertEqual(s['Data']['feptx_ofdm (per port, half_slot)'], ['56', '88', '102933', '0.98']) + self.assertEqual(s['Data']['feptx_total'], ['148', '299', '102933', '1.00']) + # TODO this one is not found, the CI does not recover this?? + #self.assertEqual(s['Data']['DL & UL scheduling timing'], ['17', '97', '159371', '1.14']) + + def test_analze_services_no_service(self): + service_desc = {} # nothing to analyze + to_analyze = ["testserv"] # some service requested to be analyze that does not exist + status = cls_analysis.AnalyzeServices(self.html, service_desc, to_analyze) + self.assertFalse(status) + + # TODO this does not work: we don't evaluate return codes yet + #def test_analze_service_exit_nonnull(self): + # service_desc = {} + # service_desc["testserv"] = {'returncode': 129, 'logfile': 'tests/log-analysis/empty.log'} + # to_analyze = ["testserv"] # some service requested to be analyze that does not exist + # status = cls_analysis.AnalyzeServices(self.html, service_desc, to_analyze) + # self.assertFalse(status) + +if __name__ == '__main__': + unittest.main() diff --git a/ci-scripts/tests/analysis/gnb_phytest.success.nrL1.log b/ci-scripts/tests/analysis/gnb_phytest.success.nrL1.log new file mode 100644 index 0000000000..066a71978b --- /dev/null +++ b/ci-scripts/tests/analysis/gnb_phytest.success.nrL1.log @@ -0,0 +1,37 @@ +Blacklisted PRBs 0/273 +-5.-6.-4.-4.-4.-6.-6.-6.-8.-6.-6.-4.-4.-4.-3.-6.-6.-11.-8.-8.-5.-4.-5.-4.-5. +-6.-11.-11.-8.-8.-5.-5.-4.-4.-6.-8.-11.-11.-8.-8.-6.-5.-6.-6.-8.-11.-11.-11.-8.-6. +-6.-6.-6.-11.-11.-11.-11.-11.-11.-6.-6.-6.-8.-3.-11.-11.-11.-8.-8.-6.-6.-6.-11.-11.-11. +-11.-8.-6.-5.-6.-6.-8.-11.-11.-11.-6.-6.-5.-4.-4.-5.-8.-8.-8.-6.-5.-4.-3.-3.-4. +-4.-6.-6.-5.-3.-3.-2.-2.-2.-3.-3.-4.-5.-5.-3.-2.-1.-1.-2.-5.-4.-4.-4.-3.-2. + 0.-1.-1.-1.-2.-2.-3.-2.-1. 0. 2.-11.13. 8. 7. 6. 5. 0. 0.-1. 0. 0. 0.-1.-3. +-3.-3.-3.-1.-1.-1.-1.-1.-1.-2.-3.-1.-1.-1.-1.-2.-1.-3.-4.-5.-8.-6.-3.-3.-3. +-3.-3.-5.-8.-8.-8.-6.-4.-4.-3.-4.-6.-6.-8.-8.-8.-6.-5.-6.-5.-6.-8.-11.-11.-11. +-8.-5.-5.-8.-6.-6.-8.-11.-11.-11.-8.-6.-6.-6.-8.-11.-11.-11.-8.-8.-6.-6.-5.-5.-8. +-11.-11.-8.-11.-6.-5.-5.-5.-5.-6.-11.-11.-8.-6.-3.-4.-5.-5.-6.-8.-6.-6.-5.-4.-4. +-4.-5.-6.-6.-11.-8.-6.-4.-3.-4.-3.-4.-5.-6.-8.-6.-5.-4.-3.-4.-3.-6.-5. +max_IO = 24 (137), min_I0 = 0 (17), avg_I0 = 11 dB(8.8.) +PRACH I0 = 0.0 dB +DLSCH RNTI 1234: current_Qm 6, current_RI 2, total_bytes TX -1130807110 +ULSCH RNTI 1234, 389: ulsch_power[0] 41,5 ulsch_noise_power[0] 7.8, sync_pos 0 + ulsch_power[1] 26.7, ulsch_noise_power[1] 7.8 + round_trials 6434(1.0e+00):6432(1.0e+00):6432(1.0e+00):6432, DTX 0, current_Qm 6, current_RI 1, total_bytes RX/SCHED 0/385630560 + L1 Tx job: 401.013 us; 128667; 895.809 us; + L1 Rx job: 183.883 us; 128667; 980.814 us; + L1 Tx processing: 476.247 us; 102933; 872.099 us; + DLSCH encoding: 216.371 us; 77201; 544.272 us; + DLSCH scrambling: 5.524 us; 77201; 19.192 us; + DLSCH modulation: 21.137 us; 77201; 36.428 us; + DLSCH resource mapping: 1.127 us; 1003613; 12.027 us; + DLSCH precoding: 3.667 us; 1003613; 17.411 us; + L1 Rx processing: 488.478 us; 38598; 867.672 us; +UL segment deinterleaving: 5.063 us; 488870; 27.005 us; + UL segment rate recovery: 11.554 us; 488870; 517.998 us; + UL segments decoding: 134.035 us; 488870; 759.164 us; + UL Indication: 0.763 us; 38598; 9.588 us; + Slot Indication: 17.891 us; 128667; 94.446 us; + PUSCH inner-receiver: 307.193 us; 25730; 529.580 us; + feprx: 169.305 us; 38600; 310.489 us; +feptx_ofdm (per port, half_slot): 56.093 us; 102933; 88.125 us; + feptx_total: 147.889 us; 102933; 299.307 us; + diff --git a/ci-scripts/tests/analysis/gnb_phytest.success.nrMAC.log b/ci-scripts/tests/analysis/gnb_phytest.success.nrMAC.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ci-scripts/tests/deployment.py b/ci-scripts/tests/deployment.py index 10bd2c843f..4fb466acc2 100644 --- a/ci-scripts/tests/deployment.py +++ b/ci-scripts/tests/deployment.py @@ -16,7 +16,6 @@ import cls_oai_html import cls_oaicitest import cls_containerize from cls_ci_helper import TestCaseCtx -import ran import cls_cmd class TestDeploymentMethods(unittest.TestCase): @@ -39,7 +38,6 @@ class TestDeploymentMethods(unittest.TestCase): self.html.testCaseId = "000000" self.ci = cls_oaicitest.OaiCiTest() self.cont = cls_containerize.Containerize() - self.ran = ran.RANManagement() self.cont.yamlPath = '' self.cont.ranAllowMerge = True self.cont.ranBranch = '' @@ -56,7 +54,7 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath = 'tests/simple-dep/' self.cont.deploymentTag = "noble" deploy = self.cont.DeployObject(self.ctx, self.node, self.html) - undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -79,7 +77,7 @@ class TestDeploymentMethods(unittest.TestCase): stopB = self.cont.StopObject(self.ctx, self.node, self.html) # should not undeploy anything (everything already stopped) self.cont.services = None - undeployAll = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + undeployAll = self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertTrue(deploy) self.assertFalse(stopC) self.assertTrue(stopA) @@ -92,7 +90,7 @@ class TestDeploymentMethods(unittest.TestCase): old = self.cont.yamlPath self.cont.yamlPath = 'tests/simple-fail/' deploy = self.cont.DeployObject(self.ctx, self.node, self.html) - self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertFalse(deploy) self.cont.yamlPath = old @@ -101,7 +99,7 @@ class TestDeploymentMethods(unittest.TestCase): old = self.cont.yamlPath self.cont.yamlPath = 'tests/simple-fail-2svc/' deploy = self.cont.DeployObject(self.ctx, self.node, self.html) - self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertFalse(deploy) self.cont.yamlPath = old @@ -110,7 +108,7 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.services = "oai-gnb" self.cont.deploymentTag = 'develop-12345678' deploy = self.cont.DeployObject(self.ctx, self.node, self.html) - undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -119,7 +117,7 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.services = "oai-gnb oai-nr-ue" self.cont.deploymentTag = 'develop-12345678' deploy = self.cont.DeployObject(self.ctx, self.node, self.html) - undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -130,7 +128,7 @@ class TestDeploymentMethods(unittest.TestCase): deploy1 = self.cont.DeployObject(self.ctx, self.node, self.html) self.cont.services = "oai-nr-ue" deploy2 = self.cont.DeployObject(self.ctx, self.node, self.html) - undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertTrue(deploy1) self.assertTrue(deploy2) self.assertTrue(undeploy) @@ -145,5 +143,37 @@ class TestDeploymentMethods(unittest.TestCase): cmd.run(f"rm -rf {self.cont.eNBSourceCodePath}") self.assertTrue(ws) + def test_undeploy_loganalysis(self): + self.cont.yamlPath = 'yaml_files/5g_rfsimulator_tdd_dora' + self.cont.services = "oai-gnb" + self.cont.deploymentTag = 'develop-12345678' + deploy = self.cont.DeployObject(self.ctx, self.node, self.html) + analyze = ["oai-gnb"] + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, analyze) + self.assertTrue(deploy) + self.assertTrue(undeploy) + + def test_undeploy_multi_loganalysis(self): + self.cont.yamlPath = 'yaml_files/5g_rfsimulator_tdd_dora' + self.cont.services = "oai-gnb" + self.cont.deploymentTag = 'develop-12345678' + deploy = self.cont.DeployObject(self.ctx, self.node, self.html) + analyze = ["oai-gnb", "oai-gnb=ContainsString=NOTFOUND"] + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, analyze) + self.assertTrue(deploy) + self.assertFalse(undeploy) + + # TODO this does not work: we don't evaluate return codes yet + #def test_undeploy_rc_nonnull(self): + # # fails reliably + # old = self.cont.yamlPath + # self.cont.yamlPath = 'tests/simple-fail/' + # deploy = self.cont.DeployObject(self.ctx, self.node, self.html) + # # should be false because wrong exit code + # undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, ["test"]) + # self.assertFalse(deploy) + # self.assertFalse(undeploy) + # self.cont.yamlPath = old + if __name__ == '__main__': unittest.main() diff --git a/ci-scripts/tests/log-analysis.py b/ci-scripts/tests/log-analysis.py new file mode 100644 index 0000000000..2df23483d3 --- /dev/null +++ b/ci-scripts/tests/log-analysis.py @@ -0,0 +1,131 @@ +import sys +import logging +logging.basicConfig( + level=logging.DEBUG, + stream=sys.stdout, + format="[%(asctime)s] %(levelname)8s: %(message)s" +) +import os +import unittest + +sys.path.append('./') # to find OAI imports below +import cls_loganalysis + +class TestLogAnalysis(unittest.TestCase): + def setUp(self): + self.maxDiff = None + + #def test_no_file(self): + # f = "/total/fantasy/path/file.log" + # result, _ = cls_loganalysis.Default.run(f, None) + # self.assertFalse(result) + + def test_no_error(self): + f = "tests/log-analysis/empty.log" + result, _ = cls_loganalysis.Default.run(f, None) + self.assertTrue(result) + + def test_retx_check_good(self): + f = "tests/log-analysis/retx-check-good.log" + opt = "dl=1,0,0,0;ul=1,0,0,0" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3 +UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 51.0 dB CCE fail 0""" + self.assertTrue(result) + self.assertEqual(log, expected) + + def test_retx_check_good_single_val(self): + f = "tests/log-analysis/retx-check-good.log" + opt = "1,0,0,0" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3 +UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 51.0 dB CCE fail 0""" + self.assertTrue(result) + self.assertEqual(log, expected) + + def test_retx_check_good_ul_only(self): + f = "tests/log-analysis/retx-check-good.log" + opt = "ul=1,0,0,0" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3 +UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 51.0 dB CCE fail 0""" + self.assertTrue(result) + self.assertEqual(log, expected) + + def test_retx_check_bad(self): + f = "tests/log-analysis/retx-check-bad.log" + opt = "dl=10,100,100,100;ul=10,100,100,100" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0 +UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 34.0 dB CCE fail 0 +UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64 +UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104 SNR 38.0 dB CCE fail 0 +!!! Failure: UE bf9b: retx rounds DL [1] UL [1] exceeded threshold +UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0 +UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5 SNR 29.0 dB CCE fail 0""" + self.assertFalse(result) + self.assertEqual(log, expected) + + def test_retx_check_dl_only_bad(self): + f = "tests/log-analysis/retx-check-bad.log" + opt = "dl=10,80,100,100" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0 +UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 34.0 dB CCE fail 0 +UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64 +UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104 SNR 38.0 dB CCE fail 0 +!!! Failure: UE bf9b: retx rounds DL [1, 2] exceeded threshold +UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0 +UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5 SNR 29.0 dB CCE fail 0""" + self.assertFalse(result) + self.assertEqual(log, expected) + + def test_retx_check_ul_bad(self): + f = "tests/log-analysis/retx-check-bad.log" + opt = "dl=100,100,100,100;ul=10,100,100,100" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0 +UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 34.0 dB CCE fail 0 +UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64 +UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104 SNR 38.0 dB CCE fail 0 +!!! Failure: UE bf9b: retx rounds UL [1] exceeded threshold +UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0 +UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5 SNR 29.0 dB CCE fail 0""" + self.assertFalse(result) + self.assertEqual(log, expected) + + def test_last_line_contains_false(self): + f = "tests/log-analysis/arbitrary.log" + opt = "Bye." + result, log = cls_loganalysis.LastLineContains.run(f, opt) + expected = f"could not find '{opt}' in last line" + self.assertFalse(result) + self.assertEqual(log, expected) + + def test_last_line_contains_false(self): + f = "tests/log-analysis/arbitrary.log" + opt = "got sync" + result, log = cls_loganalysis.LastLineContains.run(f, opt) + expected = "" + self.assertTrue(result) + self.assertEqual(log, expected) + + def test_ends_no_bye(self): + f = "tests/log-analysis/arbitrary.log" + opt = None + result, log = cls_loganalysis.EndsWithBye.run(f, opt) + expected = "No Bye. message found, did not stop properly" + self.assertFalse(result) + self.assertEqual(log, expected) + + def test_ends_with_bye(self): + f = "tests/log-analysis/with-bye.log" + opt = None + result, log = cls_loganalysis.EndsWithBye.run(f, opt) + expected = "" + self.assertTrue(result) + self.assertEqual(log, expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/ci-scripts/tests/log-analysis/arbitrary.log b/ci-scripts/tests/log-analysis/arbitrary.log new file mode 100644 index 0000000000..095690aeb3 --- /dev/null +++ b/ci-scripts/tests/log-analysis/arbitrary.log @@ -0,0 +1,9 @@ +[UTIL] threadCreate() for L1_rx_thread: creating thread (no affinity, default priority) +[UTIL] threadCreate() for L1_tx_thread: creating thread (no affinity, default priority) +[UTIL] threadCreate() for L1_stats: creating thread (no affinity, default priority) +TYPE TO TERMINATE +[PHY] got sync (ru_thread) +[PHY] RU 0 rf device ready +[PHY] RU 0 RF started cpu_meas_enabled 0 +[HW] No connected device, generating void samples... +[PHY] got sync (L1_stats_thread) diff --git a/ci-scripts/tests/log-analysis/empty.log b/ci-scripts/tests/log-analysis/empty.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ci-scripts/tests/log-analysis/retx-check-bad.log b/ci-scripts/tests/log-analysis/retx-check-bad.log new file mode 100644 index 0000000000..7651da78cf --- /dev/null +++ b/ci-scripts/tests/log-analysis/retx-check-bad.log @@ -0,0 +1,54 @@ +9185541.336575 [NR_RRC] I UE 1 Logical Channel DL-DCCH, Generate SecurityModeCommand (bytes 3) +9185541.356794 [NR_RRC] A [UL] (cellID bc614e, UE ID 1 RNTI a491) Received Security Mode Complete +9185541.357264 [NR_RRC] I UE 1: Logical Channel DL-DCCH, Generate NR UECapabilityEnquiry (bytes 8, xid 2) +9185541.373199 [NR_RRC] A [UL] (cellID bc614e, UE ID 1 RNTI a491) Received UE capabilities +9185541.373330 [NR_RRC] I Send message to ngap: NGAP_UE_CAPABILITIES_IND +9185541.373356 [NR_RRC] A [DL] (cellID bc614e, UE ID 1 RNTI a491) Send DL Information Transfer [53 bytes] +9185541.373368 [NR_RRC] I Send message to sctp: NGAP_InitialContextSetupResponse +9185541.390034 [NR_MAC] I Frame.Slot 384.0 +UE RNTI a491 CU-UE-ID 1 in-sync PH 59 dB PCMAX 21 dBm, average RSRP -101 (4 meas) +UE a491: CQI 2, RI 2, PMI (0,0) +UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0 +UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 34.0 dB CCE fail 0 +UE a491: MAC: TX 367 RX 3664 bytes +UE a491: LCID 1: TX 213 RX 1649 bytes + +9185559.310044 [NR_MAC] I Frame.Slot 128.0 +UE RNTI bf9b CU-UE-ID 1 in-sync PH 48 dB PCMAX 21 dBm +UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64 +UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104 SNR 38.0 dB CCE fail 0 +UE bf9b: MAC: TX 10145 RX 82 bytes +UE bf9b: LCID 1: TX 5283 RX 53 bytes + +9185582.350069 [NR_MAC] I Frame.Slot 384.0 +UE RNTI 257a CU-UE-ID 2 in-sync PH 48 dB PCMAX 21 dBm, average RSRP -83 (16 meas) +UE 257a: CQI 15, RI 2, PMI (0,1) +UE 257a: UL-RI 2, TPMI 0 +UE 257a: dlsch_rounds 153/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00000 MCS (1) 0 CCE fail 0 +UE 257a: ulsch_rounds 562/2/1/0, ulsch_errors 0, ulsch_DTX 3, BLER 0.00000 MCS (1) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 29.5 dB CCE fail 0 +UE 257a: MAC: TX 13416 RX 26519 bytes +UE 257a: LCID 1: TX 523 RX 2046 bytes +UE 257a: LCID 2: TX 0 RX 0 bytes +UE 257a: LCID 4: TX 9800 RX 9945 bytes + +9185687.310202 [NR_MAC] I Frame.Slot 640.0 +UE RNTI 257a CU-UE-ID 2 in-sync PH 61 dB PCMAX 21 dBm, average RSRP -83 (16 meas) +UE 257a: CQI 15, RI 2, PMI (0,1) +UE 257a: UL-RI 2, TPMI 0 +UE 257a: dlsch_rounds 64992/129/13/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00242 MCS (1) 27 CCE fail 0 +UE 257a: ulsch_rounds 39328/2610/8/0, ulsch_errors 0, ulsch_DTX 14, BLER 0.11139 MCS (1) 9 (Qm 4 deltaMCS 0 dB) NPRB 273 SNR 25.5 dB CCE fail 0 +UE 257a: MAC: TX 3344707955 RX 818603547 bytes +UE 257a: LCID 1: TX 832 RX 4312 bytes +UE 257a: LCID 2: TX 0 RX 0 bytes +UE 257a: LCID 4: TX 3322610250 RX 815642955 bytes + +9185692.430254 [NR_MAC] I Frame.Slot 128.0 +UE RNTI 257a CU-UE-ID 2 in-sync PH 48 dB PCMAX 21 dBm, average RSRP -84 (16 meas) +UE 257a: CQI 15, RI 2, PMI (0,1) +UE 257a: UL-RI 2, TPMI 0 +UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0 +UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5 SNR 29.0 dB CCE fail 0 +UE 257a: MAC: TX 3424886787 RX 879242724 bytes +UE 257a: LCID 1: TX 847 RX 4422 bytes +UE 257a: LCID 2: TX 0 RX 0 bytes +UE 257a: LCID 4: TX 3401067134 RX 875911216 bytes diff --git a/ci-scripts/tests/log-analysis/retx-check-good.log b/ci-scripts/tests/log-analysis/retx-check-good.log new file mode 100644 index 0000000000..c586c615f3 --- /dev/null +++ b/ci-scripts/tests/log-analysis/retx-check-good.log @@ -0,0 +1,8 @@ +7799905.793901 [NR_MAC] I Frame.Slot 256.0 +UE RNTI f40a CU-UE-ID 2 in-sync PH 48 dB PCMAX 20 dBm, average RSRP -44 (16 meas) +UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3 +UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 51.0 dB CCE fail 0 +UE f40a: MAC: TX 3836 RX 13684 bytes +UE f40a: LCID 1: TX 530 RX 281 bytes +UE f40a: LCID 2: TX 3 RX 35 bytes +UE f40a: LCID 4: TX 1886 RX 2100 bytes diff --git a/ci-scripts/tests/log-analysis/with-bye.log b/ci-scripts/tests/log-analysis/with-bye.log new file mode 100644 index 0000000000..22abbfab05 --- /dev/null +++ b/ci-scripts/tests/log-analysis/with-bye.log @@ -0,0 +1,3 @@ +Some other logs +Bye. +process exited. diff --git a/ci-scripts/xml_class_list.yml b/ci-scripts/xml_class_list.yml index e8fa42dcd3..2fd2787cfd 100755 --- a/ci-scripts/xml_class_list.yml +++ b/ci-scripts/xml_class_list.yml @@ -32,3 +32,5 @@ - Custom_Command - Custom_Script - Create_Workspace + - AnalyzeRTStats + - AnalyzeRTStats_Object diff --git a/ci-scripts/xml_files/container_4g_l2sim_tdd.xml b/ci-scripts/xml_files/container_4g_l2sim_tdd.xml index 1b3ec37b81..5483a9e4c7 100644 --- a/ci-scripts/xml_files/container_4g_l2sim_tdd.xml +++ b/ci-scripts/xml_files/container_4g_l2sim_tdd.xml @@ -129,6 +129,9 @@ localhost Undeploy all OAI 4G stack ci-scripts/yaml_files/4g_l2sim_fdd + + oai_enb=EndsWithBye oai_ue1 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml index adad131ef7..57295697e4 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml @@ -147,6 +147,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml index 0879e27170..ac057f1862 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml @@ -118,6 +118,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz_noS1 + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml index acde953fe8..aca8caf06b 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml @@ -147,6 +147,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_fdd_10MHz + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml index cf6eb1d838..619fd54bec 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml @@ -147,6 +147,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_fdd_20MHz + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fembms.xml b/ci-scripts/xml_files/container_4g_rfsim_fembms.xml index f03fb12e76..41708abab2 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fembms.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fembms.xml @@ -84,6 +84,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_fembms + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_mbms.xml b/ci-scripts/xml_files/container_4g_rfsim_mbms.xml index f41aa98a9f..457adb77b8 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_mbms.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_mbms.xml @@ -84,6 +84,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_mbms + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml index f57ff9eb14..16c61edaf0 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml @@ -146,6 +146,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_tdd_05MHz + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_5g_e1_rfsim.xml b/ci-scripts/xml_files/container_5g_e1_rfsim.xml index fc3be58e2d..6c8b3940b5 100644 --- a/ci-scripts/xml_files/container_5g_e1_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_e1_rfsim.xml @@ -86,8 +86,9 @@ localhost Undeploy all OAI 5G stack ci-scripts/yaml_files/5g_rfsimulator_e1 - 1,0,0,0 - 1,0,0,0 + + oai-cucp=EndsWithBye oai-cuup=EndsWithBye oai-cuup2=EndsWithBye oai-cuup3=EndsWithBye oai-du=RetxCheck=1,0,0,0 oai-du=EndsWithBye oai-du2=RetxCheck=1,0,0,0 oai-du2=EndsWithBye oai-du3=RetxCheck=1,0,0,0 oai-du3=EndsWithBye oai-nr-ue oai-nr-ue2 oai-nr-ue3 + diff --git a/ci-scripts/xml_files/container_5g_f1_rfsim.xml b/ci-scripts/xml_files/container_5g_f1_rfsim.xml index 7385daacfc..56df31e156 100644 --- a/ci-scripts/xml_files/container_5g_f1_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_f1_rfsim.xml @@ -311,8 +311,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_f1_rfsimulator - 30,100,100,100 - 30,100,100,100 + + oai-cu=EndsWithBye oai-du-pci0=RetxCheck=30,100,100,100 oai-du-pci0=EndsWithBye oai-du-pci1=RetxCheck=30,100,100,100 oai-du-pci1=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml index b4383feeaf..09d0e7784f 100644 --- a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml @@ -113,8 +113,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_fdd_rfsimulator - 1,0,0,0 - 1,0,0,0 + + oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_flexric_rfsim.xml b/ci-scripts/xml_files/container_5g_flexric_rfsim.xml index 78fb296697..985f2b623c 100644 --- a/ci-scripts/xml_files/container_5g_flexric_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_flexric_rfsim.xml @@ -197,6 +197,14 @@ localhost ci-scripts/yaml_files/5g_rfsimulator_flexric true + + nearRT-RIC=LastLineContains=Removing E2 Node + xapp-rc-moni=LastLineContains=Test xApp run SUCCESSFULLY + xapp-kpm-moni=LastLineContains=Test xApp run SUCCESSFULLY + xapp-kpm-rc=LastLineContains=Test xApp run SUCCESSFULLY + xapp-gtp-mac-rlc-pdcp-moni=LastLineContains=Test xApp run SUCCESSFULLY + oai-gnb=EndsWithBye oai-nr-ue oai-nr-ue2 + diff --git a/ci-scripts/xml_files/container_5g_rfsim.xml b/ci-scripts/xml_files/container_5g_rfsim.xml index 58199ab5d9..2bc96b7d8c 100644 --- a/ci-scripts/xml_files/container_5g_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_rfsim.xml @@ -243,8 +243,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator - 1,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue oai-nr-ue2 oai-nr-ue3 oai-nr-ue4 oai-nr-ue5 oai-nr-ue6 oai-nr-ue7 oai-nr-ue8 oai-nr-ue9 oai-nr-ue10 + diff --git a/ci-scripts/xml_files/container_5g_rfsim_24prb.xml b/ci-scripts/xml_files/container_5g_rfsim_24prb.xml index 1e7128a6e8..f7f677cb0d 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_24prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_24prb.xml @@ -257,6 +257,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_24prb + + oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_2x2.xml b/ci-scripts/xml_files/container_5g_rfsim_2x2.xml index 06cb9b63db..3c1c33db10 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_2x2.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_2x2.xml @@ -113,8 +113,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_2x2 - 1,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml b/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml index 4fbb0f6f7e..649663ea93 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml @@ -113,8 +113,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_fdd_phytest - 10,100,100,100 - 10,100,100,100 + + oai-gnb=RetxCheck=10,100,100,100 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_fr2_66prb.xml b/ci-scripts/xml_files/container_5g_rfsim_fr2_66prb.xml index b64837c839..cb15ffbcc0 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_fr2_66prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_fr2_66prb.xml @@ -89,8 +89,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_fr2_66prb - 10,0,0,0 - 10,0,0,0 + + oai-gnb=RetxCheck=10,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_multiue.xml b/ci-scripts/xml_files/container_5g_rfsim_multiue.xml index 83bd857c4a..4010e561bc 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_multiue.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_multiue.xml @@ -159,10 +159,11 @@ Undeploy_Object true Undeploy all OAI 5G stack - ci-scripts/yaml_files/5g_rfsimulator_multiue - 1,0,0,0 - 1,0,0,0 localhost + ci-scripts/yaml_files/5g_rfsimulator_multiue + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_n2_ho.xml b/ci-scripts/xml_files/container_5g_rfsim_n2_ho.xml index 9ef14478b1..915fa0e4e6 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_n2_ho.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_n2_ho.xml @@ -227,8 +227,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_n2_ho - 10,100,100,100 - 10,100,100,100 + + oai-cucp-0=EndsWithBye oai-cuup-0=EndsWithBye oai-du-0=RetxCheck=10,100,100,100 oai-du-0=EndsWithBye oai-nr-ue oai-cucp-1=EndsWithBye oai-cuup-1=EndsWithBye oai-du-1=RetxCheck=10,100,100,100 oai-du-1=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml b/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml index 7e291b86fb..3cb18aeb44 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml @@ -138,6 +138,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_ntn_geo + + oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml b/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml index cac6f5f4ee..97d2540ce6 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml @@ -138,6 +138,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_ntn_leo + + oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_prs.xml b/ci-scripts/xml_files/container_5g_rfsim_prs.xml index 2485e7d503..6e9500c388 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_prs.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_prs.xml @@ -74,8 +74,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_prs - 30,100,100,100 - 30,100,100,100 + + oai-gnb=RetxCheck=30,100,100,100 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_sidelink.xml b/ci-scripts/xml_files/container_5g_rfsim_sidelink.xml index f434bb600c..d4d7723915 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_sidelink.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_sidelink.xml @@ -60,21 +60,16 @@ 20 - - Custom_Command - Check that UE synched - localhost - docker logs rfsim5g-oai-nr-ue-2 | grep -m 1 "PSBCH RX:OK" - - Undeploy_Object true Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_sidelink - 1,0,0,0 - 1,0,0,0 + + oai-nr-ue-1 + oai-nr-ue-2=ContainsString=PSBCH RX:OK + diff --git a/ci-scripts/xml_files/container_5g_rfsim_simple.xml b/ci-scripts/xml_files/container_5g_rfsim_simple.xml index be4912d2b2..541a2f5a29 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_simple.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_simple.xml @@ -80,8 +80,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator - 1,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue oai-nr-ue2 oai-nr-ue3 oai-nr-ue4 oai-nr-ue5 oai-nr-ue6 oai-nr-ue7 oai-nr-ue8 oai-nr-ue9 oai-nr-ue10 + diff --git a/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml b/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml index 83209d664e..dbea7321ed 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml @@ -95,8 +95,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_tdd_dora - 0,0,0,0 - 0,0,0,0 + + oai-gnb=RetxCheck=0,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml b/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml index 060f539cde..ef84c97f7c 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml @@ -174,8 +174,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_u0_25prb - 1,0,0,0 - 1,0,0,0 + + oai-vnf=RetxCheck=0,0,0,0 oai-vnf=EndsWithBye oai-pnf=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_vrtsim_chanmod.xml b/ci-scripts/xml_files/container_5g_vrtsim_chanmod.xml index c844228640..9198261754 100644 --- a/ci-scripts/xml_files/container_5g_vrtsim_chanmod.xml +++ b/ci-scripts/xml_files/container_5g_vrtsim_chanmod.xml @@ -113,8 +113,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_vrtsim_chanmod - 1,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_vrtsim_chanmod_gh.xml b/ci-scripts/xml_files/container_5g_vrtsim_chanmod_gh.xml index 3d3a9a5e41..353bd68f05 100644 --- a/ci-scripts/xml_files/container_5g_vrtsim_chanmod_gh.xml +++ b/ci-scripts/xml_files/container_5g_vrtsim_chanmod_gh.xml @@ -114,8 +114,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_vrtsim_chanmod - 1,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_vrtsim_cirdb.xml b/ci-scripts/xml_files/container_5g_vrtsim_cirdb.xml index 84a6a527f7..fe08d34adf 100644 --- a/ci-scripts/xml_files/container_5g_vrtsim_cirdb.xml +++ b/ci-scripts/xml_files/container_5g_vrtsim_cirdb.xml @@ -114,8 +114,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_vrtsim_cirdb - 10,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=dl=10,0,0,0;ul=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1.xml index 614acee309..af703bad33 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1.xml @@ -183,6 +183,9 @@ Replaces xml_files/enb_usrp210_band7_test_05mhz_tm1.xml Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_05Mhz_tm1 + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_if4_5.xml b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_if4_5.xml index 243243658b..aa3598126c 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_if4_5.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_if4_5.xml @@ -165,6 +165,9 @@ Undeploy RCC/RRU hutch ci-scripts/yaml_files/lte_b200_fdd_05Mhz_if4.5 + + rru_fdd=EndsWithBye rcc_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_no_rrc_activity.xml b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_no_rrc_activity.xml index bd7400cb13..71fd42ccf4 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_no_rrc_activity.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_no_rrc_activity.xml @@ -156,6 +156,9 @@ Replaces xml_files/enb_usrp210_band7_test_05mhz_tm1_rrc_inactivity_no_flexran.xm Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_05Mhz_tm1_no_rrc_activity + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1.xml index 92f4376676..a0ea977254 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1.xml @@ -168,6 +168,9 @@ Replaces xml_files/enb_usrp210_band7_test_10mhz_tm1.xml Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_10Mhz_tm1 + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_cdrx.xml b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_cdrx.xml index 5e3b64e01b..868a80cec3 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_cdrx.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_cdrx.xml @@ -116,6 +116,9 @@ Replaces xml_files/enb_usrp210_band7_test_10mhz_tm1.xml Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_10Mhz_tm1_cdrx + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_oaiue.xml b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_oaiue.xml index 5133c17f14..c0b0f91027 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_oaiue.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_oaiue.xml @@ -133,6 +133,9 @@ Undeploy LTE-UE carabe ci-scripts/yaml_files/lte_b200_fdd_10Mhz_oai_ue_magma + + lte_ue0 + @@ -147,6 +150,9 @@ Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_10Mhz_tm1_magma + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_20Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_fdd_20Mhz_tm1.xml index cf092b2653..6a3bf52993 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_20Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_20Mhz_tm1.xml @@ -140,6 +140,9 @@ Replaces xml_files/enb_usrp210_band7_test_10mhz_tm1.xml Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_20Mhz_tm1 + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1.xml index 849d6e0140..b20740c0e7 100644 --- a/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1.xml @@ -194,6 +194,9 @@ Replaces xml_files/enb_usrp210_band40_test_05mhz_tm1.xml Undeploy eNB starsky ci-scripts/yaml_files/lte_b200_tdd_05Mhz_tm1 + + enb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1_if4_5.xml b/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1_if4_5.xml index 3b2e5a8022..8d75c79347 100644 --- a/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1_if4_5.xml +++ b/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1_if4_5.xml @@ -174,6 +174,9 @@ Undeploy RRU/RCC starsky ci-scripts/yaml_files/lte_b200_tdd_05Mhz_if4.5 + + rru_tdd=EndsWithBye rcc_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_tdd_10Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_tdd_10Mhz_tm1.xml index a947c6774c..74eb50fe06 100644 --- a/ci-scripts/xml_files/container_lte_b200_tdd_10Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_tdd_10Mhz_tm1.xml @@ -187,6 +187,9 @@ Replaces xml_files/enb_usrp210_band40_test_10mhz_tm1.xml Undeploy eNB starsky ci-scripts/yaml_files/lte_b200_tdd_10Mhz_tm1 + + enb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1.xml index 0888891cb3..1b1f007e0e 100644 --- a/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1.xml @@ -133,6 +133,9 @@ Replaces xml_files/enb_usrp210_band40_test_20mhz_tm1.xml Undeploy eNB starsky ci-scripts/yaml_files/lte_b200_tdd_20Mhz_tm1 + + enb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1_default_scheduler.xml b/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1_default_scheduler.xml index 636abb1a66..cbd196b353 100644 --- a/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1_default_scheduler.xml +++ b/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1_default_scheduler.xml @@ -189,6 +189,9 @@ Replaces xml_files/enb_usrp210_band40_test_20mhz_tm1_default_scheduler.xml Undeploy eNB starsky ci-scripts/yaml_files/lte_b200_tdd_20Mhz_tm1_default_scheduler + + enb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm1.xml b/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm1.xml index 39ac1a7df4..0eab1b9fc1 100644 --- a/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm1.xml +++ b/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm1.xml @@ -125,6 +125,9 @@ Undeploy gNB obelix ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1 + + oai-enb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm2.xml b/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm2.xml index e2b9f7ae13..41c3c4f4a7 100644 --- a/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm2.xml +++ b/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm2.xml @@ -125,6 +125,9 @@ Undeploy gNB obelix ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2 + + oai-enb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_nsa_b200_quectel.xml b/ci-scripts/xml_files/container_nsa_b200_quectel.xml index a9e859886c..c9b941949c 100644 --- a/ci-scripts/xml_files/container_nsa_b200_quectel.xml +++ b/ci-scripts/xml_files/container_nsa_b200_quectel.xml @@ -192,8 +192,9 @@ Undeploy gNB ofqot ci-scripts/yaml_files/nsa_b200_gnb - 10,100,100,100 - 10,100,100,100 + + gnb_mono_tdd=RetxCheck=10,100,100,100 gnb_mono_tdd=EndsWithBye + @@ -202,6 +203,9 @@ Undeploy eNB nepes ci-scripts/yaml_files/nsa_b200_enb + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_aerial_quectel.xml b/ci-scripts/xml_files/container_sa_aerial_quectel.xml index 7282513bb8..b5de982e5a 100644 --- a/ci-scripts/xml_files/container_sa_aerial_quectel.xml +++ b/ci-scripts/xml_files/container_sa_aerial_quectel.xml @@ -132,8 +132,9 @@ Undeploy gNB gracehopper1-oai ci-scripts/yaml_files/sa_gnb_aerial - 20,100,100,100 - 20,100,100,100 + + oai-gnb-aerial=RetxCheck=20,100,100,100 oai-gnb-aerial=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_aerial_quectel_ul_heavy.xml b/ci-scripts/xml_files/container_sa_aerial_quectel_ul_heavy.xml index 8367316e3a..9d826d2261 100644 --- a/ci-scripts/xml_files/container_sa_aerial_quectel_ul_heavy.xml +++ b/ci-scripts/xml_files/container_sa_aerial_quectel_ul_heavy.xml @@ -121,8 +121,9 @@ Undeploy gNB gracehopper1-oai ci-scripts/yaml_files/sa_gnb_aerial_ul - 20,100,100,100 - 20,100,100,100 + + oai-gnb-aerial=RetxCheck=20,100,100,100 oai-gnb-aerial=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_aw2s_asue.xml b/ci-scripts/xml_files/container_sa_aw2s_asue.xml index 6dfbc83187..dadfad3bf5 100644 --- a/ci-scripts/xml_files/container_sa_aw2s_asue.xml +++ b/ci-scripts/xml_files/container_sa_aw2s_asue.xml @@ -177,6 +177,9 @@ Undeploy gNB avra ci-scripts/yaml_files/sa_aw2s_gnb + + oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml b/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml index c7e5ab5721..19a668ff27 100644 --- a/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml +++ b/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml @@ -235,6 +235,9 @@ Undeploy gNB avra ci-scripts/yaml_files/sa_aw2s_2x2_gnb + + oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_b200_quectel.xml b/ci-scripts/xml_files/container_sa_b200_quectel.xml index cb64146091..2891a7b117 100644 --- a/ci-scripts/xml_files/container_sa_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_b200_quectel.xml @@ -229,8 +229,9 @@ Undeploy gNB ofqot ci-scripts/yaml_files/sa_b200_gnb - 10,100,100,100 - 10,100,100,100 + + gnb_mono_tdd=RetxCheck=10,100,100,100 gnb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml b/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml index ce9e1121c3..797c6fd853 100644 --- a/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml @@ -255,8 +255,9 @@ Undeploy CUCP/CU-UP/DU ofqot ci-scripts/yaml_files/sa_e1_b200_gnb - 10,100,100,100 - 10,100,100,100 + + gnb_cucp=EndsWithBye gnb_cuup=EndsWithBye gnb_du_tdd=RetxCheck=10,100,100,100 gnb_du_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml b/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml index 35103d01e7..875e7856c7 100644 --- a/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml @@ -263,8 +263,9 @@ Undeploy CU-DU ofqot ci-scripts/yaml_files/sa_f1_b200_gnb - 10,100,100,100 - 10,100,100,100 + + gnb_cu=EndsWithBye gnb_du_fdd=RetxCheck=10,100,100,100 gnb_du_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_f1_ho_b210_quectel.xml b/ci-scripts/xml_files/container_sa_f1_ho_b210_quectel.xml index 2804e3c14e..73e1375046 100644 --- a/ci-scripts/xml_files/container_sa_f1_ho_b210_quectel.xml +++ b/ci-scripts/xml_files/container_sa_f1_ho_b210_quectel.xml @@ -191,6 +191,9 @@ Undeploy CU-DU0 groot ci-scripts/yaml_files/5g_sa_f1_b210_ho + + oai-cu=EndsWithBye oai-du0=EndsWithBye + @@ -199,6 +202,9 @@ Undeploy DU1 rocket ci-scripts/yaml_files/5g_sa_f1_b210_ho + + oai-du1=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml b/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml index 3129883744..97037157b4 100644 --- a/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml +++ b/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml @@ -127,6 +127,9 @@ Undeploy gNB cacofonix ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb + + oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml b/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml index c02a12deac..d8548077fd 100644 --- a/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml +++ b/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml @@ -138,6 +138,9 @@ Undeploy gNB cacofonix ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb + + oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n2_ho_b210_quectel.xml b/ci-scripts/xml_files/container_sa_n2_ho_b210_quectel.xml index 1f4c7c1981..42390d0f2f 100644 --- a/ci-scripts/xml_files/container_sa_n2_ho_b210_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n2_ho_b210_quectel.xml @@ -160,6 +160,9 @@ Undeploy Monolithic gNB-PCI0 groot ci-scripts/yaml_files/5g_sa_n2_ho_b210 + + oai-gnb-pci0=EndsWithBye + @@ -168,6 +171,9 @@ Undeploy Monolithic gNB-PCI1 rocket ci-scripts/yaml_files/5g_sa_n2_ho_b210 + + oai-gnb-pci1=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml index 0c3cd35b2f..6823b553c4 100644 --- a/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml @@ -123,8 +123,9 @@ Undeploy gNB matix ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz - 10,100,100,100 - 10,100,100,100 + + oai-gnb=RetxCheck=10,100,100,100 oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml index 5d1b3dcd95..dd4d6d9e96 100644 --- a/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml @@ -123,8 +123,9 @@ Undeploy gNB matix ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz - 15,100,100,100 - 20,100,100,100 + + oai-gnb=RetxCheck=dl=15,100,100,100;ul=20,100,100,100 oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml index 45d105de9e..694a6aa288 100644 --- a/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml @@ -123,8 +123,9 @@ Undeploy VNF-PNF matix ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz - 10,100,100,100 - 10,100,100,100 + + oai-vnf=RetxCheck=10,100,100,100 oai-vnf=EndsWithBye oai-pnf=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n310_nrue.xml b/ci-scripts/xml_files/container_sa_n310_nrue.xml index 64311b06cd..1db35faa6b 100644 --- a/ci-scripts/xml_files/container_sa_n310_nrue.xml +++ b/ci-scripts/xml_files/container_sa_n310_nrue.xml @@ -125,6 +125,9 @@ Undeploy nr UE caracal ci-scripts/yaml_files/5g_sa_n310_nrue + + oai-nr-ue + @@ -133,8 +136,9 @@ Undeploy gNB avra ci-scripts/yaml_files/5g_sa_n310_gnb - 20,100,100,100 - 20,100,100,100 + + oai-gnb=RetxCheck=20,100,100,100 oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n310_nrue_longrun.xml b/ci-scripts/xml_files/container_sa_n310_nrue_longrun.xml index 6241a9d78d..987f8355a5 100644 --- a/ci-scripts/xml_files/container_sa_n310_nrue_longrun.xml +++ b/ci-scripts/xml_files/container_sa_n310_nrue_longrun.xml @@ -205,6 +205,9 @@ Undeploy nr UE caracal ci-scripts/yaml_files/5g_sa_n310_nrue + + oai-nr-ue + @@ -213,8 +216,9 @@ Undeploy gNB avra ci-scripts/yaml_files/5g_sa_n310_gnb - 20,100,100,100 - 20,100,100,100 + + oai-gnb=RetxCheck=20,100,100,100 oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml b/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml index 82cb032372..d81c855f39 100644 --- a/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml @@ -183,8 +183,9 @@ Undeploy gNB ofqot ci-scripts/yaml_files/sa_sc_b200_gnb - 10,100,100,100 - 10,100,100,100 + + gnb_mono_tdd=RetxCheck=10,100,100,100 gnb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/gnb_phytest_fhi7.2_docker.xml b/ci-scripts/xml_files/gnb_phytest_fhi7.2_docker.xml new file mode 100644 index 0000000000..5386ba4d83 --- /dev/null +++ b/ci-scripts/xml_files/gnb_phytest_fhi7.2_docker.xml @@ -0,0 +1,88 @@ + + + gNB-PHY-Test-100-4x4-FHI72 + Timing phytest 100 MHz 4x4 FHI7.2 + tasks + + + Custom_Script + Setup sriov and network interfaces for dummy (no RU) + bulbul + + + + + Pull_Local_Registry + Pull Images from Local Registry + bulbul + oai-gnb-fhi72 + + + + Create_Workspace + Create new Workspace + bulbul + + + + Deploy_Object + Initialize gNB FHI7.2 + bulbul + ci-scripts/yaml_files/phytest_fhi72 + oai-gnb + + + + IdleSleep + Sleep + 60 + + + + AnalyzeRTStats_Object + Analyze Real-Time Stats + datalog_rt_stats.100.4x4.fhi72.yaml + bulbul + + + + Undeploy_Object + bulbul + true + Terminate gNB + ci-scripts/yaml_files/phytest_fhi72 + + oai-gnb=EndsWithBye + + + + + Clean_Test_Server_Images + true + Clean Test Images on Test Server + bulbul + oai-gnb-fhi72 + + + diff --git a/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml b/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml index 30f307612c..6af1753555 100644 --- a/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml @@ -37,7 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.273prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 273 -t 23 -D 127 -m 23 -M 273 -l 2 --device.name rf_emulator --rf_emulator.noise_level_dBFS -36 --log_config.global_log_options level,nocolor,time - datalog_rt_stats.100.2x2.yaml NR numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -48,12 +47,21 @@ 60 + + AnalyzeRTStats + Analyze Real-Time Stats + datalog_rt_stats.100.2x2.yaml + caracal + + Terminate_eNB caracal true Terminate gNB - NR + + nr-softmodem=EndsWithBye + diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml index f6af5ee485..fcc88f9c04 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml @@ -37,9 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.106prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 106 -t 23 -D 127 -m 28 -M 106 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - datalog_rt_stats.default.yaml - NR - 192.168.20.2 numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -49,12 +46,21 @@ 60 + + AnalyzeRTStats + Analyze Real-Time Stats + datalog_rt_stats.default.yaml + caracal + + Terminate_eNB true Terminate gNB caracal - NR + + nr-softmodem=EndsWithBye + diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml index 93053dee90..a4b57f0942 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml @@ -37,9 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.273prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 273 -t 23 -D 127 -m 23 -M 273 -l 2 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - datalog_rt_stats.100.2x2.yaml - NR - 192.168.20.2 numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -49,12 +46,21 @@ 60 + + AnalyzeRTStats + Analyze Real-Time Stats + datalog_rt_stats.100.2x2.yaml + caracal + + Terminate_eNB true Terminate gNB caracal - NR + + nr-softmodem=EndsWithBye + diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml index 2cd040a47a..098e881248 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml @@ -37,9 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.162prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 162 -t 23 -D 127 -m 23 -M 162 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - datalog_rt_stats.1x1.60.yaml - NR - 192.168.20.2 numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -49,12 +46,21 @@ 60 + + AnalyzeRTStats + Analyze Real-Time Stats + datalog_rt_stats.1x1.60.yaml + caracal + + Terminate_eNB true Terminate gNB caracal - NR + + nr-softmodem=EndsWithBye + diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml index 7e5592fb6e..0075201dcf 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml @@ -37,9 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.162prb.usrpn300.phytest-dora.conf --phy-test --gNBs.[0].pdsch_AntennaPorts_XP 2 --RUs.[0].nb_tx 2 --RUs.[0].nb_rx 2 -q -U 768 -T 162 -t 23 -D 127 -m 23 -M 162 -l 2 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - datalog_rt_stats.60.2x2.yaml - NR - 192.168.20.2 numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -49,12 +46,21 @@ 60 + + AnalyzeRTStats + Analyze Real-Time Stats + datalog_rt_stats.60.2x2.yaml + caracal + + Terminate_eNB true Terminate gNB caracal - NR + + nr-softmodem=EndsWithBye + diff --git a/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml index 487672ae1f..91597197ca 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml @@ -10,7 +10,7 @@ services: - IPC_LOCK environment: TZ: Europe/Paris - USE_ADDITIONAL_OPTIONS: --nfapi VNF + USE_ADDITIONAL_OPTIONS: --nfapi VNF --log_config.global_log_options level,nocolor,utc_time NFAPI_TRACE_LEVEL: info volumes: - ../../conf_files/gnb-vnf.sa.band77.162prb.nfapi.4x4.conf:/opt/oai-gnb/etc/gnb.conf @@ -30,7 +30,7 @@ services: - IPC_LOCK environment: TZ: Europe/Paris - USE_ADDITIONAL_OPTIONS: --nfapi PNF + USE_ADDITIONAL_OPTIONS: --nfapi PNF --log_config.global_log_options level,nocolor,utc_time NFAPI_TRACE_LEVEL: info volumes: - ../../conf_files/gnb-pnf.band77.usrpn310.4x4.conf:/opt/oai-gnb/etc/gnb.conf diff --git a/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml b/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml deleted file mode 100644 index 7963f823b1..0000000000 --- a/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml +++ /dev/null @@ -1,27 +0,0 @@ -services: - enb_mono_fdd: - image: ${REGISTRY-oaisoftwarealliance/}oai-enb:${TAG:-develop} - privileged: true - container_name: prod-enb-mono-fdd - environment: - USE_B2XX: 'yes' - USE_ADDITIONAL_OPTIONS: --RUs.[0].max_rxgain 115 --RUs.[0].max_pdschReferenceSignalPower -27 --eNBs.[0].component_carriers.[0].pucch_p0_Nominal -96 --log_config.global_log_options level,nocolor,time - volumes: - - ../../conf_files/enb.band7.25prb.usrpb200.conf:/opt/oai-enb/etc/enb.conf - networks: - public_net: - ipv4_address: 192.168.61.30 - healthcheck: - # pgrep does NOT work - test: /bin/bash -c "ps aux | grep -v grep | grep -c softmodem" - interval: 10s - timeout: 5s - retries: 5 - -networks: - public_net: - name: prod-oai-public-net - ipam: - config: - - subnet: 192.168.61.0/26 - diff --git a/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1/docker-compose.yml b/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1/docker-compose.yml index 0aefaf5ef0..0cc07d308e 100644 --- a/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1/docker-compose.yml +++ b/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1/docker-compose.yml @@ -5,7 +5,7 @@ services: container_name: oai-enb environment: TZ: Europe/Paris - USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 + USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 --log_config.global_log_options level,nocolor,utc_time volumes: - ../../conf_files/enb.band38.lte_2x2.100prb.usrpn310.conf:/opt/oai-enb/etc/enb.conf network_mode: "host" diff --git a/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2/docker-compose.yml b/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2/docker-compose.yml index db6215a148..94ceec5723 100644 --- a/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2/docker-compose.yml +++ b/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2/docker-compose.yml @@ -5,7 +5,7 @@ services: container_name: oai-enb environment: TZ: Europe/Paris - USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 + USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 --log_config.global_log_options level,nocolor,utc_time volumes: - ../../conf_files/enb.band38.lte_2x2_tm2.100prb.usrpn310.conf:/opt/oai-enb/etc/enb.conf network_mode: "host" diff --git a/ci-scripts/yaml_files/phytest_fhi72/docker-compose.yaml b/ci-scripts/yaml_files/phytest_fhi72/docker-compose.yaml new file mode 100644 index 0000000000..46fbc24302 --- /dev/null +++ b/ci-scripts/yaml_files/phytest_fhi72/docker-compose.yaml @@ -0,0 +1,22 @@ +services: + oai-gnb: + image: ${REGISTRY-oaisoftwarealliance/}${GNB_IMG:-oai-gnb-fhi72}:${TAG:-develop} + container_name: oai-gnb-phytest + cap_add: + - IPC_LOCK + - SYS_NICE + cap_drop: + - ALL + environment: + USE_ADDITIONAL_OPTIONS: --phy-test -q -M 273 -m 27 -l 4 -D 231 -T 273 -t 16 -L 2 -U 528 --log_config.global_log_options level,nocolor,time --thread-pool 16,17,18,19,20,21,22,23 + devices: + - /dev/vfio:/dev/vfio/ + volumes: + - ../../conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf:/opt/oai-gnb/etc/gnb.conf + - /dev/hugepages:/dev/hugepages + cpuset: "8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23" + healthcheck: + test: /bin/bash -c "pgrep nr-softmodem" + interval: 10s + timeout: 5s + retries: 5 diff --git a/ci-scripts/yaml_files/phytest_fhi72/setup_sriov_dummy.sh b/ci-scripts/yaml_files/phytest_fhi72/setup_sriov_dummy.sh new file mode 100755 index 0000000000..c18affb0a9 --- /dev/null +++ b/ci-scripts/yaml_files/phytest_fhi72/setup_sriov_dummy.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +pci_addr() +{ + PF_IF=$1 + VF_INDEX=$2 + SYSFS_PATH="/sys/class/net/${PF_IF}/device/virtfn${VF_INDEX}" + + if [ ! -e "$SYSFS_PATH" ]; then + echo "VF $VF_INDEX not found for interface $PF_IF" + exit 1 + fi + PCI_ADDR=$(basename "$(readlink "$SYSFS_PATH")") + echo "$PCI_ADDR" +} +IF_NAME=ens1f0np0 +## O-DU C Plane MAC ADDR and VLAN +C_U_PLANE_MAC_ADD=22:11:55:33:99:55 +C_U_PLANE_VLAN=123 +MTU=9000 +DPDK_DEVBIND_PREFIX=/home/oaicicd/test/dpdk-stable-22.11.7/bin +NUM_VFs=1 +sudo ethtool -G $IF_NAME rx 8160 tx 8160 +sudo sh -c "echo 0 > /sys/class/net/$IF_NAME/device/sriov_numvfs" +sudo sh -c "echo $NUM_VFs > /sys/class/net/$IF_NAME/device/sriov_numvfs" +C_U_PLANE_PCI=$(pci_addr $IF_NAME 0) +# this next 2 lines is for C/U planes +sudo ip link set $IF_NAME vf 0 mac $C_U_PLANE_MAC_ADD vlan $C_U_PLANE_VLAN spoofchk off mtu $MTU +sleep 1 +sudo modprobe iavf +sudo ${DPDK_DEVBIND_PREFIX}/dpdk-devbind.py --unbind $C_U_PLANE_PCI +sudo modprobe vfio-pci +sudo ${DPDK_DEVBIND_PREFIX}/dpdk-devbind.py --bind vfio-pci $C_U_PLANE_PCI +echo "Successfully configured C-PLANE and U-PLANE: + - C-PLANE MAC: $C_U_PLANE_MAC_ADD, VLAN: $C_U_PLANE_VLAN, PCI: $C_U_PLANE_PCI" +exit 0 + diff --git a/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml index 54bf6cec00..bc1d20a984 100644 --- a/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml @@ -12,6 +12,7 @@ services: core: -1 # for core dumps environment: TZ: Europe/Paris + USE_ADDITIONAL_OPTIONS: --log_config.global_log_options level,nocolor,utc_time volumes: - ../../conf_files/gnb.sa.band78.51prb.aw2s.ddsuu.conf:/opt/oai-gnb-aw2s/etc/gnb.conf healthcheck: diff --git a/executables/nr-uesoftmodem.c b/executables/nr-uesoftmodem.c index 26ce3860c8..1631332e5e 100644 --- a/executables/nr-uesoftmodem.c +++ b/executables/nr-uesoftmodem.c @@ -500,6 +500,7 @@ int main(int argc, char **argv) time_manager_finish(); free(pckg); + printf("Bye.\n"); return 0; }