From 90bf723eae568b5fc71f6202149a8e0ab9ac3331 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 24 Feb 2026 08:22:53 +0100 Subject: [PATCH 01/25] nr-uesoftmodem: end with Bye. Similarly to gNB, eNB, lteUE, end with a "Bye." that the CI (will) check for. The right way to do this would be to use the exit code, but most executables do not actually end with 0, such that we need to check for this. Thus, this commit is only to harmonize with other executables. --- executables/nr-uesoftmodem.c | 1 + 1 file changed, 1 insertion(+) diff --git a/executables/nr-uesoftmodem.c b/executables/nr-uesoftmodem.c index a0e06850b7..d19bd20da0 100644 --- a/executables/nr-uesoftmodem.c +++ b/executables/nr-uesoftmodem.c @@ -500,6 +500,7 @@ int main(int argc, char **argv) time_manager_finish(); free(pckg); + printf("Bye.\n"); return 0; } From e1b7e91f8a37eebaa9a43a3a49999946eb69857f Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 26 Feb 2026 10:26:56 +0100 Subject: [PATCH 02/25] Use nocolor in various CI pipelines --- ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml | 4 ++-- ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1/docker-compose.yml | 2 +- ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2/docker-compose.yml | 2 +- ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml index 487672ae1f..91597197ca 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml @@ -10,7 +10,7 @@ services: - IPC_LOCK environment: TZ: Europe/Paris - USE_ADDITIONAL_OPTIONS: --nfapi VNF + USE_ADDITIONAL_OPTIONS: --nfapi VNF --log_config.global_log_options level,nocolor,utc_time NFAPI_TRACE_LEVEL: info volumes: - ../../conf_files/gnb-vnf.sa.band77.162prb.nfapi.4x4.conf:/opt/oai-gnb/etc/gnb.conf @@ -30,7 +30,7 @@ services: - IPC_LOCK environment: TZ: Europe/Paris - USE_ADDITIONAL_OPTIONS: --nfapi PNF + USE_ADDITIONAL_OPTIONS: --nfapi PNF --log_config.global_log_options level,nocolor,utc_time NFAPI_TRACE_LEVEL: info volumes: - ../../conf_files/gnb-pnf.band77.usrpn310.4x4.conf:/opt/oai-gnb/etc/gnb.conf diff --git a/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1/docker-compose.yml b/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1/docker-compose.yml index 0aefaf5ef0..0cc07d308e 100644 --- a/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1/docker-compose.yml +++ b/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1/docker-compose.yml @@ -5,7 +5,7 @@ services: container_name: oai-enb environment: TZ: Europe/Paris - USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 + USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 --log_config.global_log_options level,nocolor,utc_time volumes: - ../../conf_files/enb.band38.lte_2x2.100prb.usrpn310.conf:/opt/oai-enb/etc/enb.conf network_mode: "host" diff --git a/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2/docker-compose.yml b/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2/docker-compose.yml index db6215a148..94ceec5723 100644 --- a/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2/docker-compose.yml +++ b/ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2/docker-compose.yml @@ -5,7 +5,7 @@ services: container_name: oai-enb environment: TZ: Europe/Paris - USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 + USE_ADDITIONAL_OPTIONS: --usrp-tx-thread-config 1 --thread-pool 0,2,4,6 --log_config.global_log_options level,nocolor,utc_time volumes: - ../../conf_files/enb.band38.lte_2x2_tm2.100prb.usrpn310.conf:/opt/oai-enb/etc/enb.conf network_mode: "host" diff --git a/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml index 54bf6cec00..bc1d20a984 100644 --- a/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_aw2s_gnb/docker-compose.yml @@ -12,6 +12,7 @@ services: core: -1 # for core dumps environment: TZ: Europe/Paris + USE_ADDITIONAL_OPTIONS: --log_config.global_log_options level,nocolor,utc_time volumes: - ../../conf_files/gnb.sa.band78.51prb.aw2s.ddsuu.conf:/opt/oai-gnb-aw2s/etc/gnb.conf healthcheck: From 3a9fdb674ae9c01d0a0c8e34002664ad38156b3f Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 18 Feb 2026 18:12:00 +0100 Subject: [PATCH 03/25] Shorten logging output to time I am not sure we ever need the millisecond output. The date is typically also clear from the output (we could print the date at the beginning, once, instead, but this is not done here. Finally, in Jenkins, we typically get time stamps. Thus, it's not necessary to print this by default. On the other hand, since it might be useful when running locally, make time stamps in logs configurable. --- ci-scripts/args_parse.py | 6 +++++- ci-scripts/main.py | 14 ++++++-------- ci-scripts/run_locally.sh | 6 +++--- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/ci-scripts/args_parse.py b/ci-scripts/args_parse.py index 3a3ea52fa0..48bca5644f 100644 --- a/ci-scripts/args_parse.py +++ b/ci-scripts/args_parse.py @@ -44,6 +44,7 @@ import constants as CONST def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER): force_local = False + date_fmt = None while len(argvs) > 1: myArgv = argvs.pop(1) # 0th is this file's name @@ -56,6 +57,9 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER): #consider inline parameters + elif re.match(r'^\-\-datefmt=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match(r'^\-\-datefmt=(.+)$', myArgv, re.IGNORECASE) + date_fmt = matchReg.group(1) elif re.match(r'^\-\-mode=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match(r'^\-\-mode=(.+)$', myArgv, re.IGNORECASE) mode = matchReg.group(1) @@ -168,4 +172,4 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER): HELP.GenericHelp(CONST.Version) sys.exit('Invalid Parameter: ' + myArgv) - return mode, force_local + return mode, force_local, date_fmt diff --git a/ci-scripts/main.py b/ci-scripts/main.py index d86af05871..1abdda36ed 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -62,13 +62,6 @@ import lxml.etree as ET import logging import signal import traceback -logging.basicConfig( - level=logging.DEBUG, - stream=sys.stdout, - format="[%(asctime)s] %(levelname)8s: %(message)s" -) - - #----------------------------------------------------------- @@ -349,7 +342,12 @@ CLUSTER = cls_cluster.Cluster() import args_parse # Force local execution, move all execution targets to localhost force_local = False -mode, force_local = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER) +mode, force_local, date_fmt = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER) +fmt = "%(levelname)8s: %(message)s" +if date_fmt: + fmt = "[%(asctime)s] %(levelname)s %(message)s" +logging.basicConfig(level=logging.DEBUG, stream=sys.stdout, format=fmt, datefmt=date_fmt,) + #----------------------------------------------------------- # mode amd XML class (action) analysis diff --git a/ci-scripts/run_locally.sh b/ci-scripts/run_locally.sh index 6423378e9c..645ac1a9fe 100755 --- a/ci-scripts/run_locally.sh +++ b/ci-scripts/run_locally.sh @@ -31,15 +31,15 @@ docker tag oai-nr-cuup oai-ci/oai-nr-cuup:develop-${SHORT_COMMIT_SHA} python3 main.py --mode=InitiateHtml --ranRepository=NONE --ranBranch=${CURRENT_BRANCH} \ --ranCommitID=${COMMIT_SHA} --ranAllowMerge=false \ --ranTargetBranch=NONE \ - --XMLTestFile=xml_files/${TESTCASE} --local + --XMLTestFile=xml_files/${TESTCASE} --local --datefmt="%H:%M:%S" python3 main.py --mode=TesteNB --ranRepository=NONE --ranBranch=${CURRENT_BRANCH} \ --ranCommitID=${COMMIT_SHA} --ranAllowMerge=false \ --ranTargetBranch=NONE \ --eNBSourceCodePath=${REPO_PATH} \ - --XMLTestFile=${TESTCASE} --local + --XMLTestFile=${TESTCASE} --local --datefmt="%H:%M:%S" RET=$? -python3 main.py --mode=FinalizeHtml --local +python3 main.py --mode=FinalizeHtml --local --datefmt="%H:%M:%S" exit ${RET} From 0b219bf443c0c89c161f3626b7583b16c8605f8c Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 22 Aug 2025 08:02:51 +0200 Subject: [PATCH 04/25] Fix invalid escape sequences Use a raw string as suggested: ci-scripts/cls_static_code_analysis.py:93: SyntaxWarning: invalid escape sequence '\:' result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) ci-scripts/cls_static_code_analysis.py:125: SyntaxWarning: invalid escape sequence '\.' ret = re.search('cppcheck version="(?P[0-9\.]+)"', str(line)) ci-scripts/cls_oai_html.py:221: SyntaxWarning: invalid escape sequence '\/' cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<\/span>/' test_results.html" ci-scripts/ran.py:428: SyntaxWarning: "\[" is an invalid escape sequence. Such sequences will not work in the future. Did you mean "\\["? A raw string is also an option. result = re.search('\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI 0x[0-9a-fA-F]+ received correctly, adding UE MAC Context RNTI 0x[0-9a-fA-F]+', str(line)) ci-scripts/ran.py:457: SyntaxWarning: "\[" is an invalid escape sequence. Such sequences will not work in the future. Did you mean "\\["? A raw string is also an option. result = re.search('\[PHY\]\s+problem receiving samples', str(line)) ci-scripts/ran.py:461: SyntaxWarning: "\[" is an invalid escape sequence. Such sequences will not work in the future. Did you mean "\\["? A raw string is also an option. result = re.search('\[MAC\]\s+Removing UE', str(line)) --- ci-scripts/cls_oai_html.py | 2 +- ci-scripts/cls_static_code_analysis.py | 4 ++-- ci-scripts/ran.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-scripts/cls_oai_html.py b/ci-scripts/cls_oai_html.py index 6e29063192..444fc68b5c 100644 --- a/ci-scripts/cls_oai_html.py +++ b/ci-scripts/cls_oai_html.py @@ -218,7 +218,7 @@ class HTMLManagement(): cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__//' test_results.html" subprocess.run(cmd, shell=True) else: - cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<\/span>/' test_results.html" + cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + r"__/<\/span>/' test_results.html" subprocess.run(cmd, shell=True) self.htmlFooterCreated = False diff --git a/ci-scripts/cls_static_code_analysis.py b/ci-scripts/cls_static_code_analysis.py index 5ed52e4027..cc9abedf23 100644 --- a/ci-scripts/cls_static_code_analysis.py +++ b/ci-scripts/cls_static_code_analysis.py @@ -89,7 +89,7 @@ class StaticCodeAnalysis(): logging.debug('Building on server: ' + node) cmd = cls_cmd.getConnection(node) # on RedHat/CentOS .git extension is mandatory - result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) + result = re.search(r'([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) if result is not None: full_ran_repo_name = self.ranRepository.replace('git/', 'git') else: @@ -121,7 +121,7 @@ class StaticCodeAnalysis(): xmlStart = False with open(filename, 'r') as logfile: for line in logfile: - ret = re.search('cppcheck version="(?P[0-9\.]+)"', str(line)) + ret = re.search(r'cppcheck version="(?P[0-9\.]+)"', str(line)) if ret is not None: CCR.versions[vId] = ret.group('version') if re.search('RUN cat cmake_targets/log/cppcheck.xml', str(line)) is not None: diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 04384ad4f4..9a69f2d747 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -420,7 +420,7 @@ class RANManagement(): if result is not None: mbmsRequestMsg += 1 #FR1 NSA test : add new markers to make sure gNB is used - result = re.search('\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI 0x[0-9a-fA-F]+ received correctly, adding UE MAC Context RNTI 0x[0-9a-fA-F]+', str(line)) + result = re.search(r'\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI 0x[0-9a-fA-F]+ received correctly, adding UE MAC Context RNTI 0x[0-9a-fA-F]+', str(line)) if result is not None: NSA_RAPROC_PUSCH_check = 1 @@ -449,11 +449,11 @@ class RANManagement(): #count "problem receiving samples" msg - result = re.search('\[PHY\]\s+problem receiving samples', str(line)) + result = re.search(r'\[PHY\]\s+problem receiving samples', str(line)) if result is not None: pb_receiving_samples_cnt += 1 #count "Removing UE" msg - result = re.search('\[MAC\]\s+Removing UE', str(line)) + result = re.search(r'\[MAC\]\s+Removing UE', str(line)) if result is not None: removing_ue += 1 #count "X2AP-PDU" From 7fe3183d1e1dc9b93261eda2d7520774a5909f2e Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 17 Feb 2026 19:03:26 +0100 Subject: [PATCH 05/25] Remove dead code --- ci-scripts/main.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 1abdda36ed..7dd0ef28ef 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -279,15 +279,6 @@ def ExecuteActionWithParam(action, ctx, node): return success -#check if given test is in list -#it is in list if one of the strings in 'list' is at the beginning of 'test' -def test_in_list(test, list): - for check in list: - check=check.replace('+','') - if (test.startswith(check)): - return True - return False - test_runner_abort = False def receive_signal(signum, frame): global test_runner_abort @@ -365,18 +356,7 @@ elif re.match('^TerminateSPGW$', mode, re.IGNORECASE): elif re.match('^LogCollectBuild$', mode, re.IGNORECASE): logging.warning("Option LogCollectBuild ignored") elif re.match('^LogCollecteNB$', mode, re.IGNORECASE): - if RAN.eNBSourceCodePath == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - if os.path.isdir('cmake_targets/log'): - cmd = 'zip -r enb.log.' + RAN.BuildId + '.zip cmake_targets/log' - logging.info(cmd) - try: - zipStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=60) - except subprocess.CalledProcessError as e: - logging.error("Command '{}' returned non-zero exit status {}.".format(e.cmd, e.returncode)) - logging.error("Error output:\n{}".format(e.output)) - sys.exit(0) + logging.warning("Option LogCollecteNB ignored") elif re.match('^LogCollectHSS$', mode, re.IGNORECASE): logging.warning("Option LogCollectHSS ignored") elif re.match('^LogCollectMME$', mode, re.IGNORECASE): From cc1d66319020c55987cde9d1584d3e1f50c54ec0 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 19 Feb 2026 09:44:34 +0100 Subject: [PATCH 06/25] Remove unused code in args_parse.py --- ci-scripts/args_parse.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci-scripts/args_parse.py b/ci-scripts/args_parse.py index 48bca5644f..9f181ae68e 100644 --- a/ci-scripts/args_parse.py +++ b/ci-scripts/args_parse.py @@ -33,7 +33,6 @@ #----------------------------------------------------------- import sys # arg import re # reg -import yaml import constants as CONST #----------------------------------------------------------- @@ -162,9 +161,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER): elif re.match(r'^\-\-OCRegistry=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match(r'^\-\-OCRegistry=(.+)$', myArgv, re.IGNORECASE) CLUSTER.OCRegistry = matchReg.group(1) - elif re.match(r'^\-\-BuildId=(.+)$', myArgv, re.IGNORECASE): - matchReg = re.match(r'^\-\-BuildId=(.+)$', myArgv, re.IGNORECASE) - RAN.BuildId = matchReg.group(1) elif re.match(r'^\-\-FlexRicTag=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match(r'^\-\-FlexRicTag=(.+)$', myArgv, re.IGNORECASE) CONTAINERS.flexricTag = matchReg.group(1) From 84a49886d80f584cd9bab99ef2018962aa249f8d Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 30 Oct 2024 19:41:37 +0100 Subject: [PATCH 07/25] Move unused configs to untested --- .../{ => untested}/enb.band38.nsa_2x2.100prb.usrpn310.conf | 0 .../conf_files/{ => untested}/enb.band7.tm1.50prb.usrpb210.conf | 0 .../{ => untested}/gnb.nsa.band78.106prb.usrpn310.2x2.conf | 0 .../{ => untested}/gnb.sa.band78.106prb.usrp310.2x2.conf | 0 .../{ => untested}/gnb.sa.band78.106prb.usrpn310.ddsuu-2x2.conf | 0 .../{ => untested}/gnb.sa.band78.162prb.usrpn310.2x2.conf | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename ci-scripts/conf_files/{ => untested}/enb.band38.nsa_2x2.100prb.usrpn310.conf (100%) rename ci-scripts/conf_files/{ => untested}/enb.band7.tm1.50prb.usrpb210.conf (100%) rename ci-scripts/conf_files/{ => untested}/gnb.nsa.band78.106prb.usrpn310.2x2.conf (100%) rename ci-scripts/conf_files/{ => untested}/gnb.sa.band78.106prb.usrp310.2x2.conf (100%) rename ci-scripts/conf_files/{ => untested}/gnb.sa.band78.106prb.usrpn310.ddsuu-2x2.conf (100%) rename ci-scripts/conf_files/{ => untested}/gnb.sa.band78.162prb.usrpn310.2x2.conf (100%) diff --git a/ci-scripts/conf_files/enb.band38.nsa_2x2.100prb.usrpn310.conf b/ci-scripts/conf_files/untested/enb.band38.nsa_2x2.100prb.usrpn310.conf similarity index 100% rename from ci-scripts/conf_files/enb.band38.nsa_2x2.100prb.usrpn310.conf rename to ci-scripts/conf_files/untested/enb.band38.nsa_2x2.100prb.usrpn310.conf diff --git a/ci-scripts/conf_files/enb.band7.tm1.50prb.usrpb210.conf b/ci-scripts/conf_files/untested/enb.band7.tm1.50prb.usrpb210.conf similarity index 100% rename from ci-scripts/conf_files/enb.band7.tm1.50prb.usrpb210.conf rename to ci-scripts/conf_files/untested/enb.band7.tm1.50prb.usrpb210.conf diff --git a/ci-scripts/conf_files/gnb.nsa.band78.106prb.usrpn310.2x2.conf b/ci-scripts/conf_files/untested/gnb.nsa.band78.106prb.usrpn310.2x2.conf similarity index 100% rename from ci-scripts/conf_files/gnb.nsa.band78.106prb.usrpn310.2x2.conf rename to ci-scripts/conf_files/untested/gnb.nsa.band78.106prb.usrpn310.2x2.conf diff --git a/ci-scripts/conf_files/gnb.sa.band78.106prb.usrp310.2x2.conf b/ci-scripts/conf_files/untested/gnb.sa.band78.106prb.usrp310.2x2.conf similarity index 100% rename from ci-scripts/conf_files/gnb.sa.band78.106prb.usrp310.2x2.conf rename to ci-scripts/conf_files/untested/gnb.sa.band78.106prb.usrp310.2x2.conf diff --git a/ci-scripts/conf_files/gnb.sa.band78.106prb.usrpn310.ddsuu-2x2.conf b/ci-scripts/conf_files/untested/gnb.sa.band78.106prb.usrpn310.ddsuu-2x2.conf similarity index 100% rename from ci-scripts/conf_files/gnb.sa.band78.106prb.usrpn310.ddsuu-2x2.conf rename to ci-scripts/conf_files/untested/gnb.sa.band78.106prb.usrpn310.ddsuu-2x2.conf diff --git a/ci-scripts/conf_files/gnb.sa.band78.162prb.usrpn310.2x2.conf b/ci-scripts/conf_files/untested/gnb.sa.band78.162prb.usrpn310.2x2.conf similarity index 100% rename from ci-scripts/conf_files/gnb.sa.band78.162prb.usrpn310.2x2.conf rename to ci-scripts/conf_files/untested/gnb.sa.band78.162prb.usrpn310.2x2.conf From 4a2f6cd823823188b022bd604040a7b4a0d1bf2a Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 30 Oct 2024 19:48:01 +0100 Subject: [PATCH 08/25] Remove unused docker file --- .../fr1_enb_mono_fdd_tim/docker-compose.yml | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml diff --git a/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml b/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml deleted file mode 100644 index 7963f823b1..0000000000 --- a/ci-scripts/yaml_files/fr1_enb_mono_fdd_tim/docker-compose.yml +++ /dev/null @@ -1,27 +0,0 @@ -services: - enb_mono_fdd: - image: ${REGISTRY-oaisoftwarealliance/}oai-enb:${TAG:-develop} - privileged: true - container_name: prod-enb-mono-fdd - environment: - USE_B2XX: 'yes' - USE_ADDITIONAL_OPTIONS: --RUs.[0].max_rxgain 115 --RUs.[0].max_pdschReferenceSignalPower -27 --eNBs.[0].component_carriers.[0].pucch_p0_Nominal -96 --log_config.global_log_options level,nocolor,time - volumes: - - ../../conf_files/enb.band7.25prb.usrpb200.conf:/opt/oai-enb/etc/enb.conf - networks: - public_net: - ipv4_address: 192.168.61.30 - healthcheck: - # pgrep does NOT work - test: /bin/bash -c "ps aux | grep -v grep | grep -c softmodem" - interval: 10s - timeout: 5s - retries: 5 - -networks: - public_net: - name: prod-oai-public-net - ipam: - config: - - subnet: 192.168.61.0/26 - From 92997fba2bbeea215477bf5a6726a2665083f441 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 28 Jan 2025 18:54:59 +0100 Subject: [PATCH 09/25] Update tests/README.md --- ci-scripts/tests/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-scripts/tests/README.md b/ci-scripts/tests/README.md index f583c51c08..355d046efd 100644 --- a/ci-scripts/tests/README.md +++ b/ci-scripts/tests/README.md @@ -16,6 +16,7 @@ To run individual unit tests, start them like so: python tests/deployment.py -v python tests/iperf-analysis.py -v python tests/ping-iperf.py -v + python tests/pull-clean-int-registry.py -v The logs will indicate if all tests passed. `tests/deployment.py` requires these images to be present: From 44ad255195c89c070997086db3fcbaec9f265d30 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 31 Oct 2024 17:23:06 +0100 Subject: [PATCH 10/25] Remove unused USRP_IPAddress --- ci-scripts/main.py | 2 -- ci-scripts/ran.py | 1 - ci-scripts/xml_files/gnb_phytest_usrp_run.xml | 1 - ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml | 1 - ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml | 1 - ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml | 1 - 6 files changed, 7 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 7dd0ef28ef..a67405728d 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -108,8 +108,6 @@ def ExecuteActionWithParam(action, ctx, node): else: RAN.datalog_rt_stats_file=datalog_rt_stats_file RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args') - USRPIPAddress = test.findtext('USRP_IPAddress') or '' - #local variable air_interface air_interface = test.findtext('air_interface') if (air_interface is None) or (air_interface.lower() not in ['nr','lte']): diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 9a69f2d747..850b44318c 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -70,7 +70,6 @@ class RANManagement(): self.runtime_stats= '' self.datalog_rt_stats={} self.datalog_rt_stats_file='datalog_rt_stats.default.yaml' - self.USRPIPAddress = '' #checkers from xml self.ran_checkers={} self.cmd_prefix = '' # prefix before {lte,nr}-softmodem diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml index f6af5ee485..e2bd3204aa 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml @@ -39,7 +39,6 @@ -O ci-scripts/conf_files/gnb.band79.106prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 106 -t 23 -D 127 -m 28 -M 106 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time datalog_rt_stats.default.yaml NR - 192.168.20.2 numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml index 93053dee90..c33908a92d 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml @@ -39,7 +39,6 @@ -O ci-scripts/conf_files/gnb.band79.273prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 273 -t 23 -D 127 -m 23 -M 273 -l 2 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time datalog_rt_stats.100.2x2.yaml NR - 192.168.20.2 numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml index 2cd040a47a..0348c63b35 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml @@ -39,7 +39,6 @@ -O ci-scripts/conf_files/gnb.band79.162prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 162 -t 23 -D 127 -m 23 -M 162 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time datalog_rt_stats.1x1.60.yaml NR - 192.168.20.2 numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml index 7e5592fb6e..075ba3e92f 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml @@ -39,7 +39,6 @@ -O ci-scripts/conf_files/gnb.band79.162prb.usrpn300.phytest-dora.conf --phy-test --gNBs.[0].pdsch_AntennaPorts_XP 2 --RUs.[0].nb_tx 2 --RUs.[0].nb_rx 2 -q -U 768 -T 162 -t 23 -D 127 -m 23 -M 162 -l 2 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time datalog_rt_stats.60.2x2.yaml NR - 192.168.20.2 numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 From 41a6188c0422bef4ad09db2f38ef04484c0f293b Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 21 Jan 2026 19:00:15 +0100 Subject: [PATCH 11/25] Change bash set options to improve run_locally.sh set -e makes the script exit on error. However, there is code to handle a non-zero return code of the script, and -e prevents from writing the footer. add set -x to see all the commands that are being executed. --- ci-scripts/run_locally.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci-scripts/run_locally.sh b/ci-scripts/run_locally.sh index 645ac1a9fe..f01d263433 100755 --- a/ci-scripts/run_locally.sh +++ b/ci-scripts/run_locally.sh @@ -1,6 +1,5 @@ #!/bin/bash -set -e SHORT_COMMIT_SHA=$(git rev-parse --short=8 HEAD) COMMIT_SHA=$(git rev-parse HEAD) CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) @@ -13,6 +12,8 @@ if [ $# -eq 0 ] exit 1 fi +set -x + # The script assumes you've build the following images: # # docker build . -f docker/Dockerfile.gNB.ubuntu -t oai-gnb From e809841aeaaa1fe972c9f5db203a573cc9a7e3c7 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 21 Jan 2026 15:40:21 +0100 Subject: [PATCH 12/25] Move Real-Time Stats Analysis to separate step Factor out to make existing log analysis simpler. It will also allow to more easily port this functionality to docker-based setups. --- ci-scripts/cls_analysis.py | 38 +++++++ ci-scripts/main.py | 9 +- ci-scripts/ran.py | 98 +++---------------- ci-scripts/xml_class_list.yml | 1 + .../gnb_phytest_rfemulator_run_100_2x2.xml | 8 +- ci-scripts/xml_files/gnb_phytest_usrp_run.xml | 8 +- .../gnb_phytest_usrp_run_100_2x2.xml | 8 +- .../xml_files/gnb_phytest_usrp_run_60.xml | 8 +- .../xml_files/gnb_phytest_usrp_run_60_2x2.xml | 8 +- 9 files changed, 93 insertions(+), 93 deletions(-) diff --git a/ci-scripts/cls_analysis.py b/ci-scripts/cls_analysis.py index 588490da62..3d9eb9c499 100644 --- a/ci-scripts/cls_analysis.py +++ b/ci-scripts/cls_analysis.py @@ -22,6 +22,8 @@ import re import os +import logging +import yaml import xml.etree.ElementTree as ET import json @@ -109,3 +111,39 @@ class Analysis(): test_summary['Nbpass'] = nb_tests - nb_failed test_summary['Nbfail'] = nb_failed return nb_failed == 0, test_summary, test_result + + def analyze_rt_stats(thresholds, L1_stats, MAC_stats): + with open(thresholds, 'r') as f: + datalog_rt_stats = yaml.load(f, Loader=yaml.FullLoader) + + rt_keys = datalog_rt_stats['Ref'] + real_time_stats = {} + for sf in [L1_stats, MAC_stats]: + with open(sf, 'r') as f: + for line in f.readlines(): + for k in rt_keys: + result = re.search(k, line) + if result is not None: + tmp = re.match(rf'^.*?(\b{k}\b.*)', line.rstrip()) + if tmp is not None: + real_time_stats[k] = tmp.group(1) + + # datalog_rt_stats format must align with HTML.CreateHtmlDataLogTable() + datalog_rt_stats['Data']={} + for k in real_time_stats: + tmp = re.match(r'^(?P.*):\s+(?P\d+\.\d+) us;\s+(?P\d+);\s+(?P\d+\.\d+) us;', real_time_stats[k]) + if tmp is not None: + metric = tmp.group('metric') + avg = float(tmp.group('avg')) + max = float(tmp.group('max')) + count = int(tmp.group('count')) + datalog_rt_stats['Data'][metric] = ["{:.0f}".format(avg),"{:.0f}".format(max),"{:d}".format(count),"{:.2f}".format(avg / datalog_rt_stats['Ref'][metric])] + + success = True + for k in datalog_rt_stats['Data']: + valnorm = float(datalog_rt_stats['Data'][k][3]) + dev = datalog_rt_stats['DeviationThreshold'][k] + if valnorm > 1.0 + dev or valnorm < 1.0 - dev: # condition for fail: avg/ref deviates by more than "deviation threshold" + logging.debug(f'\u001B[1;30;43m normalized metric {k}={valnorm} deviates by more than {dev}\u001B[0m') + success = False + return success, datalog_rt_stats diff --git a/ci-scripts/main.py b/ci-scripts/main.py index a67405728d..4628b1fcba 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -102,11 +102,6 @@ def ExecuteActionWithParam(action, ctx, node): success = CONTAINERS.BuildRunTests(ctx, node, HTML) elif action == 'Initialize_eNB': - datalog_rt_stats_file=test.findtext('rt_stats_cfg') - if datalog_rt_stats_file is None: - RAN.datalog_rt_stats_file='datalog_rt_stats.default.yaml' - else: - RAN.datalog_rt_stats_file=datalog_rt_stats_file RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args') #local variable air_interface air_interface = test.findtext('air_interface') @@ -271,6 +266,10 @@ def ExecuteActionWithParam(action, ctx, node): images = test.findtext('images').split() success = CLUSTER.PullClusterImage(HTML, node, images, tag_prefix=tag_prefix) + elif action == 'AnalyzeRTStats': + yaml = test.findtext('stats_cfg') + success = RAN.AnalyzeRTStats(HTML, node, ctx, yaml) + else: logging.warning(f"unknown action {action}, skip step") success = True # by default, we skip the step and print a warning diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 850b44318c..a4cd624975 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -46,6 +46,7 @@ import cls_cmd import cls_cmd import helpreadme as HELP import constants as CONST +import cls_analysis from cls_ci_helper import archiveArtifact #----------------------------------------------------------- @@ -68,8 +69,6 @@ class RANManagement(): self.eNBmbmsEnables = [False, False, False] self.eNBstatuses = [-1, -1, -1] self.runtime_stats= '' - self.datalog_rt_stats={} - self.datalog_rt_stats_file='datalog_rt_stats.default.yaml' #checkers from xml self.ran_checkers={} self.cmd_prefix = '' # prefix before {lte,nr}-softmodem @@ -151,8 +150,6 @@ class RANManagement(): logdir = os.path.dirname(logfile) file = archiveArtifact(cmd, ctx, logfile) - archiveArtifact(cmd, ctx, f"{logdir}/nrL1_stats.log") - archiveArtifact(cmd, ctx, f"{logdir}/nrMAC_stats.log") cmd.close() if file is None: logging.debug('\u001B[1;37;41m Could not copy xNB logfile to analyze it! \u001B[0m') @@ -167,12 +164,22 @@ class RANManagement(): else: HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) - #display rt stats for gNB only - if len(self.datalog_rt_stats) != 0: - HTML.CreateHtmlDataLogTable(self.datalog_rt_stats) - return logStatus >= 0 + def AnalyzeRTStats(self, HTML, node, ctx, thresholds): + logging.info(f'Analyzing realtime stats from server: {node}') + lSourcePath = self.eNBSourceCodePath + + logdir = f'{lSourcePath}/cmake_targets' + with cls_cmd.getConnection(node) as cmd: + l1_file = archiveArtifact(cmd, ctx, f"{logdir}/nrL1_stats.log") + mac_file = archiveArtifact(cmd, ctx, f"{logdir}/nrMAC_stats.log") + + logging.info(f"check against thresholds from {thresholds}") + success, datalog_rt_stats = cls_analysis.Analysis.analyze_rt_stats(thresholds, l1_file, mac_file) + HTML.CreateHtmlDataLogTable(datalog_rt_stats) + return success + def _analyzeUeRetx(self, rounds, checkers, regex): if len(rounds) == 0 or len(checkers) == 0: logging.warning(f'warning: rounds={rounds} checkers={checkers}') @@ -241,8 +248,6 @@ class RANManagement(): NSA_RAPROC_PUSCH_check = 0 #dlsch and ulsch statistics (dictionary) dlsch_ulsch_stats = {} - #real time statistics (dictionary) - real_time_stats = {} #count "problem receiving samples" msg pb_receiving_samples_cnt = 0 #count "removing UE" msg @@ -470,54 +475,6 @@ class RANManagement(): enb_log_file.close() - - #the following part takes the *_stats.log files as source (not the stdout log file) - - #the datalog config file has to be loaded - datalog_rt_stats_file=self.datalog_rt_stats_file - if (os.path.isfile(datalog_rt_stats_file)): - yaml_file=datalog_rt_stats_file - elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)): - yaml_file='ci-scripts/'+datalog_rt_stats_file - else: - logging.error("Datalog RT stats yaml file cannot be found") - sys.exit("Datalog RT stats yaml file cannot be found") - - with open(yaml_file,'r') as f: - datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader) - rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field - - # nrL1_stats.log/nrMAC_stats.log should be in the same directory as main log file - # currently the link is only implicit as below based on pattern matching - # I will rework this to give the file explicitly - l1_stats_fn = re.sub(r'-enb.log$', '-nrL1_stats.log', eNBlogFile) - mac_stats_fn = re.sub(r'-enb.log$', '-nrMAC_stats.log', eNBlogFile) - if os.path.isfile(l1_stats_fn) and os.path.isfile(mac_stats_fn): - # don't use CI-nrL1_stats.log, as this will increase the processing time for - # no reason, we just need the last occurence - nrL1_stats = open(l1_stats_fn, 'r') - nrMAC_stats = open(mac_stats_fn, 'r') - for line in nrL1_stats.readlines(): - for k in rt_keys: - result = re.search(k, line) - if result is not None: - #remove 1- all useless char before relevant info 2- trailing char - tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex - if tmp!=None: - real_time_stats[k]=tmp.group(1) - for line in nrMAC_stats.readlines(): - for k in rt_keys: - result = re.search(k, line) - if result is not None: - #remove 1- all useless char before relevant info 2- trailing char - tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex - if tmp!=None: - real_time_stats[k]=tmp.group(1) - nrL1_stats.close() - nrMAC_stats.close() - else: - logging.debug(f"NR Stats files for RT analysis not found: {l1_stats_fn}, {mac_stats_fn}") - #stdout log file and stat log files analysis completed logging.debug(' File analysis (stdout, stats) completed') @@ -601,31 +558,6 @@ class RANManagement(): retx_status[ue]['ul'] = self._analyzeUeRetx(dlulstat['ulsch_rounds'], ulcheckers, r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+ulsch_errors\s+(\d+)') - #real time statistics - datalog_rt_stats['Data']={} - if len(real_time_stats)!=0: #check if dictionary is not empty - for k in real_time_stats: - tmp=re.match(r'^(?P.*):\s+(?P\d+\.\d+) us;\s+(?P\d+);\s+(?P\d+\.\d+) us;',real_time_stats[k]) - if tmp is not None: - metric=tmp.group('metric') - avg=float(tmp.group('avg')) - max=float(tmp.group('max')) - count=int(tmp.group('count')) - datalog_rt_stats['Data'][metric]=["{:.0f}".format(avg),"{:.0f}".format(max),"{:d}".format(count),"{:.2f}".format(avg/datalog_rt_stats['Ref'][metric])] - #once all metrics are collected, store the data as a class attribute to build a dedicated HTML table afterward - self.datalog_rt_stats=datalog_rt_stats - #check if there is a fail => will render the test as failed - for k in datalog_rt_stats['Data']: - valnorm = float(datalog_rt_stats['Data'][k][3]) - dev = datalog_rt_stats['DeviationThreshold'][k] - if valnorm > 1.0 + dev or valnorm < 1.0 - dev: # condition for fail : avg/ref deviates by more than "deviation threshold" - logging.debug(f'\u001B[1;30;43m normalized datalog_rt_stats metric {k}={valnorm} deviates by more than {dev}\u001B[0m') - RealTimeProcessingIssue = True - else: - statMsg = 'No real time stats found in the log file\n' - logging.debug('No real time stats found in the log file') - htmleNBFailureMsg += statMsg - if not showedByeMsg: logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB did not show "Bye." message at end, it likely did not stop properly! \u001B[0m') htmleNBFailureMsg += 'No Bye. message found, did not stop properly\n' diff --git a/ci-scripts/xml_class_list.yml b/ci-scripts/xml_class_list.yml index e8fa42dcd3..7dba0d247a 100755 --- a/ci-scripts/xml_class_list.yml +++ b/ci-scripts/xml_class_list.yml @@ -32,3 +32,4 @@ - Custom_Command - Custom_Script - Create_Workspace + - AnalyzeRTStats diff --git a/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml b/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml index 30f307612c..620d275d37 100644 --- a/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml @@ -37,7 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.273prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 273 -t 23 -D 127 -m 23 -M 273 -l 2 --device.name rf_emulator --rf_emulator.noise_level_dBFS -36 --log_config.global_log_options level,nocolor,time - datalog_rt_stats.100.2x2.yaml NR numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -48,6 +47,13 @@ 60 + + AnalyzeRTStats + Analyze Real-Time Stats + datalog_rt_stats.100.2x2.yaml + caracal + + Terminate_eNB caracal diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml index e2bd3204aa..f2c140faac 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml @@ -37,7 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.106prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 106 -t 23 -D 127 -m 28 -M 106 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - datalog_rt_stats.default.yaml NR numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -48,6 +47,13 @@ 60 + + AnalyzeRTStats + Analyze Real-Time Stats + datalog_rt_stats.default.yaml + caracal + + Terminate_eNB true diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml index c33908a92d..5204653223 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml @@ -37,7 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.273prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 273 -t 23 -D 127 -m 23 -M 273 -l 2 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - datalog_rt_stats.100.2x2.yaml NR numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -48,6 +47,13 @@ 60 + + AnalyzeRTStats + Analyze Real-Time Stats + datalog_rt_stats.100.2x2.yaml + caracal + + Terminate_eNB true diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml index 0348c63b35..e939cb1e07 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml @@ -37,7 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.162prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 162 -t 23 -D 127 -m 23 -M 162 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - datalog_rt_stats.1x1.60.yaml NR numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -48,6 +47,13 @@ 60 + + AnalyzeRTStats + Analyze Real-Time Stats + datalog_rt_stats.1x1.60.yaml + caracal + + Terminate_eNB true diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml index 075ba3e92f..1adcd042ac 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml @@ -37,7 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.162prb.usrpn300.phytest-dora.conf --phy-test --gNBs.[0].pdsch_AntennaPorts_XP 2 --RUs.[0].nb_tx 2 --RUs.[0].nb_rx 2 -q -U 768 -T 162 -t 23 -D 127 -m 23 -M 162 -l 2 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - datalog_rt_stats.60.2x2.yaml NR numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -48,6 +47,13 @@ 60 + + AnalyzeRTStats + Analyze Real-Time Stats + datalog_rt_stats.60.2x2.yaml + caracal + + Terminate_eNB true From 65dab3e027d80f40521666986a4742607450d995 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 29 Jan 2025 09:47:34 +0100 Subject: [PATCH 13/25] Add unit test phytest analysis --- ci-scripts/tests/README.md | 1 + ci-scripts/tests/analysis.py | 57 +++++++++++++++++++ .../analysis/gnb_phytest.success.nrL1.log | 37 ++++++++++++ .../analysis/gnb_phytest.success.nrMAC.log | 0 4 files changed, 95 insertions(+) create mode 100644 ci-scripts/tests/analysis.py create mode 100644 ci-scripts/tests/analysis/gnb_phytest.success.nrL1.log create mode 100644 ci-scripts/tests/analysis/gnb_phytest.success.nrMAC.log diff --git a/ci-scripts/tests/README.md b/ci-scripts/tests/README.md index 355d046efd..48a48b8a31 100644 --- a/ci-scripts/tests/README.md +++ b/ci-scripts/tests/README.md @@ -10,6 +10,7 @@ directory, i.e., `ci-scripts/`, and run To run individual unit tests, start them like so: + python tests/analysis.py -v python tests/build.py -v python tests/cmd.py -v python tests/corenetwork.py -v diff --git a/ci-scripts/tests/analysis.py b/ci-scripts/tests/analysis.py new file mode 100644 index 0000000000..37e4d3499e --- /dev/null +++ b/ci-scripts/tests/analysis.py @@ -0,0 +1,57 @@ +import sys +import logging +logging.basicConfig( + level=logging.DEBUG, + stream=sys.stdout, + format="[%(asctime)s] %(levelname)8s: %(message)s" +) +import os +import unittest +import yaml + +sys.path.append('./') # to find OAI imports below +import cls_analysis +import cls_oai_html +import constants + +class TestAnalysis(unittest.TestCase): + def setUp(self): + self.html = cls_oai_html.HTMLManagement() + self.html.testCaseId = "000000" + self.maxDiff = 10000 # to compare long files below + + def test_gnb_phytest(self): + rtsf = "datalog_rt_stats.100.2x2.yaml" + l1f = "tests/analysis/gnb_phytest.success.nrL1.log" + macf = "tests/analysis/gnb_phytest.success.nrMAC.log" + status, s = cls_analysis.Analysis.analyze_rt_stats(rtsf, l1f, macf) + self.assertTrue(status) + self.assertEqual(s['Title'], "Processing Time (us) from datalog_rt_stats.100.2x2.yaml") + self.assertEqual(s['ColNames'], ["Metric", "Average; Max; Count", "Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold)"]) + with open(rtsf, 'r') as f: + rt_stats = yaml.load(f, Loader=yaml.FullLoader) + # check that analyze_rt_stats() picked up all Ref/DeviationThreshold values from yaml file, e.g. + # s['Ref']['L1 Tx processing'] == corresponding value in yaml + # s['DeviationThreshold']['L1 Tx processing'] == corresponding value in yaml + for k, v in rt_stats['Ref'].items(): + self.assertEqual(s['Ref'][k], v) + for k, v in rt_stats['DeviationThreshold'].items(): + self.assertEqual(s['DeviationThreshold'][k], v) + for k, v in s['Ref'].items(): + self.assertEqual(v, rt_stats['Ref'][k]) + for k, v in s['DeviationThreshold'].items(): + self.assertEqual(v, rt_stats['DeviationThreshold'][k]) + self.assertEqual(s['Data']['L1 Tx processing'], ['476', '872', '102933', '1.00']) + self.assertEqual(s['Data']['DLSCH encoding'], ['216', '544', '77201', '1.08']) + self.assertEqual(s['Data']['L1 Rx processing'], ['488', '868', '38598', '0.92']) + self.assertEqual(s['Data']['UL Indication'], ['1', '10', '38598', '0.76']) + self.assertEqual(s['Data']['Slot Indication'], ['18', '94', '128667', '1.38']) + self.assertEqual(s['Data']['PUSCH inner-receiver'], ['307', '530', '25730', '0.85']) + self.assertEqual(s['Data']['feprx'], ['169', '310', '38600', '1.12']) + self.assertEqual(s['Data']['feptx_ofdm (per port, half_slot)'], ['56', '88', '102933', '0.98']) + self.assertEqual(s['Data']['feptx_total'], ['148', '299', '102933', '1.00']) + # TODO this one is not found, the CI does not recover this?? + #self.assertEqual(s['Data']['DL & UL scheduling timing'], ['17', '97', '159371', '1.14']) + +if __name__ == '__main__': + unittest.main() diff --git a/ci-scripts/tests/analysis/gnb_phytest.success.nrL1.log b/ci-scripts/tests/analysis/gnb_phytest.success.nrL1.log new file mode 100644 index 0000000000..066a71978b --- /dev/null +++ b/ci-scripts/tests/analysis/gnb_phytest.success.nrL1.log @@ -0,0 +1,37 @@ +Blacklisted PRBs 0/273 +-5.-6.-4.-4.-4.-6.-6.-6.-8.-6.-6.-4.-4.-4.-3.-6.-6.-11.-8.-8.-5.-4.-5.-4.-5. +-6.-11.-11.-8.-8.-5.-5.-4.-4.-6.-8.-11.-11.-8.-8.-6.-5.-6.-6.-8.-11.-11.-11.-8.-6. +-6.-6.-6.-11.-11.-11.-11.-11.-11.-6.-6.-6.-8.-3.-11.-11.-11.-8.-8.-6.-6.-6.-11.-11.-11. +-11.-8.-6.-5.-6.-6.-8.-11.-11.-11.-6.-6.-5.-4.-4.-5.-8.-8.-8.-6.-5.-4.-3.-3.-4. +-4.-6.-6.-5.-3.-3.-2.-2.-2.-3.-3.-4.-5.-5.-3.-2.-1.-1.-2.-5.-4.-4.-4.-3.-2. + 0.-1.-1.-1.-2.-2.-3.-2.-1. 0. 2.-11.13. 8. 7. 6. 5. 0. 0.-1. 0. 0. 0.-1.-3. +-3.-3.-3.-1.-1.-1.-1.-1.-1.-2.-3.-1.-1.-1.-1.-2.-1.-3.-4.-5.-8.-6.-3.-3.-3. +-3.-3.-5.-8.-8.-8.-6.-4.-4.-3.-4.-6.-6.-8.-8.-8.-6.-5.-6.-5.-6.-8.-11.-11.-11. +-8.-5.-5.-8.-6.-6.-8.-11.-11.-11.-8.-6.-6.-6.-8.-11.-11.-11.-8.-8.-6.-6.-5.-5.-8. +-11.-11.-8.-11.-6.-5.-5.-5.-5.-6.-11.-11.-8.-6.-3.-4.-5.-5.-6.-8.-6.-6.-5.-4.-4. +-4.-5.-6.-6.-11.-8.-6.-4.-3.-4.-3.-4.-5.-6.-8.-6.-5.-4.-3.-4.-3.-6.-5. +max_IO = 24 (137), min_I0 = 0 (17), avg_I0 = 11 dB(8.8.) +PRACH I0 = 0.0 dB +DLSCH RNTI 1234: current_Qm 6, current_RI 2, total_bytes TX -1130807110 +ULSCH RNTI 1234, 389: ulsch_power[0] 41,5 ulsch_noise_power[0] 7.8, sync_pos 0 + ulsch_power[1] 26.7, ulsch_noise_power[1] 7.8 + round_trials 6434(1.0e+00):6432(1.0e+00):6432(1.0e+00):6432, DTX 0, current_Qm 6, current_RI 1, total_bytes RX/SCHED 0/385630560 + L1 Tx job: 401.013 us; 128667; 895.809 us; + L1 Rx job: 183.883 us; 128667; 980.814 us; + L1 Tx processing: 476.247 us; 102933; 872.099 us; + DLSCH encoding: 216.371 us; 77201; 544.272 us; + DLSCH scrambling: 5.524 us; 77201; 19.192 us; + DLSCH modulation: 21.137 us; 77201; 36.428 us; + DLSCH resource mapping: 1.127 us; 1003613; 12.027 us; + DLSCH precoding: 3.667 us; 1003613; 17.411 us; + L1 Rx processing: 488.478 us; 38598; 867.672 us; +UL segment deinterleaving: 5.063 us; 488870; 27.005 us; + UL segment rate recovery: 11.554 us; 488870; 517.998 us; + UL segments decoding: 134.035 us; 488870; 759.164 us; + UL Indication: 0.763 us; 38598; 9.588 us; + Slot Indication: 17.891 us; 128667; 94.446 us; + PUSCH inner-receiver: 307.193 us; 25730; 529.580 us; + feprx: 169.305 us; 38600; 310.489 us; +feptx_ofdm (per port, half_slot): 56.093 us; 102933; 88.125 us; + feptx_total: 147.889 us; 102933; 299.307 us; + diff --git a/ci-scripts/tests/analysis/gnb_phytest.success.nrMAC.log b/ci-scripts/tests/analysis/gnb_phytest.success.nrMAC.log new file mode 100644 index 0000000000..e69de29bb2 From 9e35799ca6b28c7e58bbe31e2237c8f56e27b914 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 31 Oct 2024 15:27:14 +0100 Subject: [PATCH 14/25] Remove X2HO eNBmbmsEnables variables - it is spaghetti code: difficult to understand which code paths are taken - the only effect seems to be in the UE log analysis function: it might make the log fail. However, we test that iperf is successful (Iperf2_Unidir()), and would detect before the log analysis if MBMS traffic does not work - the rest is only messages that likely nobody reads The real reason for this change is to reduce the number of class members in the ran.py class, to simplify the functioning of the eNB analysis function (no self) to reduce dependencies on the ran.py class. --- ci-scripts/cls_oaicitest.py | 14 ----------- ci-scripts/ran.py | 46 ------------------------------------- 2 files changed, 60 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 237e795937..67dc5c5e4d 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -604,7 +604,6 @@ class OaiCiTest(): nrDecodeMib = 0 nrFoundDCI = 0 nrCRCOK = 0 - mbms_messages = 0 nbPduSessAccept = 0 nbPduDiscard = 0 HTML.htmlUEFailureMsg='' @@ -678,10 +677,6 @@ class OaiCiTest(): result = re.search('No cell synchronization found, abandoning', str(line)) if result is not None: no_cell_sync_found = True - if RAN.eNBmbmsEnables[0]: - result = re.search('TRIED TO PUSH MBMS DATA', str(line)) - if result is not None: - mbms_messages += 1 result = re.search(r"MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P\d{1,3}), N_RB_DL (?P\d{1,3}), PHICH DURATION (?P\d), PHICH RESOURCE (?P.{1,4}), TX_ANT (?P\d)", str(line)) if result is not None and (not mib_found): try: @@ -812,15 +807,6 @@ class OaiCiTest(): statMsg = f'UE showed {fatalErrorCount} "MAC BSR Triggered ReTxBSR Timer expiry" message(s)' logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if RAN.eNBmbmsEnables[0]: - if mbms_messages > 0: - statMsg = f'UE showed {mbms_messages} "TRIED TO PUSH MBMS DATA" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - else: - statMsg = 'UE did NOT SHOW "TRIED TO PUSH MBMS DATA" message(s)' - logging.debug(f'\u001B[1;30;41m{statMsg}\u001B[0m') - global_status = CONST.OAI_UE_PROCESS_NO_MBMS_MSGS - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' if foundSegFault: logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m') if not nrUEFlag: diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index a4cd624975..ef790b45c8 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -66,7 +66,6 @@ class RANManagement(): self.imageKind = '' self.air_interface = '' self.eNBOptions = ['', '', ''] - self.eNBmbmsEnables = [False, False, False] self.eNBstatuses = [-1, -1, -1] self.runtime_stats= '' #checkers from xml @@ -234,9 +233,6 @@ class RANManagement(): isRRU = False isSlave = False slaveReceivesFrameResyncCmd = False - X2HO_state = CONST.X2_HO_REQ_STATE__IDLE - X2HO_inNbProcedures = 0 - X2HO_outNbProcedures = 0 global_status = CONST.ALL_PROCESSES_OK # Runtime statistics runTime = '' @@ -297,31 +293,6 @@ class RANManagement(): if result is not None: fields=line.split(':') nbContextSwitches = 'nbContextSwitches : ' + fields[1].replace('\n','') - if X2HO_state == CONST.X2_HO_REQ_STATE__IDLE: - result = re.search('target eNB Receives X2 HO Req X2AP_HANDOVER_REQ', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ - result = re.search('source eNB receives the X2 HO ACK X2AP_HANDOVER_REQ_ACK', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK - if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ: - result = re.search('Received LTE_RRCConnectionReconfigurationComplete from UE', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE - if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE: - result = re.search('issue rrc_eNB_send_PATH_SWITCH_REQ', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ - if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ: - result = re.search('received path switch ack S1AP_PATH_SWITCH_REQ_ACK', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__IDLE - X2HO_inNbProcedures += 1 - if X2HO_state == CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK: - result = re.search('source eNB receives the X2 UE CONTEXT RELEASE X2AP_UE_CONTEXT_RELEASE', str(line)) - if result is not None: - X2HO_state = CONST.X2_HO_REQ_STATE__IDLE - X2HO_outNbProcedures += 1 if self.eNBOptions[0] != '': res1 = re.search('max_rxgain (?P[0-9]+)', self.eNBOptions[0]) @@ -419,10 +390,6 @@ class RANManagement(): result = re.search('dropping, not enough RBs', str(line)) if result is not None: dropNotEnoughRBs += 1 - if self.eNBmbmsEnables[0]: - result = re.search('MBMS USER-PLANE.*Requesting.*bytes from RLC', str(line)) - if result is not None: - mbmsRequestMsg += 1 #FR1 NSA test : add new markers to make sure gNB is used result = re.search(r'\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI 0x[0-9a-fA-F]+ received correctly, adding UE MAC Context RNTI 0x[0-9a-fA-F]+', str(line)) if result is not None: @@ -659,19 +626,6 @@ class RANManagement(): rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected' logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') htmleNBFailureMsg += rrcMsg + '\n' - if self.eNBmbmsEnables[0]: - if mbmsRequestMsg > 0: - rrcMsg = 'eNB requested ' + str(mbmsRequestMsg) + ' times the RLC for MBMS USER-PLANE' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if X2HO_inNbProcedures > 0: - rrcMsg = 'eNB completed ' + str(X2HO_inNbProcedures) + ' X2 Handover Connection procedure(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if X2HO_outNbProcedures > 0: - rrcMsg = 'eNB completed ' + str(X2HO_outNbProcedures) + ' X2 Handover Release procedure(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' if self.eNBOptions[0] != '': res1 = re.search('drx_Config_present prSetup', self.eNBOptions[0]) if res1 is not None: From 2e85f676461c6786b2e6945282742f258499d6a9 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 17 Feb 2026 18:58:02 +0100 Subject: [PATCH 15/25] Remove air_interface: it's not used --- ci-scripts/cls_oaicitest.py | 1 - ci-scripts/main.py | 14 -------------- ci-scripts/ran.py | 7 +++---- .../gnb_phytest_rfemulator_run_100_2x2.xml | 1 - ci-scripts/xml_files/gnb_phytest_usrp_run.xml | 2 -- .../xml_files/gnb_phytest_usrp_run_100_2x2.xml | 2 -- ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml | 2 -- .../xml_files/gnb_phytest_usrp_run_60_2x2.xml | 2 -- 8 files changed, 3 insertions(+), 28 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 67dc5c5e4d..850961c00b 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -301,7 +301,6 @@ class OaiCiTest(): self.iperf_profile = '' self.iperf_tcp_rate_target = '' self.finalStatus = False - self.air_interface='' self.ue_ids = [] self.svr_node = None self.svr_id = None diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 4628b1fcba..75f92bba3b 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -103,13 +103,6 @@ def ExecuteActionWithParam(action, ctx, node): elif action == 'Initialize_eNB': RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args') - #local variable air_interface - air_interface = test.findtext('air_interface') - if (air_interface is None) or (air_interface.lower() not in ['nr','lte']): - RAN.air_interface = 'lte-softmodem' - else: - RAN.air_interface = air_interface.lower() +'-softmodem' - cmd_prefix = test.findtext('cmd_prefix') if cmd_prefix is not None: RAN.cmd_prefix = cmd_prefix success = RAN.InitializeeNB(ctx, node, HTML) @@ -122,13 +115,6 @@ def ExecuteActionWithParam(action, ctx, node): string_field=test.findtext('u_retx_th') if (string_field is not None): RAN.ran_checkers['u_retx_th'] = [float(x) for x in string_field.split(',')] - - #local variable air_interface - air_interface = test.findtext('air_interface') - if (air_interface is None) or (air_interface.lower() not in ['nr','lte']): - RAN.air_interface = 'lte-softmodem' - else: - RAN.air_interface = air_interface.lower() +'-softmodem' success = RAN.TerminateeNB(ctx, node, HTML) elif action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Terminate_UE' or action == 'CheckStatusUE' or action == 'DataEnable_UE' or action == 'DataDisable_UE': diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index ef790b45c8..020ec5949f 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -64,7 +64,6 @@ class RANManagement(): self.eNBSourceCodePath = '' self.Initialize_eNB_args = '' self.imageKind = '' - self.air_interface = '' self.eNBOptions = ['', '', ''] self.eNBstatuses = [-1, -1, -1] self.runtime_stats= '' @@ -99,7 +98,7 @@ class RANManagement(): logfile = f'{lSourcePath}/cmake_targets/enb.log' cmd.cd(f"{lSourcePath}/cmake_targets/") # important: set wd so nrL1_stats.log etc are logged here - cmd.run(f'sudo -E stdbuf -o0 {self.cmd_prefix} {lSourcePath}/cmake_targets/ran_build/build/{self.air_interface} -O {lSourcePath}/{full_config_file} {extra_options} > {logfile} 2>&1 &') + cmd.run(f'sudo -E stdbuf -o0 {self.cmd_prefix} {lSourcePath}/cmake_targets/ran_build/build/nr-softmodem -O {lSourcePath}/{full_config_file} {extra_options} > {logfile} 2>&1 &') if extra_options != '': self.eNBOptions = extra_options @@ -113,12 +112,12 @@ class RANManagement(): enbDidSync = True break if not enbDidSync: - cmd.run(f'sudo killall -9 {self.air_interface}') # in case it did not stop automatically + cmd.run(f'sudo killall -9 nr-softmodem') # in case it did not stop automatically archiveArtifact(cmd, ctx, logfile) cmd.close() - msg = f'{self.cmd_prefix} {self.air_interface} -O {config_file} {extra_options}' + msg = f'{self.cmd_prefix} nr-softmodem -O {config_file} {extra_options}' if enbDidSync: logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m') HTML.CreateHtmlTestRowQueue(msg, 'OK', []) diff --git a/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml b/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml index 620d275d37..c91b9c8fc5 100644 --- a/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml @@ -59,7 +59,6 @@ caracal true Terminate gNB - NR diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml index f2c140faac..f2ad827d29 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml @@ -37,7 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.106prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 106 -t 23 -D 127 -m 28 -M 106 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - NR numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -59,7 +58,6 @@ true Terminate gNB caracal - NR diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml index 5204653223..d298873582 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml @@ -37,7 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.273prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 273 -t 23 -D 127 -m 23 -M 273 -l 2 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - NR numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -59,7 +58,6 @@ true Terminate gNB caracal - NR diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml index e939cb1e07..7354fa3596 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml @@ -37,7 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.162prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 162 -t 23 -D 127 -m 23 -M 162 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - NR numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -59,7 +58,6 @@ true Terminate gNB caracal - NR diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml index 1adcd042ac..7dd50ca83f 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml @@ -37,7 +37,6 @@ Initialize gNB USRP caracal -O ci-scripts/conf_files/gnb.band79.162prb.usrpn300.phytest-dora.conf --phy-test --gNBs.[0].pdsch_AntennaPorts_XP 2 --RUs.[0].nb_tx 2 --RUs.[0].nb_rx 2 -q -U 768 -T 162 -t 23 -D 127 -m 23 -M 162 -l 2 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time - NR numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0 @@ -59,7 +58,6 @@ true Terminate gNB caracal - NR From 03f56c10e7759bab5239819e2c2f5609bd9032c8 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 16 Feb 2026 23:48:31 +0100 Subject: [PATCH 16/25] GetDeployedServices(): return container ID will be relevant to get the return code in the next commit. --- ci-scripts/cls_containerize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 7826a32a61..c56bdb33d2 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -186,7 +186,7 @@ def GetDeployedServices(ssh, file): else: c = ret.stdout logging.info(f'service {s} with container id {c}') - deployed_services.append(s) + deployed_services.append((s, c)) return deployed_services def CheckLogs(self, filename, HTML, RAN): @@ -790,7 +790,7 @@ class Containerize(): wd_yaml = f'{wd}/docker-compose.y*ml' with cls_cmd.getConnection(node) as ssh: ExistEnvFilePrint(ssh, wd) - services = GetDeployedServices(ssh, wd_yaml) + services = [s for s, _ in GetDeployedServices(ssh, wd_yaml)] success = [] fail = [] for s in reqServices: @@ -820,7 +820,7 @@ class Containerize(): copyin_res = None ssh.run(f'docker compose -f {wd_yaml} stop') if services is not None: - copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, wd_yaml, ctx) for s in services] + copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, wd_yaml, ctx) for s, _ in services] else: logging.warning('could not identify services to stop => no log file') ssh.run(f'docker compose -f {wd_yaml} down -v') From fe464ec6d00002304fc7bf8180355d5e533c4188 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 17 Feb 2026 01:04:00 +0100 Subject: [PATCH 17/25] Initial code for log analysis - we agreed to basically throw away the old log checker (as it is unmaintainable) - now: provide list of analysis to do * all checkers in cls_log_analysis, can be unit tested (see also next commit) * looked up dynamically to simplify adding of new tests + config * configured with DESCRIPTION [DESCRIPTION...] DESCRIPTION ... where DESCRIPTION follows service=func[=options]: - service is name of service to check - func is function to call, which receives filename - option are arbitary options to pass to func is whitespace delimited (so can take multiple service analysis definitions) exists to allow service definitions with whitespace changes that would not work in - I initially planned to check return code, but most softmodems actually exit with non-zero return code, so this is still TODO - Default analyzer (checking for assertions, ...) is always run if a service is listed - checks for file size - add unit tests --- ci-scripts/cls_analysis.py | 98 +++++++++++++++++++ ci-scripts/cls_containerize.py | 21 ++-- ci-scripts/cls_loganalysis.py | 40 ++++++++ ci-scripts/main.py | 24 +++-- ci-scripts/ran.py | 14 ++- ci-scripts/tests/README.md | 1 + ci-scripts/tests/analysis.py | 14 +++ ci-scripts/tests/deployment.py | 48 +++++++-- ci-scripts/tests/log-analysis.py | 26 +++++ ci-scripts/tests/log-analysis/empty.log | 0 .../gnb_phytest_rfemulator_run_100_2x2.xml | 3 + 11 files changed, 257 insertions(+), 32 deletions(-) create mode 100644 ci-scripts/cls_loganalysis.py create mode 100644 ci-scripts/tests/log-analysis.py create mode 100644 ci-scripts/tests/log-analysis/empty.log diff --git a/ci-scripts/cls_analysis.py b/ci-scripts/cls_analysis.py index 3d9eb9c499..866a261352 100644 --- a/ci-scripts/cls_analysis.py +++ b/ci-scripts/cls_analysis.py @@ -24,10 +24,13 @@ import re import os import logging import yaml +import signal import xml.etree.ElementTree as ET import json +import importlib, inspect + class Analysis(): def _get_test_description(properties): @@ -147,3 +150,98 @@ class Analysis(): logging.debug(f'\u001B[1;30;43m normalized metric {k}={valnorm} deviates by more than {dev}\u001B[0m') success = False return success, datalog_rt_stats + + +# returns tuple of (service, analyzer, option string) +def _lookupServiceAnalyzerOpt(s, analyzers): + res = s.split("=", 2) + name = res[0] + if len(res) == 1: + return res[0], analyzers["Default"], None + opt = res[2] if len(res) > 2 else None + func = res[1] + a = analyzers[func] if func in analyzers else None + return name, a, opt + +# groups requested service analysis (service=func[=option]) on per service +# basis and looks up analyzer func. If log analysis is requested, will always +# run Default log analysis +def _groupServices(to_analyze): + req_analysis = {} + # get content of cls_loganalysis module, then get all analyzers (classes) in this module + mod = importlib.import_module("cls_loganalysis") + analyzers = {name:cl for name, cl in inspect.getmembers(mod, inspect.isclass)} + for req in to_analyze: + s, func, opt = _lookupServiceAnalyzerOpt(req, analyzers) + logging.debug(f"requested check '{req}' => service {s}, function {func.__name__}, options '{opt}'") + # always put default analyzer first + l = req_analysis[s] if s in req_analysis else [(analyzers["Default"], None, "default")] + if func is not analyzers["Default"]: + l.append((func, opt, req)) + req_analysis[s] = l + return req_analysis + +def _describe_exit_code(code): + if code > 128: + sig = code - 128 + try: + return f"terminated by signal {sig} ({signal.Signals(sig).name})" + except ValueError: + return f"terminated by unknown signal {sig}" + else: + return f"custom exit code" + +def AnalyzeServices(HTML, service_desc, to_analyze): + success = True + # hack: we want to give as a description "log analysis", but the description + # is set outside, so retain what was set before + orig_html_desc = HTML.desc + # group analysis on a per-service basis, then iterate + for serv, list_analysis in _groupServices(to_analyze).items(): + HTML.desc = f"Log analysis for service {serv}" + if serv not in service_desc: + success = False + logging.error(f"requested service {serv} not in list of services") + HTML.CreateHtmlTestRowQueue("N/A", 'KO', ["service not detected"]) + continue + # pre-initialize with return code + rc = service_desc[serv]["returncode"] + logging.info(f"analyze service {serv}: return code {rc}") + service_success = True # TODO rc == 0: too many functions (eNB, lteUE, RIC, nrUE) fail non-zero + logs = [] + if rc != 0: + logs.append(f"=> return code {rc}, likely " + _describe_exit_code(rc) + " [ignored by CI]") + # skip if the file is too big + logfile = service_desc[serv]["logfile"] + filename = os.path.basename(logfile) + b = os.path.getsize(logfile) + logging.debug(f"using logfile {logfile} of size {b} bytes") + if b > 10 * 1024 * 1024: + success = False + logs.append(f"logfile too big (>10MB)") + logging.error(logs) + HTML.CreateHtmlTestRowQueue(filename, 'KO', ["\n".join(logs)]) + continue + # run each analyzer with its options on the logfile + for (func, opt, desc) in list_analysis: + if func is None: + service_success = False + logging.error(f"request analysis function for desc {desc} not found") + HTML.CreateHtmlTestRowQueue(serv, 'KO', [f"no analysis function for {desc}"]) + continue + result, l = func.run(logfile, opt) + logging.info(f"service {serv}: analysis with func {func}, result {result}, logs '{l}'") + service_success = service_success and result + if not result: + logs.append(f"=> {func.__name__} check (options '{opt}') FAILED") + logs.append(l) + logs = '\n'.join(logs) + if not service_success: + logging.error(l) + else: + logging.info(l) + all_funcs = ", ".join([f.__name__ for (f, _, _) in list_analysis]) + HTML.CreateHtmlTestRowQueue(f"Check {all_funcs} on {filename}", 'OK' if service_success else 'KO', [logs]) + success = success and service_success + HTML.desc = orig_html_desc + return success diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index c56bdb33d2..16fc89613d 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -46,6 +46,7 @@ import cls_cmd import helpreadme as HELP import constants as CONST import cls_oaicitest +import cls_analysis from cls_ci_helper import archiveArtifact from collections import deque @@ -808,7 +809,7 @@ class Containerize(): HTML.CreateHtmlTestRowQueue(self.services, 'KO', [f'Failed stopping {" ".join(fail)}, succeeded {" ".join(success)}']) return success - def UndeployObject(self, ctx, node, HTML, RAN): + def UndeployObject(self, ctx, node, HTML, to_analyze): lSourcePath = self.eNBSourceCodePath logging.info(f'\u001B[1m Undeploying all objects from server {node}\u001B[0m') yaml = self.yamlPath.strip('/') @@ -817,21 +818,27 @@ class Containerize(): with cls_cmd.getConnection(node) as ssh: ExistEnvFilePrint(ssh, wd) services = GetDeployedServices(ssh, wd_yaml) - copyin_res = None + all_logs = True ssh.run(f'docker compose -f {wd_yaml} stop') if services is not None: - copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, wd_yaml, ctx) for s, _ in services] + service_desc = {} + for s, c in services: + ret = ssh.run(f'docker inspect {c} --format="{{{{.State.ExitCode}}}}"') + rc = int(ret.stdout.strip()) + f = CopyinServiceLog(ssh, lSourcePath, s, wd_yaml, ctx) + all_logs = all_logs and f is not None + service_desc[s] = {'returncode': rc, 'logfile': f} else: logging.warning('could not identify services to stop => no log file') ssh.run(f'docker compose -f {wd_yaml} down -v') + HTML.CreateHtmlTestRowQueue(node, 'OK', ['Undeployment successful']) ssh.run(f'rm {wd}/.env') - if not copyin_res: + if not all_logs: HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)']) - logging.error(f"could not copy all files: {copyin_res=} {services=}") + logging.error(f"could not copy all files: {all_logs=} {services=}") success = False else: - log_results = [CheckLogs(self, f, HTML, RAN) for f in copyin_res] - success = all(log_results) + success = cls_analysis.AnalyzeServices(HTML, service_desc, to_analyze) if success: logging.info('\u001B[1m Undeploying objects Pass\u001B[0m') else: diff --git a/ci-scripts/cls_loganalysis.py b/ci-scripts/cls_loganalysis.py new file mode 100644 index 0000000000..0003b8be69 --- /dev/null +++ b/ci-scripts/cls_loganalysis.py @@ -0,0 +1,40 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- + +import re +import os +import logging + +class Default: + def run(file, opt=None): + # TODO add check on assertions, etc + if not os.path.exists(file): + return False, f"cannot find file {file}" + return True, "" + +class ContainsString: + def run(file, needle): + with open(file, "r") as f: + for line in f.readlines(): + if re.search(needle, line): + return True, "" + return False, f"could not find '{needle}' in logs" diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 75f92bba3b..bbce96f304 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -115,7 +115,14 @@ def ExecuteActionWithParam(action, ctx, node): string_field=test.findtext('u_retx_th') if (string_field is not None): RAN.ran_checkers['u_retx_th'] = [float(x) for x in string_field.split(',')] - success = RAN.TerminateeNB(ctx, node, HTML) + services = [] + analysis = test.find("analysis") + if analysis is not None: + # services: multiple services to analyse, separated by whitespace + services = analysis.findtext("services", default="").split() + # service: individual services to analyze, in case they have whitespace + services = services + [s.text for s in analysis.findall("service")] + success = RAN.TerminateeNB(ctx, node, HTML, services) elif action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Terminate_UE' or action == 'CheckStatusUE' or action == 'DataEnable_UE' or action == 'DataDisable_UE': CiTestObj.ue_ids = test.findtext('id').split(' ') @@ -188,12 +195,6 @@ def ExecuteActionWithParam(action, ctx, node): elif action == 'Deploy_Object' or action == 'Undeploy_Object' or action == "Create_Workspace" or action == "Stop_Object": CONTAINERS.yamlPath = test.findtext('yaml_path') - string_field=test.findtext('d_retx_th') - if (string_field is not None): - CONTAINERS.ran_checkers['d_retx_th'] = [float(x) for x in string_field.split(',')] - string_field=test.findtext('u_retx_th') - if (string_field is not None): - CONTAINERS.ran_checkers['u_retx_th'] = [float(x) for x in string_field.split(',')] CONTAINERS.services = test.findtext('services') CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1) CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge) @@ -202,7 +203,14 @@ def ExecuteActionWithParam(action, ctx, node): elif action == 'Stop_Object': success = CONTAINERS.StopObject(ctx, node, HTML) elif action == 'Undeploy_Object': - success = CONTAINERS.UndeployObject(ctx, node, HTML, RAN) + analysis = test.find("analysis") + services = [] + if analysis is not None: + # services: multiple services to analyse, separated by whitespace + services = analysis.findtext("services", default="").split() + # service: individual services to analyze, in case they have whitespace + services = services + [s.text for s in analysis.findall("service")] + success = CONTAINERS.UndeployObject(ctx, node, HTML, services) elif action == 'Create_Workspace': if force_local: # Do not create a working directory when running locally. Current repo directory will be used diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 020ec5949f..27b309cafd 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -127,7 +127,7 @@ class RANManagement(): return enbDidSync - def TerminateeNB(self, ctx, node, HTML): + def TerminateeNB(self, ctx, node, HTML, to_analyze): logging.debug('Stopping eNB/gNB on server: ' + node) lSourcePath = self.eNBSourceCodePath cmd = cls_cmd.getConnection(node) @@ -142,6 +142,7 @@ class RANManagement(): if result is not None: cmd.run('sudo -S killall --signal SIGKILL -r .*-softmodem') time.sleep(5) + HTML.CreateHtmlTestRowQueue(node, 'OK', ['Undeployment successful']) # see InitializeeNB() logfile = f'{lSourcePath}/cmake_targets/enb.log' @@ -156,13 +157,10 @@ class RANManagement(): return False logging.debug('\u001B[1m Analyzing xNB logfile \u001B[0m ' + file) - logStatus = self.AnalyzeLogFile_eNB(file, HTML, self.ran_checkers) - if logStatus < 0: - HTML.CreateHtmlTestRow('N/A', 'KO', logStatus) - else: - HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) - - return logStatus >= 0 + service_desc = {} + service_desc["nr-softmodem"] = {'returncode': 0, 'logfile': file} + success = cls_analysis.AnalyzeServices(HTML, service_desc, to_analyze) + return success def AnalyzeRTStats(self, HTML, node, ctx, thresholds): logging.info(f'Analyzing realtime stats from server: {node}') diff --git a/ci-scripts/tests/README.md b/ci-scripts/tests/README.md index 48a48b8a31..ac3aa2115f 100644 --- a/ci-scripts/tests/README.md +++ b/ci-scripts/tests/README.md @@ -16,6 +16,7 @@ To run individual unit tests, start them like so: python tests/corenetwork.py -v python tests/deployment.py -v python tests/iperf-analysis.py -v + python tests/log-analysis.py -v python tests/ping-iperf.py -v python tests/pull-clean-int-registry.py -v diff --git a/ci-scripts/tests/analysis.py b/ci-scripts/tests/analysis.py index 37e4d3499e..e77bf14603 100644 --- a/ci-scripts/tests/analysis.py +++ b/ci-scripts/tests/analysis.py @@ -53,5 +53,19 @@ class TestAnalysis(unittest.TestCase): # TODO this one is not found, the CI does not recover this?? #self.assertEqual(s['Data']['DL & UL scheduling timing'], ['17', '97', '159371', '1.14']) + def test_analze_services_no_service(self): + service_desc = {} # nothing to analyze + to_analyze = ["testserv"] # some service requested to be analyze that does not exist + status = cls_analysis.AnalyzeServices(self.html, service_desc, to_analyze) + self.assertFalse(status) + + # TODO this does not work: we don't evaluate return codes yet + #def test_analze_service_exit_nonnull(self): + # service_desc = {} + # service_desc["testserv"] = {'returncode': 129, 'logfile': 'tests/log-analysis/empty.log'} + # to_analyze = ["testserv"] # some service requested to be analyze that does not exist + # status = cls_analysis.AnalyzeServices(self.html, service_desc, to_analyze) + # self.assertFalse(status) + if __name__ == '__main__': unittest.main() diff --git a/ci-scripts/tests/deployment.py b/ci-scripts/tests/deployment.py index 10bd2c843f..4fb466acc2 100644 --- a/ci-scripts/tests/deployment.py +++ b/ci-scripts/tests/deployment.py @@ -16,7 +16,6 @@ import cls_oai_html import cls_oaicitest import cls_containerize from cls_ci_helper import TestCaseCtx -import ran import cls_cmd class TestDeploymentMethods(unittest.TestCase): @@ -39,7 +38,6 @@ class TestDeploymentMethods(unittest.TestCase): self.html.testCaseId = "000000" self.ci = cls_oaicitest.OaiCiTest() self.cont = cls_containerize.Containerize() - self.ran = ran.RANManagement() self.cont.yamlPath = '' self.cont.ranAllowMerge = True self.cont.ranBranch = '' @@ -56,7 +54,7 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath = 'tests/simple-dep/' self.cont.deploymentTag = "noble" deploy = self.cont.DeployObject(self.ctx, self.node, self.html) - undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -79,7 +77,7 @@ class TestDeploymentMethods(unittest.TestCase): stopB = self.cont.StopObject(self.ctx, self.node, self.html) # should not undeploy anything (everything already stopped) self.cont.services = None - undeployAll = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + undeployAll = self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertTrue(deploy) self.assertFalse(stopC) self.assertTrue(stopA) @@ -92,7 +90,7 @@ class TestDeploymentMethods(unittest.TestCase): old = self.cont.yamlPath self.cont.yamlPath = 'tests/simple-fail/' deploy = self.cont.DeployObject(self.ctx, self.node, self.html) - self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertFalse(deploy) self.cont.yamlPath = old @@ -101,7 +99,7 @@ class TestDeploymentMethods(unittest.TestCase): old = self.cont.yamlPath self.cont.yamlPath = 'tests/simple-fail-2svc/' deploy = self.cont.DeployObject(self.ctx, self.node, self.html) - self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertFalse(deploy) self.cont.yamlPath = old @@ -110,7 +108,7 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.services = "oai-gnb" self.cont.deploymentTag = 'develop-12345678' deploy = self.cont.DeployObject(self.ctx, self.node, self.html) - undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -119,7 +117,7 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.services = "oai-gnb oai-nr-ue" self.cont.deploymentTag = 'develop-12345678' deploy = self.cont.DeployObject(self.ctx, self.node, self.html) - undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -130,7 +128,7 @@ class TestDeploymentMethods(unittest.TestCase): deploy1 = self.cont.DeployObject(self.ctx, self.node, self.html) self.cont.services = "oai-nr-ue" deploy2 = self.cont.DeployObject(self.ctx, self.node, self.html) - undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran) + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, []) self.assertTrue(deploy1) self.assertTrue(deploy2) self.assertTrue(undeploy) @@ -145,5 +143,37 @@ class TestDeploymentMethods(unittest.TestCase): cmd.run(f"rm -rf {self.cont.eNBSourceCodePath}") self.assertTrue(ws) + def test_undeploy_loganalysis(self): + self.cont.yamlPath = 'yaml_files/5g_rfsimulator_tdd_dora' + self.cont.services = "oai-gnb" + self.cont.deploymentTag = 'develop-12345678' + deploy = self.cont.DeployObject(self.ctx, self.node, self.html) + analyze = ["oai-gnb"] + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, analyze) + self.assertTrue(deploy) + self.assertTrue(undeploy) + + def test_undeploy_multi_loganalysis(self): + self.cont.yamlPath = 'yaml_files/5g_rfsimulator_tdd_dora' + self.cont.services = "oai-gnb" + self.cont.deploymentTag = 'develop-12345678' + deploy = self.cont.DeployObject(self.ctx, self.node, self.html) + analyze = ["oai-gnb", "oai-gnb=ContainsString=NOTFOUND"] + undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, analyze) + self.assertTrue(deploy) + self.assertFalse(undeploy) + + # TODO this does not work: we don't evaluate return codes yet + #def test_undeploy_rc_nonnull(self): + # # fails reliably + # old = self.cont.yamlPath + # self.cont.yamlPath = 'tests/simple-fail/' + # deploy = self.cont.DeployObject(self.ctx, self.node, self.html) + # # should be false because wrong exit code + # undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, ["test"]) + # self.assertFalse(deploy) + # self.assertFalse(undeploy) + # self.cont.yamlPath = old + if __name__ == '__main__': unittest.main() diff --git a/ci-scripts/tests/log-analysis.py b/ci-scripts/tests/log-analysis.py new file mode 100644 index 0000000000..dba7e9caf4 --- /dev/null +++ b/ci-scripts/tests/log-analysis.py @@ -0,0 +1,26 @@ +import sys +import logging +logging.basicConfig( + level=logging.DEBUG, + stream=sys.stdout, + format="[%(asctime)s] %(levelname)8s: %(message)s" +) +import os +import unittest + +sys.path.append('./') # to find OAI imports below +import cls_loganalysis + +class TestLogAnalysis(unittest.TestCase): + def test_no_file(self): + f = "/total/fantasy/path/file.log" + result, _ = cls_loganalysis.Default.run(f, None) + self.assertFalse(result) + + def test_no_error(self): + f = "tests/log-analysis/empty.log" + result, _ = cls_loganalysis.Default.run(f, None) + self.assertTrue(result) + +if __name__ == '__main__': + unittest.main() diff --git a/ci-scripts/tests/log-analysis/empty.log b/ci-scripts/tests/log-analysis/empty.log new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml b/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml index c91b9c8fc5..8429baa1ba 100644 --- a/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml @@ -59,6 +59,9 @@ caracal true Terminate gNB + + nr-softmodem=ContainsString=Bye. + From 707cb560d56511724ea8d2abd3a7208cb26e6ca9 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 18 Feb 2026 17:17:10 +0100 Subject: [PATCH 18/25] Implement retransmission checker, include unit test --- ci-scripts/cls_containerize.py | 2 - ci-scripts/cls_loganalysis.py | 94 +++++++++++++++++++ ci-scripts/main.py | 7 -- ci-scripts/ran.py | 66 ------------- ci-scripts/tests/log-analysis.py | 72 ++++++++++++++ .../tests/log-analysis/retx-check-bad.log | 54 +++++++++++ .../tests/log-analysis/retx-check-good.log | 8 ++ .../xml_files/container_5g_fdd_rfsim.xml | 2 - 8 files changed, 228 insertions(+), 77 deletions(-) create mode 100644 ci-scripts/tests/log-analysis/retx-check-bad.log create mode 100644 ci-scripts/tests/log-analysis/retx-check-good.log diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 16fc89613d..235b5cdce9 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -267,8 +267,6 @@ class Containerize(): self.dockerfileprefix = '' self.host = '' - #checkers from xml - self.ran_checkers={} self.num_attempts = 1 self.flexricTag = '' diff --git a/ci-scripts/cls_loganalysis.py b/ci-scripts/cls_loganalysis.py index 0003b8be69..2369f1115f 100644 --- a/ci-scripts/cls_loganalysis.py +++ b/ci-scripts/cls_loganalysis.py @@ -38,3 +38,97 @@ class ContainsString: if re.search(needle, line): return True, "" return False, f"could not find '{needle}' in logs" + +class RetxCheck(): + def _parseList(s): + if not s: + return [] + return [int(n) for n in s.split(",")] + + def _get_ul_dl(s): + dl, ul = (None, None) + for o in s.split(";", 1): + if o.startswith("dl="): + dl = o[3:] + elif o.startswith("ul="): + ul = o[3:] + else: + raise ValueError(f"unrecognized option {s}: needs format '[ul=,dl=]X,Y,...[;ul=Z,V,...]'") + return dl, ul + + def _parseRetxCheckers(options): + if not "=" in options: + dlul = RetxCheck._parseList(options) + return dlul, dlul + dl, ul = RetxCheck._get_ul_dl(options) + return RetxCheck._parseList(dl), RetxCheck._parseList(ul) + + def _analyzeUeRetx(rounds, checkers, regex): + perc = list(0 for i in checkers) # results in % + stats = list(False for i in checkers) # status if succeeded + tmp = re.match(regex, rounds) + if tmp is None: + logging.error('_analyzeUeRetx: did not match regex for DL retx analysis') + return stats + retx_data = [float(x) for x in tmp.groups()] + for i in range(0, len(perc)): + #case where numerator > denumerator with denum ==0 is disregarded, cannot hapen in principle, will lead to 0% + perc[i] = 0 if (retx_data[i] == 0) else 100 * retx_data[i + 1] / retx_data[i] + #treating % > 100 , % > requirement + stats[i] = perc[i] <= 100 and perc[i] <= checkers[i] + return stats + + def _filter_retx_rounds(l): + # map to round number if False, else None + l = [i+1 if x is False else None for i, x in enumerate(l)] + # filter out None + return [x for x in l if x is not None] + + def run(file, options): + # aggregate statistics per UE + dlsch_ulsch_stats = {} + status = True + with open(file, "r") as f: + for line in f.readlines(): + keys = {'dlsch_rounds','ulsch_rounds'} + for k in keys: + result = re.search(k, line) + if result is None: + continue + result = re.search('UE (?:RNTI )?([0-9a-f]{4})', line) + if result is None: + logging.error(f'did not find RNTI while matching key {k}') + status = False + continue + rnti = result.group(1) + if not rnti in dlsch_ulsch_stats: + dlsch_ulsch_stats[rnti] = {} + dlsch_ulsch_stats[rnti][k]=re.sub(r'^.*\]\s+', r'' , line.rstrip()) + + dl, ul = RetxCheck._parseRetxCheckers(options) + msgs = [] + for ue in dlsch_ulsch_stats: + dlulstat = dlsch_ulsch_stats[ue] + + for key in dlulstat: + msgs.append(dlulstat[key]) + + retx_dl = RetxCheck._analyzeUeRetx(dlulstat['dlsch_rounds'], dl, r'^.*dlsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+dlsch_errors\s+(\d+)') + retx_ul = RetxCheck._analyzeUeRetx(dlulstat['ulsch_rounds'], ul, r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+ulsch_errors\s+(\d+)') + + if False in retx_dl or False in retx_ul: + dlstr = RetxCheck._filter_retx_rounds(retx_dl) + ulstr = RetxCheck._filter_retx_rounds(retx_ul) + msg = f"UE {ue}: retx rounds" + if dlstr: + msg = f"{msg} DL {dlstr}" + if ulstr: + msg = f"{msg} UL {ulstr}" + msg = f"{msg} exceeded threshold" + msgs.append('!!! Failure: ' + msg) + logging.error(f'\u001B[1;37;41m {msg}\u001B[0m') + status = False + else: + msg = f"UE {ue} retransmissions within bounds" + logging.debug(msg) + return status, "\n".join(msgs) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index bbce96f304..3b40941b91 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -108,13 +108,6 @@ def ExecuteActionWithParam(action, ctx, node): success = RAN.InitializeeNB(ctx, node, HTML) elif action == 'Terminate_eNB': - #retx checkers - string_field = test.findtext('d_retx_th') - if (string_field is not None): - RAN.ran_checkers['d_retx_th'] = [float(x) for x in string_field.split(',')] - string_field=test.findtext('u_retx_th') - if (string_field is not None): - RAN.ran_checkers['u_retx_th'] = [float(x) for x in string_field.split(',')] services = [] analysis = test.find("analysis") if analysis is not None: diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 27b309cafd..864c2463a5 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -67,8 +67,6 @@ class RANManagement(): self.eNBOptions = ['', '', ''] self.eNBstatuses = [-1, -1, -1] self.runtime_stats= '' - #checkers from xml - self.ran_checkers={} self.cmd_prefix = '' # prefix before {lte,nr}-softmodem self.node = '' self.command = '' @@ -176,25 +174,6 @@ class RANManagement(): HTML.CreateHtmlDataLogTable(datalog_rt_stats) return success - def _analyzeUeRetx(self, rounds, checkers, regex): - if len(rounds) == 0 or len(checkers) == 0: - logging.warning(f'warning: rounds={rounds} checkers={checkers}') - return [] - - perc = list(0 for i in checkers) # results in % - stats = list(False for i in checkers) # status if succeeded - tmp = re.match(regex, rounds) - if tmp is None: - logging.error('_analyzeUeRetx: did not match regex for DL retx analysis') - return stats - retx_data = [float(x) for x in tmp.groups()] - for i in range(0, len(perc)): - #case where numerator > denumerator with denum ==0 is disregarded, cannot hapen in principle, will lead to 0% - perc[i] = 0 if (retx_data[i] == 0) else 100 * retx_data[i + 1] / retx_data[i] - #treating % > 100 , % > requirement - stats[i] = perc[i] <= 100 and perc[i] <= checkers[i] - return stats - def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}): if (not os.path.isfile(eNBlogFile)): return -1 @@ -239,8 +218,6 @@ class RANManagement(): nbContextSwitches = '' #NSA FR1 check NSA_RAPROC_PUSCH_check = 0 - #dlsch and ulsch statistics (dictionary) - dlsch_ulsch_stats = {} #count "problem receiving samples" msg pb_receiving_samples_cnt = 0 #count "removing UE" msg @@ -392,22 +369,6 @@ class RANManagement(): if result is not None: NSA_RAPROC_PUSCH_check = 1 - # Collect information on UE DLSCH and ULSCH statistics - keys = {'dlsch_rounds','ulsch_rounds'} - for k in keys: - result = re.search(k, line) - if result is None: - continue - result = re.search('UE (?:RNTI )?([0-9a-f]{4})', line) - if result is None: - logging.error(f'did not find RNTI while matching key {k}') - continue - rnti = result.group(1) - - #remove 1- all useless char before relevant info (ulsch or dlsch) 2- trailing char - if not rnti in dlsch_ulsch_stats: dlsch_ulsch_stats[rnti] = {} - dlsch_ulsch_stats[rnti][k]=re.sub(r'^.*\]\s+', r'' , line.rstrip()) - result = re.search('Received NR_RRCReconfigurationComplete from UE', str(line)) if result is not None: nrRrcRcfgComplete += 1 @@ -505,23 +466,6 @@ class RANManagement(): logging.debug(statMsg) htmleNBFailureMsg += htmlMsg - #ulsch and dlsch statistics and checkers - for ue in dlsch_ulsch_stats: - dlulstat = dlsch_ulsch_stats[ue] - #print statistics into html - statMsg='' - for key in dlulstat: - statMsg += dlulstat[key] + '\n' - logging.debug(dlulstat[key]) - htmleNBFailureMsg += statMsg - - retx_status[ue] = {} - dlcheckers = [] if 'd_retx_th' not in checkers else checkers['d_retx_th'] - retx_status[ue]['dl'] = self._analyzeUeRetx(dlulstat['dlsch_rounds'], dlcheckers, r'^.*dlsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+dlsch_errors\s+(\d+)') - ulcheckers = [] if 'u_retx_th' not in checkers else checkers['u_retx_th'] - retx_status[ue]['ul'] = self._analyzeUeRetx(dlulstat['ulsch_rounds'], ulcheckers, r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+ulsch_errors\s+(\d+)') - - if not showedByeMsg: logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB did not show "Bye." message at end, it likely did not stop properly! \u001B[0m') htmleNBFailureMsg += 'No Bye. message found, did not stop properly\n' @@ -557,16 +501,6 @@ class RANManagement(): logging.debug(statMsg) htmleNBFailureMsg += htmlMsg - for ue in retx_status: - msg = f"retransmissions for UE {ue}: DL {retx_status[ue]['dl']} UL {retx_status[ue]['ul']}" - if False in retx_status[ue]['dl'] or False in retx_status[ue]['ul']: - msg = 'Failure: ' + msg - logging.error(f'\u001B[1;37;41m {msg}\u001B[0m') - htmleNBFailureMsg += f'{msg}\n' - global_status = CONST.ENB_RETX_ISSUE - else: - logging.debug(msg) - if RealTimeProcessingIssue: logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with real time processing issue! \u001B[0m') htmleNBFailureMsg += 'Fail due to real time processing issue\n' diff --git a/ci-scripts/tests/log-analysis.py b/ci-scripts/tests/log-analysis.py index dba7e9caf4..d7e754a054 100644 --- a/ci-scripts/tests/log-analysis.py +++ b/ci-scripts/tests/log-analysis.py @@ -12,6 +12,9 @@ sys.path.append('./') # to find OAI imports below import cls_loganalysis class TestLogAnalysis(unittest.TestCase): + def setUp(self): + self.maxDiff = None + def test_no_file(self): f = "/total/fantasy/path/file.log" result, _ = cls_loganalysis.Default.run(f, None) @@ -22,5 +25,74 @@ class TestLogAnalysis(unittest.TestCase): result, _ = cls_loganalysis.Default.run(f, None) self.assertTrue(result) + def test_retx_check_good(self): + f = "tests/log-analysis/retx-check-good.log" + opt = "dl=1,0,0,0;ul=1,0,0,0" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3 +UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 51.0 dB CCE fail 0""" + self.assertTrue(result) + self.assertEqual(log, expected) + + def test_retx_check_good_single_val(self): + f = "tests/log-analysis/retx-check-good.log" + opt = "1,0,0,0" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3 +UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 51.0 dB CCE fail 0""" + self.assertTrue(result) + self.assertEqual(log, expected) + + def test_retx_check_good_ul_only(self): + f = "tests/log-analysis/retx-check-good.log" + opt = "ul=1,0,0,0" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3 +UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 51.0 dB CCE fail 0""" + self.assertTrue(result) + self.assertEqual(log, expected) + + def test_retx_check_bad(self): + f = "tests/log-analysis/retx-check-bad.log" + opt = "dl=10,100,100,100;ul=10,100,100,100" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0 +UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 34.0 dB CCE fail 0 +UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64 +UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104 SNR 38.0 dB CCE fail 0 +!!! Failure: UE bf9b: retx rounds DL [1] UL [1] exceeded threshold +UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0 +UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5 SNR 29.0 dB CCE fail 0""" + self.assertFalse(result) + self.assertEqual(log, expected) + + def test_retx_check_dl_only_bad(self): + f = "tests/log-analysis/retx-check-bad.log" + opt = "dl=10,80,100,100" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0 +UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 34.0 dB CCE fail 0 +UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64 +UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104 SNR 38.0 dB CCE fail 0 +!!! Failure: UE bf9b: retx rounds DL [1, 2] exceeded threshold +UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0 +UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5 SNR 29.0 dB CCE fail 0""" + self.assertFalse(result) + self.assertEqual(log, expected) + + def test_retx_check_ul_bad(self): + f = "tests/log-analysis/retx-check-bad.log" + opt = "dl=100,100,100,100;ul=10,100,100,100" + result, log = cls_loganalysis.RetxCheck.run(f, opt) + expected = """UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0 +UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 34.0 dB CCE fail 0 +UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64 +UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104 SNR 38.0 dB CCE fail 0 +!!! Failure: UE bf9b: retx rounds UL [1] exceeded threshold +UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0 +UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5 SNR 29.0 dB CCE fail 0""" + self.assertFalse(result) + self.assertEqual(log, expected) + if __name__ == '__main__': unittest.main() diff --git a/ci-scripts/tests/log-analysis/retx-check-bad.log b/ci-scripts/tests/log-analysis/retx-check-bad.log new file mode 100644 index 0000000000..7651da78cf --- /dev/null +++ b/ci-scripts/tests/log-analysis/retx-check-bad.log @@ -0,0 +1,54 @@ +9185541.336575 [NR_RRC] I UE 1 Logical Channel DL-DCCH, Generate SecurityModeCommand (bytes 3) +9185541.356794 [NR_RRC] A [UL] (cellID bc614e, UE ID 1 RNTI a491) Received Security Mode Complete +9185541.357264 [NR_RRC] I UE 1: Logical Channel DL-DCCH, Generate NR UECapabilityEnquiry (bytes 8, xid 2) +9185541.373199 [NR_RRC] A [UL] (cellID bc614e, UE ID 1 RNTI a491) Received UE capabilities +9185541.373330 [NR_RRC] I Send message to ngap: NGAP_UE_CAPABILITIES_IND +9185541.373356 [NR_RRC] A [DL] (cellID bc614e, UE ID 1 RNTI a491) Send DL Information Transfer [53 bytes] +9185541.373368 [NR_RRC] I Send message to sctp: NGAP_InitialContextSetupResponse +9185541.390034 [NR_MAC] I Frame.Slot 384.0 +UE RNTI a491 CU-UE-ID 1 in-sync PH 59 dB PCMAX 21 dBm, average RSRP -101 (4 meas) +UE a491: CQI 2, RI 2, PMI (0,0) +UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0 +UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 34.0 dB CCE fail 0 +UE a491: MAC: TX 367 RX 3664 bytes +UE a491: LCID 1: TX 213 RX 1649 bytes + +9185559.310044 [NR_MAC] I Frame.Slot 128.0 +UE RNTI bf9b CU-UE-ID 1 in-sync PH 48 dB PCMAX 21 dBm +UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64 +UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104 SNR 38.0 dB CCE fail 0 +UE bf9b: MAC: TX 10145 RX 82 bytes +UE bf9b: LCID 1: TX 5283 RX 53 bytes + +9185582.350069 [NR_MAC] I Frame.Slot 384.0 +UE RNTI 257a CU-UE-ID 2 in-sync PH 48 dB PCMAX 21 dBm, average RSRP -83 (16 meas) +UE 257a: CQI 15, RI 2, PMI (0,1) +UE 257a: UL-RI 2, TPMI 0 +UE 257a: dlsch_rounds 153/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00000 MCS (1) 0 CCE fail 0 +UE 257a: ulsch_rounds 562/2/1/0, ulsch_errors 0, ulsch_DTX 3, BLER 0.00000 MCS (1) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 29.5 dB CCE fail 0 +UE 257a: MAC: TX 13416 RX 26519 bytes +UE 257a: LCID 1: TX 523 RX 2046 bytes +UE 257a: LCID 2: TX 0 RX 0 bytes +UE 257a: LCID 4: TX 9800 RX 9945 bytes + +9185687.310202 [NR_MAC] I Frame.Slot 640.0 +UE RNTI 257a CU-UE-ID 2 in-sync PH 61 dB PCMAX 21 dBm, average RSRP -83 (16 meas) +UE 257a: CQI 15, RI 2, PMI (0,1) +UE 257a: UL-RI 2, TPMI 0 +UE 257a: dlsch_rounds 64992/129/13/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00242 MCS (1) 27 CCE fail 0 +UE 257a: ulsch_rounds 39328/2610/8/0, ulsch_errors 0, ulsch_DTX 14, BLER 0.11139 MCS (1) 9 (Qm 4 deltaMCS 0 dB) NPRB 273 SNR 25.5 dB CCE fail 0 +UE 257a: MAC: TX 3344707955 RX 818603547 bytes +UE 257a: LCID 1: TX 832 RX 4312 bytes +UE 257a: LCID 2: TX 0 RX 0 bytes +UE 257a: LCID 4: TX 3322610250 RX 815642955 bytes + +9185692.430254 [NR_MAC] I Frame.Slot 128.0 +UE RNTI 257a CU-UE-ID 2 in-sync PH 48 dB PCMAX 21 dBm, average RSRP -84 (16 meas) +UE 257a: CQI 15, RI 2, PMI (0,1) +UE 257a: UL-RI 2, TPMI 0 +UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0 +UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5 SNR 29.0 dB CCE fail 0 +UE 257a: MAC: TX 3424886787 RX 879242724 bytes +UE 257a: LCID 1: TX 847 RX 4422 bytes +UE 257a: LCID 2: TX 0 RX 0 bytes +UE 257a: LCID 4: TX 3401067134 RX 875911216 bytes diff --git a/ci-scripts/tests/log-analysis/retx-check-good.log b/ci-scripts/tests/log-analysis/retx-check-good.log new file mode 100644 index 0000000000..c586c615f3 --- /dev/null +++ b/ci-scripts/tests/log-analysis/retx-check-good.log @@ -0,0 +1,8 @@ +7799905.793901 [NR_MAC] I Frame.Slot 256.0 +UE RNTI f40a CU-UE-ID 2 in-sync PH 48 dB PCMAX 20 dBm, average RSRP -44 (16 meas) +UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3 +UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5 SNR 51.0 dB CCE fail 0 +UE f40a: MAC: TX 3836 RX 13684 bytes +UE f40a: LCID 1: TX 530 RX 281 bytes +UE f40a: LCID 2: TX 3 RX 35 bytes +UE f40a: LCID 4: TX 1886 RX 2100 bytes diff --git a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml index b4383feeaf..eb616d9e35 100644 --- a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml @@ -113,8 +113,6 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_fdd_rfsimulator - 1,0,0,0 - 1,0,0,0 From 30a27432f0eea0c7da5a2bffeed1042b5c5d92bf Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 17 Feb 2026 18:56:28 +0100 Subject: [PATCH 19/25] Remove unused logging and unused imports --- ci-scripts/cls_containerize.py | 60 ----- ci-scripts/cls_oaicitest.py | 259 ------------------- ci-scripts/ran.py | 441 --------------------------------- 3 files changed, 760 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 235b5cdce9..e21cb3771b 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -31,24 +31,17 @@ #----------------------------------------------------------- # Import #----------------------------------------------------------- -import sys # arg import re # reg import logging import os -import shutil -import time -from zipfile import ZipFile #----------------------------------------------------------- # OAI Testing modules #----------------------------------------------------------- import cls_cmd -import helpreadme as HELP import constants as CONST -import cls_oaicitest import cls_analysis from cls_ci_helper import archiveArtifact -from collections import deque #----------------------------------------------------------- # Helper functions used here and in other classes @@ -190,59 +183,6 @@ def GetDeployedServices(ssh, file): deployed_services.append((s, c)) return deployed_services -def CheckLogs(self, filename, HTML, RAN): - success = True - name = os.path.basename(filename) - if (any(sub in name for sub in ['oai_ue','oai-nr-ue','lte_ue'])): - logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m') - logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(filename, HTML, RAN) - opt = f"UE log analysis ({name})" - # usage of htmlUEFailureMsg/htmleNBFailureMsg is because Analyze log files - # abuse HTML to store their reports, and we here want to put custom options, - # which is not possible with CreateHtmlTestRow - # solution: use HTML templates, where we don't need different HTML write funcs - if (logStatus < 0): - HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmlUEFailureMsg]) - success = False - else: - HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmlUEFailureMsg]) - HTML.htmlUEFailureMsg = "" - elif 'nv-cubb' in name: - msg = 'Undeploy PNF/Nvidia CUBB' - HTML.CreateHtmlTestRow(msg, 'OK', CONST.ALL_PROCESSES_OK) - elif (any(sub in name for sub in ['enb','rru','rcc','cu','du','gnb','vnf'])): - logging.debug(f'\u001B[1m Analyzing XnB logfile {filename}\u001B[0m') - logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers) - opt = f"xNB log analysis ({name})" - if (logStatus < 0): - HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmleNBFailureMsg]) - success = False - else: - HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg]) - HTML.htmleNBFailureMsg = "" - elif 'xapp' in name: - opt = f"Undeploy {name}" - with open(f'{filename}', "r") as f: - last_line = deque(f, maxlen=1).pop() - if ('Test xApp run SUCCESSFULLY' in last_line): - HTML.CreateHtmlTestRowQueue(opt, 'OK', ["xApp run successfully"]) - else: - HTML.CreateHtmlTestRowQueue(opt, 'KO', ["xApp didn't run successfully"]) - success = False - elif 'RIC' in name: - opt = f"Undeploy {name}" - with open(f'{filename}', 'r') as f: - last_line = deque(f, maxlen=1).pop() - if ('Removing E2 Node' in last_line): - HTML.CreateHtmlTestRowQueue(opt, 'OK', ["nearRT-RIC run successfully"]) - else: - HTML.CreateHtmlTestRowQueue(opt, 'KO', ["nearRT-RIC didn't run successfully"]) - success = False - else: - logging.info(f"Skipping analysis of log '{filename}': no submatch for xNB/UE") - logging.debug(f"log check: file {filename} passed analysis {success}") - return success - #----------------------------------------------------------- # Class Declaration #----------------------------------------------------------- diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 850961c00b..26ac91e668 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -32,7 +32,6 @@ #----------------------------------------------------------- # Import Libs #----------------------------------------------------------- -import sys # arg import re # reg import time # sleep import os @@ -41,7 +40,6 @@ import concurrent.futures import json #import our libs -import helpreadme as HELP import constants as CONST import cls_module @@ -578,263 +576,6 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', [f'{ue_header}\n{msg}']) return success - def AnalyzeLogFile_UE(self, UElogFile,HTML,RAN): - if (not os.path.isfile(f'{UElogFile}')): - return -1 - ue_log_file = open(f'{UElogFile}', 'r') - exitSignalReceived = False - foundAssertion = False - msgAssertion = '' - msgLine = 0 - foundSegFault = False - foundRealTimeIssue = False - uciStatMsgCount = 0 - pdcpDataReqFailedCount = 0 - badDciCount = 0 - f1aRetransmissionCount = 0 - fatalErrorCount = 0 - macBsrTimerExpiredCount = 0 - rrcConnectionRecfgComplete = 0 - no_cell_sync_found = False - mib_found = False - frequency_found = False - plmn_found = False - nrUEFlag = False - nrDecodeMib = 0 - nrFoundDCI = 0 - nrCRCOK = 0 - nbPduSessAccept = 0 - nbPduDiscard = 0 - HTML.htmlUEFailureMsg='' - global_status = CONST.ALL_PROCESSES_OK - for line in ue_log_file.readlines(): - result = re.search('nr_synchro_time|Starting NR UE soft modem', str(line)) - sidelink = re.search('sl-mode', str(line)) - if result is not None: - nrUEFlag = True - if sidelink is not None: - nrUEFlag = False - if nrUEFlag: - result = re.search('decode mib', str(line)) - if result is not None: - nrDecodeMib += 1 - result = re.search('found 1 DCIs', str(line)) - if result is not None: - nrFoundDCI += 1 - result = re.search('CRC OK', str(line)) - if result is not None: - nrCRCOK += 1 - result = re.search('Received PDU Session Establishment Accept', str(line)) - if result is not None: - nbPduSessAccept += 1 - result = re.search('warning: discard PDU, sn out of window', str(line)) - if result is not None: - nbPduDiscard += 1 - result = re.search('--nfapi STANDALONE_PNF --node-number 2', str(line)) - if result is not None: - frequency_found = True - result = re.search('Exiting OAI softmodem', str(line)) - if result is not None: - exitSignalReceived = True - result = re.search('System error|[Ss]egmentation [Ff]ault|======= Backtrace: =========|======= Memory map: ========', str(line)) - if result is not None and not exitSignalReceived: - foundSegFault = True - result = re.search('[Cc]ore [dD]ump', str(line)) - if result is not None and not exitSignalReceived: - foundSegFault = True - result = re.search('[Aa]ssertion', str(line)) - if result is not None and not exitSignalReceived: - foundAssertion = True - result = re.search('LLL', str(line)) - if result is not None and not exitSignalReceived: - foundRealTimeIssue = True - if foundAssertion and (msgLine < 3): - msgLine += 1 - msgAssertion += str(line) - result = re.search('uci->stat', str(line)) - if result is not None and not exitSignalReceived: - uciStatMsgCount += 1 - result = re.search('PDCP data request failed', str(line)) - if result is not None and not exitSignalReceived: - pdcpDataReqFailedCount += 1 - result = re.search('bad DCI 1', str(line)) - if result is not None and not exitSignalReceived: - badDciCount += 1 - result = re.search('Format1A Retransmission but TBS are different', str(line)) - if result is not None and not exitSignalReceived: - f1aRetransmissionCount += 1 - result = re.search('FATAL ERROR', str(line)) - if result is not None and not exitSignalReceived: - fatalErrorCount += 1 - result = re.search('MAC BSR Triggered ReTxBSR Timer expiry', str(line)) - if result is not None and not exitSignalReceived: - macBsrTimerExpiredCount += 1 - result = re.search('Generating RRCConnectionReconfigurationComplete', str(line)) - if result is not None: - rrcConnectionRecfgComplete += 1 - # No cell synchronization found, abandoning - result = re.search('No cell synchronization found, abandoning', str(line)) - if result is not None: - no_cell_sync_found = True - result = re.search(r"MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P\d{1,3}), N_RB_DL (?P\d{1,3}), PHICH DURATION (?P\d), PHICH RESOURCE (?P.{1,4}), TX_ANT (?P\d)", str(line)) - if result is not None and (not mib_found): - try: - mibMsg = "MIB Information: " + result.group(1) + ', ' + result.group(2) - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - mibMsg = " nidcell = " + result.group('nidcell') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg - logging.debug(f'\033[94m{mibMsg}\033[0m') - mibMsg = " n_rb_dl = " + result.group('n_rb_dl') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - mibMsg = " phich_duration = " + result.group('phich_duration') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg - logging.debug(f'\033[94m{mibMsg}\033[0m') - mibMsg = " phich_resource = " + result.group('phich_resource') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - mibMsg = " tx_ant = " + result.group('tx_ant') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - mib_found = True - except Exception as e: - logging.error(f'\033[91m MIB marker was not found \033[0m') - result = re.search("Initial sync: pbch decoded sucessfully", str(line)) - if result is not None and (not frequency_found): - try: - mibMsg = f"UE decoded PBCH successfully" - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - frequency_found = True - except Exception as e: - logging.error(f'\033[91m UE did not find PBCH\033[0m') - result = re.search(r"PLMN MCC (?P\d{1,3}), MNC (?P\d{1,3}), TAC", str(line)) - if result is not None and (not plmn_found): - try: - mibMsg = f"PLMN MCC = {result.group('mcc')} MNC = {result.group('mnc')}" - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - plmn_found = True - except Exception as e: - logging.error(f'\033[91m PLMN not found \033[0m') - result = re.search(r"Found (?P[\w,\s]{1,15}) \(name from internal table\)", str(line)) - if result is not None: - try: - mibMsg = f"The operator is: {result.group('operator')}" - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n' - logging.debug(f'\033[94m{mibMsg}\033[0m') - except Exception as e: - logging.error(f'\033[91m Operator name not found \033[0m') - result = re.search("SIB5 InterFreqCarrierFreq element (.{1,4})/(.{1,4})", str(line)) - if result is not None: - try: - mibMsg = f'SIB5 InterFreqCarrierFreq element {result.group(1)}/{result.group(2)}' - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + ' -> ' - logging.debug(f'\033[94m{mibMsg}\033[0m') - except Exception as e: - logging.error(f'\033[91m SIB5 InterFreqCarrierFreq element not found \033[0m') - result = re.search(r"DL Carrier Frequency/ARFCN : \-*(?P\d{1,15}/\d{1,4})", str(line)) - if result is not None: - try: - freq = result.group('carrier_frequency') - new_freq = re.sub('/[0-9]+','',freq) - float_freq = float(new_freq) / 1000000 - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'DL Freq: ' + ('%.1f' % float_freq) + ' MHz' - logging.debug(f'\033[94m DL Carrier Frequency is: {freq}\033[0m') - except Exception as e: - logging.error(f'\033[91m DL Carrier Frequency not found \033[0m') - result = re.search(r"AllowedMeasBandwidth : (?P\d{1,7})", str(line)) - if result is not None: - try: - prb = result.group('allowed_bandwidth') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + ' -- PRB: ' + prb + '\n' - logging.debug(f'\033[94m AllowedMeasBandwidth: {prb}\033[0m') - except Exception as e: - logging.error(f'\033[91m AllowedMeasBandwidth not found \033[0m') - ue_log_file.close() - if rrcConnectionRecfgComplete > 0: - statMsg = f'UE connected to eNB ({rrcConnectionRecfgComplete}) RRCConnectionReconfigurationComplete message(s) generated)' - logging.debug(f'\033[94m{statMsg}\033[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if nrUEFlag: - if nrDecodeMib > 0: - statMsg = f'UE showed {nrDecodeMib} "MIB decode" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if nrFoundDCI > 0: - statMsg = f'UE showed {nrFoundDCI} "DCI found" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if nrCRCOK > 0: - statMsg = f'UE showed {nrCRCOK} "PDSCH decoding" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if not frequency_found: - statMsg = 'NR-UE could NOT synch!' - logging.error(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if nbPduSessAccept > 0: - statMsg = f'UE showed {nbPduSessAccept} "Received PDU Session Establishment Accept" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if nbPduDiscard > 0: - statMsg = f'UE showed {nbPduDiscard} "warning: discard PDU, sn out of window" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if uciStatMsgCount > 0: - statMsg = f'UE showed {uciStatMsgCount} "uci->stat" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if pdcpDataReqFailedCount > 0: - statMsg = f'UE showed {pdcpDataReqFailedCount} "PDCP data request failed" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if badDciCount > 0: - statMsg = f'UE showed {badDciCount} "bad DCI 1(A)" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if f1aRetransmissionCount > 0: - statMsg = f'UE showed {f1aRetransmissionCount} "Format1A Retransmission but TBS are different" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if fatalErrorCount > 0: - statMsg = f'UE showed {fatalErrorCount} "FATAL ERROR:" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if macBsrTimerExpiredCount > 0: - statMsg = f'UE showed {fatalErrorCount} "MAC BSR Triggered ReTxBSR Timer expiry" message(s)' - logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n' - if foundSegFault: - logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m') - if not nrUEFlag: - global_status = CONST.OAI_UE_PROCESS_SEG_FAULT - else: - if not frequency_found: - global_status = CONST.OAI_UE_PROCESS_SEG_FAULT - if foundAssertion: - logging.debug('\u001B[1;30;43m UE showed an assertion! \u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE showed an assertion!\n' - if not nrUEFlag: - if not mib_found or not frequency_found: - global_status = CONST.OAI_UE_PROCESS_ASSERTION - else: - if not frequency_found: - global_status = CONST.OAI_UE_PROCESS_ASSERTION - if foundRealTimeIssue: - logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE faced real time issues!\n' - if nrUEFlag: - if not frequency_found: - global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC - else: - if no_cell_sync_found and not mib_found: - logging.debug('\u001B[1;37;41m UE could not synchronize ! \u001B[0m') - HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE could not synchronize!\n' - global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC - return global_status - def TerminateUE(self, ctx, node, HTML): ues = [cls_module.Module_UE(n.strip(), node) for n in self.ue_ids] with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor: diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 864c2463a5..950b868ad3 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -31,21 +31,15 @@ #----------------------------------------------------------- # Import #----------------------------------------------------------- -import sys # arg import re # reg import logging import os import time -import yaml -import cls_cmd - #----------------------------------------------------------- # OAI Testing modules #----------------------------------------------------------- import cls_cmd -import helpreadme as HELP -import constants as CONST import cls_analysis from cls_ci_helper import archiveArtifact @@ -173,438 +167,3 @@ class RANManagement(): success, datalog_rt_stats = cls_analysis.Analysis.analyze_rt_stats(thresholds, l1_file, mac_file) HTML.CreateHtmlDataLogTable(datalog_rt_stats) return success - - def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}): - if (not os.path.isfile(eNBlogFile)): - return -1 - enb_log_file = open(eNBlogFile, 'r') - exitSignalReceived = False - foundAssertion = False - msgAssertion = '' - msgLine = 0 - foundSegFault = False - foundRealTimeIssue = False - foundRealTimeIssue_cnt = 0 - rrcSetupComplete = 0 - rrcReleaseRequest = 0 - rrcReconfigRequest = 0 - rrcReconfigComplete = 0 - rrcReestablishRequest = 0 - rrcReestablishComplete = 0 - rrcReestablishReject = 0 - rlcDiscardBuffer = 0 - rachCanceledProcedure = 0 - uciStatMsgCount = 0 - pdcpFailure = 0 - ulschFailure = 0 - ulschAllocateCCEerror = 0 - uplinkSegmentsAborted = 0 - ulschReceiveOK = 0 - gnbRxTxWakeUpFailure = 0 - gnbTxWriteThreadEnabled = False - cdrxActivationMessageCount = 0 - dropNotEnoughRBs = 0 - mbmsRequestMsg = 0 - htmleNBFailureMsg = '' - isRRU = False - isSlave = False - slaveReceivesFrameResyncCmd = False - global_status = CONST.ALL_PROCESSES_OK - # Runtime statistics - runTime = '' - userTime = '' - systemTime = '' - maxPhyMemUsage = '' - nbContextSwitches = '' - #NSA FR1 check - NSA_RAPROC_PUSCH_check = 0 - #count "problem receiving samples" msg - pb_receiving_samples_cnt = 0 - #count "removing UE" msg - removing_ue = 0 - #count"X2AP-PDU" - x2ap_pdu = 0 - #gnb specific log markers - gnb_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': [], 'Detected UL Failure on PUSCH':[]} - nodeB_prefix_found = False - RealTimeProcessingIssue = False - retx_status = {} - nrRrcRcfgComplete = 0 - harqFeedbackPast = 0 - showedByeMsg = False # last line is Bye. -> stopped properly - - line_cnt=0 #log file line counter - for line in enb_log_file.readlines(): - line_cnt+=1 - # Detection of eNB/gNB from a container log - result = re.search('Starting eNB soft modem', str(line)) - if result is not None: - nodeB_prefix_found = True - nodeB_prefix = 'e' - result = re.search('Starting gNB soft modem', str(line)) - if result is not None: - nodeB_prefix_found = True - nodeB_prefix = 'g' - result = re.search('Run time:' ,str(line)) - # Runtime statistics - result = re.search('Run time:' ,str(line)) - if result is not None: - runTime = str(line).strip() - if runTime != '': - result = re.search('Time executing user inst', str(line)) - if result is not None: - fields=line.split(':') - userTime = 'userTime : ' + fields[1].replace('\n','') - result = re.search('Time executing system inst', str(line)) - if result is not None: - fields=line.split(':') - systemTime = 'systemTime : ' + fields[1].replace('\n','') - result = re.search('Max. Phy. memory usage:', str(line)) - if result is not None: - fields=line.split(':') - maxPhyMemUsage = 'maxPhyMemUsage : ' + fields[1].replace('\n','') - result = re.search('Number of context switch.*process origin', str(line)) - if result is not None: - fields=line.split(':') - nbContextSwitches = 'nbContextSwitches : ' + fields[1].replace('\n','') - - if self.eNBOptions[0] != '': - res1 = re.search('max_rxgain (?P[0-9]+)', self.eNBOptions[0]) - res2 = re.search('max_rxgain (?P[0-9]+)', str(line)) - if res1 is not None and res2 is not None: - requested_option = int(res1.group('requested_option')) - applied_option = int(res2.group('applied_option')) - if requested_option == applied_option: - htmleNBFailureMsg += ' Command line option(s) correctly applied ' + self.eNBOptions[0] + '\n\n' - else: - htmleNBFailureMsg += ' Command line option(s) NOT applied ' + self.eNBOptions[0] + '\n\n' - result = re.search('Exiting OAI softmodem|Caught SIGTERM, shutting down', str(line)) - if result is not None: - exitSignalReceived = True - result = re.search('[Ss]egmentation [Ff]ault', str(line)) - if result is not None and not exitSignalReceived: - foundSegFault = True - result = re.search('[Cc]ore [dD]ump', str(line)) - if result is not None and not exitSignalReceived: - foundSegFault = True - result = re.search('[Aa]ssertion', str(line)) - if result is not None and not exitSignalReceived: - foundAssertion = True - result = re.search('LLL', str(line)) - if result is not None and not exitSignalReceived: - foundRealTimeIssue = True - foundRealTimeIssue_cnt += 1 - if foundAssertion and (msgLine < 3): - msgLine += 1 - msgAssertion += str(line) - result = re.search('Setting function for RU', str(line)) - if result is not None: - isRRU = True - if isRRU: - result = re.search('RU 0 is_slave=yes', str(line)) - if result is not None: - isSlave = True - if isSlave: - result = re.search('Received RRU_frame_resynch command', str(line)) - if result is not None: - slaveReceivesFrameResyncCmd = True - result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line)) - if result is not None: - rrcSetupComplete += 1 - result = re.search('Generate LTE_RRCConnectionRelease|Generate RRCConnectionRelease', str(line)) - if result is not None: rrcReleaseRequest += 1 - result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line)) - if result is not None: - rrcReconfigRequest += 1 - result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line)) - if result is not None: - rrcReconfigComplete += 1 - result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line)) - if result is not None: - rrcReestablishRequest += 1 - result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line)) - if result is not None: - rrcReestablishComplete += 1 - result = re.search('LTE_RRCConnectionReestablishmentReject', str(line)) - if result is not None: - rrcReestablishReject += 1 - result = re.search('CDRX configuration activated after RRC Connection', str(line)) - if result is not None: - cdrxActivationMessageCount += 1 - result = re.search('uci->stat', str(line)) - if result is not None: - uciStatMsgCount += 1 - result = re.search('PDCP.*Out of Resources.*reason', str(line)) - if result is not None: - pdcpFailure += 1 - result = re.search('could not wakeup gNB rxtx process', str(line)) - if result is not None: - gnbRxTxWakeUpFailure += 1 - result = re.search('tx write thread ready', str(line)) - if result is not None: - gnbTxWriteThreadEnabled = True - result = re.search('ULSCH in error in round|ULSCH 0 in error', str(line)) - if result is not None: - ulschFailure += 1 - result = re.search('ERROR ALLOCATING CCEs', str(line)) - if result is not None: - ulschAllocateCCEerror += 1 - result = re.search('uplink segment error.*aborted [1-9] segments', str(line)) - if result is not None: - uplinkSegmentsAborted += 1 - result = re.search('ULSCH received ok', str(line)) - if result is not None: - ulschReceiveOK += 1 - result = re.search('BAD all_segments_received', str(line)) - if result is not None: - rlcDiscardBuffer += 1 - result = re.search('Canceled RA procedure for UE rnti', str(line)) - if result is not None: - rachCanceledProcedure += 1 - result = re.search('dropping, not enough RBs', str(line)) - if result is not None: - dropNotEnoughRBs += 1 - #FR1 NSA test : add new markers to make sure gNB is used - result = re.search(r'\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI 0x[0-9a-fA-F]+ received correctly, adding UE MAC Context RNTI 0x[0-9a-fA-F]+', str(line)) - if result is not None: - NSA_RAPROC_PUSCH_check = 1 - - result = re.search('Received NR_RRCReconfigurationComplete from UE', str(line)) - if result is not None: - nrRrcRcfgComplete += 1 - result = re.search('HARQ feedback is in the past', str(line)) - if result is not None: - harqFeedbackPast += 1 - - - #count "problem receiving samples" msg - result = re.search(r'\[PHY\]\s+problem receiving samples', str(line)) - if result is not None: - pb_receiving_samples_cnt += 1 - #count "Removing UE" msg - result = re.search(r'\[MAC\]\s+Removing UE', str(line)) - if result is not None: - removing_ue += 1 - #count "X2AP-PDU" - result = re.search('X2AP-PDU', str(line)) - if result is not None: - x2ap_pdu += 1 - #gnb markers logging - for k in gnb_markers: - result = re.search(k, line) - if result is not None: - gnb_markers[k].append(line_cnt) - - # check whether e/gNB log finishes with "Bye." message - showedByeMsg |= re.search(r'^Bye.\n', str(line), re.MULTILINE) is not None - - enb_log_file.close() - - #stdout log file and stat log files analysis completed - logging.debug(' File analysis (stdout, stats) completed') - - #post processing depending on the node type - if not nodeB_prefix_found: - if self.air_interface == 'lte-softmodem': - nodeB_prefix = 'e' - else: - nodeB_prefix = 'g' - - if nodeB_prefix == 'g': - if ulschReceiveOK > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if gnbRxTxWakeUpFailure > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(gnbRxTxWakeUpFailure) + ' "could not wakeup gNB rxtx process" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if gnbTxWriteThreadEnabled: - statMsg = nodeB_prefix + 'NB ran with TX Write thread enabled' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if nrRrcRcfgComplete > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(nrRrcRcfgComplete) + ' "Received NR_RRCReconfigurationComplete from UE" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if harqFeedbackPast > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(harqFeedbackPast) + ' "HARQ feedback is in the past" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - #FR1 NSA test : add new markers to make sure gNB is used - if NSA_RAPROC_PUSCH_check: - statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : PASS ' - htmlMsg = statMsg+'\n' - else: - statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : FAIL or not relevant' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - #problem receiving samples log - statMsg = '[PHY] problem receiving samples msg count = '+str(pb_receiving_samples_cnt) - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - #gnb markers - statMsg = 'logfile line count = ' + str(line_cnt) - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - if len(gnb_markers['SgNBReleaseRequestAcknowledge'])!=0: - statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequestAcknowledge'][0]) - else: - statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - statMsg = 'FAILURE = ' + str(len(gnb_markers['FAILURE'])) + ' occurences' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - statMsg = 'Detected UL Failure on PUSCH = ' + str(len(gnb_markers['Detected UL Failure on PUSCH'])) + ' occurences' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - - if not showedByeMsg: - logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB did not show "Bye." message at end, it likely did not stop properly! \u001B[0m') - htmleNBFailureMsg += 'No Bye. message found, did not stop properly\n' - global_status = CONST.ENB_SHUTDOWN_NO_BYE - else: - logging.debug('"Bye." message found at end.') - - else: - #Removing UE log - statMsg = '[MAC] Removing UE msg count = '+str(removing_ue) - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - #X2AP-PDU log - statMsg = 'X2AP-PDU msg count = '+str(x2ap_pdu) - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - #nsa markers - statMsg = 'logfile line count = ' + str(line_cnt) - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - if len(gnb_markers['SgNBReleaseRequest'])!=0: - statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequest'][0]) - else: - statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - statMsg = 'scgFailureInformationNR-r15 = ' + str(len(gnb_markers['scgFailureInformationNR-r15'])) + ' occurences' - htmlMsg = statMsg+'\n' - logging.debug(statMsg) - htmleNBFailureMsg += htmlMsg - - if RealTimeProcessingIssue: - logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with real time processing issue! \u001B[0m') - htmleNBFailureMsg += 'Fail due to real time processing issue\n' - global_status = CONST.ENB_REAL_TIME_PROCESSING_ISSUE - if uciStatMsgCount > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if pdcpFailure > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if ulschFailure > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if ulschAllocateCCEerror > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(ulschAllocateCCEerror) + ' "eNB_dlsch_ulsch_scheduler(); ERROR ALLOCATING CCEs" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if uplinkSegmentsAborted > 0: - statMsg = nodeB_prefix + 'NB showed ' + str(uplinkSegmentsAborted) + ' "uplink segment error 0/2, aborted * segments" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if dropNotEnoughRBs > 0: - statMsg = 'eNB showed ' + str(dropNotEnoughRBs) + ' "dropping, not enough RBs" message(s)' - logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') - htmleNBFailureMsg += statMsg + '\n' - if rrcSetupComplete > 0: - rrcMsg = nodeB_prefix + 'NB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if rrcReleaseRequest > 0: - rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if rrcReconfigRequest > 0 or rrcReconfigComplete > 0: - rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0: - rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if self.eNBOptions[0] != '': - res1 = re.search('drx_Config_present prSetup', self.eNBOptions[0]) - if res1 is not None: - if cdrxActivationMessageCount > 0: - rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - else: - rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration' - logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m') - htmleNBFailureMsg += rrcMsg + '\n' - if rachCanceledProcedure > 0: - rachMsg = nodeB_prefix + 'NB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)' - logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m') - htmleNBFailureMsg += rachMsg + '\n' - if isRRU: - if isSlave: - if slaveReceivesFrameResyncCmd: - rruMsg = 'Slave RRU received the RRU_frame_resynch command from RAU' - logging.debug('\u001B[1;30;43m ' + rruMsg + ' \u001B[0m') - htmleNBFailureMsg += rruMsg + '\n' - else: - rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU' - logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m') - htmleNBFailureMsg += rruMsg + '\n' - global_status = CONST.ENB_PROCESS_SLAVE_RRU_NOT_SYNCED - if foundSegFault: - logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with a Segmentation Fault! \u001B[0m') - global_status = CONST.ENB_PROCESS_SEG_FAULT - if foundAssertion: - logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with an assertion! \u001B[0m') - htmleNBFailureMsg += msgAssertion - global_status = CONST.ENB_PROCESS_ASSERTION - if foundRealTimeIssue: - logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB faced real time issues! \u001B[0m') - htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues! COUNT = '+ str(foundRealTimeIssue_cnt) +' lines\n' - if rlcDiscardBuffer > 0: - rlcMsg = nodeB_prefix + 'NB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)' - logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m') - htmleNBFailureMsg += rlcMsg + '\n' - global_status = CONST.ENB_PROCESS_REALTIME_ISSUE - HTML.htmleNBFailureMsg=htmleNBFailureMsg - # Runtime statistics for console output and HTML - if runTime != '': - logging.debug(runTime) - logging.debug(userTime) - logging.debug(systemTime) - logging.debug(maxPhyMemUsage) - logging.debug(nbContextSwitches) - self.runtime_stats='
'+runTime + '\n'+ userTime + '\n' + systemTime + '\n' + maxPhyMemUsage + '\n' + nbContextSwitches+'
' - return global_status From 366df408a1073908f816fbde36555e5a1bf70d4a Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 23 Feb 2026 14:26:41 +0100 Subject: [PATCH 20/25] Default checker: check for assertions --- ci-scripts/cls_loganalysis.py | 13 +++++++++---- ci-scripts/tests/log-analysis.py | 8 ++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ci-scripts/cls_loganalysis.py b/ci-scripts/cls_loganalysis.py index 2369f1115f..f4de1258b8 100644 --- a/ci-scripts/cls_loganalysis.py +++ b/ci-scripts/cls_loganalysis.py @@ -26,10 +26,15 @@ import logging class Default: def run(file, opt=None): - # TODO add check on assertions, etc - if not os.path.exists(file): - return False, f"cannot find file {file}" - return True, "" + success = True + logs = [] + with open(file, "r") as f: + for line in f.readlines(): + result = re.search('[Aa]ssertion', line) + if result: + logs.append(line) + success = False + return success, "\n".join(logs) class ContainsString: def run(file, needle): diff --git a/ci-scripts/tests/log-analysis.py b/ci-scripts/tests/log-analysis.py index d7e754a054..6d5f6c5bd7 100644 --- a/ci-scripts/tests/log-analysis.py +++ b/ci-scripts/tests/log-analysis.py @@ -15,10 +15,10 @@ class TestLogAnalysis(unittest.TestCase): def setUp(self): self.maxDiff = None - def test_no_file(self): - f = "/total/fantasy/path/file.log" - result, _ = cls_loganalysis.Default.run(f, None) - self.assertFalse(result) + #def test_no_file(self): + # f = "/total/fantasy/path/file.log" + # result, _ = cls_loganalysis.Default.run(f, None) + # self.assertFalse(result) def test_no_error(self): f = "tests/log-analysis/empty.log" From 735058f5c4b0f8af7883de77e010c081bd57c181 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 19 Feb 2026 11:28:54 +0100 Subject: [PATCH 21/25] New log-analysis checker LastLineContains --- ci-scripts/cls_loganalysis.py | 9 +++++++++ ci-scripts/tests/log-analysis.py | 16 ++++++++++++++++ ci-scripts/tests/log-analysis/arbitrary.log | 9 +++++++++ 3 files changed, 34 insertions(+) create mode 100644 ci-scripts/tests/log-analysis/arbitrary.log diff --git a/ci-scripts/cls_loganalysis.py b/ci-scripts/cls_loganalysis.py index f4de1258b8..38c36202bd 100644 --- a/ci-scripts/cls_loganalysis.py +++ b/ci-scripts/cls_loganalysis.py @@ -23,6 +23,7 @@ import re import os import logging +from collections import deque class Default: def run(file, opt=None): @@ -44,6 +45,14 @@ class ContainsString: return True, "" return False, f"could not find '{needle}' in logs" +class LastLineContains: + def run(file, string): + with open(file, "r") as f: + last_line = deque(f, maxlen=1).pop() + success = string in last_line + log = "" if success else f"could not find '{string}' in last line" + return success, log + class RetxCheck(): def _parseList(s): if not s: diff --git a/ci-scripts/tests/log-analysis.py b/ci-scripts/tests/log-analysis.py index 6d5f6c5bd7..a4416305a8 100644 --- a/ci-scripts/tests/log-analysis.py +++ b/ci-scripts/tests/log-analysis.py @@ -94,5 +94,21 @@ UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.1145 self.assertFalse(result) self.assertEqual(log, expected) + def test_last_line_contains_false(self): + f = "tests/log-analysis/arbitrary.log" + opt = "Bye." + result, log = cls_loganalysis.LastLineContains.run(f, opt) + expected = f"could not find '{opt}' in last line" + self.assertFalse(result) + self.assertEqual(log, expected) + + def test_last_line_contains_false(self): + f = "tests/log-analysis/arbitrary.log" + opt = "got sync" + result, log = cls_loganalysis.LastLineContains.run(f, opt) + expected = "" + self.assertTrue(result) + self.assertEqual(log, expected) + if __name__ == '__main__': unittest.main() diff --git a/ci-scripts/tests/log-analysis/arbitrary.log b/ci-scripts/tests/log-analysis/arbitrary.log new file mode 100644 index 0000000000..095690aeb3 --- /dev/null +++ b/ci-scripts/tests/log-analysis/arbitrary.log @@ -0,0 +1,9 @@ +[UTIL] threadCreate() for L1_rx_thread: creating thread (no affinity, default priority) +[UTIL] threadCreate() for L1_tx_thread: creating thread (no affinity, default priority) +[UTIL] threadCreate() for L1_stats: creating thread (no affinity, default priority) +TYPE TO TERMINATE +[PHY] got sync (ru_thread) +[PHY] RU 0 rf device ready +[PHY] RU 0 RF started cpu_meas_enabled 0 +[HW] No connected device, generating void samples... +[PHY] got sync (L1_stats_thread) From 516583405176d5a73543f1bd273a94362d64fa8c Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 26 Feb 2026 13:36:17 +0100 Subject: [PATCH 22/25] Introduce EndsWithBye log analysis check It was initially foreseen to just check the program exit code, but most executables don't return with 0 (so the CI would always fail). Add a separate check that will check for the Bye., only going through the last X lines instead of the entire file. --- ci-scripts/cls_loganalysis.py | 13 +++++++++++++ ci-scripts/tests/log-analysis.py | 17 +++++++++++++++++ ci-scripts/tests/log-analysis/with-bye.log | 3 +++ 3 files changed, 33 insertions(+) create mode 100644 ci-scripts/tests/log-analysis/with-bye.log diff --git a/ci-scripts/cls_loganalysis.py b/ci-scripts/cls_loganalysis.py index 38c36202bd..b28e217707 100644 --- a/ci-scripts/cls_loganalysis.py +++ b/ci-scripts/cls_loganalysis.py @@ -53,6 +53,19 @@ class LastLineContains: log = "" if success else f"could not find '{string}' in last line" return success, log +class EndsWithBye: + def run(file, option): + # no option: we ignore any + with open(file, "r") as f: + n = 0 + for line in reversed(f.readlines()): + if re.search(r'^Bye.\n', line, re.MULTILINE) is not None: + return True, "" + n = n + 1 + if n > 50: # if not found in last 50 lines, assume it does not exist + break + return False, "No Bye. message found, did not stop properly" + class RetxCheck(): def _parseList(s): if not s: diff --git a/ci-scripts/tests/log-analysis.py b/ci-scripts/tests/log-analysis.py index a4416305a8..2df23483d3 100644 --- a/ci-scripts/tests/log-analysis.py +++ b/ci-scripts/tests/log-analysis.py @@ -110,5 +110,22 @@ UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.1145 self.assertTrue(result) self.assertEqual(log, expected) + def test_ends_no_bye(self): + f = "tests/log-analysis/arbitrary.log" + opt = None + result, log = cls_loganalysis.EndsWithBye.run(f, opt) + expected = "No Bye. message found, did not stop properly" + self.assertFalse(result) + self.assertEqual(log, expected) + + def test_ends_with_bye(self): + f = "tests/log-analysis/with-bye.log" + opt = None + result, log = cls_loganalysis.EndsWithBye.run(f, opt) + expected = "" + self.assertTrue(result) + self.assertEqual(log, expected) + + if __name__ == '__main__': unittest.main() diff --git a/ci-scripts/tests/log-analysis/with-bye.log b/ci-scripts/tests/log-analysis/with-bye.log new file mode 100644 index 0000000000..22abbfab05 --- /dev/null +++ b/ci-scripts/tests/log-analysis/with-bye.log @@ -0,0 +1,3 @@ +Some other logs +Bye. +process exited. From c6bb50edb11d6a9af1830767872d3f57fc101d20 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 24 Feb 2026 08:16:21 +0100 Subject: [PATCH 23/25] Add LogAnalysis in all XML files I try to retain/match the previous log checks, without adding new checks. --- ci-scripts/xml_files/container_4g_l2sim_tdd.xml | 3 +++ .../xml_files/container_4g_rfsim_fdd_05MHz.xml | 3 +++ .../xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml | 3 +++ .../xml_files/container_4g_rfsim_fdd_10MHz.xml | 3 +++ .../xml_files/container_4g_rfsim_fdd_20MHz.xml | 3 +++ ci-scripts/xml_files/container_4g_rfsim_fembms.xml | 3 +++ ci-scripts/xml_files/container_4g_rfsim_mbms.xml | 3 +++ .../xml_files/container_4g_rfsim_tdd_05MHz.xml | 3 +++ ci-scripts/xml_files/container_5g_e1_rfsim.xml | 5 +++-- ci-scripts/xml_files/container_5g_f1_rfsim.xml | 5 +++-- ci-scripts/xml_files/container_5g_fdd_rfsim.xml | 3 +++ ci-scripts/xml_files/container_5g_flexric_rfsim.xml | 8 ++++++++ ci-scripts/xml_files/container_5g_rfsim.xml | 5 +++-- ci-scripts/xml_files/container_5g_rfsim_24prb.xml | 3 +++ ci-scripts/xml_files/container_5g_rfsim_2x2.xml | 5 +++-- .../xml_files/container_5g_rfsim_fdd_phytest.xml | 5 +++-- .../xml_files/container_5g_rfsim_fr2_66prb.xml | 5 +++-- ci-scripts/xml_files/container_5g_rfsim_multiue.xml | 7 ++++--- ci-scripts/xml_files/container_5g_rfsim_n2_ho.xml | 5 +++-- ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml | 3 +++ ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml | 3 +++ ci-scripts/xml_files/container_5g_rfsim_prs.xml | 5 +++-- .../xml_files/container_5g_rfsim_sidelink.xml | 13 ++++--------- ci-scripts/xml_files/container_5g_rfsim_simple.xml | 5 +++-- .../xml_files/container_5g_rfsim_tdd_dora.xml | 5 +++-- .../xml_files/container_5g_rfsim_u0_25prb.xml | 5 +++-- .../xml_files/container_5g_vrtsim_chanmod.xml | 5 +++-- .../xml_files/container_5g_vrtsim_chanmod_gh.xml | 5 +++-- ci-scripts/xml_files/container_5g_vrtsim_cirdb.xml | 5 +++-- .../xml_files/container_lte_b200_fdd_05Mhz_tm1.xml | 3 +++ .../container_lte_b200_fdd_05Mhz_tm1_if4_5.xml | 3 +++ ...ainer_lte_b200_fdd_05Mhz_tm1_no_rrc_activity.xml | 3 +++ .../xml_files/container_lte_b200_fdd_10Mhz_tm1.xml | 3 +++ .../container_lte_b200_fdd_10Mhz_tm1_cdrx.xml | 3 +++ .../container_lte_b200_fdd_10Mhz_tm1_oaiue.xml | 6 ++++++ .../xml_files/container_lte_b200_fdd_20Mhz_tm1.xml | 3 +++ .../xml_files/container_lte_b200_tdd_05Mhz_tm1.xml | 3 +++ .../container_lte_b200_tdd_05Mhz_tm1_if4_5.xml | 3 +++ .../xml_files/container_lte_b200_tdd_10Mhz_tm1.xml | 3 +++ .../xml_files/container_lte_b200_tdd_20Mhz_tm1.xml | 3 +++ ...ner_lte_b200_tdd_20Mhz_tm1_default_scheduler.xml | 3 +++ .../xml_files/container_lte_n3xx_tdd_2x2_tm1.xml | 3 +++ .../xml_files/container_lte_n3xx_tdd_2x2_tm2.xml | 3 +++ ci-scripts/xml_files/container_nsa_b200_quectel.xml | 8 ++++++-- .../xml_files/container_sa_aerial_quectel.xml | 5 +++-- .../container_sa_aerial_quectel_ul_heavy.xml | 5 +++-- ci-scripts/xml_files/container_sa_aw2s_asue.xml | 3 +++ ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml | 3 +++ ci-scripts/xml_files/container_sa_b200_quectel.xml | 5 +++-- .../xml_files/container_sa_e1_b200_quectel.xml | 5 +++-- .../xml_files/container_sa_f1_b200_quectel.xml | 5 +++-- .../xml_files/container_sa_f1_ho_b210_quectel.xml | 6 ++++++ .../container_sa_fhi72_metanoia_up2_2x2.xml | 3 +++ .../xml_files/container_sa_fhi72_vvdn_up2.xml | 3 +++ .../xml_files/container_sa_n2_ho_b210_quectel.xml | 6 ++++++ .../container_sa_n310_2X2_100MHz_quectel.xml | 5 +++-- .../container_sa_n310_2X2_60MHz_quectel.xml | 5 +++-- .../container_sa_n310_4X4_60MHz_quectel.xml | 5 +++-- ci-scripts/xml_files/container_sa_n310_nrue.xml | 8 ++++++-- .../xml_files/container_sa_n310_nrue_longrun.xml | 8 ++++++-- .../xml_files/container_sa_sc_b200_quectel.xml | 5 +++-- .../gnb_phytest_rfemulator_run_100_2x2.xml | 2 +- ci-scripts/xml_files/gnb_phytest_usrp_run.xml | 3 +++ .../xml_files/gnb_phytest_usrp_run_100_2x2.xml | 3 +++ ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml | 3 +++ .../xml_files/gnb_phytest_usrp_run_60_2x2.xml | 3 +++ 66 files changed, 221 insertions(+), 65 deletions(-) diff --git a/ci-scripts/xml_files/container_4g_l2sim_tdd.xml b/ci-scripts/xml_files/container_4g_l2sim_tdd.xml index 1b3ec37b81..5483a9e4c7 100644 --- a/ci-scripts/xml_files/container_4g_l2sim_tdd.xml +++ b/ci-scripts/xml_files/container_4g_l2sim_tdd.xml @@ -129,6 +129,9 @@ localhost Undeploy all OAI 4G stack ci-scripts/yaml_files/4g_l2sim_fdd + + oai_enb=EndsWithBye oai_ue1 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml index adad131ef7..57295697e4 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml @@ -147,6 +147,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml index 0879e27170..ac057f1862 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml @@ -118,6 +118,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz_noS1 + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml index acde953fe8..aca8caf06b 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml @@ -147,6 +147,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_fdd_10MHz + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml index cf6eb1d838..619fd54bec 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml @@ -147,6 +147,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_fdd_20MHz + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fembms.xml b/ci-scripts/xml_files/container_4g_rfsim_fembms.xml index f03fb12e76..41708abab2 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fembms.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fembms.xml @@ -84,6 +84,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_fembms + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_mbms.xml b/ci-scripts/xml_files/container_4g_rfsim_mbms.xml index f41aa98a9f..457adb77b8 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_mbms.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_mbms.xml @@ -84,6 +84,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_mbms + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml index f57ff9eb14..16c61edaf0 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml @@ -146,6 +146,9 @@ Undeploy all OAI 4G stack localhost ci-scripts/yaml_files/4g_rfsimulator_tdd_05MHz + + oai_enb0=EndsWithBye oai_ue0 + diff --git a/ci-scripts/xml_files/container_5g_e1_rfsim.xml b/ci-scripts/xml_files/container_5g_e1_rfsim.xml index fc3be58e2d..6c8b3940b5 100644 --- a/ci-scripts/xml_files/container_5g_e1_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_e1_rfsim.xml @@ -86,8 +86,9 @@ localhost Undeploy all OAI 5G stack ci-scripts/yaml_files/5g_rfsimulator_e1 - 1,0,0,0 - 1,0,0,0 + + oai-cucp=EndsWithBye oai-cuup=EndsWithBye oai-cuup2=EndsWithBye oai-cuup3=EndsWithBye oai-du=RetxCheck=1,0,0,0 oai-du=EndsWithBye oai-du2=RetxCheck=1,0,0,0 oai-du2=EndsWithBye oai-du3=RetxCheck=1,0,0,0 oai-du3=EndsWithBye oai-nr-ue oai-nr-ue2 oai-nr-ue3 + diff --git a/ci-scripts/xml_files/container_5g_f1_rfsim.xml b/ci-scripts/xml_files/container_5g_f1_rfsim.xml index 7385daacfc..56df31e156 100644 --- a/ci-scripts/xml_files/container_5g_f1_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_f1_rfsim.xml @@ -311,8 +311,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_f1_rfsimulator - 30,100,100,100 - 30,100,100,100 + + oai-cu=EndsWithBye oai-du-pci0=RetxCheck=30,100,100,100 oai-du-pci0=EndsWithBye oai-du-pci1=RetxCheck=30,100,100,100 oai-du-pci1=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml index eb616d9e35..09d0e7784f 100644 --- a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml @@ -113,6 +113,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_fdd_rfsimulator + + oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_flexric_rfsim.xml b/ci-scripts/xml_files/container_5g_flexric_rfsim.xml index 78fb296697..985f2b623c 100644 --- a/ci-scripts/xml_files/container_5g_flexric_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_flexric_rfsim.xml @@ -197,6 +197,14 @@ localhost ci-scripts/yaml_files/5g_rfsimulator_flexric true + + nearRT-RIC=LastLineContains=Removing E2 Node + xapp-rc-moni=LastLineContains=Test xApp run SUCCESSFULLY + xapp-kpm-moni=LastLineContains=Test xApp run SUCCESSFULLY + xapp-kpm-rc=LastLineContains=Test xApp run SUCCESSFULLY + xapp-gtp-mac-rlc-pdcp-moni=LastLineContains=Test xApp run SUCCESSFULLY + oai-gnb=EndsWithBye oai-nr-ue oai-nr-ue2 + diff --git a/ci-scripts/xml_files/container_5g_rfsim.xml b/ci-scripts/xml_files/container_5g_rfsim.xml index 58199ab5d9..2bc96b7d8c 100644 --- a/ci-scripts/xml_files/container_5g_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_rfsim.xml @@ -243,8 +243,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator - 1,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue oai-nr-ue2 oai-nr-ue3 oai-nr-ue4 oai-nr-ue5 oai-nr-ue6 oai-nr-ue7 oai-nr-ue8 oai-nr-ue9 oai-nr-ue10 + diff --git a/ci-scripts/xml_files/container_5g_rfsim_24prb.xml b/ci-scripts/xml_files/container_5g_rfsim_24prb.xml index 1e7128a6e8..f7f677cb0d 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_24prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_24prb.xml @@ -257,6 +257,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_24prb + + oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_2x2.xml b/ci-scripts/xml_files/container_5g_rfsim_2x2.xml index 06cb9b63db..3c1c33db10 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_2x2.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_2x2.xml @@ -113,8 +113,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_2x2 - 1,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml b/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml index 4fbb0f6f7e..649663ea93 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml @@ -113,8 +113,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_fdd_phytest - 10,100,100,100 - 10,100,100,100 + + oai-gnb=RetxCheck=10,100,100,100 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_fr2_66prb.xml b/ci-scripts/xml_files/container_5g_rfsim_fr2_66prb.xml index b64837c839..cb15ffbcc0 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_fr2_66prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_fr2_66prb.xml @@ -89,8 +89,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_fr2_66prb - 10,0,0,0 - 10,0,0,0 + + oai-gnb=RetxCheck=10,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_multiue.xml b/ci-scripts/xml_files/container_5g_rfsim_multiue.xml index 83bd857c4a..4010e561bc 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_multiue.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_multiue.xml @@ -159,10 +159,11 @@ Undeploy_Object true Undeploy all OAI 5G stack - ci-scripts/yaml_files/5g_rfsimulator_multiue - 1,0,0,0 - 1,0,0,0 localhost + ci-scripts/yaml_files/5g_rfsimulator_multiue + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_n2_ho.xml b/ci-scripts/xml_files/container_5g_rfsim_n2_ho.xml index 9ef14478b1..915fa0e4e6 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_n2_ho.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_n2_ho.xml @@ -227,8 +227,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_n2_ho - 10,100,100,100 - 10,100,100,100 + + oai-cucp-0=EndsWithBye oai-cuup-0=EndsWithBye oai-du-0=RetxCheck=10,100,100,100 oai-du-0=EndsWithBye oai-nr-ue oai-cucp-1=EndsWithBye oai-cuup-1=EndsWithBye oai-du-1=RetxCheck=10,100,100,100 oai-du-1=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml b/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml index 7e291b86fb..3cb18aeb44 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml @@ -138,6 +138,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_ntn_geo + + oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml b/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml index cac6f5f4ee..97d2540ce6 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml @@ -138,6 +138,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_ntn_leo + + oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_prs.xml b/ci-scripts/xml_files/container_5g_rfsim_prs.xml index 2485e7d503..6e9500c388 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_prs.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_prs.xml @@ -74,8 +74,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_prs - 30,100,100,100 - 30,100,100,100 + + oai-gnb=RetxCheck=30,100,100,100 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_sidelink.xml b/ci-scripts/xml_files/container_5g_rfsim_sidelink.xml index f434bb600c..d4d7723915 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_sidelink.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_sidelink.xml @@ -60,21 +60,16 @@ 20 - - Custom_Command - Check that UE synched - localhost - docker logs rfsim5g-oai-nr-ue-2 | grep -m 1 "PSBCH RX:OK" - - Undeploy_Object true Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_sidelink - 1,0,0,0 - 1,0,0,0 + + oai-nr-ue-1 + oai-nr-ue-2=ContainsString=PSBCH RX:OK + diff --git a/ci-scripts/xml_files/container_5g_rfsim_simple.xml b/ci-scripts/xml_files/container_5g_rfsim_simple.xml index be4912d2b2..541a2f5a29 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_simple.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_simple.xml @@ -80,8 +80,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator - 1,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue oai-nr-ue2 oai-nr-ue3 oai-nr-ue4 oai-nr-ue5 oai-nr-ue6 oai-nr-ue7 oai-nr-ue8 oai-nr-ue9 oai-nr-ue10 + diff --git a/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml b/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml index 83209d664e..dbea7321ed 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml @@ -95,8 +95,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_tdd_dora - 0,0,0,0 - 0,0,0,0 + + oai-gnb=RetxCheck=0,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml b/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml index 060f539cde..ef84c97f7c 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml @@ -174,8 +174,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_rfsimulator_u0_25prb - 1,0,0,0 - 1,0,0,0 + + oai-vnf=RetxCheck=0,0,0,0 oai-vnf=EndsWithBye oai-pnf=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_vrtsim_chanmod.xml b/ci-scripts/xml_files/container_5g_vrtsim_chanmod.xml index c844228640..9198261754 100644 --- a/ci-scripts/xml_files/container_5g_vrtsim_chanmod.xml +++ b/ci-scripts/xml_files/container_5g_vrtsim_chanmod.xml @@ -113,8 +113,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_vrtsim_chanmod - 1,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_vrtsim_chanmod_gh.xml b/ci-scripts/xml_files/container_5g_vrtsim_chanmod_gh.xml index 3d3a9a5e41..353bd68f05 100644 --- a/ci-scripts/xml_files/container_5g_vrtsim_chanmod_gh.xml +++ b/ci-scripts/xml_files/container_5g_vrtsim_chanmod_gh.xml @@ -114,8 +114,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_vrtsim_chanmod - 1,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_5g_vrtsim_cirdb.xml b/ci-scripts/xml_files/container_5g_vrtsim_cirdb.xml index 84a6a527f7..fe08d34adf 100644 --- a/ci-scripts/xml_files/container_5g_vrtsim_cirdb.xml +++ b/ci-scripts/xml_files/container_5g_vrtsim_cirdb.xml @@ -114,8 +114,9 @@ Undeploy all OAI 5G stack localhost ci-scripts/yaml_files/5g_vrtsim_cirdb - 10,0,0,0 - 1,0,0,0 + + oai-gnb=RetxCheck=dl=10,0,0,0;ul=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1.xml index 614acee309..af703bad33 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1.xml @@ -183,6 +183,9 @@ Replaces xml_files/enb_usrp210_band7_test_05mhz_tm1.xml Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_05Mhz_tm1 + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_if4_5.xml b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_if4_5.xml index 243243658b..aa3598126c 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_if4_5.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_if4_5.xml @@ -165,6 +165,9 @@ Undeploy RCC/RRU hutch ci-scripts/yaml_files/lte_b200_fdd_05Mhz_if4.5 + + rru_fdd=EndsWithBye rcc_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_no_rrc_activity.xml b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_no_rrc_activity.xml index bd7400cb13..71fd42ccf4 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_no_rrc_activity.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_no_rrc_activity.xml @@ -156,6 +156,9 @@ Replaces xml_files/enb_usrp210_band7_test_05mhz_tm1_rrc_inactivity_no_flexran.xm Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_05Mhz_tm1_no_rrc_activity + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1.xml index 92f4376676..a0ea977254 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1.xml @@ -168,6 +168,9 @@ Replaces xml_files/enb_usrp210_band7_test_10mhz_tm1.xml Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_10Mhz_tm1 + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_cdrx.xml b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_cdrx.xml index 5e3b64e01b..868a80cec3 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_cdrx.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_cdrx.xml @@ -116,6 +116,9 @@ Replaces xml_files/enb_usrp210_band7_test_10mhz_tm1.xml Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_10Mhz_tm1_cdrx + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_oaiue.xml b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_oaiue.xml index 5133c17f14..c0b0f91027 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_oaiue.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_oaiue.xml @@ -133,6 +133,9 @@ Undeploy LTE-UE carabe ci-scripts/yaml_files/lte_b200_fdd_10Mhz_oai_ue_magma + + lte_ue0 + @@ -147,6 +150,9 @@ Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_10Mhz_tm1_magma + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_fdd_20Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_fdd_20Mhz_tm1.xml index cf092b2653..6a3bf52993 100644 --- a/ci-scripts/xml_files/container_lte_b200_fdd_20Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_fdd_20Mhz_tm1.xml @@ -140,6 +140,9 @@ Replaces xml_files/enb_usrp210_band7_test_10mhz_tm1.xml Undeploy eNB hutch ci-scripts/yaml_files/lte_b200_fdd_20Mhz_tm1 + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1.xml index 849d6e0140..b20740c0e7 100644 --- a/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1.xml @@ -194,6 +194,9 @@ Replaces xml_files/enb_usrp210_band40_test_05mhz_tm1.xml Undeploy eNB starsky ci-scripts/yaml_files/lte_b200_tdd_05Mhz_tm1 + + enb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1_if4_5.xml b/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1_if4_5.xml index 3b2e5a8022..8d75c79347 100644 --- a/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1_if4_5.xml +++ b/ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1_if4_5.xml @@ -174,6 +174,9 @@ Undeploy RRU/RCC starsky ci-scripts/yaml_files/lte_b200_tdd_05Mhz_if4.5 + + rru_tdd=EndsWithBye rcc_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_tdd_10Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_tdd_10Mhz_tm1.xml index a947c6774c..74eb50fe06 100644 --- a/ci-scripts/xml_files/container_lte_b200_tdd_10Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_tdd_10Mhz_tm1.xml @@ -187,6 +187,9 @@ Replaces xml_files/enb_usrp210_band40_test_10mhz_tm1.xml Undeploy eNB starsky ci-scripts/yaml_files/lte_b200_tdd_10Mhz_tm1 + + enb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1.xml b/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1.xml index 0888891cb3..1b1f007e0e 100644 --- a/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1.xml +++ b/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1.xml @@ -133,6 +133,9 @@ Replaces xml_files/enb_usrp210_band40_test_20mhz_tm1.xml Undeploy eNB starsky ci-scripts/yaml_files/lte_b200_tdd_20Mhz_tm1 + + enb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1_default_scheduler.xml b/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1_default_scheduler.xml index 636abb1a66..cbd196b353 100644 --- a/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1_default_scheduler.xml +++ b/ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1_default_scheduler.xml @@ -189,6 +189,9 @@ Replaces xml_files/enb_usrp210_band40_test_20mhz_tm1_default_scheduler.xml Undeploy eNB starsky ci-scripts/yaml_files/lte_b200_tdd_20Mhz_tm1_default_scheduler + + enb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm1.xml b/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm1.xml index 39ac1a7df4..0eab1b9fc1 100644 --- a/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm1.xml +++ b/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm1.xml @@ -125,6 +125,9 @@ Undeploy gNB obelix ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1 + + oai-enb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm2.xml b/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm2.xml index e2b9f7ae13..41c3c4f4a7 100644 --- a/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm2.xml +++ b/ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm2.xml @@ -125,6 +125,9 @@ Undeploy gNB obelix ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2 + + oai-enb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_nsa_b200_quectel.xml b/ci-scripts/xml_files/container_nsa_b200_quectel.xml index a9e859886c..c9b941949c 100644 --- a/ci-scripts/xml_files/container_nsa_b200_quectel.xml +++ b/ci-scripts/xml_files/container_nsa_b200_quectel.xml @@ -192,8 +192,9 @@ Undeploy gNB ofqot ci-scripts/yaml_files/nsa_b200_gnb - 10,100,100,100 - 10,100,100,100 + + gnb_mono_tdd=RetxCheck=10,100,100,100 gnb_mono_tdd=EndsWithBye + @@ -202,6 +203,9 @@ Undeploy eNB nepes ci-scripts/yaml_files/nsa_b200_enb + + enb_mono_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_aerial_quectel.xml b/ci-scripts/xml_files/container_sa_aerial_quectel.xml index 7282513bb8..b5de982e5a 100644 --- a/ci-scripts/xml_files/container_sa_aerial_quectel.xml +++ b/ci-scripts/xml_files/container_sa_aerial_quectel.xml @@ -132,8 +132,9 @@ Undeploy gNB gracehopper1-oai ci-scripts/yaml_files/sa_gnb_aerial - 20,100,100,100 - 20,100,100,100 + + oai-gnb-aerial=RetxCheck=20,100,100,100 oai-gnb-aerial=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_aerial_quectel_ul_heavy.xml b/ci-scripts/xml_files/container_sa_aerial_quectel_ul_heavy.xml index 8367316e3a..9d826d2261 100644 --- a/ci-scripts/xml_files/container_sa_aerial_quectel_ul_heavy.xml +++ b/ci-scripts/xml_files/container_sa_aerial_quectel_ul_heavy.xml @@ -121,8 +121,9 @@ Undeploy gNB gracehopper1-oai ci-scripts/yaml_files/sa_gnb_aerial_ul - 20,100,100,100 - 20,100,100,100 + + oai-gnb-aerial=RetxCheck=20,100,100,100 oai-gnb-aerial=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_aw2s_asue.xml b/ci-scripts/xml_files/container_sa_aw2s_asue.xml index 6dfbc83187..dadfad3bf5 100644 --- a/ci-scripts/xml_files/container_sa_aw2s_asue.xml +++ b/ci-scripts/xml_files/container_sa_aw2s_asue.xml @@ -177,6 +177,9 @@ Undeploy gNB avra ci-scripts/yaml_files/sa_aw2s_gnb + + oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml b/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml index c7e5ab5721..19a668ff27 100644 --- a/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml +++ b/ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml @@ -235,6 +235,9 @@ Undeploy gNB avra ci-scripts/yaml_files/sa_aw2s_2x2_gnb + + oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_b200_quectel.xml b/ci-scripts/xml_files/container_sa_b200_quectel.xml index cb64146091..2891a7b117 100644 --- a/ci-scripts/xml_files/container_sa_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_b200_quectel.xml @@ -229,8 +229,9 @@ Undeploy gNB ofqot ci-scripts/yaml_files/sa_b200_gnb - 10,100,100,100 - 10,100,100,100 + + gnb_mono_tdd=RetxCheck=10,100,100,100 gnb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml b/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml index ce9e1121c3..797c6fd853 100644 --- a/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_e1_b200_quectel.xml @@ -255,8 +255,9 @@ Undeploy CUCP/CU-UP/DU ofqot ci-scripts/yaml_files/sa_e1_b200_gnb - 10,100,100,100 - 10,100,100,100 + + gnb_cucp=EndsWithBye gnb_cuup=EndsWithBye gnb_du_tdd=RetxCheck=10,100,100,100 gnb_du_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml b/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml index 35103d01e7..875e7856c7 100644 --- a/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_f1_b200_quectel.xml @@ -263,8 +263,9 @@ Undeploy CU-DU ofqot ci-scripts/yaml_files/sa_f1_b200_gnb - 10,100,100,100 - 10,100,100,100 + + gnb_cu=EndsWithBye gnb_du_fdd=RetxCheck=10,100,100,100 gnb_du_fdd=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_f1_ho_b210_quectel.xml b/ci-scripts/xml_files/container_sa_f1_ho_b210_quectel.xml index 2804e3c14e..73e1375046 100644 --- a/ci-scripts/xml_files/container_sa_f1_ho_b210_quectel.xml +++ b/ci-scripts/xml_files/container_sa_f1_ho_b210_quectel.xml @@ -191,6 +191,9 @@ Undeploy CU-DU0 groot ci-scripts/yaml_files/5g_sa_f1_b210_ho + + oai-cu=EndsWithBye oai-du0=EndsWithBye + @@ -199,6 +202,9 @@ Undeploy DU1 rocket ci-scripts/yaml_files/5g_sa_f1_b210_ho + + oai-du1=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml b/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml index 3129883744..97037157b4 100644 --- a/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml +++ b/ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml @@ -127,6 +127,9 @@ Undeploy gNB cacofonix ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb + + oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml b/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml index c02a12deac..d8548077fd 100644 --- a/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml +++ b/ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml @@ -138,6 +138,9 @@ Undeploy gNB cacofonix ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb + + oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n2_ho_b210_quectel.xml b/ci-scripts/xml_files/container_sa_n2_ho_b210_quectel.xml index 1f4c7c1981..42390d0f2f 100644 --- a/ci-scripts/xml_files/container_sa_n2_ho_b210_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n2_ho_b210_quectel.xml @@ -160,6 +160,9 @@ Undeploy Monolithic gNB-PCI0 groot ci-scripts/yaml_files/5g_sa_n2_ho_b210 + + oai-gnb-pci0=EndsWithBye + @@ -168,6 +171,9 @@ Undeploy Monolithic gNB-PCI1 rocket ci-scripts/yaml_files/5g_sa_n2_ho_b210 + + oai-gnb-pci1=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml index 0c3cd35b2f..6823b553c4 100644 --- a/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml @@ -123,8 +123,9 @@ Undeploy gNB matix ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz - 10,100,100,100 - 10,100,100,100 + + oai-gnb=RetxCheck=10,100,100,100 oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml index 5d1b3dcd95..dd4d6d9e96 100644 --- a/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml @@ -123,8 +123,9 @@ Undeploy gNB matix ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz - 15,100,100,100 - 20,100,100,100 + + oai-gnb=RetxCheck=dl=15,100,100,100;ul=20,100,100,100 oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml index 45d105de9e..694a6aa288 100644 --- a/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml @@ -123,8 +123,9 @@ Undeploy VNF-PNF matix ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz - 10,100,100,100 - 10,100,100,100 + + oai-vnf=RetxCheck=10,100,100,100 oai-vnf=EndsWithBye oai-pnf=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n310_nrue.xml b/ci-scripts/xml_files/container_sa_n310_nrue.xml index 64311b06cd..1db35faa6b 100644 --- a/ci-scripts/xml_files/container_sa_n310_nrue.xml +++ b/ci-scripts/xml_files/container_sa_n310_nrue.xml @@ -125,6 +125,9 @@ Undeploy nr UE caracal ci-scripts/yaml_files/5g_sa_n310_nrue + + oai-nr-ue + @@ -133,8 +136,9 @@ Undeploy gNB avra ci-scripts/yaml_files/5g_sa_n310_gnb - 20,100,100,100 - 20,100,100,100 + + oai-gnb=RetxCheck=20,100,100,100 oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_n310_nrue_longrun.xml b/ci-scripts/xml_files/container_sa_n310_nrue_longrun.xml index 6241a9d78d..987f8355a5 100644 --- a/ci-scripts/xml_files/container_sa_n310_nrue_longrun.xml +++ b/ci-scripts/xml_files/container_sa_n310_nrue_longrun.xml @@ -205,6 +205,9 @@ Undeploy nr UE caracal ci-scripts/yaml_files/5g_sa_n310_nrue + + oai-nr-ue + @@ -213,8 +216,9 @@ Undeploy gNB avra ci-scripts/yaml_files/5g_sa_n310_gnb - 20,100,100,100 - 20,100,100,100 + + oai-gnb=RetxCheck=20,100,100,100 oai-gnb=EndsWithBye + diff --git a/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml b/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml index 82cb032372..d81c855f39 100644 --- a/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_sc_b200_quectel.xml @@ -183,8 +183,9 @@ Undeploy gNB ofqot ci-scripts/yaml_files/sa_sc_b200_gnb - 10,100,100,100 - 10,100,100,100 + + gnb_mono_tdd=RetxCheck=10,100,100,100 gnb_mono_tdd=EndsWithBye + diff --git a/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml b/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml index 8429baa1ba..6af1753555 100644 --- a/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml @@ -60,7 +60,7 @@ true Terminate gNB - nr-softmodem=ContainsString=Bye. + nr-softmodem=EndsWithBye diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml index f2ad827d29..fcc88f9c04 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml @@ -58,6 +58,9 @@ true Terminate gNB caracal + + nr-softmodem=EndsWithBye + diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml index d298873582..a4b57f0942 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml @@ -58,6 +58,9 @@ true Terminate gNB caracal + + nr-softmodem=EndsWithBye + diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml index 7354fa3596..098e881248 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml @@ -58,6 +58,9 @@ true Terminate gNB caracal + + nr-softmodem=EndsWithBye + diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml index 7dd50ca83f..0075201dcf 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml @@ -58,6 +58,9 @@ true Terminate gNB caracal + + nr-softmodem=EndsWithBye + From 378d3a7a6b79163efc1404e88ed393ccc8b64cce Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 22 Jan 2026 07:54:42 +0100 Subject: [PATCH 24/25] Add AnalyzeRTStatsObject() XML step --- ci-scripts/cls_containerize.py | 20 ++++++++++++++++++++ ci-scripts/main.py | 4 ++++ ci-scripts/xml_class_list.yml | 1 + 3 files changed, 25 insertions(+) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index e21cb3771b..86ee1c2dff 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -782,3 +782,23 @@ class Containerize(): else: logging.error('\u001B[1m Undeploying objects Failed\u001B[0m') return success + + def AnalyzeRTStatsObject(self, HTML, node, ctx, thresholds): + logging.info(f'Analyzing realtime stats from server: {node}') + yaml = self.yamlPath.strip('/') + wd = f'{self.eNBSourceCodePath}/{yaml}' + wd_yaml = f'{wd}/docker-compose.y*ml' + + with cls_cmd.getConnection(node) as cmd: + services = GetDeployedServices(cmd, wd_yaml) + s, _ = services[0] # we simply assume something is deployed, if not Exception will report + # similar to BuildRunTests(), use docker cp to avoid problems with permissions + cmd.run(f'docker compose -f {wd_yaml} cp {s}:/opt/oai-gnb/nrL1_stats.log {wd}/') + l1_file = archiveArtifact(cmd, ctx, f"{wd}/nrL1_stats.log") + cmd.run(f'docker compose -f {wd_yaml} cp {s}:/opt/oai-gnb/nrMAC_stats.log {wd}/') + mac_file = archiveArtifact(cmd, ctx, f"{wd}/nrMAC_stats.log") + + logging.info(f"check against thresholds from {thresholds}") + success, datalog_rt_stats = cls_analysis.Analysis.analyze_rt_stats(thresholds, l1_file, mac_file) + HTML.CreateHtmlDataLogTable(datalog_rt_stats) + return success diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 3b40941b91..9e1401fcf0 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -257,6 +257,10 @@ def ExecuteActionWithParam(action, ctx, node): yaml = test.findtext('stats_cfg') success = RAN.AnalyzeRTStats(HTML, node, ctx, yaml) + elif action == 'AnalyzeRTStats_Object': + yaml = test.findtext('stats_cfg') + success = CONTAINERS.AnalyzeRTStatsObject(HTML, node, ctx, yaml) + else: logging.warning(f"unknown action {action}, skip step") success = True # by default, we skip the step and print a warning diff --git a/ci-scripts/xml_class_list.yml b/ci-scripts/xml_class_list.yml index 7dba0d247a..2fd2787cfd 100755 --- a/ci-scripts/xml_class_list.yml +++ b/ci-scripts/xml_class_list.yml @@ -33,3 +33,4 @@ - Custom_Script - Create_Workspace - AnalyzeRTStats + - AnalyzeRTStats_Object From da287084595e4e8be4d91819dd1628c2d3f539c9 Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Thu, 22 Jan 2026 07:55:22 +0100 Subject: [PATCH 25/25] CI: Add config files for phytest with 7.2 --- ...nb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf | 266 ++++++++++++++++++ .../datalog_rt_stats.100.4x4.fhi72.yaml | 28 ++ .../xml_files/gnb_phytest_fhi7.2_docker.xml | 88 ++++++ .../phytest_fhi72/docker-compose.yaml | 22 ++ .../phytest_fhi72/setup_sriov_dummy.sh | 38 +++ 5 files changed, 442 insertions(+) create mode 100644 ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf create mode 100644 ci-scripts/datalog_rt_stats.100.4x4.fhi72.yaml create mode 100644 ci-scripts/xml_files/gnb_phytest_fhi7.2_docker.xml create mode 100644 ci-scripts/yaml_files/phytest_fhi72/docker-compose.yaml create mode 100755 ci-scripts/yaml_files/phytest_fhi72/setup_sriov_dummy.sh diff --git a/ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf b/ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf new file mode 100644 index 0000000000..0b087e7406 --- /dev/null +++ b/ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf @@ -0,0 +1,266 @@ +Active_gNBs = ( "gNB-OAI"); +# Asn1_verbosity, choice in: none, info, annoying +Asn1_verbosity = "none"; + +gNBs = +( + { + ////////// Identification parameters: + gNB_ID = 0xe00; + gNB_name = "gNB-OAI"; + + // Tracking area code, 0x0000 and 0xfffe are reserved values + tracking_area_code = 1; + plmn_list = ({ mcc = 208; mnc = 99; mnc_length = 2; snssaiList = ( { sst = 1; }); }); + + nr_cellid = 1; + + ////////// Physical parameters: + + pdsch_AntennaPorts_XP = 2; + pdsch_AntennaPorts_N1 = 2; + maxMIMO_layers = 4; + pusch_AntennaPorts = 4; + do_CSIRS = 0; + do_SRS = 0; + + servingCellConfigCommon = ( + { + #spCellConfigCommon + physCellId = 0; +# downlinkConfigCommon + #frequencyInfoDL + # center frequency = 4049.76 MHz + # selected SSB frequency = 4049.76 MHz + absoluteFrequencySSB = 668640; #669984; + dl_frequencyBand = 77; + # frequency point A = 4000.62 MHz + dl_absoluteFrequencyPointA = 666708; + #scs-SpecificCarrierList + dl_offstToCarrier = 0; +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + dl_subcarrierSpacing = 1; + dl_carrierBandwidth = 273; + #initialDownlinkBWP + #genericParameters + initialDLBWPlocationAndBandwidth = 1099; #38.101-1 Table 5.3.2-1 + # +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + initialDLBWPsubcarrierSpacing = 1; + #pdcch-ConfigCommon + initialDLBWPcontrolResourceSetZero = 11; + initialDLBWPsearchSpaceZero = 0; + + #uplinkConfigCommon + #frequencyInfoUL + ul_frequencyBand = 77; + #scs-SpecificCarrierList + ul_offstToCarrier = 0; +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + ul_subcarrierSpacing = 1; + ul_carrierBandwidth = 273; + pMax = 23; + #initialUplinkBWP + #genericParameters + initialULBWPlocationAndBandwidth = 1099; +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + initialULBWPsubcarrierSpacing = 1; + #rach-ConfigCommon + #rach-ConfigGeneric + prach_ConfigurationIndex = 159; +#prach_msg1_FDM +#0 = one, 1=two, 2=four, 3=eight + prach_msg1_FDM = 0; + prach_msg1_FrequencyStart = 0; + zeroCorrelationZoneConfig = 0; + preambleReceivedTargetPower = -100; +#preamblTransMax (0...10) = (3,4,5,6,7,8,10,20,50,100,200) + preambleTransMax = 7; +#powerRampingStep +# 0=dB0,1=dB2,2=dB4,3=dB6 + powerRampingStep = 2; +#ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR +#1=oneeighth,2=onefourth,3=half,4=one,5=two,6=four,7=eight,8=sixteen + ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR = 4; +#one (0..15) 4,8,12,16,...60,64 + ssb_perRACH_OccasionAndCB_PreamblesPerSSB = 15; +#ra_ContentionResolutionTimer +#(0..7) 8,16,24,32,40,48,56,64 + ra_ContentionResolutionTimer = 7; + rsrp_ThresholdSSB = 19; +#prach-RootSequenceIndex_PR +#1 = 839, 2 = 139 + prach_RootSequenceIndex_PR = 2; + prach_RootSequenceIndex = 1; + # SCS for msg1, can only be 15 for 30 kHz < 6 GHz, takes precendence over the one derived from prach-ConfigIndex + # + msg1_SubcarrierSpacing = 1, +# restrictedSetConfig +# 0=unrestricted, 1=restricted type A, 2=restricted type B + restrictedSetConfig = 0, + +# this is the offset between the last PRACH preamble power and the Msg3 PUSCH, 2 times the field value in dB + msg3_DeltaPreamble = 2; + p0_NominalWithGrant = -100; + +# pucch-ConfigCommon setup : +# pucchGroupHopping +# 0 = neither, 1= group hopping, 2=sequence hopping + pucchGroupHopping = 0; + hoppingId = 0; + p0_nominal = -96; + + ssb_PositionsInBurst_Bitmap = 0x1; + +# ssb_periodicityServingCell +# 0 = ms5, 1=ms10, 2=ms20, 3=ms40, 4=ms80, 5=ms160, 6=spare2, 7=spare1 + ssb_periodicityServingCell = 2; + +# dmrs_TypeA_position +# 0 = pos2, 1 = pos3 + dmrs_TypeA_Position = 0; + +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + subcarrierSpacing = 1; + + + #tdd-UL-DL-ConfigurationCommon +# subcarrierSpacing +# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 + referenceSubcarrierSpacing = 1; + # pattern1 + # dl_UL_TransmissionPeriodicity + # 0=ms0p5, 1=ms0p625, 2=ms1, 3=ms1p25, 4=ms2, 5=ms2p5, 6=ms5, 7=ms10 + dl_UL_TransmissionPeriodicity = 5; + nrofDownlinkSlots = 3; + nrofDownlinkSymbols = 6; + nrofUplinkSlots = 1; + nrofUplinkSymbols = 4; + + ssPBCH_BlockPower = -11; + } + + ); + + + # ------- SCTP definitions + SCTP : + { + # Number of streams to use in input/output + SCTP_INSTREAMS = 2; + SCTP_OUTSTREAMS = 2; + }; + + + ////////// AMF parameters: + amf_ip_address = ({ ipv4 = "172.21.6.5"; }); + + NETWORK_INTERFACES : + { + GNB_IPV4_ADDRESS_FOR_NG_AMF = "172.21.16.209"; + GNB_IPV4_ADDRESS_FOR_NGU = "172.21.16.209"; + GNB_PORT_FOR_S1U = 2152; # Spec 2152 + }; + } +); + +MACRLCs = ( +{ + num_cc = 1; + tr_s_preference = "local_L1"; + tr_n_preference = "local_RRC"; + pusch_TargetSNRx10 = 220; + pucch_TargetSNRx10 = 220; + pusch_FailureThres = 100; +} +); + +L1s = ( +{ + num_cc = 1; + tr_n_preference = "local_mac"; + prach_dtx_threshold = 120 + pucch0_dtx_threshold = 80; + pusch_dtx_threshold = -100; + tx_amp_backoff_dB = 3; + L1_rx_thread_core = 8; + L1_tx_thread_core = 9; + max_ldpc_iterations = 20; + phase_compensation = 0; # needs to match O-RU configuration +} +); + +RUs = ( +{ + local_rf = "no"; + nb_tx = 4; + nb_rx = 4; + att_tx = 0; + att_rx = 0; + bands = [77]; + max_pdschReferenceSignalPower = -27; + max_rxgain = 75; + sf_extension = 0; + eNB_instances = [0]; + ru_thread_core = 10; + sl_ahead = 5; + tr_preference = "raw_if4p5"; # important: activate FHI7.2 + do_precoding = 0; # needs to match O-RU configuration +} +); + +security = { + # preferred ciphering algorithms + # the first one of the list that an UE supports in chosen + # valid values: nea0, nea1, nea2, nea3 + ciphering_algorithms = ( "nea0" ); + + # preferred integrity algorithms + # the first one of the list that an UE supports in chosen + # valid values: nia0, nia1, nia2, nia3 + integrity_algorithms = ( "nia2", "nia0" ); + + # setting 'drb_ciphering' to "no" disables ciphering for DRBs, no matter + # what 'ciphering_algorithms' configures; same thing for 'drb_integrity' + drb_ciphering = "yes"; + drb_integrity = "no"; +}; + +log_config : +{ + global_log_level = "info"; + hw_log_level = "info"; + phy_log_level = "info"; + mac_log_level = "info"; + rlc_log_level = "info"; + pdcp_log_level = "info"; + rrc_log_level = "info"; + ngap_log_level = "info"; + f1ap_log_level = "info"; +}; + +fhi_72 = { + dpdk_devices = ("0000:01:01.0"); + dpdk_iova_mode = "VA"; + system_core = 11; + io_core = 12; + worker_cores = (13); + ru_addr = ("98:ae:71:04:83:e3"); + mtu = 9600; + fh_config = ({ + T1a_cp_dl = (285, 470); + T1a_cp_ul = (285, 429); + T1a_up = (125, 350); + Ta4 = (110, 180); + ru_config = { + iq_width = 9; + iq_width_prach = 9; + }; + }); +}; + diff --git a/ci-scripts/datalog_rt_stats.100.4x4.fhi72.yaml b/ci-scripts/datalog_rt_stats.100.4x4.fhi72.yaml new file mode 100644 index 0000000000..b907b081c1 --- /dev/null +++ b/ci-scripts/datalog_rt_stats.100.4x4.fhi72.yaml @@ -0,0 +1,28 @@ +#this is a configuration file +#used to build real time processing statistics +#for 5G NR phy test (gNB terminate) +Title : Processing Time (us) from datalog_rt_stats.4x4.fhi72.yaml +ColNames : + - Metric + - Average; Max; Count + - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold) +Ref : + tx_fhaul : 40.2 + feptx_prec (per port, half_slot) : 12.3 + L1 Tx processing : 221.0 + DLSCH encoding : 141.0 + L1 Rx processing : 332.5 + UL segments decoding : 55.2 + UL Indication : 0.4 + Slot Indication : 14.3 + PUSCH inner-receiver : 421.4 +DeviationThreshold : + tx_fhaul : 0.25 + feptx_prec (per port, half_slot) : 0.25 + L1 Tx processing : 0.25 + DLSCH encoding : 0.25 + L1 Rx processing : 0.25 + UL segments decoding : 0.25 + UL Indication : 1.00 + Slot Indication : 0.50 + PUSCH inner-receiver : 0.25 diff --git a/ci-scripts/xml_files/gnb_phytest_fhi7.2_docker.xml b/ci-scripts/xml_files/gnb_phytest_fhi7.2_docker.xml new file mode 100644 index 0000000000..5386ba4d83 --- /dev/null +++ b/ci-scripts/xml_files/gnb_phytest_fhi7.2_docker.xml @@ -0,0 +1,88 @@ + + + gNB-PHY-Test-100-4x4-FHI72 + Timing phytest 100 MHz 4x4 FHI7.2 + tasks + + + Custom_Script + Setup sriov and network interfaces for dummy (no RU) + bulbul + + + + + Pull_Local_Registry + Pull Images from Local Registry + bulbul + oai-gnb-fhi72 + + + + Create_Workspace + Create new Workspace + bulbul + + + + Deploy_Object + Initialize gNB FHI7.2 + bulbul + ci-scripts/yaml_files/phytest_fhi72 + oai-gnb + + + + IdleSleep + Sleep + 60 + + + + AnalyzeRTStats_Object + Analyze Real-Time Stats + datalog_rt_stats.100.4x4.fhi72.yaml + bulbul + + + + Undeploy_Object + bulbul + true + Terminate gNB + ci-scripts/yaml_files/phytest_fhi72 + + oai-gnb=EndsWithBye + + + + + Clean_Test_Server_Images + true + Clean Test Images on Test Server + bulbul + oai-gnb-fhi72 + + + diff --git a/ci-scripts/yaml_files/phytest_fhi72/docker-compose.yaml b/ci-scripts/yaml_files/phytest_fhi72/docker-compose.yaml new file mode 100644 index 0000000000..46fbc24302 --- /dev/null +++ b/ci-scripts/yaml_files/phytest_fhi72/docker-compose.yaml @@ -0,0 +1,22 @@ +services: + oai-gnb: + image: ${REGISTRY-oaisoftwarealliance/}${GNB_IMG:-oai-gnb-fhi72}:${TAG:-develop} + container_name: oai-gnb-phytest + cap_add: + - IPC_LOCK + - SYS_NICE + cap_drop: + - ALL + environment: + USE_ADDITIONAL_OPTIONS: --phy-test -q -M 273 -m 27 -l 4 -D 231 -T 273 -t 16 -L 2 -U 528 --log_config.global_log_options level,nocolor,time --thread-pool 16,17,18,19,20,21,22,23 + devices: + - /dev/vfio:/dev/vfio/ + volumes: + - ../../conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf:/opt/oai-gnb/etc/gnb.conf + - /dev/hugepages:/dev/hugepages + cpuset: "8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23" + healthcheck: + test: /bin/bash -c "pgrep nr-softmodem" + interval: 10s + timeout: 5s + retries: 5 diff --git a/ci-scripts/yaml_files/phytest_fhi72/setup_sriov_dummy.sh b/ci-scripts/yaml_files/phytest_fhi72/setup_sriov_dummy.sh new file mode 100755 index 0000000000..c18affb0a9 --- /dev/null +++ b/ci-scripts/yaml_files/phytest_fhi72/setup_sriov_dummy.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +pci_addr() +{ + PF_IF=$1 + VF_INDEX=$2 + SYSFS_PATH="/sys/class/net/${PF_IF}/device/virtfn${VF_INDEX}" + + if [ ! -e "$SYSFS_PATH" ]; then + echo "VF $VF_INDEX not found for interface $PF_IF" + exit 1 + fi + PCI_ADDR=$(basename "$(readlink "$SYSFS_PATH")") + echo "$PCI_ADDR" +} +IF_NAME=ens1f0np0 +## O-DU C Plane MAC ADDR and VLAN +C_U_PLANE_MAC_ADD=22:11:55:33:99:55 +C_U_PLANE_VLAN=123 +MTU=9000 +DPDK_DEVBIND_PREFIX=/home/oaicicd/test/dpdk-stable-22.11.7/bin +NUM_VFs=1 +sudo ethtool -G $IF_NAME rx 8160 tx 8160 +sudo sh -c "echo 0 > /sys/class/net/$IF_NAME/device/sriov_numvfs" +sudo sh -c "echo $NUM_VFs > /sys/class/net/$IF_NAME/device/sriov_numvfs" +C_U_PLANE_PCI=$(pci_addr $IF_NAME 0) +# this next 2 lines is for C/U planes +sudo ip link set $IF_NAME vf 0 mac $C_U_PLANE_MAC_ADD vlan $C_U_PLANE_VLAN spoofchk off mtu $MTU +sleep 1 +sudo modprobe iavf +sudo ${DPDK_DEVBIND_PREFIX}/dpdk-devbind.py --unbind $C_U_PLANE_PCI +sudo modprobe vfio-pci +sudo ${DPDK_DEVBIND_PREFIX}/dpdk-devbind.py --bind vfio-pci $C_U_PLANE_PCI +echo "Successfully configured C-PLANE and U-PLANE: + - C-PLANE MAC: $C_U_PLANE_MAC_ADD, VLAN: $C_U_PLANE_VLAN, PCI: $C_U_PLANE_PCI" +exit 0 +