From 55cebe6edf6d566cf335982730348400df6dbaa3 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Sat, 6 Sep 2025 15:07:23 +0200 Subject: [PATCH] Provide add_timed_physim_test() in cmake Add two new macros that allow to register physim tests for ctest, including timing thresholds that should be fulfilled. First, add_timed_physim_test() registers a new test using a helper script RunTimedTest.cmake. This is because after test execution, we need to analyze logs, and the recommended way to do multiple steps in one test is via a helper cmake script: https://cmake.org/pipermail/cmake-developers/2016-February/027816.html Second, check_physim_threshold() adds new thresholds. It takes a text to be parsed, and a condition (e.g., "< 20") to check for the number following the threshold (which is assumed to be present right after the threshold). It uses a test property to count the total number of checks (limiting them to 10), and sets environment variables for the script. I initially planned to use a test property for checks, but those are only valid in the same directory, and RunTimedTest.cmake seems to be assumed by cmake to be "elsewhere", hence I needed to resort to environment variables. RunTimedTest.cmake is called through cmake with the test parameters and checks. It re-constructs a list of checks [1], runs the test, and pipes the log into a separate script that is passed all checks (see below). Afterwards, it verifies that both the test and script passed. A script analyze-timing.sh builds an awk script from the checks passed. The script analyzes each line of the test output for the threshold, and compares against the threshold. analyze-timing.sh returns success if all checks passed. [1] I did not manage to pass a "list" of checks in a single environment variable through the check_physim_threshold(), which would be simpler. --- doc/physical-simulators.md | 38 ++++++++++ openair1/SIMULATION/tests/CMakeLists.txt | 72 ++++++++++++++++++ openair1/SIMULATION/tests/RunTimedTest.cmake | 41 ++++++++++ openair1/SIMULATION/tests/analyze-timing.sh | 78 ++++++++++++++++++++ 4 files changed, 229 insertions(+) create mode 100644 openair1/SIMULATION/tests/RunTimedTest.cmake create mode 100755 openair1/SIMULATION/tests/analyze-timing.sh diff --git a/doc/physical-simulators.md b/doc/physical-simulators.md index dab457d38f..cd573a8686 100644 --- a/doc/physical-simulators.md +++ b/doc/physical-simulators.md @@ -103,6 +103,7 @@ To define a new test or modify existing ones, update the following file: openair1/SIMULATION/tests/CMakeLists.txt ``` +## `add_physim_test()` Use the `add_physim_test()` macro with the following arguments: @@ -123,6 +124,43 @@ For instance, a PRACHsim looks like this: These tests are run automatically as part of the following pipelines: [RAN-PhySim-Cluster-4G](https://jenkins-oai.eurecom.fr/job/RAN-PhySim-Cluster-4G/) and [RAN-PhySim-Cluster-5G](https://jenkins-oai.eurecom.fr/job/RAN-PhySim-Cluster-5G/) +## `add_timed_physim_test()` + +Use the `add_timed_physim_test()` macro to add a test the same way as with +`add_physim_test()` above. Additionally, it allows to check for thresholds with +`check_threshold()`: + + check_threshold( ) + +where: +- `` is any test that must have been added with + `add_timed_physim_test()` +- `` is a threshold to check for, e.g., `PHY tx proc`, and +- `` a condition to check, e.g. `< 200` + +There are two convenience functions to simplify the use of `check_threshold()`: + + check_threshold_range( LOWER UPPER ) + check_threshold_variance( AVG ABS_VAR ) + +where +- `` and `` are a lower and upper threshold, respectively, where + either one or both variables can be provided, and +- `` and `` are average and the variation in absolute numbers + (not a percentage!) can be provided. + +Both functions internally use `check_threshold()`. + +Thus upon execution of the test, the test will be run, but additionally ctest +will check for a match of `PHY tx proc ` (where ` is of format +`[0-9]+(\.[0-9]+)?`), and a matching number will be checked against condition +`< 200`. + +For instance, this could look like this: + + add_timed_physim_test(physim.5g.nr_dlsim.test3 "Some description" nr_dlsim -P) + check_physim_threshold(physim.5g.nr_dlsim.test3 "DLSCH encoding time" "< 50") + ### How to rerun failed CI tests using `ctest` Ctest automatically logs the failed tests in LastTestsFailed.log. This log is archived in diff --git a/openair1/SIMULATION/tests/CMakeLists.txt b/openair1/SIMULATION/tests/CMakeLists.txt index 4d57ada955..a7532e142d 100644 --- a/openair1/SIMULATION/tests/CMakeLists.txt +++ b/openair1/SIMULATION/tests/CMakeLists.txt @@ -17,6 +17,78 @@ macro(add_physim_test test_name test_description test_exec) add_dependencies(tests ${test_exec}) endmacro() +define_property(TEST PROPERTY CHECK_COUNT BRIEF_DOCS "helper property to enumerate checks in environment") +function(add_timed_physim_test test_name test_description test_exec) + # catch all the arguments past the last expected arqument and store them in the options_list + if (NOT TARGET ${test_exec}) + message(FATAL_ERROR "test executable ${test_exec} is not an executable") + endif() + set(test_invocation $ ${ARGN}) + add_test( + NAME ${test_name} + COMMAND ${CMAKE_COMMAND} "-DTEST_CMD=${test_invocation}" "-DCHECK_SCRIPT=${CMAKE_CURRENT_SOURCE_DIR}/analyze-timing.sh" -P ${CMAKE_CURRENT_SOURCE_DIR}/RunTimedTest.cmake + ) + set_tests_properties(${test_name} PROPERTIES + LABELS "${test_exec}" + TEST_DESCRIPTION "${test_description}" + # pass test description also through environment variable: for cmake < 3.30, + # in JSON export, we cannot recover the description otherwise + # see also https://gitlab.kitware.com/cmake/cmake/-/issues/21490 + ENVIRONMENT "LD_LIBRARY_PATH=.;TEST_DESCRIPTION=${test_description}" + ) + set_tests_properties(${test_name} PROPERTIES CHECK_COUNT 0) +endfunction() + +function(check_threshold testname threshold condition) + # check that threshold and condition don't have a colon (;), because that + # would interfere with cmake's list management + string(FIND "${threshold}" ";" pos) + if (pos GREATER -1) + message(FATAL_ERROR "colon not allowed in threshold, but have \"${threshold}\"") + endif() + string(FIND "${condition}" ";" pos) + if (pos GREATER -1) + message(FATAL_ERROR "colon not allowed in condition, but have \"${condition}\"") + endif() + set(THRCOND "${threshold}\;${condition}") + + get_test_property(${testname} CHECK_COUNT count) + #message(STATUS "add check ${count} ${THRCOND}") + if (${count} GREATER 10) + message(FATAL_ERROR "only maximum of 10 checks per test allowed") + endif() + + # add a new environment variable CHECK_X with this threshold+condition, then + # increase test property regarding check count + set_property(TEST ${testname} APPEND PROPERTY ENVIRONMENT "CHECK_${count}=${THRCOND}") + MATH(EXPR count "${count}+1") + set_tests_properties(${testname} PROPERTIES CHECK_COUNT ${count}) +endfunction() + +function(check_threshold_range testname threshold) + cmake_parse_arguments(RANGE "" "LOWER;UPPER" "" ${ARGN}) + if (NOT RANGE_LOWER AND NOT RANGE_UPPER) + message(FATAL_ERROR "need at least one LOWER or one UPPER threshold") + endif() + if (RANGE_LOWER) + check_threshold(${testname} ${threshold} "> ${RANGE_LOWER}") + endif() + if (RANGE_UPPER) + check_threshold(${testname} ${threshold} "< ${RANGE_UPPER}") + endif() +endfunction() + +function(check_threshold_variance testname threshold) + cmake_parse_arguments(VARIANCE "" "AVG;ABS_VAR" "" ${ARGN}) + if (NOT VARIANCE_AVG AND NOT VARIANCE_ABS_VAR) + message(FATAL_ERROR "need both AVG and ABS_VAR") + endif() + MATH(EXPR upper "${VARIANCE_AVG}+${VARIANCE_ABS_VAR}") + MATH(EXPR lower "${VARIANCE_AVG}-${VARIANCE_ABS_VAR}") + check_threshold_range(${testname} ${threshold} LOWER ${lower} UPPER ${upper}) +endfunction() + + #################################################################################### ###### dlsim unit test ###### #################################################################################### diff --git a/openair1/SIMULATION/tests/RunTimedTest.cmake b/openair1/SIMULATION/tests/RunTimedTest.cmake new file mode 100644 index 0000000000..779d781bec --- /dev/null +++ b/openair1/SIMULATION/tests/RunTimedTest.cmake @@ -0,0 +1,41 @@ +# this script expects +# - TEST_CMD (program + options) +# - CHECK_SCRIPT (path to analyze-timing.sh) +# further, it analyzes the (test-specific) environment for CHECK_0, CHECK_1, +# etc. to construct the input to the check script. + +# the calling process writes env vars CHECK_0, CHECK_1, ... +# each has "condition;threshold" as it's content +# combine all necessary CHECK0, CHECK1, ... into a single array CHECKS +# to be passed to the check script +foreach(count RANGE 9) + set(CHECK "CHECK_${count}") + if (NOT DEFINED ENV{${CHECK}}) + break() + endif() + list(APPEND CHECKS "$ENV{${CHECK}}") +endforeach() + +# execute the actual test command TEST_CMD and pipe its output into the +# check script. Afterwards, check both commands return codes. +execute_process(COMMAND ${TEST_CMD} + COMMAND ${CHECK_SCRIPT} ${CHECKS} + COMMAND_ECHO STDOUT + RESULTS_VARIABLE RET_CODES +) +list(LENGTH RET_CODES LEN_RET_CODES) +if(NOT LEN_RET_CODES EQUAL 2) + message(SEND_ERROR "execute_process() did not run both commands!") +endif() + +list(GET RET_CODES 0 TEST_RET_CODE) +message(STATUS "test command finished with ${TEST_RET_CODE}") +if(NOT ${TEST_RET_CODE} MATCHES "0") + message(SEND_ERROR " => test failed!") +endif() + +list(GET RET_CODES 1 CHECK_RET_CODE) +message(STATUS "check command finished with ${CHECK_RET_CODE}") +if(NOT ${CHECK_RET_CODE} MATCHES "0") + message(SEND_ERROR " => check failed!") +endif() diff --git a/openair1/SIMULATION/tests/analyze-timing.sh b/openair1/SIMULATION/tests/analyze-timing.sh new file mode 100755 index 0000000000..2ca45f346f --- /dev/null +++ b/openair1/SIMULATION/tests/analyze-timing.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# This bash script builds an awk script. The awk script tries to match +# (provided) patterns and compares them against given thresholds condition +# (e.g., threshold is "< 10"). + +function die() { echo $@ 1>&2; exit 1; } + +# RC will be the return code. If any rule fails, it will set RC=1, which will +# make the script fail. Print also every line, because the logs are piped into +# this script, but a user is typically also interested into the raw logs. +SCRIPT=' + BEGIN { RC = 0; } + + { print $0 } +' + +NUM=0 +# for each pair of /, add corresponding rules. +while [ $# -gt 0 ]; do + [ $# -ne 1 ] || die "unmatched /" + PATTERN=${1} + COND=${2} + shift 2 + + # Add a rule that searches for a PATTERN + number, and checks against + # CONDition. If the condition does not hold, it is counted as a failure (sets + # RC to signal error). In both cases, the result is logged in an array to + # output at the end of the script. + # + # Example: pattern "PHY proc tx", condition "< 200" + # The awk script tries to match every line for "PHY proc tx _NUMBER_" (where + # number is a decimal), and compares _NUMBER_ against condition "< 200", + # i.e., "_NUMBER_ < 200". + # If the condition holds, will set "CHECK PHY proc tx _NUMBER_ < 200 SUCCESS". + # If the condition fails, will set "CHECK PHY proc tx _NUMBER_ < 200 FAIL". + SCRIPT+=' + match($0, /'${PATTERN}' +([0-9]+(\.[0-9]+)?)/, n) { + if (n[1] '${COND}') { + r = "SUCCESS"; + } else { + r = "FAIL"; + RC = 1; + } + RESULTS['${NUM}']=sprintf("CHECK %-35s %7.2f %-8s %s", "'${PATTERN}'", n[1], " '${COND}'", r); + } +' + + # Generate an additional rule for the end of the logs: if the pattern is not + # found, it is counted as a failure and logged appropriately. + # Following the above example "PHY proc tx" and "< 200": if such log is not + # found, will set "CHECK PHY proc tx < 200 NOTFOUND" + SCRIPT+=' + END { + if (!RESULTS['${NUM}']) { + RESULTS['${NUM}']=sprintf("CHECK %-35s %-7s NOTFOUND", "'${PATTERN}'", "'${COND}'"); + RC = 1; + } + } +' + + let NUM=${NUM}+1 +done + +# After passing through all logs, print all results, and exit with return code +# (0 on success, i.e..all conditions checked, otherwise 1 on failure). +SCRIPT+=' + END { + for (i = 0; i < '${NUM}'; ++i) { + print RESULTS[i] + } + exit RC + } +' + +# Read from separate file descriptor 3: this allows us to pipe the actual log +# to analyze into awk (by using the script like so: analyze-timing.sh < log) +awk -f /dev/fd/3 3<<< ${SCRIPT}