From 003567675c8ebb35e30b13e011f4a8aa06ca31d4 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 7 Feb 2024 13:51:58 +0100 Subject: [PATCH] CI: phytest-timing: detect deviation from normalized values above/below The CI would only detect a deviation from the normalized value based on a configurable threshold, typically 1.25 (so we would tolerate a 25% increase). However, it often happens that a timing gets shorter (e.g., forgot to put the timing measurements). This commit introduces a "Deviation Threshold" (smaller than 1) to not only check that we exceed 1.0 + "Deviation Threshold" (as before), but also that are not below 1.0 - "Deviation Threshold". In other words, assuming a maximum divergence of 25%, instead of just checking if val > 1.25, we now also check if val < 0.75 (and flag error). --- ci-scripts/cls_oai_html.py | 14 +++++++---- ci-scripts/datalog_rt_stats.1x1.60.yaml | 30 ++++++++++++------------ ci-scripts/datalog_rt_stats.2x2.yaml | 30 +++++++++++++----------- ci-scripts/datalog_rt_stats.60.2x2.yaml | 30 ++++++++++++------------ ci-scripts/datalog_rt_stats.default.yaml | 30 ++++++++++++------------ ci-scripts/ran.py | 6 +++-- 6 files changed, 75 insertions(+), 65 deletions(-) diff --git a/ci-scripts/cls_oai_html.py b/ci-scripts/cls_oai_html.py index 81b01cd530..04971485a6 100644 --- a/ci-scripts/cls_oai_html.py +++ b/ci-scripts/cls_oai_html.py @@ -437,13 +437,19 @@ class HTMLManagement(): for k in DataLog['Data']: # TestRow + avg = DataLog['Data'][k][0] + maxval = DataLog['Data'][k][1] + count = DataLog['Data'][k][2] + valnorm = float(DataLog['Data'][k][3]) + dev = DataLog['DeviationThreshold'][k] + ref = DataLog['Ref'][k] self.htmlFile.write(' \n') self.htmlFile.write(' ' + k + ' \n') - self.htmlFile.write(' ' + DataLog['Data'][k][0] + '; ' + DataLog['Data'][k][1] + '; ' + DataLog['Data'][k][2] + ' \n') - if float(DataLog['Data'][k][3])> DataLog['Threshold'][k]: - self.htmlFile.write(' ' + DataLog['Data'][k][3] + ' (Ref = ' + str(DataLog['Ref'][k]) + ' ; Thres = ' +str(DataLog['Threshold'][k])+') ' + '\n') + self.htmlFile.write(f' {avg}; {maxval}; {count}\n') + if valnorm > 1.0 + dev or valnorm < 1.0 - dev: + self.htmlFile.write(f' {valnorm} (Avg over Ref = {avg} over {ref}; max allowed deviation = {dev})\n') else: - self.htmlFile.write(' ' + DataLog['Data'][k][3] + ' (Ref = ' + str(DataLog['Ref'][k]) + ' ; Thres = ' +str(DataLog['Threshold'][k])+') ' + '\n') + self.htmlFile.write(f' {valnorm} (Avg over Ref = {avg} over {ref}; max allowed deviation = {dev})\n') self.htmlFile.write(' \n') self.htmlFile.close() diff --git a/ci-scripts/datalog_rt_stats.1x1.60.yaml b/ci-scripts/datalog_rt_stats.1x1.60.yaml index 9160573853..5e2bf90943 100644 --- a/ci-scripts/datalog_rt_stats.1x1.60.yaml +++ b/ci-scripts/datalog_rt_stats.1x1.60.yaml @@ -5,7 +5,7 @@ Title : Processing Time (us) ColNames : - Metric - Average; Max; Count - - Average vs Reference Deviation (Reference Value; Acceptability Threshold) + - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold) Ref : feprx : 46.0 feptx_prec : 15.0 @@ -20,17 +20,17 @@ Ref : DL & UL scheduling timing : 13.0 UL Indication : 3.0 Slot Indication : 60.0 -Threshold : - feprx : 1.25 - feptx_prec : 1.25 - feptx_ofdm : 1.25 - feptx_total : 1.25 - L1 Tx processing : 1.25 - DLSCH encoding : 1.25 - L1 Rx processing : 1.25 - PUSCH inner-receiver : 1.25 - PUSCH decoding : 1.25 - Schedule Response : 1.25 - DL & UL scheduling timing : 1.25 - UL Indication : 1.25 - Slot Indication : 1.25 +DeviationThreshold : + feprx : 0.25 + feptx_prec : 0.25 + feptx_ofdm : 0.25 + feptx_total : 0.25 + L1 Tx processing : 0.25 + DLSCH encoding : 0.25 + L1 Rx processing : 0.25 + PUSCH inner-receiver : 0.25 + PUSCH decoding : 0.25 + Schedule Response : 0.25 + DL & UL scheduling timing : 0.25 + UL Indication : 0.25 + Slot Indication : 0.25 diff --git a/ci-scripts/datalog_rt_stats.2x2.yaml b/ci-scripts/datalog_rt_stats.2x2.yaml index 1d86d0e4a6..f99e09737f 100644 --- a/ci-scripts/datalog_rt_stats.2x2.yaml +++ b/ci-scripts/datalog_rt_stats.2x2.yaml @@ -5,7 +5,7 @@ Title : Processing Time (us) ColNames : - Metric - Average; Max; Count - - Average vs Reference Deviation (Reference Value; Acceptability Threshold) + - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold) Ref : feprx : 120.0 feptx_prec : 30.0 @@ -16,19 +16,21 @@ Ref : L1 Rx processing : 175.0 PUSCH inner-receiver : 100.0 PUSCH decoding : 180.0 + Schedule Response : 111.0 DL & UL scheduling timing : 37.0 UL Indication : 3.0 Slot Indication : 38.0 -Threshold : - feprx : 1.25 - feptx_prec : 1.25 - feptx_ofdm : 1.25 - feptx_total : 1.25 - L1 Tx processing : 1.25 - DLSCH encoding : 1.25 - L1 Rx processing : 1.25 - PUSCH inner-receiver : 1.25 - PUSCH decoding : 1.25 - DL & UL scheduling timing : 1.25 - UL Indication : 1.25 - Slot Indication : 1.25 +DeviationThreshold : + feprx : 0.25 + feptx_prec : 0.25 + feptx_ofdm : 0.25 + feptx_total : 0.25 + L1 Tx processing : 0.25 + DLSCH encoding : 0.25 + L1 Rx processing : 0.25 + PUSCH inner-receiver : 0.25 + PUSCH decoding : 0.25 + Schedule Response : 0.25 + DL & UL scheduling timing : 0.25 + UL Indication : 0.25 + Slot Indication : 0.25 diff --git a/ci-scripts/datalog_rt_stats.60.2x2.yaml b/ci-scripts/datalog_rt_stats.60.2x2.yaml index 4f86ebe7bd..66d01f859d 100644 --- a/ci-scripts/datalog_rt_stats.60.2x2.yaml +++ b/ci-scripts/datalog_rt_stats.60.2x2.yaml @@ -5,7 +5,7 @@ Title : Processing Time (us) ColNames : - Metric - Average; Max; Count - - Average vs Reference Deviation (Reference Value; Acceptability Threshold) + - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold) Ref : feprx : 84.0 feptx_prec : 14.0 @@ -20,17 +20,17 @@ Ref : DL & UL scheduling timing : 20.0 UL Indication : 5.0 Slot Indication : 127.0 -Threshold : - feprx : 1.25 - feptx_prec : 1.25 - feptx_ofdm : 1.25 - feptx_total : 1.25 - L1 Tx processing : 1.25 - DLSCH encoding : 1.25 - L1 Rx processing : 1.25 - PUSCH inner-receiver : 1.25 - PUSCH decoding : 1.25 - Schedule Response : 48.0 - DL & UL scheduling timing : 1.25 - UL Indication : 1.25 - Slot Indication : 1.25 +DeviationThreshold : + feprx : 0.25 + feptx_prec : 0.25 + feptx_ofdm : 0.25 + feptx_total : 0.25 + L1 Tx processing : 0.25 + DLSCH encoding : 0.25 + L1 Rx processing : 0.25 + PUSCH inner-receiver : 0.25 + PUSCH decoding : 0.25 + Schedule Response : 0.25 + DL & UL scheduling timing : 0.25 + UL Indication : 0.25 + Slot Indication : 0.25 diff --git a/ci-scripts/datalog_rt_stats.default.yaml b/ci-scripts/datalog_rt_stats.default.yaml index 970fdda184..73a5f82232 100644 --- a/ci-scripts/datalog_rt_stats.default.yaml +++ b/ci-scripts/datalog_rt_stats.default.yaml @@ -5,7 +5,7 @@ Title : Processing Time (us) ColNames : - Metric - Average; Max; Count - - Average vs Reference Deviation (Reference Value; Acceptability Threshold) + - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold) Ref : feprx : 43.0 feptx_prec : 13.0 @@ -20,17 +20,17 @@ Ref : DL & UL scheduling timing : 10.0 UL Indication : 3.0 Slot Indication : 40.0 -Threshold : - feprx : 1.25 - feptx_prec : 1.25 - feptx_ofdm : 1.25 - feptx_total : 1.25 - L1 Tx processing : 1.25 - DLSCH encoding : 1.25 - L1 Rx processing : 1.25 - PUSCH inner-receiver : 1.25 - PUSCH decoding : 1.25 - Schedule Response : 1.25 - DL & UL scheduling timing : 1.25 - UL Indication : 1.25 - Slot Indication : 1.25 +DeviationThreshold : + feprx : 0.25 + feptx_prec : 0.25 + feptx_ofdm : 0.25 + feptx_total : 0.25 + L1 Tx processing : 0.25 + DLSCH encoding : 0.25 + L1 Rx processing : 0.25 + PUSCH inner-receiver : 0.25 + PUSCH decoding : 0.25 + Schedule Response : 0.25 + DL & UL scheduling timing : 0.25 + UL Indication : 0.25 + Slot Indication : 0.25 diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 17a8b54d6d..52b9c79abf 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -1179,8 +1179,10 @@ class RANManagement(): self.datalog_rt_stats=datalog_rt_stats #check if there is a fail => will render the test as failed for k in datalog_rt_stats['Data']: - if float(datalog_rt_stats['Data'][k][3])> datalog_rt_stats['Threshold'][k]: #condition for fail : avg/ref is greater than the fixed threshold - logging.debug('\u001B[1;30;43m datalog_rt_stats metric ' + k + '=' + datalog_rt_stats['Data'][k][3] + ' > threshold ' + str(datalog_rt_stats['Threshold'][k]) + ' \u001B[0m') + valnorm = float(datalog_rt_stats['Data'][k][3]) + dev = datalog_rt_stats['DeviationThreshold'][k] + if valnorm > 1.0 + dev or valnorm < 1.0 - dev: # condition for fail : avg/ref deviates by more than "deviation threshold" + logging.debug(f'\u001B[1;30;43m normalized datalog_rt_stats metric {k}={valnorm} deviates by more than {dev}\u001B[0m') RealTimeProcessingIssue = True else: statMsg = 'No real time stats found in the log file\n'