From 67b9e92a5c875b0739a5156b7ee3a6da667d6ea2 Mon Sep 17 00:00:00 2001 From: hardy Date: Tue, 27 Apr 2021 16:54:42 +0200 Subject: [PATCH 1/5] improving stats appearance --- ci-scripts/datalog_rt_stats.yaml | 30 ++++++++++++++++ ci-scripts/html.py | 31 +++++++++++++++++ ci-scripts/ran.py | 59 ++++++++++++++++++++++++++------ 3 files changed, 109 insertions(+), 11 deletions(-) create mode 100644 ci-scripts/datalog_rt_stats.yaml diff --git a/ci-scripts/datalog_rt_stats.yaml b/ci-scripts/datalog_rt_stats.yaml new file mode 100644 index 0000000000..692962ed35 --- /dev/null +++ b/ci-scripts/datalog_rt_stats.yaml @@ -0,0 +1,30 @@ +Title : Processing Time (us) +ColNames : + - Metric + - Average + - Max + - Deviation (Avg vs Ref) +Ref : + feprx : 60.0 + feptx_prec : 8.0 + feptx_ofdm : 85.0 + feptx_total : 75.0 + L1 Tx processing : 300.0 + DLSCH encoding : 230.0 + L1 Rx processing : 175.0 + PUSCH inner-receiver : 100.0 + PUSCH decoding : 140.0 + DL & UL scheduling timing stats : 15.0 + UL Indication : 20.0 +Threshold : + feprx : 1.25 + feptx_prec : 1.25 + feptx_ofdm : 1.25 + feptx_total : 1.25 + L1 Tx processing : 1.25 + DLSCH encoding : 1.25 + L1 Rx processing : 1.25 + PUSCH inner-receiver : 1.25 + PUSCH decoding : 1.25 + DL & UL scheduling timing stats : 1.25 + UL Indication : 1.25 \ No newline at end of file diff --git a/ci-scripts/html.py b/ci-scripts/html.py index a0ac638573..aca717c0d4 100644 --- a/ci-scripts/html.py +++ b/ci-scripts/html.py @@ -446,6 +446,37 @@ class HTMLManagement(): self.htmlFile.write(' \n') self.htmlFile.close() + #for the moment it is limited to 4 columns, to be made generic later + def CreateHtmlDataLogTable(self, DataLog): + if (self.htmlFooterCreated or (not self.htmlHeaderCreated)): + return + self.htmlFile = open('test_results.html', 'a') + + # TabHeader + self.htmlFile.write(' \n') + self.htmlFile.write(' ---- ' + DataLog['Title'] + ' ---- \n') + self.htmlFile.write(' \n') + self.htmlFile.write(' \n') + self.htmlFile.write(' '+ DataLog['ColNames'][0] +'\n') + self.htmlFile.write(' ' + DataLog['ColNames'][1] + '\n') + self.htmlFile.write(' ' + DataLog['ColNames'][2] + '\n') + self.htmlFile.write(' '+ DataLog['ColNames'][3] +'\n') + self.htmlFile.write(' \n') + + for k in DataLog['Data']: + # TestRow + self.htmlFile.write(' \n') + self.htmlFile.write(' ' + k + ' \n') + self.htmlFile.write(' ' + DataLog['Data'][k][0] + ' \n') + self.htmlFile.write(' ' + DataLog['Data'][k][1] + ' \n') + if float(DataLog['Data'][k][2])> DataLog['Threshold'][k]: + self.htmlFile.write(' ' + DataLog['Data'][k][2] + ' ('+str(DataLog['Threshold'][k])+') ' + '\n') + else: + self.htmlFile.write(' ' + DataLog['Data'][k][2] + ' ('+str(DataLog['Threshold'][k])+') ' + '\n') + self.htmlFile.write(' \n') + self.htmlFile.close() + + def CreateHtmlTestRowQueue(self, options, status, ue_status, ue_queue): if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)): self.htmlFile = open('test_results.html', 'a') diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index f1f79a2435..8e378afbe9 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -37,6 +37,8 @@ import logging import os import time from multiprocessing import Process, Lock, SimpleQueue +import yaml + #----------------------------------------------------------- # OAI Testing modules @@ -90,6 +92,7 @@ class RANManagement(): self.testCase_id = '' self.epcPcapFile = '' self.runtime_stats= '' + self.datalog_rt_stats={} @@ -633,6 +636,8 @@ class RANManagement(): HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) else: HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) + if len(self.datalog_rt_stats)!=0: + HTML.CreateHtmlDataLogTable(self.datalog_rt_stats) self.eNBmbmsEnables[int(self.eNB_instance)] = False self.eNBstatuses[int(self.eNB_instance)] = -1 @@ -699,6 +704,21 @@ class RANManagement(): #count "problem receiving samples" msg pb_receiving_samples_cnt = 0 + #the datalog config file has to be loaded + datalog_rt_stats_file='datalog_rt_stats.yaml' + if (os.path.isfile(datalog_rt_stats_file)): + yaml_file=datalog_rt_stats_file + elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)): + yaml_file='ci-scripts/'+datalog_rt_stats_file + else: + logging.error("Datalog RT stats yaml file cannot be found") + sys.exit("Datalog RT stats yaml file cannot be found") + + with open(yaml_file,'r') as f: + datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader) + rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field + + for line in enb_log_file.readlines(): # Runtime statistics result = re.search('Run time:' ,str(line)) @@ -857,21 +877,21 @@ class RANManagement(): #keys below are the markers we are loooking for, loop over this keys list #everytime these markers are found in the log file, the previous ones are overwritten in the dict #eventually we record and print only the last occurence - keys = {'UE ID','dlsch_rounds','dlsch_total_bytes','ulsch_rounds','ulsch_total_bytes_scheduled', 'scheduling timing stats'} + keys = {'UE ID','dlsch_rounds','dlsch_total_bytes','ulsch_rounds','ulsch_total_bytes_scheduled'} for k in keys: result = re.search(k, line) if result is not None: #remove 1- all useless char before relevant info (ulsch or dlsch) 2- trailing char dlsch_ulsch_stats[k]=re.sub(r'^.*\]\s+', r'' , line.rstrip()) - #real time statistics - #same method as above - keys = {'feprx','feptx_prec','feptx_ofdm','feptx_total','L1 Tx processing','DLSCH encoding','L1 Rx processing','PUSCH inner-receiver','PUSCH decoding'} - for k in keys: + #real time statistics for gNB + for k in rt_keys: result = re.search(k, line) if result is not None: #remove 1- all useless char before relevant info 2- trailing char + line=line.replace('[0m','') tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex real_time_stats[k]=tmp.group(1) + #count "problem receiving samples" msg result = re.search('\[PHY\]\s+problem receiving samples', str(line)) if result is not None: @@ -920,13 +940,30 @@ class RANManagement(): logging.debug(dlsch_ulsch_stats[key]) htmleNBFailureMsg += statMsg - #real time statistics statistics + #real time statistics (old but working version) + #if len(real_time_stats)!=0: #check if dictionary is not empty + # statMsg='' + # for k in real_time_stats: + # tmp=re.match(r'^(?P.*):\s+(?P\d+\.\d+) us;\s+\d+;\s+(?P\d+\.\d+) us;',real_time_stats[k]) + # if tmp is not None: + # metric=tmp.group('metric') + # avg=tmp.group('avg') + # max=tmp.group('max') + # statMsg += metric + ' : avg = ' + avg + 'us ; max = ' + max +'us\n' + # htmleNBFailureMsg += statMsg + + #real time statistics + datalog_rt_stats['Data']={} if len(real_time_stats)!=0: #check if dictionary is not empty - statMsg='' - for key in real_time_stats: #for each dictionary key - statMsg += real_time_stats[key] + '\n' - logging.debug(real_time_stats[key]) - htmleNBFailureMsg += statMsg + for k in real_time_stats: + tmp=re.match(r'^(?P.*):\s+(?P\d+\.\d+) us;\s+\d+;\s+(?P\d+\.\d+) us;',real_time_stats[k]) + if tmp is not None: + metric=tmp.group('metric') + avg=float(tmp.group('avg')) + max=float(tmp.group('max')) + datalog_rt_stats['Data'][metric]=["{:.0f}".format(avg),"{:.0f}".format(max),"{:.2f}".format(avg/datalog_rt_stats['Ref'][metric])] + #store the data as a class attribute to build a dedicated HTML table afterward + self.datalog_rt_stats=datalog_rt_stats else: statMsg = 'No real time stats found in the log file\n' logging.debug('No real time stats found in the log file') From ccf6624f79e7736c9053d9979e39fdb315e55381 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 29 Apr 2021 10:26:10 +0200 Subject: [PATCH 2/5] changing col name in html report --- ci-scripts/datalog_rt_stats.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ci-scripts/datalog_rt_stats.yaml b/ci-scripts/datalog_rt_stats.yaml index 692962ed35..e21cbc4088 100644 --- a/ci-scripts/datalog_rt_stats.yaml +++ b/ci-scripts/datalog_rt_stats.yaml @@ -1,9 +1,12 @@ +#this is a configuration file +#used to build real time processing statistics +#for 5G NR phy test (gNB terminate) Title : Processing Time (us) ColNames : - Metric - Average - Max - - Deviation (Avg vs Ref) + - Average vs Reference Deviation (Acceptability Threshold) Ref : feprx : 60.0 feptx_prec : 8.0 @@ -25,6 +28,6 @@ Threshold : DLSCH encoding : 1.25 L1 Rx processing : 1.25 PUSCH inner-receiver : 1.25 - PUSCH decoding : 1.25 + PUSCH decoding : 1.25 DL & UL scheduling timing stats : 1.25 - UL Indication : 1.25 \ No newline at end of file + UL Indication : 1.25 From 37cc4d60331ef4aabb0a2ae61a00feafae921028 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 28 Apr 2021 21:00:35 +0200 Subject: [PATCH 3/5] fixing html columns --- ci-scripts/html.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-scripts/html.py b/ci-scripts/html.py index aca717c0d4..e51516d606 100644 --- a/ci-scripts/html.py +++ b/ci-scripts/html.py @@ -457,7 +457,7 @@ class HTMLManagement(): self.htmlFile.write(' ---- ' + DataLog['Title'] + ' ---- \n') self.htmlFile.write(' \n') self.htmlFile.write(' \n') - self.htmlFile.write(' '+ DataLog['ColNames'][0] +'\n') + self.htmlFile.write(' '+ DataLog['ColNames'][0] +'\n') self.htmlFile.write(' ' + DataLog['ColNames'][1] + '\n') self.htmlFile.write(' ' + DataLog['ColNames'][2] + '\n') self.htmlFile.write(' '+ DataLog['ColNames'][3] +'\n') @@ -466,13 +466,13 @@ class HTMLManagement(): for k in DataLog['Data']: # TestRow self.htmlFile.write(' \n') - self.htmlFile.write(' ' + k + ' \n') - self.htmlFile.write(' ' + DataLog['Data'][k][0] + ' \n') - self.htmlFile.write(' ' + DataLog['Data'][k][1] + ' \n') + self.htmlFile.write(' ' + k + ' \n') + self.htmlFile.write(' ' + DataLog['Data'][k][0] + ' \n') + self.htmlFile.write(' ' + DataLog['Data'][k][1] + ' \n') if float(DataLog['Data'][k][2])> DataLog['Threshold'][k]: - self.htmlFile.write(' ' + DataLog['Data'][k][2] + ' ('+str(DataLog['Threshold'][k])+') ' + '\n') + self.htmlFile.write(' ' + DataLog['Data'][k][2] + ' ('+str(DataLog['Threshold'][k])+') ' + '\n') else: - self.htmlFile.write(' ' + DataLog['Data'][k][2] + ' ('+str(DataLog['Threshold'][k])+') ' + '\n') + self.htmlFile.write(' ' + DataLog['Data'][k][2] + ' ('+str(DataLog['Threshold'][k])+') ' + '\n') self.htmlFile.write(' \n') self.htmlFile.close() From 7ca887221625bbef414f976eabc94ce30c3a94f8 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 29 Apr 2021 21:59:47 +0200 Subject: [PATCH 4/5] fix test status fail/pass + test update + cleaning --- ci-scripts/ran.py | 21 +++++++------------ ci-scripts/xml_files/gnb_phytest_usrp_run.xml | 4 ++-- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 8e378afbe9..f5f53c20b7 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -940,18 +940,6 @@ class RANManagement(): logging.debug(dlsch_ulsch_stats[key]) htmleNBFailureMsg += statMsg - #real time statistics (old but working version) - #if len(real_time_stats)!=0: #check if dictionary is not empty - # statMsg='' - # for k in real_time_stats: - # tmp=re.match(r'^(?P.*):\s+(?P\d+\.\d+) us;\s+\d+;\s+(?P\d+\.\d+) us;',real_time_stats[k]) - # if tmp is not None: - # metric=tmp.group('metric') - # avg=tmp.group('avg') - # max=tmp.group('max') - # statMsg += metric + ' : avg = ' + avg + 'us ; max = ' + max +'us\n' - # htmleNBFailureMsg += statMsg - #real time statistics datalog_rt_stats['Data']={} if len(real_time_stats)!=0: #check if dictionary is not empty @@ -962,8 +950,13 @@ class RANManagement(): avg=float(tmp.group('avg')) max=float(tmp.group('max')) datalog_rt_stats['Data'][metric]=["{:.0f}".format(avg),"{:.0f}".format(max),"{:.2f}".format(avg/datalog_rt_stats['Ref'][metric])] - #store the data as a class attribute to build a dedicated HTML table afterward - self.datalog_rt_stats=datalog_rt_stats + #once all metrics are collected, store the data as a class attribute to build a dedicated HTML table afterward + self.datalog_rt_stats=datalog_rt_stats + #check if there is a fail => will render the test as failed + for k in datalog_rt_stats['Data']: + if float(datalog_rt_stats['Data'][k][2])> datalog_rt_stats['Threshold'][k]: #condition for fail : avg/ref is greater than the fixed threshold + #setting prematureExit is ok although not the best option + self.prematureExit=True else: statMsg = 'No real time stats found in the log file\n' logging.debug('No real time stats found in the log file') diff --git a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml index 00610d430e..9e798afdc4 100644 --- a/ci-scripts/xml_files/gnb_phytest_usrp_run.xml +++ b/ci-scripts/xml_files/gnb_phytest_usrp_run.xml @@ -24,7 +24,7 @@ gNB-PHY-Test Run-gNB-PHY-Test tasks - 2 + 3 090101 000001 090109 @@ -40,7 +40,7 @@ IdleSleep Sleep - 180 + 300 From 7050044fcdec39b8e13d03d614bd4dfdcf87b3b0 Mon Sep 17 00:00:00 2001 From: hardy Date: Fri, 30 Apr 2021 10:24:44 +0200 Subject: [PATCH 5/5] improve understanding of results --- ci-scripts/datalog_rt_stats.yaml | 2 +- ci-scripts/html.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-scripts/datalog_rt_stats.yaml b/ci-scripts/datalog_rt_stats.yaml index e21cbc4088..3680d2a19b 100644 --- a/ci-scripts/datalog_rt_stats.yaml +++ b/ci-scripts/datalog_rt_stats.yaml @@ -6,7 +6,7 @@ ColNames : - Metric - Average - Max - - Average vs Reference Deviation (Acceptability Threshold) + - Average vs Reference Deviation (Reference Value ; Acceptability Threshold) Ref : feprx : 60.0 feptx_prec : 8.0 diff --git a/ci-scripts/html.py b/ci-scripts/html.py index e51516d606..02ba2eee2c 100644 --- a/ci-scripts/html.py +++ b/ci-scripts/html.py @@ -470,9 +470,9 @@ class HTMLManagement(): self.htmlFile.write(' ' + DataLog['Data'][k][0] + ' \n') self.htmlFile.write(' ' + DataLog['Data'][k][1] + ' \n') if float(DataLog['Data'][k][2])> DataLog['Threshold'][k]: - self.htmlFile.write(' ' + DataLog['Data'][k][2] + ' ('+str(DataLog['Threshold'][k])+') ' + '\n') + self.htmlFile.write(' ' + DataLog['Data'][k][2] + ' (Ref = ' + str(DataLog['Ref'][k]) + ' ; Thres = ' +str(DataLog['Threshold'][k])+') ' + '\n') else: - self.htmlFile.write(' ' + DataLog['Data'][k][2] + ' ('+str(DataLog['Threshold'][k])+') ' + '\n') + self.htmlFile.write(' ' + DataLog['Data'][k][2] + ' (Ref = ' + str(DataLog['Ref'][k]) + ' ; Thres = ' +str(DataLog['Threshold'][k])+') ' + '\n') self.htmlFile.write(' \n') self.htmlFile.close()