diff --git a/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf b/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf index 900566d29c..aac4f5d220 100644 --- a/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf +++ b/ci-scripts/conf_files/gnb.band78.nsa_2x2.106PRB.usrpn310.conf @@ -47,7 +47,7 @@ gNBs = dl_carrierBandwidth = 106; #initialDownlinkBWP #genericParameters - # this is RBstart=84,L=13 (275*(L-1))+RBstart + # this is RBstart=43,L=24 (275*(L-1))+RBstart initialDLBWPlocationAndBandwidth = 6368; # subcarrierSpacing # 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120 @@ -234,7 +234,7 @@ L1s = ( num_cc = 1; tr_n_preference = "local_mac"; pusch_proc_threads = 8; - prach_dtx_threshold = 100; + prach_dtx_threshold = 120; } ); diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 6d8d00610c..d29936d1bb 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -39,7 +39,6 @@ import time from multiprocessing import Process, Lock, SimpleQueue import yaml - #----------------------------------------------------------- # OAI Testing modules #----------------------------------------------------------- @@ -482,13 +481,14 @@ class RANManagement(): mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10) - #tentative + #stats monitoring during runtime time.sleep(20) + monitor_file='stats_monitor.py' if self.eNB_Stats=='yes': - monitor_file='stats_monitor.py' - #mySSH.command('echo ' + lPassWord + ' | sudo -S cp ' + self.eNBSourceCodePath + '/ci-scripts/'+ monitor_file + ' ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/.','\$', 5) - #mySSH.command('echo $USER; nohup python3 ' + self.eNBSourceCodePath + '/cmake_targets/ran_build/build/' + monitor_file + ' 2>&1 &', '\$', 5) - mySSH.command('echo $USER; nohup python3 ../ci-scripts/' + monitor_file + ' 2>&1 > stats_monitor_execution.log &', '\$', 5) + if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'): + mySSH.command('echo $USER; nohup python3 ../ci-scripts/' + monitor_file + ' enb 2>&1 > enb_stats_monitor_execution.log &', '\$', 5) + else: + mySSH.command('echo $USER; nohup python3 ../ci-scripts/' + monitor_file + ' gnb 2>&1 > gnb_stats_monitor_execution.log &', '\$', 5) @@ -686,6 +686,11 @@ class RANManagement(): fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)] self.eNBLogFiles[int(self.eNB_instance)] = '' if analyzeFile: + #*stats.log files + pickle + png + mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*stats.log', '.') + mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.pickle', '.') + mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.png', '.') + # copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.') if (copyin_res == -1): logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m') @@ -694,6 +699,14 @@ class RANManagement(): self.eNBmbmsEnables[int(self.eNB_instance)] = False return if self.eNB_serverId[self.eNB_instance] != '0': + #*stats.log files + pickle + png + + #debug / tentative + mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrL1_stats.log', self.eNBSourceCodePath + '/cmake_targets/') + mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrMAC_stats.log', self.eNBSourceCodePath + '/cmake_targets/') + mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './gnb_stats_monitor.pickle.pickle', self.eNBSourceCodePath + '/cmake_targets/') + mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './gnb_stats_monitor.png', self.eNBSourceCodePath + '/cmake_targets/') + # mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/') logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze) logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML) @@ -719,11 +732,9 @@ class RANManagement(): mySSH.command('cd cmake_targets', '\$', 5) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/*monitor.pickle .','\$',20) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/*monitor.png .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor.png', '\$', 60) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 5) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *.pickle *.png', '\$', 5) mySSH.close() def AnalyzeLogFile_eNB(self, eNBlogFile, HTML): diff --git a/ci-scripts/stats_monitor.py b/ci-scripts/stats_monitor.py index 7ce00dab46..ae9b39bfa6 100755 --- a/ci-scripts/stats_monitor.py +++ b/ci-scripts/stats_monitor.py @@ -8,8 +8,11 @@ import pickle import numpy as np import os -def collect(d): - cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log' +def collect(d, node_type): + if node_type=='enb': + cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' + else: #'gnb' + cmd='cat nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log' process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) output = process.stdout.readlines() for l in output: @@ -22,7 +25,7 @@ def collect(d): d['mcs'].append(int(result.group(4))) -def graph(d): +def graph(d, node_type): figure, axis = plt.subplots(4, 1,figsize=(10, 10)) @@ -61,11 +64,12 @@ def graph(d): plt.tight_layout() # Combine all the operations and display - plt.savefig('/tmp/stats_monitor.png') + plt.savefig(node_type+'_stats_monitor.png') plt.show() if __name__ == "__main__": + node_type = sys.argv[1]#enb or gnb d={} d['PHR']=[] @@ -74,17 +78,17 @@ if __name__ == "__main__": d['mcs']=[] - cmd='ps aux | grep mode | grep -v grep' + cmd='ps aux | grep modem | grep -v grep' process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) output = process.stdout.readlines() while len(output)!=0 : - collect(d) + collect(d, node_type) process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) output = process.stdout.readlines() time.sleep(1) print('process stopped') - with open('/tmp/stats_monitor.pickle', 'wb') as handle: + with open(node_type+'_stats_monitor.pickle', 'wb') as handle: pickle.dump(d, handle, protocol=pickle.HIGHEST_PROTOCOL) - graph(d) + graph(d, node_type) diff --git a/ci-scripts/stats_monitor_conf.yaml b/ci-scripts/stats_monitor_conf.yaml new file mode 100644 index 0000000000..3da7c1e8ba --- /dev/null +++ b/ci-scripts/stats_monitor_conf.yaml @@ -0,0 +1,11 @@ +enb : + PHR: + bler: + mcsoff: + mcs: + +gnb : + PHR: + bler: + mcsoff: + mcs: \ No newline at end of file diff --git a/ci-scripts/stats_monitor_dev.py b/ci-scripts/stats_monitor_dev.py new file mode 100755 index 0000000000..cbf0d20880 --- /dev/null +++ b/ci-scripts/stats_monitor_dev.py @@ -0,0 +1,100 @@ +import subprocess +import time +import shlex +import re +import sys +import matplotlib.pyplot as plt +import pickle +import numpy as np +import os +import yaml + + +class Stat_Monitor(): + def __init__(self,): + with open('stats_monitor_conf.yaml','r') as f: + self.d = yaml.load(f) + for node in self.d: + for metric in self.d[node]: + self.d[node][metric]=[] + + + def process_enb (self,node_type,output): + for l in output: + tmp=l.decode("utf-8") + result=re.match(rf'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp) + if result is not None: + self.d[node_type]['PHR'].append(int(result.group(1))) + self.d[node_type]['bler'].append(float(result.group(2))) + self.d[node_type]['mcsoff'].append(int(result.group(3))) + self.d[node_type]['mcs'].append(int(result.group(4))) + + + + def process_gnb (self,node_type,output): + for l in output: + tmp=l.decode("utf-8") + result=re.match(rf'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp) + if result is not None: + self.d[node_type]['PHR'].append(int(result.group(1))) + self.d[node_type]['bler'].append(float(result.group(2))) + self.d[node_type]['mcsoff'].append(int(result.group(3))) + self.d[node_type]['mcs'].append(int(result.group(4))) + + + + + + def collect(self,node_type): + if node_type=='enb': + cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' + else: #'gnb' + cmd='cat nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log' + process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + output = process.stdout.readlines() + if node_type=='enb': + self.process_enb(node_type,output) + else: #'gnb' + self.process_gnb(node_type,output) + + def graph(self,node_type): + + col = 1 + figure, axis = plt.subplots(len(self.d[node_type]), col ,figsize=(10, 10)) + i=0 + for metric in self.d[node_type]: + major_ticks = np.arange(0, len(self.d[node_type][metric])+1, 1) + axis[i].set_xticks(major_ticks) + axis[i].set_xticklabels([]) + axis[i].plot(self.d[node_type][metric],marker='o') + axis[i].set_xlabel('time') + axis[i].set_ylabel(metric) + axis[i].set_title(metric) + i+=1 + + plt.tight_layout() + # Combine all the operations and display + plt.savefig(node_type+'_stats_monitor.png') + plt.show() + + +if __name__ == "__main__": + + node_type = sys.argv[1]#enb or gnb + mon=Stat_Monitor() + + #collecting stats when modem process is stopped + cmd='ps aux | grep mode | grep -v grep' + process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + output = process.stdout.readlines() + while len(output)!=0 : + mon.collect(node_type) + process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + output = process.stdout.readlines() + time.sleep(1) + print('Process stopped') + with open(node_type+'_stats_monitor.pickle', 'wb') as handle: + pickle.dump(mon.d, handle, protocol=pickle.HIGHEST_PROTOCOL) + mon.graph(node_type) + + diff --git a/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml b/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml index 9e1c7afde3..1bc7e918f5 100644 --- a/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_2x2_quectel.xml @@ -65,6 +65,7 @@ 0 lte yes + yes 192.168.18.241 @@ -76,6 +77,7 @@ 1 1 nr + yes 192.168.18.240 @@ -111,7 +113,7 @@ Iperf iperf (DL/20Mbps/UDP)(60 sec)(single-ue profile) - -u -b 20M -t 60 + -u -b 20M -t 10 DL nrmodule2_quectel 5 @@ -120,8 +122,8 @@ Iperf - iperf (UL/3Mbps/UDP)(60 sec)(single-ue profile) - -u -b 3M -t 60 + iperf (UL/1Mbps/UDP)(60 sec)(single-ue profile) + -u -b 1M -t 10 UL nrmodule2_quectel 5 diff --git a/ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml b/ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml index 31eee2ca3d..0005ce8844 100644 --- a/ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml +++ b/ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml @@ -143,6 +143,7 @@ services: TAC: 1 GW_ID: 1 REALM: openairinterface.org + UE_MTU_IPV4: 1500 healthcheck: test: /bin/bash -c "pgrep oai_spgwc" interval: 10s