995 lines
46 KiB
Python
995 lines
46 KiB
Python
#/*
|
|
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
|
# * contributor license agreements. See the NOTICE file distributed with
|
|
# * this work for additional information regarding copyright ownership.
|
|
# * The OpenAirInterface Software Alliance licenses this file to You under
|
|
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
|
# * except in compliance with the License.
|
|
# * You may obtain a copy of the License at
|
|
# *
|
|
# * http://www.openairinterface.org/?page_id=698
|
|
# *
|
|
# * Unless required by applicable law or agreed to in writing, software
|
|
# * distributed under the License is distributed on an "AS IS" BASIS,
|
|
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# * See the License for the specific language governing permissions and
|
|
# * limitations under the License.
|
|
# *-------------------------------------------------------------------------------
|
|
# * For more information about the OpenAirInterface (OAI) Software Alliance:
|
|
# * contact@openairinterface.org
|
|
# */
|
|
#---------------------------------------------------------------------
|
|
# Python for CI of OAI-eNB + COTS-UE
|
|
#
|
|
# Required Python Version
|
|
# Python 3.x
|
|
#
|
|
# Required Python Package
|
|
# pexpect
|
|
#---------------------------------------------------------------------
|
|
|
|
#-----------------------------------------------------------
|
|
# Import
|
|
#-----------------------------------------------------------
|
|
import sys # arg
|
|
import re # reg
|
|
import logging
|
|
import os
|
|
import time
|
|
from multiprocessing import Process, Lock, SimpleQueue
|
|
|
|
#-----------------------------------------------------------
|
|
# OAI Testing modules
|
|
#-----------------------------------------------------------
|
|
import sshconnection as SSH
|
|
import epc
|
|
import helpreadme as HELP
|
|
import constants as CONST
|
|
import html
|
|
|
|
#-----------------------------------------------------------
|
|
# Class Declaration
|
|
#-----------------------------------------------------------
|
|
class RANManagement():
|
|
|
|
def __init__(self):
|
|
|
|
self.prematureExit = False
|
|
self.ranRepository = ''
|
|
self.ranBranch = ''
|
|
self.ranAllowMerge = False
|
|
self.ranCommitID = ''
|
|
self.ranTargetBranch = ''
|
|
self.eNBIPAddress = ''
|
|
self.eNBUserName = ''
|
|
self.eNBPassword = ''
|
|
self.eNBSourceCodePath = ''
|
|
self.eNB1IPAddress = ''
|
|
self.eNB1UserName = ''
|
|
self.eNB1Password = ''
|
|
self.eNB1SourceCodePath = ''
|
|
self.eNB2IPAddress = ''
|
|
self.eNB2UserName = ''
|
|
self.eNB2Password = ''
|
|
self.eNB2SourceCodePath = ''
|
|
self.Build_eNB_args = ''
|
|
self.backgroundBuild = False
|
|
self.backgroundBuildTestId = ['', '', '']
|
|
self.Build_eNB_forced_workspace_cleanup = False
|
|
self.Initialize_eNB_args = ''
|
|
self.air_interface = ['', '', ''] #changed from 'lte' to '' may lead to side effects in main
|
|
self.eNB_instance = 0
|
|
self.eNB_serverId = ''
|
|
self.eNBLogFiles = ['', '', '']
|
|
self.eNBOptions = ['', '', '']
|
|
self.eNBmbmsEnables = [False, False, False]
|
|
self.eNBstatuses = [-1, -1, -1]
|
|
self.flexranCtrlInstalled = False
|
|
self.flexranCtrlStarted = False
|
|
self.testCase_id = ''
|
|
self.epcPcapFile = ''
|
|
self.htmlObj = None
|
|
self.epcObj = None
|
|
self.runtime_stats= ''
|
|
|
|
|
|
|
|
#-----------------------------------------------------------
|
|
# RAN management functions
|
|
#-----------------------------------------------------------
|
|
|
|
def BuildeNB(self):
|
|
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
if self.eNB_serverId == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.eNB_serverId == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.eNB_serverId == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
mySSH = SSH.SSHConnection()
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
|
|
# Check if we build an 5G-NR gNB or an LTE eNB or an OCP eNB
|
|
result = re.search('--eNBocp', self.Build_eNB_args)
|
|
if result is not None:
|
|
self.air_interface[self.eNB_instance] = 'ocp-enb'
|
|
else:
|
|
result = re.search('--gNB', self.Build_eNB_args)
|
|
if result is not None:
|
|
self.air_interface[self.eNB_instance] = 'nr-softmodem'
|
|
else:
|
|
self.air_interface[self.eNB_instance] = 'lte-softmodem'
|
|
|
|
# Worakround for some servers, we need to erase completely the workspace
|
|
if self.Build_eNB_forced_workspace_cleanup:
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15)
|
|
if self.htmlObj is not None:
|
|
self.testCase_id = self.htmlObj.testCase_id
|
|
else:
|
|
self.testCase_id = '000000'
|
|
# on RedHat/CentOS .git extension is mandatory
|
|
result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
|
|
if result is not None:
|
|
full_ran_repo_name = self.ranRepository
|
|
else:
|
|
full_ran_repo_name = self.ranRepository + '.git'
|
|
mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
|
|
mySSH.command('cd ' + lSourcePath, '\$', 5)
|
|
mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
|
|
# Raphael: here add a check if git clone or git fetch went smoothly
|
|
mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
|
|
mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
|
|
# Checking the BUILD INFO file
|
|
if not self.backgroundBuild:
|
|
mySSH.command('ls *.txt', '\$', 5)
|
|
result = re.search('LAST_BUILD_INFO', mySSH.getBefore())
|
|
if result is not None:
|
|
mismatch = False
|
|
mySSH.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
|
|
result = re.search(self.ranCommitID, mySSH.getBefore())
|
|
if result is None:
|
|
mismatch = True
|
|
mySSH.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
|
|
if (self.ranAllowMerge):
|
|
result = re.search('YES', mySSH.getBefore())
|
|
if result is None:
|
|
mismatch = True
|
|
mySSH.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
|
|
if self.ranTargetBranch == '':
|
|
result = re.search('develop', mySSH.getBefore())
|
|
else:
|
|
result = re.search(self.ranTargetBranch, mySSH.getBefore())
|
|
if result is None:
|
|
mismatch = True
|
|
else:
|
|
result = re.search('NO', mySSH.getBefore())
|
|
if result is None:
|
|
mismatch = True
|
|
if not mismatch:
|
|
mySSH.close()
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
|
|
return
|
|
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
|
|
# if the commit ID is provided use it to point to it
|
|
if self.ranCommitID != '':
|
|
mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5)
|
|
# if the branch is not develop, then it is a merge request and we need to do
|
|
# the potential merge. Note that merge conflicts should already been checked earlier
|
|
if (self.ranAllowMerge):
|
|
if self.ranTargetBranch == '':
|
|
if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
|
|
mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
|
|
else:
|
|
logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
|
|
mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
|
|
mySSH.command('source oaienv', '\$', 5)
|
|
mySSH.command('cd cmake_targets', '\$', 5)
|
|
mySSH.command('mkdir -p log', '\$', 5)
|
|
mySSH.command('chmod 777 log', '\$', 5)
|
|
# no need to remove in log (git clean did the trick)
|
|
if self.backgroundBuild:
|
|
mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5)
|
|
mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5)
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S ls', '\$', 5)
|
|
mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-build.sh' + ' > ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ' + ' 2>&1 &', lUserName, 5)
|
|
mySSH.close()
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
|
|
self.backgroundBuildTestId[int(self.eNB_instance)] = self.testCase_id
|
|
return
|
|
mySSH.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests|build have failed', 1500)
|
|
mySSH.close()
|
|
self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.testCase_id)
|
|
|
|
def WaitBuildeNBisFinished(self):
|
|
if self.eNB_serverId == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.eNB_serverId == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.eNB_serverId == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
mySSH = SSH.SSHConnection()
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
count = 40
|
|
buildOAIprocess = True
|
|
while (count > 0) and buildOAIprocess:
|
|
mySSH.command('ps aux | grep --color=never build_ | grep -v grep', '\$', 6)
|
|
result = re.search('build_oai', mySSH.getBefore())
|
|
if result is None:
|
|
buildOAIprocess = False
|
|
else:
|
|
count -= 1
|
|
time.sleep(30)
|
|
mySSH.close()
|
|
self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.backgroundBuildTestId[int(self.eNB_instance)])
|
|
|
|
def checkBuildeNB(self, lIpAddr, lUserName, lPassWord, lSourcePath, testcaseId):
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.testCase_id=testcaseId
|
|
|
|
mySSH = SSH.SSHConnection()
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 3)
|
|
mySSH.command('ls ran_build/build', '\$', 3)
|
|
mySSH.command('ls ran_build/build', '\$', 3)
|
|
|
|
#check if we have the build corresponding to the air interface keywords (nr-softmode, lte-softmodem, ocp-enb)
|
|
logging.info('CHECK Build with IP='+lIpAddr+' SourcePath='+lSourcePath)
|
|
result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore())
|
|
if result is None:
|
|
buildStatus = False #if not, build failed
|
|
else:
|
|
buildStatus = True
|
|
# Generating a BUILD INFO file
|
|
mySSH.command('echo "SRC_BRANCH: ' + self.ranBranch + '" > ../LAST_BUILD_INFO.txt', '\$', 2)
|
|
mySSH.command('echo "SRC_COMMIT: ' + self.ranCommitID + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
|
|
if (self.ranAllowMerge):
|
|
mySSH.command('echo "MERGED_W_TGT_BRANCH: YES" >> ../LAST_BUILD_INFO.txt', '\$', 2)
|
|
if self.ranTargetBranch == '':
|
|
mySSH.command('echo "TGT_BRANCH: develop" >> ../LAST_BUILD_INFO.txt', '\$', 2)
|
|
else:
|
|
mySSH.command('echo "TGT_BRANCH: ' + self.ranTargetBranch + '" >> ../LAST_BUILD_INFO.txt', '\$', 2)
|
|
else:
|
|
mySSH.command('echo "MERGED_W_TGT_BRANCH: NO" >> ../LAST_BUILD_INFO.txt', '\$', 2)
|
|
|
|
|
|
mySSH.command('mkdir -p build_log_' + testcaseId, '\$', 5)
|
|
mySSH.command('mv log/* ' + 'build_log_' + testcaseId, '\$', 5)
|
|
mySSH.command('mv compile_oai_enb.log ' + 'build_log_' + testcaseId, '\$', 5)
|
|
if self.eNB_serverId != '0':
|
|
mySSH.command('cd cmake_targets', '\$', 5)
|
|
mySSH.command('if [ -e tmp_build' + testcaseId + '.zip ]; then rm -f tmp_build' + testcaseId + '.zip; fi', '\$', 5)
|
|
mySSH.command('zip -r -qq tmp_build' + testcaseId + '.zip build_log_' + testcaseId, '\$', 5)
|
|
mySSH.close()
|
|
if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
|
|
os.remove('./tmp_build' + testcaseId + '.zip')
|
|
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/tmp_build' + testcaseId + '.zip', '.')
|
|
if (os.path.isfile('./tmp_build' + testcaseId + '.zip')):
|
|
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './tmp_build' + testcaseId + '.zip', self.eNBSourceCodePath + '/cmake_targets/.')
|
|
os.remove('./tmp_build' + testcaseId + '.zip')
|
|
mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
|
|
mySSH.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5)
|
|
#-qq quiet / -u update orcreate files
|
|
mySSH.command('unzip -u -qq -DD tmp_build' + testcaseId + '.zip', '\$', 5)
|
|
mySSH.command('rm -f tmp_build' + testcaseId + '.zip', '\$', 5)
|
|
mySSH.close()
|
|
else:
|
|
mySSH.close()
|
|
|
|
#generate logging info depending on buildStatus and air interface
|
|
if buildStatus:
|
|
logging.info('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Pass\u001B[0m')
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK)
|
|
else:
|
|
logging.error('\u001B[1m Building OAI ' + self.air_interface[self.eNB_instance] + ' Failed\u001B[0m')
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow(self.Build_eNB_args, 'KO', CONST.ALL_PROCESSES_OK)
|
|
self.htmlObj.CreateHtmlTabFooter(False)
|
|
sys.exit(1)
|
|
|
|
def InitializeeNB(self):
|
|
if self.eNB_serverId == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.eNB_serverId == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.eNB_serverId == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
|
|
if self.htmlObj is not None:
|
|
self.testCase_id = self.htmlObj.testCase_id
|
|
else:
|
|
self.testCase_id = '000000'
|
|
mySSH = SSH.SSHConnection()
|
|
|
|
if (self.pStatus < 0):
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus)
|
|
self.htmlObj.CreateHtmlTabFooter(False)
|
|
sys.exit(1)
|
|
# If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB
|
|
result = re.search('T_stdout', str(self.Initialize_eNB_args))
|
|
if (result is not None) and (self.epcObj is not None):
|
|
localEpcIpAddr = self.epcObj.IPAddress
|
|
localEpcUserName = self.epcObj.UserName
|
|
localEpcPassword = self.epcObj.Password
|
|
mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
|
|
mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5)
|
|
result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore())
|
|
if result is not None:
|
|
eth_interface = result.group('eth_interface')
|
|
logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m')
|
|
self.epcPcapFile = 'enb_' + self.testCase_id + '_s1log.pcap'
|
|
mySSH.command('echo ' + localEpcPassword + ' | sudo -S rm -f /tmp/' + self.epcPcapFile , '\$', 5)
|
|
mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5)
|
|
mySSH.close()
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
mySSH.command('cd ' + lSourcePath, '\$', 5)
|
|
# Initialize_eNB_args usually start with -O and followed by the location in repository
|
|
full_config_file = self.Initialize_eNB_args.replace('-O ','')
|
|
extra_options = ''
|
|
extIdx = full_config_file.find('.conf')
|
|
if (extIdx > 0):
|
|
extra_options = full_config_file[extIdx + 5:]
|
|
# if tracer options is on, compiling and running T Tracer
|
|
result = re.search('T_stdout', str(extra_options))
|
|
if result is not None:
|
|
logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m')
|
|
mySSH.command('cd common/utils/T/tracer', '\$', 5)
|
|
mySSH.command('make', '\$', 10)
|
|
mySSH.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', lUserName, 5)
|
|
mySSH.command('cd ' + lSourcePath, '\$', 5)
|
|
full_config_file = full_config_file[:extIdx + 5]
|
|
config_path, config_file = os.path.split(full_config_file)
|
|
else:
|
|
sys.exit('Insufficient Parameter')
|
|
ci_full_config_file = config_path + '/ci-' + config_file
|
|
rruCheck = False
|
|
result = re.search('^rru|^rcc|^du.band', str(config_file))
|
|
if result is not None:
|
|
rruCheck = True
|
|
# do not reset board twice in IF4.5 case
|
|
result = re.search('^rru|^enb|^du.band', str(config_file))
|
|
if result is not None:
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60)
|
|
result = re.search('type: b200', mySSH.getBefore())
|
|
if result is not None:
|
|
logging.debug('Found a B2xx device --> resetting it')
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10)
|
|
# Reloading FGPA bin firmware
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60)
|
|
# Make a copy and adapt to EPC / eNB IP addresses
|
|
mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5)
|
|
if self.epcObj is not None:
|
|
localMmeIpAddr = self.epcObj.MmeIPAddress
|
|
mySSH.command('sed -i -e \'s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
|
|
mySSH.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
|
|
mySSH.command('sed -i -e \'s/CI_GNB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2);
|
|
mySSH.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
|
|
mySSH.command('sed -i -e \'s/CI_RRU1_IP_ADDR/' + self.eNB1IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
|
|
mySSH.command('sed -i -e \'s/CI_RRU2_IP_ADDR/' + self.eNB2IPAddress + '/\' ' + ci_full_config_file, '\$', 2);
|
|
mySSH.command('sed -i -e \'s/CI_FR1_CTL_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2);
|
|
if self.flexranCtrlInstalled and self.flexranCtrlStarted:
|
|
mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED = "yes";/\' ' + ci_full_config_file, '\$', 2);
|
|
else:
|
|
mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED = "no";/\' ' + ci_full_config_file, '\$', 2);
|
|
self.eNBmbmsEnables[int(self.eNB_instance)] = False
|
|
mySSH.command('grep enable_enb_m2 ' + ci_full_config_file, '\$', 2);
|
|
result = re.search('yes', mySSH.getBefore())
|
|
if result is not None:
|
|
self.eNBmbmsEnables[int(self.eNB_instance)] = True
|
|
logging.debug('\u001B[1m MBMS is enabled on this eNB\u001B[0m')
|
|
result = re.search('noS1', str(self.Initialize_eNB_args))
|
|
eNBinNoS1 = False
|
|
if result is not None:
|
|
eNBinNoS1 = True
|
|
logging.debug('\u001B[1m eNB is in noS1 configuration \u001B[0m')
|
|
# Launch eNB with the modified config file
|
|
mySSH.command('source oaienv', '\$', 5)
|
|
mySSH.command('cd cmake_targets', '\$', 5)
|
|
if self.air_interface == 'nr':
|
|
mySSH.command('if [ -e rbconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm rbconfig.raw; fi', '\$', 5)
|
|
mySSH.command('if [ -e reconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm reconfig.raw; fi', '\$', 5)
|
|
# NOTE: WE SHALL do a check if the executable is present (in case build went wrong)
|
|
mySSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
|
mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5)
|
|
mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10)
|
|
self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log'
|
|
if extra_options != '':
|
|
self.eNBOptions[int(self.eNB_instance)] = extra_options
|
|
time.sleep(6)
|
|
doLoop = True
|
|
loopCounter = 20
|
|
enbDidSync = False
|
|
while (doLoop):
|
|
loopCounter = loopCounter - 1
|
|
if (loopCounter == 0):
|
|
# In case of T tracer recording, we may need to kill it
|
|
result = re.search('T_stdout', str(self.Initialize_eNB_args))
|
|
if result is not None:
|
|
mySSH.command('killall --signal SIGKILL record', '\$', 5)
|
|
mySSH.close()
|
|
doLoop = False
|
|
logging.error('\u001B[1;37;41m eNB/gNB/ocp-eNB logging system did not show got sync! \u001B[0m')
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'KO', CONST.ALL_PROCESSES_OK)
|
|
# In case of T tracer recording, we need to kill tshark on EPC side
|
|
result = re.search('T_stdout', str(self.Initialize_eNB_args))
|
|
if (result is not None) and (self.epcObj is not None):
|
|
localEpcIpAddr = self.epcObj.IPAddress
|
|
localEpcUserName = self.epcObj.UserName
|
|
localEpcPassword = self.epcObj.Password
|
|
mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
|
|
logging.debug('\u001B[1m Stopping tshark \u001B[0m')
|
|
mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
|
|
if self.epcPcapFile != '':
|
|
time.sleep(0.5)
|
|
mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
|
|
mySSH.close()
|
|
time.sleep(1)
|
|
if self.epcPcapFile != '':
|
|
copyin_res = mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
|
|
if (copyin_res == 0):
|
|
mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
|
|
self.prematureExit = True
|
|
return
|
|
else:
|
|
mySSH.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting"', '\$', 4)
|
|
if rruCheck:
|
|
result = re.search('wait RUs', mySSH.getBefore())
|
|
else:
|
|
result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore())
|
|
if result is None:
|
|
time.sleep(6)
|
|
else:
|
|
doLoop = False
|
|
enbDidSync = True
|
|
time.sleep(10)
|
|
|
|
rruCheck = False
|
|
result = re.search('^rru|^du.band', str(config_file))
|
|
if result is not None:
|
|
rruCheck = True
|
|
if enbDidSync and eNBinNoS1 and not rruCheck:
|
|
mySSH.command('ifconfig oaitun_enb1', '\$', 4)
|
|
mySSH.command('ifconfig oaitun_enb1', '\$', 4)
|
|
result = re.search('inet addr:1|inet 1', mySSH.getBefore())
|
|
if result is not None:
|
|
logging.debug('\u001B[1m oaitun_enb1 interface is mounted and configured\u001B[0m')
|
|
else:
|
|
logging.error('\u001B[1m oaitun_enb1 interface is either NOT mounted or NOT configured\u001B[0m')
|
|
if self.eNBmbmsEnables[int(self.eNB_instance)]:
|
|
mySSH.command('ifconfig oaitun_enm1', '\$', 4)
|
|
result = re.search('inet addr', mySSH.getBefore())
|
|
if result is not None:
|
|
logging.debug('\u001B[1m oaitun_enm1 interface is mounted and configured\u001B[0m')
|
|
else:
|
|
logging.error('\u001B[1m oaitun_enm1 interface is either NOT mounted or NOT configured\u001B[0m')
|
|
if enbDidSync:
|
|
self.eNBstatuses[int(self.eNB_instance)] = int(self.eNB_serverId)
|
|
|
|
mySSH.close()
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'OK', CONST.ALL_PROCESSES_OK)
|
|
logging.debug('\u001B[1m Initialize eNB/gNB/ocp-eNB Completed\u001B[0m')
|
|
|
|
def CheckeNBProcess(self, status_queue):
|
|
try:
|
|
# At least the instance 0 SHALL be on!
|
|
if self.eNBstatuses[0] == 0:
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
elif self.eNBstatuses[0] == 1:
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
elif self.eNBstatuses[0] == 2:
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
else:
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
mySSH = SSH.SSHConnection()
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + self.air_interface[self.eNB_instance] + ' | grep -v grep', '\$', 5)
|
|
result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore())
|
|
if result is None:
|
|
logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
|
|
status_queue.put(CONST.ENB_PROCESS_FAILED)
|
|
else:
|
|
status_queue.put(CONST.ENB_PROCESS_OK)
|
|
mySSH.close()
|
|
except:
|
|
os.kill(os.getppid(),signal.SIGUSR1)
|
|
|
|
def TerminateeNB(self):
|
|
if self.eNB_serverId == '0':
|
|
lIpAddr = self.eNBIPAddress
|
|
lUserName = self.eNBUserName
|
|
lPassWord = self.eNBPassword
|
|
lSourcePath = self.eNBSourceCodePath
|
|
elif self.eNB_serverId == '1':
|
|
lIpAddr = self.eNB1IPAddress
|
|
lUserName = self.eNB1UserName
|
|
lPassWord = self.eNB1Password
|
|
lSourcePath = self.eNB1SourceCodePath
|
|
elif self.eNB_serverId == '2':
|
|
lIpAddr = self.eNB2IPAddress
|
|
lUserName = self.eNB2UserName
|
|
lPassWord = self.eNB2Password
|
|
lSourcePath = self.eNB2SourceCodePath
|
|
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
|
HELP.GenericHelp(CONST.Version)
|
|
sys.exit('Insufficient Parameter')
|
|
mySSH = SSH.SSHConnection()
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
|
|
if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
|
|
nodeB_prefix = 'e'
|
|
else:
|
|
nodeB_prefix = 'g'
|
|
mySSH.command('stdbuf -o0 ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5)
|
|
result = re.search('(-softmodem|ocp)', mySSH.getBefore())
|
|
if result is not None:
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT -r .*-softmodem ocp-enb || true', '\$', 5)
|
|
time.sleep(10)
|
|
mySSH.command('stdbuf -o0 ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5)
|
|
result = re.search('(-softmodem|ocp)', mySSH.getBefore())
|
|
if result is not None:
|
|
mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5)
|
|
time.sleep(5)
|
|
mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5)
|
|
mySSH.close()
|
|
# If tracer options is on, stopping tshark on EPC side
|
|
result = re.search('T_stdout', str(self.Initialize_eNB_args))
|
|
if (result is not None) and (self.epcObj is not None):
|
|
localEpcIpAddr = self.epcObj.IPAddress
|
|
localEpcUserName = self.epcObj.UserName
|
|
localEpcPassword = self.epcObj.Password
|
|
mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword)
|
|
logging.debug('\u001B[1m Stopping tshark \u001B[0m')
|
|
mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5)
|
|
time.sleep(1)
|
|
if self.epcPcapFile != '':
|
|
mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5)
|
|
mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.')
|
|
mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.')
|
|
mySSH.close()
|
|
logging.debug('\u001B[1m Replaying RAW record file\u001B[0m')
|
|
mySSH.open(lIpAddr, lUserName, lPassWord)
|
|
mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5)
|
|
enbLogFile = self.eNBLogFiles[int(self.eNB_instance)]
|
|
raw_record_file = enbLogFile.replace('.log', '_record.raw')
|
|
replay_log_file = enbLogFile.replace('.log', '_replay.log')
|
|
extracted_txt_file = enbLogFile.replace('.log', '_extracted_messages.txt')
|
|
extracted_log_file = enbLogFile.replace('.log', '_extracted_messages.log')
|
|
mySSH.command('./extract_config -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + extracted_txt_file, '\$', 5)
|
|
mySSH.command('echo $USER; nohup ./replay -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', lUserName, 5)
|
|
mySSH.command('./textlog -d ' + lSourcePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + lSourcePath + '/cmake_targets/' + extracted_log_file, '\$', 5)
|
|
mySSH.close()
|
|
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + extracted_log_file, '.')
|
|
logging.debug('\u001B[1m Analyzing eNB replay logfile \u001B[0m')
|
|
logStatus = self.AnalyzeLogFile_eNB(extracted_log_file)
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
|
|
self.eNBLogFiles[int(self.eNB_instance)] = ''
|
|
else:
|
|
analyzeFile = False
|
|
if self.eNBLogFiles[int(self.eNB_instance)] != '':
|
|
analyzeFile = True
|
|
fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
|
|
self.eNBLogFiles[int(self.eNB_instance)] = ''
|
|
if analyzeFile:
|
|
copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
|
|
if (copyin_res == -1):
|
|
logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
|
|
self.htmlObj.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
|
|
self.eNBmbmsEnables[int(self.eNB_instance)] = False
|
|
return
|
|
if self.eNB_serverId != '0':
|
|
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
|
|
logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze)
|
|
logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze)
|
|
if (logStatus < 0):
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow('N/A', 'KO', logStatus)
|
|
self.preamtureExit = True
|
|
self.eNBmbmsEnables[int(self.eNB_instance)] = False
|
|
return
|
|
else:
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
|
|
else:
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
|
|
self.eNBmbmsEnables[int(self.eNB_instance)] = False
|
|
self.eNBstatuses[int(self.eNB_instance)] = -1
|
|
|
|
def LogCollecteNB(self):
|
|
mySSH = SSH.SSHConnection()
|
|
mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword)
|
|
mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5)
|
|
mySSH.command('cd cmake_targets', '\$', 5)
|
|
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5)
|
|
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log', '\$', 60)
|
|
mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 5)
|
|
mySSH.close()
|
|
|
|
def AnalyzeLogFile_eNB(self, eNBlogFile):
|
|
if (not os.path.isfile('./' + eNBlogFile)):
|
|
return -1
|
|
enb_log_file = open('./' + eNBlogFile, 'r')
|
|
exitSignalReceived = False
|
|
foundAssertion = False
|
|
msgAssertion = ''
|
|
msgLine = 0
|
|
foundSegFault = False
|
|
foundRealTimeIssue = False
|
|
rrcSetupComplete = 0
|
|
rrcReleaseRequest = 0
|
|
rrcReconfigRequest = 0
|
|
rrcReconfigComplete = 0
|
|
rrcReestablishRequest = 0
|
|
rrcReestablishComplete = 0
|
|
rrcReestablishReject = 0
|
|
rlcDiscardBuffer = 0
|
|
rachCanceledProcedure = 0
|
|
uciStatMsgCount = 0
|
|
pdcpFailure = 0
|
|
ulschFailure = 0
|
|
ulschAllocateCCEerror = 0
|
|
uplinkSegmentsAborted = 0
|
|
ulschReceiveOK = 0
|
|
gnbRxTxWakeUpFailure = 0
|
|
gnbTxWriteThreadEnabled = False
|
|
cdrxActivationMessageCount = 0
|
|
dropNotEnoughRBs = 0
|
|
mbmsRequestMsg = 0
|
|
htmleNBFailureMsg = ''
|
|
isRRU = False
|
|
isSlave = False
|
|
slaveReceivesFrameResyncCmd = False
|
|
X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
|
|
X2HO_inNbProcedures = 0
|
|
X2HO_outNbProcedures = 0
|
|
global_status = CONST.ALL_PROCESSES_OK
|
|
# Runtime statistics
|
|
runTime = ''
|
|
userTime = ''
|
|
systemTime = ''
|
|
maxPhyMemUsage = ''
|
|
nbContextSwitches = ''
|
|
for line in enb_log_file.readlines():
|
|
# Runtime statistics
|
|
result = re.search('Run time:' ,str(line))
|
|
if result is not None:
|
|
runTime = str(line).strip()
|
|
if runTime != '':
|
|
result = re.search('Time executing user inst', str(line))
|
|
if result is not None:
|
|
fields=line.split(':')
|
|
userTime = 'userTime : ' + fields[1].replace('\n','')
|
|
result = re.search('Time executing system inst', str(line))
|
|
if result is not None:
|
|
fields=line.split(':')
|
|
systemTime = 'systemTime : ' + fields[1].replace('\n','')
|
|
result = re.search('Max. Phy. memory usage:', str(line))
|
|
if result is not None:
|
|
fields=line.split(':')
|
|
maxPhyMemUsage = 'maxPhyMemUsage : ' + fields[1].replace('\n','')
|
|
result = re.search('Number of context switch.*process origin', str(line))
|
|
if result is not None:
|
|
fields=line.split(':')
|
|
nbContextSwitches = 'nbContextSwitches : ' + fields[1].replace('\n','')
|
|
if X2HO_state == CONST.X2_HO_REQ_STATE__IDLE:
|
|
result = re.search('target eNB Receives X2 HO Req X2AP_HANDOVER_REQ', str(line))
|
|
if result is not None:
|
|
X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ
|
|
result = re.search('source eNB receives the X2 HO ACK X2AP_HANDOVER_REQ_ACK', str(line))
|
|
if result is not None:
|
|
X2HO_state = CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK
|
|
if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ:
|
|
result = re.search('Received LTE_RRCConnectionReconfigurationComplete from UE', str(line))
|
|
if result is not None:
|
|
X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE
|
|
if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE:
|
|
result = re.search('issue rrc_eNB_send_PATH_SWITCH_REQ', str(line))
|
|
if result is not None:
|
|
X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ
|
|
if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ:
|
|
result = re.search('received path switch ack S1AP_PATH_SWITCH_REQ_ACK', str(line))
|
|
if result is not None:
|
|
X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
|
|
X2HO_inNbProcedures += 1
|
|
if X2HO_state == CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK:
|
|
result = re.search('source eNB receives the X2 UE CONTEXT RELEASE X2AP_UE_CONTEXT_RELEASE', str(line))
|
|
if result is not None:
|
|
X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
|
|
X2HO_outNbProcedures += 1
|
|
|
|
if self.eNBOptions[int(self.eNB_instance)] != '':
|
|
res1 = re.search('max_rxgain (?P<requested_option>[0-9]+)', self.eNBOptions[int(self.eNB_instance)])
|
|
res2 = re.search('max_rxgain (?P<applied_option>[0-9]+)', str(line))
|
|
if res1 is not None and res2 is not None:
|
|
requested_option = int(res1.group('requested_option'))
|
|
applied_option = int(res2.group('applied_option'))
|
|
if requested_option == applied_option:
|
|
htmleNBFailureMsg += '<span class="glyphicon glyphicon-ok-circle"></span> Command line option(s) correctly applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
|
|
else:
|
|
htmleNBFailureMsg += '<span class="glyphicon glyphicon-ban-circle"></span> Command line option(s) NOT applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[int(self.eNB_instance)] + '\n\n'
|
|
result = re.search('Exiting OAI softmodem', str(line))
|
|
if result is not None:
|
|
exitSignalReceived = True
|
|
result = re.search('[Ss]egmentation [Ff]ault', str(line))
|
|
if result is not None and not exitSignalReceived:
|
|
foundSegFault = True
|
|
result = re.search('[Cc]ore [dD]ump', str(line))
|
|
if result is not None and not exitSignalReceived:
|
|
foundSegFault = True
|
|
result = re.search('./ran_build/build/lte-softmodem', str(line))
|
|
if result is not None and not exitSignalReceived:
|
|
foundSegFault = True
|
|
result = re.search('[Aa]ssertion', str(line))
|
|
if result is not None and not exitSignalReceived:
|
|
foundAssertion = True
|
|
result = re.search('LLL', str(line))
|
|
if result is not None and not exitSignalReceived:
|
|
foundRealTimeIssue = True
|
|
if foundAssertion and (msgLine < 3):
|
|
msgLine += 1
|
|
msgAssertion += str(line)
|
|
result = re.search('Setting function for RU', str(line))
|
|
if result is not None:
|
|
isRRU = True
|
|
if isRRU:
|
|
result = re.search('RU 0 is_slave=yes', str(line))
|
|
if result is not None:
|
|
isSlave = True
|
|
if isSlave:
|
|
result = re.search('Received RRU_frame_resynch command', str(line))
|
|
if result is not None:
|
|
slaveReceivesFrameResyncCmd = True
|
|
result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line))
|
|
if result is not None:
|
|
rrcSetupComplete += 1
|
|
result = re.search('Generate LTE_RRCConnectionRelease|Generate RRCConnectionRelease', str(line))
|
|
if result is not None: rrcReleaseRequest += 1
|
|
result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line))
|
|
if result is not None:
|
|
rrcReconfigRequest += 1
|
|
result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line))
|
|
if result is not None:
|
|
rrcReconfigComplete += 1
|
|
result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line))
|
|
if result is not None:
|
|
rrcReestablishRequest += 1
|
|
result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line))
|
|
if result is not None:
|
|
rrcReestablishComplete += 1
|
|
result = re.search('LTE_RRCConnectionReestablishmentReject', str(line))
|
|
if result is not None:
|
|
rrcReestablishReject += 1
|
|
result = re.search('CDRX configuration activated after RRC Connection', str(line))
|
|
if result is not None:
|
|
cdrxActivationMessageCount += 1
|
|
result = re.search('uci->stat', str(line))
|
|
if result is not None:
|
|
uciStatMsgCount += 1
|
|
result = re.search('PDCP.*Out of Resources.*reason', str(line))
|
|
if result is not None:
|
|
pdcpFailure += 1
|
|
result = re.search('could not wakeup gNB rxtx process', str(line))
|
|
if result is not None:
|
|
gnbRxTxWakeUpFailure += 1
|
|
result = re.search('tx write thread ready', str(line))
|
|
if result is not None:
|
|
gnbTxWriteThreadEnabled = True
|
|
result = re.search('ULSCH in error in round|ULSCH 0 in error', str(line))
|
|
if result is not None:
|
|
ulschFailure += 1
|
|
result = re.search('ERROR ALLOCATING CCEs', str(line))
|
|
if result is not None:
|
|
ulschAllocateCCEerror += 1
|
|
result = re.search('uplink segment error.*aborted [1-9] segments', str(line))
|
|
if result is not None:
|
|
uplinkSegmentsAborted += 1
|
|
result = re.search('ULSCH received ok', str(line))
|
|
if result is not None:
|
|
ulschReceiveOK += 1
|
|
result = re.search('BAD all_segments_received', str(line))
|
|
if result is not None:
|
|
rlcDiscardBuffer += 1
|
|
result = re.search('Canceled RA procedure for UE rnti', str(line))
|
|
if result is not None:
|
|
rachCanceledProcedure += 1
|
|
result = re.search('dropping, not enough RBs', str(line))
|
|
if result is not None:
|
|
dropNotEnoughRBs += 1
|
|
if self.eNBmbmsEnables[int(self.eNB_instance)]:
|
|
result = re.search('MBMS USER-PLANE.*Requesting.*bytes from RLC', str(line))
|
|
if result is not None:
|
|
mbmsRequestMsg += 1
|
|
enb_log_file.close()
|
|
logging.debug(' File analysis completed')
|
|
if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'):
|
|
nodeB_prefix = 'e'
|
|
else:
|
|
nodeB_prefix = 'g'
|
|
if self.air_interface[self.eNB_instance] == 'nr-softmodem':
|
|
if ulschReceiveOK > 0:
|
|
statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)'
|
|
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += statMsg + '\n'
|
|
if gnbRxTxWakeUpFailure > 0:
|
|
statMsg = nodeB_prefix + 'NB showed ' + str(gnbRxTxWakeUpFailure) + ' "could not wakeup gNB rxtx process" message(s)'
|
|
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += statMsg + '\n'
|
|
if gnbTxWriteThreadEnabled:
|
|
statMsg = nodeB_prefix + 'NB ran with TX Write thread enabled'
|
|
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += statMsg + '\n'
|
|
if uciStatMsgCount > 0:
|
|
statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
|
|
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += statMsg + '\n'
|
|
if pdcpFailure > 0:
|
|
statMsg = nodeB_prefix + 'NB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
|
|
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += statMsg + '\n'
|
|
if ulschFailure > 0:
|
|
statMsg = nodeB_prefix + 'NB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
|
|
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += statMsg + '\n'
|
|
if ulschAllocateCCEerror > 0:
|
|
statMsg = nodeB_prefix + 'NB showed ' + str(ulschAllocateCCEerror) + ' "eNB_dlsch_ulsch_scheduler(); ERROR ALLOCATING CCEs" message(s)'
|
|
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += statMsg + '\n'
|
|
if uplinkSegmentsAborted > 0:
|
|
statMsg = nodeB_prefix + 'NB showed ' + str(uplinkSegmentsAborted) + ' "uplink segment error 0/2, aborted * segments" message(s)'
|
|
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += statMsg + '\n'
|
|
if dropNotEnoughRBs > 0:
|
|
statMsg = 'eNB showed ' + str(dropNotEnoughRBs) + ' "dropping, not enough RBs" message(s)'
|
|
logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += statMsg + '\n'
|
|
if rrcSetupComplete > 0:
|
|
rrcMsg = nodeB_prefix + 'NB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
if rrcReleaseRequest > 0:
|
|
rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
|
|
rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0:
|
|
rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
if self.eNBmbmsEnables[int(self.eNB_instance)]:
|
|
if mbmsRequestMsg > 0:
|
|
rrcMsg = 'eNB requested ' + str(mbmsRequestMsg) + ' times the RLC for MBMS USER-PLANE'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
if X2HO_inNbProcedures > 0:
|
|
rrcMsg = 'eNB completed ' + str(X2HO_inNbProcedures) + ' X2 Handover Connection procedure(s)'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
if X2HO_outNbProcedures > 0:
|
|
rrcMsg = 'eNB completed ' + str(X2HO_outNbProcedures) + ' X2 Handover Release procedure(s)'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
if self.eNBOptions[int(self.eNB_instance)] != '':
|
|
res1 = re.search('drx_Config_present prSetup', self.eNBOptions[int(self.eNB_instance)])
|
|
if res1 is not None:
|
|
if cdrxActivationMessageCount > 0:
|
|
rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
|
|
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
else:
|
|
rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration'
|
|
logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rrcMsg + '\n'
|
|
if rachCanceledProcedure > 0:
|
|
rachMsg = nodeB_prefix + 'NB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
|
|
logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rachMsg + '\n'
|
|
if isRRU:
|
|
if isSlave:
|
|
if slaveReceivesFrameResyncCmd:
|
|
rruMsg = 'Slave RRU received the RRU_frame_resynch command from RAU'
|
|
logging.debug('\u001B[1;30;43m ' + rruMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rruMsg + '\n'
|
|
else:
|
|
rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
|
|
logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rruMsg + '\n'
|
|
self.prematureExit = True
|
|
global_status = CONST.ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
|
|
if foundSegFault:
|
|
logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with a Segmentation Fault! \u001B[0m')
|
|
global_status = CONST.ENB_PROCESS_SEG_FAULT
|
|
if foundAssertion:
|
|
logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with an assertion! \u001B[0m')
|
|
htmleNBFailureMsg += msgAssertion
|
|
global_status = CONST.ENB_PROCESS_ASSERTION
|
|
if foundRealTimeIssue:
|
|
logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB faced real time issues! \u001B[0m')
|
|
htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues!\n'
|
|
if rlcDiscardBuffer > 0:
|
|
rlcMsg = nodeB_prefix + 'NB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
|
|
logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m')
|
|
htmleNBFailureMsg += rlcMsg + '\n'
|
|
global_status = CONST.ENB_PROCESS_REALTIME_ISSUE
|
|
if self.htmlObj is not None:
|
|
self.htmlObj.htmleNBFailureMsg=htmleNBFailureMsg
|
|
# Runtime statistics for console output and HTML
|
|
if runTime != '':
|
|
logging.debug(runTime)
|
|
logging.debug(userTime)
|
|
logging.debug(systemTime)
|
|
logging.debug(maxPhyMemUsage)
|
|
logging.debug(nbContextSwitches)
|
|
self.runtime_stats='<pre>'+runTime + '\n'+ userTime + '\n' + systemTime + '\n' + maxPhyMemUsage + '\n' + nbContextSwitches+'</pre>'
|
|
return global_status
|