From c2b51255d66f9eff8fc76a572f329c4418d1c24b Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Fri, 8 Mar 2019 09:42:33 +0100 Subject: [PATCH 01/26] WIP for adding OAI UE Testing. Signed-off-by: Boris Djalal --- ci-scripts/main.py | 190 ++++++++++++++++++ ci-scripts/xml_files/ue_band20_test_10mhz.xml | 32 +++ 2 files changed, 222 insertions(+) create mode 100644 ci-scripts/xml_files/ue_band20_test_10mhz.xml diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 5e89384ebb..0a63795bc4 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -127,6 +127,22 @@ class SSHConnection(): self.eNBCpuModel = '' self.eNBCpuMHz = '' + #self.UEOsVersion = '' + #self.UEKernelVersion = '' + #self.UEUhdVersion = '' + #self.UECpuNb = '' + #self.UECpuModel = '' + self.UEIPAddress = '' + self.UERepository = '' + self.UEBranch = '' + #self.UE_AllowMerge = False + self.UECommitID = '' + #self.UETargetBranch = '' + self.UEUserName = '' + self.UEPassword = '' + #self.UESourceCodePath = '' + #self.UECpuMHz = '' + def open(self, ipaddress, username, password): count = 0 connect_status = False @@ -326,6 +342,12 @@ class SSHConnection(): self.close() self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK) + def BuildOAIUE(self): + if self.UEIPAddress == '' or self.UERepository == '' or self.UEBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': + Usage() + sys.exit('Insufficient Parameter') + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + def InitializeHSS(self): if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '': Usage() @@ -517,6 +539,108 @@ class SSHConnection(): job.join() self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) + def InitializeOAIUE(self): + if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': + Usage() + sys.exit('Insufficient Parameter') + initialize_UE_flag = True + pStatus = self.CheckProcessExist(initialize_UE_flag) + if (pStatus < 0): + self.CreateHtmlTestRow(self.Initialize_UE_args, 'KO', pStatus) + self.CreateHtmlTabFooter(False) + sys.exit(1) + # If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB + result = re.search('T_stdout', str(self.Initialize_eNB_args)) + if result is not None: + self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) + self.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5) + result = re.search('interfaceToUse=(?P[a-zA-Z0-9\-\_]+)done', str(self.ssh.before)) + if result is not None: + eth_interface = result.group('eth_interface') + logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m') + self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f /tmp/enb_' + self.testCase_id + '_s1log.pcap', '\$', 5) + self.command('echo $USER; nohup sudo tshark -f "host ' + self.eNBIPAddress +'" -i ' + eth_interface + ' -w /tmp/enb_' + self.testCase_id + '_s1log.pcap > /tmp/tshark.log 2>&1 &', self.EPCUserName, 5) + self.close() + self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) + self.command('cd ' + self.eNBSourceCodePath, '\$', 5) + # Initialize_eNB_args usually start with -O and followed by the location in repository + full_config_file = self.Initialize_eNB_args.replace('-O ','') + extIdx = full_config_file.find('.conf') + if (extIdx > 0): + extra_options = full_config_file[extIdx + 5:] + # if tracer options is on, compiling and running T Tracer + result = re.search('T_stdout', str(extra_options)) + if result is not None: + logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m') + self.command('cd common/utils/T/tracer', '\$', 5) + self.command('make', '\$', 10) + self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', self.eNBUserName, 5) + self.command('cd ' + self.eNBSourceCodePath, '\$', 5) + full_config_file = full_config_file[:extIdx + 5] + config_path, config_file = os.path.split(full_config_file) + else: + sys.exit('Insufficient Parameter') + ci_full_config_file = config_path + '/ci-' + config_file + rruCheck = False + result = re.search('rru', str(config_file)) + if result is not None: + rruCheck = True + # Make a copy and adapt to EPC / eNB IP addresses + self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5) + self.command('sed -i -e \'s/CI_MME_IP_ADDR/' + self.EPCIPAddress + '/\' ' + ci_full_config_file, '\$', 2); + self.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2); + # Launch eNB with the modified config file + self.command('source oaienv', '\$', 5) + self.command('cd cmake_targets', '\$', 5) + self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.eNBSourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh ', '\$', 5) + self.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh ', '\$', 5) + self.command('echo ' + self.eNBPassword + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5) + self.command('echo ' + self.eNBPassword + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + self.eNBSourceCodePath + '/cmake_targets -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + if not rruCheck: + self.eNBLogFile = 'enb_' + self.testCase_id + '.log' + time.sleep(6) + doLoop = True + loopCounter = 10 + while (doLoop): + loopCounter = loopCounter - 1 + if (loopCounter == 0): + # In case of T tracer recording, we may need to kill it + result = re.search('T_stdout', str(self.Initialize_eNB_args)) + if result is not None: + self.command('killall --signal SIGKILL record', '\$', 5) + self.close() + doLoop = False + logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m') + self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK) + self.CreateHtmlTabFooter(False) + # In case of T tracer recording, we need to kill tshark on EPC side + result = re.search('T_stdout', str(self.Initialize_eNB_args)) + if result is not None: + self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) + logging.debug('\u001B[1m Stopping tshark \u001B[0m') + self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) + self.close() + time.sleep(1) + pcap_log_file = 'enb_' + self.testCase_id + '_s1log.pcap' + copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + pcap_log_file, '.') + if (copyin_res == 0): + self.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, pcap_log_file, self.eNBSourceCodePath + '/cmake_targets/.') + sys.exit(1) + else: + self.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4) + if rruCheck: + result = re.search('wait RUs', str(self.ssh.before)) + else: + result = re.search('got sync', str(self.ssh.before)) + if result is None: + time.sleep(6) + else: + doLoop = False + self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'OK', ALL_PROCESSES_OK) + logging.debug('\u001B[1m Initialize eNB Completed\u001B[0m') + + self.close() + def checkDevTTYisUnlocked(self): self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword) count = 0 @@ -2467,6 +2591,16 @@ while len(argvs) > 1: matchReg = re.match('^\-\-XMLTestFile=(.+)$', myArgv, re.IGNORECASE) SSH.testXMLfiles.append(matchReg.group(1)) SSH.nbTestXMLfiles += 1 + elif re.match('^\-\-UEIPAddress=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UEIPAddress=(.+)$', myArgv, re.IGNORECASE) + SSH.testXMLfiles.append(matchReg.group(1)) + SSH.nbTestXMLfiles += 1 + elif re.match('^\-\-UEUserName=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UEUserName=(.+)$', myArgv, re.IGNORECASE) + SSH.UEUserName = matchReg.group(1) + elif re.match('^\-\-UEPassword=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UEPassword=(.+)$', myArgv, re.IGNORECASE) + SSH.UEPassword = matchReg.group(1) elif re.match('^\-\-finalStatus=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-finalStatus=(.+)$', myArgv, re.IGNORECASE) finalStatus = matchReg.group(1) @@ -2675,6 +2809,62 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE): sys.exit('Invalid action') SSH.CreateHtmlTabFooter(True) +elif re.match('^TestUE$', mode, re.IGNORECASE): + if SSH.UEIPAddress == '' or SSH.UERepository == '' or SSH.UEBranch == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '': + Usage() + sys.exit('Insufficient Parameter') + #read test_case_list.xml file + # if no parameters for XML file, use default value + if (SSH.nbTestXMLfiles != 1): + xml_test_file = sys.path[0] + "/test_case_list.xml" + else: + xml_test_file = sys.path[0] + "/" + SSH.testXMLfiles[0] + + xmlTree = ET.parse(xml_test_file) + xmlRoot = xmlTree.getroot() + + exclusion_tests=xmlRoot.findtext('TestCaseExclusionList',default='') + requested_tests=xmlRoot.findtext('TestCaseRequestedList',default='') + if (SSH.nbTestXMLfiles == 1): + SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-0')) + all_tests=xmlRoot.findall('testCase') + + exclusion_tests=exclusion_tests.split() + requested_tests=requested_tests.split() + + #check that exclusion tests are well formatted + #(6 digits or less than 6 digits followed by +) + for test in exclusion_tests: + if (not re.match('^[0-9]{6}$', test) and + not re.match('^[0-9]{1,5}\+$', test)): + logging.debug('ERROR: exclusion test is invalidly formatted: ' + test) + sys.exit(1) + else: + logging.debug(test) + + #check that requested tests are well formatted + #(6 digits or less than 6 digits followed by +) + #be verbose + for test in requested_tests: + if (re.match('^[0-9]{6}$', test) or + re.match('^[0-9]{1,5}\+$', test)): + logging.debug('INFO: test group/case requested: ' + test) + else: + logging.debug('ERROR: requested test is invalidly formatted: ' + test) + sys.exit(1) + + #get the list of tests to be done + todo_tests=[] + for test in requested_tests: + if (test_in_list(test, exclusion_tests)): + logging.debug('INFO: test will be skipped: ' + test) + else: + #logging.debug('INFO: test will be run: ' + test) + todo_tests.append(test) + + signal.signal(signal.SIGUSR1, receive_signal) + + SSH.CreateHtmlTabHeader() else: Usage() sys.exit('Invalid mode') diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz.xml b/ci-scripts/xml_files/ue_band20_test_10mhz.xml new file mode 100644 index 0000000000..7841664246 --- /dev/null +++ b/ci-scripts/xml_files/ue_band20_test_10mhz.xml @@ -0,0 +1,32 @@ + + + 010101 + + + IdleSleep + Sleep OAI UE + 1 + + + From a22102bf3453ab9f854c068ea3f0996c02a8f1e1 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Mon, 11 Mar 2019 09:37:28 +0100 Subject: [PATCH 02/26] Temporary hack to make main.py command work with the new xml file, ue_band20_test_10mhz.xml. Signed-off-by: Boris Djalal --- ci-scripts/main.py | 259 ++++++++++-------- ci-scripts/xml_files/ue_band20_test_10mhz.xml | 4 +- 2 files changed, 154 insertions(+), 109 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 0a63795bc4..355d75cd1d 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -126,7 +126,6 @@ class SSHConnection(): self.eNBCpuNb = '' self.eNBCpuModel = '' self.eNBCpuMHz = '' - #self.UEOsVersion = '' #self.UEKernelVersion = '' #self.UEUhdVersion = '' @@ -142,6 +141,9 @@ class SSHConnection(): self.UEPassword = '' #self.UESourceCodePath = '' #self.UECpuMHz = '' + self.Build_eNB_args = '' + self.Initialize_eNB_args = '' + def open(self, ipaddress, username, password): count = 0 @@ -347,6 +349,37 @@ class SSHConnection(): Usage() sys.exit('Insufficient Parameter') self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('mkdir -p ' + self.UESourceCodePath, '\$', 5) + self.command('cd ' + self.UESourceCodePath, '\$', 5) + self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.UERepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600) + # here add a check if git clone or git fetch went smoothly + self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) + self.command('git config user.name "OAI Jenkins"', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S git clean -x -d -ff', '\$', 30) + # if the commit ID is provided use it to point to it + if self.UECommitID != '': + self.command('git checkout -f ' + self.UECommitID, '\$', 5) + # if the branch is not develop, then it is a merge request and we need to do + # the potential merge. Note that merge conflicts should already been checked earlier + if (self.UE_AllowMerge): + if self.UETargetBranch == '': + if (self.UEBranch != 'develop') and (self.UEBranch != 'origin/develop'): + self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + else: + logging.debug('Merging with the target branch: ' + self.UETargetBranch) + self.command('git merge --ff origin/' + self.UETargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + self.command('source oaienv', '\$', 5) + self.command('cd cmake_targets', '\$', 5) + self.command('mkdir -p log', '\$', 5) + self.command('chmod 777 log', '\$', 5) + # no need to remove in log (git clean did the trick) + self.command('stdbuf -o0 ./build_oai ' + self.Build_UE_args + ' 2>&1 | stdbuf -o0 tee -a compile_oai_ue.log', 'Bypassing the Tests', 600) + self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) + self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) + self.command('mv compile_oai_ue.log ' + 'build_log_' + self.testCase_id, '\$', 5) + self.close() + self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK) + def InitializeHSS(self): if self.EPCIPAddress == '' or self.EPCUserName == '' or self.EPCPassword == '' or self.EPCSourceCodePath == '' or self.EPCType == '': @@ -528,6 +561,7 @@ class SSHConnection(): def InitializeUE(self): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': Usage() + print('InitializeUE') sys.exit('Insufficient Parameter') multi_jobs = [] for device_id in self.UEDevices: @@ -542,6 +576,7 @@ class SSHConnection(): def InitializeOAIUE(self): if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': Usage() + print('DEBUG InitializeOAIUE 1') sys.exit('Insufficient Parameter') initialize_UE_flag = True pStatus = self.CheckProcessExist(initialize_UE_flag) @@ -549,22 +584,10 @@ class SSHConnection(): self.CreateHtmlTestRow(self.Initialize_UE_args, 'KO', pStatus) self.CreateHtmlTabFooter(False) sys.exit(1) - # If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB - result = re.search('T_stdout', str(self.Initialize_eNB_args)) - if result is not None: - self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) - self.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5) - result = re.search('interfaceToUse=(?P[a-zA-Z0-9\-\_]+)done', str(self.ssh.before)) - if result is not None: - eth_interface = result.group('eth_interface') - logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m') - self.command('echo ' + self.EPCPassword + ' | sudo -S rm -f /tmp/enb_' + self.testCase_id + '_s1log.pcap', '\$', 5) - self.command('echo $USER; nohup sudo tshark -f "host ' + self.eNBIPAddress +'" -i ' + eth_interface + ' -w /tmp/enb_' + self.testCase_id + '_s1log.pcap > /tmp/tshark.log 2>&1 &', self.EPCUserName, 5) - self.close() - self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) - self.command('cd ' + self.eNBSourceCodePath, '\$', 5) - # Initialize_eNB_args usually start with -O and followed by the location in repository - full_config_file = self.Initialize_eNB_args.replace('-O ','') + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('cd ' + self.UESourceCodePath, '\$', 5) + # Initialize_UE_args usually start with -O and followed by the location in repository + full_config_file = self.Initialize_UE_args.replace('-O ','') extIdx = full_config_file.find('.conf') if (extIdx > 0): extra_options = full_config_file[extIdx + 5:] @@ -574,30 +597,30 @@ class SSHConnection(): logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m') self.command('cd common/utils/T/tracer', '\$', 5) self.command('make', '\$', 10) - self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', self.eNBUserName, 5) - self.command('cd ' + self.eNBSourceCodePath, '\$', 5) + self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.UESourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.UESourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', self.UEUserName, 5) + self.command('cd ' + self.UESourceCodePath, '\$', 5) full_config_file = full_config_file[:extIdx + 5] config_path, config_file = os.path.split(full_config_file) else: + print('DEBUG InitializeOAIUE 2') sys.exit('Insufficient Parameter') ci_full_config_file = config_path + '/ci-' + config_file rruCheck = False result = re.search('rru', str(config_file)) if result is not None: rruCheck = True - # Make a copy and adapt to EPC / eNB IP addresses + # Make a copy and adapt to EPC / UE IP addresses self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5) - self.command('sed -i -e \'s/CI_MME_IP_ADDR/' + self.EPCIPAddress + '/\' ' + ci_full_config_file, '\$', 2); - self.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2); - # Launch eNB with the modified config file + self.command('sed -i -e \'s/CI_UE_IP_ADDR/' + self.UEIPAddress + '/\' ' + ci_full_config_file, '\$', 2); + # Launch UE with the modified config file self.command('source oaienv', '\$', 5) self.command('cd cmake_targets', '\$', 5) - self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.eNBSourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh ', '\$', 5) - self.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh ', '\$', 5) - self.command('echo ' + self.eNBPassword + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5) - self.command('echo ' + self.eNBPassword + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + self.eNBSourceCodePath + '/cmake_targets -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.UESourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.UE_instance) + '.sh ', '\$', 5) + self.command('chmod 775 ./my-lte-softmodem-run' + str(self.UE_instance) + '.sh ', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf ue_' + self.testCase_id + '.log', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S -E daemon --inherit --unsafe --name=ue' + str(self.UE_instance) + '_daemon --chdir=' + self.UESourceCodePath + '/cmake_targets -o ' + self.UESourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) if not rruCheck: - self.eNBLogFile = 'enb_' + self.testCase_id + '.log' + self.UELogFile = 'enb_' + self.testCase_id + '.log' time.sleep(6) doLoop = True loopCounter = 10 @@ -605,16 +628,16 @@ class SSHConnection(): loopCounter = loopCounter - 1 if (loopCounter == 0): # In case of T tracer recording, we may need to kill it - result = re.search('T_stdout', str(self.Initialize_eNB_args)) + result = re.search('T_stdout', str(self.Initialize_UE_args)) if result is not None: self.command('killall --signal SIGKILL record', '\$', 5) self.close() doLoop = False - logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m') + logging.error('\u001B[1;37;41m UE logging system did not show got sync! \u001B[0m') self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK) self.CreateHtmlTabFooter(False) # In case of T tracer recording, we need to kill tshark on EPC side - result = re.search('T_stdout', str(self.Initialize_eNB_args)) + result = re.search('T_stdout', str(self.Initialize_UE_args)) if result is not None: self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) logging.debug('\u001B[1m Stopping tshark \u001B[0m') @@ -624,7 +647,7 @@ class SSHConnection(): pcap_log_file = 'enb_' + self.testCase_id + '_s1log.pcap' copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + pcap_log_file, '.') if (copyin_res == 0): - self.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, pcap_log_file, self.eNBSourceCodePath + '/cmake_targets/.') + self.copyout(self.UEIPAddress, self.UEUserName, self.UEPassword, pcap_log_file, self.UESourceCodePath + '/cmake_targets/.') sys.exit(1) else: self.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4) @@ -637,7 +660,7 @@ class SSHConnection(): else: doLoop = False self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'OK', ALL_PROCESSES_OK) - logging.debug('\u001B[1m Initialize eNB Completed\u001B[0m') + logging.debug('\u001B[1m Initialize UE Completed\u001B[0m') self.close() @@ -1998,6 +2021,69 @@ class SSHConnection(): job.join() self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) + def TerminateOAIUE(self): + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S daemon --name=ue' + str(self.UE_instance) + '_daemon --stop', '\$', 5) + self.command('rm -f my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGINT lte-uesoftmodem || true', '\$', 5) + time.sleep(5) + self.command('stdbuf -o0 ps -aux | grep -v grep | grep lte-uesoftmodem', '\$', 5) + result = re.search('lte-uesoftmodem', str(self.ssh.before)) + if result is not None: + self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGKILL lte-uesoftmodem || true', '\$', 5) + time.sleep(5) + self.close() + # If tracer options is on, stopping tshark on EPC side + result = re.search('T_stdout', str(self.Initialize_UE_args)) + if result is not None: + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + logging.debug('\u001B[1m Stopping tshark \u001B[0m') + self.command('echo ' + self.UEPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) + time.sleep(1) + pcap_log_file = self.UELogFile.replace('.log', '_s1log.pcap') + self.command('echo ' + self.UEPassword + ' | sudo -S chmod 666 /tmp/' + pcap_log_file, '\$', 5) + self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, '/tmp/' + pcap_log_file, '.') + self.copyout(self.UEIPAddress, self.UEUserName, self.UEPassword, pcap_log_file, self.UESourceCodePath + '/cmake_targets/.') + self.close() + logging.debug('\u001B[1m Replaying RAW record file\u001B[0m') + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('cd ' + self.UESourceCodePath + '/common/utils/T/tracer/', '\$', 5) + raw_record_file = self.UELogFile.replace('.log', '_record.raw') + replay_log_file = self.UELogFile.replace('.log', '_replay.log') + extracted_txt_file = self.UELogFile.replace('.log', '_extracted_messages.txt') + extracted_log_file = self.UELogFile.replace('.log', '_extracted_messages.log') + self.command('./extract_config -i ' + self.UESourceCodePath + '/cmake_targets/' + raw_record_file + ' > ' + self.UESourceCodePath + '/cmake_targets/' + extracted_txt_file, '\$', 5) + self.command('echo $USER; nohup ./replay -i ' + self.UESourceCodePath + '/cmake_targets/' + raw_record_file + ' > ' + self.UESourceCodePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', self.UEUserName, 5) + self.command('./textlog -d ' + self.UESourceCodePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + self.UESourceCodePath + '/cmake_targets/' + extracted_log_file, '\$', 5) + self.close() + self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + extracted_log_file, '.') + logging.debug('\u001B[1m Analyzing UE replay logfile \u001B[0m') + logStatus = self.AnalyzeLogFile_UE(extracted_log_file) + self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) + self.UELogFile = '' + else: + result = re.search('enb_', str(self.UELogFile)) + if result is not None: + copyin_res = self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.') + if (copyin_res == -1): + logging.debug('\u001B[1;37;41m Could not copy UE logfile to analyze it! \u001B[0m') + self.htmlUEFailureMsg = 'Could not copy UE logfile to analyze it!' + self.CreateHtmlTestRow('N/A', 'KO', ENB_PROCESS_NOLOGFILE_TO_ANALYZE) + self.UELogFile = '' + return + logging.debug('\u001B[1m Analyzing UE logfile \u001B[0m') + logStatus = self.AnalyzeLogFile_UE(self.UELogFile) + if (logStatus < 0): + self.CreateHtmlTestRow('N/A', 'KO', logStatus) + self.CreateHtmlTabFooter(False) + sys.exit(1) + else: + self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) + self.UELogFile = '' + else: + self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) + def AutoTerminateUEandeNB(self): self.testCase_id = 'AUTO-KILL-UE' self.desc = 'Automatic Termination of UE' @@ -2213,10 +2299,6 @@ class SSHConnection(): if (self.ADBIPAddress != 'none'): self.GetAllUEDevices(terminate_ue_flag) self.GetAllCatMDevices(terminate_ue_flag) - else: - self.UEDevices.append('doughq9rehg') - self.UEDevices.append('dnsgiuahgia') - self.UEDevices.append('uehgieng9') self.htmlUEConnected = len(self.UEDevices) self.htmlFile.write('

' + str(len(self.UEDevices)) + ' UE(s) is(are) connected to ADB bench server

\n') @@ -2259,10 +2341,6 @@ class SSHConnection(): if (self.ADBIPAddress != 'none'): self.GetAllUEDevices(terminate_ue_flag) self.GetAllCatMDevices(terminate_ue_flag) - else: - self.UEDevices.append('doughq9rehg') - self.UEDevices.append('dnsgiuahgia') - self.UEDevices.append('uehgieng9') self.htmlUEConnected = len(self.UEDevices) i = 0 @@ -2593,14 +2671,25 @@ while len(argvs) > 1: SSH.nbTestXMLfiles += 1 elif re.match('^\-\-UEIPAddress=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-UEIPAddress=(.+)$', myArgv, re.IGNORECASE) - SSH.testXMLfiles.append(matchReg.group(1)) - SSH.nbTestXMLfiles += 1 + SSH.UEIPAddress = matchReg.group(1) + elif re.match('^\-\-UERepository=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UERepository=(.+)$', myArgv, re.IGNORECASE) + SSH.UERepository = matchReg.group(1) elif re.match('^\-\-UEUserName=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-UEUserName=(.+)$', myArgv, re.IGNORECASE) SSH.UEUserName = matchReg.group(1) elif re.match('^\-\-UEPassword=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-UEPassword=(.+)$', myArgv, re.IGNORECASE) SSH.UEPassword = matchReg.group(1) + elif re.match('^\-\-UESourceCodePath=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UESourceCodePath=(.+)$', myArgv, re.IGNORECASE) + SSH.UESourceCodePath = matchReg.group(1) + elif re.match('^\-\-UEBranch=(.+)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UEBranch=(.+)$', myArgv, re.IGNORECASE) + SSH.UEBranch = matchReg.group(1) + elif re.match('^\-\-UECommitID=(.*)$', myArgv, re.IGNORECASE): + matchReg = re.match('^\-\-UECommitID=(.*)$', myArgv, re.IGNORECASE) + SSH.UECommitID = matchReg.group(1) elif re.match('^\-\-finalStatus=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-finalStatus=(.+)$', myArgv, re.IGNORECASE) finalStatus = matchReg.group(1) @@ -2616,7 +2705,7 @@ if re.match('^TerminateeNB$', mode, re.IGNORECASE): sys.exit('Insufficient Parameter') SSH.TerminateeNB() elif re.match('^TerminateUE$', mode, re.IGNORECASE): - if SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '': + if SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '': Usage() sys.exit('Insufficient Parameter') signal.signal(signal.SIGUSR1, receive_signal) @@ -2687,14 +2776,26 @@ elif re.match('^InitiateHtml$', mode, re.IGNORECASE): SSH.CreateHtmlHeader() elif re.match('^FinalizeHtml$', mode, re.IGNORECASE): SSH.CreateHtmlFooter(SSH.finalStatus) -elif re.match('^TesteNB$', mode, re.IGNORECASE): - if SSH.eNBIPAddress == '' or SSH.eNBRepository == '' or SSH.eNBBranch == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '' or SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '' or SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '': - Usage() - sys.exit('Insufficient Parameter') +elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re.IGNORECASE): + if re.match('^TesteNB$', mode, re.IGNORECASE): + if SSH.eNBIPAddress == '' or SSH.eNBRepository == '' or SSH.eNBBranch == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '' or SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '' or SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '': + Usage() + sys.exit('Insufficient Parameter') + + if (SSH.EPCIPAddress != 'none'): + SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/tcp_iperf_stats.awk", "/tmp") + SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/active_net_interfaces.awk", "/tmp") + else: + print('SSH.UEIPAddress=' + SSH.UEIPAddress) + print('SSH.UERepository=' + SSH.UERepository) + print('SSH.UEBranch=' + SSH.UEBranch) + print('SSH.UEUserName=' + SSH.UEUserName) + print('SSH.UEPassword=' + SSH.UEPassword) + print('SSH.UESourceCodePath=' + SSH.UESourceCodePath) + if SSH.UEIPAddress == '' or SSH.UERepository == '' or SSH.UEBranch == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '': + Usage() + sys.exit('UE: Insufficient Parameter') - if (SSH.EPCIPAddress != 'none'): - SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/tcp_iperf_stats.awk", "/tmp") - SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/active_net_interfaces.awk", "/tmp") #read test_case_list.xml file # if no parameters for XML file, use default value if (SSH.nbTestXMLfiles != 1): @@ -2809,62 +2910,6 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE): sys.exit('Invalid action') SSH.CreateHtmlTabFooter(True) -elif re.match('^TestUE$', mode, re.IGNORECASE): - if SSH.UEIPAddress == '' or SSH.UERepository == '' or SSH.UEBranch == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '': - Usage() - sys.exit('Insufficient Parameter') - #read test_case_list.xml file - # if no parameters for XML file, use default value - if (SSH.nbTestXMLfiles != 1): - xml_test_file = sys.path[0] + "/test_case_list.xml" - else: - xml_test_file = sys.path[0] + "/" + SSH.testXMLfiles[0] - - xmlTree = ET.parse(xml_test_file) - xmlRoot = xmlTree.getroot() - - exclusion_tests=xmlRoot.findtext('TestCaseExclusionList',default='') - requested_tests=xmlRoot.findtext('TestCaseRequestedList',default='') - if (SSH.nbTestXMLfiles == 1): - SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-0')) - all_tests=xmlRoot.findall('testCase') - - exclusion_tests=exclusion_tests.split() - requested_tests=requested_tests.split() - - #check that exclusion tests are well formatted - #(6 digits or less than 6 digits followed by +) - for test in exclusion_tests: - if (not re.match('^[0-9]{6}$', test) and - not re.match('^[0-9]{1,5}\+$', test)): - logging.debug('ERROR: exclusion test is invalidly formatted: ' + test) - sys.exit(1) - else: - logging.debug(test) - - #check that requested tests are well formatted - #(6 digits or less than 6 digits followed by +) - #be verbose - for test in requested_tests: - if (re.match('^[0-9]{6}$', test) or - re.match('^[0-9]{1,5}\+$', test)): - logging.debug('INFO: test group/case requested: ' + test) - else: - logging.debug('ERROR: requested test is invalidly formatted: ' + test) - sys.exit(1) - - #get the list of tests to be done - todo_tests=[] - for test in requested_tests: - if (test_in_list(test, exclusion_tests)): - logging.debug('INFO: test will be skipped: ' + test) - else: - #logging.debug('INFO: test will be run: ' + test) - todo_tests.append(test) - - signal.signal(signal.SIGUSR1, receive_signal) - - SSH.CreateHtmlTabHeader() else: Usage() sys.exit('Invalid mode') diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz.xml b/ci-scripts/xml_files/ue_band20_test_10mhz.xml index 7841664246..72b6a81ec1 100644 --- a/ci-scripts/xml_files/ue_band20_test_10mhz.xml +++ b/ci-scripts/xml_files/ue_band20_test_10mhz.xml @@ -21,9 +21,9 @@ --> - 010101 + 080101 - + IdleSleep Sleep OAI UE 1 From df49d8a90845c52a35ed9317d5b065cff1fffb72 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Mon, 11 Mar 2019 15:00:36 +0100 Subject: [PATCH 03/26] No more 'none' value. Successfully writing the log file ue_080101.log and finding the sync marker. Initialize OAI UE is working. BuildOAIUE action seems ok. Must fix the log file path for the InitializeOAIUE action. Doing the work in /tmp, not in the repository of the main.py script being executed. Signed-off-by: Boris Djalal --- ci-scripts/main.py | 406 +++++++++++++----- ci-scripts/xml_files/ue_band20_test_10mhz.xml | 35 +- ci-scripts/xml_files/ue_sfr_796.xml | 44 ++ 3 files changed, 378 insertions(+), 107 deletions(-) create mode 100644 ci-scripts/xml_files/ue_sfr_796.xml diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 355d75cd1d..5e113dd413 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -1,4 +1,3 @@ -#/* # * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more # * contributor license agreements. See the NOTICE file distributed with # * this work for additional information regarding copyright ownership. @@ -132,17 +131,17 @@ class SSHConnection(): #self.UECpuNb = '' #self.UECpuModel = '' self.UEIPAddress = '' - self.UERepository = '' self.UEBranch = '' #self.UE_AllowMerge = False self.UECommitID = '' #self.UETargetBranch = '' self.UEUserName = '' self.UEPassword = '' + self.UE_instance = '' #self.UESourceCodePath = '' #self.UECpuMHz = '' - self.Build_eNB_args = '' - self.Initialize_eNB_args = '' + self.Build_OAI_UE_args = '' + self.Initialize_OAI_UE_args = '' def open(self, ipaddress, username, password): @@ -345,40 +344,41 @@ class SSHConnection(): self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK) def BuildOAIUE(self): - if self.UEIPAddress == '' or self.UERepository == '' or self.UEBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': + return + if self.UEIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': Usage() sys.exit('Insufficient Parameter') self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) self.command('mkdir -p ' + self.UESourceCodePath, '\$', 5) self.command('cd ' + self.UESourceCodePath, '\$', 5) - self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.UERepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600) + self.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + self.eNBRepository + ' .; else stdbuf -o0 git fetch; fi', '\$', 600) # here add a check if git clone or git fetch went smoothly self.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) self.command('git config user.name "OAI Jenkins"', '\$', 5) self.command('echo ' + self.UEPassword + ' | sudo -S git clean -x -d -ff', '\$', 30) # if the commit ID is provided use it to point to it - if self.UECommitID != '': - self.command('git checkout -f ' + self.UECommitID, '\$', 5) + if self.eNBCommitID != '': + self.command('git checkout -f ' + self.eNBCommitID, '\$', 5) # if the branch is not develop, then it is a merge request and we need to do # the potential merge. Note that merge conflicts should already been checked earlier - if (self.UE_AllowMerge): - if self.UETargetBranch == '': - if (self.UEBranch != 'develop') and (self.UEBranch != 'origin/develop'): + if (self.eNB_AllowMerge): + if self.eNBTargetBranch == '': + if (self.eNBBranch != 'develop') and (self.eNBBranch != 'origin/develop'): self.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) else: - logging.debug('Merging with the target branch: ' + self.UETargetBranch) - self.command('git merge --ff origin/' + self.UETargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + logging.debug('Merging with the target branch: ' + self.eNBTargetBranch) + self.command('git merge --ff origin/' + self.eNBTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) self.command('source oaienv', '\$', 5) self.command('cd cmake_targets', '\$', 5) self.command('mkdir -p log', '\$', 5) self.command('chmod 777 log', '\$', 5) # no need to remove in log (git clean did the trick) - self.command('stdbuf -o0 ./build_oai ' + self.Build_UE_args + ' 2>&1 | stdbuf -o0 tee -a compile_oai_ue.log', 'Bypassing the Tests', 600) + self.command('stdbuf -o0 ./build_oai ' + self.Build_OAI_UE_args + ' 2>&1 | stdbuf -o0 tee -a compile_oai_ue.log', 'Bypassing the Tests', 600) self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) self.command('mv compile_oai_ue.log ' + 'build_log_' + self.testCase_id, '\$', 5) self.close() - self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK) + self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', ALL_PROCESSES_OK) def InitializeHSS(self): @@ -493,8 +493,8 @@ class SSHConnection(): # Launch eNB with the modified config file self.command('source oaienv', '\$', 5) self.command('cd cmake_targets', '\$', 5) - self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.eNBSourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh ', '\$', 5) - self.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh ', '\$', 5) + self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.eNBSourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + self.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) self.command('echo ' + self.eNBPassword + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5) self.command('echo ' + self.eNBPassword + ' | sudo -S -E daemon --inherit --unsafe --name=enb' + str(self.eNB_instance) + '_daemon --chdir=' + self.eNBSourceCodePath + '/cmake_targets -o ' + self.eNBSourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) if not rruCheck: @@ -561,7 +561,6 @@ class SSHConnection(): def InitializeUE(self): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': Usage() - print('InitializeUE') sys.exit('Insufficient Parameter') multi_jobs = [] for device_id in self.UEDevices: @@ -576,92 +575,84 @@ class SSHConnection(): def InitializeOAIUE(self): if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': Usage() - print('DEBUG InitializeOAIUE 1') sys.exit('Insufficient Parameter') - initialize_UE_flag = True - pStatus = self.CheckProcessExist(initialize_UE_flag) - if (pStatus < 0): - self.CreateHtmlTestRow(self.Initialize_UE_args, 'KO', pStatus) - self.CreateHtmlTabFooter(False) - sys.exit(1) + #initialize_OAI_UE_flag = True + #pStatus = self.CheckOAIUEProcessExist(initialize_OAI_UE_flag) + #if (pStatus < 0): + # self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', pStatus) + # self.CreateHtmlTabFooter(False) + # sys.exit(1) self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) self.command('cd ' + self.UESourceCodePath, '\$', 5) - # Initialize_UE_args usually start with -O and followed by the location in repository - full_config_file = self.Initialize_UE_args.replace('-O ','') - extIdx = full_config_file.find('.conf') - if (extIdx > 0): - extra_options = full_config_file[extIdx + 5:] - # if tracer options is on, compiling and running T Tracer - result = re.search('T_stdout', str(extra_options)) - if result is not None: - logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m') - self.command('cd common/utils/T/tracer', '\$', 5) - self.command('make', '\$', 10) - self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.UESourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.UESourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', self.UEUserName, 5) - self.command('cd ' + self.UESourceCodePath, '\$', 5) - full_config_file = full_config_file[:extIdx + 5] - config_path, config_file = os.path.split(full_config_file) - else: - print('DEBUG InitializeOAIUE 2') - sys.exit('Insufficient Parameter') - ci_full_config_file = config_path + '/ci-' + config_file - rruCheck = False - result = re.search('rru', str(config_file)) - if result is not None: - rruCheck = True - # Make a copy and adapt to EPC / UE IP addresses - self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5) - self.command('sed -i -e \'s/CI_UE_IP_ADDR/' + self.UEIPAddress + '/\' ' + ci_full_config_file, '\$', 2); + # Initialize_OAI_UE_args usually start with -C and followed by the location in repository + #full_config_file = self.Initialize_OAI_UE_args.replace('-O ','') + #extIdx = full_config_file.find('.conf') + #if (extIdx > 0): + # extra_options = full_config_file[extIdx + 5:] + # # if tracer options is on, compiling and running T Tracer + # result = re.search('T_stdout', str(extra_options)) + ## if result is not None: + # logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m') + # self.command('cd common/utils/T/tracer', '\$', 5) + # self.command('make', '\$', 10) + # self.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + self.UESourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', self.UEUserName, 5) + # self.command('cd ' + self.UESourceCodePath, '\$', 5) + # full_config_file = full_config_file[:extIdx + 5] + # config_path, config_file = os.path.split(full_config_file) + #ci_full_config_file = config_path + '/ci-' + config_file + #rruCheck = False + #result = re.search('rru', str(config_file)) + #if result is not None: + # rruCheck = True + ## Make a copy and adapt to EPC / UE IP addresses + #self.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5) + #self.command('sed -i -e \'s/CI_UE_IP_ADDR/' + self.UEIPAddress + '/\' ' + ci_full_config_file, '\$', 2); # Launch UE with the modified config file self.command('source oaienv', '\$', 5) - self.command('cd cmake_targets', '\$', 5) - self.command('echo "ulimit -c unlimited && ./lte_build_oai/build/lte-softmodem -O ' + self.UESourceCodePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.UE_instance) + '.sh ', '\$', 5) - self.command('chmod 775 ./my-lte-softmodem-run' + str(self.UE_instance) + '.sh ', '\$', 5) - self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf ue_' + self.testCase_id + '.log', '\$', 5) - self.command('echo ' + self.UEPassword + ' | sudo -S -E daemon --inherit --unsafe --name=ue' + str(self.UE_instance) + '_daemon --chdir=' + self.UESourceCodePath + '/cmake_targets -o ' + self.UESourceCodePath + '/cmake_targets/enb_' + self.testCase_id + '.log ./my-lte-softmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) - if not rruCheck: - self.UELogFile = 'enb_' + self.testCase_id + '.log' + self.command('cd cmake_targets/lte_build_oai/build', '\$', 5) + self.command('echo "ulimit -c unlimited && ./lte-uesoftmodem ' + self.Initialize_OAI_UE_args + '" > ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) + self.command('chmod 775 ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S rm -Rf ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S -E daemon --inherit --unsafe --name=ue' + str(self.UE_instance) + '_daemon --chdir=' + self.UESourceCodePath + '/cmake_targets/lte_build_oai/build -o ' + self.UESourceCodePath + '/cmake_targets/ue_' + self.testCase_id + '.log ./my-lte-uesoftmodem-run' + str(self.UE_instance) + '.sh', '\$', 5) + self.UELogFile = 'ue_' + self.testCase_id + '.log' time.sleep(6) + self.command('cd ../..', '\$', 5) doLoop = True loopCounter = 10 while (doLoop): loopCounter = loopCounter - 1 if (loopCounter == 0): # In case of T tracer recording, we may need to kill it - result = re.search('T_stdout', str(self.Initialize_UE_args)) - if result is not None: - self.command('killall --signal SIGKILL record', '\$', 5) + #result = re.search('T_stdout', str(self.Initialize_OAI_UE_args)) + #if result is not None: + # self.command('killall --signal SIGKILL record', '\$', 5) self.close() doLoop = False logging.error('\u001B[1;37;41m UE logging system did not show got sync! \u001B[0m') - self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK) + self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', ALL_PROCESSES_OK) self.CreateHtmlTabFooter(False) - # In case of T tracer recording, we need to kill tshark on EPC side - result = re.search('T_stdout', str(self.Initialize_UE_args)) - if result is not None: - self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) - logging.debug('\u001B[1m Stopping tshark \u001B[0m') - self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) - self.close() - time.sleep(1) - pcap_log_file = 'enb_' + self.testCase_id + '_s1log.pcap' - copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + pcap_log_file, '.') - if (copyin_res == 0): - self.copyout(self.UEIPAddress, self.UEUserName, self.UEPassword, pcap_log_file, self.UESourceCodePath + '/cmake_targets/.') + ## In case of T tracer recording, we need to kill tshark on EPC side + #result = re.search('T_stdout', str(self.Initialize_OAI_UE_args)) + #if result is not None: + # self.open(self.EPCIPAddress, self.EPCUserName, self.EPCPassword) + # logging.debug('\u001B[1m Stopping tshark \u001B[0m') + # self.command('echo ' + self.EPCPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) + # self.close() + # time.sleep(1) + # pcap_log_file = 'enb_' + self.testCase_id + '_s1log.pcap' + # copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + pcap_log_file, '.') + # if (copyin_res == 0): + # self.copyout(self.UEIPAddress, self.UEUserName, self.UEPassword, pcap_log_file, self.UESourceCodePath + '/cmake_targets/.') sys.exit(1) else: - self.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4) - if rruCheck: - result = re.search('wait RUs', str(self.ssh.before)) - else: - result = re.search('got sync', str(self.ssh.before)) + self.command('stdbuf -o0 cat ue_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync"', '\$', 4) + result = re.search('got sync', str(self.ssh.before)) if result is None: time.sleep(6) else: doLoop = False - self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'OK', ALL_PROCESSES_OK) - logging.debug('\u001B[1m Initialize UE Completed\u001B[0m') - + self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'OK', ALL_PROCESSES_OK) + logging.debug('\u001B[1m Initialize OAI UE Completed\u001B[0m') self.close() def checkDevTTYisUnlocked(self): @@ -1674,6 +1665,48 @@ class SSHConnection(): result = logStatus return result + def CheckOAIUEProcessExist(self, initialize_OAI_UE_flag): + multi_jobs = [] + status_queue = SimpleQueue() + if initialize_OAI_UE_flag == False: + p = Process(target = SSH.CheckOAIUEProcess, args = (status_queue,)) + p.daemon = True + p.start() + multi_jobs.append(p) + for job in multi_jobs: + job.join() + + if (status_queue.empty()): + return -15 + else: + result = 0 + while (not status_queue.empty()): + status = status_queue.get() + if (status < 0): + result = status + if result == OAI_UE_PROCESS_FAILED: + fileCheck = re.search('enb_', str(self.UELogFile)) + if fileCheck is not None: + self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.') + logStatus = self.AnalyzeLogFile_UE(self.UELogFile) + if logStatus < 0: + result = logStatus + return result + + def CheckOAIUEProcess(self, status_queue): + try: + self.open(self.OAIUEIPAddress, self.OAIUEUserName, self.OAIUEPassword) + self.command('stdbuf -o0 ps -aux | grep -v grep | grep --color=never lte-uesoftmodem', '\$', 5) + result = re.search('lte-uesoftmodem', str(self.ssh.before)) + if result is None: + logging.debug('\u001B[1;37;41m OAI UE Process Not Found! \u001B[0m') + status_queue.put(OAI_UE_PROCESS_FAILED) + else: + status_queue.put(OAI_UE_PROCESS_OK) + self.close() + except: + os.kill(os.getppid(),signal.SIGUSR1) + def CheckeNBProcess(self, status_queue): try: self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) @@ -1881,6 +1914,162 @@ class SSHConnection(): return ENB_PROCESS_REALTIME_ISSUE return 0 + def AnalyzeLogFile_UE(self, UElogFile): + if (not os.path.isfile('./' + UElogFile)): + return -1 + ue_log_file = open('./' + UElogFile, 'r') + foundAssertion = False + msgAssertion = '' + msgLine = 0 + foundSegFault = False + foundRealTimeIssue = False + rrcSetupRequest = 0 + rrcSetupComplete = 0 + rrcReleaseRequest = 0 + rrcReconfigRequest = 0 + rrcReconfigComplete = 0 + rrcReestablishRequest = 0 + rrcReestablishComplete = 0 + rrcReestablishReject = 0 + rlcDiscardBuffer = 0 + rachCanceledProcedure = 0 + uciStatMsgCount = 0 + pdcpFailure = 0 + ulschFailure = 0 + for line in ue_log_file.readlines(): + result = re.search('[Ss]egmentation [Ff]ault', str(line)) + if result is not None: + foundSegFault = True + result = re.search('[Cc]ore [dD]ump', str(line)) + if result is not None: + foundSegFault = True + result = re.search('[Aa]ssertion', str(line)) + if result is not None: + foundAssertion = True + result = re.search('LLL', str(line)) + if result is not None: + foundRealTimeIssue = True + if foundAssertion and (msgLine < 3): + msgLine += 1 + msgAssertion += str(line) + result = re.search('uci->stat', str(line)) + if result is not None: + uciStatMsgCount += 1 + # full pattern + # No cell synchronization found, abandoning + + # MIB Information => FDD, NORMAL, NidCell 421, N_RB_DL 50, PHICH DURATION 0, PHICH RESOURCE 1/6, TX_ANT 2 + # mask --> FDD NORMAl 421 50 0 1/6 2 + # Measured Carrier Frequency 816000688 Hz (offset 12 Hz) + # mask --> 816000688 + # Found Orange FR (name from internal table) + # mask Orange FR + # SIB5 InterFreqCarrierFreq element 0/3 + # mask -- 0 and 3 + # DL Carrier Frequency/ARFCN : 2645000000/3000 + # mask 2645000000 + # AllowedMeasBandwidth : 100 + # mask 100 + result = re.search('Generating LTE_RRCConnectionSetup', str(line)) + if result is not None: + rrcSetupRequest += 1 + result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line)) + if result is not None: + rrcSetupComplete += 1 + result = re.search('Generate LTE_RRCConnectionRelease', str(line)) + if result is not None: + rrcReleaseRequest += 1 + result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line)) + if result is not None: + rrcReconfigRequest += 1 + result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line)) + if result is not None: + rrcReconfigComplete += 1 + result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line)) + if result is not None: + rrcReestablishRequest += 1 + result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line)) + if result is not None: + rrcReestablishComplete += 1 + result = re.search('LTE_RRCConnectionReestablishmentReject', str(line)) + if result is not None: + rrcReestablishReject += 1 + result = re.search('PDCP.*Out of Resources.*reason', str(line)) + if result is not None: + pdcpFailure += 1 + result = re.search('ULSCH in error in round', str(line)) + if result is not None: + ulschFailure += 1 + result = re.search('BAD all_segments_received', str(line)) + if result is not None: + rlcDiscardBuffer += 1 + result = re.search('Canceled RA procedure for UE rnti', str(line)) + if result is not None: + rachCanceledProcedure += 1 + ue_log_file.close() + self.htmlUEFailureMsg = '' + if uciStatMsgCount > 0: + statMsg = 'UE showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)' + logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') + self.htmlUEFailureMsg += statMsg + '\n' + if pdcpFailure > 0: + statMsg = 'UE showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)' + logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') + self.htmlUEFailureMsg += statMsg + '\n' + if ulschFailure > 0: + statMsg = 'UE showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)' + logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') + self.htmlUEFailureMsg += statMsg + '\n' + if rrcSetupRequest > 0 or rrcSetupComplete > 0: + rrcMsg = 'UE requested ' + str(rrcSetupRequest) + ' RRC Connection Setup(s)' + logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rrcMsg + '\n' + rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed' + logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rrcMsg + '\n' + if rrcReleaseRequest > 0: + rrcMsg = 'UE requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)' + logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rrcMsg + '\n' + if rrcReconfigRequest > 0 or rrcReconfigComplete > 0: + rrcMsg = 'UE requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)' + logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rrcMsg + '\n' + rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed' + logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rrcMsg + '\n' + if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0: + rrcMsg = 'UE requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)' + logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rrcMsg + '\n' + rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed' + logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rrcMsg + '\n' + rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected' + logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rrcMsg + '\n' + if rachCanceledProcedure > 0: + rachMsg = 'UE cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)' + logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rachMsg + '\n' + if foundSegFault: + logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m') + return ENB_PROCESS_SEG_FAULT + if foundAssertion: + logging.debug('\u001B[1;37;41m UE ended with an assertion! \u001B[0m') + self.htmlUEFailureMsg += msgAssertion + return ENB_PROCESS_ASSERTION + if foundRealTimeIssue: + logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m') + self.htmlUEFailureMsg += 'UE faced real time issues!\n' + #return ENB_PROCESS_REALTIME_ISSUE + if rlcDiscardBuffer > 0: + rlcMsg = 'UE RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)' + logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m') + self.htmlUEFailureMsg += rlcMsg + '\n' + return ENB_PROCESS_REALTIME_ISSUE + return 0 + def TerminateeNB(self): self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) self.command('cd ' + self.eNBSourceCodePath + '/cmake_targets', '\$', 5) @@ -2035,7 +2224,7 @@ class SSHConnection(): time.sleep(5) self.close() # If tracer options is on, stopping tshark on EPC side - result = re.search('T_stdout', str(self.Initialize_UE_args)) + result = re.search('T_stdout', str(self.Initialize_OAI_UE_args)) if result is not None: self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) logging.debug('\u001B[1m Stopping tshark \u001B[0m') @@ -2063,7 +2252,7 @@ class SSHConnection(): self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) self.UELogFile = '' else: - result = re.search('enb_', str(self.UELogFile)) + result = re.search('ue_', str(self.UELogFile)) if result is not None: copyin_res = self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.') if (copyin_res == -1): @@ -2528,7 +2717,7 @@ def Usage(): print('------------------------------------------------------------') def CheckClassValidity(action,id): - if action != 'Build_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW' and action != 'Initialize_CatM_module' and action != 'Terminate_CatM_module' and action != 'Attach_CatM_module' and action != 'Detach_CatM_module' and action != 'IdleSleep': + if action != 'Build_eNB' and action != 'Initialize_eNB' and action != 'Terminate_eNB' and action != 'Initialize_UE' and action != 'Terminate_UE' and action != 'Attach_UE' and action != 'Detach_UE' and action != 'Build_OAI_UE' and action != 'Initialize_OAI_UE' and action != 'Terminate_OAI_UE' and action != 'Ping' and action != 'Iperf' and action != 'Reboot_UE' and action != 'Initialize_HSS' and action != 'Terminate_HSS' and action != 'Initialize_MME' and action != 'Terminate_MME' and action != 'Initialize_SPGW' and action != 'Terminate_SPGW' and action != 'Initialize_CatM_module' and action != 'Terminate_CatM_module' and action != 'Attach_CatM_module' and action != 'Detach_CatM_module' and action != 'IdleSleep': logging.debug('ERROR: test-case ' + id + ' has wrong class ' + action) return False return True @@ -2555,6 +2744,20 @@ def GetParametersFromXML(action): else: SSH.nbMaxUEtoAttach = int(nbMaxUEtoAttach) + if action == 'Build_OAI_UE': + SSH.Build_OAI_UE_args = test.findtext('Build_OAI_UE_args') + + if action == 'Initialize_OAI_UE': + SSH.Initialize_OAI_UE_args = test.findtext('Initialize_OAI_UE_args') + SSH.UE_instance = test.findtext('UE_instance') + if (SSH.UE_instance is None): + SSH.UE_instance = '0' + + if action == 'Terminate_OAI_UE': + SSH.eNB_instance = test.findtext('UE_instance') + if (SSH.UE_instance is None): + SSH.UE_instance = '0' + if action == 'Ping': SSH.ping_args = test.findtext('ping_args') SSH.ping_packetloss_threshold = test.findtext('ping_packetloss_threshold') @@ -2672,9 +2875,6 @@ while len(argvs) > 1: elif re.match('^\-\-UEIPAddress=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-UEIPAddress=(.+)$', myArgv, re.IGNORECASE) SSH.UEIPAddress = matchReg.group(1) - elif re.match('^\-\-UERepository=(.+)$', myArgv, re.IGNORECASE): - matchReg = re.match('^\-\-UERepository=(.+)$', myArgv, re.IGNORECASE) - SSH.UERepository = matchReg.group(1) elif re.match('^\-\-UEUserName=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-UEUserName=(.+)$', myArgv, re.IGNORECASE) SSH.UEUserName = matchReg.group(1) @@ -2684,12 +2884,6 @@ while len(argvs) > 1: elif re.match('^\-\-UESourceCodePath=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-UESourceCodePath=(.+)$', myArgv, re.IGNORECASE) SSH.UESourceCodePath = matchReg.group(1) - elif re.match('^\-\-UEBranch=(.+)$', myArgv, re.IGNORECASE): - matchReg = re.match('^\-\-UEBranch=(.+)$', myArgv, re.IGNORECASE) - SSH.UEBranch = matchReg.group(1) - elif re.match('^\-\-UECommitID=(.*)$', myArgv, re.IGNORECASE): - matchReg = re.match('^\-\-UECommitID=(.*)$', myArgv, re.IGNORECASE) - SSH.UECommitID = matchReg.group(1) elif re.match('^\-\-finalStatus=(.+)$', myArgv, re.IGNORECASE): matchReg = re.match('^\-\-finalStatus=(.+)$', myArgv, re.IGNORECASE) finalStatus = matchReg.group(1) @@ -2710,6 +2904,12 @@ elif re.match('^TerminateUE$', mode, re.IGNORECASE): sys.exit('Insufficient Parameter') signal.signal(signal.SIGUSR1, receive_signal) SSH.TerminateUE() +elif re.match('^TerminateOAIUE$', mode, re.IGNORECASE): + if SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '': + Usage() + sys.exit('Insufficient Parameter') + signal.signal(signal.SIGUSR1, receive_signal) + SSH.TerminateUE() elif re.match('^TerminateHSS$', mode, re.IGNORECASE): if SSH.EPCIPAddress == '' or SSH.EPCUserName == '' or SSH.EPCPassword == '' or SSH.EPCType == '' or SSH.EPCSourceCodePath == '': Usage() @@ -2761,7 +2961,7 @@ elif re.match('^LogCollectIperf$', mode, re.IGNORECASE): sys.exit('Insufficient Parameter') SSH.LogCollectIperf() elif re.match('^InitiateHtml$', mode, re.IGNORECASE): - if SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '': + if (SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '') and (SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == ''): Usage() sys.exit('Insufficient Parameter') count = 0 @@ -2782,17 +2982,11 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re Usage() sys.exit('Insufficient Parameter') - if (SSH.EPCIPAddress != 'none'): + if (SSH.EPCIPAddress != ''): SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/tcp_iperf_stats.awk", "/tmp") SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/active_net_interfaces.awk", "/tmp") else: - print('SSH.UEIPAddress=' + SSH.UEIPAddress) - print('SSH.UERepository=' + SSH.UERepository) - print('SSH.UEBranch=' + SSH.UEBranch) - print('SSH.UEUserName=' + SSH.UEUserName) - print('SSH.UEPassword=' + SSH.UEPassword) - print('SSH.UESourceCodePath=' + SSH.UESourceCodePath) - if SSH.UEIPAddress == '' or SSH.UERepository == '' or SSH.UEBranch == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '': + if SSH.UEIPAddress == '' or SSH.eNBRepository == '' or SSH.eNBBranch == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '': Usage() sys.exit('UE: Insufficient Parameter') @@ -2878,6 +3072,12 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re SSH.AttachUE() elif action == 'Detach_UE': SSH.DetachUE() + elif action == 'Build_OAI_UE': + SSH.BuildOAIUE() + elif action == 'Initialize_OAI_UE': + SSH.InitializeOAIUE() + elif action == 'Terminate_OAI_UE': + SSH.TerminateOAIUE() elif action == 'Initialize_CatM_module': SSH.InitializeCatM() elif action == 'Terminate_CatM_module': diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz.xml b/ci-scripts/xml_files/ue_band20_test_10mhz.xml index 72b6a81ec1..38e1aa6b7a 100644 --- a/ci-scripts/xml_files/ue_band20_test_10mhz.xml +++ b/ci-scripts/xml_files/ue_band20_test_10mhz.xml @@ -21,12 +21,39 @@ --> - 080101 + +090101 +090102 000001 090104 +090103 000001 090104 + - + + Build_OAI_UE + Build OAI UE + -w USRP --UE + + + + Initialize_OAI_UE + Initialize OAI UE -- sniffing SFR frequency + -C 806000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect + + + + Initialize_OAI_UE + Initialize OAI UE -- sniffing Orange frequency + -C 816000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect + + + IdleSleep - Sleep OAI UE - 1 + Sleep + 30 + + + + Terminate_OAI_UE + Terminate OAI UE diff --git a/ci-scripts/xml_files/ue_sfr_796.xml b/ci-scripts/xml_files/ue_sfr_796.xml new file mode 100644 index 0000000000..7600d371c1 --- /dev/null +++ b/ci-scripts/xml_files/ue_sfr_796.xml @@ -0,0 +1,44 @@ + + + 080102 080103 080104 + + + Initialize_OAI_UE + Initialize OAI UE + -C 796000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect + 0 + + + + IdleSleep + Sleep + + + + Terminate_OAI_UE + Terminate OAI UE + 0 + + + From 6d93c9c522c1cba6dc6a7b467296b83fc1d546c4 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Tue, 19 Mar 2019 09:49:04 +0100 Subject: [PATCH 04/26] Using os.getcwd() instead of sys.path[0] to work with pdb. Signed-off-by: Boris Djalal --- ci-scripts/main.py | 101 ++++++++++----------------------------------- 1 file changed, 22 insertions(+), 79 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 5e113dd413..89ead0a016 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -1923,19 +1923,13 @@ class SSHConnection(): msgLine = 0 foundSegFault = False foundRealTimeIssue = False - rrcSetupRequest = 0 - rrcSetupComplete = 0 - rrcReleaseRequest = 0 - rrcReconfigRequest = 0 - rrcReconfigComplete = 0 - rrcReestablishRequest = 0 - rrcReestablishComplete = 0 - rrcReestablishReject = 0 rlcDiscardBuffer = 0 rachCanceledProcedure = 0 uciStatMsgCount = 0 pdcpFailure = 0 ulschFailure = 0 + no_cell_sync_found = False + marker1 = False for line in ue_log_file.readlines(): result = re.search('[Ss]egmentation [Ff]ault', str(line)) if result is not None: @@ -1957,55 +1951,29 @@ class SSHConnection(): uciStatMsgCount += 1 # full pattern # No cell synchronization found, abandoning - + result = re.search('No cell synchronization found, abandoning', str(line)) + if result is not None: + no_cell_sync_found = True # MIB Information => FDD, NORMAL, NidCell 421, N_RB_DL 50, PHICH DURATION 0, PHICH RESOURCE 1/6, TX_ANT 2 # mask --> FDD NORMAl 421 50 0 1/6 2 + result = re.search("MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P\d{1,3}), N_RB_DL (?P\d{1,3}), PHICH DURATION (?P\d), PHICH RESOURCE (?P\d/\d), TX_ANT (?P\d)", str(line)) + if result is not None: + if result.group(0) != 'FDD' or result.group(1) != 'NORMAL' or result.group('nidcell') != '421' or result.group('n_rb_dl') != '421' or result.group('phich_duration') != '0' or result.group('phich_resource') != '1/6' or result.group('tx_ant') != '2': + marker1 = True # Measured Carrier Frequency 816000688 Hz (offset 12 Hz) - # mask --> 816000688 + # mask --> 816000688i + # Found Orange FR (name from internal table) # mask Orange FR + # SIB5 InterFreqCarrierFreq element 0/3 # mask -- 0 and 3 + # DL Carrier Frequency/ARFCN : 2645000000/3000 # mask 2645000000 + # AllowedMeasBandwidth : 100 # mask 100 - result = re.search('Generating LTE_RRCConnectionSetup', str(line)) - if result is not None: - rrcSetupRequest += 1 - result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line)) - if result is not None: - rrcSetupComplete += 1 - result = re.search('Generate LTE_RRCConnectionRelease', str(line)) - if result is not None: - rrcReleaseRequest += 1 - result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line)) - if result is not None: - rrcReconfigRequest += 1 - result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line)) - if result is not None: - rrcReconfigComplete += 1 - result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line)) - if result is not None: - rrcReestablishRequest += 1 - result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line)) - if result is not None: - rrcReestablishComplete += 1 - result = re.search('LTE_RRCConnectionReestablishmentReject', str(line)) - if result is not None: - rrcReestablishReject += 1 - result = re.search('PDCP.*Out of Resources.*reason', str(line)) - if result is not None: - pdcpFailure += 1 - result = re.search('ULSCH in error in round', str(line)) - if result is not None: - ulschFailure += 1 - result = re.search('BAD all_segments_received', str(line)) - if result is not None: - rlcDiscardBuffer += 1 - result = re.search('Canceled RA procedure for UE rnti', str(line)) - if result is not None: - rachCanceledProcedure += 1 ue_log_file.close() self.htmlUEFailureMsg = '' if uciStatMsgCount > 0: @@ -2020,34 +1988,6 @@ class SSHConnection(): statMsg = 'UE showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)' logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m') self.htmlUEFailureMsg += statMsg + '\n' - if rrcSetupRequest > 0 or rrcSetupComplete > 0: - rrcMsg = 'UE requested ' + str(rrcSetupRequest) + ' RRC Connection Setup(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - self.htmlUEFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - self.htmlUEFailureMsg += rrcMsg + '\n' - if rrcReleaseRequest > 0: - rrcMsg = 'UE requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - self.htmlUEFailureMsg += rrcMsg + '\n' - if rrcReconfigRequest > 0 or rrcReconfigComplete > 0: - rrcMsg = 'UE requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - self.htmlUEFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - self.htmlUEFailureMsg += rrcMsg + '\n' - if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0: - rrcMsg = 'UE requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - self.htmlUEFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - self.htmlUEFailureMsg += rrcMsg + '\n' - rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') - self.htmlUEFailureMsg += rrcMsg + '\n' if rachCanceledProcedure > 0: rachMsg = 'UE cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)' logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m') @@ -2068,6 +2008,8 @@ class SSHConnection(): logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m') self.htmlUEFailureMsg += rlcMsg + '\n' return ENB_PROCESS_REALTIME_ISSUE + if marker1: + logging.debug('\u001B[1;37;41m Wrong MIB Information line! \u001B[0m') return 0 def TerminateeNB(self): @@ -2800,6 +2742,7 @@ SSH = SSHConnection() argvs = sys.argv argc = len(argvs) +cwd = os.getcwd() while len(argvs) > 1: myArgv = argvs.pop(1) # 0th is this file's name @@ -2966,7 +2909,7 @@ elif re.match('^InitiateHtml$', mode, re.IGNORECASE): sys.exit('Insufficient Parameter') count = 0 while (count < SSH.nbTestXMLfiles): - xml_test_file = sys.path[0] + "/" + SSH.testXMLfiles[count] + xml_test_file = cwd + "/" + SSH.testXMLfiles[count] xmlTree = ET.parse(xml_test_file) xmlRoot = xmlTree.getroot() SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-' + str(count))) @@ -2983,8 +2926,8 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re sys.exit('Insufficient Parameter') if (SSH.EPCIPAddress != ''): - SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/tcp_iperf_stats.awk", "/tmp") - SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, sys.path[0] + "/active_net_interfaces.awk", "/tmp") + SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, cwd + "/tcp_iperf_stats.awk", "/tmp") + SSH.copyout(SSH.EPCIPAddress, SSH.EPCUserName, SSH.EPCPassword, cwd + "/active_net_interfaces.awk", "/tmp") else: if SSH.UEIPAddress == '' or SSH.eNBRepository == '' or SSH.eNBBranch == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '': Usage() @@ -2993,9 +2936,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re #read test_case_list.xml file # if no parameters for XML file, use default value if (SSH.nbTestXMLfiles != 1): - xml_test_file = sys.path[0] + "/test_case_list.xml" + xml_test_file = cwd + "/test_case_list.xml" else: - xml_test_file = sys.path[0] + "/" + SSH.testXMLfiles[0] + xml_test_file = cwd + "/" + SSH.testXMLfiles[0] xmlTree = ET.parse(xml_test_file) xmlRoot = xmlTree.getroot() From 1affd5ff59a8e2215683507a22e56eedcf5c97ea Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Tue, 19 Mar 2019 16:11:17 +0100 Subject: [PATCH 05/26] Printing information from the UE log file. Signed-off-by: Boris Djalal --- ci-scripts/main.py | 54 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 89ead0a016..a1851fcec8 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -1877,7 +1877,7 @@ class SSHConnection(): self.htmleNBFailureMsg += rrcMsg + '\n' if rrcReconfigRequest > 0 or rrcReconfigComplete > 0: rrcMsg = 'eNB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)' - logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') + logging.debug('\u001B[1;30;43m ' + rrcMsig + ' \u001B[0m') self.htmleNBFailureMsg += rrcMsg + '\n' rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed' logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m') @@ -1929,7 +1929,6 @@ class SSHConnection(): pdcpFailure = 0 ulschFailure = 0 no_cell_sync_found = False - marker1 = False for line in ue_log_file.readlines(): result = re.search('[Ss]egmentation [Ff]ault', str(line)) if result is not None: @@ -1955,25 +1954,58 @@ class SSHConnection(): if result is not None: no_cell_sync_found = True # MIB Information => FDD, NORMAL, NidCell 421, N_RB_DL 50, PHICH DURATION 0, PHICH RESOURCE 1/6, TX_ANT 2 - # mask --> FDD NORMAl 421 50 0 1/6 2 - result = re.search("MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P\d{1,3}), N_RB_DL (?P\d{1,3}), PHICH DURATION (?P\d), PHICH RESOURCE (?P\d/\d), TX_ANT (?P\d)", str(line)) + # mask --> FDD NORMAL 421 50 0 1/6 2 + result = re.search("MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P\d{1,3}), N_RB_DL (?P\d{1,3}), PHICH DURATION (?P\d), PHICH RESOURCE (?P.{1,4}), TX_ANT (?P\d)", str(line)) if result is not None: - if result.group(0) != 'FDD' or result.group(1) != 'NORMAL' or result.group('nidcell') != '421' or result.group('n_rb_dl') != '421' or result.group('phich_duration') != '0' or result.group('phich_resource') != '1/6' or result.group('tx_ant') != '2': - marker1 = True + try: + print('\033[94m' + "MIB Information: " + result.group(1) + ', ' + result.group(2) + '\033[0m') + print('\033[94m' + "nidcell = " + result.group('nidcell') + '\033[0m') + print('\033[94m' + "n_rb_dl = " + result.group('n_rb_dl') + '\033[0m') + print('\033[94m' + "phich_duration = " + result.group('phich_duration') + '\033[0m') + print('\033[94m' + "phich_resource = " + result.group('phich_resource') + '\033[0m') + print('\033[94m' + "tx_ant = " + result.group('tx_ant') + '\033[0m') + except Exception as e: + print('\033[91m' + "a marker was not found" + '\033[0m') # Measured Carrier Frequency 816000688 Hz (offset 12 Hz) # mask --> 816000688i - + result = re.search("Measured Carrier Frequency (?P\d{1,15}) Hz", str(line)) + if result is not None: + try: + print('\033[94m' + "Measured Carrier Frequency = " + result.group('measured_carrier_frequency') + '\033[0m') + except Exception as e: + print('\033[91m' + "Measured Carrier Frequency not found" + '\033[0m') # Found Orange FR (name from internal table) # mask Orange FR - + result = re.search("Found (?P[\w,\s]{1,15}) \(name from internal table\)", str(line)) + if result is not None: + try: + print('\033[94m' + "The operator is: " + result.group('operator') + '\033[0m') + except Exception as e: + print('\033[91m' + "Operator name not found" + '\033[0m') # SIB5 InterFreqCarrierFreq element 0/3 # mask -- 0 and 3 - + result = re.search("SIB5 InterFreqCarrierFreq element (.{1,4})/(.{1,4})", str(line)) + if result is not None: + try: + print('\033[94m' + "SIB5 InterFreqCarrierFreq element " + result.group(1) + '/' + result.group(2) + '\033[0m') + except Exception as e: + print('\033[91m' + "SIB5 InterFreqCarrierFreq element not found" + '\033[0m') # DL Carrier Frequency/ARFCN : 2645000000/3000 # mask 2645000000 - + result = re.search("DL Carrier Frequency/ARFCN : (?P)\d{1,15}/\d{1,4}", str(line)) + if result is not None: + try: + print('\033[94m' + "DL Carrier Frequency is: " + result.group('carrier_frequency') + '\033[0m') + except Exception as e: + print('\033[91m' + "DL Carrier Frequency not found" + '\033[0m') # AllowedMeasBandwidth : 100 # mask 100 + result = re.search("AllowedMeasBandwidth : (?P\d{1,7})", str(line)) + if result is not None: + try: + print('\033[94m' + "AllowedMeasBandwidth: " + result.group('allowed_bandwidth') + '\033[0m') + except Exception as e: + print('\033[91m' + "AllowedMeasBandwidth not found" + '\033[0m') ue_log_file.close() self.htmlUEFailureMsg = '' if uciStatMsgCount > 0: @@ -2008,8 +2040,6 @@ class SSHConnection(): logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m') self.htmlUEFailureMsg += rlcMsg + '\n' return ENB_PROCESS_REALTIME_ISSUE - if marker1: - logging.debug('\u001B[1;37;41m Wrong MIB Information line! \u001B[0m') return 0 def TerminateeNB(self): From 32be9dcbb8edb814060a90b64eacdd535c573fda Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Wed, 20 Mar 2019 10:10:24 +0100 Subject: [PATCH 06/26] Testing 2 frequencies for each operator (Orange and SFR). Signed-off-by: Boris Djalal --- ci-scripts/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index a1851fcec8..f3caa1f23f 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -344,7 +344,6 @@ class SSHConnection(): self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK) def BuildOAIUE(self): - return if self.UEIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': Usage() sys.exit('Insufficient Parameter') From dccf64ac571d6de310e86df0bebd2fa3f5cedb07 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Wed, 20 Mar 2019 15:48:03 +0100 Subject: [PATCH 07/26] Deleting ue_sfr_796.xml. Signed-off-by: Boris Djalal --- ci-scripts/xml_files/ue_sfr_796.xml | 44 ----------------------------- 1 file changed, 44 deletions(-) delete mode 100644 ci-scripts/xml_files/ue_sfr_796.xml diff --git a/ci-scripts/xml_files/ue_sfr_796.xml b/ci-scripts/xml_files/ue_sfr_796.xml deleted file mode 100644 index 7600d371c1..0000000000 --- a/ci-scripts/xml_files/ue_sfr_796.xml +++ /dev/null @@ -1,44 +0,0 @@ - - - 080102 080103 080104 - - - Initialize_OAI_UE - Initialize OAI UE - -C 796000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect - 0 - - - - IdleSleep - Sleep - - - - Terminate_OAI_UE - Terminate OAI UE - 0 - - - From c396e42ab2e6af266eb5c62c5de5b29e6345bba8 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Thu, 21 Mar 2019 11:49:10 +0100 Subject: [PATCH 08/26] Adding the reset procedure using b2xx_fx3_utils. Signed-off-by: Boris Djalal --- ci-scripts/main.py | 98 +++++++++++-------- .../xml_files/ue_band20_test_10mhz_orange.xml | 62 ++++++++++++ .../xml_files/ue_band20_test_10mhz_sfr.xml | 62 ++++++++++++ 3 files changed, 181 insertions(+), 41 deletions(-) create mode 100644 ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml create mode 100644 ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml diff --git a/ci-scripts/main.py b/ci-scripts/main.py index f3caa1f23f..ec2d526910 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -119,17 +119,12 @@ class SSHConnection(): self.htmlTabNames = [] self.htmlTabIcons = [] self.finalStatus = False - self.eNBOsVersion = '' - self.eNBKernelVersion = '' - self.eNBUhdVersion = '' - self.eNBCpuNb = '' - self.eNBCpuModel = '' - self.eNBCpuMHz = '' - #self.UEOsVersion = '' - #self.UEKernelVersion = '' - #self.UEUhdVersion = '' - #self.UECpuNb = '' - #self.UECpuModel = '' + self.OsVersion = '' + self.KernelVersion = '' + self.UhdVersion = '' + self.CpuNb = '' + self.CpuModel = '' + self.CpuMHz = '' self.UEIPAddress = '' self.UEBranch = '' #self.UE_AllowMerge = False @@ -344,6 +339,7 @@ class SSHConnection(): self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK) def BuildOAIUE(self): + return if self.UEIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': Usage() sys.exit('Insufficient Parameter') @@ -582,6 +578,16 @@ class SSHConnection(): # self.CreateHtmlTabFooter(False) # sys.exit(1) self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + # b2xx_fx3_utils reset procedure + self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 5) + result = re.search('type: b200', str(self.ssh.before)) + if result is not None: + logging.debug('Found a B2xx device --> resetting it') + self.command('echo ' + self.UEPassword + ' | sudo -S sudo b2xx_fx3_utils --reset-device', '\$', 5) + # Reloading FGPA bin firmware + self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 5) + else: + logging.debug('Did not find any B2xx device') self.command('cd ' + self.UESourceCodePath, '\$', 5) # Initialize_OAI_UE_args usually start with -C and followed by the location in repository #full_config_file = self.Initialize_OAI_UE_args.replace('-O ','') @@ -2336,41 +2342,52 @@ class SSHConnection(): def RetrieveSystemVersion(self): if self.eNBIPAddress == 'none': - self.eNBOsVersion = 'Ubuntu 16.04.5 LTS' - self.eNBKernelVersion = '4.15.0-45-generic' - self.eNBUhdVersion = '3.13.0.1-0' - self.eNBCpuNb = '4' - self.eNBCpuModel = 'Intel(R) Core(TM) i5-6200U' - self.eNBCpuMHz = '2399.996 MHz' + self.OsVersion = 'Ubuntu 16.04.5 LTS' + self.KernelVersion = '4.15.0-45-generic' + self.UhdVersion = '3.13.0.1-0' + self.CpuNb = '4' + self.CpuModel = 'Intel(R) Core(TM) i5-6200U' + self.CpuMHz = '2399.996 MHz' return - if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '': + machine = None + if self.eNBIPAddress != '' and self.eNBUserName != '' and self.eNBPassword != '': + machine = 'eNB' + self.IPAddress = self.eNBIPAddress + self.UserName = self.eNBUserName + self.Password = self.eNBPassword + if self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != '': + machine = 'UE' + self.IPAddress = self.UEIPAddress + self.UserName = self.UEUserName + self.Password = self.UEPassword + if machine is None: Usage() sys.exit('Insufficient Parameter') - self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) + self.open(self.IPAddress, self.UserName, self.Password) self.command('lsb_release -a', '\$', 5) result = re.search('Description:\\\\t(?P[a-zA-Z0-9\-\_\.\ ]+)', str(self.ssh.before)) if result is not None: - self.eNBOsVersion = result.group('os_type') - logging.debug('OS is: ' + self.eNBOsVersion) + self.OsVersion = result.group('os_type') + logging.debug('OS is: ' + self.OsVersion) self.command('uname -r', '\$', 5) result = re.search('uname -r\\\\r\\\\n(?P[a-zA-Z0-9\-\_\.]+)', str(self.ssh.before)) if result is not None: - self.eNBKernelVersion = result.group('kernel_version') - logging.debug('Kernel Version is: ' + self.eNBKernelVersion) + self.KernelVersion = result.group('kernel_version') + logging.debug('Kernel Version is: ' + self.KernelVersion) self.command('dpkg --list | egrep --color=never libuhd003', '\$', 5) result = re.search('libuhd003:amd64 *(?P[0-9\.]+)', str(self.ssh.before)) if result is not None: - self.eNBUhdVersion = result.group('uhd_version') - logging.debug('UHD Version is: ' + self.eNBUhdVersion) + self.UhdVersion = result.group('uhd_version') + logging.debug('UHD Version is: ' + self.UhdVersion) self.command('lscpu', '\$', 5) result = re.search('CPU\(s\): *(?P[0-9]+).*Model name: *(?P[a-zA-Z0-9\-\_\.\ \(\)]+).*CPU MHz: *(?P[0-9\.]+)', str(self.ssh.before)) if result is not None: - self.eNBCpuNb = result.group('nb_cpus') - logging.debug('nb_cpus: ' + self.eNBCpuNb) - self.eNBCpuModel = result.group('model') - logging.debug('model: ' + self.eNBCpuModel) - self.eNBCpuMHz = result.group('cpu_mhz') + ' MHz' - logging.debug('cpu_mhz: ' + self.eNBCpuMHz) + self.CpuNb = result.group('nb_cpus') + logging.debug('nb_cpus: ' + self.CpuNb) + self.CpuModel = result.group('model') + logging.debug('model: ' + self.CpuModel) + self.CpuMHz = result.group('cpu_mhz') + ' MHz' + logging.debug('cpu_mhz: ' + self.CpuMHz) self.close() #----------------------------------------------------------- @@ -2459,10 +2476,9 @@ class SSHConnection(): if (self.ADBIPAddress != 'none'): self.GetAllUEDevices(terminate_ue_flag) self.GetAllCatMDevices(terminate_ue_flag) - self.htmlUEConnected = len(self.UEDevices) - - self.htmlFile.write('

' + str(len(self.UEDevices)) + ' UE(s) is(are) connected to ADB bench server

\n') - self.htmlFile.write('

' + str(len(self.CatMDevices)) + ' CAT-M UE(s) is(are) connected to bench server

\n') + self.htmlUEConnected = len(self.UEDevices) + self.htmlFile.write('

' + str(len(self.UEDevices)) + ' UE(s) is(are) connected to ADB bench server

\n') + self.htmlFile.write('

' + str(len(self.CatMDevices)) + ' CAT-M UE(s) is(are) connected to bench server

\n') self.htmlFile.write('
\n') self.htmlFile.write('
    \n') count = 0 @@ -2535,19 +2551,19 @@ class SSHConnection(): self.htmlFile.write(' \n') self.htmlFile.write(' \n') self.htmlFile.write(' OS Version\n') - self.htmlFile.write(' ' + self.eNBOsVersion + '\n') + self.htmlFile.write(' ' + self.OsVersion + '\n') self.htmlFile.write(' Kernel Version\n') - self.htmlFile.write(' ' + self.eNBKernelVersion + '\n') + self.htmlFile.write(' ' + self.KernelVersion + '\n') self.htmlFile.write(' UHD Version\n') - self.htmlFile.write(' ' + self.eNBUhdVersion + '\n') + self.htmlFile.write(' ' + self.UhdVersion + '\n') self.htmlFile.write(' \n') self.htmlFile.write(' \n') self.htmlFile.write(' Nb CPUs\n') - self.htmlFile.write(' ' + self.eNBCpuNb + '\n') + self.htmlFile.write(' ' + self.CpuNb + '\n') self.htmlFile.write(' CPU Model Name\n') - self.htmlFile.write(' ' + self.eNBCpuModel + '\n') + self.htmlFile.write(' ' + self.CpuModel + '\n') self.htmlFile.write(' CPU Frequency\n') - self.htmlFile.write(' ' + self.eNBCpuMHz + '\n') + self.htmlFile.write(' ' + self.CpuMHz + '\n') self.htmlFile.write(' \n') self.htmlFile.write(' \n') self.htmlFile.write(' Final Status\n') diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml new file mode 100644 index 0000000000..4b5e2137de --- /dev/null +++ b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml @@ -0,0 +1,62 @@ + + + test-10mhz-orange + Test-10Mhz-Orange + tasks + +090101 +090102 000001 090104 +090103 000001 090104 + + + + Build_OAI_UE + Build OAI UE + -w USRP --UE + + + + Initialize_OAI_UE + Initialize OAI UE -- sniffing Orange frequency + -C 816000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect + + + + Initialize_OAI_UE + Initialize OAI UE -- sniffing Orange frequency + -C 812000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect + + + + IdleSleep + Sleep + 30 + + + + Terminate_OAI_UE + Terminate OAI UE + + + diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml new file mode 100644 index 0000000000..481f6b2699 --- /dev/null +++ b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml @@ -0,0 +1,62 @@ + + + test-10mHz-sfr + Test-10MHz-SFR + tasks + +090101 +090102 000001 090104 +090103 000001 090104 + + + + Build_OAI_UE + Build OAI UE + -w USRP --UE + + + + Initialize_OAI_UE + Initialize OAI UE -- sniffing SFR frequency + -C 806000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect + + + + Initialize_OAI_UE + Initialize OAI UE -- sniffing SFR frequency + -C 802000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect + + + + IdleSleep + Sleep + 30 + + + + Terminate_OAI_UE + Terminate OAI UE + + + From e360232e56fda4a480aeb1d7025df1e68a9a2c08 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Thu, 21 Mar 2019 13:58:20 +0100 Subject: [PATCH 09/26] Fixing a typo. Signed-off-by: Boris Djalal --- ci-scripts/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 5153ef51b1..d98bb60eca 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -348,7 +348,6 @@ class SSHConnection(): self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK) def BuildOAIUE(self): - return if self.UEIPAddress == '' or self.eNBRepository == '' or self.eNBBranch == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': Usage() sys.exit('Insufficient Parameter') @@ -2543,7 +2542,7 @@ class SSHConnection(): if result is not None: self.UhdVersion = result.group('uhd_version') logging.debug('UHD Version is: ' + self.UhdVersion) - self.command('echo ' + self.eNBPassword + ' | sudo -S uhd_find_devices', '\$', 5) + self.command('echo ' + self.Password + ' | sudo -S uhd_find_devices', '\$', 5) result = re.search('product: (?P[0-9A-Za-z]+)\\\\r\\\\n', str(self.ssh.before)) if result is not None: self.UsrpBoard = result.group('usrp_board') From 982aa940a76010206b3e5c6003243872f8da17f1 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Thu, 21 Mar 2019 15:50:00 +0100 Subject: [PATCH 10/26] First test of the pipeline. Signed-off-by: Boris Djalal --- ci-scripts/Jenkinsfile-tmp-ue | 302 ++++++++++++++++++++++++++++++++++ ci-scripts/main.py | 4 +- 2 files changed, 304 insertions(+), 2 deletions(-) create mode 100644 ci-scripts/Jenkinsfile-tmp-ue diff --git a/ci-scripts/Jenkinsfile-tmp-ue b/ci-scripts/Jenkinsfile-tmp-ue new file mode 100644 index 0000000000..f7e39f7e83 --- /dev/null +++ b/ci-scripts/Jenkinsfile-tmp-ue @@ -0,0 +1,302 @@ +#!/bin/groovy +/* + * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The OpenAirInterface Software Alliance licenses this file to You under + * the OAI Public License, Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.openairinterface.org/?page_id=698 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + *------------------------------------------------------------------------------- + * For more information about the OpenAirInterface (OAI) Software Alliance: + * contact@openairinterface.org + */ + +// Template Jenkins Declarative Pipeline script to run Test w/ RF HW + +// Location of the python executor node shall be in the same subnet as the others servers +def pythonExecutor = params.pythonExecutor + +// Location of the test XML file to be run +def testXMLFile = params.pythonTestXmlFile +def mainPythonAllXmlFiles = "" +def buildStageStatus = true + +// Name of the test stage +def testStageName = params.pipelineTestStageName + +// Name of the phone resource +def ciSmartPhoneResource = params.smartphonesResource + +// Terminate Status +def termUE = 0 +def termENB = 1 +def termSPGW = 2 +def termMME = 3 +def termHSS = 4 +def termStatusArray = new Boolean[termHSS + 1] +termStatusArray[termUE] = false +termStatusArray[termENB] = false +termStatusArray[termSPGW] = false +termStatusArray[termMME] = false +termStatusArray[termHSS] = false + +// Global Parameters. Normally they should be populated when the master job +// triggers the slave job with parameters +def eNB_Repository +def eNB_Branch +def eNB_CommitID +def eNB_AllowMergeRequestProcess = false +def eNB_TargetBranch + +pipeline { + agent { + label pythonExecutor + } + options { + disableConcurrentBuilds() + ansiColor('xterm') + lock (ciSmartPhoneResource) + } + // the following parameter options are commented out so it shows the ones + // that you SHALL have to run the job. + // You can use them as template + /* + parameters { + //node-test parameters + string(name: 'pythonExecutor', defaultValue: 'nodea', description: 'Node where the pipeline - python scripts will be executed') + string(name: 'pythonTestXmlFile', defaultValue: 'enb_usrpB210_band7_50PRB.xml', description: 'Location of the Test XML to be run') + string(name: 'pipelineTestStageName', defaultValue: 'Test COTS-UE - OAI eNB - LTEBOX EPC', description: 'Naming of the Test Stage') + booleanParam(name: 'pipelineZipsConsoleLog', defaultValue: 'True', description: 'If true, the pipeline script retrieves the job console log, zips it and archives it as artifact') + string(name: 'smartphonesResource', defaultValue: 'CI-Bench-1-Phones', description: 'Lockeable Resource to prevent multiple jobs to run simultaneously with the same resource') + + //eNB parameters + string(name: 'eNB_IPAddress', defaultValue: '192.168.XX.XX', description: 'IP Address of eNB') + credentials(name: 'eNB_Credentials', defaultValue: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', credentialType: "Username with password", required: true, description: 'Credentials for eNB') + string(name: 'eNB_SourceCodePath', defaultValue: '/tmp/CI-enb', description: 'Full path of eNB source code') + + //EPC parameters + string(name: 'EPC_IPAddress', defaultValue: '192.168.XX.XX', description: 'IP Address of EPC') + string(name: 'EPC_Type', defaultValue: 'ltebox', description: 'EPC type: OAI or ltebox') + credentials(name: 'EPC_Credentials', defaultValue: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', credentialType: "Username with password", required: true, description: 'Credentials for EPC') + string(name: 'EPC_SourceCodePath', defaultValue: '/tmp/CI-enb', description: 'Full path of EPC source code') + + //ADB server parameters + string(name: 'ADB_IPAddress', defaultValue: '192.168.XX.XX', description: 'IP Address of ADB server') + credentials(name: 'ADB_Credentials', defaultValue: 'XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX', credentialType: "Username with password", required: true, description: 'Credentials for ADB') + } + */ + + stages { + stage ("Verify Parameters") { + steps { + script { + echo '\u2705 \u001B[32mVerify Parameters\u001B[0m' + def allParametersPresent = true + + // It is already to late to check it + if (params.pythonExecutor != null) { + echo "eNB CI executor node : ${pythonExecutor}" + } + // If not present picking a default XML file + if (params.pythonTestXmlFile == null) { + // picking default + testXMLFile = 'xml_files/enb_usrpB210_band7_50PRB.xml' + echo "Test XML file(default): ${testXMLFile}" + mainPythonAllXmlFiles += "--XMLTestFile=" + testXMLFile + " " + } else { + String[] myXmlTestSuite = testXMLFile.split("\\r?\\n") + for (xmlFile in myXmlTestSuite) { + if (fileExists("ci-scripts/" + xmlFile)) { + mainPythonAllXmlFiles += "--XMLTestFile=" + xmlFile + " " + echo "Test XML file : ${xmlFile}" + } + } + } + // If not present picking a default Stage Name + if (params.pipelineTestStageName == null) { + // picking default + testStageName = 'Template Test Stage' + } + + if (params.smartphonesResource == null) { + allParametersPresent = false + } + if (params.UE_IPAddress == null) { + allParametersPresent = false + } + if (params.UE_SourceCodePath == null) { + allParametersPresent = false + } + if (params.UE_Credentials == null) { + allParametersPresent = false + } + // the following 4 parameters should be pushed by the master trigger + // if not present, take the job GIT variables (used for developing) + if (params.eNB_Repository == null) { + eNB_Repository = env.GIT_URL + } else { + eNB_Repository = params.eNB_Repository + } + echo "eNB_Repository : ${eNB_Repository}" + if (params.eNB_Branch == null) { + eNB_Branch = env.GIT_BRANCH + } else { + eNB_Branch = params.eNB_Branch + } + echo "eNB_Branch : ${eNB_Branch}" + if (params.eNB_CommitID == null) { + eNB_CommitID = env.GIT_COMMIT + } else { + eNB_CommitID = params.eNB_CommitID + } + echo "eNB_CommitID : ${eNB_CommitID}" + if (params.eNB_mergeRequest != null) { + eNB_AllowMergeRequestProcess = params.eNB_mergeRequest + if (eNB_AllowMergeRequestProcess) { + if (params.eNB_TargetBranch != null) { + eNB_TargetBranch = params.eNB_TargetBranch + } else { + eNB_TargetBranch = 'develop' + } + echo "eNB_TargetBranch : ${eNB_TargetBranch}" + } + } +/* + if (params.EPC_IPAddress == null) { + allParametersPresent = false + } + if (params.EPC_Type == null) { + allParametersPresent = false + } + if (params.EPC_SourceCodePath == null) { + allParametersPresent = false + } + if (params.EPC_Credentials == null) { + allParametersPresent = false + } +*/ + if (params.ADB_IPAddress == null) { + allParametersPresent = false + } + if (params.ADB_Credentials == null) { + allParametersPresent = false + } + + if (allParametersPresent) { + echo "All parameters are present" + if (eNB_AllowMergeRequestProcess) { + sh "git fetch" + sh "./ci-scripts/doGitLabMerge.sh --src-branch ${eNB_Branch} --src-commit ${eNB_CommitID} --target-branch ${eNB_TargetBranch} --target-commit latest" + } else { + sh "git fetch" + sh "git checkout -f ${eNB_CommitID}" + } + } else { + echo "Some parameters are missing" + sh "./ci-scripts/fail.sh" + } + } + } + } + stage ("Build and Test") { + steps { + script { + dir ('ci-scripts') { + echo "\u2705 \u001B[32m${testStageName}\u001B[0m" + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'], + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.ADB_Credentials}", usernameVariable: 'ADB_Username', passwordVariable: 'ADB_Password'] + ]) { + sh "python3 main.py --mode=InitiateHtml --eNBRepository=${eNB_Repository} --eNBBranch=${eNB_Branch} --eNBCommitID=${eNB_CommitID} --eNB_AllowMerge=${eNB_AllowMergeRequestProcess} --eNBTargetBranch=${eNB_TargetBranch} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} ${mainPythonAllXmlFiles}" + String[] myXmlTestSuite = testXMLFile.split("\\r?\\n") + for (xmlFile in myXmlTestSuite) { + if (fileExists(xmlFile)) { + try { + sh "python3 main.py --mode=TestUE --UEIPAddress=${params.UE_IPAddress} --eNBRepository=${eNB_Repository} --eNBBranch=${eNB_Branch} --eNBCommitID=${eNB_CommitID} --eNB_AllowMerge=${eNB_AllowMergeRequestProcess} --eNBTargetBranch=${eNB_TargetBranch} --UEUserName=${UE_Username} --eNBPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} --XMLTestFile=${xmlFile}" + } catch (Exception e) { + currentBuild.result = 'FAILURE' + buildStageStatus = false + } + } + } + sh "python3 main.py --mode=FinalizeHtml --finalStatus=${buildStageStatus} --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password}" + } + } + } + } + } + stage('Log Collection') { + parallel { + stage('Log Collection (eNB - Build)') { + steps { + echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' +/* + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] + ]) { + sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" + + echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' + sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("build.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "build.log.${env.BUILD_ID}.zip" + } +*/ + } + } + } + stage('Log Collection (eNB - Run)') { + steps { + echo '\u2705 \u001B[32mLog Collection (eNB - Run)\u001B[0m' +// withCredentials([ +// [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] +// ]) { +// sh "python3 ci-scripts/main.py --mode=LogCollecteNB --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" +// +// echo '\u2705 \u001B[32mLog Transfer (eNB - Run)\u001B[0m' +// sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/enb.log.zip ./enb.log.${env.BUILD_ID}.zip || true" +// } + script { + if(fileExists("enb.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "enb.log.${env.BUILD_ID}.zip" + } + if(fileExists("ci-scripts/test_results.html")) { + sh "mv ci-scripts/test_results.html test_results-${JOB_NAME}.html" + sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: ${BUILD_ID}#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results-${JOB_NAME}.html" + archiveArtifacts "test_results-${JOB_NAME}.html" + } + } + } + } + } + } + } + + post { + always { + script { + if (params.pipelineZipsConsoleLog != null) { + if (params.pipelineZipsConsoleLog) { + echo "Archiving Jenkins console log" + sh "wget --no-check-certificate --no-proxy ${env.JENKINS_URL}/job/${env.JOB_NAME}/${env.BUILD_ID}/consoleText -O consoleText.log || true" + sh "zip -m consoleText.log.${env.BUILD_ID}.zip consoleText.log || true" + if(fileExists("consoleText.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "consoleText.log.${env.BUILD_ID}.zip" + } + } + } + } + } + } +} diff --git a/ci-scripts/main.py b/ci-scripts/main.py index d98bb60eca..f79665eb38 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -2640,8 +2640,8 @@ class SSHConnection(): self.htmlFile.write(' \n') self.htmlFile.write(' \n') - terminate_ue_flag = True if (self.ADBIPAddress != 'none'): + terminate_ue_flag = True self.GetAllUEDevices(terminate_ue_flag) self.GetAllCatMDevices(terminate_ue_flag) self.htmlUEConnected = len(self.UEDevices) @@ -3121,7 +3121,7 @@ elif re.match('^LogCollectIperf$', mode, re.IGNORECASE): sys.exit('Insufficient Parameter') SSH.LogCollectIperf() elif re.match('^InitiateHtml$', mode, re.IGNORECASE): - if (SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == '') and (SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == ''): + if (SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == ''): Usage() sys.exit('Insufficient Parameter') count = 0 From 82b742368ea8c32126fcede47c8207eb22ad2f4e Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 21 Mar 2019 15:57:14 +0100 Subject: [PATCH 11/26] fix Signed-off-by: Raphael Defosseux --- ci-scripts/Jenkinsfile-tmp-ue | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/ci-scripts/Jenkinsfile-tmp-ue b/ci-scripts/Jenkinsfile-tmp-ue index f7e39f7e83..c8508fb9ad 100644 --- a/ci-scripts/Jenkinsfile-tmp-ue +++ b/ci-scripts/Jenkinsfile-tmp-ue @@ -239,21 +239,19 @@ pipeline { stage('Log Collection (eNB - Build)') { steps { echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' -/* - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("build.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "build.log.${env.BUILD_ID}.zip" - } -*/ - } +// withCredentials([ +// [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] +// ]) { +// sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" +// +// echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' +// sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" +// } +// script { +// if(fileExists("build.log.${env.BUILD_ID}.zip")) { +// archiveArtifacts "build.log.${env.BUILD_ID}.zip" +// } +// } } } stage('Log Collection (eNB - Run)') { From 77582d06c56cb9d366450d4e918637fadf23c3c9 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 21 Mar 2019 16:03:26 +0100 Subject: [PATCH 12/26] fix again Signed-off-by: Raphael Defosseux --- ci-scripts/Jenkinsfile-tmp-ue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/Jenkinsfile-tmp-ue b/ci-scripts/Jenkinsfile-tmp-ue index c8508fb9ad..16ba3ef4c6 100644 --- a/ci-scripts/Jenkinsfile-tmp-ue +++ b/ci-scripts/Jenkinsfile-tmp-ue @@ -221,7 +221,7 @@ pipeline { for (xmlFile in myXmlTestSuite) { if (fileExists(xmlFile)) { try { - sh "python3 main.py --mode=TestUE --UEIPAddress=${params.UE_IPAddress} --eNBRepository=${eNB_Repository} --eNBBranch=${eNB_Branch} --eNBCommitID=${eNB_CommitID} --eNB_AllowMerge=${eNB_AllowMergeRequestProcess} --eNBTargetBranch=${eNB_TargetBranch} --UEUserName=${UE_Username} --eNBPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} --XMLTestFile=${xmlFile}" + sh "python3 main.py --mode=TestUE --UEIPAddress=${params.UE_IPAddress} --eNBRepository=${eNB_Repository} --eNBBranch=${eNB_Branch} --eNBCommitID=${eNB_CommitID} --eNB_AllowMerge=${eNB_AllowMergeRequestProcess} --eNBTargetBranch=${eNB_TargetBranch} --UEUserName=${UE_Username} --UEPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} --XMLTestFile=${xmlFile}" } catch (Exception e) { currentBuild.result = 'FAILURE' buildStageStatus = false From f4bdf53c4e1fec9b285e9f303d6177a928b3a49d Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Fri, 22 Mar 2019 09:40:09 +0100 Subject: [PATCH 13/26] Splitting two xml files into three xml files. Signed-off-by: Boris Djalal --- ci-scripts/xml_files/ue_band20_test_10mhz.xml | 25 ------------------- .../xml_files/ue_band20_test_10mhz_orange.xml | 7 +++--- .../xml_files/ue_band20_test_10mhz_sfr.xml | 11 ++++---- 3 files changed, 10 insertions(+), 33 deletions(-) diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz.xml b/ci-scripts/xml_files/ue_band20_test_10mhz.xml index 38e1aa6b7a..eeea2e359c 100644 --- a/ci-scripts/xml_files/ue_band20_test_10mhz.xml +++ b/ci-scripts/xml_files/ue_band20_test_10mhz.xml @@ -23,8 +23,6 @@ 090101 -090102 000001 090104 -090103 000001 090104 @@ -33,27 +31,4 @@ -w USRP --UE - - Initialize_OAI_UE - Initialize OAI UE -- sniffing SFR frequency - -C 806000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect - - - - Initialize_OAI_UE - Initialize OAI UE -- sniffing Orange frequency - -C 816000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect - - - - IdleSleep - Sleep - 30 - - - - Terminate_OAI_UE - Terminate OAI UE - - diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml index 4b5e2137de..f49a26c2f7 100644 --- a/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml +++ b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml @@ -26,9 +26,10 @@ tasks 090101 -090102 000001 090104 -090103 000001 090104 +090102 000001 090109 +090103 000001 090109 + 090101 Build_OAI_UE @@ -54,7 +55,7 @@ 30 - + Terminate_OAI_UE Terminate OAI UE diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml index 481f6b2699..f37df7ef42 100644 --- a/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml +++ b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml @@ -26,9 +26,10 @@ tasks 090101 -090102 000001 090104 -090103 000001 090104 +090104 000001 090109 +090105 000001 090109 + 090101 Build_OAI_UE @@ -36,13 +37,13 @@ -w USRP --UE - + Initialize_OAI_UE Initialize OAI UE -- sniffing SFR frequency -C 806000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect - + Initialize_OAI_UE Initialize OAI UE -- sniffing SFR frequency -C 802000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect @@ -54,7 +55,7 @@ 30 - + Terminate_OAI_UE Terminate OAI UE From d8bc658ab4db31fba46ba94a72b5e89559300386 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Fri, 22 Mar 2019 10:51:11 +0100 Subject: [PATCH 14/26] Adding CollectOAIUE mode. Signed-off-by: Boris Djalal --- ci-scripts/main.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index f79665eb38..480a0ad661 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -2195,8 +2195,8 @@ class SSHConnection(): logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m') return ENB_PROCESS_SEG_FAULT if foundAssertion: - logging.debug('\u001B[1;37;41m UE ended with an assertion! \u001B[0m') - self.htmlUEFailureMsg += msgAssertion + logging.debug('\u001B[1;37;43m UE ended with an assertion! \u001B[0m') + self.htmlUEFailureMsg += msgAssertion return ENB_PROCESS_ASSERTION if foundRealTimeIssue: logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m') @@ -2428,7 +2428,15 @@ class SSHConnection(): self.CreateHtmlTestRow(str(self.idle_sleep_time) + ' sec', 'OK', ALL_PROCESSES_OK) def LogCollectBuild(self): - self.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) + if (self.eNBIPAddress != '' and self.eNBUserName != '' and self.eNBPassword != ''): + self.IPAddress = self.eNBIPAddress + self.UserName = self.eNBUserName + self.Password = self.eNBPassword + elif (self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != ''): + self.IPAddress = self.UEIPAddress + self.UserName = self.UEUserName + self.Password = self.UEPassword + self.open(self.IPAddress, self.UserName, self.Password) self.command('cd ' + self.eNBSourceCodePath, '\$', 5) self.command('cd cmake_targets', '\$', 5) self.command('rm -f build.log.zip', '\$', 5) @@ -2502,6 +2510,15 @@ class SSHConnection(): self.command('zip spgw.log.zip xGwLog.0', '\$', 60) self.close() + def LogCollectOAIUE(self): + self.open(self.UEIPAddress, self.UEUserName, self.UEPassword) + self.command('cd ' + self.UESourceCodePath, '\$', 5) + self.command('cd cmake_targets', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S rm -f ue.log.zip', '\$', 5) + self.command('echo ' + self.UEPassword + ' | sudo -S zip ue.log.zip ue*.log core* ue_*record.raw ue_*.pcap ue_*txt', '\$', 60) + self.command('echo ' + self.UEPassword + ' | sudo -S rm ue*.log core* ue_*record.raw ue_*.pcap ue_*txt', '\$', 5) + self.close() + def RetrieveSystemVersion(self): if self.eNBIPAddress == 'none': self.OsVersion = 'Ubuntu 16.04.5 LTS' @@ -3086,7 +3103,7 @@ elif re.match('^TerminateSPGW$', mode, re.IGNORECASE): sys.exit('Insufficient Parameter') SSH.TerminateSPGW() elif re.match('^LogCollectBuild$', mode, re.IGNORECASE): - if SSH.eNBIPAddress == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '': + if (SSH.eNBIPAddress == '' or SSH.eNBUserName == '' or SSH.eNBPassword == '' or SSH.eNBSourceCodePath == '') and (SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == ''): Usage() sys.exit('Insufficient Parameter') SSH.LogCollectBuild() @@ -3120,6 +3137,11 @@ elif re.match('^LogCollectIperf$', mode, re.IGNORECASE): Usage() sys.exit('Insufficient Parameter') SSH.LogCollectIperf() +elif re.match('^LogCollectOAIUE$', mode, re.IGNORECASE): + if SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '' or SSH.UESourceCodePath == '': + Usage() + sys.exit('Insufficient Parameter') + SSH.LogCollectOAIUE() elif re.match('^InitiateHtml$', mode, re.IGNORECASE): if (SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == ''): Usage() From 0146f8bf59824861819c4469f65dda52d8094dee Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Fri, 22 Mar 2019 11:34:53 +0100 Subject: [PATCH 15/26] Renamed xml_files/ue_band20_test_10mhz.xml to xml_files/ue_band20_build.xml. Signed-off-by: Boris Djalal --- .../xml_files/{ue_band20_test_10mhz.xml => ue_band20_build.xml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ci-scripts/xml_files/{ue_band20_test_10mhz.xml => ue_band20_build.xml} (100%) diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz.xml b/ci-scripts/xml_files/ue_band20_build.xml similarity index 100% rename from ci-scripts/xml_files/ue_band20_test_10mhz.xml rename to ci-scripts/xml_files/ue_band20_build.xml From b53e7e65462868f157859f2a7dd7fb0f4bba7154 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Fri, 22 Mar 2019 11:38:05 +0100 Subject: [PATCH 16/26] Removing 2 frequencies. Signed-off-by: Boris Djalal --- ci-scripts/Jenkinsfile-gitlab | 6 ++++++ ci-scripts/xml_files/ue_band20_build.xml | 4 ++++ ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml | 7 ------- ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml | 7 ------- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab index 5fa46ddcc0..c1675e5992 100644 --- a/ci-scripts/Jenkinsfile-gitlab +++ b/ci-scripts/Jenkinsfile-gitlab @@ -133,6 +133,7 @@ pipeline { script { def message = "OAI " + JOB_NAME + " build (" + BUILD_ID + "): Merge Conflicts -- Cannot perform CI" addGitLabMRComment comment: message + currentBuild.result = 'FAILURE' } } } @@ -283,6 +284,11 @@ pipeline { } } post { + failure { + script { + currentBuild.result = 'FAILURE' + } + } always { script { dir ('archives') { diff --git a/ci-scripts/xml_files/ue_band20_build.xml b/ci-scripts/xml_files/ue_band20_build.xml index eeea2e359c..9ca1a2b234 100644 --- a/ci-scripts/xml_files/ue_band20_build.xml +++ b/ci-scripts/xml_files/ue_band20_build.xml @@ -21,9 +21,13 @@ --> + build-tab + Build + wrench 090101 + Build_OAI_UE diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml index f49a26c2f7..6e5e26c4a5 100644 --- a/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml +++ b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml @@ -27,7 +27,6 @@ 090101 090102 000001 090109 -090103 000001 090109 090101 @@ -43,12 +42,6 @@ -C 816000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect - - Initialize_OAI_UE - Initialize OAI UE -- sniffing Orange frequency - -C 812000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect - - IdleSleep Sleep diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml index f37df7ef42..02b310f9c3 100644 --- a/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml +++ b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml @@ -27,7 +27,6 @@ 090101 090104 000001 090109 -090105 000001 090109 090101 @@ -43,12 +42,6 @@ -C 806000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect - - Initialize_OAI_UE - Initialize OAI UE -- sniffing SFR frequency - -C 802000000 -r 50 --ue-rxgain 120 --ue-scan-carrier --no-L2-connect - - IdleSleep Sleep From 2c2d34dc85abb90c8dcb31cd0260d5505b558dbe Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Fri, 22 Mar 2019 11:59:22 +0100 Subject: [PATCH 17/26] CI: fix when XML file is missing during HtmlInit Signed-off-by: Raphael Defosseux --- ci-scripts/main.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 480a0ad661..3cdfae3af7 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -3147,14 +3147,20 @@ elif re.match('^InitiateHtml$', mode, re.IGNORECASE): Usage() sys.exit('Insufficient Parameter') count = 0 + foundCount = 0 while (count < SSH.nbTestXMLfiles): xml_test_file = cwd + "/" + SSH.testXMLfiles[count] - xmlTree = ET.parse(xml_test_file) - xmlRoot = xmlTree.getroot() - SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-' + str(count))) - SSH.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-' + str(count))) - SSH.htmlTabIcons.append(xmlRoot.findtext('htmlTabIcon',default='info-sign')) + xml_test_file = sys.path[0] + "/" + SSH.testXMLfiles[count] + if (os.path.isfile(xml_test_file)): + xmlTree = ET.parse(xml_test_file) + xmlRoot = xmlTree.getroot() + SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-' + str(count))) + SSH.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-' + str(count))) + SSH.htmlTabIcons.append(xmlRoot.findtext('htmlTabIcon',default='info-sign')) + foundCount += 1 count += 1 + if foundCount != SSH.nbTestXMLfiles: + SSH.nbTestXMLfiles = foundCount SSH.CreateHtmlHeader() elif re.match('^FinalizeHtml$', mode, re.IGNORECASE): SSH.CreateHtmlFooter(SSH.finalStatus) From 290a22e1baa4db042ed593e38845ebdd02f88f09 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Fri, 22 Mar 2019 13:26:39 +0100 Subject: [PATCH 18/26] Adapting log collection in Jenkins script. Signed-off-by: Boris Djalal --- ci-scripts/Jenkinsfile-tmp-ue | 54 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/ci-scripts/Jenkinsfile-tmp-ue b/ci-scripts/Jenkinsfile-tmp-ue index 16ba3ef4c6..617d7c25ae 100644 --- a/ci-scripts/Jenkinsfile-tmp-ue +++ b/ci-scripts/Jenkinsfile-tmp-ue @@ -236,38 +236,38 @@ pipeline { } stage('Log Collection') { parallel { - stage('Log Collection (eNB - Build)') { + stage('Log Collection (OAI UE - Build)') { steps { - echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' -// withCredentials([ -// [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] -// ]) { -// sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" -// -// echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' -// sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" -// } -// script { -// if(fileExists("build.log.${env.BUILD_ID}.zip")) { -// archiveArtifacts "build.log.${env.BUILD_ID}.zip" -// } -// } + echo '\u2705 \u001B[32mLog Collection (OAI UE - Build)\u001B[0m' + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'] + ]) { + sh "python3 ci-scripts/main.py --mode=LogCollectBuild --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath}" + + echo '\u2705 \u001B[32mLog Transfer (UE - Build)\u001B[0m' + sh "sshpass -p \'${UE_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${UE_Username}@${params.UE_IPAddress}:${UE_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" + } + script { + if(fileExists("build.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "build.log.${env.BUILD_ID}.zip" + } + } } } - stage('Log Collection (eNB - Run)') { + stage('Log Collection (OAI UE - Run)') { steps { - echo '\u2705 \u001B[32mLog Collection (eNB - Run)\u001B[0m' -// withCredentials([ -// [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] -// ]) { -// sh "python3 ci-scripts/main.py --mode=LogCollecteNB --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" -// -// echo '\u2705 \u001B[32mLog Transfer (eNB - Run)\u001B[0m' -// sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/enb.log.zip ./enb.log.${env.BUILD_ID}.zip || true" -// } + echo '\u2705 \u001B[32mLog Collection (OAI UE - Run)\u001B[0m' + withCredentials([ + [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'] + ]) { + sh "python3 ci-scripts/main.py --mode=LogCollectUE --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath}" + + echo '\u2705 \u001B[32mLog Transfer (UE - Run)\u001B[0m' + sh "sshpass -p \'${UE_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${UE_Username}@${params.UE_IPAddress}:${UE_SourceCodePath}/cmake_targets/ue.log.zip ./ue.log.${env.BUILD_ID}.zip || true" + } script { - if(fileExists("enb.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "enb.log.${env.BUILD_ID}.zip" + if(fileExists("ue.log.${env.BUILD_ID}.zip")) { + archiveArtifacts "ue.log.${env.BUILD_ID}.zip" } if(fileExists("ci-scripts/test_results.html")) { sh "mv ci-scripts/test_results.html test_results-${JOB_NAME}.html" From a282bea31c46f938cb29e38014efdc287ba5c2f8 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Fri, 22 Mar 2019 13:42:05 +0100 Subject: [PATCH 19/26] fix Signed-off-by: Boris Djalal --- ci-scripts/main.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 3cdfae3af7..aa3c890649 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -2432,16 +2432,20 @@ class SSHConnection(): self.IPAddress = self.eNBIPAddress self.UserName = self.eNBUserName self.Password = self.eNBPassword + self.SourceCodePath = self.eNBSourceCodePath elif (self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != ''): self.IPAddress = self.UEIPAddress self.UserName = self.UEUserName self.Password = self.UEPassword + self.SourceCodePath = self.UESourceCodePath + else: + sys.exit('Insufficient Parameter') self.open(self.IPAddress, self.UserName, self.Password) - self.command('cd ' + self.eNBSourceCodePath, '\$', 5) + self.command('cd ' + self.SourceCodePath, '\$', 5) self.command('cd cmake_targets', '\$', 5) self.command('rm -f build.log.zip', '\$', 5) self.command('zip build.log.zip build_log_*/*', '\$', 60) - self.command('echo ' + self.eNBPassword + ' | sudo -S rm -rf build_log_*', '\$', 5) + self.command('echo ' + self.Password + ' | sudo -S rm -rf build_log_*', '\$', 5) self.close() def LogCollecteNB(self): From b15ad72f846cf3661d058cdb23bb0d3ea3ee01a1 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Fri, 22 Mar 2019 13:54:50 +0100 Subject: [PATCH 20/26] fix Signed-off-by: Boris Djalal --- ci-scripts/Jenkinsfile-tmp-ue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/Jenkinsfile-tmp-ue b/ci-scripts/Jenkinsfile-tmp-ue index 617d7c25ae..70fe29879f 100644 --- a/ci-scripts/Jenkinsfile-tmp-ue +++ b/ci-scripts/Jenkinsfile-tmp-ue @@ -260,7 +260,7 @@ pipeline { withCredentials([ [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.UE_Credentials}", usernameVariable: 'UE_Username', passwordVariable: 'UE_Password'] ]) { - sh "python3 ci-scripts/main.py --mode=LogCollectUE --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath}" + sh "python3 ci-scripts/main.py --mode=LogCollectOAIUE --UEIPAddress=${params.UE_IPAddress} --UEUserName=${UE_Username} --UEPassword=${UE_Password} --UESourceCodePath=${params.UE_SourceCodePath}" echo '\u2705 \u001B[32mLog Transfer (UE - Run)\u001B[0m' sh "sshpass -p \'${UE_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${UE_Username}@${params.UE_IPAddress}:${UE_SourceCodePath}/cmake_targets/ue.log.zip ./ue.log.${env.BUILD_ID}.zip || true" From a7f841ad741cf8ec82eb2f63931072f61693a92c Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Fri, 22 Mar 2019 15:24:12 +0100 Subject: [PATCH 21/26] Printing UE in report and eNB by default. Signed-off-by: Boris Djalal --- ci-scripts/main.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index aa3c890649..851977a375 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -381,7 +381,7 @@ class SSHConnection(): self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) self.command('mv compile_oai_ue.log ' + 'build_log_' + self.testCase_id, '\$', 5) self.close() - self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', ALL_PROCESSES_OK) + self.CreateHtmlTestRow(self.Build_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE') def InitializeHSS(self): @@ -657,7 +657,7 @@ class SSHConnection(): self.close() doLoop = False logging.error('\u001B[1;37;41m UE logging system did not show got sync! \u001B[0m') - self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', ALL_PROCESSES_OK) + self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', ALL_PROCESSES_OK, 'OAI UE') self.CreateHtmlTabFooter(False) ## In case of T tracer recording, we need to kill tshark on EPC side #result = re.search('T_stdout', str(self.Initialize_OAI_UE_args)) @@ -679,7 +679,7 @@ class SSHConnection(): time.sleep(6) else: doLoop = False - self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'OK', ALL_PROCESSES_OK) + self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'OK', ALL_PROCESSES_OK, 'OAI UE') logging.debug('\u001B[1m Initialize OAI UE Completed\u001B[0m') self.close() @@ -2388,7 +2388,10 @@ class SSHConnection(): self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + extracted_log_file, '.') logging.debug('\u001B[1m Analyzing UE replay logfile \u001B[0m') logStatus = self.AnalyzeLogFile_UE(extracted_log_file) - self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) + html_cell = '
    BLA\nBLA BLA '
+			html_cell += '
    ' + html_queue.put(html_cell) + self.CreateHtmlTestRow(html_queue, 'OK', ALL_PROCESSES_OK) self.UELogFile = '' else: result = re.search('ue_', str(self.UELogFile)) @@ -2772,7 +2775,7 @@ class SSHConnection(): self.htmlFile.write('\n') self.htmlFile.close() - def CreateHtmlTestRow(self, options, status, processesStatus): + def CreateHtmlTestRow(self, options, status, processesStatus, machine='eNB'): if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)): self.htmlFile.write(' \n') self.htmlFile.write(' ' + self.testCase_id + '\n') @@ -2784,13 +2787,13 @@ class SSHConnection(): if (processesStatus == 0): self.htmlFile.write(' ' + str(status) + '\n') elif (processesStatus == ENB_PROCESS_FAILED): - self.htmlFile.write(' KO - eNB process not found\n') + self.htmlFile.write(' KO - ' + machine + ' process not found\n') elif (processesStatus == ENB_PROCESS_SEG_FAULT): - self.htmlFile.write(' KO - eNB process ended in Segmentation Fault\n') + self.htmlFile.write(' KO - ' + machine + ' process ended in Segmentation Fault\n') elif (processesStatus == ENB_PROCESS_ASSERTION): - self.htmlFile.write(' KO - eNB process ended in Assertion\n') + self.htmlFile.write(' KO - ' + machine + ' process ended in Assertion\n') elif (processesStatus == ENB_PROCESS_REALTIME_ISSUE): - self.htmlFile.write(' KO - eNB process faced Real Time issue(s)\n') + self.htmlFile.write(' KO - ' + machine + ' process faced Real Time issue(s)\n') elif (processesStatus == ENB_PROCESS_NOLOGFILE_TO_ANALYZE): self.htmlFile.write(' OK\n') elif (processesStatus == HSS_PROCESS_FAILED): From 20b13d64fbac2667a14b44259db2ca9cf532ec03 Mon Sep 17 00:00:00 2001 From: Boris Djalal Date: Fri, 22 Mar 2019 16:17:42 +0100 Subject: [PATCH 22/26] Replacing N/A with success message in cell. Signed-off-by: Boris Djalal --- ci-scripts/main.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 851977a375..e116620e93 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -2388,9 +2388,6 @@ class SSHConnection(): self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + extracted_log_file, '.') logging.debug('\u001B[1m Analyzing UE replay logfile \u001B[0m') logStatus = self.AnalyzeLogFile_UE(extracted_log_file) - html_cell = '
    BLA\nBLA BLA '
-			html_cell += '
    ' - html_queue.put(html_cell) self.CreateHtmlTestRow(html_queue, 'OK', ALL_PROCESSES_OK) self.UELogFile = '' else: @@ -2410,10 +2407,10 @@ class SSHConnection(): self.CreateHtmlTabFooter(False) sys.exit(1) else: - self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK) + self.CreateHtmlTestRow('
    sniffing successful/pre>', 'OK', ALL_PROCESSES_OK)
 				self.UELogFile = ''
 			else:
-				self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
+				self.CreateHtmlTestRow('
    sniffing successful/pre>', 'OK', ALL_PROCESSES_OK)
 
 	def AutoTerminateUEandeNB(self):
 		self.testCase_id = 'AUTO-KILL-UE'

From ee06bb4fd1ee216a919510ecfe7890db82c8d5e1 Mon Sep 17 00:00:00 2001
From: Boris Djalal 
Date: Fri, 22 Mar 2019 16:29:08 +0100
Subject: [PATCH 23/26] typo

Signed-off-by: Boris Djalal 
---
 ci-scripts/main.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci-scripts/main.py b/ci-scripts/main.py
index e116620e93..1c90b6119a 100644
--- a/ci-scripts/main.py
+++ b/ci-scripts/main.py
@@ -2407,10 +2407,10 @@ class SSHConnection():
 					self.CreateHtmlTabFooter(False)
 					sys.exit(1)
 				else:
-					self.CreateHtmlTestRow('
    sniffing successful/pre>', 'OK', ALL_PROCESSES_OK)
+					self.CreateHtmlTestRow('
    sniffing successful
    ', 'OK', ALL_PROCESSES_OK)
 				self.UELogFile = ''
 			else:
-				self.CreateHtmlTestRow('
    sniffing successful/pre>', 'OK', ALL_PROCESSES_OK)
+				self.CreateHtmlTestRow('
    sniffing successful
    ', 'OK', ALL_PROCESSES_OK)
 
 	def AutoTerminateUEandeNB(self):
 		self.testCase_id = 'AUTO-KILL-UE'

From dac4ceb7cba2a477f4712757fa16f9fc02e2fa40 Mon Sep 17 00:00:00 2001
From: Raphael Defosseux 
Date: Fri, 22 Mar 2019 16:48:58 +0100
Subject: [PATCH 24/26] CI: adding call to new slave UE job from master
 pipeline

Signed-off-by: Raphael Defosseux 
---
 ci-scripts/Jenkinsfile-gitlab | 55 +++++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 2 deletions(-)

diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab
index c1675e5992..cd2d783638 100644
--- a/ci-scripts/Jenkinsfile-gitlab
+++ b/ci-scripts/Jenkinsfile-gitlab
@@ -398,7 +398,7 @@ pipeline {
                         }
                     }
                 }
-                stage ("Test FDD - Band 7 - B210") {
+                stage ("Test MONOLITHIC - FDD - Band 7 - B210") {
                     steps {
                         script {
                             if ("MERGE".equals(env.gitlabActionType)) {
@@ -449,7 +449,7 @@ pipeline {
                         }
                     }
                 }
-                stage ("Test TDD - Band 40 - B210") {
+                stage ("Test MONOLITHIC - TDD - Band 40 - B210") {
                     steps {
                         script {
                             if ("MERGE".equals(env.gitlabActionType)) {
@@ -653,6 +653,57 @@ pipeline {
                         }
                     }
                 }
+                stage ("Test OAI UE Sniffing - FDD - Band 20 - B200") {
+                    steps {
+                        script {
+                            if ("MERGE".equals(env.gitlabActionType)) {
+                                //gitlabCommitStatus(name: "Test-OAI-UE-FDD-Band20") {
+                                    build job: 'UE-CI-FDD-Band20-B200',
+                                       parameters: [
+                                           string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)),
+                                           string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)),
+                                           string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)),
+                                           booleanParam(name: 'eNB_mergeRequest', value: true),
+                                           string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch))
+                                       ]
+                                //}
+                            } else {
+                                //gitlabCommitStatus(name: "Test-OAI-UE-FDD-Band20") {
+                                    build job: 'UE-CI-FDD-Band20-B200',
+                                       parameters: [
+                                           string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)),
+                                           string(name: 'eNB_Branch', value: String.valueOf(GIT_BRANCH)),
+                                           string(name: 'eNB_CommitID', value: String.valueOf(GIT_COMMIT)),
+                                           booleanParam(name: 'eNB_mergeRequest', value: false)
+                                       ]
+                                //}
+                            }
+                        }
+                    }
+                    post {
+                        // In case of any non-success, we are retrieving the HTML report of the last completed
+                        // slave job.
+                        // The only drop-back is that we may retrieve the HTML report of a previous build
+                        always {
+                            script {
+                                if (!fileExists('test_results-UE-CI-FDD-Band20-B200.html')) {
+                                    copyArtifacts(projectName: 'UE-CI-FDD-Band20-B200',
+                                                  filter: 'test_results*.html',
+                                                  selector: lastCompleted())
+                                    if (fileExists('test_results-UE-CI-FDD-Band20-B200.html')) {
+                                        sh "sed -i -e 's#TEMPLATE_BUILD_TIME#${JOB_TIMESTAMP}#' test_results-UE-CI-FDD-Band20-B200.html"
+                                        archiveArtifacts artifacts: 'test_results-UE-CI-FDD-Band20-B200.html'
+                                    }
+                                }
+                            }
+                        }
+                        failure {
+                            script {
+                                currentBuild.result = 'FAILURE'
+                            }
+                        }
+                    }
+                }
             }
             post {
                 always {

From 27a8abfe51de12377cc5025e54a63dba52f8fe7b Mon Sep 17 00:00:00 2001
From: Boris Djalal 
Date: Mon, 25 Mar 2019 15:50:19 +0100
Subject: [PATCH 25/26] Removing a wrong change in match TerminateUE. Code
 cleaning.

Signed-off-by: Boris Djalal 
---
 ci-scripts/main.py | 64 +++++++++++++++++++---------------------------
 1 file changed, 27 insertions(+), 37 deletions(-)

diff --git a/ci-scripts/main.py b/ci-scripts/main.py
index 1c90b6119a..430d64a6c0 100644
--- a/ci-scripts/main.py
+++ b/ci-scripts/main.py
@@ -42,6 +42,7 @@ ENB_PROCESS_SEG_FAULT = -11
 ENB_PROCESS_ASSERTION = -12
 ENB_PROCESS_REALTIME_ISSUE = -13
 ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14
+UE_PROCESS_NOLOGFILE_TO_ANALYZE = -20
 HSS_PROCESS_FAILED = -2
 HSS_PROCESS_OK = +2
 MME_PROCESS_FAILED = -3
@@ -129,24 +130,13 @@ class SSHConnection():
 		self.CpuModel = ''
 		self.CpuMHz = ''
 		self.UEIPAddress = ''
-		self.UEBranch = ''
-		#self.UE_AllowMerge = False
-		self.UECommitID = ''
-		#self.UETargetBranch = ''
 		self.UEUserName = ''
 		self.UEPassword = ''
 		self.UE_instance = ''
-		#self.UESourceCodePath = ''
-		#self.UECpuMHz = ''
+		self.UESourceCodePath = ''
 		self.Build_OAI_UE_args = ''
 		self.Initialize_OAI_UE_args = ''
 		self.eNBOsVersion = ''
-		#self.eNBKernelVersion = ''
-		#self.eNBUsrpBoard = ''
-		#self.eNBUhdVersion = ''
-		#self.eNBCpuNb = ''
-		#self.eNBCpuModel = ''
-		#self.eNBCpuMHz = ''
 
 	def open(self, ipaddress, username, password):
 		count = 0
@@ -610,7 +600,7 @@ class SSHConnection():
 			# Reloading FGPA bin firmware
 			self.command('echo ' + self.UEPassword + ' | sudo -S uhd_find_devices', '\$', 5)
 		else:
-			logging.debug('Did not find any B2xx device')		
+			logging.debug('Did not find any B2xx device')
 		self.command('cd ' + self.UESourceCodePath, '\$', 5)
 		# Initialize_OAI_UE_args usually start with -C and followed by the location in repository
 		#full_config_file = self.Initialize_OAI_UE_args.replace('-O ','')
@@ -2044,7 +2034,7 @@ class SSHConnection():
 			self.htmleNBFailureMsg += rrcMsg + '\n'
 		if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
 			rrcMsg = 'eNB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
-			logging.debug('\u001B[1;30;43m ' + rrcMsig + ' \u001B[0m')
+			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
 			self.htmleNBFailureMsg += rrcMsg + '\n'
 			rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
 			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
@@ -2397,7 +2387,7 @@ class SSHConnection():
 				if (copyin_res == -1):
 					logging.debug('\u001B[1;37;41m Could not copy UE logfile to analyze it! \u001B[0m')
 					self.htmlUEFailureMsg = 'Could not copy UE logfile to analyze it!'
-					self.CreateHtmlTestRow('N/A', 'KO', ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
+					self.CreateHtmlTestRow('N/A', 'KO', UE_PROCESS_NOLOGFILE_TO_ANALYZE)
 					self.UELogFile = ''
 					return
 				logging.debug('\u001B[1m Analyzing UE logfile \u001B[0m')
@@ -2429,23 +2419,23 @@ class SSHConnection():
 
 	def LogCollectBuild(self):
 		if (self.eNBIPAddress != '' and self.eNBUserName != '' and self.eNBPassword != ''):
-			self.IPAddress = self.eNBIPAddress
-			self.UserName = self.eNBUserName
-			self.Password = self.eNBPassword
-			self.SourceCodePath = self.eNBSourceCodePath
+			IPAddress = self.eNBIPAddress
+			UserName = self.eNBUserName
+			Password = self.eNBPassword
+			SourceCodePath = self.eNBSourceCodePath
 		elif (self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != ''):
-			self.IPAddress = self.UEIPAddress
-			self.UserName = self.UEUserName
-			self.Password = self.UEPassword
-			self.SourceCodePath = self.UESourceCodePath
+			IPAddress = self.UEIPAddress
+			UserName = self.UEUserName
+			Password = self.UEPassword
+			SourceCodePath = self.UESourceCodePath
 		else:
 			sys.exit('Insufficient Parameter')
-		self.open(self.IPAddress, self.UserName, self.Password)
-		self.command('cd ' + self.SourceCodePath, '\$', 5)
+		self.open(IPAddress, UserName, Password)
+		self.command('cd ' + SourceCodePath, '\$', 5)
 		self.command('cd cmake_targets', '\$', 5)
 		self.command('rm -f build.log.zip', '\$', 5)
 		self.command('zip build.log.zip build_log_*/*', '\$', 60)
-		self.command('echo ' + self.Password + ' | sudo -S rm -rf build_log_*', '\$', 5)
+		self.command('echo ' + Password + ' | sudo -S rm -rf build_log_*', '\$', 5)
 		self.close()
 
 	def LogCollecteNB(self):
@@ -2536,18 +2526,18 @@ class SSHConnection():
 		machine = None
 		if self.eNBIPAddress != '' and self.eNBUserName != '' and self.eNBPassword != '':
 			machine = 'eNB'
-			self.IPAddress = self.eNBIPAddress
-			self.UserName = self.eNBUserName
-			self.Password = self.eNBPassword	
-		if self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != '':
+			IPAddress = self.eNBIPAddress
+			UserName = self.eNBUserName
+			Password = self.eNBPassword
+		elif self.UEIPAddress != '' and self.UEUserName != '' and self.UEPassword != '':
 			machine = 'UE'
-			self.IPAddress = self.UEIPAddress
-			self.UserName = self.UEUserName
-			self.Password = self.UEPassword
+			IPAddress = self.UEIPAddress
+			UserName = self.UEUserName
+			Password = self.UEPassword
 		if machine is None:
 			Usage()
 			sys.exit('Insufficient Parameter')
-		self.open(self.IPAddress, self.UserName, self.Password)
+		self.open(IPAddress, UserName, Password)
 		self.command('lsb_release -a', '\$', 5)
 		result = re.search('Description:\\\\t(?P[a-zA-Z0-9\-\_\.\ ]+)', str(self.ssh.before))
 		if result is not None:
@@ -2563,7 +2553,7 @@ class SSHConnection():
 		if result is not None:
 			self.UhdVersion = result.group('uhd_version')
 			logging.debug('UHD Version is: ' + self.UhdVersion)
-		self.command('echo ' + self.Password + ' | sudo -S uhd_find_devices', '\$', 5)
+		self.command('echo ' + Password + ' | sudo -S uhd_find_devices', '\$', 5)
 		result = re.search('product: (?P[0-9A-Za-z]+)\\\\r\\\\n', str(self.ssh.before))
 		if result is not None:
 			self.UsrpBoard = result.group('usrp_board')
@@ -2791,7 +2781,7 @@ class SSHConnection():
 					self.htmlFile.write('        KO - ' + machine + ' process ended in Assertion\n')
 				elif (processesStatus == ENB_PROCESS_REALTIME_ISSUE):
 					self.htmlFile.write('        KO - ' + machine + ' process faced Real Time issue(s)\n')
-				elif (processesStatus == ENB_PROCESS_NOLOGFILE_TO_ANALYZE):
+				elif (processesStatus == ENB_PROCESS_NOLOGFILE_TO_ANALYZE) or (processesStatus == UE_PROCESS_NOLOGFILE_TO_ANALYZE):
 					self.htmlFile.write('        OK\n')
 				elif (processesStatus == HSS_PROCESS_FAILED):
 					self.htmlFile.write('        KO - HSS process not found\n')
@@ -3080,7 +3070,7 @@ if re.match('^TerminateeNB$', mode, re.IGNORECASE):
 		sys.exit('Insufficient Parameter')
 	SSH.TerminateeNB()
 elif re.match('^TerminateUE$', mode, re.IGNORECASE):
-	if SSH.UEIPAddress == '' or SSH.UEUserName == '' or SSH.UEPassword == '':
+	if (SSH.ADBIPAddress == '' or SSH.ADBUserName == '' or SSH.ADBPassword == ''):
 		Usage()
 		sys.exit('Insufficient Parameter')
 	signal.signal(signal.SIGUSR1, receive_signal)

From 783583880d69a2aded3ccccef2acb790b948e076 Mon Sep 17 00:00:00 2001
From: Raphael Defosseux 
Date: Wed, 27 Mar 2019 15:06:01 +0100
Subject: [PATCH 26/26] CI: improved HTML reporting on OAI UE

Signed-off-by: Raphael Defosseux 
---
 ci-scripts/main.py                            | 126 +++++++++++-------
 .../xml_files/ue_band20_test_10mhz_orange.xml |   9 +-
 .../xml_files/ue_band20_test_10mhz_sfr.xml    |   9 +-
 3 files changed, 83 insertions(+), 61 deletions(-)

diff --git a/ci-scripts/main.py b/ci-scripts/main.py
index 430d64a6c0..96d7397e46 100644
--- a/ci-scripts/main.py
+++ b/ci-scripts/main.py
@@ -43,6 +43,8 @@ ENB_PROCESS_ASSERTION = -12
 ENB_PROCESS_REALTIME_ISSUE = -13
 ENB_PROCESS_NOLOGFILE_TO_ANALYZE = -14
 UE_PROCESS_NOLOGFILE_TO_ANALYZE = -20
+UE_PROCESS_COULD_NOT_SYNC = -21
+UE_PROCESS_ASSERTION = -22
 HSS_PROCESS_FAILED = -2
 HSS_PROCESS_OK = +2
 MME_PROCESS_FAILED = -3
@@ -332,8 +334,6 @@ class SSHConnection():
 		self.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
 		self.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
 		self.command('mv compile_oai_enb.log ' + 'build_log_' + self.testCase_id, '\$', 5)
-		# Workaround to run with develop-nr
-		self.command('if [ -e ran_build ]; then cp -rf ran_build lte_build_oai; fi', '\$', 30)
 		self.close()
 		self.CreateHtmlTestRow(self.Build_eNB_args, 'OK', ALL_PROCESSES_OK)
 
@@ -2086,6 +2086,9 @@ class SSHConnection():
 		pdcpFailure = 0
 		ulschFailure = 0
 		no_cell_sync_found = False
+		mib_found = False
+		frequency_found = False
+		self.htmlUEFailureMsg = ''
 		for line in ue_log_file.readlines():
 			result = re.search('[Ss]egmentation [Ff]ault', str(line))
 			if result is not None:
@@ -2105,66 +2108,78 @@ class SSHConnection():
 			result = re.search('uci->stat', str(line))
 			if result is not None:
 				uciStatMsgCount += 1
-			# full pattern
 			# No cell synchronization found, abandoning
 			result = re.search('No cell synchronization found, abandoning', str(line))
 			if result is not None:
 				no_cell_sync_found = True
-			# MIB Information => FDD, NORMAL, NidCell 421, N_RB_DL 50, PHICH DURATION 0, PHICH RESOURCE 1/6, TX_ANT 2
-			# mask --> FDD  NORMAL 421 50  0 1/6 2
 			result = re.search("MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P\d{1,3}), N_RB_DL (?P\d{1,3}), PHICH DURATION (?P\d), PHICH RESOURCE (?P.{1,4}), TX_ANT (?P\d)", str(line))
-			if result is not None:
+			if result is not None and (not mib_found):
 				try:
-					print('\033[94m' + "MIB Information: " + result.group(1) + ', ' + result.group(2) + '\033[0m')
-					print('\033[94m' + "nidcell = " + result.group('nidcell') + '\033[0m')
-					print('\033[94m' + "n_rb_dl = " + result.group('n_rb_dl') + '\033[0m')
-					print('\033[94m' + "phich_duration = " + result.group('phich_duration') + '\033[0m')
-					print('\033[94m' + "phich_resource = " + result.group('phich_resource') + '\033[0m')
-					print('\033[94m' + "tx_ant = " + result.group('tx_ant') + '\033[0m')
+					mibMsg = "MIB Information: " + result.group(1) + ', ' + result.group(2)
+					self.htmlUEFailureMsg += mibMsg + '\n'
+					logging.debug('\033[94m' + mibMsg + '\033[0m')
+					mibMsg = "    nidcell = " + result.group('nidcell')
+					self.htmlUEFailureMsg += mibMsg
+					logging.debug('\033[94m' + mibMsg + '\033[0m')
+					mibMsg = "    n_rb_dl = " + result.group('n_rb_dl')
+					self.htmlUEFailureMsg += mibMsg + '\n'
+					logging.debug('\033[94m' + mibMsg + '\033[0m')
+					mibMsg = "    phich_duration = " + result.group('phich_duration')
+					self.htmlUEFailureMsg += mibMsg
+					logging.debug('\033[94m' + mibMsg + '\033[0m')
+					mibMsg = "    phich_resource = " + result.group('phich_resource')
+					self.htmlUEFailureMsg += mibMsg + '\n'
+					logging.debug('\033[94m' + mibMsg + '\033[0m')
+					mibMsg = "    tx_ant = " + result.group('tx_ant')
+					self.htmlUEFailureMsg += mibMsg + '\n'
+					logging.debug('\033[94m' + mibMsg + '\033[0m')
+					mib_found = True
 				except Exception as e:
-					print('\033[91m' + "a marker was not found" + '\033[0m')
-			# Measured Carrier Frequency 816000688 Hz (offset 12 Hz)
-			# mask --> 816000688i
+					logging.error('\033[91m' + "MIB marker was not found" + '\033[0m')
 			result = re.search("Measured Carrier Frequency (?P\d{1,15}) Hz", str(line))
-			if result is not None:
+			if result is not None and (not frequency_found):
 				try:
-					print('\033[94m' + "Measured Carrier Frequency = " + result.group('measured_carrier_frequency') + '\033[0m')
+					mibMsg = "Measured Carrier Frequency = " + result.group('measured_carrier_frequency') + ' Hz'
+					self.htmlUEFailureMsg += mibMsg + '\n'
+					logging.debug('\033[94m' + mibMsg + '\033[0m')
+					frequency_found = True
 				except Exception as e:
-					print('\033[91m' + "Measured Carrier Frequency not found" + '\033[0m')
-			# Found Orange FR (name from internal table)
-			# mask Orange FR
+					logging.error('\033[91m' + "Measured Carrier Frequency not found" + '\033[0m')
 			result = re.search("Found (?P[\w,\s]{1,15}) \(name from internal table\)", str(line))
 			if result is not None:
 				try:
-					print('\033[94m' + "The operator is: " + result.group('operator') + '\033[0m')
+					mibMsg = "The operator is: " + result.group('operator')
+					self.htmlUEFailureMsg += mibMsg + '\n'
+					logging.debug('\033[94m' + mibMsg + '\033[0m')
 				except Exception as e:
-					print('\033[91m' + "Operator name not found" + '\033[0m')
-			# SIB5 InterFreqCarrierFreq element 0/3
-			# mask -- 0 and 3
+					logging.error('\033[91m' + "Operator name not found" + '\033[0m')
 			result = re.search("SIB5 InterFreqCarrierFreq element (.{1,4})/(.{1,4})", str(line))
 			if result is not None:
 				try:
-					print('\033[94m' + "SIB5 InterFreqCarrierFreq element " + result.group(1) + '/' + result.group(2) + '\033[0m')
+					mibMsg = "SIB5 InterFreqCarrierFreq element " + result.group(1) + '/' + result.group(2)
+					self.htmlUEFailureMsg += mibMsg + ' -> '
+					logging.debug('\033[94m' + mibMsg + '\033[0m')
 				except Exception as e:
-					print('\033[91m' + "SIB5 InterFreqCarrierFreq element not found" + '\033[0m')
-			# DL Carrier Frequency/ARFCN : 2645000000/3000
-			# mask 2645000000
-			result = re.search("DL Carrier Frequency/ARFCN : (?P)\d{1,15}/\d{1,4}", str(line))
+					logging.error('\033[91m' + "SIB5 InterFreqCarrierFreq element not found" + '\033[0m')
+			result = re.search("DL Carrier Frequency/ARFCN : (?P\d{1,15}/\d{1,4})", str(line))
 			if result is not None:
 				try:
-					print('\033[94m' + "DL Carrier Frequency is: " + result.group('carrier_frequency') + '\033[0m')
+					freq = result.group('carrier_frequency')
+					new_freq = re.sub('/[0-9]+','',freq)
+					float_freq = float(new_freq) / 1000000
+					self.htmlUEFailureMsg += 'DL Freq: ' + ('%.1f' % float_freq) + ' MHz'
+					logging.debug('\033[94m' + "    DL Carrier Frequency is: " + freq + '\033[0m')
 				except Exception as e:
-					print('\033[91m' + "DL Carrier Frequency not found" + '\033[0m')
-			# AllowedMeasBandwidth : 100
-			# mask 100
+					logging.error('\033[91m' + "    DL Carrier Frequency not found" + '\033[0m')
 			result = re.search("AllowedMeasBandwidth : (?P\d{1,7})", str(line))
 			if result is not None:
 				try:
-					print('\033[94m' + "AllowedMeasBandwidth: " + result.group('allowed_bandwidth') + '\033[0m')
+					prb = result.group('allowed_bandwidth')
+					self.htmlUEFailureMsg += ' -- PRB: ' + prb + '\n'
+					logging.debug('\033[94m' + "    AllowedMeasBandwidth: " + prb + '\033[0m')
 				except Exception as e:
-					print('\033[91m' + "AllowedMeasBandwidth not found" + '\033[0m')
+					logging.error('\033[91m' + "    AllowedMeasBandwidth not found" + '\033[0m')
 		ue_log_file.close()
-		self.htmlUEFailureMsg = ''
 		if uciStatMsgCount > 0:
 			statMsg = 'UE showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
 			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
@@ -2186,12 +2201,19 @@ class SSHConnection():
 			return ENB_PROCESS_SEG_FAULT
 		if foundAssertion:
 			logging.debug('\u001B[1;37;43m UE ended with an assertion! \u001B[0m')
-			self.htmlUEFailureMsg += msgAssertion  
-			return ENB_PROCESS_ASSERTION
+			# removed for esthetics
+			#self.htmlUEFailureMsg += msgAssertion
+			self.htmlUEFailureMsg += 'UE ended with an assertion!\n'
+			if not mib_found or not frequency_found:
+				return UE_PROCESS_ASSERTION
 		if foundRealTimeIssue:
 			logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m')
 			self.htmlUEFailureMsg += 'UE faced real time issues!\n'
 			#return ENB_PROCESS_REALTIME_ISSUE
+		if no_cell_sync_found and not mib_found:
+			logging.debug('\u001B[1;37;41m UE could not synchronize ! \u001B[0m')
+			self.htmlUEFailureMsg += 'UE could not synchronize!\n'
+			return UE_PROCESS_COULD_NOT_SYNC
 		if rlcDiscardBuffer > 0:
 			rlcMsg = 'UE RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
 			logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m')
@@ -2386,21 +2408,27 @@ class SSHConnection():
 				copyin_res = self.copyin(self.UEIPAddress, self.UEUserName, self.UEPassword, self.UESourceCodePath + '/cmake_targets/' + self.UELogFile, '.')
 				if (copyin_res == -1):
 					logging.debug('\u001B[1;37;41m Could not copy UE logfile to analyze it! \u001B[0m')
-					self.htmlUEFailureMsg = 'Could not copy UE logfile to analyze it!'
-					self.CreateHtmlTestRow('N/A', 'KO', UE_PROCESS_NOLOGFILE_TO_ANALYZE)
+					optionsMsg = '
    Could not copy UE logfile to analyze it!
    '
+					self.CreateHtmlTestRow(optionsMsg, 'KO', UE_PROCESS_NOLOGFILE_TO_ANALYZE, 'UE')
 					self.UELogFile = ''
 					return
 				logging.debug('\u001B[1m Analyzing UE logfile \u001B[0m')
 				logStatus = self.AnalyzeLogFile_UE(self.UELogFile)
 				if (logStatus < 0):
-					self.CreateHtmlTestRow('N/A', 'KO', logStatus)
+					optionsMsg = '
    Sniffing Unsuccessful\n'
+					optionsMsg += self.htmlUEFailureMsg
+					optionsMsg += '
    '
+					self.CreateHtmlTestRow(optionsMsg, 'KO', logStatus, 'UE')
 					self.CreateHtmlTabFooter(False)
 					sys.exit(1)
 				else:
-					self.CreateHtmlTestRow('
    sniffing successful
    ', 'OK', ALL_PROCESSES_OK)
+					optionsMsg = '
    Sniffing Successful\n'
+					optionsMsg += self.htmlUEFailureMsg
+					optionsMsg += '
    '
+					self.CreateHtmlTestRow(optionsMsg, 'OK', ALL_PROCESSES_OK)
 				self.UELogFile = ''
 			else:
-				self.CreateHtmlTestRow('
    sniffing successful
    ', 'OK', ALL_PROCESSES_OK)
+				self.CreateHtmlTestRow('
    No Log File to analyze
    ', 'OK', ALL_PROCESSES_OK)
 
 	def AutoTerminateUEandeNB(self):
 		self.testCase_id = 'AUTO-KILL-UE'
@@ -2574,6 +2602,9 @@ class SSHConnection():
 #-----------------------------------------------------------
 	def CreateHtmlHeader(self):
 		if (not self.htmlHeaderCreated):
+			logging.debug('\u001B[1m----------------------------------------\u001B[0m')
+			logging.debug('\u001B[1m  Creating HTML header \u001B[0m')
+			logging.debug('\u001B[1m----------------------------------------\u001B[0m')
 			self.htmlFile = open('test_results.html', 'w')
 			self.htmlFile.write('\n')
 			self.htmlFile.write('\n')
@@ -2720,6 +2751,9 @@ class SSHConnection():
 
 	def CreateHtmlFooter(self, passStatus):
 		if (os.path.isfile('test_results.html')):
+			logging.debug('\u001B[1m----------------------------------------\u001B[0m')
+			logging.debug('\u001B[1m  Creating HTML footer \u001B[0m')
+			logging.debug('\u001B[1m----------------------------------------\u001B[0m')
 			self.RetrieveSystemVersion()
 			self.htmlFile = open('test_results.html', 'a')
 			self.htmlFile.write('\n')
@@ -2777,12 +2811,14 @@ class SSHConnection():
 					self.htmlFile.write('        KO - ' + machine + ' process not found\n')
 				elif (processesStatus == ENB_PROCESS_SEG_FAULT):
 					self.htmlFile.write('        KO - ' + machine + ' process ended in Segmentation Fault\n')
-				elif (processesStatus == ENB_PROCESS_ASSERTION):
+				elif (processesStatus == ENB_PROCESS_ASSERTION) or (processesStatus == UE_PROCESS_ASSERTION):
 					self.htmlFile.write('        KO - ' + machine + ' process ended in Assertion\n')
 				elif (processesStatus == ENB_PROCESS_REALTIME_ISSUE):
 					self.htmlFile.write('        KO - ' + machine + ' process faced Real Time issue(s)\n')
 				elif (processesStatus == ENB_PROCESS_NOLOGFILE_TO_ANALYZE) or (processesStatus == UE_PROCESS_NOLOGFILE_TO_ANALYZE):
-					self.htmlFile.write('        OK\n')
+					self.htmlFile.write('        OK?\n')
+				elif (processesStatus == UE_PROCESS_COULD_NOT_SYNC):
+					self.htmlFile.write('        KO - UE could not sync\n')
 				elif (processesStatus == HSS_PROCESS_FAILED):
 					self.htmlFile.write('        KO - HSS process not found\n')
 				elif (processesStatus == MME_PROCESS_FAILED):
diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml
index 6e5e26c4a5..0e61e75d56 100644
--- a/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml
+++ b/ci-scripts/xml_files/ue_band20_test_10mhz_orange.xml
@@ -25,16 +25,9 @@
         Test-10Mhz-Orange
         tasks	
 	
-090101
 090102 000001 090109
 	
-	090101
-
-	
-		Build_OAI_UE
-		Build OAI UE
-		-w USRP --UE
-	
+	
 
 	
 		Initialize_OAI_UE
diff --git a/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml
index 02b310f9c3..ae8aa0e668 100644
--- a/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml
+++ b/ci-scripts/xml_files/ue_band20_test_10mhz_sfr.xml
@@ -25,16 +25,9 @@
         Test-10MHz-SFR
         tasks
 	
-090101
 090104 000001 090109
 	
-	090101
-
-	
-		Build_OAI_UE
-		Build OAI UE
-		-w USRP --UE
-	
+	
 
 	
 		Initialize_OAI_UE