CI: adding support to repeat scenario in case of failure.

nice when the tested feature is known to be unstable
    CDRX testing is now separated from normal FDD 10MHz testing

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>
This commit is contained in:
Raphael Defosseux
2019-07-18 22:40:12 +02:00
parent 2d6d1dd965
commit 65002b7fb1
9 changed files with 236 additions and 171 deletions
+137 -132
View File
@@ -87,6 +87,7 @@ logging.basicConfig(
#-----------------------------------------------------------
class SSHConnection():
def __init__(self):
self.prematureExit = False
self.ranRepository = ''
self.ranBranch = ''
self.ranAllowMerge = False
@@ -146,6 +147,7 @@ class SSHConnection():
self.htmlTabRefs = []
self.htmlTabNames = []
self.htmlTabIcons = []
self.repeatCounts = []
self.finalStatus = False
self.OsVersion = ''
self.KernelVersion = ''
@@ -745,7 +747,6 @@ class SSHConnection():
doLoop = False
logging.error('\u001B[1;37;41m eNB logging system did not show got sync! \u001B[0m')
self.CreateHtmlTestRow('-O ' + config_file + extra_options, 'KO', ALL_PROCESSES_OK)
self.CreateHtmlTabFooter(False)
# In case of T tracer recording, we need to kill tshark on EPC side
result = re.search('T_stdout', str(self.Initialize_eNB_args))
if result is not None:
@@ -761,7 +762,8 @@ class SSHConnection():
copyin_res = self.copyin(self.EPCIPAddress, self.EPCUserName, self.EPCPassword, '/tmp/' + self.EPC_PcapFileName, '.')
if (copyin_res == 0):
self.copyout(lIpAddr, lUserName, lPassWord, self.EPC_PcapFileName, lSourcePath + '/cmake_targets/.')
sys.exit(1)
self.prematureExit = True
return
else:
self.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting"', '\$', 4)
if rruCheck:
@@ -938,8 +940,6 @@ class SSHConnection():
self.CreateHtmlTestRow(self.Initialize_OAI_UE_args, 'KO', OAI_UE_PROCESS_NO_TUNNEL_INTERFACE, 'OAI UE')
logging.error('\033[91mInitialize OAI UE Failed! \033[0m')
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def checkDevTTYisUnlocked(self):
self.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
@@ -1090,8 +1090,8 @@ class SSHConnection():
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE)
if (pStatus < 0):
self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.CreateHtmlTabFooter(False)
sys.exit(1)
self.prematureExit = True
return
try:
statusQueue = SimpleQueue()
lock = Lock()
@@ -1173,8 +1173,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
except:
os.kill(os.getppid(),signal.SIGUSR1)
@@ -1235,8 +1233,7 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow('N/A', 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
multi_jobs = []
status_queue = SimpleQueue()
lock = Lock()
@@ -1254,9 +1251,8 @@ class SSHConnection():
if (status_queue.empty()):
self.CreateHtmlTestRow('N/A', 'KO', ALL_PROCESSES_OK)
self.CreateHtmlTabFooter(False)
self.AutoTerminateUEandeNB()
sys.exit(1)
return
else:
attach_status = True
html_queue = SimpleQueue()
@@ -1285,8 +1281,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def DetachUE_common(self, device_id):
try:
@@ -1307,8 +1301,7 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow('N/A', 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
multi_jobs = []
cnt = 0
for device_id in self.UEDevices:
@@ -1560,8 +1553,6 @@ class SSHConnection():
if (status_queue.empty()):
self.CreateHtmlTestRow(htmlOptions, 'KO', ALL_PROCESSES_OK)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
else:
check_status = True
html_queue = SimpleQueue()
@@ -1578,8 +1569,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue(htmlOptions, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def GetAllUEIPAddresses(self):
if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
@@ -1725,8 +1714,7 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args))
if ping_from_eNB is not None:
if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '':
@@ -1829,14 +1817,12 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
ueIpStatus = self.GetAllUEIPAddresses()
if (ueIpStatus < 0):
self.CreateHtmlTestRow(self.ping_args, 'KO', UE_IP_ADDRESS_ISSUE)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
multi_jobs = []
i = 0
lock = Lock()
@@ -1854,8 +1840,6 @@ class SSHConnection():
if (status_queue.empty()):
self.CreateHtmlTestRow(self.ping_args, 'KO', ALL_PROCESSES_OK)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
else:
ping_status = True
html_queue = SimpleQueue()
@@ -1873,8 +1857,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def Iperf_ComputeTime(self):
result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args))
@@ -2352,8 +2334,6 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
if self.eNBIPAddress == '' or self.eNBUserName == '' or self.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
Usage()
sys.exit('Insufficient Parameter')
@@ -2443,8 +2423,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def Iperf(self):
result = re.search('noS1', str(self.Initialize_eNB_args))
@@ -2463,14 +2441,12 @@ class SSHConnection():
if (pStatus < 0):
self.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
ueIpStatus = self.GetAllUEIPAddresses()
if (ueIpStatus < 0):
self.CreateHtmlTestRow(self.iperf_args, 'KO', UE_IP_ADDRESS_ISSUE)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
return
multi_jobs = []
i = 0
ue_num = len(self.UEIPAddresses)
@@ -2489,8 +2465,6 @@ class SSHConnection():
if (status_queue.empty()):
self.CreateHtmlTestRow(self.iperf_args, 'KO', ALL_PROCESSES_OK)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
else:
iperf_status = True
iperf_noperf = False
@@ -2513,8 +2487,6 @@ class SSHConnection():
else:
self.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
def CheckProcessExist(self, check_eNB, check_OAI_UE):
multi_jobs = []
@@ -2834,10 +2806,17 @@ class SSHConnection():
rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
self.htmleNBFailureMsg += rrcMsg + '\n'
if cdrxActivationMessageCount > 0:
rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
self.htmleNBFailureMsg += rrcMsg + '\n'
if self.eNBOptions[int(self.eNB_instance)] != '':
res1 = re.search('drx_Config_present prSetup', self.eNBOptions[int(self.eNB_instance)])
if res1 is not None:
if cdrxActivationMessageCount > 0:
rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
self.htmleNBFailureMsg += rrcMsg + '\n'
else:
rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration'
logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
self.htmleNBFailureMsg += rrcMsg + '\n'
if rachCanceledProcedure > 0:
rachMsg = 'eNB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
@@ -2852,6 +2831,7 @@ class SSHConnection():
rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')
self.htmleNBFailureMsg += rruMsg + '\n'
self.prematureExit = True
return ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
if foundSegFault:
logging.debug('\u001B[1;37;41m eNB ended with a Segmentation Fault! \u001B[0m')
@@ -3121,8 +3101,8 @@ class SSHConnection():
logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze)
if (logStatus < 0):
self.CreateHtmlTestRow('N/A', 'KO', logStatus)
self.CreateHtmlTabFooter(False)
sys.exit(1)
self.preamtureExit = True
return
else:
self.CreateHtmlTestRow('N/A', 'OK', ALL_PROCESSES_OK)
else:
@@ -3261,8 +3241,6 @@ class SSHConnection():
if (logStatus != OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'):
self.Initialize_OAI_UE_args = ''
self.AutoTerminateUEandeNB()
self.CreateHtmlTabFooter(False)
sys.exit(1)
else:
logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m')
self.htmlUEFailureMsg = '<b>' + ueAction + ' Completed</b>\n' + self.htmlUEFailureMsg
@@ -3288,6 +3266,7 @@ class SSHConnection():
self.ShowTestID()
self.eNB_instance = '0'
self.TerminateeNB()
self.prematureExit = True
def IdleSleep(self):
time.sleep(self.idle_sleep_time)
@@ -3548,6 +3527,7 @@ class SSHConnection():
pillMsg = ' <li><a data-toggle="pill" href="#'
pillMsg += self.htmlTabRefs[count]
pillMsg += '">'
pillMsg += '__STATE_' + self.htmlTabNames[count] + '__'
pillMsg += self.htmlTabNames[count]
pillMsg += ' <span class="glyphicon glyphicon-'
pillMsg += self.htmlTabIcons[count]
@@ -3601,6 +3581,14 @@ class SSHConnection():
self.htmlFile.write(' </tr>\n')
self.htmlFile.write(' </table>\n')
self.htmlFile.write(' </div>\n')
self.htmlFile.close()
time.sleep(1)
if passStatus:
cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__//' test_results.html"
subprocess.run(cmd, shell=True)
else:
cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<span class=\"glyphicon glyphicon-remove\"><\/span>/' test_results.html"
subprocess.run(cmd, shell=True)
self.htmlFooterCreated = False
def CreateHtmlFooter(self, passStatus):
@@ -4178,6 +4166,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
requested_tests=xmlRoot.findtext('TestCaseRequestedList',default='')
if (SSH.nbTestXMLfiles == 1):
SSH.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-0'))
SSH.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-0'))
repeatCount = xmlRoot.findtext('repeatCount',default='1')
SSH.repeatCounts.append(int(repeatCount))
all_tests=xmlRoot.findall('testCase')
exclusion_tests=exclusion_tests.split()
@@ -4219,88 +4210,102 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
SSH.CreateHtmlTabHeader()
for test_case_id in todo_tests:
for test in all_tests:
id = test.get('id')
if test_case_id != id:
continue
SSH.testCase_id = id
SSH.desc = test.findtext('desc')
action = test.findtext('class')
if (CheckClassValidity(action, id) == False):
continue
SSH.ShowTestID()
GetParametersFromXML(action)
if action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Ping' or action == 'Iperf' or action == 'Reboot_UE' or action == 'DataDisable_UE' or action == 'DataEnable_UE' or action == 'CheckStatusUE':
if (SSH.ADBIPAddress != 'none'):
terminate_ue_flag = False
SSH.GetAllUEDevices(terminate_ue_flag)
if action == 'Build_eNB':
SSH.BuildeNB()
elif action == 'WaitEndBuild_eNB':
SSH.WaitBuildeNBisFinished()
elif action == 'Initialize_eNB':
SSH.InitializeeNB()
elif action == 'Terminate_eNB':
SSH.TerminateeNB()
elif action == 'Initialize_UE':
SSH.InitializeUE()
elif action == 'Terminate_UE':
SSH.TerminateUE()
elif action == 'Attach_UE':
SSH.AttachUE()
elif action == 'Detach_UE':
SSH.DetachUE()
elif action == 'DataDisable_UE':
SSH.DataDisableUE()
elif action == 'DataEnable_UE':
SSH.DataEnableUE()
elif action == 'CheckStatusUE':
SSH.CheckStatusUE()
elif action == 'Build_OAI_UE':
SSH.BuildOAIUE()
elif action == 'Initialize_OAI_UE':
SSH.InitializeOAIUE()
elif action == 'Terminate_OAI_UE':
SSH.TerminateOAIUE()
elif action == 'Initialize_CatM_module':
SSH.InitializeCatM()
elif action == 'Terminate_CatM_module':
SSH.TerminateCatM()
elif action == 'Attach_CatM_module':
SSH.AttachCatM()
elif action == 'Detach_CatM_module':
SSH.TerminateCatM()
elif action == 'Ping_CatM_module':
SSH.PingCatM()
elif action == 'Ping':
SSH.Ping()
elif action == 'Iperf':
SSH.Iperf()
elif action == 'Reboot_UE':
SSH.RebootUE()
elif action == 'Initialize_HSS':
SSH.InitializeHSS()
elif action == 'Terminate_HSS':
SSH.TerminateHSS()
elif action == 'Initialize_MME':
SSH.InitializeMME()
elif action == 'Terminate_MME':
SSH.TerminateMME()
elif action == 'Initialize_SPGW':
SSH.InitializeSPGW()
elif action == 'Terminate_SPGW':
SSH.TerminateSPGW()
elif action == 'Initialize_FlexranCtrl':
SSH.InitializeFlexranCtrl()
elif action == 'Terminate_FlexranCtrl':
SSH.TerminateFlexranCtrl()
elif action == 'IdleSleep':
SSH.IdleSleep()
else:
sys.exit('Invalid action')
SSH.CreateHtmlTabFooter(True)
cnt = 0
SSH.prematureExit = True
while cnt < SSH.repeatCounts[0] and SSH.prematureExit:
SSH.prematureExit = False
for test_case_id in todo_tests:
for test in all_tests:
id = test.get('id')
if test_case_id != id:
continue
SSH.testCase_id = id
SSH.desc = test.findtext('desc')
action = test.findtext('class')
if (CheckClassValidity(action, id) == False):
continue
SSH.ShowTestID()
GetParametersFromXML(action)
if action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Ping' or action == 'Iperf' or action == 'Reboot_UE' or action == 'DataDisable_UE' or action == 'DataEnable_UE' or action == 'CheckStatusUE':
if (SSH.ADBIPAddress != 'none'):
terminate_ue_flag = False
SSH.GetAllUEDevices(terminate_ue_flag)
if action == 'Build_eNB':
SSH.BuildeNB()
elif action == 'WaitEndBuild_eNB':
SSH.WaitBuildeNBisFinished()
elif action == 'Initialize_eNB':
SSH.InitializeeNB()
elif action == 'Terminate_eNB':
SSH.TerminateeNB()
elif action == 'Initialize_UE':
SSH.InitializeUE()
elif action == 'Terminate_UE':
SSH.TerminateUE()
elif action == 'Attach_UE':
SSH.AttachUE()
elif action == 'Detach_UE':
SSH.DetachUE()
elif action == 'DataDisable_UE':
SSH.DataDisableUE()
elif action == 'DataEnable_UE':
SSH.DataEnableUE()
elif action == 'CheckStatusUE':
SSH.CheckStatusUE()
elif action == 'Build_OAI_UE':
SSH.BuildOAIUE()
elif action == 'Initialize_OAI_UE':
SSH.InitializeOAIUE()
elif action == 'Terminate_OAI_UE':
SSH.TerminateOAIUE()
elif action == 'Initialize_CatM_module':
SSH.InitializeCatM()
elif action == 'Terminate_CatM_module':
SSH.TerminateCatM()
elif action == 'Attach_CatM_module':
SSH.AttachCatM()
elif action == 'Detach_CatM_module':
SSH.TerminateCatM()
elif action == 'Ping_CatM_module':
SSH.PingCatM()
elif action == 'Ping':
SSH.Ping()
elif action == 'Iperf':
SSH.Iperf()
elif action == 'Reboot_UE':
SSH.RebootUE()
elif action == 'Initialize_HSS':
SSH.InitializeHSS()
elif action == 'Terminate_HSS':
SSH.TerminateHSS()
elif action == 'Initialize_MME':
SSH.InitializeMME()
elif action == 'Terminate_MME':
SSH.TerminateMME()
elif action == 'Initialize_SPGW':
SSH.InitializeSPGW()
elif action == 'Terminate_SPGW':
SSH.TerminateSPGW()
elif action == 'Initialize_FlexranCtrl':
SSH.InitializeFlexranCtrl()
elif action == 'Terminate_FlexranCtrl':
SSH.TerminateFlexranCtrl()
elif action == 'IdleSleep':
SSH.IdleSleep()
else:
sys.exit('Invalid action')
if SSH.prematureExit:
break
if SSH.prematureExit:
break
cnt += 1
if cnt == SSH.repeatCounts[0] and SSH.prematureExit:
logging.debug('Testsuite failed ' + str(cnt) + ' time(s)')
SSH.CreateHtmlTabFooter(False)
sys.exit('Failed Scenario')
else:
logging.info('Testsuite passed after ' + str(cnt) + ' time(s)')
SSH.CreateHtmlTabFooter(True)
else:
Usage()
sys.exit('Invalid mode')