From 0a4cfc2173e3cda2a459546d72f2a17ba2365870 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 22 Oct 2020 19:02:25 +0200 Subject: [PATCH] CI: scenarios that are known as unstable won't be seen as failures Signed-off-by: Raphael Defosseux --- ci-scripts/cls_oaicitest.py | 19 +++++++++++++----- ci-scripts/html.py | 18 +++++++++++++++-- ci-scripts/main.py | 20 ++++++++++++++++++- .../enb_usrp210_band13_test_10mhz_tm1.xml | 1 + .../enb_usrp210_band40_test_05mhz_tm2.xml | 2 ++ .../enb_usrp210_band40_test_10mhz_tm1.xml | 2 ++ .../enb_usrp210_band40_test_20mhz_tm1.xml | 2 ++ ...and40_test_20mhz_tm1_default_scheduler.xml | 2 ++ .../if4p5_usrp210_band40_test_10mhz.xml | 2 ++ ...x2ho_enb_usrp210_band13_test_05mhz_tm1.xml | 1 + 10 files changed, 61 insertions(+), 8 deletions(-) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index ff27a0ed6c..68e1597f28 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -72,6 +72,9 @@ class OaiCiTest(): self.ADBCentralized = True self.testCase_id = '' self.testXMLfiles = [] + self.testUnstable = False + self.testMinStableId = '999999' + self.testStabilityPointReached = False self.desc = '' self.ping_args = '' self.ping_packetloss_threshold = '' @@ -211,7 +214,7 @@ class OaiCiTest(): logging.error('\u001B[1m Building OAI UE Failed\u001B[0m') HTML.CreateHtmlTestRow(self.Build_OAI_UE_args, 'KO', CONST.ALL_PROCESSES_OK, 'OAI UE') HTML.CreateHtmlTabFooter(False) - sys.exit(1) + self.ConditionalExit() def CheckFlexranCtrlInstallation(self,RAN,EPC): if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '': @@ -335,7 +338,7 @@ class OaiCiTest(): if (pStatus < 0): HTML.CreateHtmlTestRow(self.air_interface + ' ' + self.Initialize_OAI_UE_args, 'KO', pStatus) HTML.CreateHtmlTabFooter(False) - sys.exit(1) + self.ConditionalExit() UE_prefix = '' else: UE_prefix = 'NR ' @@ -719,7 +722,7 @@ class OaiCiTest(): logging.debug('Using the OAI EPC HSS: not implemented yet') HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) HTML.CreateHtmlTabFooter(False) - sys.exit(1) + self.ConditionalExit() else: SSH.command('egrep --color=never "Allocated ipv4 addr" /opt/ltebox/var/log/xGwLog.0', '\$', 5) result = re.search('Allocated ipv4 addr: (?P[0-9\.]+) from Pool', SSH.getBefore()) @@ -1046,7 +1049,7 @@ class OaiCiTest(): if (pStatus < 0): HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) HTML.CreateHtmlTabFooter(False) - sys.exit(1) + self.ConditionalExit() multi_jobs = [] for device_id in self.UEDevices: p = Process(target = self.RebootUE_common, args = (device_id,)) @@ -1248,7 +1251,7 @@ class OaiCiTest(): if (pStatus < 0): HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) HTML.CreateHtmlTabFooter(False) - sys.exit(1) + self.ConditionalExit() multi_jobs = [] lock = Lock() status_queue = SimpleQueue() @@ -3169,6 +3172,12 @@ class OaiCiTest(): HTML.CpuMHz[idx]=CpuMHz SSH.close() + def ConditionalExit(self): + if self.testUnstable: + if self.testStabilityPointReached or self.testMinStableId == '999999': + sys.exit(0) + sys.exit(1) + def ShowTestID(self): logging.debug('\u001B[1m----------------------------------------\u001B[0m') logging.debug('\u001B[1mTest ID:' + self.testCase_id + '\u001B[0m') diff --git a/ci-scripts/html.py b/ci-scripts/html.py index 8f2d38cced..f50899be4f 100644 --- a/ci-scripts/html.py +++ b/ci-scripts/html.py @@ -64,6 +64,10 @@ class HTMLManagement(): self.htmlTabIcons = [] self.testXMLfiles = [] + self.testUnstable = False + self.testMinStableId = '999999' + self.testStabilityPointReached = False + self.htmleNBFailureMsg = '' self.htmlUEFailureMsg = '' @@ -230,13 +234,20 @@ class HTMLManagement(): def CreateHtmlTabFooter(self, passStatus): if ((not self.htmlFooterCreated) and (self.htmlHeaderCreated)): + testOkEvenIfUnstable = False + if self.testUnstable and not passStatus: + if self.testStabilityPointReached or self.testMinStableId == '999999': + testOkEvenIfUnstable = True self.htmlFile = open('test_results.html', 'a') self.htmlFile.write(' \n') self.htmlFile.write(' Final Tab Status\n') if passStatus: self.htmlFile.write(' PASS \n') else: - self.htmlFile.write(' FAIL \n') + if testOkEvenIfUnstable: + self.htmlFile.write(' KNOWN UNSTABLE SCENARIO \n') + else: + self.htmlFile.write(' FAIL \n') self.htmlFile.write(' \n') self.htmlFile.write(' \n') self.htmlFile.write(' \n') @@ -246,7 +257,10 @@ class HTMLManagement(): cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__//' test_results.html" subprocess.run(cmd, shell=True) else: - cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<\/span>/' test_results.html" + if testOkEvenIfUnstable: + cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<\/span>/' test_results.html" + else: + cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<\/span>/' test_results.html" subprocess.run(cmd, shell=True) self.htmlFooterCreated = False diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 4b001f8064..b149da6cc5 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -494,6 +494,9 @@ elif re.match('^FinalizeHtml$', mode, re.IGNORECASE): CiTestObj.RetrieveSystemVersion('UE',HTML,RAN) HTML.CreateHtmlFooter(CiTestObj.finalStatus) elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re.IGNORECASE): + logging.debug('\u001B[1m----------------------------------------\u001B[0m') + logging.debug('\u001B[1m Starting Scenario \u001B[0m') + logging.debug('\u001B[1m----------------------------------------\u001B[0m') if re.match('^TesteNB$', mode, re.IGNORECASE): if RAN.eNBIPAddress == '' or RAN.ranRepository == '' or RAN.ranBranch == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '' or EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.Type == '' or EPC.SourceCodePath == '' or CiTestObj.ADBIPAddress == '' or CiTestObj.ADBUserName == '' or CiTestObj.ADBPassword == '': HELP.GenericHelp(CONST.Version) @@ -529,7 +532,14 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re HTML.htmlTabRefs.append(xmlRoot.findtext('htmlTabRef',default='test-tab-0')) HTML.htmlTabNames.append(xmlRoot.findtext('htmlTabName',default='Test-0')) repeatCount = xmlRoot.findtext('repeatCount',default='1') + testStability = xmlRoot.findtext('TestUnstable',default='False') CiTestObj.repeatCounts.append(int(repeatCount)) + if testStability == 'True': + CiTestObj.testUnstable = True + HTML.testUnstable = True + CiTestObj.testMinStableId = xmlRoot.findtext('TestMinId',default='999999') + HTML.testMinStableId = CiTestObj.testMinStableId + logging.debug('Test is tagged as Unstable -- starting from TestID ' + str(CiTestObj.testMinStableId)) all_tests=xmlRoot.findall('testCase') exclusion_tests=exclusion_tests.split() @@ -686,11 +696,19 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re HTML=ldpc.Run_PhySim(HTML,CONST,id) else: sys.exit('Invalid class (action) from xml') + if not RAN.prematureExit: + if CiTestObj.testCase_id == CiTestObj.testMinStableId: + logging.debug('Scenario has reached minimal stability point') + CiTestObj.testStabilityPointReached = True + HTML.testStabilityPointReached = True CiTestObj.FailReportCnt += 1 if CiTestObj.FailReportCnt == CiTestObj.repeatCounts[0] and RAN.prematureExit: logging.debug('Testsuite failed ' + str(CiTestObj.FailReportCnt) + ' time(s)') HTML.CreateHtmlTabFooter(False) - sys.exit('Failed Scenario') + if CiTestObj.testUnstable and (CiTestObj.testStabilityPointReached or CiTestObj.testMinStableId == '999999'): + logging.debug('Scenario has reached minimal stability point -- Not a Failure') + else: + sys.exit('Failed Scenario') else: logging.info('Testsuite passed after ' + str(CiTestObj.FailReportCnt) + ' time(s)') HTML.CreateHtmlTabFooter(True) diff --git a/ci-scripts/xml_files/enb_usrp210_band13_test_10mhz_tm1.xml b/ci-scripts/xml_files/enb_usrp210_band13_test_10mhz_tm1.xml index 611492b0bc..cdba0723df 100644 --- a/ci-scripts/xml_files/enb_usrp210_band13_test_10mhz_tm1.xml +++ b/ci-scripts/xml_files/enb_usrp210_band13_test_10mhz_tm1.xml @@ -25,6 +25,7 @@ Test-LTE-M-10MHz-TM1 tasks 1 + True 030201 040102 diff --git a/ci-scripts/xml_files/enb_usrp210_band40_test_05mhz_tm2.xml b/ci-scripts/xml_files/enb_usrp210_band40_test_05mhz_tm2.xml index 09aa0b04a2..5ece10d37e 100644 --- a/ci-scripts/xml_files/enb_usrp210_band40_test_05mhz_tm2.xml +++ b/ci-scripts/xml_files/enb_usrp210_band40_test_05mhz_tm2.xml @@ -26,6 +26,8 @@ Test-05MHz-TM2 tasks 2 + True + 040608 030201 040101 diff --git a/ci-scripts/xml_files/enb_usrp210_band40_test_10mhz_tm1.xml b/ci-scripts/xml_files/enb_usrp210_band40_test_10mhz_tm1.xml index 29699d3b3d..b1d87e598e 100644 --- a/ci-scripts/xml_files/enb_usrp210_band40_test_10mhz_tm1.xml +++ b/ci-scripts/xml_files/enb_usrp210_band40_test_10mhz_tm1.xml @@ -24,6 +24,8 @@ test-10-tm1 Test-10MHz-TM1 tasks + True + 040615 030201 040101 diff --git a/ci-scripts/xml_files/enb_usrp210_band40_test_20mhz_tm1.xml b/ci-scripts/xml_files/enb_usrp210_band40_test_20mhz_tm1.xml index f98ed1600a..d75704aceb 100644 --- a/ci-scripts/xml_files/enb_usrp210_band40_test_20mhz_tm1.xml +++ b/ci-scripts/xml_files/enb_usrp210_band40_test_20mhz_tm1.xml @@ -25,6 +25,8 @@ Test-20MHz-TM1 tasks 4 + True + 040623 030201 040101 diff --git a/ci-scripts/xml_files/enb_usrp210_band40_test_20mhz_tm1_default_scheduler.xml b/ci-scripts/xml_files/enb_usrp210_band40_test_20mhz_tm1_default_scheduler.xml index fd018f7a9c..a6530a920c 100644 --- a/ci-scripts/xml_files/enb_usrp210_band40_test_20mhz_tm1_default_scheduler.xml +++ b/ci-scripts/xml_files/enb_usrp210_band40_test_20mhz_tm1_default_scheduler.xml @@ -24,6 +24,8 @@ test-20-tm1-defSched Test-20MHz-TM1-default-scheduler tasks + True + 040522 030201 040101 diff --git a/ci-scripts/xml_files/if4p5_usrp210_band40_test_10mhz.xml b/ci-scripts/xml_files/if4p5_usrp210_band40_test_10mhz.xml index 8276294e9a..9224c67fe7 100644 --- a/ci-scripts/xml_files/if4p5_usrp210_band40_test_10mhz.xml +++ b/ci-scripts/xml_files/if4p5_usrp210_band40_test_10mhz.xml @@ -25,6 +25,8 @@ Test-10MHz tasks 2 + True + 040612 030201 040101 diff --git a/ci-scripts/xml_files/x2ho_enb_usrp210_band13_test_05mhz_tm1.xml b/ci-scripts/xml_files/x2ho_enb_usrp210_band13_test_05mhz_tm1.xml index c43be75a87..7256c985c5 100644 --- a/ci-scripts/xml_files/x2ho_enb_usrp210_band13_test_05mhz_tm1.xml +++ b/ci-scripts/xml_files/x2ho_enb_usrp210_band13_test_05mhz_tm1.xml @@ -26,6 +26,7 @@ Test-X2-HO-FDD-05MHz-TM1 tasks 2 + True 030201 030202 040101