fix(ci): implement proper automatic un-deploy of RAN containers

Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>
This commit is contained in:
Raphael Defosseux
2021-11-16 15:03:01 +01:00
parent 4ab1c44172
commit ce1bc7cefb
6 changed files with 141 additions and 66 deletions
+59 -49
View File
@@ -368,7 +368,7 @@ class OaiCiTest():
except:
os.kill(os.getppid(),signal.SIGUSR1)
def InitializeUE(self,HTML,RAN,EPC, COTS_UE, InfraUE,ue_trace):
def InitializeUE(self,HTML,RAN,EPC, COTS_UE, InfraUE,ue_trace,CONTAINERS):
if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB
if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
HELP.GenericHelp(CONST.Version)
@@ -424,12 +424,12 @@ class OaiCiTest():
Module_UE.CheckModuleMTU()
else: #status==-1 failed to retrieve IP address
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.UE_IP_ADDRESS_ISSUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE,InfraUE):
def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS):
if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
@@ -653,7 +653,7 @@ class OaiCiTest():
HTML.htmlUEFailureMsg='nr-uesoftmodem did NOT synced'
HTML.CreateHtmlTestRow(self.air_interface + ' ' + self.Initialize_OAI_UE_args, 'KO', CONST.OAI_UE_PROCESS_COULD_NOT_SYNC, 'OAI UE')
logging.error('\033[91mInitialize OAI UE Failed! \033[0m')
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
def checkDevTTYisUnlocked(self):
SSH = sshconnection.SSHConnection()
@@ -731,7 +731,7 @@ class OaiCiTest():
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
self.checkDevTTYisUnlocked()
def AttachCatM(self,HTML,RAN,COTS_UE,EPC,InfraUE):
def AttachCatM(self,HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS):
if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
@@ -804,9 +804,9 @@ class OaiCiTest():
html_cell = '<pre style="background-color:white">CAT-M module Attachment Failed</pre>'
html_queue.put(html_cell)
HTML.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
def PingCatM(self,HTML,RAN,EPC,COTS_UE,InfraUE):
def PingCatM(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS):
if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
@@ -815,7 +815,7 @@ class OaiCiTest():
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
if (pStatus < 0):
HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
try:
statusQueue = SimpleQueue()
@@ -836,7 +836,7 @@ class OaiCiTest():
moduleIPAddr = result.group('ipaddr')
else:
HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
ping_time = re.findall("-c (\d+)",str(self.ping_args))
device_id = 'catm'
@@ -900,7 +900,7 @@ class OaiCiTest():
HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', 1, statusQueue)
else:
HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
except:
os.kill(os.getppid(),signal.SIGUSR1)
@@ -992,7 +992,7 @@ class OaiCiTest():
except:
os.kill(os.getppid(),signal.SIGUSR1)
def AttachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE):
def AttachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS):
if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB
if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
HELP.GenericHelp(CONST.Version)
@@ -1002,7 +1002,7 @@ class OaiCiTest():
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
if (pStatus < 0):
HTML.CreateHtmlTestRow('N/A', 'KO', pStatus)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
multi_jobs = []
status_queue = SimpleQueue()
@@ -1021,7 +1021,7 @@ class OaiCiTest():
if (status_queue.empty()):
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
else:
attach_status = True
@@ -1050,7 +1050,7 @@ class OaiCiTest():
time.sleep(5)
else:
HTML.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
else: #if an ID is specified, it is a module from the yaml infrastructure file
#Attention, as opposed to InitializeUE, the connect manager process is not checked as it is supposed to be active already
@@ -1086,7 +1086,7 @@ class OaiCiTest():
Module_UE.CheckModuleMTU()
else: #status==-1 failed to retrieve IP address
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.UE_IP_ADDRESS_ISSUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
def DetachUE_common(self, device_id, idx,COTS_UE):
@@ -1111,7 +1111,7 @@ class OaiCiTest():
except:
os.kill(os.getppid(),signal.SIGUSR1)
def DetachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE):
def DetachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS):
if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB
if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
HELP.GenericHelp(CONST.Version)
@@ -1121,7 +1121,7 @@ class OaiCiTest():
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
if (pStatus < 0):
HTML.CreateHtmlTestRow('N/A', 'KO', pStatus)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
multi_jobs = []
cnt = 0
@@ -1396,7 +1396,7 @@ class OaiCiTest():
except:
os.kill(os.getppid(),signal.SIGUSR1)
def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE,InfraUE):
def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS):
if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
HELP.GenericHelp(CONST.Version)
sys.exit('Insufficient Parameter')
@@ -1443,7 +1443,7 @@ class OaiCiTest():
if (status_queue.empty()):
HTML.CreateHtmlTestRow(htmlOptions, 'KO', CONST.ALL_PROCESSES_OK)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
else:
check_status = True
html_queue = SimpleQueue()
@@ -1459,7 +1459,7 @@ class OaiCiTest():
HTML.CreateHtmlTestRowQueue(htmlOptions, 'OK', len(self.UEDevices), html_queue)
else:
HTML.CreateHtmlTestRowQueue(htmlOptions, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
def GetAllUEIPAddresses(self):
SSH = sshconnection.SSHConnection()
@@ -1686,14 +1686,14 @@ class OaiCiTest():
html_queue.put(html_cell)
HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
def PingNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE):
def PingNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS):
SSH=sshconnection.SSHConnection()
check_eNB = True
check_OAI_UE = True
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
if (pStatus < 0):
HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args))
if ping_from_eNB is not None:
@@ -1779,10 +1779,10 @@ class OaiCiTest():
except:
os.kill(os.getppid(),signal.SIGUSR1)
def Ping(self,HTML,RAN,EPC,COTS_UE, InfraUE):
def Ping(self,HTML,RAN,EPC,COTS_UE, InfraUE, CONTAINERS):
result = re.search('noS1', str(RAN.Initialize_eNB_args))
if result is not None:
self.PingNoS1(HTML,RAN,EPC,COTS_UE,InfraUE)
self.PingNoS1(HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS)
return
if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '':
HELP.GenericHelp(CONST.Version)
@@ -1795,7 +1795,7 @@ class OaiCiTest():
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
if (pStatus < 0):
HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
if self.ue_id=="":
@@ -1803,7 +1803,7 @@ class OaiCiTest():
ueIpStatus = self.GetAllUEIPAddresses()
if (ueIpStatus < 0):
HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
else:
self.UEIPAddresses=[]
@@ -1830,7 +1830,7 @@ class OaiCiTest():
if (status_queue.empty()):
HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.ALL_PROCESSES_OK)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
else:
ping_status = True
html_queue = SimpleQueue()
@@ -1847,7 +1847,7 @@ class OaiCiTest():
HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue)
else:
HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
def Iperf_ComputeTime(self):
result = re.search('-t (?P<iperf_time>\d+)', str(self.iperf_args))
@@ -2647,7 +2647,7 @@ class OaiCiTest():
except:
os.kill(os.getppid(),signal.SIGUSR1)
def IperfNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE):
def IperfNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS):
SSH = sshconnection.SSHConnection()
if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '':
HELP.GenericHelp(CONST.Version)
@@ -2657,7 +2657,7 @@ class OaiCiTest():
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
if (pStatus < 0):
HTML.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
server_on_enb = re.search('-R', str(self.iperf_args))
if server_on_enb is not None:
@@ -2756,12 +2756,12 @@ class OaiCiTest():
HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue)
else:
HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
def Iperf(self,HTML,RAN,EPC,COTS_UE, InfraUE):
def Iperf(self,HTML,RAN,EPC,COTS_UE, InfraUE,CONTAINERS):
result = re.search('noS1', str(RAN.Initialize_eNB_args))
if result is not None:
self.IperfNoS1(HTML,RAN,EPC,COTS_UE,InfraUE)
self.IperfNoS1(HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS)
return
if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '' or self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '':
HELP.GenericHelp(CONST.Version)
@@ -2774,14 +2774,14 @@ class OaiCiTest():
pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC)
if (pStatus < 0):
HTML.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
if self.ue_id=="":#is not a module, follow legacy code
ueIpStatus = self.GetAllUEIPAddresses()
if (ueIpStatus < 0):
HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
return
else: #is a module
self.UEIPAddresses=[]
@@ -2826,7 +2826,7 @@ class OaiCiTest():
if (status_queue.empty()):
HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.ALL_PROCESSES_OK)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
else:
iperf_status = True
iperf_noperf = False
@@ -2848,7 +2848,7 @@ class OaiCiTest():
HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue)
else:
HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
def CheckProcessExist(self, check_eNB, check_OAI_UE,RAN,EPC):
multi_jobs = []
@@ -3280,7 +3280,7 @@ class OaiCiTest():
else:
HTML.CreateHtmlTestRow('QLog trace is disabled', 'OK', CONST.ALL_PROCESSES_OK)
def TerminateOAIUE(self,HTML,RAN,COTS_UE,EPC, InfraUE):
def TerminateOAIUE(self,HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS):
SSH = sshconnection.SSHConnection()
SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword)
SSH.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5)
@@ -3321,11 +3321,11 @@ class OaiCiTest():
# Not an error then
if (logStatus != CONST.OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'):
self.Initialize_OAI_UE_args = ''
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
else:
if (logStatus == CONST.OAI_UE_PROCESS_COULD_NOT_SYNC):
self.Initialize_OAI_UE_args = ''
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE)
self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS)
else:
logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m')
HTML.htmlUEFailureMsg='<b>' + ueAction + ' Completed</b>\n' + HTML.htmlUEFailureMsg
@@ -3334,26 +3334,26 @@ class OaiCiTest():
else:
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC,InfraUE):
def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS):
if (self.ADBIPAddress != 'none'):
self.testCase_id = 'AUTO-KILL-UE'
HTML.testCase_id=self.testCase_id
HTML.testCase_id = self.testCase_id
self.desc = 'Automatic Termination of UE'
HTML.desc='Automatic Termination of UE'
HTML.desc = self.desc
self.ShowTestID()
self.TerminateUE(HTML,COTS_UE,InfraUE,self.ue_trace)
if (self.Initialize_OAI_UE_args != ''):
self.testCase_id = 'AUTO-KILL-OAI-UE'
HTML.testCase_id=self.testCase_id
HTML.testCase_id = self.testCase_id
self.desc = 'Automatic Termination of OAI-UE'
HTML.desc='Automatic Termination of OAI-UE'
HTML.desc = self.desc
self.ShowTestID()
self.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE)
if (RAN.Initialize_eNB_args != ''):
self.testCase_id = 'AUTO-KILL-RAN'
HTML.testCase_id=self.testCase_id
HTML.testCase_id = self.testCase_id
self.desc = 'Automatic Termination of all RAN nodes'
HTML.desc='Automatic Termination of RAN nodes'
HTML.desc = self.desc
self.ShowTestID()
#terminate all RAN nodes eNB/gNB/OCP
for instance in range(0, len(RAN.air_interface)):
@@ -3363,11 +3363,21 @@ class OaiCiTest():
RAN.TerminateeNB(HTML,EPC)
if RAN.flexranCtrlInstalled and RAN.flexranCtrlStarted:
self.testCase_id = 'AUTO-KILL-flexran-ctl'
HTML.testCase_id=self.testCase_id
HTML.testCase_id = self.testCase_id
self.desc = 'Automatic Termination of FlexRan CTL'
HTML.desc='Automatic Termination of FlexRan CTL'
HTML.desc = self.desc
self.ShowTestID()
self.TerminateFlexranCtrl(HTML,RAN,EPC)
if CONTAINERS.yamlPath[0] != '':
self.testCase_id = 'AUTO-KILL-CONTAINERS'
HTML.testCase_id = self.testCase_id
self.desc = 'Automatic Termination of all RAN containers'
HTML.desc = self.desc
self.ShowTestID()
for instance in range(0, len(CONTAINERS.yamlPath)):
if CONTAINERS.yamlPath[instance]!='':
CONTAINERS.eNB_instance=instance
CONTAINERS.UndeployObject(HTML,RAN)
RAN.prematureExit=True
def IdleSleep(self,HTML):