+
+
+ 
+ 
+ 
CAT-M module Attachment Failed' html_queue.put(html_cell) HTML.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) - def PingCatM(self,HTML,RAN,EPC,COTS_UE,InfraUE): + def PingCatM(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS): if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -815,7 +815,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) return try: statusQueue = SimpleQueue() @@ -836,7 +836,7 @@ class OaiCiTest(): moduleIPAddr = result.group('ipaddr') else: HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) return ping_time = re.findall("-c (\d+)",str(self.ping_args)) device_id = 'catm' @@ -900,7 +900,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', 1, statusQueue) else: HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) except: os.kill(os.getppid(),signal.SIGUSR1) @@ -992,7 +992,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def AttachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): + def AttachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS): if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) @@ -1002,7 +1002,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) return multi_jobs = [] status_queue = SimpleQueue() @@ -1021,7 +1021,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) return else: attach_status = True @@ -1050,7 +1050,7 @@ class OaiCiTest(): time.sleep(5) else: HTML.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) else: #if an ID is specified, it is a module from the yaml infrastructure file #Attention, as opposed to InitializeUE, the connect manager process is not checked as it is supposed to be active already @@ -1086,7 +1086,7 @@ class OaiCiTest(): Module_UE.CheckModuleMTU() else: #status==-1 failed to retrieve IP address HTML.CreateHtmlTestRow('N/A', 'KO', CONST.UE_IP_ADDRESS_ISSUE) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) return def DetachUE_common(self, device_id, idx,COTS_UE): @@ -1111,7 +1111,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def DetachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): + def DetachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS): if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) @@ -1121,7 +1121,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) return multi_jobs = [] cnt = 0 @@ -1328,7 +1328,7 @@ class OaiCiTest(): SSH = sshconnection.SSHConnection() SSH.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword) if self.ADBCentralized: - SSH.command('lsusb | egrep "Future Technology Devices International, Ltd FT2232C" | sed -e "s#:.*##" -e "s# #_#g"', '\$', 15) + SSH.command('lsusb | egrep --colour=never "Future Technology Devices International, Ltd FT2232C" | sed -e "s#:.*##" -e "s# #_#g"', '\$', 15) #self.CatMDevices = re.findall("\\\\r\\\\n([A-Za-z0-9_]+)",SSH.getBefore()) self.CatMDevices = re.findall("\\\\r\\\\n([A-Za-z0-9_]+)",SSH.getBefore()) else: @@ -1396,7 +1396,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): + def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -1443,7 +1443,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(htmlOptions, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) else: check_status = True html_queue = SimpleQueue() @@ -1459,7 +1459,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(htmlOptions, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(htmlOptions, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) def GetAllUEIPAddresses(self): SSH = sshconnection.SSHConnection() @@ -1552,7 +1552,7 @@ class OaiCiTest(): SSH.command('cd scripts', '\$', 5) # In case of a docker-based deployment, we need to ping from the trf-gen container launchFromTrfContainer = False - if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): + if (re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE)) or (re.match('OAICN5G', EPC.Type, re.IGNORECASE)): launchFromTrfContainer = True if launchFromTrfContainer: ping_status = SSH.command('docker exec -it prod-trf-gen /bin/bash -c "ping ' + self.ping_args + ' ' + UE_IPAddress + '" 2>&1 | tee ping_' + self.testCase_id + '_' + device_id + '.log', '\$', int(ping_time[0])*1.5) @@ -1583,7 +1583,7 @@ class OaiCiTest(): if re.match('OAI-Rel14-Docker', EPC.Type, re.IGNORECASE): Target = EPC.MmeIPAddress elif re.match('OAICN5G', EPC.Type, re.IGNORECASE): - Target = '8.8.8.8' + Target = EPC.MmeIPAddress else: Target = EPC.IPAddress #ping from module NIC rather than IP address to make sure round trip is over the air @@ -1686,14 +1686,14 @@ class OaiCiTest(): html_queue.put(html_cell) HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue) - def PingNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE): + def PingNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS): SSH=sshconnection.SSHConnection() check_eNB = True check_OAI_UE = True pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) return ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args)) if ping_from_eNB is not None: @@ -1779,10 +1779,10 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def Ping(self,HTML,RAN,EPC,COTS_UE, InfraUE): + def Ping(self,HTML,RAN,EPC,COTS_UE, InfraUE, CONTAINERS): result = re.search('noS1', str(RAN.Initialize_eNB_args)) if result is not None: - self.PingNoS1(HTML,RAN,EPC,COTS_UE,InfraUE) + self.PingNoS1(HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS) return if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) @@ -1795,7 +1795,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) return if self.ue_id=="": @@ -1803,7 +1803,7 @@ class OaiCiTest(): ueIpStatus = self.GetAllUEIPAddresses() if (ueIpStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) return else: self.UEIPAddresses=[] @@ -1830,7 +1830,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) else: ping_status = True html_queue = SimpleQueue() @@ -1847,7 +1847,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) def Iperf_ComputeTime(self): result = re.search('-t (?P
\n'
if re.match('ltebox', self.Type, re.IGNORECASE):
logging.debug('Using the SABOX simulated HSS')
mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5)
@@ -229,16 +246,48 @@ class EPCManagement():
logging.debug('Using the sabox')
mySSH.command('cd /opt/ltebox/tools', '\$', 5)
mySSH.command('echo ' + self.Password + ' | sudo -S ./start_sabox', '\$', 5)
+ html_cell += 'N/A\n'
elif re.match('OAICN5G', self.Type, re.IGNORECASE):
logging.debug('Starting OAI CN5G')
mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5)
mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5)
mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5)
mySSH.command('./core-network.sh start nrf spgwu', '\$', 60)
+ time.sleep(2)
+ mySSH.command('docker-compose -p 5gcn ps -a', '\$', 60)
+ if mySSH.getBefore().count('Up (healthy)') != 6:
+ logging.error('Not all container healthy')
+ else:
+ logging.debug('OK')
+ mySSH.command('docker-compose config | grep --colour=never image', '\$', 10)
+ listOfImages = mySSH.getBefore()
+ for imageLine in listOfImages.split('\\r\\n'):
+ res1 = re.search('image: (?P[a-zA-Z0-9\-]+):(?P[a-zA-Z0-9\-]+)', str(imageLine))
+ res2 = re.search('mysql', str(imageLine))
+ if res1 is not None and res2 is None:
+ html_cell += res1.group('name') + ':' + res1.group('tag') + ' '
+ nbChars = len(res1.group('name')) + len(res1.group('tag')) + 2
+ while (nbChars < 32):
+ html_cell += ' '
+ nbChars += 1
+ mySSH.command('docker image inspect --format="Size = {{.Size}} bytes" ' + res1.group('name') + ':' + res1.group('tag'), '\$', 10)
+ res3 = re.search('Size *= *(?P[0-9\-]*) *bytes', mySSH.getBefore())
+ if res3 is not None:
+ imageSize = int(res3.group('size'))
+ imageSize = int(imageSize/(1024*1024))
+ html_cell += str(imageSize) + ' MBytes '
+ mySSH.command('docker image inspect --format="Date = {{.Created}}" ' + res1.group('name') + ':' + res1.group('tag'), '\$', 10)
+ res4 = re.search('Date *= *(?P[0-9\-]*)T', mySSH.getBefore())
+ if res4 is not None:
+ html_cell += '(' + res4.group('date') + ')'
+ html_cell += '\n'
else:
logging.error('This option should not occur!')
+ html_cell += ' '
mySSH.close()
- HTML.CreateHtmlTestRow(self.Type, 'OK', CONST.ALL_PROCESSES_OK)
+ html_queue = SimpleQueue()
+ html_queue.put(html_cell)
+ HTML.CreateHtmlTestRowQueue(self.Type, 'OK', 1, html_queue)
def SetAmfIPAddress(self):
# Not an error if we don't need an 5GCN
@@ -289,7 +338,15 @@ class EPCManagement():
mySSH = SSH.SSHConnection()
mySSH.open(self.IPAddress, self.UserName, self.Password)
if re.match('OAI-Rel14-Docker', self.Type, re.IGNORECASE):
- mySSH.command('docker top ' + self.containerPrefix + '-oai-mme', '\$', 5)
+ self.isMagmaUsed = False
+ mySSH.command('docker ps -a', '\$', 5)
+ result = re.search('magma', mySSH.getBefore())
+ if result is not None:
+ self.isMagmaUsed = True
+ if self.isMagmaUsed:
+ mySSH.command('docker top ' + self.containerPrefix + '-magma-mme', '\$', 5)
+ else:
+ mySSH.command('docker top ' + self.containerPrefix + '-oai-mme', '\$', 5)
else:
mySSH.command('stdbuf -o0 ps -aux | grep --color=never mme | grep -v grep', '\$', 5)
if re.match('OAI-Rel14-Docker', self.Type, re.IGNORECASE):
@@ -354,7 +411,7 @@ class EPCManagement():
elif re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE):
mySSH.command('echo ' + self.Password + ' | sudo -S killall --signal SIGINT oai_hss || true', '\$', 5)
time.sleep(2)
- mySSH.command('stdbuf -o0 ps -aux | grep hss | grep -v grep', '\$', 5)
+ mySSH.command('stdbuf -o0 ps -aux | grep --colour=never hss | grep -v grep', '\$', 5)
result = re.search('oai_hss -j', mySSH.getBefore())
if result is not None:
mySSH.command('echo ' + self.Password + ' | sudo -S killall --signal SIGKILL oai_hss || true', '\$', 5)
@@ -362,7 +419,7 @@ class EPCManagement():
elif re.match('OAI', self.Type, re.IGNORECASE):
mySSH.command('echo ' + self.Password + ' | sudo -S killall --signal SIGINT run_hss oai_hss || true', '\$', 5)
time.sleep(2)
- mySSH.command('stdbuf -o0 ps -aux | grep hss | grep -v grep', '\$', 5)
+ mySSH.command('stdbuf -o0 ps -aux | grep --colour=never hss | grep -v grep', '\$', 5)
result = re.search('\/bin\/bash .\/run_', mySSH.getBefore())
if result is not None:
mySSH.command('echo ' + self.Password + ' | sudo -S killall --signal SIGKILL run_hss oai_hss || true', '\$', 5)
@@ -371,6 +428,11 @@ class EPCManagement():
mySSH.command('cd scripts', '\$', 5)
time.sleep(1)
mySSH.command('echo ' + self.Password + ' | sudo -S screen -S simulated_hss -X quit', '\$', 5)
+ time.sleep(5)
+ mySSH.command('ps aux | grep --colour=never hss_sim | grep -v grep', '\$', 5, silent=True)
+ result = re.search('hss_sim s6as diam_hss', mySSH.getBefore())
+ if result is not None:
+ mySSH.command('echo ' + self.Password + ' | sudo -S killall hss_sim', '\$', 5)
else:
logging.error('This should not happen!')
mySSH.close()
@@ -397,6 +459,7 @@ class EPCManagement():
elif re.match('ltebox', self.Type, re.IGNORECASE):
mySSH.command('cd /opt/ltebox/tools', '\$', 5)
mySSH.command('echo ' + self.Password + ' | sudo -S ./stop_mme', '\$', 5)
+ time.sleep(5)
else:
logging.error('This should not happen!')
mySSH.close()
@@ -448,6 +511,7 @@ class EPCManagement():
def Terminate5GCN(self, HTML):
mySSH = SSH.SSHConnection()
mySSH.open(self.IPAddress, self.UserName, self.Password)
+ message = ''
if re.match('ltebox', self.Type, re.IGNORECASE):
logging.debug('Terminating SA BOX')
mySSH.command('cd /opt/ltebox/tools', '\$', 5)
@@ -458,15 +522,31 @@ class EPCManagement():
time.sleep(1)
mySSH.command('echo ' + self.Password + ' | sudo -S screen -S simulated_5g_hss -X quit', '\$', 5)
elif re.match('OAICN5G', self.Type, re.IGNORECASE):
- self.LogCollectOAICN5G()
+ logging.debug('OAI CN5G Collecting Log files to workspace')
+ mySSH.command('echo ' + self.Password + ' | sudo rm -rf ' + self.SourceCodePath + '/logs', '\$', 5)
+ mySSH.command('mkdir ' + self.SourceCodePath + '/logs','\$', 5)
+ containers_list=['oai-smf','oai-spgwu','oai-amf','oai-nrf']
+ for c in containers_list:
+ mySSH.command('docker logs ' + c + ' > ' + self.SourceCodePath + '/logs/' + c + '.log', '\$', 5)
+
logging.debug('Terminating OAI CN5G')
mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5)
- mySSH.command('docker-compose down', '\$', 5)
mySSH.command('./core-network.sh stop nrf spgwu', '\$', 60)
+ time.sleep(2)
+ mySSH.command('tshark -r /tmp/oai-cn5g.pcap | egrep --colour=never "Tracking area update" ','\$', 30)
+ result = re.search('Tracking area update request', mySSH.getBefore())
+ if result is not None:
+ message = 'UE requested ' + str(mySSH.getBefore().count('Tracking area update request')) + 'Tracking area update request(s)'
+ else:
+ message = 'No Tracking area update request'
+ logging.debug(message)
else:
logging.error('This should not happen!')
mySSH.close()
- HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
+ html_queue = SimpleQueue()
+ html_cell = '' + message + '' + html_queue.put(html_cell) + HTML.CreateHtmlTestRowQueue(self.Type, 'OK', 1, html_queue) def DeployEpc(self, HTML): logging.debug('Trying to deploy') @@ -489,6 +569,12 @@ class EPCManagement(): HTML.CreateHtmlTabFooter(False) sys.exit('docker-compose not installed on ' + self.IPAddress) + # Checking if it is a MAGMA deployment + self.isMagmaUsed = False + if os.path.isfile('./' + self.yamlPath + '/redis_extern.conf'): + self.isMagmaUsed = True + logging.debug('MAGMA MME is used!') + mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5) mySSH.command('if [ -d ' + self.SourceCodePath + '/logs ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/logs ; fi', '\$', 5) mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts ' + self.SourceCodePath + '/logs', '\$', 5) @@ -500,10 +586,19 @@ class EPCManagement(): # - docker-compose config | grep container_name mySSH.command('cd ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/docker-compose.yml', self.SourceCodePath + '/scripts') + if self.isMagmaUsed: + mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/entrypoint.sh', self.SourceCodePath + '/scripts') + mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/mme.conf', self.SourceCodePath + '/scripts') + mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/mme_fd.sprint.conf', self.SourceCodePath + '/scripts') + mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/redis_extern.conf', self.SourceCodePath + '/scripts') + mySSH.command('chmod a+x ' + self.SourceCodePath + '/scripts/entrypoint.sh', '\$', 5) + else: + mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/entrypoint.sh', self.SourceCodePath + '/scripts') + mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/mme.conf', self.SourceCodePath + '/scripts') + mySSH.command('chmod 775 entrypoint.sh', '\$', 60) mySSH.command('wget --quiet --tries=3 --retry-connrefused https://raw.githubusercontent.com/OPENAIRINTERFACE/openair-hss/develop/src/hss_rel14/db/oai_db.cql', '\$', 30) mySSH.command('docker-compose down', '\$', 60) mySSH.command('docker-compose up -d db_init', '\$', 60) - # databases take time... time.sleep(10) cnt = 0 @@ -525,8 +620,12 @@ class EPCManagement(): # deploying EPC cNFs mySSH.command('docker-compose up -d oai_spgwu', '\$', 60) - listOfContainers = 'prod-cassandra prod-oai-hss prod-oai-mme prod-oai-spgwc prod-oai-spgwu-tiny' - expectedHealthyContainers = 5 + if self.isMagmaUsed: + listOfContainers = 'prod-cassandra prod-oai-hss prod-magma-mme prod-oai-spgwc prod-oai-spgwu-tiny prod-redis' + expectedHealthyContainers = 6 + else: + listOfContainers = 'prod-cassandra prod-oai-hss prod-oai-mme prod-oai-spgwc prod-oai-spgwu-tiny' + expectedHealthyContainers = 5 # Checking for additional services mySSH.command('docker-compose config', '\$', 5) @@ -540,6 +639,30 @@ class EPCManagement(): listOfContainers += ' prod-trf-gen' expectedHealthyContainers += 1 + mySSH.command('docker-compose config | grep --colour=never image', '\$', 10) + html_cell = '
\n'
+ listOfImages = mySSH.getBefore()
+ for imageLine in listOfImages.split('\\r\\n'):
+ res1 = re.search('image: (?P[a-zA-Z0-9\-]+):(?P[a-zA-Z0-9\-]+)', str(imageLine))
+ res2 = re.search('cassandra|redis', str(imageLine))
+ if res1 is not None and res2 is None:
+ html_cell += res1.group('name') + ':' + res1.group('tag') + ' '
+ nbChars = len(res1.group('name')) + len(res1.group('tag')) + 2
+ while (nbChars < 32):
+ html_cell += ' '
+ nbChars += 1
+ mySSH.command('docker image inspect --format="Size = {{.Size}} bytes" ' + res1.group('name') + ':' + res1.group('tag'), '\$', 10)
+ res3 = re.search('Size *= *(?P[0-9\-]*) *bytes', mySSH.getBefore())
+ if res3 is not None:
+ imageSize = int(res3.group('size'))
+ imageSize = int(imageSize/(1024*1024))
+ html_cell += str(imageSize) + ' MBytes '
+ mySSH.command('docker image inspect --format="Date = {{.Created}}" ' + res1.group('name') + ':' + res1.group('tag'), '\$', 10)
+ res4 = re.search('Date *= *(?P[0-9\-]*)T', mySSH.getBefore())
+ if res4 is not None:
+ html_cell += '(' + res4.group('date') + ')'
+ html_cell += '\n'
+ html_cell += ' '
# Checking if all are healthy
cnt = 0
while (cnt < 3):
@@ -555,19 +678,24 @@ class EPCManagement():
logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)')
logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)')
logging.debug(' -- ' + str(startingNb) + ' still starting container(s)')
+ html_queue = SimpleQueue()
+ html_queue.put(html_cell)
if healthyNb == expectedHealthyContainers:
mySSH.command('docker exec -d prod-oai-hss /bin/bash -c "nohup tshark -i any -f \'port 9042 or port 3868\' -w /tmp/hss_check_run.pcap 2>&1 > /dev/null"', '\$', 5)
- mySSH.command('docker exec -d prod-oai-mme /bin/bash -c "nohup tshark -i any -f \'port 3868 or port 2123 or port 36412\' -w /tmp/mme_check_run.pcap 2>&1 > /dev/null"', '\$', 10)
+ if self.isMagmaUsed:
+ mySSH.command('docker exec -d prod-magma-mme /bin/bash -c "nohup tshark -i any -f \'port 3868 or port 2123 or port 36412\' -w /tmp/mme_check_run.pcap 2>&1 > /dev/null"', '\$', 10)
+ else:
+ mySSH.command('docker exec -d prod-oai-mme /bin/bash -c "nohup tshark -i any -f \'port 3868 or port 2123 or port 36412\' -w /tmp/mme_check_run.pcap 2>&1 > /dev/null"', '\$', 10)
mySSH.command('docker exec -d prod-oai-spgwc /bin/bash -c "nohup tshark -i any -f \'port 2123 or port 8805\' -w /tmp/spgwc_check_run.pcap 2>&1 > /dev/null"', '\$', 10)
# on SPGW-U, not capturing on SGI to avoid huge file
mySSH.command('docker exec -d prod-oai-spgwu-tiny /bin/bash -c "nohup tshark -i any -f \'port 8805\' -w /tmp/spgwu_check_run.pcap 2>&1 > /dev/null"', '\$', 10)
mySSH.close()
logging.debug('Deployment OK')
- HTML.CreateHtmlTestRow(self.Type, 'OK', CONST.ALL_PROCESSES_OK)
+ HTML.CreateHtmlTestRowQueue(self.Type, 'OK', 1, html_queue)
else:
mySSH.close()
logging.debug('Deployment went wrong')
- HTML.CreateHtmlTestRow(self.Type, 'KO', CONST.INVALID_PARAMETER)
+ HTML.CreateHtmlTestRowQueue(self.Type, 'KO', 1, html_queue)
def UndeployEpc(self, HTML):
logging.debug('Trying to undeploy')
@@ -575,24 +703,52 @@ class EPCManagement():
mySSH = SSH.SSHConnection()
mySSH.open(self.IPAddress, self.UserName, self.Password)
+ # Checking if it is a MAGMA deployment.
+ mySSH.command('cd ' + self.SourceCodePath + '/scripts', '\$', 5)
+ mySSH.command('docker-compose ps -a', '\$', 5)
+ self.isMagmaUsed = False
+ result = re.search('magma', mySSH.getBefore())
+ if result is not None:
+ self.isMagmaUsed = True
+ logging.debug('MAGMA MME is used!')
# Recovering logs and pcap files
mySSH.command('cd ' + self.SourceCodePath + '/logs', '\$', 5)
mySSH.command('docker exec -it prod-oai-hss /bin/bash -c "killall --signal SIGINT oai_hss tshark"', '\$', 5)
- mySSH.command('docker exec -it prod-oai-mme /bin/bash -c "killall --signal SIGINT tshark"', '\$', 5)
+ if self.isMagmaUsed:
+ mySSH.command('docker exec -it prod-magma-mme /bin/bash -c "killall --signal SIGINT tshark"', '\$', 5)
+ else:
+ mySSH.command('docker exec -it prod-oai-mme /bin/bash -c "killall --signal SIGINT tshark"', '\$', 5)
mySSH.command('docker exec -it prod-oai-spgwc /bin/bash -c "killall --signal SIGINT oai_spgwc tshark"', '\$', 5)
mySSH.command('docker exec -it prod-oai-spgwu-tiny /bin/bash -c "killall --signal SIGINT tshark"', '\$', 5)
mySSH.command('docker logs prod-oai-hss > hss_' + self.testCase_id + '.log', '\$', 5)
- mySSH.command('docker logs prod-oai-mme > mme_' + self.testCase_id + '.log', '\$', 5)
+ if self.isMagmaUsed:
+ mySSH.command('docker cp --follow-link prod-magma-mme:/var/log/mme.log mme_' + self.testCase_id + '.log', '\$', 15)
+ else:
+ mySSH.command('docker logs prod-oai-mme > mme_' + self.testCase_id + '.log', '\$', 5)
mySSH.command('docker logs prod-oai-spgwc > spgwc_' + self.testCase_id + '.log', '\$', 5)
mySSH.command('docker logs prod-oai-spgwu-tiny > spgwu_' + self.testCase_id + '.log', '\$', 5)
mySSH.command('docker cp prod-oai-hss:/tmp/hss_check_run.pcap hss_' + self.testCase_id + '.pcap', '\$', 60)
- mySSH.command('docker cp prod-oai-mme:/tmp/mme_check_run.pcap mme_' + self.testCase_id + '.pcap', '\$', 60)
+ if self.isMagmaUsed:
+ mySSH.command('docker cp prod-magma-mme:/tmp/mme_check_run.pcap mme_' + self.testCase_id + '.pcap', '\$', 60)
+ else:
+ mySSH.command('docker cp prod-oai-mme:/tmp/mme_check_run.pcap mme_' + self.testCase_id + '.pcap', '\$', 60)
+ mySSH.command('tshark -r mme_' + self.testCase_id + '.pcap | egrep --colour=never "Tracking area update"', '\$', 60)
+ result = re.search('Tracking area update request', mySSH.getBefore())
+ if result is not None:
+ message = 'UE requested ' + str(mySSH.getBefore().count('Tracking area update request')) + 'Tracking area update request(s)'
+ else:
+ message = 'No Tracking area update request'
+ logging.debug(message)
mySSH.command('docker cp prod-oai-spgwc:/tmp/spgwc_check_run.pcap spgwc_' + self.testCase_id + '.pcap', '\$', 60)
mySSH.command('docker cp prod-oai-spgwu-tiny:/tmp/spgwu_check_run.pcap spgwu_' + self.testCase_id + '.pcap', '\$', 60)
# Remove all
mySSH.command('cd ' + self.SourceCodePath + '/scripts', '\$', 5)
- listOfContainers = 'prod-cassandra prod-oai-hss prod-oai-mme prod-oai-spgwc prod-oai-spgwu-tiny'
- nbContainers = 5
+ if self.isMagmaUsed:
+ listOfContainers = 'prod-cassandra prod-oai-hss prod-magma-mme prod-oai-spgwc prod-oai-spgwu-tiny prod-redis'
+ nbContainers = 6
+ else:
+ listOfContainers = 'prod-cassandra prod-oai-hss prod-oai-mme prod-oai-spgwc prod-oai-spgwu-tiny'
+ nbContainers = 5
# Checking for additional services
mySSH.command('docker-compose config', '\$', 5)
configResponse = mySSH.getBefore()
@@ -609,12 +765,15 @@ class EPCManagement():
mySSH.command('docker inspect --format=\'{{.Name}}\' prod-oai-public-net prod-oai-private-net', '\$', 10)
noMoreNetworkNb = mySSH.getBefore().count('No such object')
mySSH.close()
+ html_queue = SimpleQueue()
+ html_cell = '' + message + '' + html_queue.put(html_cell) if noMoreContainerNb == nbContainers and noMoreNetworkNb == 2: logging.debug('Undeployment OK') - HTML.CreateHtmlTestRow(self.Type, 'OK', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlTestRowQueue(self.Type, 'OK', 1, html_queue) else: logging.debug('Undeployment went wrong') - HTML.CreateHtmlTestRow(self.Type, 'KO', CONST.INVALID_PARAMETER) + HTML.CreateHtmlTestRowQueue(self.Type, 'KO', 1, html_queue) def LogCollectHSS(self): mySSH = SSH.SSHConnection() @@ -633,6 +792,8 @@ class EPCManagement(): mySSH.command('docker cp ' + self.containerPrefix + '-oai-hss:/openair-hss/hss_check_run.log .', '\$', 60) mySSH.command('docker cp ' + self.containerPrefix + '-oai-hss:/tmp/hss_check_run.pcap .', '\$', 60) mySSH.command('zip hss.log.zip hss_check_run.*', '\$', 60) + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + logging.debug('LogCollect is bypassed for that variant') elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): mySSH.command('zip hss.log.zip hss*.log', '\$', 60) mySSH.command('echo ' + self.Password + ' | sudo -S rm hss*.log', '\$', 5) @@ -663,6 +824,11 @@ class EPCManagement(): mySSH.command('docker cp ' + self.containerPrefix + '-oai-mme:/openair-mme/mme_check_run.log .', '\$', 60) mySSH.command('docker cp ' + self.containerPrefix + '-oai-mme:/tmp/mme_check_run.pcap .', '\$', 60) mySSH.command('zip mme.log.zip mme_check_run.*', '\$', 60) + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + mySSH.command('cd ' + self.SourceCodePath + '/logs','\$', 5) + mySSH.command('cp -f /tmp/oai-cn5g.pcap .','\$', 30) + mySSH.command('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g.pcap','\$', 30) + mySSH.command('mv mme.log.zip ' + self.SourceCodePath + '/scripts','\$', 30) elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): mySSH.command('zip mme.log.zip mme*.log', '\$', 60) mySSH.command('echo ' + self.Password + ' | sudo -S rm mme*.log', '\$', 5) @@ -692,6 +858,10 @@ class EPCManagement(): mySSH.command('docker cp ' + self.containerPrefix + '-oai-spgwc:/tmp/spgwc_check_run.pcap .', '\$', 60) mySSH.command('docker cp ' + self.containerPrefix + '-oai-spgwu-tiny:/tmp/spgwu_check_run.pcap .', '\$', 60) mySSH.command('zip spgw.log.zip spgw*_check_run.*', '\$', 60) + elif re.match('OAICN5G', self.Type, re.IGNORECASE): + mySSH.command('cd ' + self.SourceCodePath + '/logs','\$', 5) + mySSH.command('zip spgw.log.zip oai-smf.log oai-spgwu.log','\$', 30) + mySSH.command('mv spgw.log.zip ' + self.SourceCodePath + '/scripts','\$', 30) elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): mySSH.command('zip spgw.log.zip spgw*.log', '\$', 60) mySSH.command('echo ' + self.Password + ' | sudo -S rm spgw*.log', '\$', 5) @@ -701,17 +871,3 @@ class EPCManagement(): else: logging.error('This option should not occur!') mySSH.close() - - def LogCollectOAICN5G(self): - mySSH = SSH.SSHConnection() - mySSH.open(self.IPAddress, self.UserName, self.Password) - logging.debug('OAI CN5G Collecting Log files to workspace') - mySSH.command('echo ' + self.Password + ' | sudo rm -rf ' + self.SourceCodePath + '/logs', '\$', 5) - mySSH.command('mkdir ' + self.SourceCodePath + '/logs','\$', 5) - containers_list=['oai-smf','oai-spgwu','oai-amf','oai-nrf'] - for c in containers_list: - mySSH.command('docker logs ' + c + ' > ' + self.SourceCodePath + '/logs/' + c + '.log', '\$', 5) - mySSH.command('cd ' + self.SourceCodePath + '/logs', '\$', 5) - mySSH.command('zip oai-cn5g.log.zip *.log', '\$', 60) - mySSH.close() - diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 9923d515a4..e9acdc9152 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -410,6 +410,17 @@ def GetParametersFromXML(action): if (string_field is not None): CONTAINERS.cliOptions = string_field + elif action == 'Copy_Image_to_Test': + string_field = test.findtext('image_name') + if (string_field is not None): + CONTAINERS.imageToCopy = string_field + string_field = test.findtext('registry_svr_id') + if (string_field is not None): + CONTAINERS.registrySvrId = string_field + string_field = test.findtext('test_svr_id') + if (string_field is not None): + CONTAINERS.testSvrId = string_field + else: # ie action == 'Run_PhySim': ldpc.runargs = test.findtext('physim_run_args') @@ -533,7 +544,7 @@ elif re.match('^TerminateOAIUE$', mode, re.IGNORECASE): HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') signal.signal(signal.SIGUSR1, receive_signal) - CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) + CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) elif re.match('^TerminateHSS$', mode, re.IGNORECASE): if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.Type == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) @@ -734,6 +745,22 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re HTML.SethtmlUEConnected(len(CiTestObj.UEDevices) + len(CiTestObj.CatMDevices)) HTML.CreateHtmlTabHeader() + # On CI bench w/ containers, we need to validate if IP routes are set + if EPC.IPAddress == '192.168.18.210': + CONTAINERS.CheckAndAddRoute('porcepix', EPC.IPAddress, EPC.UserName, EPC.Password) + if CONTAINERS.eNBIPAddress == '192.168.18.194': + CONTAINERS.CheckAndAddRoute('asterix', CONTAINERS.eNBIPAddress, CONTAINERS.eNBUserName, CONTAINERS.eNBPassword) + if CONTAINERS.eNB1IPAddress == '192.168.18.194': + CONTAINERS.CheckAndAddRoute('asterix', CONTAINERS.eNB1IPAddress, CONTAINERS.eNB1UserName, CONTAINERS.eNB1Password) + if CONTAINERS.eNBIPAddress == '192.168.18.193': + CONTAINERS.CheckAndAddRoute('obelix', CONTAINERS.eNBIPAddress, CONTAINERS.eNBUserName, CONTAINERS.eNBPassword) + if CONTAINERS.eNB1IPAddress == '192.168.18.193': + CONTAINERS.CheckAndAddRoute('obelix', CONTAINERS.eNB1IPAddress, CONTAINERS.eNB1UserName, CONTAINERS.eNB1Password) + if CONTAINERS.eNBIPAddress == '192.168.18.209': + CONTAINERS.CheckAndAddRoute('nepes', CONTAINERS.eNBIPAddress, CONTAINERS.eNBUserName, CONTAINERS.eNBPassword) + if CONTAINERS.eNB1IPAddress == '192.168.18.209': + CONTAINERS.CheckAndAddRoute('nepes', CONTAINERS.eNB1IPAddress, CONTAINERS.eNB1UserName, CONTAINERS.eNB1Password) + CiTestObj.FailReportCnt = 0 RAN.prematureExit=True HTML.startTime=int(round(time.time() * 1000)) @@ -782,39 +809,39 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re elif action == 'Terminate_eNB': RAN.TerminateeNB(HTML, EPC) elif action == 'Initialize_UE': - CiTestObj.InitializeUE(HTML,RAN, EPC, COTS_UE, InfraUE, CiTestObj.ue_trace) + CiTestObj.InitializeUE(HTML,RAN, EPC, COTS_UE, InfraUE, CiTestObj.ue_trace, CONTAINERS) elif action == 'Terminate_UE': CiTestObj.TerminateUE(HTML,COTS_UE, InfraUE, CiTestObj.ue_trace) elif action == 'Attach_UE': - CiTestObj.AttachUE(HTML,RAN,EPC,COTS_UE,InfraUE) + CiTestObj.AttachUE(HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS) elif action == 'Detach_UE': - CiTestObj.DetachUE(HTML,RAN,EPC,COTS_UE,InfraUE) + CiTestObj.DetachUE(HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS) elif action == 'DataDisable_UE': CiTestObj.DataDisableUE(HTML) elif action == 'DataEnable_UE': CiTestObj.DataEnableUE(HTML) elif action == 'CheckStatusUE': - CiTestObj.CheckStatusUE(HTML,RAN,EPC,COTS_UE,InfraUE) + CiTestObj.CheckStatusUE(HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS) elif action == 'Build_OAI_UE': CiTestObj.BuildOAIUE(HTML) elif action == 'Initialize_OAI_UE': - CiTestObj.InitializeOAIUE(HTML,RAN,EPC,COTS_UE,InfraUE) + CiTestObj.InitializeOAIUE(HTML,RAN,EPC,COTS_UE,InfraUE,CONTAINERS) elif action == 'Terminate_OAI_UE': - CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) + CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) elif action == 'Initialize_CatM_module': CiTestObj.InitializeCatM(HTML) elif action == 'Terminate_CatM_module': CiTestObj.TerminateCatM(HTML) elif action == 'Attach_CatM_module': - CiTestObj.AttachCatM(HTML,RAN,COTS_UE,EPC,InfraUE) + CiTestObj.AttachCatM(HTML,RAN,COTS_UE,EPC,InfraUE,CONTAINERS) elif action == 'Detach_CatM_module': CiTestObj.TerminateCatM(HTML) elif action == 'Ping_CatM_module': - CiTestObj.PingCatM(HTML,RAN,EPC,COTS_UE,EPC,InfraUE) + CiTestObj.PingCatM(HTML,RAN,EPC,COTS_UE,EPC,InfraUE,CONTAINERS) elif action == 'Ping': - CiTestObj.Ping(HTML,RAN,EPC,COTS_UE, InfraUE) + CiTestObj.Ping(HTML,RAN,EPC,COTS_UE, InfraUE, CONTAINERS) elif action == 'Iperf': - CiTestObj.Iperf(HTML,RAN,EPC,COTS_UE, InfraUE) + CiTestObj.Iperf(HTML,RAN,EPC,COTS_UE, InfraUE, CONTAINERS) elif action == 'Reboot_UE': CiTestObj.RebootUE(HTML,RAN,EPC) elif action == 'Initialize_HSS': @@ -853,6 +880,8 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re HTML=ldpc.Run_PhySim(HTML,CONST,id) elif action == 'Build_Image': CONTAINERS.BuildImage(HTML) + elif action == 'Copy_Image_to_Test': + CONTAINERS.Copy_Image_to_Test_Server(HTML) elif action == 'Deploy_Object': CONTAINERS.DeployObject(HTML, EPC) elif action == 'Undeploy_Object': diff --git a/ci-scripts/mysql4testresults/docker-compose.yml b/ci-scripts/mysql4testresults/docker-compose.yml new file mode 100644 index 0000000000..1c60057ab2 --- /dev/null +++ b/ci-scripts/mysql4testresults/docker-compose.yml @@ -0,0 +1,12 @@ +version: '2' +services: + mysql: + container_name: oaicicd_mysql + restart: always + image: mysql:latest + environment: + MYSQL_ROOT_PASSWORD: 'ucZBc2XRYdvEm59F' + ports: + - "3307:3306" + volumes: + - /home/oaicicd/mysql/data:/var/lib/mysql diff --git a/ci-scripts/mysql4testresults/sql_connect.py b/ci-scripts/mysql4testresults/sql_connect.py new file mode 100644 index 0000000000..333e573e78 --- /dev/null +++ b/ci-scripts/mysql4testresults/sql_connect.py @@ -0,0 +1,33 @@ +import pymysql +import sys +from datetime import datetime + +#This is the script used to write the test results to the mysql DB +#Called by Jenkins pipeline (jenkinsfile) +#Must be located in /home/oaicicid/mysql on the database host +#Usage from Jenkinsfile : +#python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} '' + +class SQLConnect: + def __init__(self): + self.connection = pymysql.connect( + host='172.22.0.2', + user='root', + password = 'ucZBc2XRYdvEm59F', + db='oaicicd_tests', + port=3306 + ) + + def put(self,TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS): + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + cur=self.connection.cursor() + cur.execute ('INSERT INTO test_results (TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS,DATE) VALUES (%s,%s,%s,%s,%s,%s,%s);' , (TEST, MR, BRANCH, BUILD, BUILD_LINK, STATUS, now)) + self.connection.commit() + self.connection.close() + + +if __name__ == "__main__": + mydb=SQLConnect() + mydb.put(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6]) + + diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 9acb1de34e..207e62bf9c 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -165,16 +165,16 @@ class RANManagement(): result = re.search('LAST_BUILD_INFO', mySSH.getBefore()) if result is not None: mismatch = False - mySSH.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2) + mySSH.command('grep --colour=never SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2) result = re.search(self.ranCommitID, mySSH.getBefore()) if result is None: mismatch = True - mySSH.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2) + mySSH.command('grep --colour=never MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2) if (self.ranAllowMerge): result = re.search('YES', mySSH.getBefore()) if result is None: mismatch = True - mySSH.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2) + mySSH.command('grep --colour=never TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2) if self.ranTargetBranch == '': result = re.search('develop', mySSH.getBefore()) else: @@ -374,7 +374,7 @@ class RANManagement(): logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m') pcapfile = pcapfile_prefix + self.testCase_id + '_log.pcap' mySSH.command('echo ' + lPassWord + ' | sudo -S rm -f /tmp/' + pcapfile , '\$', 5) - mySSH.command('echo $USER; nohup sudo -E tshark -i ' + eth_interface + ' -w /tmp/' + pcapfile + ' 2>&1 &','\$', 5) + mySSH.command('echo $USER; nohup sudo -E tshark -i ' + eth_interface + ' -w /tmp/' + pcapfile + ' > /dev/null 2>&1 &','\$', 5) mySSH.close() @@ -423,13 +423,13 @@ class RANManagement(): # do not reset board twice in IF4.5 case result = re.search('^rru|^enb|^du.band', str(config_file)) if result is not None: - mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 90) + mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 180) result = re.search('type: b200', mySSH.getBefore()) if result is not None: logging.debug('Found a B2xx device --> resetting it') mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10) # Reloading FGPA bin firmware - mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 90) + mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 180) # Make a copy and adapt to EPC / eNB IP addresses mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5) localMmeIpAddr = EPC.MmeIPAddress @@ -446,7 +446,7 @@ class RANManagement(): else: mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED = "no";/\' ' + ci_full_config_file, '\$', 2); self.eNBmbmsEnables[int(self.eNB_instance)] = False - mySSH.command('grep enable_enb_m2 ' + ci_full_config_file, '\$', 2); + mySSH.command('grep --colour=never enable_enb_m2 ' + ci_full_config_file, '\$', 2); result = re.search('yes', mySSH.getBefore()) if result is not None: self.eNBmbmsEnables[int(self.eNB_instance)] = True @@ -593,8 +593,12 @@ class RANManagement(): lPassWord = self.eNBPassword mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) - mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + self.air_interface[self.eNB_instance] + ' | grep -v grep', '\$', 5) - result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore()) + if self.air_interface[self.eNB_instance] == '': + pattern = 'softmodem' + else: + pattern = self.air_interface[self.eNB_instance] + mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + pattern + ' | grep -v grep', '\$', 5) + result = re.search(pattern, mySSH.getBefore()) if result is None: logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m') status_queue.put(CONST.ENB_PROCESS_FAILED) @@ -734,8 +738,8 @@ class RANManagement(): mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor.png', '\$', 60) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *.pickle *.png', '\$', 5) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 60) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png log/*/*.log log/*/*.pcap', '\$', 15) mySSH.close() def AnalyzeLogFile_eNB(self, eNBlogFile, HTML): @@ -792,26 +796,25 @@ class RANManagement(): pb_receiving_samples_cnt = 0 #count "removing UE" msg removing_ue = 0 + #count"X2AP-PDU" + x2ap_pdu = 0 #NSA specific log markers nsa_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': []} + nodeB_prefix_found = False - #the datalog config file has to be loaded - datalog_rt_stats_file='datalog_rt_stats.yaml' - if (os.path.isfile(datalog_rt_stats_file)): - yaml_file=datalog_rt_stats_file - elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)): - yaml_file='ci-scripts/'+datalog_rt_stats_file - else: - logging.error("Datalog RT stats yaml file cannot be found") - sys.exit("Datalog RT stats yaml file cannot be found") - - with open(yaml_file,'r') as f: - datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader) - rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field - line_cnt=0 #log file line counter for line in enb_log_file.readlines(): line_cnt+=1 + # Detection of eNB/gNB from a container log + result = re.search('Starting eNB soft modem', str(line)) + if result is not None: + nodeB_prefix_found = True + nodeB_prefix = 'e' + result = re.search('Starting gNB soft modem', str(line)) + if result is not None: + nodeB_prefix_found = True + nodeB_prefix = 'g' + result = re.search('Run time:' ,str(line)) # Runtime statistics result = re.search('Run time:' ,str(line)) if result is not None: @@ -962,7 +965,7 @@ class RANManagement(): if result is not None: mbmsRequestMsg += 1 #FR1 NSA test : add new markers to make sure gNB is used - result = re.search('\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI [0-9a-fA-F]+ received correctly, adding UE MAC Context UE_id [0-9]+\/RNTI [0-9a-fA-F]+', str(line)) + result = re.search('\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI 0x[0-9a-fA-F]+ received correctly, adding UE MAC Context UE_id [0-9]+\/RNTI 0x[0-9a-fA-F]+', str(line)) if result is not None: NSA_RAPROC_PUSCH_check = 1 #dlsch and ulsch statistics @@ -975,15 +978,7 @@ class RANManagement(): if result is not None: #remove 1- all useless char before relevant info (ulsch or dlsch) 2- trailing char dlsch_ulsch_stats[k]=re.sub(r'^.*\]\s+', r'' , line.rstrip()) - #real time statistics for gNB - for k in rt_keys: - result = re.search(k, line) - if result is not None: - #remove 1- all useless char before relevant info 2- trailing char - line=line.replace('[0m','') - tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex - if tmp!=None: #with ULULULUULULULLLL at the head of the line, we skip it - real_time_stats[k]=tmp.group(1) + #count "problem receiving samples" msg result = re.search('\[PHY\]\s+problem receiving samples', str(line)) @@ -993,7 +988,10 @@ class RANManagement(): result = re.search('\[MAC\]\s+Removing UE', str(line)) if result is not None: removing_ue += 1 - + #count "X2AP-PDU" + result = re.search('X2AP-PDU', str(line)) + if result is not None: + x2ap_pdu += 1 #nsa markers logging for k in nsa_markers: result = re.search(k, line) @@ -1001,11 +999,60 @@ class RANManagement(): nsa_markers[k].append(line_cnt) enb_log_file.close() - logging.debug(' File analysis completed') - if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'): - nodeB_prefix = 'e' + + + #the following part takes the *_stats.log files as source (not the stdout log file) + + #the datalog config file has to be loaded + datalog_rt_stats_file='datalog_rt_stats.yaml' + if (os.path.isfile(datalog_rt_stats_file)): + yaml_file=datalog_rt_stats_file + elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)): + yaml_file='ci-scripts/'+datalog_rt_stats_file else: - nodeB_prefix = 'g' + logging.error("Datalog RT stats yaml file cannot be found") + sys.exit("Datalog RT stats yaml file cannot be found") + + with open(yaml_file,'r') as f: + datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader) + rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field + + if (os.path.isfile('./nrL1_stats.log')) and (os.path.isfile('./nrL1_stats.log')): + stat_files_present=True + else: + stat_files_present=False + logging.debug("NR Stats files for RT analysis not found") + if stat_files_present: + nrL1_stats = open('./nrL1_stats.log', 'r') + nrMAC_stats = open('./nrMAC_stats.log', 'r') + for line in nrL1_stats.readlines(): + for k in rt_keys: + result = re.search(k, line) + if result is not None: + #remove 1- all useless char before relevant info 2- trailing char + tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex + if tmp!=None: + real_time_stats[k]=tmp.group(1) + for line in nrMAC_stats.readlines(): + for k in rt_keys: + result = re.search(k, line) + if result is not None: + #remove 1- all useless char before relevant info 2- trailing char + tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex + if tmp!=None: + real_time_stats[k]=tmp.group(1) + nrL1_stats.close() + nrMAC_stats.close() + + #stdout log file and stat log files analysis completed + logging.debug(' File analysis (stdout, stats) completed') + + #post processing depending on the node type + if not nodeB_prefix_found: + if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'): + nodeB_prefix = 'e' + else: + nodeB_prefix = 'g' if nodeB_prefix == 'g': if ulschReceiveOK > 0: @@ -1087,6 +1134,11 @@ class RANManagement(): htmlMsg = statMsg+'\n' logging.debug(statMsg) htmleNBFailureMsg += htmlMsg + #X2AP-PDU log + statMsg = 'X2AP-PDU msg count = '+str(x2ap_pdu) + htmlMsg = statMsg+'\n' + logging.debug(statMsg) + htmleNBFailureMsg += htmlMsg #nsa markers statMsg = 'logfile line count = ' + str(line_cnt) htmlMsg = statMsg+'\n' diff --git a/ci-scripts/ran_dashboard/Hdashboard.py b/ci-scripts/ran_dashboard/Hdashboard.py new file mode 100644 index 0000000000..5e34ae50bf --- /dev/null +++ b/ci-scripts/ran_dashboard/Hdashboard.py @@ -0,0 +1,401 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# Merge Requests Dashboard for RAN on googleSheet +# +# Required Python Version +# Python 3.x +# +#--------------------------------------------------------------------- + +#----------------------------------------------------------- +# Import +#----------------------------------------------------------- + + +#Author Remi +import boto3 +import shlex +import subprocess +import json #json structures +import datetime #now() and date formating +from datetime import datetime +import re +import gitlab +import yaml +import os +import time + + +from sqlconnect import SQLConnect + +#----------------------------------------------------------- +# Class Declaration +#----------------------------------------------------------- + +class Dashboard: + def __init__(self): + + + #init with data sources : git, yaml config file, test results databases + print("Collecting Data") + cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ + self.git = self.__getGitData(cmd) #git data from Gitlab + self.tests = self.__loadCfg('ran_dashboard_cfg.yaml') #tests table setup from yaml + self.db = self.__loadFromDB() #test results from database + + + def __loadCfg(self,yaml_file): + with open(yaml_file,'r') as f: + tests = yaml.load(f) + return tests + + def __getGitData(self,cmd): + #cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ + process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + output = process.stdout.readline() + tmp=output.decode("utf-8") + d = json.loads(tmp) + return d + + def __loadFromDB(self): + mr_list=[] + for x in range(len(self.git)): + mr_list.append(str(self.git[x]['iid'])) + mydb=SQLConnect() + for MR in mr_list: + mydb.get(MR) + mydb.close_connection() + return mydb.data + + def Test_initHTML(self, date): + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('
| OAI RAN TEST Status Dashboard | \n') + self.f_html.write('\n') + self.f_html.write('
| Merge Requests Dashboard | \n') + self.f_html.write('\n') + self.f_html.write('
| Update : '+date+' | \n') + self.f_html.write('\n') + self.f_html.write('