maki/openairinterface5g
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ci): display state of container when failure, better tshark filtering, auto-undeploy if deploy went wrong

Browse Source 
Signed-off-by: Raphael Defosseux <raphael.defosseux@eurecom.fr>
This commit is contained in:
Raphael Defosseux
2022-02-21 16:36:34 +01:00
parent 454aae1d96
commit 0b7be7466e
9 changed files with 44 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci-scripts/cls_containerize.py
+21 -6
View File
@@ -636,7 +636,7 @@ class Containerize():
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + self.eNB_logFile[self.eNB_instance], self.eNBSourceCodePath + '/cmake_targets/') mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + self.eNB_logFile[self.eNB_instance], self.eNBSourceCodePath + '/cmake_targets/')
logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
def DeployGenObject(self, HTML): def DeployGenObject(self, HTML, RAN, UE):
self.exitStatus = 0 self.exitStatus = 0
logging.info('\u001B[1m Checking Services to deploy\u001B[0m') logging.info('\u001B[1m Checking Services to deploy\u001B[0m')
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose config --services' cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose config --services'
@@ -677,14 +677,17 @@ class Containerize():
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps -a' cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps -a'
count = 0 count = 0
healthy = 0 healthy = 0
containerStatus = []
while (count < 10): while (count < 10):
count += 1 count += 1
containerStatus = []
deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30)
healthy = 0 healthy = 0
for state in deployStatus.split('\n'): for state in deployStatus.split('\n'):
res = re.search('Up \(healthy\)', state) res = re.search('Up \(healthy\)', state)
if res is not None: if res is not None:
healthy += 1 healthy += 1
containerStatus.append(state)
if healthy == self.nb_healthy[0]: if healthy == self.nb_healthy[0]:
count = 100 count = 100
else: else:
@@ -694,27 +697,39 @@ class Containerize():
if self.tsharkStarted == False: if self.tsharkStarted == False:
logging.debug('Starting tshark on public network') logging.debug('Starting tshark on public network')
self.CaptureOnDockerNetworks() self.CaptureOnDockerNetworks()
self.tsharkStarted = True
HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK)
logging.info('\u001B[1m Deploying OAI Object(s) PASS\u001B[0m') logging.info('\u001B[1m Deploying OAI Object(s) PASS\u001B[0m')
else: else:
self.exitStatus = 1
HTML.CreateHtmlTestRow('Could not deploy in time', 'KO', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlTestRow('Could not deploy in time', 'KO', CONST.ALL_PROCESSES_OK)
for cState in containerStatus:
logging.debug(cState)
logging.error('\u001B[1m Deploying OAI Object(s) FAILED\u001B[0m') logging.error('\u001B[1m Deploying OAI Object(s) FAILED\u001B[0m')
HTML.testCase_id = 'AUTO-UNDEPLOY'
UE.testCase_id = 'AUTO-UNDEPLOY'
HTML.desc = 'Automatic Undeployment'
UE.desc = 'Automatic Undeployment'
UE.ShowTestID()
self.UndeployGenObject(HTML, RAN, UE)
self.exitStatus = 1
def CaptureOnDockerNetworks(self): def CaptureOnDockerNetworks(self):
cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"' cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"'
networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
cmd = 'sudo nohup tshark -f "not tcp and not arp and not port 53 and not host archive.ubuntu.com and not host security.ubuntu.com and not port 2152"' if re.search('4g.*rfsimulator', self.yamlPath[0]) is not None:
cmd = 'sudo nohup tshark -f "(host 192.168.61.11 and icmp) or (not host 192.168.61.11 and not host 192.168.61.30 and not arp and not port 53 and not port 2152)"'
elif re.search('5g.*rfsimulator', self.yamlPath[0]) is not None:
cmd = 'sudo nohup tshark -f "(host 192.168.72.135 and icmp) or (not host 192.168.72.135 and not host 192.168.71.150 and not arp and not port 53 and not port 2152 and not port 2153)"'
else:
return
for name in networkNames.split('\n'): for name in networkNames.split('\n'):
res = re.search('rfsim', name) if re.search('rfsim', name) is not None:
if res is not None:
cmd += ' -i ' + name cmd += ' -i ' + name
cmd += ' -w /tmp/capture_' cmd += ' -w /tmp/capture_'
ymlPath = self.yamlPath[0].split('/') ymlPath = self.yamlPath[0].split('/')
cmd += ymlPath[1] + '.pcap > /tmp/tshark.log 2>&1 &' cmd += ymlPath[1] + '.pcap > /tmp/tshark.log 2>&1 &'
logging.debug(cmd) logging.debug(cmd)
networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10)
self.tsharkStarted = True
def UndeployGenObject(self, HTML, RAN, UE): def UndeployGenObject(self, HTML, RAN, UE):
self.exitStatus = 0 self.exitStatus = 0
ci-scripts/main.py
+1 -1
View File
@@ -910,7 +910,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
elif action == 'Deploy_Run_PhySim': elif action == 'Deploy_Run_PhySim':
PHYSIM.Deploy_PhySim(HTML, RAN) PHYSIM.Deploy_PhySim(HTML, RAN)
elif action == 'DeployGenObject': elif action == 'DeployGenObject':
CONTAINERS.DeployGenObject(HTML) CONTAINERS.DeployGenObject(HTML, RAN, CiTestObj)
if CONTAINERS.exitStatus==1: if CONTAINERS.exitStatus==1:
RAN.prematureExit = True RAN.prematureExit = True
elif action == 'UndeployGenObject': elif action == 'UndeployGenObject':
ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml
+1 -1
View File
@@ -80,7 +80,7 @@
<testCase id="000014"> <testCase id="000014">
<class>DeployGenObject</class> <class>DeployGenObject</class>
<desc>Deploy OAI 4G NR-UE RF sim (FDD 05MHz)</desc> <desc>Deploy OAI 4G LTE-UE RF sim (FDD 05MHz)</desc>
<yaml_path>yaml_files/4g_rfsimulator_fdd_05MHz</yaml_path> <yaml_path>yaml_files/4g_rfsimulator_fdd_05MHz</yaml_path>
<services>oai_ue0</services> <services>oai_ue0</services>
<nb_healthy>8</nb_healthy> <nb_healthy>8</nb_healthy>
ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml
+1 -1
View File
@@ -60,7 +60,7 @@
<testCase id="000014"> <testCase id="000014">
<class>DeployGenObject</class> <class>DeployGenObject</class>
<desc>Deploy OAI 4G NR-UE RF sim (FDD 05MHz)</desc> <desc>Deploy OAI 4G LTE-UE RF sim (FDD 05MHz)</desc>
<yaml_path>yaml_files/4g_rfsimulator_fdd_05MHz_noS1</yaml_path> <yaml_path>yaml_files/4g_rfsimulator_fdd_05MHz_noS1</yaml_path>
<services>oai_ue0</services> <services>oai_ue0</services>
<nb_healthy>2</nb_healthy> <nb_healthy>2</nb_healthy>
ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml
+1 -1
View File
@@ -80,7 +80,7 @@
<testCase id="000014"> <testCase id="000014">
<class>DeployGenObject</class> <class>DeployGenObject</class>
<desc>Deploy OAI 4G NR-UE RF sim (FDD 10MHz)</desc> <desc>Deploy OAI 4G LTE-UE RF sim (FDD 10MHz)</desc>
<yaml_path>yaml_files/4g_rfsimulator_fdd_10MHz</yaml_path> <yaml_path>yaml_files/4g_rfsimulator_fdd_10MHz</yaml_path>
<services>oai_ue0</services> <services>oai_ue0</services>
<nb_healthy>8</nb_healthy> <nb_healthy>8</nb_healthy>
ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml
+1 -1
View File
@@ -80,7 +80,7 @@
<testCase id="000014"> <testCase id="000014">
<class>DeployGenObject</class> <class>DeployGenObject</class>
<desc>Deploy OAI 4G NR-UE RF sim (FDD 20MHz)</desc> <desc>Deploy OAI 4G LTE-UE RF sim (FDD 20MHz)</desc>
<yaml_path>yaml_files/4g_rfsimulator_fdd_20MHz</yaml_path> <yaml_path>yaml_files/4g_rfsimulator_fdd_20MHz</yaml_path>
<services>oai_ue0</services> <services>oai_ue0</services>
<nb_healthy>8</nb_healthy> <nb_healthy>8</nb_healthy>
ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml
+1 -1
View File
@@ -81,7 +81,7 @@
<testCase id="000014"> <testCase id="000014">
<class>DeployGenObject</class> <class>DeployGenObject</class>
<desc>Deploy OAI 4G NR-UE RF sim (TDD 05MHz)</desc> <desc>Deploy OAI 4G LTE-UE RF sim (TDD 05MHz)</desc>
<yaml_path>yaml_files/4g_rfsimulator_tdd_05MHz</yaml_path> <yaml_path>yaml_files/4g_rfsimulator_tdd_05MHz</yaml_path>
<services>oai_ue0</services> <services>oai_ue0</services>
<nb_healthy>8</nb_healthy> <nb_healthy>8</nb_healthy>
ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml
+12 -12
View File
@@ -241,12 +241,12 @@ services:
NSSAI_SD1: 112233 NSSAI_SD1: 112233
AMF_IP_ADDRESS: 192.168.71.132 AMF_IP_ADDRESS: 192.168.71.132
GNB_NGA_IF_NAME: eth0 GNB_NGA_IF_NAME: eth0
GNB_NGA_IP_ADDRESS: 192.168.71.136 GNB_NGA_IP_ADDRESS: 192.168.71.140
GNB_NGU_IF_NAME: eth0 GNB_NGU_IF_NAME: eth0
GNB_NGU_IP_ADDRESS: 192.168.71.136 GNB_NGU_IP_ADDRESS: 192.168.71.140
F1_IF_NAME: eth0 F1_IF_NAME: eth0
F1_CU_IP_ADDRESS: 192.168.71.136 F1_CU_IP_ADDRESS: 192.168.71.140
F1_DU_IP_ADDRESS: 192.168.71.138 F1_DU_IP_ADDRESS: 192.168.71.142
F1_CU_D_PORT: 2153 F1_CU_D_PORT: 2153
F1_DU_D_PORT: 2153 F1_DU_D_PORT: 2153
USE_ADDITIONAL_OPTIONS: --sa --rfsim --log_config.global_log_options level,nocolor,time USE_ADDITIONAL_OPTIONS: --sa --rfsim --log_config.global_log_options level,nocolor,time
@@ -254,7 +254,7 @@ services:
- oai-ext-dn - oai-ext-dn
networks: networks:
public_net: public_net:
ipv4_address: 192.168.71.136 ipv4_address: 192.168.71.140
healthcheck: healthcheck:
test: /bin/bash -c "pgrep nr-softmodem" test: /bin/bash -c "pgrep nr-softmodem"
interval: 10s interval: 10s
@@ -277,12 +277,12 @@ services:
NSSAI_SD1: 112233 NSSAI_SD1: 112233
AMF_IP_ADDRESS: 192.168.71.132 AMF_IP_ADDRESS: 192.168.71.132
GNB_NGA_IF_NAME: eth0 GNB_NGA_IF_NAME: eth0
GNB_NGA_IP_ADDRESS: 192.168.71.138 GNB_NGA_IP_ADDRESS: 192.168.71.142
GNB_NGU_IF_NAME: eth0 GNB_NGU_IF_NAME: eth0
GNB_NGU_IP_ADDRESS: 192.168.71.138 GNB_NGU_IP_ADDRESS: 192.168.71.142
F1_IF_NAME: eth0 F1_IF_NAME: eth0
F1_CU_IP_ADDRESS: 192.168.71.136 F1_CU_IP_ADDRESS: 192.168.71.140
F1_DU_IP_ADDRESS: 192.168.71.138 F1_DU_IP_ADDRESS: 192.168.71.142
F1_CU_D_PORT: 2153 F1_CU_D_PORT: 2153
F1_DU_D_PORT: 2153 F1_DU_D_PORT: 2153
USE_ADDITIONAL_OPTIONS: --sa --rfsim --log_config.global_log_options level,nocolor,time USE_ADDITIONAL_OPTIONS: --sa --rfsim --log_config.global_log_options level,nocolor,time
@@ -290,7 +290,7 @@ services:
- oai-cu - oai-cu
networks: networks:
public_net: public_net:
ipv4_address: 192.168.71.138 ipv4_address: 192.168.71.142
healthcheck: healthcheck:
test: /bin/bash -c "pgrep nr-softmodem" test: /bin/bash -c "pgrep nr-softmodem"
interval: 10s interval: 10s
@@ -301,7 +301,7 @@ services:
privileged: true privileged: true
container_name: rfsim5g-oai-nr-ue container_name: rfsim5g-oai-nr-ue
environment: environment:
RFSIMULATOR: 192.168.71.138 RFSIMULATOR: 192.168.71.142
FULL_IMSI: '208990100001100' FULL_IMSI: '208990100001100'
FULL_KEY: 'fec86ba6eb707ed08905757b1bb44b8f' FULL_KEY: 'fec86ba6eb707ed08905757b1bb44b8f'
OPC: 'C42449363BBAD02B66D16BC975D77CC1' OPC: 'C42449363BBAD02B66D16BC975D77CC1'
@@ -313,7 +313,7 @@ services:
- oai-du - oai-du
networks: networks:
public_net: public_net:
ipv4_address: 192.168.71.137 ipv4_address: 192.168.71.150
healthcheck: healthcheck:
test: /bin/bash -c "pgrep nr-uesoftmodem" test: /bin/bash -c "pgrep nr-uesoftmodem"
interval: 10s interval: 10s
ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml
+5 -5
View File
@@ -241,15 +241,15 @@ services:
NSSAI_SD1: 112233 NSSAI_SD1: 112233
AMF_IP_ADDRESS: 192.168.71.132 AMF_IP_ADDRESS: 192.168.71.132
GNB_NGA_IF_NAME: eth0 GNB_NGA_IF_NAME: eth0
GNB_NGA_IP_ADDRESS: 192.168.71.136 GNB_NGA_IP_ADDRESS: 192.168.71.140
GNB_NGU_IF_NAME: eth0 GNB_NGU_IF_NAME: eth0
GNB_NGU_IP_ADDRESS: 192.168.71.136 GNB_NGU_IP_ADDRESS: 192.168.71.140
USE_ADDITIONAL_OPTIONS: --sa -E --rfsim --log_config.global_log_options level,nocolor,time USE_ADDITIONAL_OPTIONS: --sa -E --rfsim --log_config.global_log_options level,nocolor,time
depends_on: depends_on:
- oai-ext-dn - oai-ext-dn
networks: networks:
public_net: public_net:
ipv4_address: 192.168.71.136 ipv4_address: 192.168.71.140
healthcheck: healthcheck:
test: /bin/bash -c "pgrep nr-softmodem" test: /bin/bash -c "pgrep nr-softmodem"
interval: 10s interval: 10s
@@ -260,7 +260,7 @@ services:
privileged: true privileged: true
container_name: rfsim5g-oai-nr-ue container_name: rfsim5g-oai-nr-ue
environment: environment:
RFSIMULATOR: 192.168.71.136 RFSIMULATOR: 192.168.71.140
FULL_IMSI: '208990100001100' FULL_IMSI: '208990100001100'
FULL_KEY: 'fec86ba6eb707ed08905757b1bb44b8f' FULL_KEY: 'fec86ba6eb707ed08905757b1bb44b8f'
OPC: 'C42449363BBAD02B66D16BC975D77CC1' OPC: 'C42449363BBAD02B66D16BC975D77CC1'
@@ -272,7 +272,7 @@ services:
- oai-gnb - oai-gnb
networks: networks:
public_net: public_net:
ipv4_address: 192.168.71.137 ipv4_address: 192.168.71.150
healthcheck: healthcheck:
test: /bin/bash -c "pgrep nr-uesoftmodem" test: /bin/bash -c "pgrep nr-uesoftmodem"
interval: 10s interval: 10s