diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 817555554d..12355ef6d7 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -152,19 +152,6 @@ def GetImageName(ssh, svcName, file): else: return ret.stdout.strip() -def GetServiceHealth(ssh, svcName, file): - if svcName is None: - return False, f"Service {svcName} not found in {file}" - image = GetImageName(ssh, svcName, file) - if 'db_init' in svcName or 'db-init' in svcName: # exits with 0, there cannot be healthy - return True, f"Service {svcName} healthy, image {image}" - for _ in range(8): - result = ssh.run(f"docker compose -f {file} ps --format json {svcName} | jq -r 'if type==\"array\" then .[0].Health else .Health end'", silent=True) - if result.stdout == 'healthy': - return True, f"Service {svcName} healthy, image {image}" - time.sleep(5) - return False, f"Failed to deploy: service {svcName}" - def ExistEnvFilePrint(ssh, wd, prompt='env vars in existing'): ret = ssh.run(f'cat {wd}/.env', silent=True, reportNonZero=False) if ret.returncode != 0: @@ -311,7 +298,6 @@ class Containerize(): self.yamlPath = ['', '', ''] self.services = ['', '', ''] self.deploymentTag = '' - self.eNB_logFile = ['', '', ''] self.testCase_id = '' @@ -320,17 +306,6 @@ class Containerize(): self.dockerfileprefix = '' self.host = '' - self.deployedContainers = [] - self.tsharkStarted = False - self.pingContName = '' - self.pingOptions = '' - self.pingLossThreshold = '' - self.svrContName = '' - self.svrOptions = '' - self.cliContName = '' - self.cliOptions = '' - - self.imageToCopy = '' #checkers from xml self.ran_checkers={} self.num_attempts = 1 @@ -872,31 +847,24 @@ class Containerize(): if num_attempts <= 0: raise ValueError(f'Invalid value for num_attempts: {num_attempts}, must be greater than 0') for attempt in range(num_attempts): - imagesInfo = [] - healthInfo = [] logging.info(f'will start services {services}') - status = ssh.run(f'docker compose -f {wd_yaml} up -d -- {services}') - if status.returncode != 0: + status = ssh.run(f'docker compose -f {wd_yaml} up -d --wait --wait-timeout 60 -- {services}') + info = ssh.run(f"docker compose -f {wd_yaml} ps --all --format=\'table {{{{.Service}}}} [{{{{.Image}}}}] {{{{.Status}}}}\' -- {services} | column -t") + deployed = status.returncode == 0 + if not deployed: msg = f'cannot deploy services {services}: {status.stdout}' logging.error(msg) - HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [msg]) - return False - for svc in services.split(): - health, msg = GetServiceHealth(ssh, svc, f'{wd_yaml}') - logging.info(msg) - imagesInfo.append(msg) - healthInfo.append(health) - deployed = all(healthInfo) - if deployed: + else: break - elif (attempt < num_attempts - 1): - warning_msg = f'Restart services {services}' + if (attempt < num_attempts - 1): + warning_msg = (f'Failed to deploy on attempt {attempt}, restart services {services}') logging.warning(warning_msg) - imagesInfo.append(warning_msg) - HTML.CreateHtmlTestRowQueue('N/A', 'NOK', ['\n'.join(imagesInfo)]) + HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [warning_msg]) for svc in services.split(): CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log') ssh.run(f'docker compose -f {wd_yaml} down -- {services}') + imagesInfo = info.stdout.splitlines()[1:] + logging.debug(f'{info.stdout.splitlines()[1:]}') if deployed: HTML.CreateHtmlTestRowQueue('N/A', 'OK', ['\n'.join(imagesInfo)]) else: diff --git a/ci-scripts/cls_module.py b/ci-scripts/cls_module.py index 3ec6ffc9e5..4ef5c67ccd 100644 --- a/ci-scripts/cls_module.py +++ b/ci-scripts/cls_module.py @@ -107,7 +107,7 @@ class Module_UE: return self._logCollect() return None - def attach(self, attach_tries = 4, attach_timeout = 60): + def attach(self, attach_tries = 2, attach_timeout = 60): ip = None while attach_tries > 0: self._command(self.cmd_dict["attach"]) diff --git a/ci-scripts/xml_files/container_5g_rfsim_fr2_32prb.xml b/ci-scripts/xml_files/container_5g_rfsim_fr2_32prb.xml index db10ff6478..1965f95aa0 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_fr2_32prb.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_fr2_32prb.xml @@ -29,6 +29,7 @@ 800813 000000 000001 + 000013 020001 020002 100001 @@ -68,6 +69,13 @@ 0 + + Attach_UE + Attach OAI UE (Wait for IP) + rfsim5g_ue + cacofonix + + Ping Ping ext-dn from NR-UE diff --git a/ci-scripts/yaml_files/4g_l2sim_fdd/docker-compose.yml b/ci-scripts/yaml_files/4g_l2sim_fdd/docker-compose.yml index 9f40fb856d..0d810a1d4b 100644 --- a/ci-scripts/yaml_files/4g_l2sim_fdd/docker-compose.yml +++ b/ci-scripts/yaml_files/4g_l2sim_fdd/docker-compose.yml @@ -10,7 +10,7 @@ services: CASSANDRA_ENDPOINT_SNITCH: GossipingPropertyFileSnitch healthcheck: test: /bin/bash -c "nodetool status" - interval: 10s + interval: 20s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml b/ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml index fb1d5a95ce..b4bec0e486 100644 --- a/ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_f1_rfsimulator/docker-compose.yaml @@ -2,6 +2,7 @@ services: mysql: container_name: "rfsim5g-mysql" image: mysql:8.0 + init: true volumes: - ../5g_rfsimulator/oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql - ../5g_rfsimulator/mysql-healthcheck.sh:/tmp/mysql-healthcheck.sh @@ -13,6 +14,8 @@ services: - MYSQL_ROOT_PASSWORD=linux healthcheck: test: /bin/bash -c "/tmp/mysql-healthcheck.sh" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 30 @@ -46,6 +49,7 @@ services: oai-upf: container_name: "rfsim5g-oai-upf" image: oaisoftwarealliance/oai-upf:v2.1.9 + init: true environment: - TZ=Europe/Paris volumes: @@ -68,6 +72,7 @@ services: privileged: true container_name: rfsim5g-oai-ext-dn image: oaisoftwarealliance/trf-gen-cn5g:focal + init: true entrypoint: /bin/bash -c \ "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\ "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; sleep infinity" @@ -78,6 +83,8 @@ services: ipv4_address: 192.168.72.135 healthcheck: test: /bin/bash -c "ping -c 2 192.168.72.134" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -103,6 +110,8 @@ services: - ../../conf_files/gnb-cu.sa.band78.106prb.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -130,6 +139,8 @@ services: - ../../conf_files/gnb-du.sa.band78.106prb.rfsim.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -161,6 +172,8 @@ services: - ../../conf_files/gnb-du.sa.band78.106prb.rfsim.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -193,6 +206,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_fdd_rfsimulator/docker-compose.yaml b/ci-scripts/yaml_files/5g_fdd_rfsimulator/docker-compose.yaml index a40033be7f..dd306c15f6 100644 --- a/ci-scripts/yaml_files/5g_fdd_rfsimulator/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_fdd_rfsimulator/docker-compose.yaml @@ -2,6 +2,7 @@ services: mysql: container_name: "rfsim5g-mysql" image: mysql:8.0 + init: true volumes: - ../5g_rfsimulator/oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql - ../5g_rfsimulator/mysql-healthcheck.sh:/tmp/mysql-healthcheck.sh @@ -15,6 +16,8 @@ services: test: /bin/bash -c "/tmp/mysql-healthcheck.sh" interval: 10s timeout: 5s + start_period: 10s + start_interval: 500ms retries: 30 networks: public_net: @@ -46,6 +49,7 @@ services: oai-upf: container_name: "rfsim5g-oai-upf" image: oaisoftwarealliance/oai-upf:v2.1.9 + init: true environment: - TZ=Europe/Paris volumes: @@ -67,6 +71,7 @@ services: privileged: true container_name: rfsim5g-oai-ext-dn image: oaisoftwarealliance/trf-gen-cn5g:focal + init: true entrypoint: /bin/bash -c \ "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\ "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; sleep infinity" @@ -80,6 +85,8 @@ services: interval: 10s timeout: 5s retries: 5 + start_period: 10s + start_interval: 500ms oai-gnb: image: ${REGISTRY:-oaisoftwarealliance}/${GNB_IMG:-oai-gnb}:${TAG:-develop} container_name: rfsim5g-oai-gnb @@ -99,6 +106,8 @@ services: test: /bin/bash -c "pgrep nr-softmodem" interval: 10s timeout: 5s + start_period: 10s + start_interval: 500ms retries: 5 ipc: host oai-nr-ue: @@ -126,6 +135,8 @@ services: interval: 10s timeout: 5s retries: 5 + start_period: 10s + start_interval: 500ms ipc: host networks: diff --git a/ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml b/ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml index 4733737b50..ac89f0d036 100644 --- a/ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_rfsimulator/docker-compose.yaml @@ -2,6 +2,7 @@ services: mysql: container_name: "rfsim5g-mysql" image: mysql:8.0 + init: true volumes: - ./oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql - ./mysql-healthcheck.sh:/tmp/mysql-healthcheck.sh @@ -13,6 +14,8 @@ services: - MYSQL_ROOT_PASSWORD=linux healthcheck: test: /bin/bash -c "/tmp/mysql-healthcheck.sh" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 30 @@ -46,6 +49,7 @@ services: oai-upf: container_name: "rfsim5g-oai-upf" image: oaisoftwarealliance/oai-upf:v2.1.9 + init: true environment: - TZ=Europe/Paris volumes: @@ -67,6 +71,7 @@ services: privileged: true container_name: rfsim5g-oai-ext-dn image: oaisoftwarealliance/trf-gen-cn5g:focal + init: true entrypoint: /bin/bash -c \ "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\ "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; sleep infinity" @@ -77,6 +82,8 @@ services: ipv4_address: 192.168.72.135 healthcheck: test: /bin/bash -c "ping -c 2 192.168.72.134" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -97,6 +104,8 @@ services: - ../../conf_files/gnb.sa.band78.106prb.rfsim.yaml:/opt/oai-gnb/etc/gnb.yaml healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -121,6 +130,8 @@ services: - ../../conf_files/nrue.uicc.yaml:/opt/oai-nr-ue/etc/nr-ue.yaml healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -145,6 +156,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -169,6 +182,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -193,6 +208,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -217,6 +234,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -241,6 +260,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -265,6 +286,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -289,6 +312,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -313,6 +338,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -337,6 +364,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_rfsimulator_24prb/docker-compose.yaml b/ci-scripts/yaml_files/5g_rfsimulator_24prb/docker-compose.yaml index 165d2791e7..8dfe5a5980 100644 --- a/ci-scripts/yaml_files/5g_rfsimulator_24prb/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_rfsimulator_24prb/docker-compose.yaml @@ -2,6 +2,7 @@ services: mysql: container_name: "rfsim5g-mysql" image: mysql:8.0 + init: true volumes: - ../5g_rfsimulator/oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql - ../5g_rfsimulator/mysql-healthcheck.sh:/tmp/mysql-healthcheck.sh @@ -13,6 +14,8 @@ services: - MYSQL_ROOT_PASSWORD=linux healthcheck: test: /bin/bash -c "/tmp/mysql-healthcheck.sh" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 30 @@ -46,6 +49,7 @@ services: oai-upf: container_name: "rfsim5g-oai-upf" image: oaisoftwarealliance/oai-upf:v2.1.9 + init: true environment: - TZ=Europe/Paris volumes: @@ -67,6 +71,7 @@ services: privileged: true container_name: rfsim5g-oai-ext-dn image: oaisoftwarealliance/trf-gen-cn5g:focal + init: true entrypoint: /bin/bash -c \ "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\ "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; sleep infinity" @@ -77,6 +82,8 @@ services: ipv4_address: 192.168.72.135 healthcheck: test: /bin/bash -c "ping -c 2 192.168.72.134" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -97,6 +104,8 @@ services: - ../../conf_files/gnb.sa.band78.24prb.rfsim.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -124,6 +133,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_rfsimulator_2x2/docker-compose.yaml b/ci-scripts/yaml_files/5g_rfsimulator_2x2/docker-compose.yaml index b051712a4c..3231a2f76e 100644 --- a/ci-scripts/yaml_files/5g_rfsimulator_2x2/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_rfsimulator_2x2/docker-compose.yaml @@ -2,6 +2,7 @@ services: mysql: container_name: "rfsim5g-mysql" image: mysql:8.0 + init: true volumes: - ../5g_rfsimulator/oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql - ../5g_rfsimulator/mysql-healthcheck.sh:/tmp/mysql-healthcheck.sh @@ -13,6 +14,8 @@ services: - MYSQL_ROOT_PASSWORD=linux healthcheck: test: /bin/bash -c "/tmp/mysql-healthcheck.sh" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 30 @@ -46,6 +49,7 @@ services: oai-upf: container_name: "rfsim5g-oai-upf" image: oaisoftwarealliance/oai-upf:v2.1.9 + init: true environment: - TZ=Europe/Paris volumes: @@ -67,6 +71,7 @@ services: privileged: true container_name: rfsim5g-oai-ext-dn image: oaisoftwarealliance/trf-gen-cn5g:focal + init: true entrypoint: /bin/bash -c \ "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\ "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; sleep infinity" @@ -77,6 +82,8 @@ services: ipv4_address: 192.168.72.135 healthcheck: test: /bin/bash -c "ping -c 2 192.168.72.134" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -97,6 +104,8 @@ services: - ../../conf_files/gnb.sa.band78.273prb.rfsim.2x2.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -123,6 +132,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_rfsimulator_e1/docker-compose.yaml b/ci-scripts/yaml_files/5g_rfsimulator_e1/docker-compose.yaml index 69d27de863..66b746b6f2 100644 --- a/ci-scripts/yaml_files/5g_rfsimulator_e1/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_rfsimulator_e1/docker-compose.yaml @@ -2,6 +2,7 @@ services: mysql: container_name: "rfsim5g-mysql" image: mysql:8.0 + init: true volumes: - ../5g_rfsimulator/oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql - ../5g_rfsimulator/mysql-healthcheck.sh:/tmp/mysql-healthcheck.sh @@ -13,6 +14,8 @@ services: - MYSQL_ROOT_PASSWORD=linux healthcheck: test: /bin/bash -c "/tmp/mysql-healthcheck.sh" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 30 @@ -46,6 +49,7 @@ services: oai-upf: container_name: "rfsim5g-oai-upf" image: oaisoftwarealliance/oai-upf:v2.1.9 + init: true environment: - TZ=Europe/Paris volumes: @@ -87,6 +91,8 @@ services: - ../../conf_files/gnb-cucp.sa.f1.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -116,6 +122,8 @@ services: - ../../conf_files/gnb-cuup.sa.f1.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-cuup" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -148,6 +156,8 @@ services: - ../../conf_files/gnb-cuup.sa.f1.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-cuup" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -180,6 +190,8 @@ services: - ../../conf_files/gnb-cuup.sa.f1.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-cuup" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -209,6 +221,8 @@ services: - ../../conf_files/gnb-du.sa.band78.106prb.rfsim.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -241,6 +255,8 @@ services: - ../../conf_files/gnb-du.sa.band78.106prb.rfsim.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -273,6 +289,8 @@ services: - ../../conf_files/gnb-du.sa.band78.106prb.rfsim.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -301,6 +319,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -329,6 +349,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -358,6 +380,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_rfsimulator_fr2_32prb/docker-compose.yaml b/ci-scripts/yaml_files/5g_rfsimulator_fr2_32prb/docker-compose.yaml index 627fde5dce..413fdb1358 100644 --- a/ci-scripts/yaml_files/5g_rfsimulator_fr2_32prb/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_rfsimulator_fr2_32prb/docker-compose.yaml @@ -2,6 +2,7 @@ services: mysql: container_name: "rfsim5g-mysql" image: mysql:8.0 + init: true volumes: - ../5g_rfsimulator/oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql - ../5g_rfsimulator/mysql-healthcheck.sh:/tmp/mysql-healthcheck.sh @@ -13,6 +14,8 @@ services: - MYSQL_ROOT_PASSWORD=linux healthcheck: test: /bin/bash -c "/tmp/mysql-healthcheck.sh" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 30 @@ -46,6 +49,7 @@ services: oai-upf: container_name: "rfsim5g-oai-upf" image: oaisoftwarealliance/oai-upf:v2.1.9 + init: true environment: - TZ=Europe/Paris volumes: @@ -67,6 +71,7 @@ services: privileged: true container_name: rfsim5g-oai-ext-dn image: oaisoftwarealliance/trf-gen-cn5g:focal + init: true entrypoint: /bin/bash -c \ "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\ "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; sleep infinity" @@ -77,6 +82,8 @@ services: ipv4_address: 192.168.72.135 healthcheck: test: /bin/bash -c "ping -c 2 192.168.72.134" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -96,6 +103,8 @@ services: - ../../conf_files/gnb.sa.band261.u3.32prb.rfsim.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -124,6 +133,8 @@ services: ipv4_address: 192.168.71.150 healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_rfsimulator_ntn_geo/docker-compose.yaml b/ci-scripts/yaml_files/5g_rfsimulator_ntn_geo/docker-compose.yaml index 31390cde9c..79240aa876 100644 --- a/ci-scripts/yaml_files/5g_rfsimulator_ntn_geo/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_rfsimulator_ntn_geo/docker-compose.yaml @@ -2,6 +2,7 @@ services: mysql: container_name: "rfsim5g-mysql" image: mysql:8.0 + init: true volumes: - ../5g_rfsimulator/oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql - ../5g_rfsimulator/mysql-healthcheck.sh:/tmp/mysql-healthcheck.sh @@ -13,6 +14,8 @@ services: - MYSQL_ROOT_PASSWORD=linux healthcheck: test: /bin/bash -c "/tmp/mysql-healthcheck.sh" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 30 @@ -46,6 +49,7 @@ services: oai-upf: container_name: "rfsim5g-oai-upf" image: oaisoftwarealliance/oai-upf:v2.1.9 + init: true environment: - TZ=Europe/Paris volumes: @@ -67,6 +71,7 @@ services: privileged: true container_name: rfsim5g-oai-ext-dn image: oaisoftwarealliance/trf-gen-cn5g:focal + init: true entrypoint: /bin/bash -c \ "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\ "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; sleep infinity" @@ -77,6 +82,8 @@ services: ipv4_address: 192.168.72.135 healthcheck: test: /bin/bash -c "ping -c 2 192.168.72.134" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -97,6 +104,8 @@ services: - ../../conf_files/gnb.sa.band254.u0.25prb.rfsim.ntn.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -122,6 +131,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_rfsimulator_ntn_leo/docker-compose.yaml b/ci-scripts/yaml_files/5g_rfsimulator_ntn_leo/docker-compose.yaml index 29be9d6900..65de72fb51 100644 --- a/ci-scripts/yaml_files/5g_rfsimulator_ntn_leo/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_rfsimulator_ntn_leo/docker-compose.yaml @@ -13,6 +13,8 @@ services: - MYSQL_ROOT_PASSWORD=linux healthcheck: test: /bin/bash -c "/tmp/mysql-healthcheck.sh" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 30 @@ -77,6 +79,8 @@ services: ipv4_address: 192.168.72.135 healthcheck: test: /bin/bash -c "ping -c 2 192.168.72.134" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -97,6 +101,8 @@ services: - ../../conf_files/gnb.sa.band254.u0.25prb.rfsim.ntn-leo.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -129,6 +135,8 @@ services: - ../../conf_files/nrue.uicc.ntn-leo.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_rfsimulator_sidelink/docker-compose.yaml b/ci-scripts/yaml_files/5g_rfsimulator_sidelink/docker-compose.yaml index 35e6a4eee9..cae436e001 100644 --- a/ci-scripts/yaml_files/5g_rfsimulator_sidelink/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_rfsimulator_sidelink/docker-compose.yaml @@ -18,6 +18,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -43,6 +45,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_rfsimulator_u0_25prb/docker-compose.yaml b/ci-scripts/yaml_files/5g_rfsimulator_u0_25prb/docker-compose.yaml index cd0365b4ec..2dca6cc25f 100644 --- a/ci-scripts/yaml_files/5g_rfsimulator_u0_25prb/docker-compose.yaml +++ b/ci-scripts/yaml_files/5g_rfsimulator_u0_25prb/docker-compose.yaml @@ -2,6 +2,7 @@ services: mysql: container_name: "rfsim5g-mysql" image: mysql:8.0 + init: true volumes: - ../5g_rfsimulator/oai_db.sql:/docker-entrypoint-initdb.d/oai_db.sql - ../5g_rfsimulator/mysql-healthcheck.sh:/tmp/mysql-healthcheck.sh @@ -13,6 +14,8 @@ services: - MYSQL_ROOT_PASSWORD=linux healthcheck: test: /bin/bash -c "/tmp/mysql-healthcheck.sh" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 30 @@ -46,6 +49,7 @@ services: oai-upf: container_name: "rfsim5g-oai-upf" image: oaisoftwarealliance/oai-upf:v2.1.9 + init: true environment: - TZ=Europe/Paris volumes: @@ -67,6 +71,7 @@ services: privileged: true container_name: rfsim5g-oai-ext-dn image: oaisoftwarealliance/trf-gen-cn5g:focal + init: true entrypoint: /bin/bash -c \ "iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE;"\ "ip route add 12.1.1.0/24 via 192.168.72.134 dev eth0; sleep infinity" @@ -77,6 +82,8 @@ services: ipv4_address: 192.168.72.135 healthcheck: test: /bin/bash -c "ping -c 2 192.168.72.134" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -99,6 +106,8 @@ services: - ../../conf_files/gnb-vnf.sa.band66.u0.25prb.nfapi.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -121,6 +130,8 @@ services: - ../../conf_files/gnb-pnf.band66.rfsim.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 @@ -146,6 +157,8 @@ services: - ../../conf_files/nrue.uicc.conf:/opt/oai-nr-ue/etc/nr-ue.conf healthcheck: test: /bin/bash -c "pgrep nr-uesoftmodem" + start_period: 10s + start_interval: 500ms interval: 10s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz/docker-compose.yml index 6d597d3a86..11ecb20f08 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz/docker-compose.yml @@ -14,7 +14,9 @@ services: volumes: - ../../conf_files/gnb.sa.band77.273prb.usrpn310.2x2.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz/docker-compose.yml index 43211d7381..332fb23503 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz/docker-compose.yml @@ -14,7 +14,9 @@ services: volumes: - ../../conf_files/gnb.sa.band77.162prb.usrpn310.2x2.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml index 0ecba5f39e..d3d2ddf582 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz/docker-compose.yml @@ -16,7 +16,7 @@ services: - ../../conf_files/gnb-vnf.sa.band77.162prb.nfapi.4x4.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: test: /bin/bash -c "pgrep nr-softmodem" - interval: 15s + interval: 5s timeout: 5s retries: 5 oai-pnf: @@ -35,7 +35,9 @@ services: volumes: - ../../conf_files/gnb-pnf.band77.usrpn310.4x4.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 15s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s retries: 5 diff --git a/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml b/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml index e93152bd37..562b72561a 100644 --- a/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/5g_sa_n310_gnb/docker-compose.yml @@ -16,7 +16,9 @@ services: volumes: - ../../conf_files/gnb.sa.band78.106prb.n310.7ds2u.conf:/opt/oai-gnb/etc/gnb.conf healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s - retries: 5 + retries: 10 diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml index a0bc415730..b70229ac94 100644 --- a/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_fhi_7.2_benetel550_gnb/docker-compose.yml @@ -22,10 +22,12 @@ services: oai-net: ipv4_address: 172.21.18.20 healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s - retries: 5 + retries: 10 networks: oai-net: driver: macvlan diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb/docker-compose.yml index 0dd0a6c99d..06273747bf 100644 --- a/ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb/docker-compose.yml @@ -21,10 +21,12 @@ services: oai-net: ipv4_address: 172.21.18.20 healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s - retries: 5 + retries: 10 networks: oai-net: driver: macvlan diff --git a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml index a74b583e2b..ff06893fa0 100644 --- a/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml +++ b/ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb/docker-compose.yml @@ -21,10 +21,13 @@ services: oai-net: ipv4_address: 172.21.18.20 healthcheck: - test: /bin/bash -c "pgrep nr-softmodem" - interval: 10s + test: /bin/bash -c "/opt/oai-gnb/bin/check-prach-io.sh" + start_period: 15s + start_interval: 500ms + interval: 5s timeout: 5s - retries: 5 + retries: 10 + networks: oai-net: driver: macvlan diff --git a/docker/Dockerfile.gNB.fhi72.rhel9 b/docker/Dockerfile.gNB.fhi72.rhel9 index 5f68599873..122b3b668d 100644 --- a/docker/Dockerfile.gNB.fhi72.rhel9 +++ b/docker/Dockerfile.gNB.fhi72.rhel9 @@ -71,6 +71,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/librfsimulator.so \ diff --git a/docker/Dockerfile.gNB.fhi72.rocky b/docker/Dockerfile.gNB.fhi72.rocky index 3fdc2f97af..f95310059d 100644 --- a/docker/Dockerfile.gNB.fhi72.rocky +++ b/docker/Dockerfile.gNB.fhi72.rocky @@ -59,6 +59,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/librfsimulator.so \ diff --git a/docker/Dockerfile.gNB.fhi72.ubuntu22 b/docker/Dockerfile.gNB.fhi72.ubuntu22 index 069055d63b..eb781281ec 100644 --- a/docker/Dockerfile.gNB.fhi72.ubuntu22 +++ b/docker/Dockerfile.gNB.fhi72.ubuntu22 @@ -71,6 +71,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/librfsimulator.so \ diff --git a/docker/Dockerfile.gNB.rhel9 b/docker/Dockerfile.gNB.rhel9 index 824e3b3833..d02b6907d7 100644 --- a/docker/Dockerfile.gNB.rhel9 +++ b/docker/Dockerfile.gNB.rhel9 @@ -60,6 +60,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/liboai_eth_transpro.so \ diff --git a/docker/Dockerfile.gNB.rocky b/docker/Dockerfile.gNB.rocky index 225432fdab..912de17ede 100644 --- a/docker/Dockerfile.gNB.rocky +++ b/docker/Dockerfile.gNB.rocky @@ -57,6 +57,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/liboai_eth_transpro.so \ diff --git a/docker/Dockerfile.gNB.ubuntu22 b/docker/Dockerfile.gNB.ubuntu22 index 68c903450a..ea7b538c41 100644 --- a/docker/Dockerfile.gNB.ubuntu22 +++ b/docker/Dockerfile.gNB.ubuntu22 @@ -94,6 +94,7 @@ WORKDIR /opt/oai-gnb/bin COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./ COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh +COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/liboai_eth_transpro.so \ diff --git a/docker/scripts/check-prach-io.sh b/docker/scripts/check-prach-io.sh new file mode 100755 index 0000000000..1e1544dadf --- /dev/null +++ b/docker/scripts/check-prach-io.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -eo pipefail + +test -f /opt/oai-gnb/nrL1_stats.log && grep -oP 'PRACH I0 = \K[0-9.]*' /opt/oai-gnb/nrL1_stats.log | awk '{if ($1 > 0.0) exit 0; else exit 1}'