diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index a9c4b5be08..b1c5120097 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -145,10 +145,6 @@ def AnalyzeBuildLogs(buildRoot, images, globalStatus): collectInfo[image] = files return collectInfo -def GetContainerName(ssh, svcName, file): - ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".container_name'", silent=True) - return ret.stdout - def GetImageName(ssh, svcName, file): ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".image'", silent=True) if ret.returncode != 0: @@ -156,17 +152,18 @@ def GetImageName(ssh, svcName, file): else: return ret.stdout.strip() -def GetContainerHealth(ssh, containerName): - if containerName is None: - return False - if 'db_init' in containerName or 'db-init' in containerName: # exits with 0, there cannot be healthy - return True +def GetServiceHealth(ssh, svcName, file): + if svcName is None: + return False, f"Service {svcName} not found in {file}" + image = GetImageName(ssh, svcName, file) + if 'db_init' in svcName or 'db-init' in svcName: # exits with 0, there cannot be healthy + return True, f"Service {svcName} healthy, image {image}" for _ in range(8): - result = ssh.run(f'docker inspect --format="{{{{.State.Health.Status}}}}" {containerName}', silent=True) + result = ssh.run(f"docker compose -f {file} ps --format json {svcName} | jq -r 'if type==\"array\" then .[0].Health else .Health end'", silent=True) if result.stdout == 'healthy': - return True + return True, f"Service {svcName} healthy, image {image}" time.sleep(5) - return False + return False, f"Failed to deploy: service {svcName}" def ExistEnvFilePrint(ssh, wd, prompt='env vars in existing'): ret = ssh.run(f'cat {wd}/.env', silent=True, reportNonZero=False) @@ -218,9 +215,9 @@ def GetServices(ssh, requested, file): else: return requested -def CopyinContainerLog(ssh, lSourcePath, yaml, containerName, filename): +def CopyinServiceLog(ssh, lSourcePath, yaml, svcName, wd_yaml, filename): remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}" - ssh.run(f'docker logs {containerName} &> {remote_filename}') + ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}') local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}" local_filename = f"{local_dir}/{filename}" return ssh.copyin(remote_filename, local_filename) @@ -336,6 +333,7 @@ class Containerize(): self.imageToCopy = '' #checkers from xml self.ran_checkers={} + self.num_attempts = 1 self.flexricTag = '' @@ -843,50 +841,55 @@ class Containerize(): def DeployObject(self, HTML): svr = self.eNB_serverId[self.eNB_instance] + num_attempts = self.num_attempts lIpAddr, lSourcePath = self.GetCredentials(svr) - logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m') + logging.debug(f'Deploying OAI Object on server: {lIpAddr}') yaml = self.yamlPath[self.eNB_instance].strip('/') # creating the log folder by default local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}" os.system(f'mkdir -p {local_dir}') wd = f'{lSourcePath}/{yaml}' - + wd_yaml = f'{wd}/docker-compose.y*ml' + yaml_dir = yaml.split('/')[-1] with cls_cmd.getConnection(lIpAddr) as ssh: - services = GetServices(ssh, self.services[self.eNB_instance], f"{wd}/docker-compose.y*ml") + services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml) if services == [] or services == ' ' or services == None: - msg = "Cannot determine services to start" + msg = 'Cannot determine services to start' logging.error(msg) HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg]) return False - ExistEnvFilePrint(ssh, wd) WriteEnvFile(ssh, services, wd, self.deploymentTag, self.flexricTag) - - logging.info(f"will start services {services}") - status = ssh.run(f'docker compose -f {wd}/docker-compose.y*ml up -d -- {services}') - if status.returncode != 0: - msg = f"cannot deploy services {services}: {status.stdout}" - logging.error(msg) - HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg]) - return False - - imagesInfo = [] - fstatus = True - for svc in services.split(): - containerName = GetContainerName(ssh, svc, f"{wd}/docker-compose.y*ml") - healthy = GetContainerHealth(ssh, containerName) - if not healthy: - imagesInfo += [f"Failed to deploy: service {svc}"] - fstatus = False - else: - image = GetImageName(ssh, svc, f"{wd}/docker-compose.y*ml") - logging.info(f"service {svc} healthy, container {containerName}, image {image}") - imagesInfo += [f"service {svc} healthy, image {image}"] - if fstatus: + if num_attempts <= 0: + raise ValueError(f'Invalid value for num_attempts: {num_attempts}, must be greater than 0') + for attempt in range(num_attempts): + imagesInfo = [] + healthInfo = [] + logging.info(f'will start services {services}') + status = ssh.run(f'docker compose -f {wd_yaml} up -d -- {services}') + if status.returncode != 0: + msg = f'cannot deploy services {services}: {status.stdout}' + logging.error(msg) + HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [msg]) + return False + for svc in services.split(): + health, msg = GetServiceHealth(ssh, svc, f'{wd_yaml}') + logging.info(msg) + imagesInfo.append(msg) + healthInfo.append(health) + deployed = all(healthInfo) + if deployed: + break + elif (attempt < num_attempts - 1): + logging.warning(f'Failed to deploy on attempt {attempt}, restart services {services}') + for svc in services.split(): + CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log') + ssh.run(f'docker compose -f {wd_yaml} down -- {services}') + if deployed: HTML.CreateHtmlTestRowQueue('N/A', 'OK', ['\n'.join(imagesInfo)]) else: HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)]) - return fstatus + return deployed def UndeployObject(self, HTML, RAN): svr = self.eNB_serverId[self.eNB_instance] @@ -902,7 +905,7 @@ class Containerize(): if services is not None: all_serv = " ".join([s for s, _ in services]) ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}') - copyin_res = all(CopyinContainerLog(ssh, lSourcePath, yaml_dir, c, f'{s}-{HTML.testCase_id}.log') for s, c in services) + copyin_res = all(CopyinServiceLog(ssh, lSourcePath, yaml_dir, s, f"{wd}/docker-compose.y*ml", f'{s}-{HTML.testCase_id}.log') for s, c in services) else: logging.warning('could not identify services to stop => no log file') ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v') diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 9bf196338c..bba0f3f57f 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -349,6 +349,7 @@ def ExecuteActionWithParam(action): string_field = test.findtext('services') if string_field is not None: CONTAINERS.services[CONTAINERS.eNB_instance] = string_field + CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1) CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge) if action == 'Deploy_Object': success = CONTAINERS.DeployObject(HTML) diff --git a/ci-scripts/tests/deployment.py b/ci-scripts/tests/deployment.py index 47e459d6a4..a7337f6e52 100644 --- a/ci-scripts/tests/deployment.py +++ b/ci-scripts/tests/deployment.py @@ -47,6 +47,7 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.eNBUserName = None self.cont.eNBPassword = None self.cont.eNBSourceCodePath = os.getcwd() + self.cont.num_attempts = 3 def test_deploy(self): self.cont.yamlPath[0] = 'tests/simple-dep/' @@ -65,6 +66,15 @@ class TestDeploymentMethods(unittest.TestCase): self.assertFalse(deploy) self.cont.yamlPath = old + def test_deployfails_2svc(self): + # fails reliably + old = self.cont.yamlPath + self.cont.yamlPath[0] = 'tests/simple-fail-2svc/' + deploy = self.cont.DeployObject(self.html) + self.cont.UndeployObject(self.html, self.ran) + self.assertFalse(deploy) + self.cont.yamlPath = old + def test_deploy_ran(self): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb" diff --git a/ci-scripts/tests/simple-fail-2svc/docker-compose.yml b/ci-scripts/tests/simple-fail-2svc/docker-compose.yml new file mode 100644 index 0000000000..8b30c4aa08 --- /dev/null +++ b/ci-scripts/tests/simple-fail-2svc/docker-compose.yml @@ -0,0 +1,23 @@ +services: + test1: + image: ubuntu:jammy + container_name: test_container1 + cap_drop: + - ALL + entrypoint: /bin/bash -c "false" + healthcheck: + test: /bin/bash -c "true" + interval: 1s + timeout: 1s + retries: 1 + test2: + image: ubuntu:jammy + container_name: test_container2 + cap_drop: + - ALL + entrypoint: /bin/bash -c "sleep infinity" + healthcheck: + test: /bin/bash -c "true" + interval: 1s + timeout: 1s + retries: 1 diff --git a/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml index 3fd9c83522..15a9ad4065 100644 --- a/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml @@ -89,6 +89,7 @@ ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz 0 0 + 3 diff --git a/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml index d4d7568ec4..52b05df661 100644 --- a/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml @@ -89,6 +89,7 @@ ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz 0 0 + 3 diff --git a/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml index b89891f496..e9833887df 100644 --- a/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml @@ -89,6 +89,7 @@ ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz 0 0 + 3 diff --git a/ci-scripts/xml_files/container_sa_n310_nrue.xml b/ci-scripts/xml_files/container_sa_n310_nrue.xml index e1e79ccec9..e6b1ba1643 100644 --- a/ci-scripts/xml_files/container_sa_n310_nrue.xml +++ b/ci-scripts/xml_files/container_sa_n310_nrue.xml @@ -80,6 +80,7 @@ ci-scripts/yaml_files/5g_sa_n310_gnb 0 0 + 3 Create_Workspace @@ -93,6 +94,7 @@ ci-scripts/yaml_files/5g_sa_n310_nrue 1 1 + 3