From e1d06cfef8d49ab3ad75abb90d415e1febd193ae Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Sun, 1 Dec 2024 22:50:38 +0100 Subject: [PATCH 1/5] CI: add CI unit test for simple fail deployment with 2 services --- ci-scripts/tests/deployment.py | 9 ++++++++ .../tests/simple-fail-2svc/docker-compose.yml | 23 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 ci-scripts/tests/simple-fail-2svc/docker-compose.yml diff --git a/ci-scripts/tests/deployment.py b/ci-scripts/tests/deployment.py index 47e459d6a4..45bffe9a0a 100644 --- a/ci-scripts/tests/deployment.py +++ b/ci-scripts/tests/deployment.py @@ -65,6 +65,15 @@ class TestDeploymentMethods(unittest.TestCase): self.assertFalse(deploy) self.cont.yamlPath = old + def test_deployfails_2svc(self): + # fails reliably + old = self.cont.yamlPath + self.cont.yamlPath[0] = 'tests/simple-fail-2svc/' + deploy = self.cont.DeployObject(self.html) + self.cont.UndeployObject(self.html, self.ran) + self.assertFalse(deploy) + self.cont.yamlPath = old + def test_deploy_ran(self): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb" diff --git a/ci-scripts/tests/simple-fail-2svc/docker-compose.yml b/ci-scripts/tests/simple-fail-2svc/docker-compose.yml new file mode 100644 index 0000000000..8b30c4aa08 --- /dev/null +++ b/ci-scripts/tests/simple-fail-2svc/docker-compose.yml @@ -0,0 +1,23 @@ +services: + test1: + image: ubuntu:jammy + container_name: test_container1 + cap_drop: + - ALL + entrypoint: /bin/bash -c "false" + healthcheck: + test: /bin/bash -c "true" + interval: 1s + timeout: 1s + retries: 1 + test2: + image: ubuntu:jammy + container_name: test_container2 + cap_drop: + - ALL + entrypoint: /bin/bash -c "sleep infinity" + healthcheck: + test: /bin/bash -c "true" + interval: 1s + timeout: 1s + retries: 1 From a9ca4a5481b479b8f1586d02b9d26ecfa89af564 Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Wed, 4 Dec 2024 08:21:44 +0100 Subject: [PATCH 2/5] CI: create GetServiceHealth() Rework GetContainerHealth() to work with docker compose and services, instead of using docker and containers. Return health and message about deployment status (used for console and HTML logging) for a given service. --- ci-scripts/cls_containerize.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index a9c4b5be08..2b00a5c2ae 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -156,17 +156,18 @@ def GetImageName(ssh, svcName, file): else: return ret.stdout.strip() -def GetContainerHealth(ssh, containerName): - if containerName is None: - return False - if 'db_init' in containerName or 'db-init' in containerName: # exits with 0, there cannot be healthy - return True +def GetServiceHealth(ssh, svcName, file): + if svcName is None: + return False, f"Service {svcName} not found in {file}" + image = GetImageName(ssh, svcName, file) + if 'db_init' in svcName or 'db-init' in svcName: # exits with 0, there cannot be healthy + return True, f"Service {svcName} healthy, image {image}" for _ in range(8): - result = ssh.run(f'docker inspect --format="{{{{.State.Health.Status}}}}" {containerName}', silent=True) + result = ssh.run(f"docker compose -f {file} ps --format json {svcName} | jq -r 'if type==\"array\" then .[0].Health else .Health end'", silent=True) if result.stdout == 'healthy': - return True + return True, f"Service {svcName} healthy, image {image}" time.sleep(5) - return False + return False, f"Failed to deploy: service {svcName}" def ExistEnvFilePrint(ssh, wd, prompt='env vars in existing'): ret = ssh.run(f'cat {wd}/.env', silent=True, reportNonZero=False) From f078e6f9c75dac9e4e78cd697d80bf3742b18990 Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Wed, 4 Dec 2024 08:36:32 +0100 Subject: [PATCH 3/5] CI: modify and rename CopyinContainerLog to use docker services --- ci-scripts/cls_containerize.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 2b00a5c2ae..d38aaa8c3e 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -219,9 +219,9 @@ def GetServices(ssh, requested, file): else: return requested -def CopyinContainerLog(ssh, lSourcePath, yaml, containerName, filename): +def CopyinServiceLog(ssh, lSourcePath, yaml, svcName, wd_yaml, filename): remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}" - ssh.run(f'docker logs {containerName} &> {remote_filename}') + ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}') local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}" local_filename = f"{local_dir}/{filename}" return ssh.copyin(remote_filename, local_filename) @@ -903,7 +903,7 @@ class Containerize(): if services is not None: all_serv = " ".join([s for s, _ in services]) ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}') - copyin_res = all(CopyinContainerLog(ssh, lSourcePath, yaml_dir, c, f'{s}-{HTML.testCase_id}.log') for s, c in services) + copyin_res = all(CopyinServiceLog(ssh, lSourcePath, yaml_dir, s, f"{wd}/docker-compose.y*ml", f'{s}-{HTML.testCase_id}.log') for s, c in services) else: logging.warning('could not identify services to stop => no log file') ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v') From f786d92bc907e95503f888c06fb99ab56ea80b4f Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Thu, 5 Dec 2024 14:03:11 +0100 Subject: [PATCH 4/5] CI: Allow restart of the container if it fails on the first attempt In CI we sometimes encouter fail of the gNB/UE deployment (and fail of the pipeline) caused by unsuccsessful initialization of the USRP N310. Restart the container if the health check fails during initialization. Introduce a configurable parameter (from XML) to set max number of attempts for the container deployment. Store logs from failed deployment attempts. --- ci-scripts/cls_containerize.py | 62 ++++++++++--------- ci-scripts/main.py | 1 + ci-scripts/tests/deployment.py | 1 + .../container_sa_n310_2X2_100MHz_quectel.xml | 1 + .../container_sa_n310_2X2_60MHz_quectel.xml | 1 + .../container_sa_n310_4X4_60MHz_quectel.xml | 1 + .../xml_files/container_sa_n310_nrue.xml | 2 + 7 files changed, 41 insertions(+), 28 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index d38aaa8c3e..45e739cb66 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -337,6 +337,7 @@ class Containerize(): self.imageToCopy = '' #checkers from xml self.ran_checkers={} + self.num_attempts = 1 self.flexricTag = '' @@ -844,50 +845,55 @@ class Containerize(): def DeployObject(self, HTML): svr = self.eNB_serverId[self.eNB_instance] + num_attempts = self.num_attempts lIpAddr, lSourcePath = self.GetCredentials(svr) - logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m') + logging.debug(f'Deploying OAI Object on server: {lIpAddr}') yaml = self.yamlPath[self.eNB_instance].strip('/') # creating the log folder by default local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}" os.system(f'mkdir -p {local_dir}') wd = f'{lSourcePath}/{yaml}' - + wd_yaml = f'{wd}/docker-compose.y*ml' + yaml_dir = yaml.split('/')[-1] with cls_cmd.getConnection(lIpAddr) as ssh: - services = GetServices(ssh, self.services[self.eNB_instance], f"{wd}/docker-compose.y*ml") + services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml) if services == [] or services == ' ' or services == None: - msg = "Cannot determine services to start" + msg = 'Cannot determine services to start' logging.error(msg) HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg]) return False - ExistEnvFilePrint(ssh, wd) WriteEnvFile(ssh, services, wd, self.deploymentTag, self.flexricTag) - - logging.info(f"will start services {services}") - status = ssh.run(f'docker compose -f {wd}/docker-compose.y*ml up -d -- {services}') - if status.returncode != 0: - msg = f"cannot deploy services {services}: {status.stdout}" - logging.error(msg) - HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg]) - return False - - imagesInfo = [] - fstatus = True - for svc in services.split(): - containerName = GetContainerName(ssh, svc, f"{wd}/docker-compose.y*ml") - healthy = GetContainerHealth(ssh, containerName) - if not healthy: - imagesInfo += [f"Failed to deploy: service {svc}"] - fstatus = False - else: - image = GetImageName(ssh, svc, f"{wd}/docker-compose.y*ml") - logging.info(f"service {svc} healthy, container {containerName}, image {image}") - imagesInfo += [f"service {svc} healthy, image {image}"] - if fstatus: + if num_attempts <= 0: + raise ValueError(f'Invalid value for num_attempts: {num_attempts}, must be greater than 0') + for attempt in range(num_attempts): + imagesInfo = [] + healthInfo = [] + logging.info(f'will start services {services}') + status = ssh.run(f'docker compose -f {wd_yaml} up -d -- {services}') + if status.returncode != 0: + msg = f'cannot deploy services {services}: {status.stdout}' + logging.error(msg) + HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [msg]) + return False + for svc in services.split(): + health, msg = GetServiceHealth(ssh, svc, f'{wd_yaml}') + logging.info(msg) + imagesInfo.append(msg) + healthInfo.append(health) + deployed = all(healthInfo) + if deployed: + break + elif (attempt < num_attempts - 1): + logging.warning(f'Failed to deploy on attempt {attempt}, restart services {services}') + for svc in services.split(): + CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log') + ssh.run(f'docker compose -f {wd_yaml} down -- {services}') + if deployed: HTML.CreateHtmlTestRowQueue('N/A', 'OK', ['\n'.join(imagesInfo)]) else: HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)]) - return fstatus + return deployed def UndeployObject(self, HTML, RAN): svr = self.eNB_serverId[self.eNB_instance] diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 9bf196338c..bba0f3f57f 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -349,6 +349,7 @@ def ExecuteActionWithParam(action): string_field = test.findtext('services') if string_field is not None: CONTAINERS.services[CONTAINERS.eNB_instance] = string_field + CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1) CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge) if action == 'Deploy_Object': success = CONTAINERS.DeployObject(HTML) diff --git a/ci-scripts/tests/deployment.py b/ci-scripts/tests/deployment.py index 45bffe9a0a..a7337f6e52 100644 --- a/ci-scripts/tests/deployment.py +++ b/ci-scripts/tests/deployment.py @@ -47,6 +47,7 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.eNBUserName = None self.cont.eNBPassword = None self.cont.eNBSourceCodePath = os.getcwd() + self.cont.num_attempts = 3 def test_deploy(self): self.cont.yamlPath[0] = 'tests/simple-dep/' diff --git a/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml index 3fd9c83522..15a9ad4065 100644 --- a/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml @@ -89,6 +89,7 @@ ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz 0 0 + 3 diff --git a/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml index d4d7568ec4..52b05df661 100644 --- a/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml @@ -89,6 +89,7 @@ ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz 0 0 + 3 diff --git a/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml index b89891f496..e9833887df 100644 --- a/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml +++ b/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml @@ -89,6 +89,7 @@ ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz 0 0 + 3 diff --git a/ci-scripts/xml_files/container_sa_n310_nrue.xml b/ci-scripts/xml_files/container_sa_n310_nrue.xml index e1e79ccec9..e6b1ba1643 100644 --- a/ci-scripts/xml_files/container_sa_n310_nrue.xml +++ b/ci-scripts/xml_files/container_sa_n310_nrue.xml @@ -80,6 +80,7 @@ ci-scripts/yaml_files/5g_sa_n310_gnb 0 0 + 3 Create_Workspace @@ -93,6 +94,7 @@ ci-scripts/yaml_files/5g_sa_n310_nrue 1 1 + 3 From 027631eb7045bf5b7b64dd9e9f47a0877c2d36fb Mon Sep 17 00:00:00 2001 From: Jaroslava Fiedlerova Date: Wed, 4 Dec 2024 08:40:14 +0100 Subject: [PATCH 5/5] CI: Remove GetContainerName() - not used anymore No need to get container name, we work directly with docker services. --- ci-scripts/cls_containerize.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 45e739cb66..b1c5120097 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -145,10 +145,6 @@ def AnalyzeBuildLogs(buildRoot, images, globalStatus): collectInfo[image] = files return collectInfo -def GetContainerName(ssh, svcName, file): - ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".container_name'", silent=True) - return ret.stdout - def GetImageName(ssh, svcName, file): ret = ssh.run(f"docker compose -f {file} config --format json {svcName} | jq -r '.services.\"{svcName}\".image'", silent=True) if ret.returncode != 0: