diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py
index d38aaa8c3e..45e739cb66 100644
--- a/ci-scripts/cls_containerize.py
+++ b/ci-scripts/cls_containerize.py
@@ -337,6 +337,7 @@ class Containerize():
self.imageToCopy = ''
#checkers from xml
self.ran_checkers={}
+ self.num_attempts = 1
self.flexricTag = ''
@@ -844,50 +845,55 @@ class Containerize():
def DeployObject(self, HTML):
svr = self.eNB_serverId[self.eNB_instance]
+ num_attempts = self.num_attempts
lIpAddr, lSourcePath = self.GetCredentials(svr)
- logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m')
+ logging.debug(f'Deploying OAI Object on server: {lIpAddr}')
yaml = self.yamlPath[self.eNB_instance].strip('/')
# creating the log folder by default
local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}"
os.system(f'mkdir -p {local_dir}')
wd = f'{lSourcePath}/{yaml}'
-
+ wd_yaml = f'{wd}/docker-compose.y*ml'
+ yaml_dir = yaml.split('/')[-1]
with cls_cmd.getConnection(lIpAddr) as ssh:
- services = GetServices(ssh, self.services[self.eNB_instance], f"{wd}/docker-compose.y*ml")
+ services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml)
if services == [] or services == ' ' or services == None:
- msg = "Cannot determine services to start"
+ msg = 'Cannot determine services to start'
logging.error(msg)
HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg])
return False
-
ExistEnvFilePrint(ssh, wd)
WriteEnvFile(ssh, services, wd, self.deploymentTag, self.flexricTag)
-
- logging.info(f"will start services {services}")
- status = ssh.run(f'docker compose -f {wd}/docker-compose.y*ml up -d -- {services}')
- if status.returncode != 0:
- msg = f"cannot deploy services {services}: {status.stdout}"
- logging.error(msg)
- HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg])
- return False
-
- imagesInfo = []
- fstatus = True
- for svc in services.split():
- containerName = GetContainerName(ssh, svc, f"{wd}/docker-compose.y*ml")
- healthy = GetContainerHealth(ssh, containerName)
- if not healthy:
- imagesInfo += [f"Failed to deploy: service {svc}"]
- fstatus = False
- else:
- image = GetImageName(ssh, svc, f"{wd}/docker-compose.y*ml")
- logging.info(f"service {svc} healthy, container {containerName}, image {image}")
- imagesInfo += [f"service {svc} healthy, image {image}"]
- if fstatus:
+ if num_attempts <= 0:
+ raise ValueError(f'Invalid value for num_attempts: {num_attempts}, must be greater than 0')
+ for attempt in range(num_attempts):
+ imagesInfo = []
+ healthInfo = []
+ logging.info(f'will start services {services}')
+ status = ssh.run(f'docker compose -f {wd_yaml} up -d -- {services}')
+ if status.returncode != 0:
+ msg = f'cannot deploy services {services}: {status.stdout}'
+ logging.error(msg)
+ HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [msg])
+ return False
+ for svc in services.split():
+ health, msg = GetServiceHealth(ssh, svc, f'{wd_yaml}')
+ logging.info(msg)
+ imagesInfo.append(msg)
+ healthInfo.append(health)
+ deployed = all(healthInfo)
+ if deployed:
+ break
+ elif (attempt < num_attempts - 1):
+ logging.warning(f'Failed to deploy on attempt {attempt}, restart services {services}')
+ for svc in services.split():
+ CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log')
+ ssh.run(f'docker compose -f {wd_yaml} down -- {services}')
+ if deployed:
HTML.CreateHtmlTestRowQueue('N/A', 'OK', ['\n'.join(imagesInfo)])
else:
HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)])
- return fstatus
+ return deployed
def UndeployObject(self, HTML, RAN):
svr = self.eNB_serverId[self.eNB_instance]
diff --git a/ci-scripts/main.py b/ci-scripts/main.py
index 9bf196338c..bba0f3f57f 100644
--- a/ci-scripts/main.py
+++ b/ci-scripts/main.py
@@ -349,6 +349,7 @@ def ExecuteActionWithParam(action):
string_field = test.findtext('services')
if string_field is not None:
CONTAINERS.services[CONTAINERS.eNB_instance] = string_field
+ CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1)
CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge)
if action == 'Deploy_Object':
success = CONTAINERS.DeployObject(HTML)
diff --git a/ci-scripts/tests/deployment.py b/ci-scripts/tests/deployment.py
index 45bffe9a0a..a7337f6e52 100644
--- a/ci-scripts/tests/deployment.py
+++ b/ci-scripts/tests/deployment.py
@@ -47,6 +47,7 @@ class TestDeploymentMethods(unittest.TestCase):
self.cont.eNBUserName = None
self.cont.eNBPassword = None
self.cont.eNBSourceCodePath = os.getcwd()
+ self.cont.num_attempts = 3
def test_deploy(self):
self.cont.yamlPath[0] = 'tests/simple-dep/'
diff --git a/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml
index 3fd9c83522..15a9ad4065 100644
--- a/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml
+++ b/ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml
@@ -89,6 +89,7 @@
ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz
0
0
+ 3
diff --git a/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml
index d4d7568ec4..52b05df661 100644
--- a/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml
+++ b/ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml
@@ -89,6 +89,7 @@
ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz
0
0
+ 3
diff --git a/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml b/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml
index b89891f496..e9833887df 100644
--- a/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml
+++ b/ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml
@@ -89,6 +89,7 @@
ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz
0
0
+ 3
diff --git a/ci-scripts/xml_files/container_sa_n310_nrue.xml b/ci-scripts/xml_files/container_sa_n310_nrue.xml
index e1e79ccec9..e6b1ba1643 100644
--- a/ci-scripts/xml_files/container_sa_n310_nrue.xml
+++ b/ci-scripts/xml_files/container_sa_n310_nrue.xml
@@ -80,6 +80,7 @@
ci-scripts/yaml_files/5g_sa_n310_gnb
0
0
+ 3
Create_Workspace
@@ -93,6 +94,7 @@
ci-scripts/yaml_files/5g_sa_n310_nrue
1
1
+ 3