Make service configurable in DeployObject, UndeployObject stops multiple

Service name can be used to start individual services. Uses only
docker-compose primitives, which can later be used to use the
docker-compose SSH functionality to (un)deploy services.

UndeployObject stops all referenced services

ci-scripts/cls_containerize.py

@@ -773,22 +773,19 @@ class Containerize():
 			mySSH.command(f'sed -i -e "s#image: {image}:latest#image: {imageToUse}#" ci-docker-compose.yml', '\$', 2)
 		localMmeIpAddr = EPC.MmeIPAddress
 		mySSH.command('sed -i -e "s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/" ci-docker-compose.yml', '\$', 2)
-#		if self.flexranCtrlDeployed:
+
-#			mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*/FLEXRAN_ENABLED: \'yes\'/" ci-docker-compose.yml', '\$', 2)
-#			mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/" ci-docker-compose.yml', '\$', 2)
-#		else:
-#			mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*$/FLEXRAN_ENABLED: \'no\'/" ci-docker-compose.yml', '\$', 2)
-#			mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/127.0.0.1/" ci-docker-compose.yml', '\$', 2)
 		# Currently support only one
 		mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 15)
-		mySSH.command('docker-compose --file ci-docker-compose.yml config --services | sed -e "s@^@service=@" 2>&1', '\$', 10)
+		svcName = self.services[self.eNB_instance]
-		result = re.search('service=(?P<svc_name>[a-zA-Z0-9\_]+)', mySSH.getBefore())
+		if svcName == '':
-		if result is not None:
+			logging.warning('no service name given: starting all services in ci-docker-compose.yml!')
-			svcName = result.group('svc_name')
+
-			mySSH.command('docker-compose --file ci-docker-compose.yml up -d ' + svcName, '\$', 15)
+		mySSH.command(f'docker-compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30)
 
 		# Checking Status
-		mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5)
+		grep = ''
+		if svcName != '': grep = f' | grep -A3 {svcName}'
+		mySSH.command(f'docker-compose --file ci-docker-compose.yml config {grep}', '\$', 5)
 		result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
 		unhealthyNb = 0
 		healthyNb = 0
@@ -848,8 +845,8 @@ class Containerize():
 		else:
 			# containers are unhealthy, so we won't start. However, logs are stored at the end
 			# in UndeployObject so we here store the logs of the unhealthy container to report it
-			logfilename = f'{lSourcePath}/cmake_targets/{self.eNB_logFile[self.eNB_instance]}'
+			logfilename = f'{lSourcePath}/cmake_targets/log/{self.eNB_logFile[self.eNB_instance]}'
-			mySSH.command('docker logs {containerName} > {logfilename}', '\$', 30)
+			mySSH.command(f'docker logs {containerName} > {logfilename}', '\$', 30)
 			mySSH.copyin(lIpAddr, lUserName, lPassWord, logfilename, '.')
 		mySSH.close()
 
@@ -896,58 +893,67 @@ class Containerize():
 		mySSH = SSH.SSHConnection()
 		mySSH.open(lIpAddr, lUserName, lPassWord)
 		mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
-		# Currently support only one
+
-		mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5)
+		svcName = self.services[self.eNB_instance]
-		containerName = ''
+		forceDown = False
-		containerToKill = False
+		if svcName != '':
-		result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
+			logging.warning(f'service name given, but will stop all services in ci-docker-compose.yml!')
-		if self.eNB_logFile[self.eNB_instance] == '':
+			svcName = ''
-			self.eNB_logFile[self.eNB_instance] = 'enb_' + HTML.testCase_id + '.log'
+
-		if result is not None:
+		mySSH.command(f'docker-compose -f ci-docker-compose.yml config --services', '\$', 5)
-			containerName = result.group('container_name')
+		# first line has command, last line has next command prompt
-			containerToKill = True
+		allServices = mySSH.getBefore().split('\r\n')[1:-1]
-		if containerToKill:
+		services = []
-			mySSH.command('docker inspect ' + containerName, '\$', 30)
+		for s in allServices:
-			result = re.search('Error: No such object: ' + containerName, mySSH.getBefore())
+			mySSH.command(f'docker-compose -f ci-docker-compose.yml ps --all -- {s}', '\$', 5, silent=False)
-			if result is not None:
+			running = mySSH.getBefore().split('\r\n')[3:-1]
-				containerToKill = False
+			#logging.debug(f'running services: {running}')
-		if containerToKill:
+			if len(running) > 0: # something is running for that service
-			mySSH.command('docker kill --signal INT ' + containerName, '\$', 30)
+				services.append(s)
-			time.sleep(5)
+		logging.info(f'stopping services {services}')
-			mySSH.command('docker kill --signal KILL ' + containerName, '\$', 30)
+
-			time.sleep(5)
+		mySSH.command(f'docker-compose -f ci-docker-compose.yml stop', '\$', 30)
-			mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30)
+		time.sleep(5)  # give some time to running containers to stop
-			mySSH.command('docker rm -f ' + containerName, '\$', 30)
+		for svcName in services:
-		# Forcing the down now to remove the networks and any artifacts
+			# head -n -1 suppresses the final "X exited with status code Y"
-		mySSH.command('docker-compose --file ci-docker-compose.yml down', '\$', 5)
+			filename = f'{svcName}-{HTML.testCase_id}.log'
+			#mySSH.command(f'docker-compose -f ci-docker-compose.yml logs --no-log-prefix -- {svcName} | head -n -1 &> {lSourcePath}/cmake_targets/log/{filename}', '\$', 30)
+			mySSH.command(f'docker-compose -f ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}', '\$', 30)
+
+		mySSH.command('docker-compose -f ci-docker-compose.yml down', '\$', 5)
 		# Cleaning any created tmp volume
-		mySSH.command('docker volume prune --force || true', '\$', 20)
+		mySSH.command('docker volume prune --force', '\$', 20)
 
 		mySSH.close()
 
 		# Analyzing log file!
-		if containerToKill:
+		files = ','.join([f'{s}-{HTML.testCase_id}' for s in services])
-			copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '.')
+		if len(services) > 1:
-		else:
+			files = '{' + files + '}'
-			copyin_res = 0
+		copyin_res = 0
-		nodeB_prefix = 'e'
+		if len(services) > 0:
-		if (copyin_res == -1):
+			copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, f'{lSourcePath}/cmake_targets/log/{files}.log', '.')
-			HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
+		if copyin_res == -1:
+			HTML.htmleNBFailureMsg='Could not copy logfile to analyze it!'
 			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
+			self.exitStatus = 1
 		else:
-			if containerToKill:
+			for svcName in services:
-				logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance])
+				filename = f'{svcName}-{HTML.testCase_id}.log'
-				logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML, self.ran_checkers)
+				logging.debug(f'\u001B[1m Analyzing logfile {filename}\u001B[0m')
-			else:
+				logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers)
-				logStatus = 0
+				if (logStatus < 0):
-			if (logStatus < 0):
+					HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
-				HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus)
+					self.exitStatus = 1
-			else:
+				else:
-				HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
+					HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
 			# all the xNB run logs shall be on the server 0 for logCollecting
-			if containerToKill and self.eNB_serverId[self.eNB_instance] != '0':
+			if self.eNB_serverId[self.eNB_instance] != '0':
-				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + self.eNB_logFile[self.eNB_instance], self.eNBSourceCodePath + '/cmake_targets/')
+				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, f'./{files}.log', f'{self.eNBSourceCodePath}/cmake_targets/')
-		logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
+		if self.exitStatus == 0:
+			logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
+		else:
+			logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m')
 
 	def DeployGenObject(self, HTML, RAN, UE):
 		self.exitStatus = 0

ci-scripts/main.py

@@ -423,6 +423,9 @@ def GetParametersFromXML(action):
 		string_field=test.findtext('u_retx_th')
 		if (string_field is not None):
 			CONTAINERS.ran_checkers['u_retx_th'] = [float(x) for x in string_field.split(',')]
+		string_field = test.findtext('services')
+		if string_field is not None:
+			CONTAINERS.services[CONTAINERS.eNB_instance] = string_field
 
 	elif action == 'DeployGenObject' or action == 'UndeployGenObject' or action == 'StatsFromGenObject':
 		string_field=test.findtext('yaml_path')
@@ -986,6 +989,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 						RAN.prematureExit = True
 				elif action == 'Undeploy_Object':
 					CONTAINERS.UndeployObject(HTML, RAN)
+					if CONTAINERS.exitStatus == 1:
+						CiTestObj.AutoTerminateeNB(HTML,RAN,EPC,CONTAINERS)
+						RAN.prematureExit = True
 				elif action == 'Cppcheck_Analysis':
 					SCA.CppCheckAnalysis(HTML)
 				elif action == 'LicenceAndFormattingCheck':