Merge remote-tracking branch 'origin/ci-log-collection-modif' into integration_2024_w09

ci-scripts/cls_cmd.py

@@ -86,6 +86,12 @@ class Cmd(metaclass=abc.ABCMeta):
 		return
 
 class LocalCmd(Cmd):
+	def __enter__(self):
+		return self
+
+	def __exit__(self, exc_type, exc_value, exc_traceback):
+		self.close()
+
 	def __init__(self, d = None):
 		self.cwd = d
 		if self.cwd is not None:
@@ -122,12 +128,40 @@ class LocalCmd(Cmd):
 		if src[0] != '/' or tgt[0] != '/':
 			raise Exception('support only absolute file paths!')
 		opt = '-r' if recursive else ''
-		self.run(f'cp {opt} {src} {tgt}')
+		return self.run(f'cp {opt} {src} {tgt}').returncode == 0
 
 	def copyout(self, src, tgt, recursive=False):
-		self.copyin(src, tgt, recursive)
+		return self.copyin(src, tgt, recursive)
+
+def PutFile(client, src, tgt):
+	success = True
+	sftp = client.open_sftp()
+	try:
+		sftp.put(src, tgt)
+	except FileNotFoundError as error:
+		logging.error(f"error while putting {src}: {error}")
+		success = False
+	sftp.close()
+	return success
+
+def GetFile(client, src, tgt):
+	success = True
+	sftp = client.open_sftp()
+	try:
+		sftp.get(src, tgt)
+	except FileNotFoundError as error:
+		logging.error(f"error while getting {src}: {error}")
+		success = False
+	sftp.close()
+	return success
 
 class RemoteCmd(Cmd):
+	def __enter__(self):
+		return self
+
+	def __exit__(self, exc_type, exc_value, exc_traceback):
+		self.close()
+
 	def __init__(self, hostname, d=None):
 		cIdx = 0
 		logging.getLogger('paramiko').setLevel(logging.ERROR) # prevent spamming through Paramiko
@@ -193,36 +227,38 @@ class RemoteCmd(Cmd):
 	def getBefore(self):
 		return self.cp.stdout
 
+	# if recursive is True, tgt must be a directory (and src is file or directory)
+	# if recursive is False, tgt and src must be a file name
 	def copyout(self, src, tgt, recursive=False):
 		logging.debug(f"copyout: local:{src} -> remote:{tgt}")
 		if recursive:
 			tmpfile = f"{uuid.uuid4()}.tar"
 			abstmpfile = f"/tmp/{tmpfile}"
-			cmd = LocalCmd()
+			with LocalCmd() as cmd:
-			cmd.run(f"tar -cf {abstmpfile} {src}")
+				if cmd.run(f"tar -cf {abstmpfile} {src}").returncode != 0:
-			sftp = self.client.open_sftp()
+					return False
-			sftp.put(abstmpfile, abstmpfile)
+				if not PutFile(self.client, abstmpfile, abstmpfile):
-			sftp.close()
+					return False
-			cmd.run(f"rm {abstmpfile}")
+				cmd.run(f"rm {abstmpfile}")
-			self.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}")
+				ret = self.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}")
+				return ret.returncode == 0
 		else:
-			sftp = self.client.open_sftp()
+			return PutFile(self.client, src, tgt)
-			sftp.put(src, tgt)
-			sftp.close()
 
+	# if recursive is True, tgt must be a directory (and src is file or directory)
+	# if recursive is False, tgt and src must be a file name
 	def copyin(self, src, tgt, recursive=False):
 		logging.debug(f"copyin: remote:{src} -> local:{tgt}")
 		if recursive:
 			tmpfile = f"{uuid.uuid4()}.tar"
 			abstmpfile = f"/tmp/{tmpfile}"
-			self.run(f"tar -cf {abstmpfile} {src}")
+			if self.run(f"tar -cf {abstmpfile} {src}").returncode != 0:
-			sftp = self.client.open_sftp()
+				return False
-			sftp.get(abstmpfile, abstmpfile)
+			if not GetFile(self.client, abstmpfile, abstmpfile):
-			sftp.close()
+				return False
 			self.run(f"rm {abstmpfile}")
-			cmd = LocalCmd()
+			with LocalCmd() as cmd:
-			cmd.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}")
+				ret = cmd.run(f"mv {abstmpfile} {tgt}; cd {tgt} && tar -xf {tmpfile} && rm {tmpfile}")
+				return ret.returncode == 0
 		else:
-			sftp = self.client.open_sftp()
+			return GetFile(self.client, src, tgt)
-			sftp.get(src, tgt)
-			sftp.close()

ci-scripts/cls_containerize.py

@@ -955,12 +955,12 @@ class Containerize():
 		if svcName == '':
 			logging.warning('no service name given: starting all services in ci-docker-compose.yml!')
 
-		mySSH.command(f'docker-compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30)
+		mySSH.command(f'docker compose --file ci-docker-compose.yml up -d -- {svcName}', '\$', 30)
 
 		# Checking Status
 		grep = ''
 		if svcName != '': grep = f' | grep -A3 {svcName}'
-		mySSH.command(f'docker-compose --file ci-docker-compose.yml config {grep}', '\$', 5)
+		mySSH.command(f'docker compose --file ci-docker-compose.yml config {grep}', '\$', 5)
 		result = re.search('container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore())
 		unhealthyNb = 0
 		healthyNb = 0
@@ -1062,55 +1062,55 @@ class Containerize():
 		if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
 			HELP.GenericHelp(CONST.Version)
 			sys.exit('Insufficient Parameter')
-		logging.debug('\u001B[1m Undeploying OAI Object from server: ' + lIpAddr + '\u001B[0m')
+		logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m')
-		mySSH = SSH.SSHConnection()
+		mySSH = cls_cmd.getConnection(lIpAddr)
-		mySSH.open(lIpAddr, lUserName, lPassWord)
+		yamlDir = f'{lSourcePath}/{self.yamlPath[self.eNB_instance]}'
-
+		mySSH.run(f'cd {yamlDir}')
-		mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
-
 		svcName = self.services[self.eNB_instance]
 		forceDown = False
 		if svcName != '':
 			logging.warning(f'service name given, but will stop all services in ci-docker-compose.yml!')
 			svcName = ''
 
-		mySSH.command(f'docker-compose -f ci-docker-compose.yml config --services', '\$', 5)
+		ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml config --services')
+		if ret.returncode != 0:
+			HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', "cannot enumerate running services")
+			self.exitStatus = 1
+			return
 		# first line has command, last line has next command prompt
-		allServices = mySSH.getBefore().split('\r\n')[1:-1]
+		allServices = ret.stdout.splitlines()
 		services = []
 		for s in allServices:
-			mySSH.command(f'docker-compose -f ci-docker-compose.yml ps --all -- {s}', '\$', 5, silent=False)
+			# outputs the hash if the container is running
-			running = mySSH.getBefore().split('\r\n')[2:-1]
+			ret = mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml ps --all --quiet -- {s}')
+			running = ret.stdout.splitlines()
 			logging.debug(f'running services: {running}')
-			if len(running) > 0: # something is running for that service
+			if ret.stdout != "" and ret.returncode == 0: # something is running for that service
 				services.append(s)
 		logging.info(f'stopping services {services}')
 
-		mySSH.command(f'docker-compose -f ci-docker-compose.yml stop -t3', '\$', 30)
+		mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml stop -t3')
-		time.sleep(5)  # give some time to running containers to stop
+		copyin_res = True
 		for svcName in services:
 			# head -n -1 suppresses the final "X exited with status code Y"
 			filename = f'{svcName}-{HTML.testCase_id}.log'
-			mySSH.command(f'docker-compose -f ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}', '\$', 120)
+			mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml logs --no-log-prefix -- {svcName} &> {lSourcePath}/cmake_targets/log/{filename}')
+			copyin_res = mySSH.copyin(f'{lSourcePath}/cmake_targets/log/{filename}', f'{filename}') and copyin_res
+		# when nv-cubb container is available, copy L1 pcap, OAI Aerial pipeline
+		if 'nv-cubb' in allServices:
+			mySSH.run(f'cp {lSourcePath}/cmake_targets/share/gnb_nvipc.pcap {lSourcePath}/cmake_targets/gnb_nvipc.pcap')
 
-		mySSH.command('docker-compose -f ci-docker-compose.yml down -v', '\$', 5)
+		mySSH.run(f'docker compose -f {yamlDir}/ci-docker-compose.yml down -v')
-		mySSH.close()
 
 		# Analyzing log file!
-		files = ','.join([f'{s}-{HTML.testCase_id}' for s in services])
+		if not copyin_res:
-		if len(services) > 1:
+			HTML.htmleNBFailureMsg='Could not copy logfile(s) to analyze it!'
-			files = '{' + files + '}'
-		copyin_res = 0
-		if len(services) > 0:
-			copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, f'{lSourcePath}/cmake_targets/log/{files}.log', '.')
-		if copyin_res == -1:
-			HTML.htmleNBFailureMsg='Could not copy logfile to analyze it!'
 			HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
 			self.exitStatus = 1
 		# use function for UE log analysis, when oai-nr-ue container is used
 		elif 'oai-nr-ue' in services or 'lte_ue0' in services:
 			self.exitStatus == 0
-			logging.debug('\u001B[1m Analyzing UE logfile ' + filename + ' \u001B[0m')
+			logging.debug(f'Analyzing UE logfile {filename}')
 			logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(f'{filename}', HTML, RAN)
 			if (logStatus < 0):
 				fullStatus = False
@@ -1129,11 +1129,12 @@ class Containerize():
 					HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
 			# all the xNB run logs shall be on the server 0 for logCollecting
 			if self.eNB_serverId[self.eNB_instance] != '0':
-				mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, f'./{files}.log', f'{self.eNBSourceCodePath}/cmake_targets/')
+				mySSH.copyout(f'./*.log', f'{lSourcePath}/cmake_targets/', recursive=True)
 		if self.exitStatus == 0:
 			logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
 		else:
 			logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m')
+		mySSH.close()
 
 	def DeployGenObject(self, HTML, RAN, UE):
 		self.exitStatus = 0