Merge remote-tracking branch 'origin/mac-e2-ci' into integration_2025_w46 (!3752)

[MAC] Add UL total/used aggregated MAC stats; [E2 agent] Properly calculate
used UE PRBs in KPM; [CI] Simplify and improve the Undeploy_Object() and
implement Stop_Object()

- Create common dlul_mac_stats_t struct for both DL and UL total/used aggregated
  PRBs
- Use correct unit [%] for RRU.PrbTotDl and RRU.PrbTotUl , instead of [PRBs]
- Use correct unit [Mb] for DRB.PdcpSduVolumeDL and DRB.PdcpSduVolumeUL [kb]
- Implement Stop_Object() (docker compose stop = SIGTERM)
- Simplify the Undeploy_Object() function
- Improve the logging for Deploy/Undeploy_Object() functions
- OAI-FlexRIC CI: properly stop all the containers (one by one or a subgroup of
  running containers), and rename oai-flexric to nearRT-RIC container

ci-scripts/cls_containerize.py

@@ -47,6 +47,7 @@ import helpreadme as HELP
 import constants as CONST
 import cls_oaicitest
 from cls_ci_helper import archiveArtifact
+from collections import deque
 
 #-----------------------------------------------------------
 # Helper functions used here and in other classes
@@ -165,25 +166,26 @@ def CopyinServiceLog(ssh, lSourcePath, svcName, wd_yaml, ctx):
 	ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}')
 	return archiveArtifact(ssh, ctx, remote_filename)
 
-def GetRunningServices(ssh, file):
+def GetDeployedServices(ssh, file):
 	ret = ssh.run(f'docker compose -f {file} config --services')
 	if ret.returncode != 0:
+		logging.error("could not get services")
 		return None
 	allServices = ret.stdout.splitlines()
-	running_services = []
+	deployed_services = []
 	for s in allServices:
-		# outputs the hash if the container is running
-		ret = ssh.run(f'docker compose -f {file} ps --all --quiet -- {s}')
+		# outputs the hash if the container has been deployed (but might be stopped)
+		ret = ssh.run(f'docker compose -f {file} ps --all --quiet -- {s}', silent=True)
 		if ret.returncode != 0:
-			logging.info(f"service {s}: {ret.stdout}")
+			# error: should not happen as we iterate over docker-provided service list
+			logging.error(f"service {s}: {ret.stdout}")
 		elif ret.stdout == "":
-			logging.warning(f"could not retrieve information for service {s}")
+			logging.info(f"service {s} not deployed")
 		else:
 			c = ret.stdout
-			logging.debug(f'running service {s} with container id {c}')
-			running_services.append((s, c))
-	logging.info(f'stopping services: {running_services}')
-	return running_services
+			logging.info(f'service {s} with container id {c}')
+			deployed_services.append(s)
+	return deployed_services
 
 def CheckLogs(self, filename, HTML, RAN):
 	success = True
@@ -215,6 +217,24 @@ def CheckLogs(self, filename, HTML, RAN):
 		else:
 			HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg])
 		HTML.htmleNBFailureMsg = ""
+	elif 'xapp' in name:
+		opt = f"Undeploy {name}"
+		with open(f'{filename}', "r") as f:
+			last_line = deque(f, maxlen=1).pop()
+		if ('Test xApp run SUCCESSFULLY' in last_line):
+			HTML.CreateHtmlTestRowQueue(opt, 'OK', ["xApp run successfully"])
+		else:
+			HTML.CreateHtmlTestRowQueue(opt, 'KO', ["xApp didn't run successfully"])
+			success = False
+	elif 'RIC' in name:
+		opt = f"Undeploy {name}"
+		with open(f'{filename}', 'r') as f:
+			last_line = deque(f, maxlen=1).pop()
+		if ('Removing E2 Node' in last_line):
+			HTML.CreateHtmlTestRowQueue(opt, 'OK', ["nearRT-RIC run successfully"])
+		else:
+			HTML.CreateHtmlTestRowQueue(opt, 'KO', ["nearRT-RIC didn't run successfully"])
+			success = False
 	else:
 		logging.info(f"Skipping analysis of log '{filename}': no submatch for xNB/UE")
 	logging.debug(f"log check: file {filename} passed analysis {success}")
@@ -715,7 +735,6 @@ class Containerize():
 	def DeployObject(self, ctx, node, HTML):
 		num_attempts = self.num_attempts
 		lSourcePath = self.eNBSourceCodePath
-		logging.debug(f'Deploying OAI Object on server: {node}')
 		yaml = self.yamlPath.strip('/')
 		wd = f'{lSourcePath}/{yaml}'
 		wd_yaml = f'{wd}/docker-compose.y*ml'
@@ -726,6 +745,7 @@ class Containerize():
 				logging.error(msg)
 				HTML.CreateHtmlTestRowQueue('N/A', 'KO', [msg])
 				return False
+			logging.info(f'\u001B[1mDeploying object(s) "{services}" on server {node}\u001B[0m')
 			ExistEnvFilePrint(ssh, wd)
 			WriteEnvFile(ssh, services, wd, self.deploymentTag, self.flexricTag)
 			if num_attempts <= 0:
@@ -750,36 +770,68 @@ class Containerize():
 		imagesInfo = info.stdout.splitlines()[1:]
 		logging.debug(f'{info.stdout.splitlines()[1:]}')
 		if deployed:
-			HTML.CreateHtmlTestRowQueue('N/A', 'OK', ['\n'.join(imagesInfo)])
+			HTML.CreateHtmlTestRowQueue(self.services, 'OK', ['\n'.join(imagesInfo)])
+			logging.info('\u001B[1m Deploying objects Pass\u001B[0m')
 		else:
-			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)])
+			HTML.CreateHtmlTestRowQueue(self.services, 'KO', ['\n'.join(imagesInfo)])
+			logging.error('\u001B[1m Deploying objects Failed\u001B[0m')
 		return deployed
 
+	def StopObject(self, ctx, node, HTML):
+		lSourcePath = self.eNBSourceCodePath
+		if not self.services:
+			raise ValueError(f'no services provided')
+		logging.info(f'\u001B[1m Stopping objects "{self.services}" from server: {node}\u001B[0m')
+		reqServices = self.services.split()
+		yaml = self.yamlPath.strip('/')
+		wd = f'{lSourcePath}/{yaml}'
+		wd_yaml = f'{wd}/docker-compose.y*ml'
+		with cls_cmd.getConnection(node) as ssh:
+			ExistEnvFilePrint(ssh, wd)
+			services = GetDeployedServices(ssh, wd_yaml)
+			success = []
+			fail = []
+			for s in reqServices:
+				if s in services:
+					ssh.run(f'docker compose -f {wd_yaml} stop -- {s}')
+					success.append(s)
+				else:
+					logging.error(f"no such service {s}")
+					fail.append(s)
+		if success == reqServices:
+			logging.info('\u001B[1m Stopping object Pass\u001B[0m')
+			HTML.CreateHtmlTestRowQueue(self.services, 'OK', [f'Stopped {self.services}'])
+		else:
+			logging.error('\u001B[1m Stopping object Failed\u001B[0m')
+			HTML.CreateHtmlTestRowQueue(self.services, 'KO', [f'Failed stopping {" ".join(fail)}, succeeded {" ".join(success)}'])
+		return success
+
 	def UndeployObject(self, ctx, node, HTML, RAN):
 		lSourcePath = self.eNBSourceCodePath
-		logging.debug(f'\u001B[1m Undeploying OAI Object from server: {node}\u001B[0m')
+		logging.info(f'\u001B[1m Undeploying all objects from server {node}\u001B[0m')
 		yaml = self.yamlPath.strip('/')
 		wd = f'{lSourcePath}/{yaml}'
+		wd_yaml = f'{wd}/docker-compose.y*ml'
 		with cls_cmd.getConnection(node) as ssh:
 			ExistEnvFilePrint(ssh, wd)
-			services = GetRunningServices(ssh, f"{wd}/docker-compose.y*ml")
+			services = GetDeployedServices(ssh, wd_yaml)
 			copyin_res = None
+			ssh.run(f'docker compose -f {wd_yaml} stop')
 			if services is not None:
-				all_serv = " ".join([s for s, _ in services])
-				ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}')
-				copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, f"{wd}/docker-compose.y*ml", ctx) for s, _ in services]
+				copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, wd_yaml, ctx) for s in services]
 			else:
 				logging.warning('could not identify services to stop => no log file')
-			ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v')
+			ssh.run(f'docker compose -f {wd_yaml} down -v')
 			ssh.run(f'rm {wd}/.env')
 		if not copyin_res:
 			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)'])
-			return False
+			logging.error(f"could not copy all files: {copyin_res=} {services=}")
+			success = False
 		else:
 			log_results = [CheckLogs(self, f, HTML, RAN) for f in copyin_res]
 			success = all(log_results)
 		if success:
-			logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
+			logging.info('\u001B[1m Undeploying objects Pass\u001B[0m')
 		else:
-			logging.error('\u001B[1m Undeploying OAI Object Failed\u001B[0m')
+			logging.error('\u001B[1m Undeploying objects Failed\u001B[0m')
 		return success

ci-scripts/main.py

@@ -214,7 +214,7 @@ def ExecuteActionWithParam(action, ctx, node):
 		core_op = getattr(cls_oaicitest.OaiCiTest, action)
 		success = core_op(cn_id, ctx, HTML)
 
-	elif action == 'Deploy_Object' or action == 'Undeploy_Object' or action == "Create_Workspace":
+	elif action == 'Deploy_Object' or action == 'Undeploy_Object' or action == "Create_Workspace" or action == "Stop_Object":
 		CONTAINERS.yamlPath = test.findtext('yaml_path')
 		string_field=test.findtext('d_retx_th')
 		if (string_field is not None):
@@ -227,6 +227,8 @@ def ExecuteActionWithParam(action, ctx, node):
 		CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge)
 		if action == 'Deploy_Object':
 			success = CONTAINERS.DeployObject(ctx, node, HTML)
+		elif action == 'Stop_Object':
+			success = CONTAINERS.StopObject(ctx, node, HTML)
 		elif action == 'Undeploy_Object':
 			success = CONTAINERS.UndeployObject(ctx, node, HTML, RAN)
 		elif action == 'Create_Workspace':

ci-scripts/tests/deployment.py

@@ -60,6 +60,33 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.assertTrue(deploy)
 		self.assertTrue(undeploy)
 
+	def test_stop_services(self):
+		self.cont.yamlPath = 'tests/simple-undep/'
+		self.cont.deploymentTag = "noble"
+		# should deploy both testA and testB
+		deploy = self.cont.DeployObject(self.ctx, self.node, self.html)
+		# should fail (no such service)
+		self.cont.services = "testC"
+		stopC = self.cont.StopObject(self.ctx, self.node, self.html)
+		# should stop testA
+		self.cont.services = "testA"
+		stopA = self.cont.StopObject(self.ctx, self.node, self.html)
+		# should (re-)stop testA (no-op)
+		self.cont.services = "testA"
+		stopA2 = self.cont.StopObject(self.ctx, self.node, self.html)
+		# should deploy testB
+		self.cont.services = "testB"
+		stopB = self.cont.StopObject(self.ctx, self.node, self.html)
+		# should not undeploy anything (everything already stopped)
+		self.cont.services = None
+		undeployAll = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran)
+		self.assertTrue(deploy)
+		self.assertFalse(stopC)
+		self.assertTrue(stopA)
+		self.assertTrue(stopA2)
+		self.assertTrue(stopB)
+		self.assertTrue(undeployAll)
+
 	def test_deployfails(self):
 		# fails reliably
 		old = self.cont.yamlPath

ci-scripts/tests/simple-undep/docker-compose.yml

@@ -0,0 +1,23 @@
+services:
+  testA:
+    image: ubuntu:${TAG:-jammy}
+    container_name: test_container_A
+    cap_drop:
+      - ALL
+    entrypoint: /bin/bash -c "sleep infinity"
+    healthcheck:
+      test: /bin/bash -c "true"
+      interval: 1s
+      timeout: 1s
+      retries: 1
+  testB:
+    image: ubuntu:${TAG:-jammy}
+    container_name: test_container_B
+    cap_drop:
+      - ALL
+    entrypoint: /bin/bash -c "sleep infinity"
+    healthcheck:
+      test: /bin/bash -c "true"
+      interval: 1s
+      timeout: 1s
+      retries: 1

ci-scripts/xml_class_list.yml

@@ -20,6 +20,7 @@
   - Build_Image
   - Build_Run_Tests
   - Deploy_Object
+  - Stop_Object
   - Undeploy_Object
   - Cppcheck_Analysis
   - Deploy_Run_OC_PhySim

ci-scripts/xml_files/container_5g_flexric_rfsim.xml

@@ -42,6 +42,13 @@
  002006
  444444
  100002
+ 100003
+ 100004
+ 100005
+ 100006
+ 100007
+ 100008
+ 100009
  100001
  222221
  222222
@@ -95,10 +102,10 @@
 
         <testCase id="000002">
                 <class>Deploy_Object</class>
-                <desc>Deploy OAI FlexRIC</desc>
+                <desc>Deploy OAI nearRT-RIC</desc>
                 <node>localhost</node>
                 <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
-                <services>oai-flexric</services>
+                <services>nearRT-RIC</services>
         </testCase>
 
         <testCase id="000003">
@@ -151,16 +158,72 @@
         </testCase>
 
         <testCase id="100002">
+                <class>Stop_Object</class>
+                <desc>Stop RC monitoring xApp</desc>
+                <node>localhost</node>
+                <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
+                <services>xapp-rc-moni</services>
+        </testCase>
+
+        <testCase id="100003">
+                <class>Stop_Object</class>
+                <desc>Stop KPM monitoring xApp</desc>
+                <node>localhost</node>
+                <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
+                <services>xapp-kpm-moni</services>
+        </testCase>
+
+        <testCase id="100004">
+                <class>Stop_Object</class>
+                <desc>Stop KPM monitoring and RC control xApp</desc>
+                <node>localhost</node>
+                <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
+                <services>xapp-kpm-rc</services>
+        </testCase>
+
+        <testCase id="100005">
+                <class>Stop_Object</class>
+                <desc>Stop custom SMs monitoring xApp</desc>
+                <node>localhost</node>
+                <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
+                <services>xapp-gtp-mac-rlc-pdcp-moni</services>
+        </testCase>
+
+        <testCase id="100006">
+                <class>Stop_Object</class>
+                <desc>Stop OAI UEs</desc>
+                <node>localhost</node>
+                <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
+                <services>oai-nr-ue oai-nr-ue2</services>
+        </testCase>
+
+        <testCase id="100007">
+                <class>Stop_Object</class>
+                <desc>Stop OAI gNB</desc>
+                <node>localhost</node>
+                <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
+                <services>oai-gnb</services>
+        </testCase>
+
+        <testCase id="100008">
+                <class>Stop_Object</class>
+                <desc>Stop nearRT-RIC</desc>
+                <node>localhost</node>
+                <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
+                <services>nearRT-RIC</services>
+        </testCase>
+
+        <testCase id="100009">
                 <class>Undeploy_Object</class>
-                <desc>Undeploy RAN and Flexric</desc>
+                <desc>Undeploy RAN and FlexRIC</desc>
                 <node>localhost</node>
                 <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
                 <always_exec>true</always_exec>
-        </testCase>
+	</testCase>
 
         <testCase id="010000">
                 <class>Deploy_Object</class>
-                <desc>RC monitoring</desc>
+                <desc>Deploy RC monitoring</desc>
                 <node>localhost</node>
                 <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
                 <services>xapp-rc-moni</services>
@@ -168,7 +231,7 @@
 
         <testCase id="010001">
                 <class>Deploy_Object</class>
-                <desc>KPM monitoring</desc>
+                <desc>Deploy KPM monitoring</desc>
                 <node>localhost</node>
                 <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
                 <services>xapp-kpm-moni</services>
@@ -176,7 +239,7 @@
 
         <testCase id="010002">
                 <class>Deploy_Object</class>
-                <desc>KPM monitoring and RC control</desc>
+                <desc>Deploy KPM monitoring and RC control</desc>
                 <node>localhost</node>
                 <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
                 <services>xapp-kpm-rc</services>
@@ -184,7 +247,7 @@
 
         <testCase id="010003">
                 <class>Deploy_Object</class>
-                <desc>Custom SMs monitoring</desc>
+                <desc>Deploy custom SMs monitoring</desc>
                 <node>localhost</node>
                 <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
                 <services>xapp-gtp-mac-rlc-pdcp-moni</services>

ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml

@@ -144,7 +144,6 @@
                 <always_exec>true</always_exec>
                 <desc>Undeploy all OAI 5G stack</desc>
                 <node>localhost</node>
-                <services>oai-vnf oai-pnf</services>
                 <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_u0_25prb</yaml_path>
                 <d_retx_th>1,0,0,0</d_retx_th>
                 <u_retx_th>1,0,0,0</u_retx_th>

ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml

@@ -146,7 +146,6 @@
     <desc>Undeploy gNB</desc>
     <node>matix</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz</yaml_path>
-    <services>oai-gnb</services>
     <d_retx_th>10,100,100,100</d_retx_th>
     <u_retx_th>10,100,100,100</u_retx_th>
   </testCase>

ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml

@@ -146,7 +146,6 @@
     <desc>Undeploy gNB</desc>
     <node>matix</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz</yaml_path>
-    <services>oai-gnb</services>
     <d_retx_th>15,100,100,100</d_retx_th>
     <u_retx_th>20,100,100,100</u_retx_th>
   </testCase>

ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml

@@ -146,7 +146,6 @@
     <desc>Undeploy VNF-PNF</desc>
     <node>matix</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz</yaml_path>
-    <services>oai-vnf oai-pnf</services>
     <d_retx_th>10,100,100,100</d_retx_th>
     <u_retx_th>10,100,100,100</u_retx_th>
   </testCase>

ci-scripts/yaml_files/5g_rfsimulator_flexric/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
-    oai-flexric:
+    nearRT-RIC:
         image: ${REGISTRY-oaisoftwarealliance/}oai-flexric:${FLEXRIC_TAG:-develop}
-        container_name: oai-flexric
+        container_name: nearRT-RIC
         networks:
             public_net:
                 ipv4_address: 192.168.70.150
@@ -17,7 +17,7 @@ services:
         command: "stdbuf -o0 /usr/local/flexric/xApp/c/monitor/xapp_rc_moni"
         container_name: xapp-rc-moni
         depends_on:
-            - oai-flexric
+            - nearRT-RIC
             - oai-gnb
         networks:
             public_net:
@@ -34,7 +34,7 @@ services:
         command: "stdbuf -o0 /usr/local/flexric/xApp/c/monitor/xapp_kpm_moni"
         container_name: xapp-kpm-moni
         depends_on:
-            - oai-flexric
+            - nearRT-RIC
             - oai-gnb
         networks:
             public_net:
@@ -51,7 +51,7 @@ services:
         command: "stdbuf -o0 /usr/local/flexric/xApp/c/kpm_rc/xapp_kpm_rc"
         container_name: xapp-kpm-rc
         depends_on:
-            - oai-flexric
+            - nearRT-RIC
             - oai-gnb
         networks:
             public_net:
@@ -68,7 +68,7 @@ services:
         command: "stdbuf -o0 /usr/local/flexric/xApp/c/monitor/xapp_gtp_mac_rlc_pdcp_moni"
         container_name: xapp-gtp-mac-rlc-pdcp-moni
         depends_on:
-            - oai-flexric
+            - nearRT-RIC
             - oai-gnb
         networks:
             public_net:
@@ -88,7 +88,7 @@ services:
             USE_ADDITIONAL_OPTIONS: -E --rfsim --log_config.global_log_options level,nocolor,time
             ASAN_OPTIONS: detect_leaks=0
         depends_on:
-            - oai-flexric
+            - nearRT-RIC
         networks:
             public_net:
                 ipv4_address: 192.168.70.151