From bcc68d6597b84b30d3f072bf9e41178dc87870e0 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 7 Jan 2022 17:37:01 +0100 Subject: [PATCH 01/22] CI: store both core dump and executable in zip --- ci-scripts/ran.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 4caa876719..b35b73e9d3 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -738,7 +738,10 @@ class RANManagement(): mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/gnb_*.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 60) + result = re.search('core.\d+', mySSH.getBefore()) + if result is not None: + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip core* ran_build/build/{lte,nr}-softmodem', '\$', 60) # add core and executable to zip mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap gnb_*.pcap enb_*txt physim_*.log *stats.log *monitor.pickle *monitor*.png ping*.log.png log/*/*.log log/*/*.pcap', '\$', 15) mySSH.close() From f5d949742e7a786311e4625759b36bcedb85ec94 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 7 Jan 2022 18:59:20 +0100 Subject: [PATCH 02/22] CI: refine tshark capture filters The CI pcaps are too large if we have large data traffic, as they include all the GTP capture (they contain basically the data that we send in the tests). This commit reduces this: - in the RAN: capture only S1AP (port 36412), X2AP (36422), NGAP (38412) traffic - in the CN: exclude GTP traffic (port 2152) --- ci-scripts/cls_containerize.py | 2 +- ci-scripts/epc.py | 6 +++--- ci-scripts/ran.py | 10 ++++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index db51f2647c..ef382a8c6c 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -689,7 +689,7 @@ class Containerize(): def CaptureOnDockerNetworks(self): cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml config | grep com.docker.network.bridge.name | sed -e "s@^.*name: @@"' networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) - cmd = 'sudo nohup tshark -f "not tcp and not arp and not port 53 and not host archive.ubuntu.com and not host security.ubuntu.com"' + cmd = 'sudo nohup tshark -f "not tcp and not arp and not port 53 and not host archive.ubuntu.com and not host security.ubuntu.com and not port 2152"' for name in networkNames.split('\n'): res = re.search('rfsim', name) if res is not None: diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index a7d923ca90..a62bb96ed1 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -133,7 +133,7 @@ class EPCManagement(): mySSH.open(self.IPAddress, self.UserName, self.Password) if re.match('OAI-Rel14-Docker', self.Type, re.IGNORECASE): logging.debug('Using the OAI EPC Release 14 MME in Docker') - mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-mme /bin/bash -c "nohup tshark -i eth0 -i lo:s10 -w /tmp/mme_check_run.pcap 2>&1 > /dev/null"', '\$', 5) + mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-mme /bin/bash -c "nohup tshark -i eth0 -i lo:s10 -f "not port 2152" -w /tmp/mme_check_run.pcap 2>&1 > /dev/null"', '\$', 5) time.sleep(5) mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-mme /bin/bash -c "nohup ./bin/oai_mme -c ./etc/' + self.mmeConfFile + ' > mme_check_run.log 2>&1"', '\$', 5) elif re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): @@ -198,8 +198,8 @@ class EPCManagement(): mySSH.open(self.IPAddress, self.UserName, self.Password) if re.match('OAI-Rel14-Docker', self.Type, re.IGNORECASE): logging.debug('Using the OAI EPC Release 14 SPGW-CUPS in Docker') - mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwc /bin/bash -c "nohup tshark -i eth0 -i lo:p5c -i lo:s5c -w /tmp/spgwc_check_run.pcap 2>&1 > /dev/null"', '\$', 5) - mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwu-tiny /bin/bash -c "nohup tshark -i eth0 -w /tmp/spgwu_check_run.pcap 2>&1 > /dev/null"', '\$', 5) + mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwc /bin/bash -c "nohup tshark -i eth0 -i lo:p5c -i lo:s5c -f "not port 2152" -w /tmp/spgwc_check_run.pcap 2>&1 > /dev/null"', '\$', 5) + mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwu-tiny /bin/bash -c "nohup tshark -i eth0 -f "not port 2152" -w /tmp/spgwu_check_run.pcap 2>&1 > /dev/null"', '\$', 5) time.sleep(5) mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwc /bin/bash -c "nohup ./bin/oai_spgwc -o -c ./etc/spgw_c.conf > spgwc_check_run.log 2>&1"', '\$', 5) time.sleep(5) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index b35b73e9d3..3c0abccdbf 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -371,10 +371,11 @@ class RANManagement(): result = re.search('interfaceToUse=(?P[a-zA-Z0-9\-\_]+)done', mySSH.getBefore()) if result is not None: eth_interface = result.group('eth_interface') - logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m') + fltr = 'port 38412 or port 36412 or port 36422' # NGAP, S1AP, X2AP + logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + ' with filter "' + fltr + '"\u001B[0m') pcapfile = pcapfile_prefix + self.testCase_id + '_log.pcap' mySSH.command('echo ' + lPassWord + ' | sudo -S rm -f /tmp/' + pcapfile , '\$', 5) - mySSH.command('echo $USER; nohup sudo -E tshark -i ' + eth_interface + ' -w /tmp/' + pcapfile + ' > /dev/null 2>&1 &','\$', 5) + mySSH.command('echo $USER; nohup sudo -E tshark -i ' + eth_interface + ' -f "' + fltr + '" -w /tmp/' + pcapfile + ' > /dev/null 2>&1 &','\$', 5) mySSH.close() @@ -390,10 +391,11 @@ class RANManagement(): result = re.search('interfaceToUse=(?P[a-zA-Z0-9\-\_]+)done', mySSH.getBefore()) if result is not None: eth_interface = result.group('eth_interface') - logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m') + fltr = 'port 38412 or port 36412 or port 36422' # NGAP, S1AP, X2AP + logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + ' with filter "' + fltr + '"\u001B[0m') self.epcPcapFile = 'enb_' + self.testCase_id + '_s1log.pcap' mySSH.command('echo ' + localEpcPassword + ' | sudo -S rm -f /tmp/' + self.epcPcapFile , '\$', 5) - mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5) + mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -f "' + fltr + '" -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5) mySSH.close() mySSH.open(lIpAddr, lUserName, lPassWord) mySSH.command('cd ' + lSourcePath, '\$', 5) From 9240d51774e46444c2cf59544a763194ed26659a Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 1 Dec 2021 14:29:48 +0100 Subject: [PATCH 03/22] Complete build_oai usage --- cmake_targets/build_oai | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/cmake_targets/build_oai b/cmake_targets/build_oai index b427b161e5..e5a806c0ad 100755 --- a/cmake_targets/build_oai +++ b/cmake_targets/build_oai @@ -99,25 +99,23 @@ Options Makes the LTE softmodem --eNBocp Makes the OCP LTE softmodem --gNB +--gNB Makes the NR softmodem ---nrUE - Makes the NR UE softmodem --RU Makes the OAI RRU --UE Makes the UE specific parts (ue_ip, usim, nvram) from the given configuration file +--nrUE + Makes the NR UE softmodem --UE-conf-nvram [configuration file] Specify conf_nvram_path (default \"$conf_nvram_path\") --UE-gen-nvram [output path] Specify gen_nvram_path (default \"$gen_nvram_path\") ---HWLAT - Makes test program for haw latency tests --a | --agent - Enables agent for software-defined control of the eNB -w | --hardware EXMIMO, USRP, BLADERF, LMSSDR, IRIS, ADRV9371_ZC706, SIMU, AW2SORI, None (Default) Adds this RF board support (in external packages installation and in compilation) +-t | --transport + Selects the transport protocol type, options: None, ETHERNET -P | --phy_simulators Makes the unitary tests Layer 1 simulators -S | --core_simulators @@ -128,6 +126,14 @@ Options runs only specified test cases specified here. This flag is only valid with -s -V | --vcd Adds a debgging facility to the binary files: GUI with major internal synchronization events +-x | --xforms + Will compile with software oscilloscope features +--HWLAT + Makes test program for haw latency tests +--HWLAT_TEST + Compiles a hw latency test program +--verbose-ci + Compile with verbose instructions in CI Docker env --verbose-compile Shows detailed compilation instructions in makefile --cflags_processor @@ -163,6 +169,10 @@ Options Build for I/Q record-playback modes -k | --skip-shared-libraries Skip build for shared libraries to reduce compilation time when building frequently for debugging purposes +--ninja + Tell cmake to use the Ninja build system. Without, will generate make files +--sanitize-address | -fsanitize=address + Enable the address sanitizer on all targets --ittiSIM Makes the itti simulator -h | --help From 0c9c716fbc0debc86eace57867da943334d0ec7f Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Tue, 14 Dec 2021 18:08:14 +0100 Subject: [PATCH 04/22] Build all libraries in Ubuntu/RHEL containers --- docker/Dockerfile.build.rhel8.2 | 2 +- docker/Dockerfile.build.ubuntu18 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.build.rhel8.2 b/docker/Dockerfile.build.rhel8.2 index 7ddfcdc034..22486c5e68 100644 --- a/docker/Dockerfile.build.rhel8.2 +++ b/docker/Dockerfile.build.rhel8.2 @@ -35,4 +35,4 @@ COPY . . RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ - ./build_oai --eNB --gNB --RU --UE --nrUE --ninja -w USRP --verbose-ci + ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib all -w USRP --verbose-ci diff --git a/docker/Dockerfile.build.ubuntu18 b/docker/Dockerfile.build.ubuntu18 index b479ee90d9..3ba1e41442 100644 --- a/docker/Dockerfile.build.ubuntu18 +++ b/docker/Dockerfile.build.ubuntu18 @@ -35,4 +35,4 @@ COPY . . RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ - ./build_oai --eNB --gNB --RU --UE --nrUE --ninja -w USRP --verbose-ci + ./build_oai --eNB --gNB --RU --UE --nrUE --ninja --build-lib all -w USRP --verbose-ci From 8c38dc0fbf50da70317ebe7f0c77604d33aa7187 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 24 Dec 2021 13:20:32 +0100 Subject: [PATCH 05/22] Include telnet SOs in container images --- docker/Dockerfile.eNB.rhel8.2 | 2 ++ docker/Dockerfile.eNB.ubuntu18 | 2 ++ docker/Dockerfile.gNB.rhel8.2 | 1 + docker/Dockerfile.gNB.ubuntu18 | 1 + docker/Dockerfile.lteUE.rhel8.2 | 1 + docker/Dockerfile.lteUE.ubuntu18 | 1 + docker/Dockerfile.nrUE.rhel8.2 | 2 ++ docker/Dockerfile.nrUE.ubuntu18 | 2 ++ 8 files changed, 12 insertions(+) diff --git a/docker/Dockerfile.eNB.rhel8.2 b/docker/Dockerfile.eNB.rhel8.2 index bb0a9cb064..6c921a4932 100644 --- a/docker/Dockerfile.eNB.rhel8.2 +++ b/docker/Dockerfile.eNB.rhel8.2 @@ -69,6 +69,8 @@ COPY --from=enb-build \ /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \ /oai-ran/cmake_targets/ran_build/build/libdfts.so \ /oai-ran/cmake_targets/ran_build/build/liboai_iqplayer.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_enb.so \ /usr/local/lib/ COPY --from=enb-base \ diff --git a/docker/Dockerfile.eNB.ubuntu18 b/docker/Dockerfile.eNB.ubuntu18 index 23cea729ca..1f375a20dc 100644 --- a/docker/Dockerfile.eNB.ubuntu18 +++ b/docker/Dockerfile.eNB.ubuntu18 @@ -76,6 +76,8 @@ COPY --from=enb-build \ /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \ /oai-ran/cmake_targets/ran_build/build/libdfts.so \ /oai-ran/cmake_targets/ran_build/build/liboai_iqplayer.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_enb.so \ /usr/local/lib/ # Now we are copying from builder-image the UHD files. diff --git a/docker/Dockerfile.gNB.rhel8.2 b/docker/Dockerfile.gNB.rhel8.2 index 832fa17153..703f4c67b5 100644 --- a/docker/Dockerfile.gNB.rhel8.2 +++ b/docker/Dockerfile.gNB.rhel8.2 @@ -74,6 +74,7 @@ COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/libldpc_optim.so \ /oai-ran/cmake_targets/ran_build/build/libldpc_optim8seg.so \ /oai-ran/cmake_targets/ran_build/build/libldpc_orig.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \ /usr/local/lib/ COPY --from=gnb-base \ diff --git a/docker/Dockerfile.gNB.ubuntu18 b/docker/Dockerfile.gNB.ubuntu18 index 66c5d7703f..24e4bcbc7c 100644 --- a/docker/Dockerfile.gNB.ubuntu18 +++ b/docker/Dockerfile.gNB.ubuntu18 @@ -78,6 +78,7 @@ COPY --from=gnb-build \ /oai-ran/cmake_targets/ran_build/build/libldpc_optim.so \ /oai-ran/cmake_targets/ran_build/build/libldpc_optim8seg.so \ /oai-ran/cmake_targets/ran_build/build/libldpc_orig.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \ /usr/local/lib/ # Now we are copying from builder-image the UHD files. diff --git a/docker/Dockerfile.lteUE.rhel8.2 b/docker/Dockerfile.lteUE.rhel8.2 index 2580b95278..f547558bdb 100644 --- a/docker/Dockerfile.lteUE.rhel8.2 +++ b/docker/Dockerfile.lteUE.rhel8.2 @@ -73,6 +73,7 @@ COPY --from=lte-ue-build \ /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \ /oai-ran/cmake_targets/ran_build/build/libSIMU.so \ /oai-ran/cmake_targets/ran_build/build/libdfts.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \ /usr/local/lib/ COPY --from=lte-ue-base \ diff --git a/docker/Dockerfile.lteUE.ubuntu18 b/docker/Dockerfile.lteUE.ubuntu18 index 70394d8a4b..a5bf4cfe94 100644 --- a/docker/Dockerfile.lteUE.ubuntu18 +++ b/docker/Dockerfile.lteUE.ubuntu18 @@ -80,6 +80,7 @@ COPY --from=lte-ue-build \ /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \ /oai-ran/cmake_targets/ran_build/build/libSIMU.so \ /oai-ran/cmake_targets/ran_build/build/libdfts.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \ /usr/local/lib/ # Now we are copying from builder-image the UHD files. diff --git a/docker/Dockerfile.nrUE.rhel8.2 b/docker/Dockerfile.nrUE.rhel8.2 index f774638f47..cbc8d8c65f 100644 --- a/docker/Dockerfile.nrUE.rhel8.2 +++ b/docker/Dockerfile.nrUE.rhel8.2 @@ -76,6 +76,8 @@ COPY --from=nr-ue-build \ /oai-ran/cmake_targets/ran_build/build/libldpc_optim.so \ /oai-ran/cmake_targets/ran_build/build/libldpc_optim8seg.so \ /oai-ran/cmake_targets/ran_build/build/libldpc_orig.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_5Gue.so \ /usr/local/lib/ COPY --from=nr-ue-base \ diff --git a/docker/Dockerfile.nrUE.ubuntu18 b/docker/Dockerfile.nrUE.ubuntu18 index 021e8d39cf..126f268e00 100644 --- a/docker/Dockerfile.nrUE.ubuntu18 +++ b/docker/Dockerfile.nrUE.ubuntu18 @@ -81,6 +81,8 @@ COPY --from=nr-ue-build \ /oai-ran/cmake_targets/ran_build/build/libldpc_optim.so \ /oai-ran/cmake_targets/ran_build/build/libldpc_optim8seg.so \ /oai-ran/cmake_targets/ran_build/build/libldpc_orig.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \ + /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_5Gue.so \ /usr/local/lib/ # Now we are copying from builder-image the UHD files. From 2a4d4d9cbddbe76c331b3615f709c72dda4587aa Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 24 Dec 2021 14:27:37 +0100 Subject: [PATCH 06/22] Add readline library: telnetsrv depends on it --- cmake_targets/tools/build_helper | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake_targets/tools/build_helper b/cmake_targets/tools/build_helper index 06f32183b7..05075bb8ed 100755 --- a/cmake_targets/tools/build_helper +++ b/cmake_targets/tools/build_helper @@ -756,6 +756,7 @@ check_install_oai_software() { libidn11-dev \ libmysqlclient-dev \ libpython2.7-dev \ + libreadline-dev \ libsctp1 \ libsctp-dev \ libssl-dev \ @@ -842,6 +843,7 @@ check_install_oai_software() { patch \ psmisc \ python \ + readline-devel \ subversion \ xmlstarlet \ python-pip \ From bd08a828cfd7504e82dfaad7408ab03ed6226704 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 27 Dec 2021 14:42:48 +0100 Subject: [PATCH 07/22] Give container image delete some time --- ci-scripts/cls_containerize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index ef382a8c6c..60e755e9cd 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -341,7 +341,7 @@ class Containerize(): # Remove all intermediate build images if self.ranAllowMerge and forceBaseImageBuild: mySSH.command(self.cli + ' image rm ' + baseImage + ':' + baseTag + ' || true', '\$', 30) - mySSH.command(self.cli + ' image rm ran-build:' + imageTag + ' || true','\$', 5) + mySSH.command(self.cli + ' image rm ran-build:' + imageTag + ' || true','\$', 30) # Cleaning any created tmp volume mySSH.command(self.cli + ' volume prune --force || true','\$', 15) mySSH.close() From ac9ee8e206996444c0ee077cc6168b996dda07fd Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 2 Dec 2021 14:49:20 +0100 Subject: [PATCH 08/22] Increase 5G/SA/RFsimulator throughput test --- ci-scripts/xml_files/container_5g_rfsim.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-scripts/xml_files/container_5g_rfsim.xml b/ci-scripts/xml_files/container_5g_rfsim.xml index d930886d0b..d1114616a8 100644 --- a/ci-scripts/xml_files/container_5g_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_rfsim.xml @@ -93,7 +93,7 @@ rfsim5g-oai-nr-ue rfsim5g-oai-ext-dn -B 12.1.1.2 -u -i 1 -s - -c 12.1.1.2 -u -i 1 -t 30 -b 400K + -c 12.1.1.2 -u -i 1 -t 30 -b 3M @@ -102,7 +102,7 @@ rfsim5g-oai-ext-dn rfsim5g-oai-nr-ue -u -i 1 -s - -B 12.1.1.2 -c 192.168.72.135 -u -i 1 -t 30 -b 20K + -B 12.1.1.2 -c 192.168.72.135 -u -i 1 -t 30 -b 1M From d02a65c82476ec5fa89ebcdbc975239ad1026ce0 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 27 Dec 2021 15:55:42 +0100 Subject: [PATCH 09/22] Tentatively analyze log files of RFsims --- ci-scripts/cls_containerize.py | 21 +++++++++++++++++++-- ci-scripts/main.py | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 60e755e9cd..9248eba4b8 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -700,7 +700,7 @@ class Containerize(): logging.debug(cmd) networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) - def UndeployGenObject(self, HTML): + def UndeployGenObject(self, HTML, RAN): self.exitStatus = 0 ymlPath = self.yamlPath[0].split('/') logPath = '../cmake_targets/log/' + ymlPath[1] @@ -731,7 +731,24 @@ class Containerize(): logging.debug(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) if anyLogs: - cmd = 'mkdir -p '+ logPath + ' && mv ' + self.yamlPath[0] + '/*.log ' + logPath + cmd = 'mkdir -p '+ logPath + ' && cp ' + self.yamlPath[0] + '/*.log ' + logPath + logging.debug(cmd) + deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) + + # Analyzing log file! + filename = self.yamlPath[0] + '/rfsim?g_oai_?nb.log' + logging.debug('\u001B[1m Analyzing xNB logfile \u001B[0m ' + filename) + cmd = 'ls ' + filename + logfileLs = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=5) + logging.debug(logfileLs) + # For the moment just assume this exists + logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML) + if (logStatus < 0): + HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus) + else: + HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) + + cmd = 'rm ' + self.yamlPath[0] + '/*.log' logging.debug(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) if self.tsharkStarted: diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 3a5a69573d..5b5eec285c 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -909,7 +909,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re if CONTAINERS.exitStatus==1: RAN.prematureExit = True elif action == 'UndeployGenObject': - CONTAINERS.UndeployGenObject(HTML) + CONTAINERS.UndeployGenObject(HTML, RAN) if CONTAINERS.exitStatus==1: RAN.prematureExit = True elif action == 'PingFromContainer': From 06bbea7f10aa46e4acc627c881c9d1df90aae834 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 10 Jan 2022 15:44:01 +0100 Subject: [PATCH 10/22] CI: Check for correct files --- ci-scripts/ran.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 3c0abccdbf..f20539b21e 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -1021,12 +1021,9 @@ class RANManagement(): datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader) rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field - if (os.path.isfile('./nrL1_stats.log')) and (os.path.isfile('./nrL1_stats.log')): - stat_files_present=True - else: - stat_files_present=False - logging.debug("NR Stats files for RT analysis not found") - if stat_files_present: + if os.path.isfile('./nrL1_stats.log') and os.path.isfile('./nrMAC_stats.log'): + # don't use CI-nrL1_stats.log, as this will increase the processing time for + # no reason, we just need the last occurence nrL1_stats = open('./nrL1_stats.log', 'r') nrMAC_stats = open('./nrMAC_stats.log', 'r') for line in nrL1_stats.readlines(): @@ -1047,6 +1044,8 @@ class RANManagement(): real_time_stats[k]=tmp.group(1) nrL1_stats.close() nrMAC_stats.close() + else: + logging.debug("NR Stats files for RT analysis not found") #stdout log file and stat log files analysis completed logging.debug(' File analysis (stdout, stats) completed') From 74a4c2f6ef81334d2642170711c90734fa3efb57 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 10 Jan 2022 15:44:46 +0100 Subject: [PATCH 11/22] CI: Log repo cloning/merging instructions --- ci-scripts/ran.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index f20539b21e..3f9c1a8e9e 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -159,6 +159,7 @@ class RANManagement(): # Raphael: here add a check if git clone or git fetch went smoothly mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5) mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) + mySSH.command('git advice.detachedHead false', '\$', 5) # Checking the BUILD INFO file if not self.backgroundBuild: mySSH.command('ls *.txt', '\$', 5) @@ -203,6 +204,7 @@ class RANManagement(): else: logging.debug('Merging with the target branch: ' + self.ranTargetBranch) mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + logging.debug(mySSH.getBefore()) # print what git said when merging/checking out mySSH.command('source oaienv', '\$', 5) mySSH.command('cd cmake_targets', '\$', 5) mySSH.command('mkdir -p log', '\$', 5) @@ -212,6 +214,7 @@ class RANManagement(): mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5) mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5) mySSH.command('echo ' + lPassWord + ' | sudo -S ls', '\$', 5) + logging.debug(mySSH.getBefore()) # print current directory contents for verification mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-build.sh' + ' > ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ' + ' 2>&1 &', lUserName, 5) mySSH.close() HTML.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK) From 76411447abc0c9469f195df8c73b4d48c79c6b01 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 10 Jan 2022 15:46:12 +0100 Subject: [PATCH 12/22] CI: Continuously collect RAN logging file output --- ci-scripts/stats_monitor.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ci-scripts/stats_monitor.py b/ci-scripts/stats_monitor.py index 933bf13203..e732325fa1 100755 --- a/ci-scripts/stats_monitor.py +++ b/ci-scripts/stats_monitor.py @@ -61,9 +61,16 @@ class StatMonitor(): def collect(self,node_type): if node_type=='enb': - cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' + files = ["L1_stats.log", "MAC_stats.log", "PDCP_stats.log", "RRC_stats.log"] else: #'gnb' - cmd='cat nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log' + files = ["nrL1_stats.log", "nrMAC_stats.log", "nrPDCP_stats.log", "nrRRC_stats.log"] + # append each file's contents to another file (prepended with CI-) for + # post-mortem/debugging analysis + for f in files: + cmd = rf'cat {f} >> CI-{f}' + subprocess.Popen(shlex.split(cmd)) + # join the files for further processing + cmd = rf'cat {shlex.join(files)}' process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) output = process.stdout.readlines() if node_type=='enb': From 4e0548328dcef531afcf1f8b06f3a54e64ed8638 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 10 Jan 2022 15:46:45 +0100 Subject: [PATCH 13/22] CI: delete old stats_monitor files --- ci-scripts/stats_monitor.py.old | 94 ----------------------------- ci-scripts/stats_monitor_dev.py | 101 -------------------------------- 2 files changed, 195 deletions(-) delete mode 100755 ci-scripts/stats_monitor.py.old delete mode 100755 ci-scripts/stats_monitor_dev.py diff --git a/ci-scripts/stats_monitor.py.old b/ci-scripts/stats_monitor.py.old deleted file mode 100755 index ae9b39bfa6..0000000000 --- a/ci-scripts/stats_monitor.py.old +++ /dev/null @@ -1,94 +0,0 @@ -import subprocess -import time -import shlex -import re -import sys -import matplotlib.pyplot as plt -import pickle -import numpy as np -import os - -def collect(d, node_type): - if node_type=='enb': - cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' - else: #'gnb' - cmd='cat nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log' - process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - output = process.stdout.readlines() - for l in output: - tmp=l.decode("utf-8") - result=re.match(rf'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp) - if result is not None: - d['PHR'].append(int(result.group(1))) - d['bler'].append(float(result.group(2))) - d['mcsoff'].append(int(result.group(3))) - d['mcs'].append(int(result.group(4))) - - -def graph(d, node_type): - - - figure, axis = plt.subplots(4, 1,figsize=(10, 10)) - - major_ticks = np.arange(0, len(d['PHR'])+1, 1) - axis[0].set_xticks(major_ticks) - axis[0].set_xticklabels([]) - axis[0].plot(d['PHR'],marker='o') - axis[0].set_xlabel('time') - axis[0].set_ylabel('PHR') - axis[0].set_title("PHR") - - major_ticks = np.arange(0, len(d['bler'])+1, 1) - axis[1].set_xticks(major_ticks) - axis[1].set_xticklabels([]) - axis[1].plot(d['bler'],marker='o') - axis[1].set_xlabel('time') - axis[1].set_ylabel('bler') - axis[1].set_title("bler") - - major_ticks = np.arange(0, len(d['mcsoff'])+1, 1) - axis[2].set_xticks(major_ticks) - axis[2].set_xticklabels([]) - axis[2].plot(d['mcsoff'],marker='o') - axis[2].set_xlabel('time') - axis[2].set_ylabel('mcsoff') - axis[2].set_title("mcsoff") - - major_ticks = np.arange(0, len(d['mcs'])+1, 1) - axis[3].set_xticks(major_ticks) - axis[3].set_xticklabels([]) - axis[3].plot(d['mcs'],marker='o') - axis[3].set_xlabel('time') - axis[3].set_ylabel('mcs') - axis[3].set_title("mcs") - - plt.tight_layout() - # Combine all the operations and display - plt.savefig(node_type+'_stats_monitor.png') - plt.show() - -if __name__ == "__main__": - - node_type = sys.argv[1]#enb or gnb - - d={} - d['PHR']=[] - d['bler']=[] - d['mcsoff']=[] - d['mcs']=[] - - - cmd='ps aux | grep modem | grep -v grep' - process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - output = process.stdout.readlines() - while len(output)!=0 : - collect(d, node_type) - process=subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) - output = process.stdout.readlines() - time.sleep(1) - print('process stopped') - with open(node_type+'_stats_monitor.pickle', 'wb') as handle: - pickle.dump(d, handle, protocol=pickle.HIGHEST_PROTOCOL) - graph(d, node_type) - - diff --git a/ci-scripts/stats_monitor_dev.py b/ci-scripts/stats_monitor_dev.py deleted file mode 100755 index 83de20f5d2..0000000000 --- a/ci-scripts/stats_monitor_dev.py +++ /dev/null @@ -1,101 +0,0 @@ -""" -To create graphs and pickle from runtime statistics in L1,MAC,RRC,PDCP files -""" - -import subprocess -import time -import shlex -import re -import sys -import pickle -import matplotlib.pyplot as plt -import numpy as np -import yaml - - -class StatMonitor(): - def __init__(self,): - with open('stats_monitor_conf.yaml','r') as file: - self.d = yaml.load(file) - for node in self.d: - for metric in self.d[node]: - self.d[node][metric]=[] - - - def process_gnb (self,node_type,output): - for line in output: - tmp=line.decode("utf-8") - result=re.match(r'^.*\bdlsch_rounds\b ([0-9]+)\/([0-9]+).*\bdlsch_errors\b ([0-9]+)',tmp) - if result is not None: - self.d[node_type]['dlsch_err'].append(int(result.group(3))) - percentage=float(result.group(2))/float(result.group(1)) - self.d[node_type]['dlsch_err_perc_round_1'].append(percentage) - result=re.match(r'^.*\bulsch_rounds\b ([0-9]+)\/([0-9]+).*\bulsch_errors\b ([0-9]+)',tmp) - if result is not None: - self.d[node_type]['ulsch_err'].append(int(result.group(3))) - percentage=float(result.group(2))/float(result.group(1)) - self.d[node_type]['ulsch_err_perc_round_1'].append(percentage) - - - def process_enb (self,node_type,output): - for line in output: - tmp=line.decode("utf-8") - result=re.match(r'^.*\bPHR\b ([0-9]+).+\bbler\b ([0-9]+\.[0-9]+).+\bmcsoff\b ([0-9]+).+\bmcs\b ([0-9]+)',tmp) - if result is not None: - self.d[node_type]['PHR'].append(int(result.group(1))) - self.d[node_type]['bler'].append(float(result.group(2))) - self.d[node_type]['mcsoff'].append(int(result.group(3))) - self.d[node_type]['mcs'].append(int(result.group(4))) - - - def collect(self,node_type): - if node_type=='enb': - cmd='cat L1_stats.log MAC_stats.log PDCP_stats.log RRC_stats.log' - else: #'gnb' - cmd='cat nrL1_stats.log nrMAC_stats.log nrPDCP_stats.log nrRRC_stats.log' - process=subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) - output = process.stdout.readlines() - if node_type=='enb': - self.process_enb(node_type,output) - else: #'gnb' - self.process_gnb(node_type,output) - - - def graph(self,node_type): - col = 1 - figure, axis = plt.subplots(len(self.d[node_type]), col ,figsize=(10, 10)) - i=0 - for metric in self.d[node_type]: - major_ticks = np.arange(0, len(self.d[node_type][metric])+1, 1) - axis[i].set_xticks(major_ticks) - axis[i].set_xticklabels([]) - axis[i].plot(self.d[node_type][metric],marker='o') - axis[i].set_xlabel('time') - axis[i].set_ylabel(metric) - axis[i].set_title(metric) - i+=1 - - plt.tight_layout() - # Combine all the operations and display - plt.savefig(node_type+'_stats_monitor.png') - plt.show() - - -if __name__ == "__main__": - - node = sys.argv[1]#enb or gnb - mon=StatMonitor() - - #collecting stats when modem process is stopped - CMD='ps aux | grep mode | grep -v grep' - process=subprocess.Popen(CMD, shell=True, stdout=subprocess.PIPE) - output = process.stdout.readlines() - while len(output)!=0 : - mon.collect(node) - process=subprocess.Popen(CMD, shell=True, stdout=subprocess.PIPE) - output = process.stdout.readlines() - time.sleep(1) - print('Process stopped') - with open(node+'_stats_monitor.pickle', 'wb') as handle: - pickle.dump(mon.d, handle, protocol=pickle.HIGHEST_PROTOCOL) - mon.graph(node) From 396fcd435ef00c967dfbb08ad9ce15b3fd50ede8 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Wed, 12 Jan 2022 14:43:04 +0100 Subject: [PATCH 14/22] CI: force rebuild of ran_base if build_oai/build_helper/Dockerfile change --- ci-scripts/cls_containerize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 9248eba4b8..be42bc2fc5 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -221,7 +221,7 @@ class Containerize(): if (self.ranAllowMerge): imageTag = 'ci-temp' if self.ranTargetBranch == 'develop': - mySSH.command('git diff HEAD..origin/develop -- docker/Dockerfile.base' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5) + mySSH.command('git diff HEAD..origin/develop -- cmake_targets/build_oai cmake_targets/tools/build_helper docker/Dockerfile.base' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5) result = re.search('index', mySSH.getBefore()) if result is not None: forceBaseImageBuild = True From 63a5a097325342a703c63009ca535022b5500e9b Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 13 Jan 2022 18:34:53 +0100 Subject: [PATCH 15/22] Complete build_oai -t options --- cmake_targets/build_oai | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake_targets/build_oai b/cmake_targets/build_oai index e5a806c0ad..2a032079d3 100755 --- a/cmake_targets/build_oai +++ b/cmake_targets/build_oai @@ -115,7 +115,7 @@ Options EXMIMO, USRP, BLADERF, LMSSDR, IRIS, ADRV9371_ZC706, SIMU, AW2SORI, None (Default) Adds this RF board support (in external packages installation and in compilation) -t | --transport - Selects the transport protocol type, options: None, ETHERNET + Selects the transport protocol type, options: None, Ethernet, benetel4g, benetel5g -P | --phy_simulators Makes the unitary tests Layer 1 simulators -S | --core_simulators From a8041b6794a84ce9915f390b26a156d70228a0ec Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Thu, 13 Jan 2022 18:44:40 +0100 Subject: [PATCH 16/22] CI: Correct file path of logging results --- ci-scripts/cls_containerize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index be42bc2fc5..f2f5d39153 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -736,7 +736,7 @@ class Containerize(): deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) # Analyzing log file! - filename = self.yamlPath[0] + '/rfsim?g_oai_?nb.log' + filename = self.yamlPath[0] + '/rfsim?g-oai-?nb.log' logging.debug('\u001B[1m Analyzing xNB logfile \u001B[0m ' + filename) cmd = 'ls ' + filename logfileLs = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=5) From 5dae0096f00ce118163d722dcc9991ff13e7eb46 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Fri, 14 Jan 2022 18:29:17 +0100 Subject: [PATCH 17/22] CI: Rename SA OAIUE x300 test cases to not overwrite N300 results --- ci-scripts/xml_files/fr1_sa_oaiue_x300.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ci-scripts/xml_files/fr1_sa_oaiue_x300.xml b/ci-scripts/xml_files/fr1_sa_oaiue_x300.xml index 509fef9e8a..c43bdd2a7c 100644 --- a/ci-scripts/xml_files/fr1_sa_oaiue_x300.xml +++ b/ci-scripts/xml_files/fr1_sa_oaiue_x300.xml @@ -26,19 +26,19 @@ tasks 1 - 040000 + 041000 000002 - 010000 + 011000 000002 - 050000 - 050001 + 051000 + 051001 000001 010002 080000 - + Initialize_OAI_UE Initialize OAI UE (X300) nr @@ -51,7 +51,7 @@ - + Initialize_eNB Initialize gNB -O ci-scripts/conf_files/gnb.band78.sa.fr1.106PRB.usrpn310.conf --sa --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time @@ -76,14 +76,14 @@ - + Ping Ping from CN to UE: 20pings in 20sec -c 20 5 - + Ping Ping from CN to UE: 100pings in 20sec -c 100 -i 0.2 From 6ab196d03b19f6dbcd27eb7db4bf6c04d3c7be00 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Mon, 17 Jan 2022 13:26:15 +0100 Subject: [PATCH 18/22] fix(ci): no history also for single ssh commands Signed-off-by: Raphael Defosseux --- ci-scripts/sshconnection.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci-scripts/sshconnection.py b/ci-scripts/sshconnection.py index 67e0b832e6..d5b4a4c83e 100644 --- a/ci-scripts/sshconnection.py +++ b/ci-scripts/sshconnection.py @@ -157,22 +157,24 @@ class SSHConnection(): if not silent: logging.debug(commandline) self.cmd2Results = '' + noHistoryCmd = 'unset HISTFILE; ' + commandline myHost = self.username + '@' + self.ipaddress # CAUTION: THIS METHOD IMPLIES THAT THERE ARE VALID SSH KEYS # BETWEEN THE PYTHON EXECUTOR NODE AND THE REMOTE HOST # OTHERWISE IT WON'T WORK - lSsh = subprocess.Popen(["ssh", "%s" % myHost, commandline],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + lSsh = subprocess.Popen(["ssh", "%s" % myHost, noHistoryCmd],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) self.cmd2Results = str(lSsh.stdout.readlines()) def command3(self, commandline, timeout, silent=False): if not silent: logging.debug(commandline) self.cmd2Results = '' + noHistoryCmd = 'unset HISTFILE; ' + commandline myHost = self.username + '@' + self.ipaddress # CAUTION: THIS METHOD IMPLIES THAT THERE ARE VALID SSH KEYS # BETWEEN THE PYTHON EXECUTOR NODE AND THE REMOTE HOST # OTHERWISE IT WON'T WORK - lSsh = subprocess.Popen(["ssh", "%s" % myHost, commandline],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) + lSsh = subprocess.Popen(["ssh", "%s" % myHost, noHistoryCmd],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE) return lSsh.stdout.readlines() From b0e3867a5f6de4523711e49d56a6bae993e8d6d5 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 17 Jan 2022 15:01:53 +0100 Subject: [PATCH 19/22] Fix: Right RFsim CU/DU filename --- ci-scripts/cls_containerize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index f2f5d39153..cd1320b33f 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -736,7 +736,7 @@ class Containerize(): deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) # Analyzing log file! - filename = self.yamlPath[0] + '/rfsim?g-oai-?nb.log' + filename = self.yamlPath[0] + '/rfsim?g-oai-*.log' # 4g/enb, 5g/gnb, 5g/-du/-cu logging.debug('\u001B[1m Analyzing xNB logfile \u001B[0m ' + filename) cmd = 'ls ' + filename logfileLs = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=5) From 64058d84b3f74de55109206449ff6df3cd2e2fb8 Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 17 Jan 2022 16:02:26 +0100 Subject: [PATCH 20/22] Don't execute ls on files to analyze (is useless) --- ci-scripts/cls_containerize.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index cd1320b33f..894067a03f 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -738,9 +738,6 @@ class Containerize(): # Analyzing log file! filename = self.yamlPath[0] + '/rfsim?g-oai-*.log' # 4g/enb, 5g/gnb, 5g/-du/-cu logging.debug('\u001B[1m Analyzing xNB logfile \u001B[0m ' + filename) - cmd = 'ls ' + filename - logfileLs = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=5) - logging.debug(logfileLs) # For the moment just assume this exists logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML) if (logStatus < 0): From edb876460f49d1e085847c992f1d3d85ce7bbba4 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Wed, 19 Jan 2022 14:38:57 +0100 Subject: [PATCH 21/22] feat(ci): adding RAN containers analysis Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 39 +++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 894067a03f..799c58c6aa 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -730,20 +730,34 @@ class Containerize(): cmd = 'cd ' + self.yamlPath[0] + ' && docker logs ' + cName + ' > ' + cName + '.log 2>&1' logging.debug(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) + fullStatus = True if anyLogs: cmd = 'mkdir -p '+ logPath + ' && cp ' + self.yamlPath[0] + '/*.log ' + logPath logging.debug(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) # Analyzing log file! - filename = self.yamlPath[0] + '/rfsim?g-oai-*.log' # 4g/enb, 5g/gnb, 5g/-du/-cu - logging.debug('\u001B[1m Analyzing xNB logfile \u001B[0m ' + filename) - # For the moment just assume this exists - logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML) - if (logStatus < 0): - HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus) - else: - HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) + listOfPossibleRanContainers = ['enb', 'gnb', 'cu', 'du'] + for container in listOfPossibleRanContainers: + filename = self.yamlPath[0] + '/rfsim?g-oai-' + container + '.log' + cmd = 'ls ' + filename + containerStatus = True + try: + lsStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) + filename = str(lsStatus).strip() + except: + containerStatus = False + if not containerStatus: + continue + + logging.debug('\u001B[1m Analyzing xNB logfile ' + filename + ' \u001B[0m') + # For the moment just assume this exists + logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML) + if (logStatus < 0): + fullStatus = False + HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus) + else: + HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) cmd = 'rm ' + self.yamlPath[0] + '/*.log' logging.debug(cmd) @@ -761,6 +775,7 @@ class Containerize(): logging.debug(cmd) copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) + logging.debug('\u001B[1m Undeploying \u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml down' logging.debug(cmd) try: @@ -777,8 +792,12 @@ class Containerize(): logging.debug(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) - HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK) - logging.info('\u001B[1m Undeploying OAI Object(s) PASS\u001B[0m') + if fullStatus: + HTML.CreateHtmlTestRow('n/a', 'OK', CONST.ALL_PROCESSES_OK) + logging.info('\u001B[1m Undeploying OAI Object(s) PASS\u001B[0m') + else: + HTML.CreateHtmlTestRow('n/a', 'KO', CONST.ALL_PROCESSES_OK) + logging.info('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m') def PingFromContainer(self, HTML): self.exitStatus = 0 From 425bb1490093a392bbc644e1b5827d33b4eaf7c1 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Wed, 19 Jan 2022 14:42:51 +0100 Subject: [PATCH 22/22] fix(ci): better shutdown of container when using USRP Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 23 +++++++++++++++++++---- ci-scripts/epc.py | 2 ++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 799c58c6aa..ba5ce49e8d 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -586,11 +586,20 @@ class Containerize(): mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) # Currently support only one mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5) + containerName = '' + containerToKill = False result = re.search('container_name: (?P[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) if self.eNB_logFile[self.eNB_instance] == '': self.eNB_logFile[self.eNB_instance] = 'enb_' + HTML.testCase_id + '.log' if result is not None: containerName = result.group('container_name') + containerToKill = True + if containerToKill: + mySSH.command('docker inspect ' + containerName, '\$', 30) + result = re.search('Error: No such object: ' + containerName, mySSH.getBefore()) + if result is not None: + containerToKill = False + if containerToKill: mySSH.command('docker kill --signal INT ' + containerName, '\$', 30) time.sleep(5) mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30) @@ -603,20 +612,26 @@ class Containerize(): mySSH.close() # Analyzing log file! - copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '.') + if containerToKill: + copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '.') + else: + copyin_res = 0 nodeB_prefix = 'e' if (copyin_res == -1): HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!' HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) else: - logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance]) - logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML) + if containerToKill: + logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance]) + logStatus = RAN.AnalyzeLogFile_eNB(self.eNB_logFile[self.eNB_instance], HTML) + else: + logStatus = 0 if (logStatus < 0): HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus) else: HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK) # all the xNB run logs shall be on the server 0 for logCollecting - if self.eNB_serverId[self.eNB_instance] != '0': + if containerToKill and self.eNB_serverId[self.eNB_instance] != '0': mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + self.eNB_logFile[self.eNB_instance], self.eNBSourceCodePath + '/cmake_targets/') logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') diff --git a/ci-scripts/epc.py b/ci-scripts/epc.py index a62bb96ed1..9ed9f06d8e 100644 --- a/ci-scripts/epc.py +++ b/ci-scripts/epc.py @@ -532,6 +532,7 @@ class EPCManagement(): logging.debug('Terminating OAI CN5G') mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5) mySSH.command('./core-network.sh stop nrf spgwu', '\$', 60) + mySSH.command('docker volume prune --force || true', '\$', 60) time.sleep(2) mySSH.command('tshark -r /tmp/oai-cn5g.pcap | egrep --colour=never "Tracking area update" ','\$', 30) result = re.search('Tracking area update request', mySSH.getBefore()) @@ -760,6 +761,7 @@ class EPCManagement(): nbContainers += 1 mySSH.command('docker-compose down', '\$', 60) + mySSH.command('docker volume prune --force || true', '\$', 60) mySSH.command('docker inspect --format=\'{{.State.Health.Status}}\' ' + listOfContainers, '\$', 10) noMoreContainerNb = mySSH.getBefore().count('No such object') mySSH.command('docker inspect --format=\'{{.Name}}\' prod-oai-public-net prod-oai-private-net', '\$', 10)