diff --git a/ci-scripts/Jenkinsfile-tmp-full-ran b/ci-scripts/Jenkinsfile-tmp-full-ran index f1baa5b04d..3bc1d6ad8e 100644 --- a/ci-scripts/Jenkinsfile-tmp-full-ran +++ b/ci-scripts/Jenkinsfile-tmp-full-ran @@ -216,24 +216,6 @@ pipeline { } stage('Log Collection') { parallel { - stage('Log Collection (eNB - Build)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' - sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("build.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "build.log.${env.BUILD_ID}.zip" - } - } - } - } stage('Log Collection (eNB - Run)') { steps { withCredentials([ diff --git a/ci-scripts/Jenkinsfile-tmp-multi-enb b/ci-scripts/Jenkinsfile-tmp-multi-enb index 2680fbe863..7c7397f30b 100644 --- a/ci-scripts/Jenkinsfile-tmp-multi-enb +++ b/ci-scripts/Jenkinsfile-tmp-multi-enb @@ -222,24 +222,6 @@ pipeline { } stage('Log Collection') { parallel { - stage('Log Collection (eNB - Build)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' - sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("build.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "build.log.${env.BUILD_ID}.zip" - } - } - } - } stage('Log Collection (eNB - Run)') { steps { withCredentials([ diff --git a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa index 5e60b199b4..89daaec8c1 100644 --- a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa +++ b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa @@ -210,24 +210,6 @@ pipeline { } stage('Log Collection') { parallel { - stage('Log Collection (eNB - Build)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' - sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("build.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "build.log.${env.BUILD_ID}.zip" - } - } - } - } stage('Log Collection (eNB - Run)') { steps { withCredentials([ diff --git a/ci-scripts/Jenkinsfile-tmp-ran b/ci-scripts/Jenkinsfile-tmp-ran index 03c0f9a0cb..82db6641e3 100644 --- a/ci-scripts/Jenkinsfile-tmp-ran +++ b/ci-scripts/Jenkinsfile-tmp-ran @@ -227,24 +227,6 @@ pipeline { } stage('Log Collection') { parallel { - stage('Log Collection (eNB - Build)') { - steps { - withCredentials([ - [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'] - ]) { - echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m' - sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}" - - echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m' - sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true" - } - script { - if(fileExists("build.log.${env.BUILD_ID}.zip")) { - archiveArtifacts "build.log.${env.BUILD_ID}.zip" - } - } - } - } stage('Log Collection (eNB - Run)') { steps { withCredentials([ diff --git a/ci-scripts/ci_infra.yaml b/ci-scripts/ci_infra.yaml index 804852c937..e5888ef6fe 100644 --- a/ci-scripts/ci_infra.yaml +++ b/ci-scripts/ci_infra.yaml @@ -5,10 +5,12 @@ idefix: AttachScript: sudo python3 ci_ctl_qtel.py /dev/ttyUSB2 wup DetachScript: sudo python3 ci_ctl_qtel.py /dev/ttyUSB2 detach NetworkScript: ip a show dev wwan0 + #Tracing: + # Start: 'sudo rm -f ci_qlog/*; nohup sudo -b QLog/QLog -s ci_qlog -f NR5G.cfg &' + # Stop: 'sudo killall QLog' + # Collect: 'sudo mv ci_qlog/* %%log_dir%%/' IF: wwan0 MTU: 1500 - Trace: True - LogStore: /media/usb-drive/ci_qlogs up2: Host: up2 @@ -46,7 +48,6 @@ adb_ue_1: NetworkScript: adb -s R3CM40LZPHT shell "ip address show | grep rmnet_data0" CmdPrefix: adb -s R3CM40LZPHT shell MTU: 1500 - LogStore: /media/usb-drive/ci_adb_1-logs adb_ue_2: Host: nano InitScript: /home/oaicicd/ci_ctl_adb.sh initialize 5200c00db4413517 @@ -59,7 +60,6 @@ adb_ue_2: NetworkScript: adb -s 5200c00db4413517 shell "ip address show | grep rmnet" CmdPrefix: adb -s 5200c00db4413517 shell MTU: 1500 - LogStore: /media/usb-drive/ci_adb_2-logs oc-cn5g: Host: avra @@ -150,6 +150,10 @@ amarisoft_ue: InitScript: /root/lteue-linux-2025-03-15/lteue /root/oaicicd/ran_sa_aw2s_asue_20MHz/aw2s-multi-00102-20.cfg & TermScript: /root/lteue-linux-2025-03-15/ws.js -t 10 127.0.0.1:9002 '{"message":"quit"}' || killall -KILL lteue-avx2 NetworkScript: ip netns exec ue1 ip a show dev pdn0 + #Tracing: + # Start: '' # nothing to be done + # Stop: '' # nothing to be done + # Collect: 'cp /tmp/ue.log %%log_dir%%' amarisoft_ue_2x2: Host: amariue InitScript: /root/lteue-linux-2025-03-15/lteue /root/oaicicd/ran_sa_aw2s_asue_20MHz_2x2/aw2s-multi-00102-2x2-v2.cfg & diff --git a/ci-scripts/cls_analysis.py b/ci-scripts/cls_analysis.py index ac591b52fb..236a4743a1 100644 --- a/ci-scripts/cls_analysis.py +++ b/ci-scripts/cls_analysis.py @@ -84,7 +84,7 @@ class Analysis(): return success,msg - def analyze_oc_physim(result_junit, details_json): + def analyze_oc_physim(result_junit, details_json, logPath): try: tree = ET.parse(result_junit) root = tree.getroot() @@ -124,7 +124,7 @@ class Analysis(): output = test.findtext("system-out") output_check = "exceeds the threshold" not in output # collect logs - log_dir = f'../cmake_targets/log/{test_exec}' + log_dir = f'{logPath}/{test_exec}' os.makedirs(log_dir, exist_ok=True) with open(f'{log_dir}/{test_name}.log', 'w') as f: f.write(output) diff --git a/ci-scripts/cls_ci_helper.py b/ci-scripts/cls_ci_helper.py new file mode 100644 index 0000000000..eb47a32892 --- /dev/null +++ b/ci-scripts/cls_ci_helper.py @@ -0,0 +1,44 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ + +import os +from typing import NamedTuple +import logging + +class TestCaseCtx(NamedTuple): + count: int + test_id: int + logPath: str + + def Default(logPath): + return TestCaseCtx(123, 112233, logPath) + def baseFilename(self): + # typically, the test ID is of form 001234 (6 digits) + return f"{self.logPath}/{self.count}-{self.test_id:06d}" + +def archiveArtifact(cmd, ctx, remote_path): + base = os.path.basename(remote_path) + local = f"{ctx.baseFilename()}-{base}" + logging.info(f"Archive artifact '{local}'") + success = cmd.copyin(remote_path, local) + if success: + cmd.run(f'rm {remote_path}', silent=True) + return local if success else None diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 7e88c6b07b..185e90a13e 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -39,6 +39,7 @@ import constants as CONST import helpreadme as HELP import cls_containerize import cls_cmd +from cls_ci_helper import archiveArtifact IMAGE_REGISTRY_SERVICE_NAME = "image-registry.openshift-image-registry.svc" NAMESPACE = "oaicicd-ran" @@ -207,7 +208,12 @@ class Cluster: HTML.CreateHtmlTestRowQueue(param, 'KO', [msg]) return success - def BuildClusterImage(self, HTML): + def _retrieveOCLog(self, ctx, job, lSourcePath, image): + fn = f'{lSourcePath}/cmake_targets/log/{image}.log' + self.cmd.run(f'oc logs {job} &> {fn}') + return (image, archiveArtifact(self.cmd, ctx, fn)) + + def BuildClusterImage(self, ctx, HTML): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': HELP.GenericHelp(CONST.Version) raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {ranBranch} ranCommitID {self.ranCommitID}') @@ -280,22 +286,27 @@ class Cluster: self._recreate_entitlements() status = True # flag to abandon compiling if any image fails - attemptedImages = [] + log_files = [] + build_metrics = f"{lSourcePath}/cmake_targets/log/build-metrics.log" if forceBaseImageBuild: self._recreate_is_tag('ran-base', baseTag, 'openshift/ran-base-is.yaml') self._recreate_bc('ran-base', baseTag, 'openshift/ran-base-bc.yaml') ranbase_job = self._start_build('ran-base') - attemptedImages += ['ran-base'] status = ranbase_job is not None and self._wait_build_end([ranbase_job], 1000) if not status: logging.error('failure during build of ran-base') - self.cmd.run(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log') # cannot use cmd.run because of redirect + log_files.append(self._retrieveOCLog(ctx, ranbase_job, lSourcePath, 'ran-base')) + # recover logs by mounting image self._retag_image_statement('ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml') pod = self._deploy_pod('openshift/ran-base-log-retrieval.yaml') if pod is not None: - self.cmd.run(f'mkdir -p cmake_targets/log/ran-base') - self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base') + logdir = f'{lSourcePath}/cmake_targets/log/ran-base' + self.cmd.run(f'mkdir -p {logdir}') + self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ {logdir}') self._undeploy_pod('openshift/ran-base-log-retrieval.yaml') + ret = self.cmd.run(f'ls {logdir}') + for f in ret.stdout.splitlines(): + archiveArtifact(self.cmd, ctx, f"{logdir}/{f}") else: status = False @@ -304,27 +315,24 @@ class Cluster: self._recreate_bc('oai-physim', imageTag, 'openshift/oai-physim-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel9') physim_job = self._start_build('oai-physim') - attemptedImages += ['oai-physim'] self._recreate_is_tag('ran-build', imageTag, 'openshift/ran-build-is.yaml') self._recreate_bc('ran-build', imageTag, 'openshift/ran-build-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel9') ranbuild_job = self._start_build('ran-build') - attemptedImages += ['ran-build'] self._recreate_is_tag('oai-clang', imageTag, 'openshift/oai-clang-is.yaml') self._recreate_bc('oai-clang', imageTag, 'openshift/oai-clang-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.clang.rhel9') clang_job = self._start_build('oai-clang') - attemptedImages += ['oai-clang'] wait = ranbuild_job is not None and physim_job is not None and clang_job is not None and self._wait_build_end([ranbuild_job, physim_job, clang_job], 1200) if not wait: logging.error('error during build of ranbuild_job or physim_job or clang_job') status = status and wait - self.cmd.run(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log') - self.cmd.run(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log') - self.cmd.run(f'oc logs {clang_job} &> cmake_targets/log/oai-clang.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, ranbuild_job, lSourcePath, 'ran-build')) + log_files.append(self._retrieveOCLog(ctx, physim_job, lSourcePath, 'oai-physim')) + log_files.append(self._retrieveOCLog(ctx, clang_job, lSourcePath, 'oai-clang')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') if status: self._recreate_is_tag('oai-enb', imageTag, 'openshift/oai-enb-is.yaml') @@ -332,72 +340,66 @@ class Cluster: self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel9') enb_job = self._start_build('oai-enb') - attemptedImages += ['oai-enb'] self._recreate_is_tag('oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml') self._recreate_bc('oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel9') gnb_job = self._start_build('oai-gnb') - attemptedImages += ['oai-gnb'] self._recreate_is_tag('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-is.yaml') self._recreate_bc('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.aw2s.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.aw2s.rhel9') gnb_aw2s_job = self._start_build('oai-gnb-aw2s') - attemptedImages += ['oai-gnb-aw2s'] wait = enb_job is not None and gnb_job is not None and gnb_aw2s_job is not None and self._wait_build_end([enb_job, gnb_job, gnb_aw2s_job], 800) if not wait: logging.error('error during build of eNB/gNB') status = status and wait # recover logs - self.cmd.run(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log') - self.cmd.run(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log') - self.cmd.run(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log') + log_files.append(self._retrieveOCLog(ctx, enb_job, lSourcePath, 'oai-enb')) + log_files.append(self._retrieveOCLog(ctx, gnb_job, lSourcePath, 'oai-gnb')) + log_files.append(self._retrieveOCLog(ctx, gnb_aw2s_job, lSourcePath, 'oai-gnb-aw2s')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') self._recreate_is_tag('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-is.yaml') self._recreate_bc('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nr-cuup.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nr-cuup.rhel9') nr_cuup_job = self._start_build('oai-nr-cuup') - attemptedImages += ['oai-nr-cuup'] self._recreate_is_tag('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml') self._recreate_bc('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel9') lteue_job = self._start_build('oai-lte-ue') - attemptedImages += ['oai-lte-ue'] self._recreate_is_tag('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml') self._recreate_bc('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel9') self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel9') nrue_job = self._start_build('oai-nr-ue') - attemptedImages += ['oai-nr-ue'] wait = nr_cuup_job is not None and lteue_job is not None and nrue_job is not None and self._wait_build_end([nr_cuup_job, lteue_job, nrue_job], 800) if not wait: logging.error('error during build of nr-cuup/lteUE/nrUE') status = status and wait # recover logs - self.cmd.run(f'oc logs {nr_cuup_job} &> cmake_targets/log/oai-nr-cuup.log') - self.cmd.run(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log') - self.cmd.run(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, nr_cuup_job, lSourcePath, 'oai-nr-cuup')) + log_files.append(self._retrieveOCLog(ctx, lteue_job, lSourcePath, 'oai-lte-ue')) + log_files.append(self._retrieveOCLog(ctx, nrue_job, lSourcePath, 'oai-nr-ue')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') if status: self._recreate_is_tag('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-is.yaml') self._recreate_bc('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-bc.yaml') self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.fhi72.rhel9') ranbuildfhi72_job = self._start_build('ran-build-fhi72') - attemptedImages += ['ran-build-fhi72'] wait = ranbuildfhi72_job is not None and self._wait_build_end([ranbuildfhi72_job], 1200) if not wait: logging.error('error during build of ranbuildfhi72_job') status = status and wait - self.cmd.run(f'oc logs {ranbuildfhi72_job} &> cmake_targets/log/ran-build-fhi72.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, ranbuildfhi72_job, lSourcePath, 'ran-build-fhi72')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') if status: self._recreate_is_tag('oai-gnb-fhi72', imageTag, 'openshift/oai-gnb-fhi72-is.yaml') @@ -405,19 +407,17 @@ class Cluster: self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.fhi72.rhel9') self._retag_image_statement('ran-build-fhi72', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build-fhi72', imageTag, 'docker/Dockerfile.gNB.fhi72.rhel9') gnb_fhi72_job = self._start_build('oai-gnb-fhi72') - attemptedImages += ['oai-gnb-fhi72'] wait = gnb_fhi72_job is not None and self._wait_build_end([gnb_fhi72_job], 600) if not wait: logging.error('error during build of gNB-fhi72') status = status and wait # recover logs - self.cmd.run(f'oc logs {gnb_fhi72_job} &> cmake_targets/log/oai-gnb-fhi72.log') - self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log') + log_files.append(self._retrieveOCLog(ctx, gnb_fhi72_job, lSourcePath, 'oai-gnb-fhi72')) + self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}') # split and analyze logs imageSize = {} - for image in attemptedImages: - self.cmd.run(f'mkdir -p cmake_targets/log/{image}') + for image, _ in log_files: tag = imageTag if image != 'ran-base' else baseTag size = self._get_image_size(image, tag) if size <= 0: @@ -428,12 +428,14 @@ class Cluster: imageSize[image] = f'{sizeMb:.1f} Mbytes (uncompressed: ~{sizeMb*2.5:.1f} Mbytes)' logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m') - grep_exp = r"\|".join(attemptedImages) - self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log'); - self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done') - - build_log_name = f'build_log_{self.testCase_id}' - cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name) + archiveArtifact(self.cmd, ctx, build_metrics) + logfile = f'{lSourcePath}/cmake_targets/log/image_registry.log' + grep_exp = r"\|".join([i for i,f in log_files]) + self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> {logfile}'); + archiveArtifact(self.cmd, ctx, logfile) + logfile = f'{lSourcePath}/cmake_targets/log/build_pod_summary.log' + self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> {logfile}; done') + archiveArtifact(self.cmd, ctx, logfile) self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done') @@ -442,7 +444,7 @@ class Cluster: self.cmd.close() # Analyze the logs - collectInfo = cls_containerize.AnalyzeBuildLogs(build_log_name, attemptedImages, status) + collectInfo = cls_containerize.AnalyzeBuildLogs(log_files, status) for img in collectInfo: for f in collectInfo[img]: status = status and collectInfo[img][f]['status'] @@ -458,9 +460,16 @@ class Cluster: HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize) + # TODO fix groovy script, remove the following. + # the groovy scripts expects all logs in + # //ci-scripts, so copy it there + with cls_cmd.LocalCmd() as c: + c.run(f'mkdir -p {os.getcwd()}/test_log_{ctx.test_id}/') + c.run(f'cp -r {ctx.logPath} {os.getcwd()}/test_log_{ctx.test_id}/') + return status - def deploy_oc_physim(self, HTML, oc_release, svr_id): + def deploy_oc_physim(self, ctx, HTML, oc_release, svr_id): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': HELP.GenericHelp(CONST.Version) raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {self.ranBranch} ranCommitID {self.ranCommitID}') @@ -470,17 +479,13 @@ class Cluster: options = f"oaicicd-core-for-ci-ran {oc_release} {image_tag} {self.eNBSourceCodePath}" ret = cls_cmd.runScript(svr_id, script, 600, options) logging.debug(f'"{script}" finished with code {ret.returncode}, output:\n{ret.stdout}') - log_dir = f'{os.getcwd()}/../cmake_targets/log' - os.makedirs(log_dir, exist_ok=True) - result_junit = f'{oc_release}-run.xml' - details_json = f'{oc_release}-tests.json' with cls_cmd.getConnection(svr_id) as ssh: - ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{details_json}', tgt=f'{log_dir}/{details_json}') - ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{result_junit}', tgt=f'{log_dir}/{result_junit}') - ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt', tgt=f'{log_dir}/physim_log.txt') - ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt', tgt=f'{log_dir}/physim_pods_summary.txt') - ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log', tgt=f'{log_dir}/LastTestsFailed.log') - test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(f'{log_dir}/{result_junit}', f'{log_dir}/{details_json}') + details_json = archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/{oc_release}-tests.json') + result_junit = archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/{oc_release}-run.xml') + archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt') + archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt') + archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log') + test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(result_junit, details_json, ctx.logPath) if test_summary: if test_status: HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index b878e82caa..34522d9eef 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -46,6 +46,7 @@ import cls_cmd import helpreadme as HELP import constants as CONST import cls_oaicitest +from cls_ci_helper import archiveArtifact #----------------------------------------------------------- # Helper functions used here and in other classes @@ -81,67 +82,26 @@ def CreateTag(ranCommitID, ranBranch, ranAllowMerge): tagToUse = f'develop-{shortCommit}' return tagToUse -def CopyLogsToExecutor(cmd, sourcePath, log_name): - cmd.cd(f'{sourcePath}/cmake_targets') - cmd.run(f'rm -f {log_name}.zip') - cmd.run(f'mkdir -p {log_name}') - cmd.run(f'mv log/* {log_name}') - cmd.run(f'zip -r -qq {log_name}.zip {log_name}') - - # copy zip to executor for analysis - if (os.path.isfile(f'./{log_name}.zip')): - os.remove(f'./{log_name}.zip') - if (os.path.isdir(f'./{log_name}')): - shutil.rmtree(f'./{log_name}') - cmd.copyin(src=f'{sourcePath}/cmake_targets/{log_name}.zip', tgt=f'{os.getcwd()}/{log_name}.zip') - cmd.run(f'rm -f {log_name}.zip') - ZipFile(f'{log_name}.zip').extractall('.') - -def AnalyzeBuildLogs(buildRoot, images, globalStatus): +def AnalyzeBuildLogs(logfiles, globalStatus): collectInfo = {} - for image in images: + for image, lf in logfiles: files = {} - file_list = [f for f in os.listdir(f'{buildRoot}/{image}') if os.path.isfile(os.path.join(f'{buildRoot}/{image}', f)) and f.endswith('.txt')] - # Analyze the "sub-logs" of every target image - for fil in file_list: - errorandwarnings = {} - warningsNo = 0 - errorsNo = 0 - with open(f'{buildRoot}/{image}/{fil}', mode='r') as inputfile: - for line in inputfile: - result = re.search(' ERROR ', str(line)) - if result is not None: - errorsNo += 1 - result = re.search(' error:', str(line)) - if result is not None: - errorsNo += 1 - result = re.search(' WARNING ', str(line)) - if result is not None: - warningsNo += 1 - result = re.search(' warning:', str(line)) - if result is not None: - warningsNo += 1 - errorandwarnings['errors'] = errorsNo - errorandwarnings['warnings'] = warningsNo - errorandwarnings['status'] = globalStatus - files[fil] = errorandwarnings - # Analyze the target image - if os.path.isfile(f'{buildRoot}/{image}.log'): - errorandwarnings = {} - committed = False - tagged = False - with open(f'{buildRoot}/{image}.log', mode='r') as inputfile: - for line in inputfile: - lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None - lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None - tagged = tagged or lineHasTag or lineHasTag2 - # the OpenShift Cluster builder prepends image registry URL - lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None - committed = committed or lineHasCommit - errorandwarnings['errors'] = 0 if committed or tagged else 1 - errorandwarnings['warnings'] = 0 - errorandwarnings['status'] = committed or tagged - files['Target Image Creation'] = errorandwarnings + errorandwarnings = {} + committed = False + tagged = False + with open(lf, mode='r') as inputfile: + for line in inputfile: + lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None + lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None + tagged = tagged or lineHasTag or lineHasTag2 + # the OpenShift Cluster builder prepends image registry URL + lineHasCommit = re.search(r'COMMIT [a-zA-Z0-9\.:/\-]*' + image, str(line)) is not None + committed = committed or lineHasCommit + errorandwarnings['errors'] = 0 if committed or tagged else 1 + errorandwarnings['warnings'] = 0 + errorandwarnings['status'] = committed or tagged + logging.info(f"Analyzing {image}, file {lf}: {errorandwarnings}") + files['Target Image Creation'] = errorandwarnings collectInfo[image] = files return collectInfo @@ -202,12 +162,10 @@ def GetServices(ssh, requested, file): else: return requested -def CopyinServiceLog(ssh, lSourcePath, yaml, svcName, wd_yaml, filename): - remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}" +def CopyinServiceLog(ssh, lSourcePath, svcName, wd_yaml, ctx): + remote_filename = f"{lSourcePath}/cmake_targets/log/{svcName}.logs" ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}') - local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}" - local_filename = f"{local_dir}/{filename}" - return ssh.copyin(remote_filename, local_filename) + return archiveArtifact(ssh, ctx, remote_filename) def GetRunningServices(ssh, file): ret = ssh.run(f'docker compose -f {file} config --services') @@ -229,14 +187,13 @@ def GetRunningServices(ssh, file): logging.info(f'stopping services: {running_services}') return running_services -def CheckLogs(self, yaml, service_name, HTML, RAN): - logPath = f'{os.getcwd()}/../cmake_targets/log/{yaml}' - filename = f'{logPath}/{service_name}-{HTML.testCase_id}.log' +def CheckLogs(self, filename, HTML, RAN): success = True - if (any(sub in service_name for sub in ['oai_ue','oai-nr-ue','lte_ue'])): + name = os.path.basename(filename) + if (any(sub in name for sub in ['oai_ue','oai-nr-ue','lte_ue'])): logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m') logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(filename, HTML, RAN) - opt = f"UE log analysis for service {service_name}" + opt = f"UE log analysis ({name})" # usage of htmlUEFailureMsg/htmleNBFailureMsg is because Analyze log files # abuse HTML to store their reports, and we here want to put custom options, # which is not possible with CreateHtmlTestRow @@ -247,13 +204,13 @@ def CheckLogs(self, yaml, service_name, HTML, RAN): else: HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmlUEFailureMsg]) HTML.htmlUEFailureMsg = "" - elif service_name == 'nv-cubb': + elif 'nv-cubb' in name: msg = 'Undeploy PNF/Nvidia CUBB' HTML.CreateHtmlTestRow(msg, 'OK', CONST.ALL_PROCESSES_OK) - elif (any(sub in service_name for sub in ['enb','rru','rcc','cu','du','gnb'])): + elif (any(sub in name for sub in ['enb','rru','rcc','cu','du','gnb'])): logging.debug(f'\u001B[1m Analyzing XnB logfile {filename}\u001B[0m') logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers) - opt = f"xNB log analysis for service {service_name}" + opt = f"xNB log analysis ({name})" if (logStatus < 0): HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmleNBFailureMsg]) success = False @@ -261,9 +218,9 @@ def CheckLogs(self, yaml, service_name, HTML, RAN): HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg]) HTML.htmleNBFailureMsg = "" else: - logging.info(f'Skipping to analyze log for service name {service_name}') - HTML.CreateHtmlTestRowQueue(f"service {service_name}", 'OK', ["no analysis function"]) - logging.debug(f"log check: service {service_name} passed analysis {success}") + logging.info(f"Skipping analysis of log '{filename}': no submatch for xNB/UE") + HTML.CreateHtmlTestRowQueue(f"file {name}", 'OK', ["no analysis function"]) + logging.debug(f"log check: file {filename} passed analysis {success}") return success #----------------------------------------------------------- @@ -299,8 +256,6 @@ class Containerize(): self.services = ['', '', ''] self.deploymentTag = '' - self.testCase_id = '' - self.cli = '' self.cliBuildOptions = '' self.dockerfileprefix = '' @@ -330,11 +285,12 @@ class Containerize(): raise ValueError(f'Insufficient Parameter: IP/node {ip}, path {path}') return (ip, path) - def BuildImage(self, HTML): + def BuildImage(self, ctx, HTML): svr = self.eNB_serverId[self.eNB_instance] lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug('Building on server: ' + lIpAddr) cmd = cls_cmd.getConnection(lIpAddr) + log_files = [] # Checking the hostname to get adapted on cli and dockerfileprefixes cmd.run('hostnamectl') @@ -390,7 +346,6 @@ class Containerize(): imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', '')) imageNames.append(('oai-gnb-aerial', 'gNB.aerial', 'oai-gnb-aerial', '')) - self.testCase_id = HTML.testCase_id cmd.cd(lSourcePath) # if asterix, copy the entitlement and subscription manager configurations if self.host == 'Red Hat': @@ -420,13 +375,24 @@ class Containerize(): # Let's remove any previous run artifacts if still there cmd.run(f"{self.cli} image prune --force") for image,pattern,name,option in imageNames: - cmd.run(f"{self.cli} image rm {name}:{imageTag}") + cmd.run(f"{self.cli} image rm {name}:{imageTag}", reportNonZero=False) # Build the base image only on Push Events (not on Merge Requests) # On when the base image docker file is being modified. if forceBaseImageBuild: cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}") - cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> cmake_targets/log/ran-base.log", timeout=1600) + logfile = f'{lSourcePath}/cmake_targets/log/ran-base.docker.log' + cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> {logfile}", timeout=1600) + t = ("ran-base", archiveArtifact(cmd, ctx, logfile)) + log_files.append(t) + + # Recover build logs, for the moment only possible when build is successful + cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}") + cmd.run("mkdir -p cmake_targets/log/ran-base") + logfile = f'{lSourcePath}/cmake_targets/log/ran-base.log' + cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/all.txt {logfile}") + cmd.run(f"{self.cli} rm -f test") + archiveArtifact(cmd, ctx, logfile) # First verify if the base image was properly created. ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}") allImagesSize = {} @@ -441,7 +407,7 @@ class Containerize(): HTML.CreateHtmlTabFooter(False) return False else: - result = re.search('Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) + result = re.search(r'Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) if result is not None: size = float(result.group("size")) / 1000000 imageSizeStr = f'{size:.1f}' @@ -450,17 +416,9 @@ class Containerize(): else: logging.debug('ran-base size is unknown') - # Recover build logs, for the moment only possible when build is successful - cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}") - cmd.run("mkdir -p cmake_targets/log/ran-base") - cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base") - cmd.run(f"{self.cli} rm -f test") - # Build the target image(s) status = True - attemptedImages = ['ran-base'] for image,pattern,name,option in imageNames: - attemptedImages += [name] # the archived Dockerfiles have "ran-base:latest" as base image # we need to update them with proper tag cmd.run(f'git checkout -- docker/Dockerfile.{pattern}{self.dockerfileprefix}') @@ -474,15 +432,18 @@ class Containerize(): cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}') if image == 'oai-gnb-aerial': cmd.run('cp -f /opt/nvidia-ipc/nvipc.src.2025.05.20.tar.gz .') - ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > cmake_targets/log/{name}.log 2>&1', timeout=1200) + logfile = f'{lSourcePath}/cmake_targets/log/{name}.docker.log' + ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > {logfile} 2>&1', timeout=1200) + t = (name, archiveArtifact(cmd, ctx, logfile)) + log_files.append(t) if image == 'oai-gnb-aerial': cmd.run('rm -f nvipc.src.2025.05.20.tar.gz') - if image == 'ran-build' and ret.returncode == 0: + if image == 'ran-build' or image == 'ran-build-asan' or image == 'ran-build-fhi72': cmd.run(f"docker run --name test-log -d {name}:{imageTag} /bin/true") - cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/ cmake_targets/log/{name}/") + logfile = f'{lSourcePath}/{image}.ninja.log' + cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/all.txt {logfile}") cmd.run(f"docker rm -f test-log") - else: - cmd.run(f"mkdir -p cmake_targets/log/{name}") + archiveArtifact(cmd, ctx, logfile) # check the status of the build ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {name}:{imageTag}") if ret.returncode != 0: @@ -493,7 +454,7 @@ class Containerize(): allImagesSize[name] = 'N/A -- Build Failed' break else: - result = re.search('Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) + result = re.search(r'Size *= *(?P[0-9\-]+) *bytes', cmd.getBefore()) if result is not None: size = float(result.group("size")) / 1000000 # convert to MB imageSizeStr = f'{size:.1f}' @@ -516,13 +477,10 @@ class Containerize(): logging.debug(cmd.run("df -h").stdout) logging.debug(cmd.run("docker system df").stdout) - # create a zip with all logs - build_log_name = f'build_log_{self.testCase_id}' - CopyLogsToExecutor(cmd, lSourcePath, build_log_name) cmd.close() # Analyze the logs - collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status) + collectInfo = AnalyzeBuildLogs(log_files, status) if status: logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') @@ -536,13 +494,12 @@ class Containerize(): HTML.CreateHtmlTabFooter(False) return False - def BuildProxy(self, HTML): + def BuildProxy(self, ctx, HTML): svr = self.eNB_serverId[self.eNB_instance] lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug('Building on server: ' + lIpAddr) ssh = cls_cmd.getConnection(lIpAddr) - self.testCase_id = HTML.testCase_id oldRanCommidID = self.ranCommitID oldRanRepository = self.ranRepository oldRanAllowMerge = self.ranAllowMerge @@ -568,15 +525,10 @@ class Containerize(): if not success: raise Exception("could not clone proxy repository") - filename = f'build_log_{self.testCase_id}' - fullpath = f'{lSourcePath}/{filename}' + fullpath = f'{lSourcePath}/proxy_build.log' ssh.run(f'docker build --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file {lSourcePath}/docker/Dockerfile.ubuntu18.04 {lSourcePath} > {fullpath} 2>&1') - ssh.run(f'zip -r -qq {fullpath}.zip {fullpath}') - local_file = f"{os.getcwd()}/../cmake_targets/log/{filename}.zip" - ssh.copyin(f'{fullpath}.zip', local_file) - # don't delete such that we might recover the zips - #ssh.run(f'rm -f {fullpath}.zip') + archiveArtifact(ssh, ctx, fullpath) ssh.run('docker image prune --force') ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') @@ -616,7 +568,7 @@ class Containerize(): collectInfo = {} collectInfo['proxy'] = files ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}') - result = re.search('Size *= *(?P[0-9\-]+) *bytes', ret.stdout) + result = re.search(r'Size *= *(?P[0-9\-]+) *bytes', ret.stdout) # Cleaning any created tmp volume ssh.run('docker volume prune --force') ssh.close() @@ -638,7 +590,7 @@ class Containerize(): HTML.CreateHtmlTabFooter(False) return False - def BuildRunTests(self, HTML): + def BuildRunTests(self, ctx, HTML): svr = self.eNB_serverId[self.eNB_instance] lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug('Building on server: ' + lIpAddr) @@ -655,7 +607,6 @@ class Containerize(): if self.forcedWorkspaceCleanup: cmd.run(f'sudo -S rm -Rf {lSourcePath}') - self.testCase_id = HTML.testCase_id # check that ran-base image exists as we expect it baseImage = 'ran-base' @@ -675,10 +626,10 @@ class Containerize(): # build ran-unittests image dockerfile = "ci-scripts/docker/Dockerfile.unittest.ubuntu22" - ret = cmd.run(f'docker build --progress=plain --tag ran-unittests:{baseTag} --file {dockerfile} . &> {lSourcePath}/cmake_targets/log/unittest-build.log') + logfile = f'{lSourcePath}/cmake_targets/log/unittest-build.log' + ret = cmd.run(f'docker build --progress=plain --tag ran-unittests:{baseTag} --file {dockerfile} . &> {logfile}') + archiveArtifact(cmd, ctx, logfile) if ret.returncode != 0: - build_log_name = f'build_log_{self.testCase_id}' - CopyLogsToExecutor(cmd, lSourcePath, build_log_name) logging.error(f'Cannot build unit tests') HTML.CreateHtmlTestRow("Unit test build failed", 'KO', [dockerfile]) HTML.CreateHtmlTabFooter(False) @@ -687,10 +638,15 @@ class Containerize(): HTML.CreateHtmlTestRowQueue("Build unit tests", 'OK', [dockerfile]) # it worked, build and execute tests, and close connection - ret = cmd.run(f'docker run -a STDOUT --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --rm ran-unittests:{baseTag} ctest --no-label-summary -j$(nproc)') - cmd.run(f'docker rmi ran-unittests:{baseTag}') - build_log_name = f'build_log_{self.testCase_id}' - CopyLogsToExecutor(cmd, lSourcePath, build_log_name) + # I would like to run it with --rm and mount the ctest result directory to avoid 'docker cp' + # below, but then permissions are messed up and we can't remove the directory without sudo + # making the next pipeline fail + ret = cmd.run(f'docker run -a STDOUT --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --name ran-unittests ran-unittests:{baseTag} ctest --no-label-summary -j$(nproc)') + cmd.run('docker cp ran-unittests:/oai-ran/build/Testing/Temporary/LastTest.log .') + archiveArtifact(cmd, ctx, f'{lSourcePath}/LastTest.log') + cmd.run('docker cp ran-unittests:/oai-ran/build/Testing/Temporary/LastTestsFailed.log .') + archiveArtifact(cmd, ctx, f'{lSourcePath}/LastTestsFailed.log') + cmd.run('docker rm ran-unittests') cmd.close() if ret.returncode == 0: @@ -823,18 +779,14 @@ class Containerize(): HTML.CreateHtmlTestRowQueue('N/A', 'KO', ["cannot create workspace"]) return success - def DeployObject(self, HTML): + def DeployObject(self, ctx, HTML): svr = self.eNB_serverId[self.eNB_instance] num_attempts = self.num_attempts lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug(f'Deploying OAI Object on server: {lIpAddr}') yaml = self.yamlPath[self.eNB_instance].strip('/') - # creating the log folder by default - local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}" - os.system(f'mkdir -p {local_dir}') wd = f'{lSourcePath}/{yaml}' wd_yaml = f'{wd}/docker-compose.y*ml' - yaml_dir = yaml.split('/')[-1] with cls_cmd.getConnection(lIpAddr) as ssh: services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml) if services == [] or services == ' ' or services == None: @@ -861,7 +813,7 @@ class Containerize(): logging.warning(warning_msg) HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [warning_msg]) for svc in services.split(): - CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log') + CopyinServiceLog(ssh, lSourcePath, svc, wd_yaml, ctx) ssh.run(f'docker compose -f {wd_yaml} down -- {services}') imagesInfo = info.stdout.splitlines()[1:] logging.debug(f'{info.stdout.splitlines()[1:]}') @@ -871,13 +823,12 @@ class Containerize(): HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)]) return deployed - def UndeployObject(self, HTML, RAN): + def UndeployObject(self, ctx, HTML, RAN): svr = self.eNB_serverId[self.eNB_instance] lIpAddr, lSourcePath = self.GetCredentials(svr) logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m') yaml = self.yamlPath[self.eNB_instance].strip('/') wd = f'{lSourcePath}/{yaml}' - yaml_dir = yaml.split('/')[-1] with cls_cmd.getConnection(lIpAddr) as ssh: ExistEnvFilePrint(ssh, wd) services = GetRunningServices(ssh, f"{wd}/docker-compose.y*ml") @@ -885,7 +836,7 @@ class Containerize(): if services is not None: all_serv = " ".join([s for s, _ in services]) ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}') - copyin_res = all(CopyinServiceLog(ssh, lSourcePath, yaml_dir, s, f"{wd}/docker-compose.y*ml", f'{s}-{HTML.testCase_id}.log') for s, c in services) + copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, f"{wd}/docker-compose.y*ml", ctx) for s, _ in services] else: logging.warning('could not identify services to stop => no log file') ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v') @@ -894,7 +845,7 @@ class Containerize(): HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)']) return False else: - log_results = [CheckLogs(self, yaml_dir, s, HTML, RAN) for s, _ in services] + log_results = [CheckLogs(self, f, HTML, RAN) for f in copyin_res] success = all(log_results) if success: logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') diff --git a/ci-scripts/cls_corenetwork.py b/ci-scripts/cls_corenetwork.py index 579bad585b..ad83eae622 100644 --- a/ci-scripts/cls_corenetwork.py +++ b/ci-scripts/cls_corenetwork.py @@ -29,6 +29,7 @@ import yaml import re import cls_cmd +from cls_ci_helper import archiveArtifact def listify(s): if s is None: @@ -114,8 +115,7 @@ class CoreNetwork: logging.info(f'deployed core network {self}, pingable IP address {ip}') return True, output - def _collect_logs(self, log_dir): - logging.info(f'collecting logs into (local) {log_dir}') + def _collect_logs(self, ctx): remote_dir = "/tmp/cn-undeploy-logs" with cls_cmd.getConnection(self._host) as c: # create a directory for log collection @@ -134,18 +134,17 @@ class CoreNetwork: logging.error("cannot enumerate log files") return [] log_files = [] - # copy them to the executor one by one, and store in log_dir + # copy them to the executor one by one for f in ret.stdout.split("\n"): - l = f.replace(remote_dir, log_dir) - c.copyin(f, l) - log_files.append(l) + name = archiveArtifact(c, ctx, f) + log_files.append(name) c.run(f'rm -rf {remote_dir}') return log_files - def undeploy(self, log_dir=None): + def undeploy(self, ctx=None): log_files = [] - if log_dir is not None: - log_files = self._collect_logs(log_dir) + if ctx is not None: + log_files = self._collect_logs(ctx) else: logging.warning("no directory for log collection specified, cannot retrieve core network logs") logging.info(f'undeploy core network {self}') diff --git a/ci-scripts/cls_module.py b/ci-scripts/cls_module.py index 3ec6ffc9e5..cfdd96477f 100644 --- a/ci-scripts/cls_module.py +++ b/ci-scripts/cls_module.py @@ -32,6 +32,7 @@ import re import yaml import cls_cmd +from cls_ci_helper import archiveArtifact class Module_UE: @@ -59,26 +60,31 @@ class Module_UE: } self.interface = m.get('IF') self.MTU = m.get('MTU') - self.trace = m.get('trace') == True - self.logStore = m.get('LogStore') self.cmd_prefix = m.get('CmdPrefix') logging.info(f'initialized UE {self} from {filename}') + t = m.get('Tracing') + self.trace = t is not None + if self.trace: + if t.get('Start') is None or t.get('Stop') is None or t.get('Collect')is None : + raise ValueError("need to have Start/Stop/Collect for tracing") + self.cmd_dict["traceStart"] = t.get('Start') + self.cmd_dict["traceStop"] = t.get('Stop') + self._logCollect = t.get('Collect') + if "%%log_dir%%" not in self._logCollect: + raise ValueError(f"(At least one) LogCollect expression for {module_name} must contain \"%%log_dir%%\"") + def __str__(self): - return f"{self.module_name}@{self.host} [IP: {self.getIP()}]" + return f"{self.module_name}@{self.host}" def __repr__(self): return self.__str__() - def _command(self, cmd, silent = False): + def _command(self, cmd, silent=False, reportNonZero=True): if cmd is None: raise Exception("no command provided") - if self.host == "" or self.host == "localhost": - c = cls_cmd.LocalCmd() - else: - c = cls_cmd.RemoteCmd(self.host) - response = c.run(cmd, silent=silent) - c.close() + with cls_cmd.getConnection(self.host) as c: + response = c.run(cmd, silent=silent, reportNonZero=reportNonZero) return response #-----------------$ @@ -86,25 +92,23 @@ class Module_UE: #-----------------$ def initialize(self): - if self.trace: - raise Exception("UE tracing not implemented yet") - self._enableTrace() # we first terminate to make sure the UE has been stopped if self.cmd_dict["detach"]: self._command(self.cmd_dict["detach"], silent=True) self._command(self.cmd_dict["terminate"], silent=True) ret = self._command(self.cmd_dict["initialize"]) logging.info(f'For command: {ret.args} | return output: {ret.stdout} | Code: {ret.returncode}') + if self.trace: + self._enableTrace() # Here each UE returns differently for the successful initialization, requires check based on UE return ret.returncode == 0 - def terminate(self): + def terminate(self, ctx=None): self._command(self.cmd_dict["terminate"]) - if self.trace: - raise Exception("UE tracing not implemented yet") + if self.trace and ctx is not None: self._disableTrace() - return self._logCollect() + return self._collectTrace(ctx) return None def attach(self, attach_tries = 4, attach_timeout = 60): @@ -113,10 +117,11 @@ class Module_UE: self._command(self.cmd_dict["attach"]) timeout = attach_timeout logging.debug("Waiting for IP address to be assigned") + ip = self.getIP(silent=False, reportNonZero=True) while timeout > 0 and not ip: - time.sleep(5) - timeout -= 5 - ip = self.getIP() + time.sleep(1) + timeout -= 1 + ip = self.getIP(silent=True, reportNonZero=False) if ip: break logging.warning(f"UE did not receive IP address after {attach_timeout} s, detaching") @@ -160,8 +165,8 @@ class Module_UE: logging.error(message) return False - def getIP(self): - output = self._command(self.cmd_dict["getNetwork"], silent=True) + def getIP(self, silent=True, reportNonZero=True): + output = self._command(self.cmd_dict["getNetwork"], silent=silent, reportNonZero=reportNonZero) result = re.search(r'inet (?P[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)', output.stdout) if result and result.group('ip'): ip = result.group('ip') @@ -191,10 +196,33 @@ class Module_UE: return self.cmd_prefix if self.cmd_prefix else "" def _enableTrace(self): - raise Exception("not implemented") + logging.info(f'UE {self}: start UE tracing') + self._command(self.cmd_dict["traceStart"]) def _disableTrace(self): - raise Exception("not implemented") + logging.info(f'UE {self}: stop UE tracing') + self._command(self.cmd_dict["traceStop"]) - def _logCollect(self): - raise Exception("not implemented") + def _collectTrace(self, ctx): + remote_dir = "/tmp/ue-trace-logs" + with cls_cmd.getConnection(self.host) as c: + # create a directory for log collection + c.run(f'rm -rf {remote_dir}') + ret = c.run(f'mkdir {remote_dir}') + if ret.returncode != 0: + logging.error("cannot create directory for log collection") + return [] + log_cmd = self._logCollect.replace('%%log_dir%%', remote_dir) + self._command(log_cmd) + # enumerate collected files + ret = c.run(f'ls {remote_dir}/*') + if ret.returncode != 0: + logging.error("cannot enumerate log files") + return [] + log_files = [] + # copy them to the executor one by one, and store in log_dir + for f in ret.stdout.split("\n"): + name = archiveArtifact(c, ctx, f) + log_files.append(name) + c.run(f'rm -rf {remote_dir}') + return log_files diff --git a/ci-scripts/cls_native.py b/ci-scripts/cls_native.py index fe508b84e3..3dde286c3a 100644 --- a/ci-scripts/cls_native.py +++ b/ci-scripts/cls_native.py @@ -28,27 +28,24 @@ import cls_cmd import cls_oai_html import cls_analysis import constants as CONST +from cls_ci_helper import archiveArtifact -LOG_PATH_PHYSIM = 'phy_sim_logs' DPDK_PATH = '/opt/dpdk-t2-22.11.0' class Native(): - def Build(test_case, HTML, host, directory, options): + def Build(ctx, test_case, HTML, host, directory, options): logging.debug(f'Building on server: {host}') HTML.testCase_id = test_case with cls_cmd.getConnection(host) as ssh: base = f"{directory}/cmake_targets" - ret = ssh.run(f"C_INCLUDE_PATH={DPDK_PATH}/include/ PKG_CONFIG_PATH={DPDK_PATH}/lib64/pkgconfig/ {base}/build_oai {options} > {base}/log/build_oai.log", timeout=900) + ret = ssh.run(f"C_INCLUDE_PATH={DPDK_PATH}/include/ PKG_CONFIG_PATH={DPDK_PATH}/lib64/pkgconfig/ {base}/build_oai {options} > {base}/build_oai.log", timeout=900) success = ret.returncode == 0 - logs = ssh.run(f"cat {base}/log/build_oai.log", silent=True) + logs = ssh.run(f"cat {base}/build_oai.log", silent=True) logging.debug(f"build finished with code {ret.returncode}, output:\n{logs.stdout}") - # create log directory, and copy build logs - target = f"{base}/build_log_{test_case}/" - ssh.run(f"mkdir -p {target}") - ssh.run(f"mv {base}/log/* {target}") + archiveArtifact(ssh, ctx, f'{base}/build_oai.log') # check if build artifacts are there # NOTE: build_oai should fail with exit code if it could not build, but it does not @@ -68,15 +65,14 @@ class Native(): HTML.CreateHtmlTestRow(options, 'KO', CONST.ALL_PROCESSES_OK) return success - def Run_Physim(HTML, host, directory, options, physim_test, threshold): + def Run_Physim(ctx, HTML, host, directory, options, physim_test, threshold): logging.debug(f'Runnin {physim_test} on server: {host}') workSpacePath = f'{directory}/cmake_targets' - os.system(f'mkdir -p ./{LOG_PATH_PHYSIM}') - runLogFile=f'physim_{HTML.testCase_id}.log' + runLogFile=f'{workSpacePath}/physim.log' with cls_cmd.getConnection(host) as cmd: - cmd.run(f'sudo LD_LIBRARY_PATH=.:{DPDK_PATH}/lib64/ {workSpacePath}/ran_build/build/{physim_test} {options} >> {workSpacePath}/{runLogFile}') - cmd.copyin(src=f'{workSpacePath}/{runLogFile}', tgt=f'{LOG_PATH_PHYSIM}/{runLogFile}') - success, msg = cls_analysis.Analysis.analyze_physim(f'{LOG_PATH_PHYSIM}/{runLogFile}', physim_test, options, threshold) + cmd.run(f'sudo LD_LIBRARY_PATH=.:{DPDK_PATH}/lib64/ {workSpacePath}/ran_build/build/{physim_test} {options} >> {runLogFile}') + physim_file = archiveArtifact(cmd, ctx, runLogFile) + success, msg = cls_analysis.Analysis.analyze_physim(physim_file, physim_test, options, threshold) if success: HTML.CreateHtmlTestRowQueue(options, 'OK', [msg]) else: diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 5cb2708efc..37b0ca76a9 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -47,6 +47,7 @@ import constants as CONST import cls_module import cls_corenetwork import cls_cmd +from cls_ci_helper import archiveArtifact #----------------------------------------------------------- # Helper functions used here and in other classes @@ -259,9 +260,7 @@ class OaiCiTest(): self.ranAllowMerge = False self.ranTargetBranch = '' - self.testCase_id = '' self.testXMLfiles = [] - self.desc = '' self.ping_args = '' self.ping_packetloss_threshold = '' self.ping_rttavg_threshold ='' @@ -355,7 +354,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue('NA', 'OK', messages) return True - def Ping_common(self, cn, ue, logPath): + def Ping_common(self, ctx, cn, ue): ping_status = 0 ueIP = ue.getIP() if not ueIP: @@ -363,13 +362,12 @@ class OaiCiTest(): svrIP = cn.getIP() if not svrIP: return (False, f"CN {cn.getName()} has no IP address") - ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log' + ping_log_file = f'/tmp/ping_{ue.getName()}.log' ping_time = re.findall(r"-c *(\d+)",str(self.ping_args)) - local_ping_log_file = f'{logPath}/{ping_log_file}' if re.search('%cn_ip%', self.ping_args) or re.search(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+', self.ping_args): raise Exception(f"ping_args should not have IP address: {self.ping_args}") interface = f'-I {ue.getIFName()}' if ue.getIFName() else '' - ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {svrIP} 2>&1 | tee /tmp/{ping_log_file}' + ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {svrIP} 2>&1 | tee {ping_log_file}' cmd = cls_cmd.getConnection(ue.getHost()) response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5) ue_header = f'UE {ue.getName()} ({ueIP})' @@ -377,8 +375,7 @@ class OaiCiTest(): message = ue_header + ': ping crashed: TIMEOUT?' return (False, message) - #copy the ping log file to have it locally for analysis (ping stats) - cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=local_ping_log_file) + local_ping_log_file = archiveArtifact(cmd, ctx, ping_log_file) cmd.close() with open(local_ping_log_file, 'r') as f: @@ -418,18 +415,13 @@ class OaiCiTest(): return (True, message) - def Ping(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"): + def Ping(self, ctx, HTML, infra_file="ci_infra.yaml"): if self.ue_ids == [] or self.svr_id == None: raise Exception("no module names in self.ue_ids or/and self.svr_id provided") - # Creating destination log folder if needed on the python executor workspace - with cls_cmd.getConnection('localhost') as local: - ymlPath = CONTAINERS.yamlPath[0].split('/') - logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}' - local.run(f'mkdir -p {logPath}', silent=True) ues = [cls_module.Module_UE(ue_id, server_name, infra_file) for ue_id, server_name in zip(self.ue_ids, self.nodes)] cn = cls_corenetwork.CoreNetwork(self.svr_id, self.svr_node, filename=infra_file) with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor: - futures = [executor.submit(self.Ping_common, cn, ue, logPath) for ue in ues] + futures = [executor.submit(self.Ping_common, ctx, cn, ue) for ue in ues] results = [f.result() for f in futures] # each result in results is a tuple, first member goes to successes, second to messages successes, messages = map(list, zip(*results)) @@ -450,7 +442,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', messages) return success - def Iperf_Module(self, cn, ue, idx, ue_num, logPath): + def Iperf_Module(self, ctx, cn, ue, idx, ue_num): ueIP = ue.getIP() if not ueIP: return (False, f"UE {ue.getName()} has no IP address") @@ -463,7 +455,7 @@ class OaiCiTest(): serverReport = "" udpIperf = re.search('-u', iperf_opt) is not None bidirIperf = re.search('--bidir', iperf_opt) is not None - client_filename = f'iperf_client_{self.testCase_id}_{ue.getName()}.log' + client_filename = f'/tmp/iperf_client_{ue.getName()}.log' if udpIperf: target_bitrate, iperf_opt = Iperf_ComputeModifiedBW(idx, ue_num, self.iperf_profile, self.iperf_args) # note: for UDP testing we don't want to use json report - reports 0 Mbps received bitrate @@ -478,14 +470,11 @@ class OaiCiTest(): port = 5002 + idx # note: some core setups start an iperf3 server automatically, indicated in ci_infra by runIperf3Server: False` t = iperf_time * 2.5 - cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False, silent=True) + cmd_ue.run(f'rm {client_filename}', reportNonZero=False, silent=True) if cn.runIperf3Server(): cmd_svr.run(f'{cn.getCmdPrefix()} timeout -vk3 {t} iperf3 -s -B {svrIP} -p {port} -1 {jsonReport} >> /dev/null &', timeout=t) - cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> /tmp/{client_filename}', timeout=t) - # note: copy iperf3 log to the current directory for log analysis and log collection - dest_filename = f'{logPath}/{client_filename}' - cmd_ue.copyin(f'/tmp/{client_filename}', dest_filename) - cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False, silent=True) + cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> {client_filename}', timeout=t) + dest_filename = archiveArtifact(cmd_ue, ctx, client_filename) if udpIperf: status, msg = Iperf_analyzeV3UDP(dest_filename, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate) elif bidirIperf: @@ -495,20 +484,15 @@ class OaiCiTest(): return (status, f'{ue_header}\n{msg}') - def Iperf(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"): + def Iperf(self, ctx, HTML, infra_file="ci_infra.yaml"): logging.debug(f'Iperf: iperf_args "{self.iperf_args}" iperf_packetloss_threshold "{self.iperf_packetloss_threshold}" iperf_bitrate_threshold "{self.iperf_bitrate_threshold}" iperf_profile "{self.iperf_profile}" iperf_options "{self.iperf_options}"') if self.ue_ids == [] or self.svr_id == None: raise Exception("no module names in self.ue_ids or/and self.svr_id provided") - # create log directory on executor node - with cls_cmd.getConnection('localhost') as local: - ymlPath = CONTAINERS.yamlPath[0].split('/') - logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}' - local.run(f'mkdir -p {logPath}', silent=True) ues = [cls_module.Module_UE(ue_id, server_name, infra_file) for ue_id, server_name in zip(self.ue_ids, self.nodes)] cn = cls_corenetwork.CoreNetwork(self.svr_id, self.svr_node, filename=infra_file) with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor: - futures = [executor.submit(self.Iperf_Module, cn, ue, i, len(ues), logPath) for i, ue in enumerate(ues)] + futures = [executor.submit(self.Iperf_Module, ctx, cn, ue, i, len(ues)) for i, ue in enumerate(ues)] results = [f.result() for f in futures] # each result in results is a tuple, first member goes to successes, second to messages successes, messages = map(list, zip(*results)) @@ -529,7 +513,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', messages) return success - def Iperf2_Unidir(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"): + def Iperf2_Unidir(self, ctx, HTML, infra_file="ci_infra.yaml"): if self.ue_ids == [] or self.svr_id == None or len(self.ue_ids) != 1: raise Exception("no module names in self.ue_ids or/and self.svr_id provided, multi UE scenario not supported") ue = cls_module.Module_UE(self.ue_ids[0].strip(),self.nodes[0].strip(), infra_file) @@ -540,23 +524,17 @@ class OaiCiTest(): svrIP = cn.getIP() if not svrIP: return False - server_filename = f'iperf_server_{self.testCase_id}_{ue.getName()}.log' - ymlPath = CONTAINERS.yamlPath[0].split('/') - logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}' + server_filename = f'/tmp/iperf_server_{ue.getName()}.log' iperf_time = Iperf_ComputeTime(self.iperf_args) target_bitrate, iperf_opt = Iperf_ComputeModifiedBW(0, 1, self.iperf_profile, self.iperf_args) t = iperf_time*2.5 - with cls_cmd.getConnection('localhost') as local: - local.run(f'mkdir -p {logPath}') with cls_cmd.getConnection(ue.getHost()) as cmd_ue, cls_cmd.getConnection(cn.getHost()) as cmd_svr: - cmd_ue.run(f'rm /tmp/{server_filename}', reportNonZero=False) - cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} iperf -B {ueIP} -s -u -i1 >> /tmp/{server_filename} &', timeout=t) + cmd_ue.run(f'rm {server_filename}', reportNonZero=False) + cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} iperf -B {ueIP} -s -u -i1 >> {server_filename} &', timeout=t) cmd_svr.run(f'{cn.getCmdPrefix()} timeout -vk3 {t} iperf -c {ueIP} -B {svrIP} {iperf_opt} -i1 >> /dev/null', timeout=t) localPath = f'{os.getcwd()}' - # note: copy iperf2 log to the directory for log collection - cmd_ue.copyin(f'/tmp/{server_filename}', f'{logPath}/{server_filename}') - cmd_ue.run(f'rm /tmp/{server_filename}', reportNonZero=False) - success, msg = Iperf_analyzeV2UDP(f'{logPath}/{server_filename}', self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate) + local = archiveArtifact(cmd_ue, ctx, server_filename) + success, msg = Iperf_analyzeV2UDP(local, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate) ue_header = f'UE {ue.getName()} ({ueIP})' logging.info(f'\u001B[1;37;45m iperf result for {ue_header}\u001B[0m') for l in msg.split('\n'): @@ -838,17 +816,17 @@ class OaiCiTest(): global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC return global_status - def TerminateUE(self, HTML): + def TerminateUE(self, ctx, HTML): ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids] with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor: - futures = [executor.submit(ue.terminate) for ue in ues] + futures = [executor.submit(ue.terminate, ctx) for ue in ues] archives = [f.result() for f in futures] archive_info = [f'Log at: {a}' if a else 'No log available' for a in archives] messages = [f"UE {ue.getName()}: {log}" for (ue, log) in zip(ues, archive_info)] HTML.CreateHtmlTestRowQueue(f'N/A', 'OK', messages) return True - def DeployCoreNetwork(cn_id, HTML): + def DeployCoreNetwork(cn_id, ctx, HTML): core_name = cn_id.strip() cn = cls_corenetwork.CoreNetwork(core_name) success, output = cn.deploy() @@ -862,41 +840,11 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(core_name, 'KO', [msg]) return success - def UndeployCoreNetwork(cn_id, HTML): - # Ping, Iperf, DeployObject put logs into a path based on YAML. We - # can't do this here (because there is no yaml), so hardcode a path for - # "cn_logs" for the moment - logPath = f'{os.getcwd()}/../cmake_targets/log/cn_logs' - with cls_cmd.getConnection('localhost') as local: - local.run(f'mkdir -p {logPath}', silent=True) + def UndeployCoreNetwork(cn_id, ctx, HTML): core_name = cn_id.strip() cn = cls_corenetwork.CoreNetwork(core_name) - logs, output = cn.undeploy(log_dir=logPath) + logs, output = cn.undeploy(ctx=ctx) logging.info(f"undeployed core network {core_name}, logs {logs}, output:\n{output}") - message = "Log files: " + ", ".join([os.path.basename(l) for l in logs]) + message = "Log files:\n" + "\n".join([os.path.basename(l) for l in logs]) HTML.CreateHtmlTestRowQueue(core_name, 'OK', [message]) return True - - def LogCollectBuild(self,RAN): - # Some pipelines are using "none" IP / Credentials - # In that case, just forget about it - if RAN.eNBIPAddress == 'none': - sys.exit(0) - - if (RAN.eNBIPAddress != '' and RAN.eNBUserName != '' and RAN.eNBPassword != ''): - IPAddress = RAN.eNBIPAddress - UserName = RAN.eNBUserName - Password = RAN.eNBPassword - SourceCodePath = RAN.eNBSourceCodePath - else: - sys.exit('Insufficient Parameter') - with cls_cmd.getConnection(IPAddress) as cmd: - d = f'{SourceCodePath}/cmake_targets' - cmd.run(f'rm -f {d}/build.log.zip') - cmd.run(f'cd {d} && zip -r build.log.zip build_log_*/*') - - def ShowTestID(self): - logging.info(f'\u001B[1m----------------------------------------\u001B[0m') - logging.info(f'\u001B[1m Test ID: {self.testCase_id} \u001B[0m') - logging.info(f'\u001B[1m {self.desc} \u001B[0m') - logging.info(f'\u001B[1m----------------------------------------\u001B[0m') diff --git a/ci-scripts/cls_static_code_analysis.py b/ci-scripts/cls_static_code_analysis.py index f2951b0528..8b0ec05352 100644 --- a/ci-scripts/cls_static_code_analysis.py +++ b/ci-scripts/cls_static_code_analysis.py @@ -43,6 +43,7 @@ from pathlib import Path import helpreadme as HELP import constants as CONST import cls_cmd +from cls_ci_helper import archiveArtifact #----------------------------------------------------------- # Class Declaration @@ -80,7 +81,7 @@ class StaticCodeAnalysis(): self.eNBPassword = '' self.eNBSourceCodePath = '' - def CppCheckAnalysis(self, HTML): + def CppCheckAnalysis(self, ctx, HTML): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -103,7 +104,7 @@ class StaticCodeAnalysis(): full_ran_repo_name = self.ranRepository + '.git' cmd.cd(lSourcePath) - logDir = f'{lSourcePath}/cmake_targets/build_log_{self.testCase_id}' + logDir = f'{lSourcePath}/cmake_targets/log' cmd.run(f'mkdir -p {logDir}') cmd.run('docker image rm oai-cppcheck:bionic oai-cppcheck:focal') cmd.run(f'sed -e "s@xenial@bionic@" {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.xenial > {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.bionic') @@ -112,31 +113,21 @@ class StaticCodeAnalysis(): cmd.run(f'docker build --tag oai-cppcheck:focal --file {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.focal . > {logDir}/cppcheck-focal.txt 2>&1') cmd.run('docker image rm oai-cppcheck:bionic oai-cppcheck:focal') - # Analyzing the logs - cmd.copyin(f'{logDir}/cppcheck-bionic.txt', 'cppcheck-bionic.txt') - cmd.copyin(f'{logDir}/cppcheck-focal.txt', 'cppcheck-focal.txt') + bionic = archiveArtifact(cmd, ctx, f'{logDir}/cppcheck-bionic.txt') + focal = archiveArtifact(cmd, ctx, f'{logDir}/cppcheck-focal.txt') cmd.close() CCR = CppCheckResults() CCR_ref = CppCheckResults() vId = 0 for variant in CCR.variants: - refAvailable = False - if self.ranAllowMerge: - refFolder = str(Path.home()) + '/cppcheck-references' - if (os.path.isfile(refFolder + '/cppcheck-'+ variant + '.txt')): - refAvailable = True - with open(refFolder + '/cppcheck-'+ variant + '.txt', 'r') as refFile: - for line in refFile: - ret = re.search(' (?P[0-9\.]+) errors', str(line)) - if ret is not None: - CCR_ref.nbErrors[vId] = int(ret.group('nb_errors')) - ret = re.search(' (?P[0-9\.]+) warnings', str(line)) - if ret is not None: - CCR_ref.nbWarnings[vId] = int(ret.group('nb_warnings')) - if (os.path.isfile('./cppcheck-'+ variant + '.txt')): + filename = ctx.baseFilename() + '-cppcheck-'+ variant + '.txt' + logging.info(f"will check file '{filename}'") + if not os.path.isfile(filename): + raise FileNotFoundError(f"{filename} is not a file") + else: xmlStart = False - with open('./cppcheck-'+ variant + '.txt', 'r') as logfile: + with open(filename, 'r') as logfile: for line in logfile: ret = re.search('cppcheck version="(?P[0-9\.]+)"', str(line)) if ret is not None: @@ -185,26 +176,6 @@ class StaticCodeAnalysis(): vMsg += ' Wrong Scanf Nb Args: ' + str(CCR.nbWrongScanfArg[vId]) + '\n' for vLine in vMsg.split('\n'): logging.debug(vLine) - if self.ranAllowMerge and refAvailable: - if CCR_ref.nbErrors[vId] == CCR.nbErrors[vId]: - logging.debug(' No change in number of errors') - elif CCR_ref.nbErrors[vId] > CCR.nbErrors[vId]: - logging.debug(' Good! Decrease in number of errors') - else: - logging.debug(' Bad! increase in number of errors') - if CCR_ref.nbWarnings[vId] == CCR.nbWarnings[vId]: - logging.debug(' No change in number of warnings') - elif CCR_ref.nbWarnings[vId] > CCR.nbWarnings[vId]: - logging.debug(' Good! Decrease in number of warnings') - else: - logging.debug(' Bad! increase in number of warnings') - # Create new reference file - if not self.ranAllowMerge: - refFolder = str(Path.home()) + '/cppcheck-references' - if not os.path.isdir(refFolder): - os.mkdir(refFolder) - with open(refFolder + '/cppcheck-'+ variant + '.txt', 'w') as refFile: - refFile.write(vMsg) vId += 1 HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) @@ -213,7 +184,7 @@ class StaticCodeAnalysis(): return True - def LicenceAndFormattingCheck(self, HTML): + def LicenceAndFormattingCheck(self, ctx, HTML): # Workspace is no longer recreated from scratch. # It implies that this method shall be called last within a build pipeline # where workspace is already created @@ -238,7 +209,7 @@ class StaticCodeAnalysis(): else: check_options += f' --build-arg TARGET_BRANCH={self.ranTargetBranch}' - logDir = f'{lSourcePath}/cmake_targets/build_log_{self.testCase_id}' + logDir = f'{lSourcePath}/cmake_targets/log/' cmd.run(f'mkdir -p {logDir}') cmd.run('docker image rm oai-formatting-check:latest') cmd.run(f'docker build --target oai-formatting-check --tag oai-formatting-check:latest {check_options} --file {lSourcePath}/ci-scripts/docker/Dockerfile.formatting.bionic {lSourcePath} > {logDir}/oai-formatting-check.txt 2>&1') @@ -247,12 +218,11 @@ class StaticCodeAnalysis(): cmd.run('docker image prune --force') cmd.run('docker volume prune --force') - # Analyzing the logs - cmd.copyin(f'{logDir}/oai-formatting-check.txt', 'oai-formatting-check.txt') + file = archiveArtifact(cmd, ctx, f'{logDir}/oai-formatting-check.txt') cmd.close() finalStatus = 0 - if (os.path.isfile('./oai-formatting-check.txt')): + if (os.path.isfile(file)): analyzed = False nbFilesNotFormatted = 0 listFiles = False @@ -263,7 +233,7 @@ class StaticCodeAnalysis(): gnuGplLicenceFiles = [] suspectLicence = False suspectLicenceFiles = [] - with open('./oai-formatting-check.txt', 'r') as logfile: + with open(file, 'r') as logfile: for line in logfile: ret = re.search('./ci-scripts/checkCodingFormattingRules.sh', str(line)) if ret is not None: diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 18956ae28f..0fdf538052 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -43,8 +43,10 @@ import cls_containerize #class Containerize for all container-based operations import cls_static_code_analysis #class for static code analysis import cls_cluster # class for building/deploying on cluster import cls_native # class for all native/source-based operations +from cls_ci_helper import TestCaseCtx import ran +import cls_cmd import cls_oai_html @@ -91,7 +93,7 @@ def AssignParams(params_dict): setattr(RAN, key, value) setattr(HTML, key, value) -def ExecuteActionWithParam(action): +def ExecuteActionWithParam(action, ctx): global RAN global HTML global CONTAINERS @@ -134,15 +136,15 @@ def ExecuteActionWithParam(action): if proxy_commit is not None: CONTAINERS.proxyCommit = proxy_commit if action == 'Build_eNB': - success = cls_native.Native.Build(HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args) + success = cls_native.Native.Build(ctx, HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args) elif action == 'Build_Image': - success = CONTAINERS.BuildImage(HTML) + success = CONTAINERS.BuildImage(ctx, HTML) elif action == 'Build_Proxy': - success = CONTAINERS.BuildProxy(HTML) + success = CONTAINERS.BuildProxy(ctx, HTML) elif action == 'Build_Cluster_Image': - success = CLUSTER.BuildClusterImage(HTML) + success = CLUSTER.BuildClusterImage(ctx, HTML) elif action == 'Build_Run_Tests': - success = CONTAINERS.BuildRunTests(HTML) + success = CONTAINERS.BuildRunTests(ctx, HTML) elif action == 'Initialize_eNB': datalog_rt_stats_file=test.findtext('rt_stats_cfg') @@ -204,7 +206,7 @@ def ExecuteActionWithParam(action): RAN.air_interface[RAN.eNB_instance] = 'lte-softmodem' else: RAN.air_interface[RAN.eNB_instance] = air_interface.lower() +'-softmodem' - success = RAN.TerminateeNB(HTML) + success = RAN.TerminateeNB(ctx, HTML) elif action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Terminate_UE' or action == 'CheckStatusUE' or action == 'DataEnable_UE' or action == 'DataDisable_UE': CiTestObj.ue_ids = test.findtext('id').split(' ') @@ -226,7 +228,7 @@ def ExecuteActionWithParam(action): elif action == 'Detach_UE': success = CiTestObj.DetachUE(HTML) elif action == 'Terminate_UE': - success = CiTestObj.TerminateUE(HTML) + success = CiTestObj.TerminateUE(ctx, HTML) elif action == 'CheckStatusUE': success = CiTestObj.CheckStatusUE(HTML) elif action == 'DataEnable_UE': @@ -253,7 +255,7 @@ def ExecuteActionWithParam(action): else: CiTestObj.nodes = [None] * len(CiTestObj.ue_ids) ping_rttavg_threshold = test.findtext('ping_rttavg_threshold') or '' - success = CiTestObj.Ping(HTML, CONTAINERS) + success = CiTestObj.Ping(ctx, HTML) elif action == 'Iperf' or action == 'Iperf2_Unidir': CiTestObj.iperf_args = test.findtext('iperf_args') @@ -284,9 +286,9 @@ def ExecuteActionWithParam(action): logging.error('test-case has wrong option ' + CiTestObj.iperf_options) CiTestObj.iperf_options = 'check' if action == 'Iperf': - success = CiTestObj.Iperf(HTML, CONTAINERS) + success = CiTestObj.Iperf(ctx, HTML) elif action == 'Iperf2_Unidir': - success = CiTestObj.Iperf2_Unidir(HTML, CONTAINERS) + success = CiTestObj.Iperf2_Unidir(ctx, HTML) elif action == 'IdleSleep': st = test.findtext('idle_sleep_time_in_sec') or "5" @@ -295,12 +297,12 @@ def ExecuteActionWithParam(action): elif action == 'Deploy_Run_PhySim': oc_release = test.findtext('oc_release') svr_id = test.findtext('svr_id') or None - success = CLUSTER.deploy_oc_physim(HTML, oc_release, svr_id) + success = CLUSTER.deploy_oc_physim(ctx, HTML, oc_release, svr_id) elif action == 'DeployCoreNetwork' or action == 'UndeployCoreNetwork': cn_id = test.findtext('cn_id') core_op = getattr(cls_oaicitest.OaiCiTest, action) - success = core_op(cn_id, HTML) + success = core_op(cn_id, ctx, HTML) elif action == 'Deploy_Object' or action == 'Undeploy_Object' or action == "Create_Workspace": eNB_instance=test.findtext('eNB_instance') @@ -328,9 +330,9 @@ def ExecuteActionWithParam(action): CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1) CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge) if action == 'Deploy_Object': - success = CONTAINERS.DeployObject(HTML) + success = CONTAINERS.DeployObject(ctx, HTML) elif action == 'Undeploy_Object': - success = CONTAINERS.UndeployObject(HTML, RAN) + success = CONTAINERS.UndeployObject(ctx, HTML, RAN) elif action == 'Create_Workspace': if force_local: # Do not create a working directory when running locally. Current repo directory will be used @@ -341,13 +343,13 @@ def ExecuteActionWithParam(action): physim_options = test.findtext('physim_run_args') physim_test = test.findtext('physim_test') physim_threshold = test.findtext('physim_time_threshold') or 'inf' - success = cls_native.Native.Run_Physim(HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold) + success = cls_native.Native.Run_Physim(ctx, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold) elif action == 'LicenceAndFormattingCheck': - success = SCA.LicenceAndFormattingCheck(HTML) + success = SCA.LicenceAndFormattingCheck(ctx, HTML) elif action == 'Cppcheck_Analysis': - success = SCA.CppCheckAnalysis(HTML) + success = SCA.CppCheckAnalysis(ctx, HTML) elif action == 'Push_Local_Registry': svr_id = test.findtext('svr_id') @@ -407,10 +409,11 @@ def test_in_list(test, list): def receive_signal(signum, frame): sys.exit(1) - - - - +def ShowTestID(ctx, desc): + logging.info(f'\u001B[1m----------------------------------------\u001B[0m') + logging.info(f'\u001B[1m Test ID: {ctx.test_id} (#{ctx.count}) \u001B[0m') + logging.info(f'\u001B[1m {desc} \u001B[0m') + logging.info(f'\u001B[1m----------------------------------------\u001B[0m') #----------------------------------------------------------- # MAIN PART @@ -466,15 +469,7 @@ if py_param_file_present == True: cwd = os.getcwd() if re.match('^TerminateeNB$', mode, re.IGNORECASE): - if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '': - HELP.GenericHelp(CONST.Version) - sys.exit('Insufficient Parameter') - if RAN.eNBIPAddress == 'none': - sys.exit(0) - RAN.eNB_instance=0 - RAN.eNB_serverId[0]='0' - RAN.eNBSourceCodePath='/tmp/' - RAN.TerminateeNB(HTML) + logging.warning("Option TerminateeNB ignored") elif re.match('^TerminateHSS$', mode, re.IGNORECASE): logging.warning("Option TerminateHSS ignored") elif re.match('^TerminateMME$', mode, re.IGNORECASE): @@ -482,12 +477,7 @@ elif re.match('^TerminateMME$', mode, re.IGNORECASE): elif re.match('^TerminateSPGW$', mode, re.IGNORECASE): logging.warning("Option TerminateSPGW ignored") elif re.match('^LogCollectBuild$', mode, re.IGNORECASE): - if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '': - logging.warning("nothing to collect (eNBIPAddress/eNBUserName/eNBPassword/eNBSourceCodePath is '')") - sys.exit(0) - if RAN.eNBIPAddress == 'none': - sys.exit(0) - CiTestObj.LogCollectBuild(RAN) + logging.warning("Option LogCollectBuild ignored") elif re.match('^LogCollecteNB$', mode, re.IGNORECASE): if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '': HELP.GenericHelp(CONST.Version) @@ -565,6 +555,16 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re else: xml_test_file = cwd + "/" + CiTestObj.testXMLfiles[0] + # directory where all log artifacts will be placed + logPath = f"{cwd}/../cmake_targets/log/{CiTestObj.testXMLfiles[0].split('/')[-1]}.d" + # we run from within ci-scripts, but the logPath is absolute, so replace + # the ci-scripts/..; if it does not exist, nothing will happen + logPath = logPath.replace(r'/ci-scripts/..', '') + logging.info(f"placing all artifacts for this run in {logPath}/") + with cls_cmd.LocalCmd() as c: + c.run(f"rm -rf {logPath}") + c.run(f"mkdir -p {logPath}") + xmlTree = ET.parse(xml_test_file) xmlRoot = xmlTree.getroot() @@ -615,29 +615,32 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re task_set_succeeded = True HTML.startTime=int(round(time.time() * 1000)) + i = 0 for test_case_id in todo_tests: for test in all_tests: id = test.get('id') if test_case_id != id: continue + i += 1 CiTestObj.testCase_id = id + ctx = TestCaseCtx(i, int(id), logPath) HTML.testCase_id=CiTestObj.testCase_id - CiTestObj.desc = test.findtext('desc') + desc = test.findtext('desc') always_exec = test.findtext('always_exec') in ['True', 'true', 'Yes', 'yes'] may_fail = test.findtext('may_fail') in ['True', 'true', 'Yes', 'yes'] - HTML.desc=CiTestObj.desc + HTML.desc = desc action = test.findtext('class') if (CheckClassValidity(xml_class_list, action, id) == False): task_set_succeeded = False continue - CiTestObj.ShowTestID() + ShowTestID(ctx, desc) if not task_set_succeeded and not always_exec: msg = f"skipping test due to prior error" logging.warning(msg) HTML.CreateHtmlTestRowQueue(msg, "SKIP", []) break try: - test_succeeded = ExecuteActionWithParam(action) + test_succeeded = ExecuteActionWithParam(action, ctx) if not test_succeeded and may_fail: logging.warning(f"test ID {test_case_id} action {action} may or may not fail, proceeding despite error") elif not test_succeeded: diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index f0f790ba2c..48039aa2e7 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -46,6 +46,7 @@ import cls_cmd import sshconnection as SSH import helpreadme as HELP import constants as CONST +from cls_ci_helper import archiveArtifact #----------------------------------------------------------- # Class Declaration @@ -124,7 +125,6 @@ class RANManagement(): self.testCase_id = HTML.testCase_id mySSH = SSH.SSHConnection() - cwd = os.getcwd() mySSH.open(lIpAddr, lUserName, lPassWord) mySSH.command('cd ' + lSourcePath, '\$', 5) @@ -249,42 +249,7 @@ class RANManagement(): logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m') return enbDidSync - def CheckeNBProcess(self, status_queue): - # At least the instance 0 SHALL be on! - if self.eNBstatuses[0] == 0: - lIpAddr = self.eNBIPAddress - lUserName = self.eNBUserName - lPassWord = self.eNBPassword - elif self.eNBstatuses[0] == 1: - lIpAddr = self.eNB1IPAddress - lUserName = self.eNB1UserName - lPassWord = self.eNB1Password - elif self.eNBstatuses[0] == 2: - lIpAddr = self.eNB2IPAddress - lUserName = self.eNB2UserName - lPassWord = self.eNB2Password - else: - lIpAddr = self.eNBIPAddress - lUserName = self.eNBUserName - lPassWord = self.eNBPassword - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) - if self.air_interface[self.eNB_instance] == '': - pattern = 'softmodem' - else: - pattern = self.air_interface[self.eNB_instance] - mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + pattern + ' | grep -v grep', '\$', 5) - result = re.search(pattern, mySSH.getBefore()) - success = result is not None - if not success: - logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m') - status_queue.put(CONST.ENB_PROCESS_FAILED) - else: - status_queue.put(CONST.ENB_PROCESS_OK) - mySSH.close() - return success - - def TerminateeNB(self, HTML): + def TerminateeNB(self, ctx, HTML): if self.eNB_serverId[self.eNB_instance] == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName @@ -356,13 +321,11 @@ class RANManagement(): fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)] self.eNBLogFiles[int(self.eNB_instance)] = '' if analyzeFile: - #*stats.log files + pickle + png - mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*stats.log', '.') - mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.pickle', '.') - mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.png', '.') - # - copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.') - if (copyin_res == -1): + with cls_cmd.getConnection(lIpAddr) as cmd: + file = archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/' + fileToAnalyze) + archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrL1_stats.log') + archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrMAC_stats.log') + if file is None: logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m') HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!' HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) @@ -375,8 +338,8 @@ class RANManagement(): mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrL1_stats.log', self.eNBSourceCodePath + '/cmake_targets/') mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrMAC_stats.log', self.eNBSourceCodePath + '/cmake_targets/') mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/') - logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze) - logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML, self.ran_checkers) + logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + file) + logStatus = self.AnalyzeLogFile_eNB(file, HTML, self.ran_checkers) if (logStatus < 0): HTML.CreateHtmlTestRow('N/A', 'KO', logStatus) #display rt stats for gNB only @@ -734,11 +697,16 @@ class RANManagement(): datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader) rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field - if os.path.isfile('./nrL1_stats.log') and os.path.isfile('./nrMAC_stats.log'): + # nrL1_stats.log/nrMAC_stats.log should be in the same directory as main log file + # currently the link is only implicit as below based on pattern matching + # I will rework this to give the file explicitly + l1_stats_fn = re.sub(r'enb_[0-9]+', 'nrL1_stats', eNBlogFile) + mac_stats_fn = re.sub(r'enb_[0-9]+', 'nrMAC_stats', eNBlogFile) + if os.path.isfile(l1_stats_fn) and os.path.isfile(mac_stats_fn): # don't use CI-nrL1_stats.log, as this will increase the processing time for # no reason, we just need the last occurence - nrL1_stats = open('./nrL1_stats.log', 'r') - nrMAC_stats = open('./nrMAC_stats.log', 'r') + nrL1_stats = open(l1_stats_fn, 'r') + nrMAC_stats = open(mac_stats_fn, 'r') for line in nrL1_stats.readlines(): for k in rt_keys: result = re.search(k, line) @@ -758,7 +726,7 @@ class RANManagement(): nrL1_stats.close() nrMAC_stats.close() else: - logging.debug("NR Stats files for RT analysis not found") + logging.debug(f"NR Stats files for RT analysis not found: {l1_stats_fn}, {mac_stats_fn}") #stdout log file and stat log files analysis completed logging.debug(' File analysis (stdout, stats) completed') diff --git a/ci-scripts/tests/config/test_module_infra.yaml b/ci-scripts/tests/config/test_module_infra.yaml new file mode 100644 index 0000000000..7bfdfc1d9d --- /dev/null +++ b/ci-scripts/tests/config/test_module_infra.yaml @@ -0,0 +1,34 @@ +test: + Host: localhost + InitScript: "" + TermScript: "" + AttachScript: "true" + DetachScript: "true" + NetworkScript: ip a show dev lo + IF: lo + MTU: 65536 + +test-fail: + Host: localhost + InitScript: "" + TermScript: "" + AttachScript: "false" + DetachScript: "true" + NetworkScript: ip a show dev TESTDEVICEDOESNOTEXIST + IF: TESTDEVICEDOESNOTEXIST + MTU: 123 + +test-trace: + Host: localhost + InitScript: "" + TermScript: "" + AttachScript: "true" + DetachScript: "true" + Tracing: + # tttf: temp-trace-test-file + Start: "rm -f /tmp/tttf; for i in {1..8}; do echo $i >> /tmp/tttf; sleep 1; done &" + Stop: "while [[ $(wc -l /tmp/tttf) < 4 ]]; do sleep 1; done" + Collect: "cp /tmp/tttf %%log_dir%%" + NetworkScript: ip a show dev lo + IF: lo + MTU: 65536 diff --git a/ci-scripts/tests/corenetwork.py b/ci-scripts/tests/corenetwork.py index 77e5c7629e..7bc1603305 100644 --- a/ci-scripts/tests/corenetwork.py +++ b/ci-scripts/tests/corenetwork.py @@ -6,14 +6,22 @@ logging.basicConfig( format="[%(asctime)s] %(levelname)8s: %(message)s" ) import os +import tempfile import unittest sys.path.append('./') # to find OAI imports below import cls_corenetwork +from cls_ci_helper import TestCaseCtx import cls_cmd class TestCoreNetwork(unittest.TestCase): + def setUp(self): + self.ctx = TestCaseCtx.Default(tempfile.mkdtemp()) + def tearDown(self): + with cls_cmd.LocalCmd() as c: + c.run(f'rm -rf {self.ctx.logPath}') + def test_simple_core(self): c = cls_corenetwork.CoreNetwork("test", filename="tests/config/test_core_infra.yaml") success, output = c.deploy() @@ -22,14 +30,14 @@ class TestCoreNetwork(unittest.TestCase): # should not have it set in config self.assertTrue(c.runIperf3Server()) self.assertEqual(c.getIP(), "127.0.0.1") - log_files, output = c.undeploy(log_dir="/tmp") + log_files, output = c.undeploy(self.ctx) self.assertEqual(output, "undeploy") with cls_cmd.LocalCmd() as cmd: # there must be one log file for this (test) core self.assertEqual(len(log_files), 1) - # test core writes to %%log_dir%%/logs, which expands to /tmp/logs + # undeploy uses archiveArtifact(), which writes to {prefix}-logs l = log_files[0] - self.assertEqual(l, "/tmp/logs") + self.assertEqual(l, f"{self.ctx.baseFilename()}-logs") ret = cmd.run(f"cat {l}") self.assertEqual(ret.returncode, 0) # command must succeed self.assertEqual(ret.stdout, "logs") # output should be "logs" @@ -39,14 +47,14 @@ class TestCoreNetwork(unittest.TestCase): success, _ = c.deploy() self.assertTrue(success) self.assertFalse(c.runIperf3Server()) - c.undeploy(log_dir="/tmp") + c.undeploy(None) def test_core_fail(self): c = cls_corenetwork.CoreNetwork("test_fail", filename="tests/config/test_core_infra.yaml") success, _ = c.deploy() self.assertFalse(success) # undeployment should still work - c.undeploy(log_dir="/tmp") + c.undeploy(None) def test_core_script(self): c = cls_corenetwork.CoreNetwork("test_script", filename="tests/config/test_core_infra.yaml") diff --git a/ci-scripts/tests/deployment.py b/ci-scripts/tests/deployment.py index a7337f6e52..d2fb0cda57 100644 --- a/ci-scripts/tests/deployment.py +++ b/ci-scripts/tests/deployment.py @@ -6,6 +6,7 @@ logging.basicConfig( format="[%(asctime)s] %(levelname)8s: %(message)s" ) import os +import tempfile os.system(f'rm -rf cmake_targets') os.system(f'mkdir -p cmake_targets/log') import unittest @@ -14,6 +15,7 @@ sys.path.append('./') # to find OAI imports below import cls_oai_html import cls_oaicitest import cls_containerize +from cls_ci_helper import TestCaseCtx import ran import cls_cmd @@ -48,12 +50,16 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.eNBPassword = None self.cont.eNBSourceCodePath = os.getcwd() self.cont.num_attempts = 3 + self.ctx = TestCaseCtx.Default(tempfile.mkdtemp()) + def tearDown(self): + with cls_cmd.LocalCmd() as c: + c.run(f'rm -rf {self.ctx.logPath}') def test_deploy(self): self.cont.yamlPath[0] = 'tests/simple-dep/' self.cont.deploymentTag = "noble" - deploy = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -61,8 +67,8 @@ class TestDeploymentMethods(unittest.TestCase): # fails reliably old = self.cont.yamlPath self.cont.yamlPath[0] = 'tests/simple-fail/' - deploy = self.cont.DeployObject(self.html) - self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertFalse(deploy) self.cont.yamlPath = old @@ -70,8 +76,8 @@ class TestDeploymentMethods(unittest.TestCase): # fails reliably old = self.cont.yamlPath self.cont.yamlPath[0] = 'tests/simple-fail-2svc/' - deploy = self.cont.DeployObject(self.html) - self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertFalse(deploy) self.cont.yamlPath = old @@ -79,8 +85,8 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb" self.cont.deploymentTag = 'develop-12345678' - deploy = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -88,8 +94,8 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb oai-nr-ue" self.cont.deploymentTag = 'develop-12345678' - deploy = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy) self.assertTrue(undeploy) @@ -97,10 +103,10 @@ class TestDeploymentMethods(unittest.TestCase): self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora' self.cont.services[0] = "oai-gnb" self.cont.deploymentTag = 'develop-12345678' - deploy1 = self.cont.DeployObject(self.html) + deploy1 = self.cont.DeployObject(self.ctx, self.html) self.cont.services[0] = "oai-nr-ue" - deploy2 = self.cont.DeployObject(self.html) - undeploy = self.cont.UndeployObject(self.html, self.ran) + deploy2 = self.cont.DeployObject(self.ctx, self.html) + undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran) self.assertTrue(deploy1) self.assertTrue(deploy2) self.assertTrue(undeploy) diff --git a/ci-scripts/tests/module.py b/ci-scripts/tests/module.py new file mode 100644 index 0000000000..47760cfa27 --- /dev/null +++ b/ci-scripts/tests/module.py @@ -0,0 +1,64 @@ +import sys +import logging +import tempfile +logging.basicConfig( + level=logging.DEBUG, + stream=sys.stdout, + format="[%(asctime)s] %(levelname)8s: %(message)s" +) +import os + +import unittest + +sys.path.append('./') # to find OAI imports below +import cls_module +from cls_ci_helper import TestCaseCtx +import cls_cmd + +class TestModule(unittest.TestCase): + def setUp(self): + self.ctx = TestCaseCtx.Default(tempfile.mkdtemp()) + def tearDown(self): + with cls_cmd.LocalCmd() as c: + c.run(f'rm -rf {self.ctx.logPath}') + + def test_simple_module(self): + c = cls_module.Module_UE("test", filename="tests/config/test_module_infra.yaml") + self.assertFalse(c.trace) + success = c.initialize() + self.assertTrue(success) + ip = c.attach() + self.assertEqual(ip, "127.0.0.1") + self.assertTrue(c.checkMTU()) + c.detach() + logs = c.terminate() + self.assertEqual(logs, None) # no tracing + + @unittest.skip("this test takes long: it verifies the UE cannot attach") + def test_simple_fail(self): + c = cls_module.Module_UE("test-fail", filename="tests/config/test_module_infra.yaml") + success = c.initialize() + self.assertTrue(success) + ip = c.attach() + self.assertEqual(ip, None) + self.assertFalse(c.checkMTU()) + c.detach() + logs = c.terminate() + self.assertEqual(logs, None) # no tracing + + def test_simple_trace(self): + c = cls_module.Module_UE("test-trace", filename="tests/config/test_module_infra.yaml") + self.assertTrue(c.trace) + success = c.initialize() + self.assertTrue(success) + ip = c.attach() + self.assertEqual(ip, "127.0.0.1") + self.assertTrue(c.checkMTU()) + c.detach() + tmp = tempfile.mkdtemp() + log_file = c.terminate(self.ctx) + # undeploy uses archiveArtifact(), which writes to {prefix}-logs + self.assertEqual(log_file, [f"{self.ctx.baseFilename()}-tttf"]) # matches test-trace UE collection file + +if __name__ == '__main__': + unittest.main() diff --git a/ci-scripts/tests/ping-iperf.py b/ci-scripts/tests/ping-iperf.py index de68122cee..f63ce323d6 100644 --- a/ci-scripts/tests/ping-iperf.py +++ b/ci-scripts/tests/ping-iperf.py @@ -9,11 +9,13 @@ import os os.system(f'rm -rf cmake_targets') os.system(f'mkdir -p cmake_targets/log') import unittest +import tempfile sys.path.append('./') # to find OAI imports below import cls_oai_html +from cls_ci_helper import TestCaseCtx import cls_oaicitest -import cls_containerize +import cls_cmd class TestPingIperf(unittest.TestCase): def setUp(self): @@ -22,7 +24,10 @@ class TestPingIperf(unittest.TestCase): self.ci = cls_oaicitest.OaiCiTest() self.ci.ue_ids = ["test"] self.ci.nodes = ["localhost"] - self.cont = cls_containerize.Containerize() + self.ctx = TestCaseCtx.Default(tempfile.mkdtemp()) + def tearDown(self): + with cls_cmd.LocalCmd() as c: + c.run(f'rm -rf {self.ctx.logPath}') def test_ping(self): self.ci.ping_args = "-c3" @@ -30,7 +35,7 @@ class TestPingIperf(unittest.TestCase): self.ci.svr_id = "test" infra_file = "tests/config/infra_ping_iperf.yaml" # TODO Should need nothing but options and UE(s) to use - success = self.ci.Ping(self.html, self.cont, infra_file=infra_file) + success = self.ci.Ping(self.ctx, self.html, infra_file=infra_file) self.assertTrue(success) def test_iperf(self): @@ -45,7 +50,7 @@ class TestPingIperf(unittest.TestCase): self.ci.iperf_profile = "balanced" infra_file = "tests/config/infra_ping_iperf.yaml" # TODO Should need nothing but options and UE(s) to use - success = self.ci.Iperf(self.html, self.cont, infra_file=infra_file) + success = self.ci.Iperf(self.ctx, self.html, infra_file=infra_file) self.assertTrue(success) def test_iperf2_unidir(self): @@ -57,7 +62,7 @@ class TestPingIperf(unittest.TestCase): self.ci.iperf_profile = "balanced" infra_file = "tests/config/infra_ping_iperf.yaml" # TODO Should need nothing but options and UE(s) to use - success = self.ci.Iperf2_Unidir(self.html, self.cont, infra_file=infra_file) + success = self.ci.Iperf2_Unidir(self.ctx, self.html, infra_file=infra_file) self.assertTrue(success) def test_iperf_highrate(self): diff --git a/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml b/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml index 2cefc32722..88173b3659 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml @@ -26,7 +26,6 @@ wrench 111111 - 100001 800813 000001 000002 diff --git a/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml b/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml index 61bf4d5e48..0d2634192d 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml @@ -26,7 +26,6 @@ wrench 111111 - 100001 800813 000001 000002 diff --git a/ci-scripts/xml_files/gnb_usrp_build.xml b/ci-scripts/xml_files/gnb_usrp_build.xml index db6450b259..e10f378c6f 100644 --- a/ci-scripts/xml_files/gnb_usrp_build.xml +++ b/ci-scripts/xml_files/gnb_usrp_build.xml @@ -47,6 +47,7 @@ Custom_Command Reboot USRP + true caracal ssh root@192.168.20.2 reboot diff --git a/docker/Dockerfile.build.fhi72.native_arm.ubuntu22 b/docker/Dockerfile.build.fhi72.native_arm.ubuntu22 index 4ea1bcd7bb..b796122bf3 100644 --- a/docker/Dockerfile.build.fhi72.native_arm.ubuntu22 +++ b/docker/Dockerfile.build.fhi72.native_arm.ubuntu22 @@ -78,7 +78,7 @@ RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ ./build_oai \ - --gNB \ + --ninja --gNB \ --build-lib "telnetsrv enbscope uescope nrscope" \ -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \ --build-e2 --cmake-opt -DXAPP_MULTILANGUAGE=OFF --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \ diff --git a/docker/Dockerfile.build.fhi72.rhel9 b/docker/Dockerfile.build.fhi72.rhel9 index b8ad1560ba..472378935d 100644 --- a/docker/Dockerfile.build.fhi72.rhel9 +++ b/docker/Dockerfile.build.fhi72.rhel9 @@ -58,7 +58,7 @@ RUN /bin/sh oaienv && \ mkdir -p log && \ scl enable gcc-toolset-13 "\ ./build_oai \ - --gNB \ + --ninja --gNB \ --build-lib 'telnetsrv enbscope uescope nrscope' \ -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \ --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION" &&\ diff --git a/docker/Dockerfile.build.fhi72.ubuntu22 b/docker/Dockerfile.build.fhi72.ubuntu22 index 5da6d0b608..4674648285 100644 --- a/docker/Dockerfile.build.fhi72.ubuntu22 +++ b/docker/Dockerfile.build.fhi72.ubuntu22 @@ -68,7 +68,7 @@ RUN /bin/sh oaienv && \ cd cmake_targets && \ mkdir -p log && \ ./build_oai \ - --gNB \ + --ninja --gNB \ --build-lib "telnetsrv enbscope uescope nrscope" \ -t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \ --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \