From 55d13bd19a04591ba7bb4700d33490e03e6ef16a Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 24 Oct 2022 12:37:38 +0200 Subject: [PATCH 1/5] Log pod build statistics --- ci-scripts/cls_cluster.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 1d44e7799b..fc46e665a1 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -360,6 +360,7 @@ class Cluster: grep_exp = "\|".join(attemptedImages) mySSH.command(f'oc get images | grep -e \'{grep_exp}\' > cmake_targets/log/image_registry.log', '\$', 10); + mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc get pod $pod -o json >> cmake_targets/log/build_pod_summary.log; done', '\$', 60) build_log_name = f'build_log_{self.testCase_id}' cls_containerize.CopyLogsToExecutor(mySSH, lSourcePath, build_log_name, lIpAddr, 'oaicicd', CONST.CI_NO_PASSWORD) From a3aead9d9f7046a7d0e99c9b8850d65119b76dbe Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 24 Oct 2022 15:16:29 +0200 Subject: [PATCH 2/5] Assign ran-build and oai-physim build jobs to different nodes --- openshift/oai-physim-bc.yaml | 2 ++ openshift/ran-build-bc.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/openshift/oai-physim-bc.yaml b/openshift/oai-physim-bc.yaml index e013a20fd3..b158c569ad 100644 --- a/openshift/oai-physim-bc.yaml +++ b/openshift/oai-physim-bc.yaml @@ -26,6 +26,8 @@ metadata: name: "oai-physim" spec: runPolicy: "Serial" + nodeSelector: + kubernetes.io/hostname: dedale source: type: "Binary" strategy: diff --git a/openshift/ran-build-bc.yaml b/openshift/ran-build-bc.yaml index 84b0f9e589..1b93973dfd 100644 --- a/openshift/ran-build-bc.yaml +++ b/openshift/ran-build-bc.yaml @@ -26,6 +26,8 @@ metadata: name: "ran-build" spec: runPolicy: "Serial" + nodeSelector: + kubernetes.io/hostname: acamas source: type: "Binary" strategy: From 2f28b5795fd48bca91a592b39e0d6994205fad4f Mon Sep 17 00:00:00 2001 From: Robert Schmidt Date: Mon, 24 Oct 2022 15:18:22 +0200 Subject: [PATCH 3/5] Collect cluster pod metrics during build --- ci-scripts/cls_cluster.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index fc46e665a1..9677fe2891 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -291,6 +291,7 @@ class Cluster: status = status and wait mySSH.command(f'oc logs {ranbuild_job} > cmake_targets/log/ran-build.log', '\$', 10) mySSH.command(f'oc logs {physim_job} > cmake_targets/log/oai-physim.log', '\$', 10) + mySSH.command('oc get pods.metrics.k8s.io > cmake_targets/log/build-metrics.log', '\$', 10) if status: self._recreate_is_tag(mySSH, 'oai-enb', imageTag, 'openshift/oai-enb-is.yaml') @@ -342,6 +343,7 @@ class Cluster: # recover logs mySSH.command(f'oc logs {lteue_job} > cmake_targets/log/oai-lte-ue.log', '\$', 10) mySSH.command(f'oc logs {nrue_job} > cmake_targets/log/oai-nr-ue.log', '\$', 10) + mySSH.command('oc get pods.metrics.k8s.io >> cmake_targets/log/build-metrics.log', '\$', 10) # split and analyze logs imageSize = {} From 1ed52cb4d2fbe9c0aaa6aea4cb689c692a644441 Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Tue, 25 Oct 2022 14:09:51 +0200 Subject: [PATCH 4/5] Changing the cluster node name --- ci-scripts/cls_cluster.py | 18 +++++++++--------- openshift/oai-physim-bc.yaml | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 9677fe2891..68895a04d6 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -262,7 +262,7 @@ class Cluster: attemptedImages += ['ran-base'] status = ranbase_job is not None and self._wait_build_end(mySSH, [ranbase_job], 600) if not status: logging.error('failure during build of ran-base') - mySSH.command(f'oc logs {ranbase_job} > cmake_targets/log/ran-base.log', '\$', 10) + mySSH.command(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log', '\$', 10) # recover logs by mounting image self._retag_image_statement(mySSH, 'ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml') pod = self._deploy_pod(mySSH, 'openshift/ran-base-log-retrieval.yaml') @@ -289,8 +289,8 @@ class Cluster: wait = ranbuild_job is not None and physim_job is not None and self._wait_build_end(mySSH, [ranbuild_job, physim_job], 1200) if not wait: logging.error('error during build of ranbuild_job or physim_job') status = status and wait - mySSH.command(f'oc logs {ranbuild_job} > cmake_targets/log/ran-build.log', '\$', 10) - mySSH.command(f'oc logs {physim_job} > cmake_targets/log/oai-physim.log', '\$', 10) + mySSH.command(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log', '\$', 10) + mySSH.command(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log', '\$', 10) mySSH.command('oc get pods.metrics.k8s.io > cmake_targets/log/build-metrics.log', '\$', 10) if status: @@ -319,9 +319,9 @@ class Cluster: if not wait: logging.error('error during build of eNB/gNB') status = status and wait # recover logs - mySSH.command(f'oc logs {enb_job} > cmake_targets/log/oai-enb.log', '\$', 10) - mySSH.command(f'oc logs {gnb_job} > cmake_targets/log/oai-gnb.log', '\$', 10) - mySSH.command(f'oc logs {gnb_aw2s_job} > cmake_targets/log/oai-gnb-aw2s.log', '\$', 10) + mySSH.command(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log', '\$', 10) + mySSH.command(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log', '\$', 10) + mySSH.command(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log', '\$', 10) self._recreate_is_tag(mySSH, 'oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml') self._recreate_bc(mySSH, 'oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml') @@ -341,8 +341,8 @@ class Cluster: if not wait: logging.error('error during build of lteUE/nrUE') status = status and wait # recover logs - mySSH.command(f'oc logs {lteue_job} > cmake_targets/log/oai-lte-ue.log', '\$', 10) - mySSH.command(f'oc logs {nrue_job} > cmake_targets/log/oai-nr-ue.log', '\$', 10) + mySSH.command(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log', '\$', 10) + mySSH.command(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log', '\$', 10) mySSH.command('oc get pods.metrics.k8s.io >> cmake_targets/log/build-metrics.log', '\$', 10) # split and analyze logs @@ -361,7 +361,7 @@ class Cluster: logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m') grep_exp = "\|".join(attemptedImages) - mySSH.command(f'oc get images | grep -e \'{grep_exp}\' > cmake_targets/log/image_registry.log', '\$', 10); + mySSH.command(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log', '\$', 10); mySSH.command('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc get pod $pod -o json >> cmake_targets/log/build_pod_summary.log; done', '\$', 60) build_log_name = f'build_log_{self.testCase_id}' diff --git a/openshift/oai-physim-bc.yaml b/openshift/oai-physim-bc.yaml index b158c569ad..3aa6397811 100644 --- a/openshift/oai-physim-bc.yaml +++ b/openshift/oai-physim-bc.yaml @@ -27,7 +27,7 @@ metadata: spec: runPolicy: "Serial" nodeSelector: - kubernetes.io/hostname: dedale + kubernetes.io/hostname: theseus source: type: "Binary" strategy: From decaec0860f549cfcfe0e59d0bd9e9ac93012206 Mon Sep 17 00:00:00 2001 From: Sagar Arora Date: Wed, 26 Oct 2022 10:27:49 +0200 Subject: [PATCH 5/5] (fix): typo in collecting pod resource metrics --- ci-scripts/cls_cluster.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/cls_cluster.py b/ci-scripts/cls_cluster.py index 68895a04d6..2f158caa9b 100644 --- a/ci-scripts/cls_cluster.py +++ b/ci-scripts/cls_cluster.py @@ -291,7 +291,7 @@ class Cluster: status = status and wait mySSH.command(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log', '\$', 10) mySSH.command(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log', '\$', 10) - mySSH.command('oc get pods.metrics.k8s.io > cmake_targets/log/build-metrics.log', '\$', 10) + mySSH.command('oc get pods.metrics.k8s.io >> cmake_targets/log/build-metrics.log', '\$', 10) if status: self._recreate_is_tag(mySSH, 'oai-enb', imageTag, 'openshift/oai-enb-is.yaml')