diff --git a/ci-scripts/Jenkinsfile-physim-deploy b/ci-scripts/Jenkinsfile-physim-deploy index a6ad112b23..5eff0b34be 100644 --- a/ci-scripts/Jenkinsfile-physim-deploy +++ b/ci-scripts/Jenkinsfile-physim-deploy @@ -207,5 +207,24 @@ pipeline { } } } - } + stage ("Log Collection") { + steps { + script { + dir ('ci-scripts') { + // Zipping all archived log files + sh "zip -r -qq physim_deploytest_logs_${env.BUILD_ID}.zip physim_test_logs_*/*" + sh "rm -rf physim_test_logs_*/" + if (fileExists("physim_deploytest_logs_${env.BUILD_ID}.zip")) { + archiveArtifacts artifacts: "physim_deploytest_logs_${env.BUILD_ID}.zip" + } + if (fileExists("test_results.html")) { + sh "mv test_results.html test_results_${env.JOB_NAME}.html" + sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: ${BUILD_ID}#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results_${JOB_NAME}.html" + archiveArtifacts artifacts: "test_results_${env.JOB_NAME}.html" + } + } + } + } + } + } } \ No newline at end of file diff --git a/ci-scripts/cls_physim1.py b/ci-scripts/cls_physim1.py index 69e5f030cc..24f789e898 100644 --- a/ci-scripts/cls_physim1.py +++ b/ci-scripts/cls_physim1.py @@ -55,6 +55,9 @@ class PhySim: self.ranCommitID= "" self.ranAllowMerge= "" self.ranTargetBranch= "" + self.testResult = {} + self.testCount = [0,0,0] + self.testSummary = {} self.exitStatus=0 #-----------------$ @@ -101,7 +104,8 @@ class PhySim: if self.ranCommitID != '': mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) - mySSH.command("sudo podman image inspect --format='Size = {{.Size}} bytes' oai-physim:ci-temp", '\$', 60) + # Check if image is exist on the Red Hat server, before pushing it to OC cluster + mySSH.command("sudo podman image inspect --format='Size = {{.Size}} bytes' oai-physim:temp", '\$', 60) if mySSH.getBefore().count('no such image') != 0: logging.error('\u001B[1m No such image oai-physim\u001B[0m') sys.exit(-1) @@ -122,9 +126,8 @@ class PhySim: else: logging.debug('oai-physim size is unknown') - # logging to OC cluster + # logging to OC Cluster and then switch to corresponding project mySSH.command(f'oc login -u {ocUserName} -p {ocPassword}', '\$', 6) - print(mySSH.getBefore()) if mySSH.getBefore().count('Login successful.') == 0: logging.error('\u001B[1m OC Cluster Login Failed\u001B[0m') sys.exit(-1) @@ -136,25 +139,51 @@ class PhySim: sys.exit(-1) else: logging.debug(f'\u001B[1m Now using project {ocWorkspace}\u001B[0m') - + + # Tag the image and push to the OC cluster + mySSH.command('oc whoami -t | sudo podman login -u ' + ocUserName + ' --password-stdin https://default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/ --tls-verify=false', '\$', 6) + if mySSH.getBefore().count('Login Succeeded!') == 0: + logging.error('\u001B[1m Podman Login to OC Cluster Registry Failed\u001B[0m') + sys.exit(-1) + else: + logging.debug('\u001B[1m Podman Login to OC Cluster Registry Successfully\u001B[0m') + mySSH.command(f'sudo podman tag oai-physim:temp default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/{self.OCWorkspace}/oai-physim:temp', '\$', 6) + mySSH.command(f'sudo podman push default-route-openshift-image-registry.apps.5glab.nsa.eurecom.fr/{self.OCWorkspace}/oai-physim:temp --tls-verify=false', '\$', 6) + if mySSH.getBefore().count('Storing signatures') == 0: + logging.error('\u001B[1m Image "oai-physim" push to OC Cluster Registry Failed\u001B[0m') + sys.exit(-1) + else: + logging.debug('\u001B[1m Image "oai-physim" push to OC Cluster Registry Successfully\u001B[0m') + # Using helm charts deployment - mySSH.command('helm install physim ./charts/physims/', '\$', 6) + mySSH.command('helm install physim ./charts/physims/ | tee -a cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6) if mySSH.getBefore().count('STATUS: deployed') == 0: - logging.error('\u001B[1m Deploying PhySim Failed using helm chart\u001B[0m') + logging.error('\u001B[1m Deploying PhySim Failed using helm chart on OC Cluster\u001B[0m') + mySSH.command('helm uninstall physim >> cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6) + self.AnalyzeLogFile_phySim(HTML) sys.exit(-1) else: logging.debug('\u001B[1m Deployed PhySim Successfully using helm chart\u001B[0m') isRunning = False - while(isRunning == False): + count = 0 + while(count < 2 and isRunning == False): + time.sleep(60) mySSH.command('oc get pods -l app.kubernetes.io/instance=physim', '\$', 6) if mySSH.getBefore().count('Running') == 12: logging.debug('\u001B[1m Running the physim test Scenarios\u001B[0m') isRunning = True podNames = re.findall('oai[\S\d\w]+', mySSH.getBefore()) + count +=1 + if isRunning == False: + logging.error('\u001B[1m Some PODS Running FAILED \u001B[0m') + mySSH.command('oc get pods -l app.kubernetes.io/instance=physim > cmake_targets/log/physim_pods_summary.txt 2>&1', '\$', 6) + mySSH.command('helm uninstall physim >> cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6) + self.AnalyzeLogFile_phySim(HTML) + sys.exit(-1) # Waiting to complete the running test count = 0 isFinished = False - while(count < 25 or isFinished == False): + while(count < 25 and isFinished == False): time.sleep(60) mySSH.command('oc get pods -l app.kubernetes.io/instance=physim', '\$', 6) result = re.search('oai-nr-dlsim[\S\d\w]+', mySSH.getBefore()) @@ -170,38 +199,61 @@ class PhySim: # Getting the logs of each executables running in individual pods for podName in podNames: mySSH.command(f'oc logs {podName} >> cmake_targets/log/physim_test.txt 2>&1', '\$', 6) - mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5) - mySSH.command('mkdir -p physim_test_log_' + self.testCase_id, '\$', 5) - mySSH.command('mv log/* ' + 'physim_test_log_' + self.testCase_id, '\$', 5) + # UnDeploy the physical simulator pods - mySSH.command('helm uninstall physim', '\$', 10) + mySSH.command('helm uninstall physim | tee -a cmake_targets/log/physim_helm_summary.txt 2>&1', '\$', 6) if mySSH.getBefore().count('release "physim" uninstalled') != 0: logging.debug('\u001B[1m UnDeployed PhySim Successfully on OC Cluster\u001B[0m') else: logging.debug('\u001B[1m Failed to UnDeploy PhySim on OC Cluster\u001B[0m') + mySSH.command('oc logout', '\$', 6) mySSH.close() - mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/physim_test_log_' + self.testCase_id + '/*', '.') - testResult = {} - testCount = 0 - if (os.path.isfile('./physim_test.txt')): - with open('./physim_test.txt', 'r') as logfile: + self.AnalyzeLogFile_phySim(HTML) + + def AnalyzeLogFile_phySim(self, HTML): + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + mySSH.command('cd ' + lSourcePath, '\$', 5) + mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5) + mySSH.command('mkdir -p physim_test_log_' + self.testCase_id, '\$', 5) + mySSH.command('mv log/physim_* ' + 'physim_test_log_' + self.testCase_id, '\$', 5) + if not os.path.exists(f'./physim_test_logs_{self.testCase_id}'): + os.mkdir(f'./physim_test_logs_{self.testCase_id}') + mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/physim_test_log_' + self.testCase_id + '/*', './physim_test_logs_' + self.testCase_id) + mySSH.command('rm -rf ./physim_test_log_'+ self.testCase_id, '\$', 5) + mySSH.command('oc logout', '\$', 6) + mySSH.close() + # physim test log analysis + nextt = 0 + if (os.path.isfile(f'./physim_test_logs_{self.testCase_id}/physim_test.txt')): + with open(f'./physim_test_logs_{self.testCase_id}/physim_test.txt', 'r') as logfile: for line in logfile: - ret = re.search('execution', str(line)) - if ret is not None: - testCount += 1 + if re.search('execution 015', str(line)) or re.search('Bypassing compilation', str(line)): + nextt = 1 + elif nextt == 1: + if not re.search('Test Results', str(line)): + nextt = 0 + ret2 = re.search('T[^\n]*', str(line)) + if ret2 is not None: + ret3 = ret2.group() + if re.search('execution 015', str(line)): + self.testCount[0] += 1 testName = line.split() ret1 = re.search('Result = PASS', str(line)) if ret1 is not None: - testResult[testName[1]] = 'PASS' + self.testResult[testName[1]] = [ret3, 'PASS'] + self.testCount[1] += 1 else: - testResult[testName[1]] = 'FAIL' - + self.testResult[testName[1]] = [ret3, 'FAIL'] + self.testCount[2] += 1 + self.testSummary['Nbtests'] = self.testCount[0] + self.testSummary['Nbpass'] = self.testCount[1] + self.testSummary['Nbfail'] = self.testCount[2] HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) - HTML.CreateHtmlTestRowPhySimTestResult(testResult) + HTML.CreateHtmlTestRowPhySimTestResult(self.testSummary,self.testResult) logging.info('\u001B[1m Physical Simulator Pass\u001B[0m') - - return 0 - - - - + return 0 \ No newline at end of file diff --git a/ci-scripts/html.py b/ci-scripts/html.py index 538906d5bf..c54712239b 100644 --- a/ci-scripts/html.py +++ b/ci-scripts/html.py @@ -563,21 +563,49 @@ class HTMLManagement(): self.htmlFile.write(' \n') vId += 1 - def CreateHtmlTestRowPhySimTestResult(self, testResult): + def CreateHtmlTestRowPhySimTestResult(self, testSummary, testResult): if (self.htmlFooterCreated or (not self.htmlHeaderCreated)): return self.htmlFile = open('test_results.html', 'a') + if bool(testResult) == False and bool(testSummary) == False: + self.htmlFile.write(' \n') + self.htmlFile.write(' ----PHYSIM TESTING FAILED - Unable to recover the test logs ---- \n') + self.htmlFile.write(' \n') + else: # Tab header - self.htmlFile.write(' \n') - self.htmlFile.write(' Test Description\n') - self.htmlFile.write(' Result\n') - self.htmlFile.write(' \n') - for key, value in testResult.items(): + self.htmlFile.write(' \n') + self.htmlFile.write(' ---- PHYSIM TEST SUMMARY---- \n') + self.htmlFile.write(' \n') + self.htmlFile.write(' \n') + self.htmlFile.write(' LogFile Name\n') + self.htmlFile.write(' Nb Tests\n') + self.htmlFile.write(' Nb Failure\n') + self.htmlFile.write(' Nb Pass\n') + self.htmlFile.write(' \n') self.htmlFile.write(' \n') - self.htmlFile.write(' ' + key + ' \n') - if value == 'PASS': - self.htmlFile.write(' ' + value + '\n') + self.htmlFile.write(' physim_test.txt \n') + self.htmlFile.write(' ' + str(testSummary['Nbtests']) + ' \n') + if testSummary['Nbfail'] == 0: + self.htmlFile.write(' ' + str(testSummary['Nbfail']) + ' \n') else: - self.htmlFile.write(' ' + value + '\n') + self.htmlFile.write(' ' + str(testSummary['Nbfail']) + ' \n') + self.htmlFile.write(' ' + str(testSummary['Nbpass']) + ' \n') + self.htmlFile.write(' \n') + self.htmlFile.write(' \n') + self.htmlFile.write(' ---- PHYSIM TEST DETAIL INFO---- \n') + self.htmlFile.write(' \n') + self.htmlFile.write(' \n') + self.htmlFile.write(' Test Name\n') + self.htmlFile.write(' Test Description\n') + self.htmlFile.write(' Result\n') + self.htmlFile.write(' \n') + for key, value in testResult.items(): + self.htmlFile.write(' \n') + self.htmlFile.write(' ' + key + ' \n') + self.htmlFile.write(' ' + value[0] + '\n') + if 'PASS' in value: + self.htmlFile.write(' ' + value[1] + '\n') + else: + self.htmlFile.write(' ' + value[1] + '\n') self.htmlFile.close() \ No newline at end of file