Merge branch 'ci_improve_gitlab_post' into 'develop'

[CI] improve gitlab post

See merge request oai/openairinterface5g!1401

ci-scripts/Jenkinsfile-tmp-multi-enb-nsa

@@ -390,7 +390,7 @@ pipeline {
           steps {
             script {
               dir ('ci-scripts/ran_dashboard') {
-                sh "python3 Hdashboard.py testevent ${params.eNB_MR} ${JOB_NAME} ${env.BUILD_URL} ${env.BUILD_ID} ${StatusForDb} "   
+                sh "python3 Hdashboard.py testevent ${params.eNB_MR} "
               }
             }
           }

ci-scripts/Jenkinsfile-trig-nsa

@@ -27,6 +27,9 @@ def pythonExecutor = params.pythonExecutor
 
 def TARGET_BRANCH = "develop"
 def ALLOW_MERGE = true
+def GitPostArgs = ''
+def DataBaseHost = params.DataBaseHost
+def jobStatus
 
 pipeline {
     agent {
@@ -47,38 +50,71 @@ pipeline {
                         COMMIT_ID=sh returnStdout: true, script:  """curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests/${MR}" | jq ".sha" || true """
                         COMMIT_ID=COMMIT_ID.trim()
                         echo "Testing NSA on : ${MR} ${SRC_BRANCH} ${COMMIT_ID}"
+                        commit = COMMIT_ID.replace("\"","")
+                        GitPostArgs += MR + ' ' + commit + ' '
+                        echo GitPostArgs
+
+                        //calling LTE 2x2
+                        jobName = "RAN-LTE-2x2-Module-OAIEPC"
+                        jobStatus =  build job: jobName, wait : true, propagate : false, parameters: [
+                            string(name: 'eNB_MR', value: String.valueOf(MR)),
+                            string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)),
+                            string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)),
+                            string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)),
+                            booleanParam(name: 'eNB_mergeRequest', value: Boolean.valueOf(ALLOW_MERGE))
+                        ]
+                        jobResult = jobStatus.getResult()
+                        build_url = jobStatus.getAbsoluteUrl()
+                        build_id = jobStatus.getNumber().toString()
+                        GitPostArgs += jobName + ' ' + build_url + ' ' + build_id + ' ' + jobResult + ' '
+                        echo GitPostArgs
+
                         //calling NSA B200
-                        build job: "RAN-NSA-B200-Module-LTEBOX", wait : true, propagate : false, parameters: [
+                        jobName = "RAN-NSA-B200-Module-LTEBOX"
+                        jobStatus =  build job: jobName, wait : true, propagate : false, parameters: [
                             string(name: 'eNB_MR', value: String.valueOf(MR)),                          
                             string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)),
                             string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)),
                             string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)),
                             booleanParam(name: 'eNB_mergeRequest', value: Boolean.valueOf(ALLOW_MERGE))
                         ]
+                        jobResult = jobStatus.getResult()
+                        build_url = jobStatus.getAbsoluteUrl()
+                        build_id = jobStatus.getNumber().toString()
+                        GitPostArgs += jobName + ' ' + build_url + ' ' + build_id + ' ' + jobResult + ' '
+                        echo GitPostArgs
+
                         //calling NSA 2x2
-                        build job: "RAN-NSA-2x2-Module-OAIEPC", wait : true, propagate : false, parameters: [
+                        jobName = "RAN-NSA-2x2-Module-OAIEPC"
-                            string(name: 'eNB_MR', value: String.valueOf(MR)),
+                        jobStatus =  build job: jobName, wait : true, propagate : false, parameters: [
-                            string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)),
-                            string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)),
-                            string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)),
-                            booleanParam(name: 'eNB_mergeRequest', value: Boolean.valueOf(ALLOW_MERGE))
-                        ]
-                        //calling LTE 2x2
-                        build job: "RAN-LTE-2x2-Module-OAIEPC", wait : true, propagate : false, parameters: [
                             string(name: 'eNB_MR', value: String.valueOf(MR)),
                             string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)),
                             string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)),
                             string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)),
                             booleanParam(name: 'eNB_mergeRequest', value: Boolean.valueOf(ALLOW_MERGE))
                         ]
+                        jobResult = jobStatus.getResult()
+                        build_url = jobStatus.getAbsoluteUrl()
+                        build_id = jobStatus.getNumber().toString()
+                        GitPostArgs += jobName + ' ' + build_url + ' ' + build_id + ' ' + jobResult + ' '
+                        echo GitPostArgs
+
                         //calling SA 
-                        build job: "RAN-SA-Module-CN5G", wait : true, propagate : false, parameters: [
+                        jobName = "RAN-SA-Module-CN5G"
+                        jobStatus =  build job: jobName, wait : true, propagate : false, parameters: [
                             string(name: 'eNB_MR', value: String.valueOf(MR)),
                             string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)),
                             string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)),
                             string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)),
                             booleanParam(name: 'eNB_mergeRequest', value: Boolean.valueOf(ALLOW_MERGE))
                         ]
+                        jobResult = jobStatus.getResult()
+                        build_url = jobStatus.getAbsoluteUrl()
+                        build_id = jobStatus.getNumber().toString()
+                        GitPostArgs += jobName + ' ' + build_url + ' ' + build_id + ' ' + jobResult + ' '
+                        echo GitPostArgs
+
+
                         //calling OAIUE B200
                         //build job: "RAN-SA-OAIUE-B200-CN5G", wait : true, propagate : false, parameters: [
                         //    string(name: 'eNB_MR', value: String.valueOf(MR)),
@@ -95,19 +131,38 @@ pipeline {
                         //    string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)),
                         //    booleanParam(name: 'eNB_mergeRequest', value: Boolean.valueOf(ALLOW_MERGE))
                         //]
+
                         //calling OAIUE N310-X300
-                        build job: "RAN-SA-OAIUE-N310-X300-CN5G", wait : true, propagate : false, parameters: [
+                        jobName = "RAN-SA-OAIUE-N310-X300-CN5G"
+                        jobStatus =  build job: jobName, wait : true, propagate : false, parameters: [
                             string(name: 'eNB_MR', value: String.valueOf(MR)),
                             string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)),
                             string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)),
                             string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)),
                             booleanParam(name: 'eNB_mergeRequest', value: Boolean.valueOf(ALLOW_MERGE))
                         ]
-
+                        jobResult = jobStatus.getResult()
+                        build_url = jobStatus.getAbsoluteUrl()
+                        build_id = jobStatus.getNumber().toString()
+                        GitPostArgs += jobName + ' ' + build_url + ' ' + build_id + ' ' + jobResult + ' '
+                        echo GitPostArgs
                     }
                 }                                  
             }   
         }
+        stage ('Git Post') {
+          when {
+            expression { params.DataBaseHost != "none" }
+          }
+          agent {label DataBaseHost}
+          steps {
+            script {
+              dir ('ci-scripts/ran_dashboard') {
+                sh "python3 Hdashboard.py gitpost ${GitPostArgs}"
+              }
+            }
+          }
+        }
     }
 }
 

ci-scripts/ran_dashboard/Hdashboard.py

@@ -509,46 +509,87 @@ class Dashboard:
         s3.meta.client.copy(copy_source, 'oaitestdashboard', path+'/'+ 'test_styles.css')
 
 
-    def PostGitNote(self,mr,jobname,buildurl,buildid,status):
+    def PostGitNote(self,mr,commit,args):
-        gl = gitlab.Gitlab.from_config('OAI')
+        #current date and time to be posted with test results
-        project_id = 223
+        #now = datetime.now()
-        project = gl.projects.get(project_id)
+        #dt_string = now.strftime("%d/%m/%Y %H:%M")
-        editable_mr = project.mergerequests.get(int(mr))
+
-        mr_notes = editable_mr.notes.list()
+        if len(args)%4 != 0:
-        mr_note = editable_mr.notes.create({
+            print("Wrong Number of Arguments")
-            'body': 'Completed Test : '+jobname+', status: <b>'+status+'</b>, '+\
+            return
-            '(<a href="'+buildurl+'">'+buildid+'</a>)<br>'+\
+        else :
-            '<a href="https://oaitestdashboard.s3.eu-west-1.amazonaws.com/MR'+mr+'/index.html">Consolidated Test Results</a>'
+            n_tests=len(args)//4
-        })
+
-        editable_mr.save()
+            gl = gitlab.Gitlab.from_config('OAI')
+            project_id = 223
+            project = gl.projects.get(project_id)
+
+            #retrieve all the notes from the MR
+            editable_mr = project.mergerequests.get(int(mr))
+            mr_notes = editable_mr.notes.list(all=True)
+
+            body = '<a href="https://oaitestdashboard.s3.eu-west-1.amazonaws.com/MR'+mr+'/index.html">Consolidated Test Results</a><br>'+\
+                'Tested CommitID: ' + commit + '<br>'
+
+            for i in range(0,n_tests):
+                jobname = args[4*i]
+                buildurl = args[4*i+1]
+                buildid = args[4*i+2]
+                status = args[4*i+3]
+                body += jobname+', status is <b>'+status+'</b>, (<a href="'+buildurl+'">'+buildid+'</a>)<br>'
+
+            #create new note
+            mr_note = editable_mr.notes.create({
+                'body': body
+            })
+            editable_mr.save()
+
+
 
 
 def main():
 
-    #call from Jenkinsfile : sh "python3 Hdashboard.py testevent ${params.eNB_MR} ${JOB_NAME} ${env.BUILD_URL} ${env.BUILD_ID} ${StatusForDb} "  
+    #call from slave Jenkinsfile : sh "python3 Hdashboard.py testevent ${params.eNB_MR} "  
+    #call from master Jenkinsfile : sh "python3 Hdashboard.py gitpost ${GitPostArgs}"
+   
 
-    #individual MR test results + test dashboard, event based (end of jenkins pipeline)
     if len(sys.argv)>1:
+
+        #individual MR test results + test dashboard, event based (end of slave jenkins pipeline)
         if sys.argv[1]=="testevent" :
             mr=sys.argv[2]
-            jobname=sys.argv[3]
-            buildurl=sys.argv[4]
-            buildid=sys.argv[5]
-            status=sys.argv[6]
             htmlDash=Dashboard()
             if mr in htmlDash.mr_list:
+                #single MR test results
                 htmlDash.Build('singleMR',mr,'/tmp/MR'+mr+'_index.html') 
                 htmlDash.CopyToS3('/tmp/MR'+mr+'_index.html','oaitestdashboard','MR'+mr+'/index.html')
+                #all MR test results
                 htmlDash.Build('Tests','0000','/tmp/Tests_index.html') 
                 htmlDash.CopyToS3('/tmp/Tests_index.html','oaitestdashboard','index.html')
-                htmlDash.PostGitNote(mr,jobname,buildurl,buildid,status)
+
+        #git post with MR test results, event based (end of master jenkins pipeline)
+        elif sys.argv[1]=="gitpost":
+            mr=sys.argv[2]
+            commit=sys.argv[3]
+            args=[]
+            for i in range (4, len(sys.argv)): #jobname, url, id , result
+                args.append(sys.argv[i])
+            htmlDash=Dashboard()
+            if mr in htmlDash.mr_list:
+                htmlDash.PostGitNote(mr,commit, args)
             else:
                 print("Not a Merge Request => this build is for testing/debug purpose, no report to git")
-    #test and MR status dash boards, cron based
+        else:
+            print("Wrong argument at position 1")
+
+
+    #test and MR status dashboards, cron based
     else:
         htmlDash=Dashboard()
+        #all MR status dashboard
         htmlDash.Build('MR','0000','/tmp/MR_index.html') 
         htmlDash.CopyToS3('/tmp/MR_index.html','oairandashboard','index.html') 
+        #all MR test results
         htmlDash.Build('Tests','0000','/tmp/Tests_index.html') 
         htmlDash.CopyToS3('/tmp/Tests_index.html','oaitestdashboard','index.html')