Merge remote-tracking branch 'origin/ci_bring_test_results_to_dashboard' into integration_2021_wk46_c

ci-scripts/Jenkinsfile-git-dashboard

@@ -33,9 +33,12 @@ pipeline {
     stages {
         stage ("gDashboard") {
             steps {
-                script {     
+                script { 
-                    //retrieve MR data from gitlab and export to gSheet            
+                    dir ("ci-scripts/ran_dashboard") {    
-                    sh returnStdout: true, script: 'python3 ci-scripts/ran_dashboard.py'
+                              //retrieve MR data from gitlab / mySQL db, build HTML pages and load them to AWS S3 bucket (configured as static web page hosting)            
+                              //deprecated method : sh returnStdout: true, script: 'python3 ran_dashboard.py'
+                              sh returnStdout: true, script: 'python3 Hdashboard.py'
+                             }
                 }                                  
             }   
         }

ci-scripts/mysql4testresults/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '2'
+services:
+  mysql:
+    container_name: oaicicd_mysql
+    restart: always
+    image: mysql:latest
+    environment:
+      MYSQL_ROOT_PASSWORD: 'ucZBc2XRYdvEm59F' 
+    ports: 
+      - "3307:3306"
+    volumes:
+      - /home/oaicicd/mysql/data:/var/lib/mysql

ci-scripts/mysql4testresults/sql_connect.py

@@ -0,0 +1,33 @@
+import pymysql
+import sys
+from datetime import datetime
+
+#This is the script used to write the test results to the mysql DB
+#Called by Jenkins pipeline (jenkinsfile)
+#Must be located in /home/oaicicid/mysql on the database host
+#Usage from Jenkinsfile : 
+#python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} ''
+
+class SQLConnect:
+    def __init__(self):
+        self.connection = pymysql.connect(
+                host='172.22.0.2',
+                user='root', 
+                password = 'ucZBc2XRYdvEm59F',
+                db='oaicicd_tests',
+                port=3306
+                )
+
+    def put(self,TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS):
+        now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        cur=self.connection.cursor()
+        cur.execute ('INSERT INTO test_results (TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS,DATE) VALUES (%s,%s,%s,%s,%s,%s,%s);' , (TEST, MR, BRANCH, BUILD, BUILD_LINK, STATUS, now))
+        self.connection.commit()
+        self.connection.close()
+
+
+if __name__ == "__main__":
+    mydb=SQLConnect()
+    mydb.put(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6])
+
+

ci-scripts/ran_dashboard/Hdashboard.py

@@ -0,0 +1,401 @@
+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+# Merge Requests Dashboard for RAN on googleSheet 
+#
+#   Required Python Version
+#     Python 3.x
+#
+#---------------------------------------------------------------------
+
+#-----------------------------------------------------------
+# Import
+#-----------------------------------------------------------
+
+
+#Author Remi
+import boto3
+import shlex
+import subprocess
+import json       #json structures
+import datetime   #now() and date formating
+from datetime import datetime
+import re
+import gitlab
+import yaml
+import os
+import time
+
+
+from sqlconnect import SQLConnect
+
+#-----------------------------------------------------------
+# Class Declaration
+#-----------------------------------------------------------
+
+class Dashboard:
+    def __init__(self): 
+
+
+        #init with data sources : git, yaml config file, test results databases
+        print("Collecting Data")
+        cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """       
+        self.git = self.__getGitData(cmd) #git data from Gitlab
+        self.tests = self.__loadCfg('ran_dashboard_cfg.yaml') #tests table setup from yaml
+        self.db = self.__loadFromDB() #test results from database
+
+
+    def __loadCfg(self,yaml_file):
+        with open(yaml_file,'r') as f:
+            tests = yaml.load(f)
+        return tests
+
+    def __getGitData(self,cmd):
+        #cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """
+        process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
+        output = process.stdout.readline()
+        tmp=output.decode("utf-8") 
+        d = json.loads(tmp)
+        return d
+
+    def __loadFromDB(self):
+        mr_list=[] 
+        for x in range(len(self.git)):
+            mr_list.append(str(self.git[x]['iid']))
+        mydb=SQLConnect()
+        for MR in mr_list: 
+            mydb.get(MR)
+        mydb.close_connection()
+        return mydb.data
+
+    def Test_initHTML(self, date):
+        self.f_html.write('<!DOCTYPE html>\n')
+        self.f_html.write('<head>\n')
+        self.f_html.write('<link rel="stylesheet" href="test_styles.css">\n')
+        self.f_html.write('<title>Test Dashboard</title>\n')
+        self.f_html.write('</head>\n')
+        self.f_html.write('<br>\n')
+        self.f_html.write('<br>\n')
+        self.f_html.write('<table>\n')
+        self.f_html.write('<tr>\n')
+        self.f_html.write('<td class="Main">OAI RAN TEST Status Dashboard</td>\n')
+        self.f_html.write('</td>\n')
+        self.f_html.write('<tr>\n')
+        self.f_html.write('<td class="DashLink"> <a href="https://oairandashboard.s3.eu-west-1.amazonaws.com/index.html">Merge Requests Dashboard</a></td>\n')
+        self.f_html.write('</td>\n')
+        self.f_html.write('<tr></tr>\n')
+        self.f_html.write('<tr>\n')
+        self.f_html.write('<td class="Date">Update : '+date+'</td>\n')
+        self.f_html.write('</td>\n')
+        self.f_html.write('</tr>\n')
+        self.f_html.write('</table>\n')
+        self.f_html.write('<br>\n')
+        self.f_html.write('<br>\n')
+
+    def Test_terminateHTML(self):
+        self.f_html.write('</body>\n')
+        self.f_html.write('</html>\n')
+        self.f_html.close()
+
+
+    def MR_initHTML(self,date):
+        self.f_html.write('<!DOCTYPE html>\n')
+        self.f_html.write('<head>\n')
+        self.f_html.write('<link rel="stylesheet" href="mr_styles.css">\n')
+        self.f_html.write('<title>MR Dashboard</title>\n')
+        self.f_html.write('</head>\n')
+        self.f_html.write('<br>\n')
+        self.f_html.write('<br>\n')
+        self.f_html.write('<table>\n')
+        self.f_html.write('<tr>\n')
+        self.f_html.write('<td class="Main">OAI RAN MR Status Dashboard</td>\n')
+        self.f_html.write('</td>\n')
+        self.f_html.write('<tr>\n')
+        self.f_html.write('<td class="DashLink"> <a href="https://oaitestdashboard.s3.eu-west-1.amazonaws.com/index.html">Tests Dashboard</a></td>\n')
+        self.f_html.write('</td>\n')
+        self.f_html.write('<tr></tr>\n')
+        self.f_html.write('<tr>\n')
+        self.f_html.write('<td class="Date">Update : '+date+'</td>\n')
+        self.f_html.write('</td>\n')
+        self.f_html.write('</tr>\n')
+        self.f_html.write('</table>\n')
+        self.f_html.write('<br>\n')
+        self.f_html.write('<br>\n')
+        self.f_html.write('<table class="MR_Table">\n')
+        self.f_html.write('<tr>\n')
+        self.f_html.write('<th class="MR">MR</th>\n')
+        self.f_html.write('<th class="CREATED_AT">Created_At</th>\n')
+        self.f_html.write('<th class="AUTHOR">Author</th>\n')
+        self.f_html.write('<th class="TITLE">Title</th>\n')
+        self.f_html.write('<th class="ASSIGNEE">Assignee</th>\n')
+        self.f_html.write('<th class="REVIEWER">Reviewer</th>\n')
+        self.f_html.write('<th class="CAN_START">CAN START</th>\n')
+        self.f_html.write('<th class="IN_PROGRESS">IN PROGRESS</th>\n')
+        self.f_html.write('<th class="COMPLETED">COMPLETED</th>\n')
+        self.f_html.write('<th class="REVIEW_FORM">Review Form</th>\n')
+        self.f_html.write('<th class="OK_MERGE">OK Merge</th>\n')
+        self.f_html.write('<th class="MERGE_CONFLICTS">Merge Conflicts</th>\n')
+        self.f_html.write('</tr>\n')
+
+    def MR_terminateHTML(self):
+        self.f_html.write('</table> \n')
+        self.f_html.write('</body>\n')
+        self.f_html.write('</html>\n')
+        self.f_html.close()
+
+
+    def MR_rowHTML(self,row):
+        self.f_html.write('<tr>\n')
+        self.f_html.write('<td><a href=\"'+row[0]+'\">'+row[1]+'</a></td>\n')
+        self.f_html.write('<td>'+row[2]+'</td>\n')
+        self.f_html.write('<td>'+row[3]+'</td>\n')
+        self.f_html.write('<td class="title_cell">'+row[4]+'</td>\n')
+        self.f_html.write('<td>'+row[5]+'</td>\n')
+        self.f_html.write('<td>'+row[6]+'</td>\n')
+        if row[7]=='X':
+            self.f_html.write('<td style="background-color: orange;">'+row[7]+'</td>\n')
+        else:
+            self.f_html.write('<td></td>\n')
+        if row[8]=='X':
+            self.f_html.write('<td style="background-color: yellow;">'+row[8]+'</td>\n')
+        else:
+            self.f_html.write('<td></td>\n')
+        if row[9]=='X':
+            self.f_html.write('<td style="background-color: rgb(144, 221, 231);">'+row[9]+'</td>\n')
+        else:
+            self.f_html.write('<td></td>\n')
+        if row[10]=='X':
+            self.f_html.write('<td style="background-color: rgb(58, 236, 58);">'+row[10]+'</td>\n')
+        else:
+            self.f_html.write('<td></td>\n')
+        if row[11]=='X':
+            self.f_html.write('<td style="background-color: rgb(58, 236, 58);">'+row[11]+'</td>\n')
+        else:
+            self.f_html.write('<td></td>\n')
+        if row[12]=='YES':
+            self.f_html.write('<td style="background-color: red;">'+row[12]+'</td>\n')
+        else:
+            self.f_html.write('<td></td>\n')
+        self.f_html.write('</tr>\n')
+
+
+    def Build(self, type, htmlfilename):
+        if type=='MR':
+            self.Build_MR_Table(htmlfilename)
+        elif type=='Tests':
+            self.Build_Test_Table(htmlfilename)
+        else :
+            print("Undefined Dashboard Type, options : MR or Tests")
+
+
+    def Build_Test_Table(self,htmlfilename):
+        print("Building Tests Dashboard...")
+
+        self.f_html=open(htmlfilename,'w')
+
+        ###update date/time, format dd/mm/YY H:M:S
+        now = datetime.now()
+        dt_string = now.strftime("%d/%m/%Y %H:%M")	  
+        #HTML table header
+        self.Test_initHTML(dt_string)
+
+
+        #1 table per MR if test results exist
+        for x in range(len(self.git)):
+            mr=str(self.git[x]['iid'])
+            if 'PASS' not in self.db[mr]:
+                self.f_html.write('<h3><a href="https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/'+mr+'">'+mr+'</a>'+'   '+self.git[x]['title'] + '</h3>\n')
+                self.f_html.write('<table class="Test_Table">\n')
+                self.f_html.write('<tr>\n')
+                self.f_html.write('<th class="Test_Name">Test Name</th>\n')
+                self.f_html.write('<th class="Test_Descr">Bench</th> \n')  
+                self.f_html.write('<th class="Test_Descr">Test</th> \n')
+                self.f_html.write('<th class="Pass"># Pass</th>\n')
+                self.f_html.write('<th class="Fail"># Fail</th>\n')
+                self.f_html.write('<th class="Last_Pass">Last Pass</th>\n')
+                self.f_html.write('<th class="Last_Fail">Last Fail</th>\n')
+                self.f_html.write('</tr>\n')
+
+                #parsing the tests
+                for t in self.tests:
+
+                    row=[]
+                    short_name= t
+                    hyperlink= self.tests[t]['link']
+                    job=self.tests[t]['job']
+
+
+                    if job in self.db[mr]:
+                        if 'PASS' in self.db[mr][job]:
+                            row.append(self.db[mr][job]['PASS'])
+                        else:
+                            row.append('')
+                        if 'FAIL' in self.db[mr][job]:
+                            row.append(self.db[mr][job]['FAIL'])
+                        else:
+                            row.append('')
+                        #2 columns for last_pass and last_fail links
+                        if 'last_pass' in self.db[mr][job]:
+                            lastpasshyperlink=  self.db[mr][job]['last_pass'][1]
+                            lastpasstext= self.db[mr][job]['last_pass'][0]
+                        else:
+                            lastpasshyperlink=''
+                            lastpasstext=''
+
+                        if 'last_fail' in self.db[mr][job]:
+                            lastfailhyperlink=  self.db[mr][job]['last_fail'][1] 
+                            lastfailtext= self.db[mr][job]['last_fail'][0]
+                        else:
+                            lastfailhyperlink=''
+                            lastfailtext=''
+
+
+
+                        self.f_html.write('<tr>\n')
+                        self.f_html.write('<td><a href='+hyperlink+'>'+short_name+'</a></td>\n')
+                        self.f_html.write('<td>'+self.tests[t]['bench']+'</td>\n')
+                        self.f_html.write('<td>'+self.tests[t]['test']+'</td>\n')
+                        if row[0]!='':
+                            self.f_html.write('<td style="background-color: rgb(58, 236, 58);">'+str(row[0])+'</td>\n')
+                        else:
+                            self.f_html.write('<td></td>\n')
+                        if row[1]!='':
+                            self.f_html.write('<td style="background-color: red;">'+str(row[1])+'</td>\n')
+                        else:
+                            self.f_html.write('<td></td>\n')
+                        self.f_html.write('<td><a href='+lastpasshyperlink+'>'+lastpasstext+'</a></td>\n')
+                        self.f_html.write('<td><a href='+lastfailhyperlink+'>'+lastfailtext+'</a></td>\n')
+                        self.f_html.write('</tr>\n')
+
+                self.f_html.write('</table>\n')
+
+        #terminate HTML table and close file
+        self.Test_terminateHTML()
+
+
+    def Build_MR_Table(self,htmlfilename):
+
+        print("Building Merge Requests Dashboard...")
+
+        self.f_html=open(htmlfilename,'w')
+
+        ###update date/time, format dd/mm/YY H:M:S
+        now = datetime.now()
+        dt_string = now.strftime("%d/%m/%Y %H:%M")	  
+
+        #HTML table header
+        self.MR_initHTML(dt_string)
+
+
+        ###MR data lines
+        for x in range(len(self.git)):
+
+
+            hyperlink= 'https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/'+ str(self.git[x]['iid'])
+            text= str(self.git[x]['iid'])
+
+                      
+            date_time_str = self.git[x]['created_at']
+            date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ')
+    
+            milestone1=milestone2=milestone3=milestone4=""
+            if self.git[x]['milestone']!=None:
+                if self.git[x]['milestone']['title']=="REVIEW_CAN_START":
+                    milestone1="X"
+                elif self.git[x]['milestone']['title']=="REVIEW_IN_PROGRESS":
+                    milestone2="X"
+                elif self.git[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED":
+                    milestone3="X"
+                elif self.git[x]['milestone']['title']=="OK_TO_BE_MERGED": 
+                    milestone4="X" 
+                else:
+                    pass
+            else:
+                pass
+
+            #check if empty or not
+            if self.git[x]['assignee']!=None:
+                assignee = str(self.git[x]['assignee']['name'])
+            else:
+                assignee = ""
+ 
+            #check if empty or not       
+            if len(self.git[x]['reviewers'])!=0:
+                reviewer = str(self.git[x]['reviewers'][0]['name'])
+            else:
+                reviewer = ""
+
+            if self.git[x]['has_conflicts']==True:
+                conflicts = "YES"
+            else:
+                conflicts = ""
+
+
+            #add a column flagging that the review form is present
+            #we use gitlab API to parse the MR notes
+            gl = gitlab.Gitlab.from_config('OAI')
+            project_id = 223
+            project = gl.projects.get(project_id)
+            #get the opened MR in the project
+            mrs = project.mergerequests.list(state='opened',per_page=100)
+            review_form=''
+            for m in range (0,len(mrs)):
+                if mrs[m].iid==self.git[x]['iid']:#check the iid is the one we are on
+                    mr_notes = mrs[m].notes.list(all=True)
+                    n=0
+                    found=False
+                    while found==False and n<len(mr_notes):
+                        res=re.search('Code Review by',mr_notes[n].body)#this is the marker we are looking for in all notes
+                        if res!=None:
+                            review_form = "X"
+                            found=True
+                        n+=1
+
+            #build final row to be inserted
+            row =[hyperlink, text, str(date_time_obj.date()),str(self.git[x]['author']['name']), str(self.git[x]['title']),\
+            assignee, reviewer,\
+            milestone1,milestone2,milestone3,review_form,milestone4,conflicts]
+
+            self.MR_rowHTML(row)
+
+        #terminate HTML table and close file
+        self.MR_terminateHTML()
+
+    def CopyToS3(self,htmlfilename,bucket,key):
+        print("Uploading to S3 bucket")
+        #Creating Session With Boto3.
+        s3 = boto3.client('s3')
+
+        #Creating S3 Resource From the Session.
+        result = s3.upload_file(htmlfilename, bucket,key, ExtraArgs={'ACL':'public-read','ContentType': 'text/html'})
+
+def main():
+
+    htmlDash=Dashboard()
+    htmlDash.Build('MR','/tmp/MR_index.html') 
+    htmlDash.CopyToS3('/tmp/MR_index.html','oairandashboard','index.html')  
+    htmlDash.Build('Tests','/tmp/Tests_index.html') 
+    htmlDash.CopyToS3('/tmp/Tests_index.html','oaitestdashboard','index.html') 
+
+        
+if __name__ == "__main__":
+    # execute only if run as a script
+    main()      

ci-scripts/ran_dashboard/mr_styles.css

@@ -0,0 +1,136 @@
+body {
+    font-family: 'lato', sans-serif;
+  }
+
+
+.Main {
+    text-align:left;
+    font-size: 30px;
+    font-weight: bold;   
+}
+
+.DashLink {
+    text-align:left;
+    font-size: 16px;    
+}
+
+.DashLink:hover {
+    font-weight: bold;      
+}
+
+.Date {
+    text-align:left;
+    font-size: 16px;
+    font-weight: bold;      
+}
+
+
+a { text-decoration: none; }
+a:visited { text-decoration: none; color:blue}
+a:hover { text-decoration: none; font-weight: bold; color:blue}
+
+
+.MR_Table {
+    border-collapse: collapse;
+}
+
+.MR_Table tr {
+    font-size: 12px;
+    text-align:center;
+}
+
+
+.MR_Table tr:hover {      
+    background-color:lightgray;
+}
+
+.MR_Table td {
+    border: 1px solid black;
+    padding : 5px;
+}
+.MR_Table td:hover {
+font-weight : bold; 
+}
+
+.MR_Table th {
+    font-size: 14px;
+    text-align:center;
+    padding : 5px;
+}
+
+
+.title_cell {
+    text-align:left;   
+}
+
+
+.MR {
+    table-layout: fixed;
+    width: 50px;
+    background-color: rgb(143, 154, 216);
+}
+
+.CREATED_AT {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.AUTHOR {
+    table-layout: fixed;
+    width: 150px;
+    background-color: rgb(143, 154, 216);
+}
+
+.TITLE {
+    table-layout: fixed;
+    width: 500px;
+    background-color: rgb(143, 154, 216);
+}
+
+.ASSIGNEE {
+    table-layout: fixed;
+    width: 150px;
+    background-color: rgb(143, 154, 216);
+}
+
+.REVIEWER {
+    table-layout: fixed;
+    width: 150px;
+    background-color: rgb(143, 154, 216);
+}
+
+.CAN_START {
+    background-color: orange;
+    table-layout: fixed;
+    width: 150px;
+}
+.IN_PROGRESS {
+    background-color: yellow;
+    table-layout: fixed;
+    width: 150px;
+}
+.COMPLETED {
+    background-color: rgb(144, 221, 231);
+    table-layout: fixed;
+    width: 150px;
+}
+
+.REVIEW_FORM {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.OK_MERGE {
+    background-color: rgb(58, 236, 58);
+    table-layout: fixed;
+    width: 150px;
+}
+
+.MERGE_CONFLICTS {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+

ci-scripts/ran_dashboard/ran_dashboard.py

@@ -30,6 +30,8 @@
 # Import
 #-----------------------------------------------------------
 
+#author Remi
+
 #import google spreadsheet API
 import gspread
 from oauth2client.service_account import ServiceAccountCredentials
@@ -42,6 +44,13 @@ import datetime   #now() and date formating
 from datetime import datetime
 import re
 import gitlab
+import yaml
+import os
+import pickle
+import time
+
+
+from sqlconnect import SQLConnect
 
 #-----------------------------------------------------------
 # Class Declaration
@@ -57,48 +66,78 @@ class gDashboard:
         #worksheet
         self.sheet = self.ss.worksheet(worksheet)
         self.ss.del_worksheet(self.sheet) #start by deleting the old sheet
-        self.sheet = self.ss.add_worksheet(title=worksheet, rows="100", cols="20") #create a new one
+        self.sheet = self.ss.add_worksheet(title=worksheet, rows="100", cols="30") #create a new one
-        
+
-        self.d = {} #data dictionary
+        #init with data sources : git, yaml config file, test results databases
+        cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """       
+        self.git = self.__getGitData(cmd) #git data from Gitlab
+        self.tests = self.__loadCfg('ran_dashboard_cfg.yaml') #tests table setup from yaml
+        self.db = self.__loadFromDB() #test results from database
 
 
-    def fetchData(self,cmd):
+    def __loadCfg(self,yaml_file):
+        with open(yaml_file,'r') as f:
+            tests = yaml.load(f)
+        return tests
+
+    def __getGitData(self,cmd):
         #cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """
         process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE)
         output = process.stdout.readline()
         tmp=output.decode("utf-8") 
-        self.d = json.loads(tmp)
+        d = json.loads(tmp)
+        return d
+
+    def __loadFromDB(self):
+        mr_list=[] 
+        for x in range(len(self.git)):
+            mr_list.append(str(self.git[x]['iid']))
+        mydb=SQLConnect()
+        for MR in mr_list: 
+            mydb.get(MR)
+        mydb.close_connection()
+        return mydb.data
+
 
 
     def gBuild(self, destinationSheetName):
 
-        #line 1 : update date/time, format dd/mm/YY H:M:S
+        ###line 1 : update date/time, format dd/mm/YY H:M:S
         now = datetime.now()
         dt_string = "Update : " + now.strftime("%d/%m/%Y %H:%M")	
         row =[dt_string]
         self.sheet.insert_row(row, index=1, value_input_option='RAW')
 
-        #line 2 empty
+        ###line 2 is for the test short names (links to jenkins pipeline), updated at the end 
-        #line 3 is for the column names
+
+        ###line 3 is for the column names
         i=3
-        row =["MR","Created_at","Author","Title","Assignee", "Reviewer", "CAN START","IN PROGRESS","COMPLETED","Review Form","OK MERGE","Merge conflicts"]
+        row =["MR","Created_at","Author","Title","Assignee", "Reviewer", "CAN START","IN PROGRESS","COMPLETED","Review Form","OK MERGE","Merge conflicts",""]
+
+        #tests
+        for t in range(0,len(self.tests)):
+            row.append("# PASS")
+            row.append("# FAIL")
+            row.append("Last Pass")
+            row.append("Last Fail")
+
         self.sheet.insert_row(row, index=i, value_input_option='RAW')
 
-        #line 4 onward, MR data lines
+        ###line 4 onward, MR data lines
-        for x in range(len(self.d)):
+        for x in range(len(self.git)):
             i=i+1                        
-            date_time_str = self.d[x]['created_at']
+            date_time_str = self.git[x]['created_at']
             date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ')
     
             milestone1=milestone2=milestone3=milestone4=""
-            if self.d[x]['milestone']!=None:
+            if self.git[x]['milestone']!=None:
-                if self.d[x]['milestone']['title']=="REVIEW_CAN_START":
+                if self.git[x]['milestone']['title']=="REVIEW_CAN_START":
                     milestone1="X"
-                elif self.d[x]['milestone']['title']=="REVIEW_IN_PROGRESS":
+                elif self.git[x]['milestone']['title']=="REVIEW_IN_PROGRESS":
                     milestone2="X"
-                elif self.d[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED":
+                elif self.git[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED":
                     milestone3="X"
-                elif self.d[x]['milestone']['title']=="OK_TO_BE_MERGED": 
+                elif self.git[x]['milestone']['title']=="OK_TO_BE_MERGED": 
                     milestone4="X" 
                 else:
                     pass
@@ -106,18 +145,18 @@ class gDashboard:
                 pass
 
             #check if empty or not
-            if self.d[x]['assignee']!=None:
+            if self.git[x]['assignee']!=None:
-                assignee = str(self.d[x]['assignee']['name'])
+                assignee = str(self.git[x]['assignee']['name'])
             else:
                 assignee = ""
  
             #check if empty or not       
-            if len(self.d[x]['reviewers'])!=0:
+            if len(self.git[x]['reviewers'])!=0:
-                reviewer = str(self.d[x]['reviewers'][0]['name'])
+                reviewer = str(self.git[x]['reviewers'][0]['name'])
             else:
                 reviewer = ""
 
-            if self.d[x]['has_conflicts']==True:
+            if self.git[x]['has_conflicts']==True:
                 conflicts = "YES"
             else:
                 conflicts = ""
@@ -129,13 +168,13 @@ class gDashboard:
             project_id = 223
             project = gl.projects.get(project_id)
             #get the opened MR in the project
-            mrs = project.mergerequests.list(state='opened')
+            mrs = project.mergerequests.list(state='opened',per_page=100)
+            review_form=''
             for m in range (0,len(mrs)):
-                if mrs[m].iid==self.d[x]['iid']:#check the iid is the one we are on
+                if mrs[m].iid==self.git[x]['iid']:#check the iid is the one we are on
                     mr_notes = mrs[m].notes.list(all=True)
                     n=0
                     found=False
-                    review_form=""
                     while found==False and n<len(mr_notes):
                         res=re.search('Code Review by',mr_notes[n].body)#this is the marker we are looking for in all notes
                         if res!=None:
@@ -143,29 +182,81 @@ class gDashboard:
                             found=True
                         n+=1
 
-
             #build final row to be inserted, the first column is left empty for now, will be filled afterward with hyperlinks to gitlab MR
-            row =["", str(date_time_obj.date()),str(self.d[x]['author']['name']),str(self.d[x]['title']),\
+            row =["", str(date_time_obj.date()),str(self.git[x]['author']['name']),	str(self.git[x]['title']),\
             assignee, reviewer,\
-            milestone1,milestone2,milestone3,review_form,milestone4,conflicts]
+            milestone1,milestone2,milestone3,review_form,milestone4,conflicts,""]
-            
+            #and append the test results coming from self.db 
-            #insert the row to worksheet
+            mr=str(self.git[x]['iid'])
+            for t in self.tests:
+                if mr in self.db:
+                    job=self.tests[t]['job']
+                    if job in self.db[mr]:
+                        if 'PASS' in self.db[mr][job]:
+                            row.append(self.db[mr][job]['PASS'])
+                        else:
+                            row.append('')
+                        if 'FAIL' in self.db[mr][job]:
+                            row.append(self.db[mr][job]['FAIL'])
+                        else:
+                            row.append('')
+                        #leave 2 columns for last_pass and last_fail links
+                        row.append('')
+                        row.append('')
+                    else:
+                        #4 columns are empty
+                        row.append('') 
+                        row.append('')
+                        row.append('')
+                        row.append('')
+
+            #insert the final row to worksheet
             self.sheet.insert_row(row, index=i, value_input_option='RAW')
-        
+            time.sleep(10)
-        
+
+
         #add MR hyperlinks in a list of requests to be sent as one update batch; this to save API calls (quotas) 
         i=3
         requests=[]
-        for x in range(len(self.d)):              
+        for x in range(len(self.git)):              
             rowIndex=i
             colIndex=0
-            hyperlink= '\"'+"https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/"+ str(self.d[x]['iid']) +'\"'
+            hyperlink= '\"'+"https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/"+ str(self.git[x]['iid']) +'\"'
-            text= '\"'+str(self.d[x]['iid'])+'"'
+            text= '\"'+str(self.git[x]['iid'])+'"'
             requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex))
-            i=i+1
+
+            mr=str(self.git[x]['iid'])
+            colIndex=15
+            for t in self.tests:
+                job=self.tests[t]['job']
+                if job in self.db[mr]:
+                    if 'last_pass' in self.db[mr][job]:
+                        hyperlink= '\"'+ self.db[mr][job]['last_pass'][1] +'\"'
+                        text= '\"'+self.db[mr][job]['last_pass'][0]+'"'
+                        requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex))
+                    if 'last_fail' in self.db[mr][job]:
+                        hyperlink= '\"'+ self.db[mr][job]['last_fail'][1] +'\"'
+                        text= '\"'+self.db[mr][job]['last_fail'][0]+'"'
+                        requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex+1))
+                colIndex+=4 #move to next test
+            i=i+1 #increment row index for next MR
+
         body = {"requests": requests}    
         self.ss.batch_update(body)        
             
+        ###line 2 is for the test names 
+        #add MR hyperlinks in a list of requests to be sent as one update batch; this to save API calls (quotas) 
+        requests=[]
+        rowIndex=1
+        colIndex=13
+        for t in self.tests :
+            hyperlink= '\"'+self.tests[t]['link']+'\"'
+            short_name= '\"'+ t +'\"'
+            requests.append(self.addHyperlink(hyperlink, short_name, destinationSheetName, rowIndex, colIndex))
+            colIndex+=4
+
+        body = {"requests": requests}
+        self.ss.batch_update(body)
 
     
     def addHyperlink(self, hyperlink, text, destinationSheetName, rowIndex, colIndex):
@@ -207,17 +298,17 @@ class gDashboard:
                     "copyPaste": {
                         "source": {
                             "sheetId": sourceSheetId,
-                            "startRowIndex": 0,
+                            "startRowIndex": 1,
                             "endRowIndex": 40,
                             "startColumnIndex": 0,
-                            "endColumnIndex": 12
+                            "endColumnIndex": 30 
                         },
                         "destination": {
                             "sheetId": destinationSheetId,
-                            "startRowIndex": 0,
+                            "startRowIndex": 1,
                             "endRowIndex": 40,
                             "startColumnIndex": 0,
-                            "endColumnIndex": 12
+                            "endColumnIndex": 30
                         },
                         "pasteType": "PASTE_FORMAT"
                     }
@@ -280,15 +371,30 @@ class gDashboard:
                 }
         )
     
-        body = {"requests": requests}    
-        self.ss.batch_update(body)
   
+        #group MR related columns
+#        sheetId = self.ss.worksheet(destinationSheetName)._properties['sheetId']
+#        requests.append(
+#                {
+#                    "addDimensionGroup": {
+#                        "range": {
+#                            "dimension": "COLUMNS",
+#                            "sheetId": sheetId,
+#                            "startIndex": 3,
+#                            "endIndex": 12
+#                        },
+#                    }
+#                }
+#        )
+#
+        body = {"requests": requests}
+        self.ss.batch_update(body)
+
+
 
 
 def main():
     my_gDashboard=gDashboard("/opt/dashboard/g_creds.json", 'OAI RAN Dashboard', 'MR Status')
-    cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ 
-    my_gDashboard.fetchData(cmd)
     my_gDashboard.gBuild("MR Status")
     my_gDashboard.gFormat("Formating" , "MR Status")
 

ci-scripts/ran_dashboard/ran_dashboard_cfg.yaml

@@ -0,0 +1,21 @@
+LTE-2x2 : #short name used in the dashboard
+  job : 'RAN-LTE-2x2-Module-OAIEPC' #job name from Jenkins, used in the database
+  link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-LTE-2x2-Module-OAIEPC'
+  bench : 'Obelix-N310-OAIEPC-Quectel(nrmodule2)'
+  test : 'TM2, TDD, 40MHz, MCS9, 26Mb DL, 7Mb UL'
+NSA-B200 :
+  job : 'RAN-NSA-B200-Module-LTEBOX'
+  link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-NSA-B200-Module-LTEBOX'
+  bench : 'Nepes-B200-Obelix-B200-LTEBOX-Quectel(idefix)'
+  test : 'TM1, 20MHz, 60Mb DL, 3Mb UL'
+NSA-2x2 :
+  job : 'RAN-NSA-2x2-Module-OAIEPC'
+  link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-NSA-2x2-Module-OAIEPC'
+  bench : 'Asterix-N310-Obelix-N310-OAIEPC-Quectel(nrmodule2)'
+  test : 'TM2, TDD, 40MHz, 60Mb DL, 3Mb UL'
+SA-N310 :
+  job : 'RAN-SA-Module-CN5G'
+  link : 'https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-SA-Module-CN5G'
+  bench : 'Asterix-N310-OAICN5G-Quectel(nrmodule2)'
+  test : 'TM2, TDD, 40MHz, 60Mb DL, 3Mb UL'
+

ci-scripts/ran_dashboard/readme.txt

@@ -0,0 +1,3 @@
+The code ran_dashboard.py was initially developped to bring MR status and test resuts to a google sheet, using google sheets API.
+This method is now deprecated, and replaced by Hdashboard.py that builds the results as HTML page and loads it to AWS S3.
+ran_dashboard_cfg.yaml is still used by Hdashboard.py as tests config file.

ci-scripts/ran_dashboard/sqlconnect.py

@@ -0,0 +1,107 @@
+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+# Merge Requests Dashboard for RAN on googleSheet 
+#
+#   Required Python Version
+#     Python 3.x
+#
+#---------------------------------------------------------------------
+
+#author Remi
+
+
+import pymysql
+import sys
+from datetime import datetime
+import pickle
+
+#This is the script/package used by the dashboard to retrieve the MR test results from the database
+
+class SQLConnect:
+    def __init__(self):
+        self.connection = pymysql.connect(
+                host='172.22.0.2',
+                user='root', 
+                password = 'ucZBc2XRYdvEm59F',
+                db='oaicicd_tests',
+                port=3306
+                )
+        self.data={} 
+
+
+    #retrieve data from mysql database and organize it in a dictionary (per MR passed as argument)
+    def get(self,MR):
+        self.data[MR]={}
+        cur=self.connection.cursor()
+
+        #get counters per test
+        sql = "select TEST,STATUS, count(*) AS COUNT from test_results where MR=(%s)  group by TEST, STATUS;"
+        cur.execute(sql,MR)
+        response=cur.fetchall()
+        if len(response)==0:#no test results yet
+            self.data[MR]['PASS']=''
+            self.data[MR]['FAIL']=''
+        else:
+            for i in range(0,len(response)):
+                test=response[i][0]
+                status=response[i][1]
+                count=response[i][2]
+                if test in self.data[MR]:
+                    self.data[MR][test][status]=count
+                else:
+                    self.data[MR][test]={}
+                    self.data[MR][test][status]=count
+      
+        #get last failing build and link
+        sql = "select TEST,BUILD, BUILD_LINK from test_results where MR=(%s) and STATUS='FAIL' order by DATE DESC;"
+        cur.execute(sql,MR)
+        response=cur.fetchall()
+        if len(response)!=0:
+            for i in range(0,len(response)):
+                test=response[i][0]
+                build=response[i][1]
+                link=response[i][2]
+                if 'last_fail' not in self.data[MR][test]:
+                    self.data[MR][test]['last_fail']=[]
+                    self.data[MR][test]['last_fail'].append(build)
+                    self.data[MR][test]['last_fail'].append(link)
+
+        #get last passing build and link
+        sql = "select TEST,BUILD, BUILD_LINK from test_results where MR=(%s) and STATUS='PASS' order by DATE DESC;"
+        cur.execute(sql,MR)
+        response=cur.fetchall()
+        if len(response)!=0:
+            for i in range(0,len(response)):
+                test=response[i][0]
+                build=response[i][1]
+                link=response[i][2]
+                if 'last_pass' not in self.data[MR][test]:
+                    self.data[MR][test]['last_pass']=[]
+                    self.data[MR][test]['last_pass'].append(build)
+                    self.data[MR][test]['last_pass'].append(link)
+
+
+    #close database connection
+    def close_connection(self):
+        self.connection.close()
+
+

ci-scripts/ran_dashboard/test_styles.css

@@ -0,0 +1,103 @@
+body {
+    font-family: 'lato', sans-serif;
+  }
+
+.Main {
+    text-align: left;
+    font-size: 30px;
+    font-weight: bold;
+}
+
+
+.DashLink {
+    text-align:left;
+    font-size: 16px;    
+}
+
+.DashLink:hover {
+    font-weight: bold;      
+}
+
+
+.Date {
+    text-align: left;
+    font-size: 16px;
+    font-weight: bold;
+}
+
+h3 {
+    font-size: 16px;    
+}
+
+a { text-decoration: none; }
+a:visited { text-decoration: none; color:blue}
+a:hover { text-decoration: none; font-weight: bold; color:blue}
+
+
+.Test_Table {
+    border-collapse: collapse;
+}
+
+.Test_Table tr {
+    font-size: 12px;
+    text-align:center;
+}
+
+
+.Test_Table tr:hover {
+    background-color:lightgray;
+}
+
+.Test_Table td {
+    border: 1px solid black;
+    padding : 5px;
+}
+.Test_Table td:hover {
+font-weight : bold;
+}
+
+.Test_Table th {
+    font-size: 14px;
+    text-align:center;
+    padding : 5px;
+}
+
+
+
+
+.Test_Name {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.Test_Descr {
+    table-layout: fixed;
+    width: 400px;
+    background-color: rgb(143, 154, 216);
+}
+
+.Pass {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.Fail {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.Last_Pass {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+
+.Last_Fail {
+    table-layout: fixed;
+    width: 100px;
+    background-color: rgb(143, 154, 216);
+}
+