diff --git a/ci-scripts/Jenkinsfile-git-dashboard b/ci-scripts/Jenkinsfile-git-dashboard index 6ebd46bc76..87e781fe8a 100644 --- a/ci-scripts/Jenkinsfile-git-dashboard +++ b/ci-scripts/Jenkinsfile-git-dashboard @@ -33,9 +33,12 @@ pipeline { stages { stage ("gDashboard") { steps { - script { - //retrieve MR data from gitlab and export to gSheet - sh returnStdout: true, script: 'python3 ci-scripts/ran_dashboard.py' + script { + dir ("ci-scripts/ran_dashboard") { + //retrieve MR data from gitlab / mySQL db, build HTML pages and load them to AWS S3 bucket (configured as static web page hosting) + //deprecated method : sh returnStdout: true, script: 'python3 ran_dashboard.py' + sh returnStdout: true, script: 'python3 Hdashboard.py' + } } } } diff --git a/ci-scripts/mysql4testresults/docker-compose.yml b/ci-scripts/mysql4testresults/docker-compose.yml new file mode 100644 index 0000000000..1c60057ab2 --- /dev/null +++ b/ci-scripts/mysql4testresults/docker-compose.yml @@ -0,0 +1,12 @@ +version: '2' +services: + mysql: + container_name: oaicicd_mysql + restart: always + image: mysql:latest + environment: + MYSQL_ROOT_PASSWORD: 'ucZBc2XRYdvEm59F' + ports: + - "3307:3306" + volumes: + - /home/oaicicd/mysql/data:/var/lib/mysql diff --git a/ci-scripts/mysql4testresults/sql_connect.py b/ci-scripts/mysql4testresults/sql_connect.py new file mode 100644 index 0000000000..333e573e78 --- /dev/null +++ b/ci-scripts/mysql4testresults/sql_connect.py @@ -0,0 +1,33 @@ +import pymysql +import sys +from datetime import datetime + +#This is the script used to write the test results to the mysql DB +#Called by Jenkins pipeline (jenkinsfile) +#Must be located in /home/oaicicid/mysql on the database host +#Usage from Jenkinsfile : +#python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} '' + +class SQLConnect: + def __init__(self): + self.connection = pymysql.connect( + host='172.22.0.2', + user='root', + password = 'ucZBc2XRYdvEm59F', + db='oaicicd_tests', + port=3306 + ) + + def put(self,TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS): + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + cur=self.connection.cursor() + cur.execute ('INSERT INTO test_results (TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS,DATE) VALUES (%s,%s,%s,%s,%s,%s,%s);' , (TEST, MR, BRANCH, BUILD, BUILD_LINK, STATUS, now)) + self.connection.commit() + self.connection.close() + + +if __name__ == "__main__": + mydb=SQLConnect() + mydb.put(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6]) + + diff --git a/ci-scripts/ran_dashboard/Hdashboard.py b/ci-scripts/ran_dashboard/Hdashboard.py new file mode 100644 index 0000000000..5e34ae50bf --- /dev/null +++ b/ci-scripts/ran_dashboard/Hdashboard.py @@ -0,0 +1,401 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# Merge Requests Dashboard for RAN on googleSheet +# +# Required Python Version +# Python 3.x +# +#--------------------------------------------------------------------- + +#----------------------------------------------------------- +# Import +#----------------------------------------------------------- + + +#Author Remi +import boto3 +import shlex +import subprocess +import json #json structures +import datetime #now() and date formating +from datetime import datetime +import re +import gitlab +import yaml +import os +import time + + +from sqlconnect import SQLConnect + +#----------------------------------------------------------- +# Class Declaration +#----------------------------------------------------------- + +class Dashboard: + def __init__(self): + + + #init with data sources : git, yaml config file, test results databases + print("Collecting Data") + cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ + self.git = self.__getGitData(cmd) #git data from Gitlab + self.tests = self.__loadCfg('ran_dashboard_cfg.yaml') #tests table setup from yaml + self.db = self.__loadFromDB() #test results from database + + + def __loadCfg(self,yaml_file): + with open(yaml_file,'r') as f: + tests = yaml.load(f) + return tests + + def __getGitData(self,cmd): + #cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ + process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + output = process.stdout.readline() + tmp=output.decode("utf-8") + d = json.loads(tmp) + return d + + def __loadFromDB(self): + mr_list=[] + for x in range(len(self.git)): + mr_list.append(str(self.git[x]['iid'])) + mydb=SQLConnect() + for MR in mr_list: + mydb.get(MR) + mydb.close_connection() + return mydb.data + + def Test_initHTML(self, date): + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('Test Dashboard\n') + self.f_html.write('\n') + self.f_html.write('
\n') + self.f_html.write('
\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('
OAI RAN TEST Status Dashboard
Update : '+date+'
\n') + self.f_html.write('
\n') + self.f_html.write('
\n') + + def Test_terminateHTML(self): + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.close() + + + def MR_initHTML(self,date): + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('MR Dashboard\n') + self.f_html.write('\n') + self.f_html.write('
\n') + self.f_html.write('
\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('
OAI RAN MR Status Dashboard
Update : '+date+'
\n') + self.f_html.write('
\n') + self.f_html.write('
\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + + def MR_terminateHTML(self): + self.f_html.write('
MRCreated_AtAuthorTitleAssigneeReviewerCAN STARTIN PROGRESSCOMPLETEDReview FormOK MergeMerge Conflicts
\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.close() + + + def MR_rowHTML(self,row): + self.f_html.write('\n') + self.f_html.write(''+row[1]+'\n') + self.f_html.write(''+row[2]+'\n') + self.f_html.write(''+row[3]+'\n') + self.f_html.write(''+row[4]+'\n') + self.f_html.write(''+row[5]+'\n') + self.f_html.write(''+row[6]+'\n') + if row[7]=='X': + self.f_html.write(''+row[7]+'\n') + else: + self.f_html.write('\n') + if row[8]=='X': + self.f_html.write(''+row[8]+'\n') + else: + self.f_html.write('\n') + if row[9]=='X': + self.f_html.write(''+row[9]+'\n') + else: + self.f_html.write('\n') + if row[10]=='X': + self.f_html.write(''+row[10]+'\n') + else: + self.f_html.write('\n') + if row[11]=='X': + self.f_html.write(''+row[11]+'\n') + else: + self.f_html.write('\n') + if row[12]=='YES': + self.f_html.write(''+row[12]+'\n') + else: + self.f_html.write('\n') + self.f_html.write('\n') + + + def Build(self, type, htmlfilename): + if type=='MR': + self.Build_MR_Table(htmlfilename) + elif type=='Tests': + self.Build_Test_Table(htmlfilename) + else : + print("Undefined Dashboard Type, options : MR or Tests") + + + def Build_Test_Table(self,htmlfilename): + print("Building Tests Dashboard...") + + self.f_html=open(htmlfilename,'w') + + ###update date/time, format dd/mm/YY H:M:S + now = datetime.now() + dt_string = now.strftime("%d/%m/%Y %H:%M") + #HTML table header + self.Test_initHTML(dt_string) + + + #1 table per MR if test results exist + for x in range(len(self.git)): + mr=str(self.git[x]['iid']) + if 'PASS' not in self.db[mr]: + self.f_html.write('

'+mr+''+' '+self.git[x]['title'] + '

\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write(' \n') + self.f_html.write(' \n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + + #parsing the tests + for t in self.tests: + + row=[] + short_name= t + hyperlink= self.tests[t]['link'] + job=self.tests[t]['job'] + + + if job in self.db[mr]: + if 'PASS' in self.db[mr][job]: + row.append(self.db[mr][job]['PASS']) + else: + row.append('') + if 'FAIL' in self.db[mr][job]: + row.append(self.db[mr][job]['FAIL']) + else: + row.append('') + #2 columns for last_pass and last_fail links + if 'last_pass' in self.db[mr][job]: + lastpasshyperlink= self.db[mr][job]['last_pass'][1] + lastpasstext= self.db[mr][job]['last_pass'][0] + else: + lastpasshyperlink='' + lastpasstext='' + + if 'last_fail' in self.db[mr][job]: + lastfailhyperlink= self.db[mr][job]['last_fail'][1] + lastfailtext= self.db[mr][job]['last_fail'][0] + else: + lastfailhyperlink='' + lastfailtext='' + + + + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + if row[0]!='': + self.f_html.write('\n') + else: + self.f_html.write('\n') + if row[1]!='': + self.f_html.write('\n') + else: + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + + self.f_html.write('
Test NameBenchTest# Pass# FailLast PassLast Fail
'+short_name+''+self.tests[t]['bench']+''+self.tests[t]['test']+''+str(row[0])+''+str(row[1])+''+lastpasstext+''+lastfailtext+'
\n') + + #terminate HTML table and close file + self.Test_terminateHTML() + + + def Build_MR_Table(self,htmlfilename): + + print("Building Merge Requests Dashboard...") + + self.f_html=open(htmlfilename,'w') + + ###update date/time, format dd/mm/YY H:M:S + now = datetime.now() + dt_string = now.strftime("%d/%m/%Y %H:%M") + + #HTML table header + self.MR_initHTML(dt_string) + + + ###MR data lines + for x in range(len(self.git)): + + + hyperlink= 'https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/'+ str(self.git[x]['iid']) + text= str(self.git[x]['iid']) + + + date_time_str = self.git[x]['created_at'] + date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ') + + milestone1=milestone2=milestone3=milestone4="" + if self.git[x]['milestone']!=None: + if self.git[x]['milestone']['title']=="REVIEW_CAN_START": + milestone1="X" + elif self.git[x]['milestone']['title']=="REVIEW_IN_PROGRESS": + milestone2="X" + elif self.git[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED": + milestone3="X" + elif self.git[x]['milestone']['title']=="OK_TO_BE_MERGED": + milestone4="X" + else: + pass + else: + pass + + #check if empty or not + if self.git[x]['assignee']!=None: + assignee = str(self.git[x]['assignee']['name']) + else: + assignee = "" + + #check if empty or not + if len(self.git[x]['reviewers'])!=0: + reviewer = str(self.git[x]['reviewers'][0]['name']) + else: + reviewer = "" + + if self.git[x]['has_conflicts']==True: + conflicts = "YES" + else: + conflicts = "" + + + #add a column flagging that the review form is present + #we use gitlab API to parse the MR notes + gl = gitlab.Gitlab.from_config('OAI') + project_id = 223 + project = gl.projects.get(project_id) + #get the opened MR in the project + mrs = project.mergerequests.list(state='opened',per_page=100) + review_form='' + for m in range (0,len(mrs)): + if mrs[m].iid==self.git[x]['iid']:#check the iid is the one we are on + mr_notes = mrs[m].notes.list(all=True) + n=0 + found=False + while found==False and n