From 707a8000dbab537b9a4e962f461688d6bd58fcd9 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 20 Oct 2021 15:49:16 +0200 Subject: [PATCH] google sheet dashboard is deprecated, replaced by Html dashboard hosted on AWS S3 --- ci-scripts/Jenkinsfile-git-dashboard | 5 +- ci-scripts/ran_dashboard/Hdashboard.py | 401 ++++++++++++++++++ ci-scripts/ran_dashboard/mr_styles.css | 136 ++++++ ci-scripts/ran_dashboard/ran_dashboard.py | 2 +- .../ran_dashboard/ran_dashboard_cfg.yaml | 19 +- ci-scripts/ran_dashboard/readme.txt | 3 + ci-scripts/ran_dashboard/test_styles.css | 103 +++++ 7 files changed, 661 insertions(+), 8 deletions(-) create mode 100644 ci-scripts/ran_dashboard/Hdashboard.py create mode 100755 ci-scripts/ran_dashboard/mr_styles.css create mode 100644 ci-scripts/ran_dashboard/readme.txt create mode 100755 ci-scripts/ran_dashboard/test_styles.css diff --git a/ci-scripts/Jenkinsfile-git-dashboard b/ci-scripts/Jenkinsfile-git-dashboard index 8c5d773192..87e781fe8a 100644 --- a/ci-scripts/Jenkinsfile-git-dashboard +++ b/ci-scripts/Jenkinsfile-git-dashboard @@ -35,8 +35,9 @@ pipeline { steps { script { dir ("ci-scripts/ran_dashboard") { - //retrieve MR data from gitlab and export to gSheet - sh returnStdout: true, script: 'python3 ran_dashboard.py' + //retrieve MR data from gitlab / mySQL db, build HTML pages and load them to AWS S3 bucket (configured as static web page hosting) + //deprecated method : sh returnStdout: true, script: 'python3 ran_dashboard.py' + sh returnStdout: true, script: 'python3 Hdashboard.py' } } } diff --git a/ci-scripts/ran_dashboard/Hdashboard.py b/ci-scripts/ran_dashboard/Hdashboard.py new file mode 100644 index 0000000000..5e34ae50bf --- /dev/null +++ b/ci-scripts/ran_dashboard/Hdashboard.py @@ -0,0 +1,401 @@ +#/* +# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The OpenAirInterface Software Alliance licenses this file to You under +# * the OAI Public License, Version 1.1 (the "License"); you may not use this file +# * except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.openairinterface.org/?page_id=698 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# *------------------------------------------------------------------------------- +# * For more information about the OpenAirInterface (OAI) Software Alliance: +# * contact@openairinterface.org +# */ +#--------------------------------------------------------------------- +# Merge Requests Dashboard for RAN on googleSheet +# +# Required Python Version +# Python 3.x +# +#--------------------------------------------------------------------- + +#----------------------------------------------------------- +# Import +#----------------------------------------------------------- + + +#Author Remi +import boto3 +import shlex +import subprocess +import json #json structures +import datetime #now() and date formating +from datetime import datetime +import re +import gitlab +import yaml +import os +import time + + +from sqlconnect import SQLConnect + +#----------------------------------------------------------- +# Class Declaration +#----------------------------------------------------------- + +class Dashboard: + def __init__(self): + + + #init with data sources : git, yaml config file, test results databases + print("Collecting Data") + cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ + self.git = self.__getGitData(cmd) #git data from Gitlab + self.tests = self.__loadCfg('ran_dashboard_cfg.yaml') #tests table setup from yaml + self.db = self.__loadFromDB() #test results from database + + + def __loadCfg(self,yaml_file): + with open(yaml_file,'r') as f: + tests = yaml.load(f) + return tests + + def __getGitData(self,cmd): + #cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ + process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) + output = process.stdout.readline() + tmp=output.decode("utf-8") + d = json.loads(tmp) + return d + + def __loadFromDB(self): + mr_list=[] + for x in range(len(self.git)): + mr_list.append(str(self.git[x]['iid'])) + mydb=SQLConnect() + for MR in mr_list: + mydb.get(MR) + mydb.close_connection() + return mydb.data + + def Test_initHTML(self, date): + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('Test Dashboard\n') + self.f_html.write('\n') + self.f_html.write('
\n') + self.f_html.write('
\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('
OAI RAN TEST Status Dashboard
Update : '+date+'
\n') + self.f_html.write('
\n') + self.f_html.write('
\n') + + def Test_terminateHTML(self): + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.close() + + + def MR_initHTML(self,date): + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('MR Dashboard\n') + self.f_html.write('\n') + self.f_html.write('
\n') + self.f_html.write('
\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('
OAI RAN MR Status Dashboard
Update : '+date+'
\n') + self.f_html.write('
\n') + self.f_html.write('
\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + + def MR_terminateHTML(self): + self.f_html.write('
MRCreated_AtAuthorTitleAssigneeReviewerCAN STARTIN PROGRESSCOMPLETEDReview FormOK MergeMerge Conflicts
\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.close() + + + def MR_rowHTML(self,row): + self.f_html.write('\n') + self.f_html.write(''+row[1]+'\n') + self.f_html.write(''+row[2]+'\n') + self.f_html.write(''+row[3]+'\n') + self.f_html.write(''+row[4]+'\n') + self.f_html.write(''+row[5]+'\n') + self.f_html.write(''+row[6]+'\n') + if row[7]=='X': + self.f_html.write(''+row[7]+'\n') + else: + self.f_html.write('\n') + if row[8]=='X': + self.f_html.write(''+row[8]+'\n') + else: + self.f_html.write('\n') + if row[9]=='X': + self.f_html.write(''+row[9]+'\n') + else: + self.f_html.write('\n') + if row[10]=='X': + self.f_html.write(''+row[10]+'\n') + else: + self.f_html.write('\n') + if row[11]=='X': + self.f_html.write(''+row[11]+'\n') + else: + self.f_html.write('\n') + if row[12]=='YES': + self.f_html.write(''+row[12]+'\n') + else: + self.f_html.write('\n') + self.f_html.write('\n') + + + def Build(self, type, htmlfilename): + if type=='MR': + self.Build_MR_Table(htmlfilename) + elif type=='Tests': + self.Build_Test_Table(htmlfilename) + else : + print("Undefined Dashboard Type, options : MR or Tests") + + + def Build_Test_Table(self,htmlfilename): + print("Building Tests Dashboard...") + + self.f_html=open(htmlfilename,'w') + + ###update date/time, format dd/mm/YY H:M:S + now = datetime.now() + dt_string = now.strftime("%d/%m/%Y %H:%M") + #HTML table header + self.Test_initHTML(dt_string) + + + #1 table per MR if test results exist + for x in range(len(self.git)): + mr=str(self.git[x]['iid']) + if 'PASS' not in self.db[mr]: + self.f_html.write('

'+mr+''+' '+self.git[x]['title'] + '

\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write(' \n') + self.f_html.write(' \n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + + #parsing the tests + for t in self.tests: + + row=[] + short_name= t + hyperlink= self.tests[t]['link'] + job=self.tests[t]['job'] + + + if job in self.db[mr]: + if 'PASS' in self.db[mr][job]: + row.append(self.db[mr][job]['PASS']) + else: + row.append('') + if 'FAIL' in self.db[mr][job]: + row.append(self.db[mr][job]['FAIL']) + else: + row.append('') + #2 columns for last_pass and last_fail links + if 'last_pass' in self.db[mr][job]: + lastpasshyperlink= self.db[mr][job]['last_pass'][1] + lastpasstext= self.db[mr][job]['last_pass'][0] + else: + lastpasshyperlink='' + lastpasstext='' + + if 'last_fail' in self.db[mr][job]: + lastfailhyperlink= self.db[mr][job]['last_fail'][1] + lastfailtext= self.db[mr][job]['last_fail'][0] + else: + lastfailhyperlink='' + lastfailtext='' + + + + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + if row[0]!='': + self.f_html.write('\n') + else: + self.f_html.write('\n') + if row[1]!='': + self.f_html.write('\n') + else: + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + self.f_html.write('\n') + + self.f_html.write('
Test NameBenchTest# Pass# FailLast PassLast Fail
'+short_name+''+self.tests[t]['bench']+''+self.tests[t]['test']+''+str(row[0])+''+str(row[1])+''+lastpasstext+''+lastfailtext+'
\n') + + #terminate HTML table and close file + self.Test_terminateHTML() + + + def Build_MR_Table(self,htmlfilename): + + print("Building Merge Requests Dashboard...") + + self.f_html=open(htmlfilename,'w') + + ###update date/time, format dd/mm/YY H:M:S + now = datetime.now() + dt_string = now.strftime("%d/%m/%Y %H:%M") + + #HTML table header + self.MR_initHTML(dt_string) + + + ###MR data lines + for x in range(len(self.git)): + + + hyperlink= 'https://gitlab.eurecom.fr/oai/openairinterface5g/-/merge_requests/'+ str(self.git[x]['iid']) + text= str(self.git[x]['iid']) + + + date_time_str = self.git[x]['created_at'] + date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ') + + milestone1=milestone2=milestone3=milestone4="" + if self.git[x]['milestone']!=None: + if self.git[x]['milestone']['title']=="REVIEW_CAN_START": + milestone1="X" + elif self.git[x]['milestone']['title']=="REVIEW_IN_PROGRESS": + milestone2="X" + elif self.git[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED": + milestone3="X" + elif self.git[x]['milestone']['title']=="OK_TO_BE_MERGED": + milestone4="X" + else: + pass + else: + pass + + #check if empty or not + if self.git[x]['assignee']!=None: + assignee = str(self.git[x]['assignee']['name']) + else: + assignee = "" + + #check if empty or not + if len(self.git[x]['reviewers'])!=0: + reviewer = str(self.git[x]['reviewers'][0]['name']) + else: + reviewer = "" + + if self.git[x]['has_conflicts']==True: + conflicts = "YES" + else: + conflicts = "" + + + #add a column flagging that the review form is present + #we use gitlab API to parse the MR notes + gl = gitlab.Gitlab.from_config('OAI') + project_id = 223 + project = gl.projects.get(project_id) + #get the opened MR in the project + mrs = project.mergerequests.list(state='opened',per_page=100) + review_form='' + for m in range (0,len(mrs)): + if mrs[m].iid==self.git[x]['iid']:#check the iid is the one we are on + mr_notes = mrs[m].notes.list(all=True) + n=0 + found=False + while found==False and n