From d4e135960b9c309ba6ddca2be17d8a69d16f8b43 Mon Sep 17 00:00:00 2001 From: hardy Date: Sun, 10 Oct 2021 22:10:04 +0200 Subject: [PATCH] major update / refactoring for ran dashboard, including mysql db and test results table --- .../mysql4testresults/docker-compose.yml | 12 ++ ci-scripts/mysql4testresults/sql_connect.py | 33 ++++ .../{ => ran_dashboard}/ran_dashboard.py | 177 ++++++++++-------- .../ran_dashboard_cfg.yaml | 0 ci-scripts/ran_dashboard/sqlconnect.py | 64 +++++++ 5 files changed, 209 insertions(+), 77 deletions(-) create mode 100644 ci-scripts/mysql4testresults/docker-compose.yml create mode 100644 ci-scripts/mysql4testresults/sql_connect.py rename ci-scripts/{ => ran_dashboard}/ran_dashboard.py (71%) rename ci-scripts/{ => ran_dashboard}/ran_dashboard_cfg.yaml (100%) create mode 100644 ci-scripts/ran_dashboard/sqlconnect.py diff --git a/ci-scripts/mysql4testresults/docker-compose.yml b/ci-scripts/mysql4testresults/docker-compose.yml new file mode 100644 index 0000000000..1c60057ab2 --- /dev/null +++ b/ci-scripts/mysql4testresults/docker-compose.yml @@ -0,0 +1,12 @@ +version: '2' +services: + mysql: + container_name: oaicicd_mysql + restart: always + image: mysql:latest + environment: + MYSQL_ROOT_PASSWORD: 'ucZBc2XRYdvEm59F' + ports: + - "3307:3306" + volumes: + - /home/oaicicd/mysql/data:/var/lib/mysql diff --git a/ci-scripts/mysql4testresults/sql_connect.py b/ci-scripts/mysql4testresults/sql_connect.py new file mode 100644 index 0000000000..333e573e78 --- /dev/null +++ b/ci-scripts/mysql4testresults/sql_connect.py @@ -0,0 +1,33 @@ +import pymysql +import sys +from datetime import datetime + +#This is the script used to write the test results to the mysql DB +#Called by Jenkins pipeline (jenkinsfile) +#Must be located in /home/oaicicid/mysql on the database host +#Usage from Jenkinsfile : +#python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} '' + +class SQLConnect: + def __init__(self): + self.connection = pymysql.connect( + host='172.22.0.2', + user='root', + password = 'ucZBc2XRYdvEm59F', + db='oaicicd_tests', + port=3306 + ) + + def put(self,TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS): + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + cur=self.connection.cursor() + cur.execute ('INSERT INTO test_results (TEST,MR,BRANCH,BUILD,BUILD_LINK,STATUS,DATE) VALUES (%s,%s,%s,%s,%s,%s,%s);' , (TEST, MR, BRANCH, BUILD, BUILD_LINK, STATUS, now)) + self.connection.commit() + self.connection.close() + + +if __name__ == "__main__": + mydb=SQLConnect() + mydb.put(sys.argv[1],sys.argv[2],sys.argv[3],sys.argv[4],sys.argv[5],sys.argv[6]) + + diff --git a/ci-scripts/ran_dashboard.py b/ci-scripts/ran_dashboard/ran_dashboard.py similarity index 71% rename from ci-scripts/ran_dashboard.py rename to ci-scripts/ran_dashboard/ran_dashboard.py index 7525c8684e..5fddba1b87 100644 --- a/ci-scripts/ran_dashboard.py +++ b/ci-scripts/ran_dashboard/ran_dashboard.py @@ -42,6 +42,11 @@ import datetime #now() and date formating from datetime import datetime import re import gitlab +import yaml +import os +import pickle + +from sqlconnect import SQLConnect #----------------------------------------------------------- # Class Declaration @@ -58,58 +63,76 @@ class gDashboard: self.sheet = self.ss.worksheet(worksheet) self.ss.del_worksheet(self.sheet) #start by deleting the old sheet self.sheet = self.ss.add_worksheet(title=worksheet, rows="100", cols="30") #create a new one - - self.d = {} #data dictionary + + #init with data sources : git, yaml config file, test results databases + cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ + self.git = self.__getGitData(cmd) #git data from Gitlab + self.tests = self.__loadCfg('ran_dashboard_cfg.yaml') #tests table setup from yaml + self.db = self.__loadFromDB() #test results from database - def fetchData(self,cmd): + def __loadCfg(self,yaml_file): + with open(yaml_file,'r') as f: + tests = yaml.load(f) + return tests + + def __getGitData(self,cmd): #cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ process = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE) output = process.stdout.readline() tmp=output.decode("utf-8") - self.d = json.loads(tmp) + d = json.loads(tmp) + return d + + def __loadFromDB(self): + mr_list=[] + for x in range(len(self.git)): + mr_list.append(str(self.git[x]['iid'])) + mydb=SQLConnect() + for MR in mr_list: + mydb.get(MR) + mydb.close_connection() + return mydb.data + def gBuild(self, destinationSheetName): - #line 1 : update date/time, format dd/mm/YY H:M:S + ###line 1 : update date/time, format dd/mm/YY H:M:S now = datetime.now() dt_string = "Update : " + now.strftime("%d/%m/%Y %H:%M") row =[dt_string] self.sheet.insert_row(row, index=1, value_input_option='RAW') - #line 2 is for the test names (links to jenkins pipeline), updated at the end + ###line 2 is for the test short names (links to jenkins pipeline), updated at the end - #line 3 is for the column names + ###line 3 is for the column names i=3 row =["MR","Created_at","Author","Title","Assignee", "Reviewer", "CAN START","IN PROGRESS","COMPLETED","Review Form","OK MERGE","Merge conflicts"] - #test 1 - row.append("# PASS") - row.append("# FAIL") - row.append("Last Fail") - #test 2 - row.append("# PASS") - row.append("# FAIL") - row.append("Last Fail") + #tests + for t in range(0,len(self.tests)): + row.append("# PASS") + row.append("# FAIL") + row.append("Last Fail") self.sheet.insert_row(row, index=i, value_input_option='RAW') - #line 4 onward, MR data lines - for x in range(len(self.d)): + ###line 4 onward, MR data lines + for x in range(len(self.git)): i=i+1 - date_time_str = self.d[x]['created_at'] + date_time_str = self.git[x]['created_at'] date_time_obj = datetime.strptime(date_time_str, '%Y-%m-%dT%H:%M:%S.%fZ') milestone1=milestone2=milestone3=milestone4="" - if self.d[x]['milestone']!=None: - if self.d[x]['milestone']['title']=="REVIEW_CAN_START": + if self.git[x]['milestone']!=None: + if self.git[x]['milestone']['title']=="REVIEW_CAN_START": milestone1="X" - elif self.d[x]['milestone']['title']=="REVIEW_IN_PROGRESS": + elif self.git[x]['milestone']['title']=="REVIEW_IN_PROGRESS": milestone2="X" - elif self.d[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED": + elif self.git[x]['milestone']['title']=="REVIEW_COMPLETED_AND_APPROVED": milestone3="X" - elif self.d[x]['milestone']['title']=="OK_TO_BE_MERGED": + elif self.git[x]['milestone']['title']=="OK_TO_BE_MERGED": milestone4="X" else: pass @@ -117,18 +140,18 @@ class gDashboard: pass #check if empty or not - if self.d[x]['assignee']!=None: - assignee = str(self.d[x]['assignee']['name']) + if self.git[x]['assignee']!=None: + assignee = str(self.git[x]['assignee']['name']) else: assignee = "" #check if empty or not - if len(self.d[x]['reviewers'])!=0: - reviewer = str(self.d[x]['reviewers'][0]['name']) + if len(self.git[x]['reviewers'])!=0: + reviewer = str(self.git[x]['reviewers'][0]['name']) else: reviewer = "" - if self.d[x]['has_conflicts']==True: + if self.git[x]['has_conflicts']==True: conflicts = "YES" else: conflicts = "" @@ -141,12 +164,12 @@ class gDashboard: project = gl.projects.get(project_id) #get the opened MR in the project mrs = project.mergerequests.list(state='opened') + review_form='' for m in range (0,len(mrs)): - if mrs[m].iid==self.d[x]['iid']:#check the iid is the one we are on + if mrs[m].iid==self.git[x]['iid']:#check the iid is the one we are on mr_notes = mrs[m].notes.list(all=True) n=0 found=False - review_form="" while found==False and n0: - colIndex=14 - hyperlink= '\"'+ testres['fails link'][0] +'\"' - text= '\"'+testres['fails list'][0]+'"' - requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex)) - i=i+1 + mr=str(self.git[x]['iid']) + colIndex=14 + for t in self.tests: + job=self.tests[t]['job'] + if job in self.db[mr]: + if len(self.db[mr][job]['last_fail'])>0: + hyperlink= '\"'+ self.db[mr][job]['last_fail'][1] +'\"' + text= '\"'+self.db[mr][job]['last_fail'][0]+'"' + requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex)) + colIndex+=3 + i=i+1 #increment row index for next MR + + + body = {"requests": requests} self.ss.batch_update(body) - #line 2 is for the test names + ###line 2 is for the test names #add MR hyperlinks in a list of requests to be sent as one update batch; this to save API calls (quotas) requests=[] rowIndex=1 colIndex=12 - hyperlink= '\"'+"https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-NSA-Mini-Module/"+'\"' - text= '\"'+'NSA Mini'+'"' - requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex)) - rowIndex=1 - colIndex=15 - hyperlink= '\"'+"https://jenkins-oai.eurecom.fr/view/RAN/job/RAN-SA-Module/"+'\"' - text= '\"'+'NR SA'+'"' - requests.append(self.addHyperlink(hyperlink, text, destinationSheetName, rowIndex, colIndex)) + for t in self.tests : + hyperlink= '\"'+self.tests[t]['link']+'\"' + short_name= '\"'+ t +'\"' + requests.append(self.addHyperlink(hyperlink, short_name, destinationSheetName, rowIndex, colIndex)) + colIndex+=3 body = {"requests": requests} self.ss.batch_update(body) @@ -264,14 +289,14 @@ class gDashboard: "startRowIndex": 1, "endRowIndex": 40, "startColumnIndex": 0, - "endColumnIndex": 19 + "endColumnIndex": 26 }, "destination": { "sheetId": destinationSheetId, "startRowIndex": 1, "endRowIndex": 40, "startColumnIndex": 0, - "endColumnIndex": 19 + "endColumnIndex": 26 }, "pasteType": "PASTE_FORMAT" } @@ -358,8 +383,6 @@ class gDashboard: def main(): my_gDashboard=gDashboard("/opt/dashboard/g_creds.json", 'OAI RAN Dashboard', 'MR Status') - cmd="""curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100" """ - my_gDashboard.fetchData(cmd) my_gDashboard.gBuild("MR Status") my_gDashboard.gFormat("Formating" , "MR Status") diff --git a/ci-scripts/ran_dashboard_cfg.yaml b/ci-scripts/ran_dashboard/ran_dashboard_cfg.yaml similarity index 100% rename from ci-scripts/ran_dashboard_cfg.yaml rename to ci-scripts/ran_dashboard/ran_dashboard_cfg.yaml diff --git a/ci-scripts/ran_dashboard/sqlconnect.py b/ci-scripts/ran_dashboard/sqlconnect.py new file mode 100644 index 0000000000..272ff4e50f --- /dev/null +++ b/ci-scripts/ran_dashboard/sqlconnect.py @@ -0,0 +1,64 @@ +import pymysql +import sys +from datetime import datetime +import pickle + + +#This is the script/package used by the dashboard to retrieve the MR test results from the database + +class SQLConnect: + def __init__(self): + self.connection = pymysql.connect( + host='172.22.0.2', + user='root', + password = 'ucZBc2XRYdvEm59F', + db='oaicicd_tests', + port=3306 + ) + self.data={} + + + #retrieve data from mysql database and organize it in a dictionary (per MR passed as argument) + def get(self,MR): + self.data[MR]={} + cur=self.connection.cursor() + + #get counters per test + sql = "select TEST,STATUS, count(*) AS COUNT from test_results where MR=(%s) group by TEST, STATUS;" + cur.execute(sql,MR) + response=cur.fetchall() + if len(response)==0:#no test results yet + self.data[MR]['PASS']='' + self.data[MR]['FAIL']='' + else: + for i in range(0,len(response)): + test=response[i][0] + status=response[i][1] + count=response[i][2] + if test in self.data[MR]: + self.data[MR][test][status]=count + else: + self.data[MR][test]={} + self.data[MR][test][status]=count + + #get last failing build and link + sql = "select TEST,BUILD, BUILD_LINK from test_results where MR=(%s) and STATUS='FAIL' order by DATE DESC;" + cur.execute(sql,MR) + response=cur.fetchall() + if len(response)!=0: + for i in range(0,len(response)): + test=response[i][0] + build=response[i][1] + link=response[i][2] + if 'last_fail' not in self.data[MR][test]: + self.data[MR][test]['last_fail']=[] + self.data[MR][test]['last_fail'].append(build) + self.data[MR][test]['last_fail'].append(link) + + + + #close database connection + def close_connection(self): + self.connection.close() + +