545 lines
15 KiB
Python
Executable File
545 lines
15 KiB
Python
Executable File
#******************************************************************************
|
|
#
|
|
# \file lib_autotest_analyser.py
|
|
#
|
|
# \par Informations
|
|
# - \b Project : UED Autotest Framework
|
|
# - \b Software :
|
|
#
|
|
# \date 16 september 2016
|
|
#
|
|
# \version 0.1
|
|
#
|
|
# \brief library to extract metrics from autotest logs and assign a verdict
|
|
#
|
|
# \author Benoit ROBERT (benoit.robert@syrtem.com)
|
|
#
|
|
# \par Statement of Ownership
|
|
# COPYRIGHT (c) 2016 BY SYRTEM S.A.R.L
|
|
# This software is furnished under license and may be used and copied
|
|
# only in accordance with the terms of such license and with the inclusion
|
|
# of the above COPYRIGHT notice. This SOFTWARE or any other copies thereof
|
|
# may not be provided or otherwise made available to any other person.
|
|
# No title to and ownership of the SOFTWARE is hereby transferred.
|
|
#
|
|
# The information in this SOFTWARE is subject to change without notice
|
|
# and should not be constructed as a commitment by SYRTEM.
|
|
# SYRTEM assumes no responsibility for the use or reliability of its
|
|
# SOFTWARE on equipment or platform not explicitly validated by SYRTEM.
|
|
#
|
|
# *******************************************************************************
|
|
|
|
|
|
import re
|
|
from pylab import *
|
|
from matplotlib.font_manager import FontProperties
|
|
|
|
import os
|
|
from jinja2 import Environment, FileSystemLoader
|
|
|
|
|
|
PATH = os.path.dirname(os.path.abspath(__file__))
|
|
TEMPLATE_ENVIRONMENT = Environment(
|
|
autoescape=False,
|
|
loader=FileSystemLoader(os.path.join(PATH, '../templates')),
|
|
trim_blocks=False)
|
|
|
|
|
|
def render_template(template_filename, context):
|
|
return TEMPLATE_ENVIRONMENT.get_template(template_filename).render(context)
|
|
|
|
|
|
|
|
def init(args = None):
|
|
return
|
|
|
|
|
|
#
|
|
#
|
|
#
|
|
def do_extract_metrics(args):
|
|
|
|
# print ""
|
|
# print "do_extract_metrics ... "
|
|
|
|
fname = args['file']
|
|
metric = args['metric']
|
|
# print(fname)
|
|
# print 'metric id = ' + metric['id']
|
|
# print 'metric regex = ' + metric['regex']
|
|
|
|
count = 0
|
|
mmin = 0
|
|
mmin_index = 0
|
|
mmax = 0
|
|
mmax_index = 0
|
|
mean = 0
|
|
median = 0
|
|
|
|
|
|
output = np.fromregex(fname,metric['regex'], [('id', 'S20'), ('metric', np.float), ('frame', np.int)] )
|
|
# print 'T0T0 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
|
|
# print output
|
|
count = output['metric'].size
|
|
# print count
|
|
if count > 0:
|
|
mmin = np.amin(output['metric']);
|
|
mmin_index = np.argmin(output['metric']);
|
|
mmax = np.amax(output['metric']);
|
|
mmax_index = np.argmax(output['metric']);
|
|
mean = np.mean(output['metric']);
|
|
median = np.median(output['metric']);
|
|
|
|
# print ( ( (metric['min_limit'] > output['metric']).sum() / float(output['metric'].size) ) * 100 )
|
|
|
|
ret = { 'metric_count' : count,
|
|
'metric_buf' : output,
|
|
'metric_min' : mmin,
|
|
'metric_min_index' : mmin_index,
|
|
'metric_max' : mmax,
|
|
'metric_max_index' : mmax_index,
|
|
'metric_mean' : mean,
|
|
'metric_median' : median,
|
|
}
|
|
return(ret)
|
|
|
|
|
|
def do_extract_metrics_new(args):
|
|
|
|
# print ""
|
|
# print "do_extract_metrics ... "
|
|
|
|
fname = args['file']
|
|
metric = args['metric']
|
|
print(fname)
|
|
print 'metric id = ' + metric['id']
|
|
print 'metric regex = ' + metric['regex']
|
|
|
|
|
|
|
|
count = 0
|
|
mmin = 0
|
|
mmin_index = 0
|
|
mmax = 0
|
|
mmax_index = 0
|
|
mean = 0
|
|
median = 0
|
|
|
|
toto = [('id', 'S20'), ('metric', np.float), ('frame', np.int)]
|
|
print toto
|
|
|
|
|
|
np_format = []
|
|
|
|
for x in range(0, metric['nb_metric']):
|
|
np_format.append( ('id'+str(x), 'S20') )
|
|
np_format.append( ('metric'+str(x), np.float) )
|
|
np_format.append( ('uom'+str(x), 'S20') )
|
|
np_format.append( ('frame', np.int))
|
|
|
|
print np_format
|
|
|
|
output = np.fromregex(fname,metric['regex'], np_format)
|
|
print output
|
|
count = output['frame'].size
|
|
print count
|
|
if count > 0:
|
|
|
|
|
|
fontP = FontProperties()
|
|
fontP.set_size('small')
|
|
|
|
fig = plt.figure(1)
|
|
plt.figure(figsize=(10,10))
|
|
|
|
plot_xmax = np.amax(output['frame'])+np.amin(output['frame'])
|
|
|
|
for x in range(0, metric['nb_metric']):
|
|
|
|
metric_name = output['id'+str(x)][0]
|
|
metric_uom = output['uom'+str(x)][0]
|
|
|
|
mmin = np.amin(output['metric'+str(x)])
|
|
mmax = np.amax(output['metric'+str(x)])
|
|
mmean = np.mean(output['metric'+str(x)])
|
|
mmedian = np.median(output['metric'+str(x)])
|
|
|
|
plot_loc = 100*metric['nb_metric']+10+x+1
|
|
|
|
sbplt = plt.subplot(plot_loc)
|
|
sbplt.plot(output['frame'], output['metric'+str(x)], color='b' )
|
|
sbplt.set_title( metric_name+' ('+metric_uom+')')
|
|
if mmin < 0:
|
|
sbplot_ymin=mmin+mmin/10
|
|
else:
|
|
sbplot_ymin=0
|
|
sbplt.set_ylim(ymin=sbplot_ymin)
|
|
if mmax > 0:
|
|
sbplot_ymax=mmax+mmax/10
|
|
else:
|
|
sbplot_ymax=0
|
|
sbplt.set_ylim(ymax=sbplot_ymax)
|
|
|
|
sbplt.set_xlim(xmax=plot_xmax)
|
|
sbplt.set_xlim(xmin=0)
|
|
text='min: '+str(mmin)+'\nmax: '+str(mmax)+'\nmean: '+str(mmean)+'\nmedian: '+str(mmedian)
|
|
sbplt.text( plot_xmax+10,sbplot_ymin,text)
|
|
sbplt.set_xlabel('frame')
|
|
sbplt.set_ylabel(metric_name)
|
|
|
|
plt.tight_layout()
|
|
|
|
fname = "toto.png"
|
|
|
|
# lgd = plt.legend(prop=fontP, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
|
|
mng = plt.get_current_fig_manager()
|
|
plt.savefig(fname, bbox_inches='tight')
|
|
plt.close()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
mmin = np.amin(output['metric']);
|
|
mmin_index = np.argmin(output['metric']);
|
|
mmax = np.amax(output['metric']);
|
|
mmax_index = np.argmax(output['metric']);
|
|
mean = np.mean(output['metric']);
|
|
median = np.median(output['metric']);
|
|
|
|
# print ( ( (metric['min_limit'] > output['metric']).sum() / float(output['metric'].size) ) * 100 )
|
|
|
|
ret = { 'metric_count' : count,
|
|
'metric_buf' : output,
|
|
|
|
|
|
'metric_min' : mmin,
|
|
'metric_min_index' : mmin_index,
|
|
'metric_max' : mmax,
|
|
'metric_max_index' : mmax_index,
|
|
'metric_mean' : mean,
|
|
'metric_median' : median,
|
|
}
|
|
return(ret)
|
|
|
|
#
|
|
#
|
|
#
|
|
def do_check_verdict(metric_def, metric_data):
|
|
verdict = 'INCON'
|
|
|
|
pass_fail_stat = metric_def['pass_fail_stat']
|
|
|
|
if pass_fail_stat == 'max_value':
|
|
metric_stat = metric_data['metric_max']
|
|
elif pass_fail_stat == 'min_value':
|
|
metric_stat = metric_data['metric_min']
|
|
elif pass_fail_stat == 'mean_value':
|
|
metric_stat = metric_data['metric_mean']
|
|
elif pass_fail_stat == 'median_value':
|
|
metric_stat = metric_data['metric_median']
|
|
else :
|
|
print "do_check_verdict -> undef metric stat (pass_fail_stat in xml file)"
|
|
return verdict
|
|
|
|
if 'max_limit' in metric_def:
|
|
if metric_stat > metric_def['max_limit']:
|
|
verdict = 'FAIL'
|
|
else:
|
|
verdict = 'PASS'
|
|
|
|
if 'min_limit' in metric_def:
|
|
if metric_stat < metric_def['min_limit']:
|
|
verdict = 'FAIL'
|
|
else:
|
|
verdict = 'PASS'
|
|
return verdict
|
|
|
|
|
|
def do_print_metrics(metric_def, metric_data):
|
|
|
|
|
|
if metric_data['metric_count'] > 0 :
|
|
|
|
# output = np.array( metric_data['metric_buf'] , [('id', 'S20'), ('metric', np.float), ('frame', np.int)])
|
|
output = metric_data['metric_buf']
|
|
# print output
|
|
|
|
fontP = FontProperties()
|
|
fontP.set_size('small')
|
|
|
|
plt.scatter(output['frame'], output['metric'], color='b', alpha=0.33, s = 1 )
|
|
plt.plot([0, output['frame'][metric_data['metric_count']-1]],[ metric_def['min_limit'],metric_def['min_limit']], 'r-', lw=2) # Red straight line
|
|
|
|
plt.title('Physical throughput ')
|
|
plt.xlabel('frame')
|
|
plt.ylabel(metric_def['id'])
|
|
plt.legend(prop=fontP)
|
|
mng = plt.get_current_fig_manager()
|
|
plt.show()
|
|
|
|
def do_img_metrics(metric_def, metric_data, fname):
|
|
|
|
|
|
if metric_data['metric_count'] > 0 :
|
|
|
|
# output = np.array( metric_data['metric_buf'] , [('id', 'S20'), ('metric', np.float), ('frame', np.int)])
|
|
output = metric_data['metric_buf']
|
|
# print 'TITI !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
|
|
# print output
|
|
|
|
# print metric_def['min_limit']
|
|
# print metric_data['metric_count']
|
|
# print output['frame'][metric_data['metric_count']-1]
|
|
|
|
fontP = FontProperties()
|
|
fontP.set_size('small')
|
|
|
|
plt.figure()
|
|
|
|
# print output['frame'].size
|
|
# print output['metric'].size
|
|
|
|
plt.scatter(output['frame'], output['metric'], color='b', alpha=0.33, s = 1 , label=metric_def['id'])
|
|
|
|
if 'min_limit' in metric_def:
|
|
plt.plot([0, output['frame'][metric_data['metric_count']-1]],[ metric_def['min_limit'],metric_def['min_limit']], 'r-', lw=2, label='min limit') # Red straight line
|
|
|
|
plt.title(metric_def['id'] +' ('+metric_def['unit_of_meas']+')')
|
|
plt.xlabel('frame')
|
|
plt.ylabel(metric_def['id'])
|
|
|
|
# Set graphic minimum Y axis
|
|
# -------------------------
|
|
if metric_data['metric_min'] < 0:
|
|
plt.ylim(ymin=metric_data['metric_min']+metric_data['metric_min']/10)
|
|
else :
|
|
plt.ylim(ymin=0)
|
|
|
|
y_axis_max = 0
|
|
if 'min_limit' in metric_def:
|
|
if metric_data['metric_max'] > metric_def['min_limit']:
|
|
y_axis_max =metric_data['metric_max']+metric_data['metric_max']/10
|
|
else:
|
|
y_axis_max =metric_def['min_limit']+metric_def['min_limit']/10
|
|
else:
|
|
y_axis_max =metric_data['metric_max']+metric_data['metric_max']/10
|
|
|
|
|
|
plt.ylim(ymax=y_axis_max)
|
|
|
|
lgd = plt.legend(prop=fontP, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
|
|
mng = plt.get_current_fig_manager()
|
|
plt.savefig(fname, bbox_extra_artists=(lgd,), bbox_inches='tight')
|
|
plt.close()
|
|
|
|
# with open(fname, 'r') as f:
|
|
# for line in f:
|
|
# m = re.search(metric['regex'], line)
|
|
# if m :
|
|
# print m.group(1) + " -> "+m.group(2)
|
|
|
|
|
|
|
|
|
|
def do_extract_traffic_metrics(args):
|
|
|
|
fname = args['file']
|
|
|
|
# print(fname)
|
|
# print 'metric id = ' + metric['id']
|
|
|
|
#[ ID] Interval Transfer Bandwidth Jitter Lost/Total Datagrams
|
|
#[ 3] 0.0- 1.0 sec 238 KBytes 1.95 Mbits/sec 0.980 ms 2/ 174 (1.1%)
|
|
# [ 3] 0.0- 1.0 sec 63.2 KBytes 517 Kbits/sec 0.146 ms 1/ 45 (2.2%)
|
|
|
|
# iperf_regex = '[\s*(\d+)]\s+\d+\.\d+-\s*(\d+\.\d+)\s+sec\s+(\d+).+(\d+\.\d+).+(\d+\.\d+).+(\d+)\/\s*(\d+)\s+\((\d+\.\d+)\%\)'
|
|
# ID 0.0 1.0 63.2 KByte 517 Kbits/s 0.146
|
|
iperf_regex = '\[\s*(\d+)\]\s+(\d+\.*\d*)-\s*(\d+\.*\d*)\s+sec\s+(\d+\.*\d*)\s+(\D+)\s+(\d+\.*\d*)\s+(\D+)\s+(\d+\.*\d*)\s+ms\s+(\d+)\/\s*(\d+)\s+\((\d+\.*\d*)\%\)'
|
|
# print 'iperf regex = ' + iperf_regex
|
|
|
|
count = 0
|
|
bw_min = 0
|
|
bw_max = 0
|
|
bw_mean = 0
|
|
bw_median = 0
|
|
jitter_min = 0
|
|
jitter_max = 0
|
|
jitter_mean = 0
|
|
jitter_median = 0
|
|
|
|
rl_min = 0
|
|
rl_max = 0
|
|
rl_mean = 0
|
|
rl_median = 0
|
|
|
|
interval_stop_max = 0
|
|
|
|
output = np.fromregex(fname,iperf_regex, [('id', np.int) , ('interval_start', np.float), ('interval_stop', np.float), ('transfer', np.float), ('transfer_uom', 'S20') ,('bandwidth', np.float), ('bandwidth_uom', 'S20') ,('jitter', np.float), ('lost', np.int) , ('total', np.int), ('rate_lost', np.float) ] )
|
|
|
|
count = output['id'].size -1
|
|
# remove last line that is an iperf result resume
|
|
|
|
if count > 0:
|
|
|
|
output= np.delete(output, (count), axis=0 )
|
|
# print output
|
|
|
|
bw_min = np.amin(output['bandwidth']);
|
|
bw_max = np.amax(output['bandwidth']);
|
|
bw_mean = np.mean(output['bandwidth']);
|
|
bw_median = np.median(output['bandwidth']);
|
|
|
|
jitter_min = np.amin(output['jitter']);
|
|
jitter_max = np.amax(output['jitter']);
|
|
jitter_mean = np.mean(output['jitter']);
|
|
jitter_median = np.median(output['jitter']);
|
|
|
|
rl_min = np.amin(output['rate_lost']);
|
|
rl_max = np.amax(output['rate_lost']);
|
|
rl_mean = np.mean(output['rate_lost']);
|
|
rl_median = np.median(output['rate_lost']);
|
|
|
|
interval_stop_max = np.amax(output['interval_stop']);
|
|
|
|
else :
|
|
count = 0
|
|
|
|
|
|
ret = { 'traffic_count' : count,
|
|
'traffic_buf' : output,
|
|
'bw_min' : bw_min,
|
|
'bw_max' : bw_max,
|
|
'bw_mean' : bw_mean,
|
|
'bw_median' : bw_median,
|
|
'jitter_min' : jitter_min,
|
|
'jitter_max' : jitter_max,
|
|
'jitter_mean' : jitter_mean,
|
|
'jitter_median' : jitter_median,
|
|
'rl_min' : rl_min,
|
|
'rl_max' : rl_max,
|
|
'rl_mean' : rl_mean,
|
|
'rl_median' : rl_median,
|
|
'interval_stop_max' : interval_stop_max
|
|
}
|
|
return(ret)
|
|
|
|
|
|
|
|
|
|
def do_img_traffic(traffic_data, fname):
|
|
|
|
|
|
if traffic_data['traffic_count'] > 0 :
|
|
|
|
output = traffic_data['traffic_buf']
|
|
|
|
|
|
fontP = FontProperties()
|
|
fontP.set_size('small')
|
|
|
|
fig = plt.figure(1)
|
|
|
|
ax1= plt.subplot(311)
|
|
plt.plot(output['interval_stop'], output['bandwidth'], color='b' )
|
|
ax1.set_title('Bandwidth (Mbits/s)')
|
|
ax1.set_ylim(ymin=-1)
|
|
ax1.set_xlim(xmax=np.amax(output['interval_stop']))
|
|
text='min: '+str(traffic_data['bw_min'])+'\nmax: '+str(traffic_data['bw_max'])+'\nmean: '+str(traffic_data['bw_mean'])+'\nmedian: '+str(traffic_data['bw_median'])
|
|
ax1.text( np.amax(output['interval_stop'])+10,0,text)
|
|
ax1.set_xlabel('time (s)')
|
|
ax1.set_ylabel(' ')
|
|
|
|
ax2=plt.subplot(312)
|
|
plt.plot(output['interval_stop'], output['jitter'], color='b' )
|
|
ax2.set_title('Jitter (ms)')
|
|
ax2.set_xlim(xmax=np.amax(output['interval_stop']))
|
|
ax2.set_ylim(ymin=-1)
|
|
text='min: '+str(traffic_data['jitter_min'])+'\nmax: '+str(traffic_data['jitter_max'])+'\nmean: '+str(traffic_data['jitter_mean'])+'\nmedian: '+str(traffic_data['jitter_median'])
|
|
ax2.text( np.amax(output['interval_stop'])+10,0,text)
|
|
ax2.set_xlabel('time (s)')
|
|
ax2.set_ylabel(' ')
|
|
|
|
ax3=plt.subplot(313)
|
|
plt.plot(output['interval_stop'], output['rate_lost'], color='b')
|
|
ax3.set_title('Loss rate %')
|
|
ax3.set_xlim(xmax=np.amax(output['interval_stop']))
|
|
ax3.set_ylim(ymin=-1)
|
|
text='min: '+str(traffic_data['rl_min'])+'\nmax: '+str(traffic_data['rl_max'])+'\nmean: '+str(traffic_data['rl_mean'])+'\nmedian: '+str(traffic_data['rl_median'])
|
|
ax3.text( np.amax(output['interval_stop'])+10,0,text)
|
|
ax3.set_xlabel('time (s)')
|
|
ax3.set_ylabel(' ')
|
|
|
|
# plt.title('Physical throughput ('+metric_def['unit_of_meas']+')')
|
|
# plt.xlabel('time (s)')
|
|
# plt.ylabel(metric_def['id'])
|
|
|
|
# Set graphic minimum Y axis
|
|
# -------------------------
|
|
# if traffic_data['bw_min'] == 0 :
|
|
# plt.ylim(ymin=-metric_def['min_limit']/10)
|
|
# else :
|
|
# plt.ylim(ymin=0)
|
|
|
|
# y_axis_max = 0
|
|
# if traffic_data['metric_max'] > metric_def['min_limit']:
|
|
# y_axis_max =traffic_data['metric_max']+traffic_data['metric_max']/10
|
|
# else:
|
|
# y_axis_max =metric_def['min_limit']+metric_def['min_limit']/10
|
|
#
|
|
# plt.ylim(ymax=y_axis_max)
|
|
|
|
plt.tight_layout()
|
|
|
|
# lgd = plt.legend(prop=fontP, bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
|
|
mng = plt.get_current_fig_manager()
|
|
plt.savefig(fname, bbox_inches='tight')
|
|
plt.close()
|
|
|
|
|
|
|
|
|
|
def create_report_html(context):
|
|
fname = context['report_path']+"/index.html"
|
|
|
|
#
|
|
with open(fname, 'w') as f:
|
|
html = render_template('index.html', context)
|
|
f.write(html)
|
|
|
|
|
|
def create_test_report_detailed_html(context, fname ):
|
|
with open(fname, 'w') as f:
|
|
html = render_template('testcase_report.html', context)
|
|
f.write(html)
|
|
|
|
|
|
def check_cell_synchro(fname):
|
|
|
|
with open(fname, 'r') as f:
|
|
for line in f:
|
|
|
|
m = re.search('AUTOTEST Cell Sync \:', line)
|
|
if m :
|
|
#print line
|
|
return 'CELL_SYNCH'
|
|
|
|
return 'CELL_NOT_SYNCH'
|
|
|
|
|
|
def check_exec_seg_fault(fname):
|
|
|
|
with open(fname, 'r') as f:
|
|
for line in f:
|
|
m = re.search('Segmentation fault', line)
|
|
if m :
|
|
#print line
|
|
return 'SEG_FAULT'
|
|
|
|
return 'NO_SEG_FAULT'
|
|
|