1# coding=utf-8 2""" 3Copyright 2013 LinkedIn Corp. All rights reserved. 4 5Licensed under the Apache License, Version 2.0 (the "License"); 6you may not use this file except in compliance with the License. 7You may obtain a copy of the License at 8 9 http://www.apache.org/licenses/LICENSE-2.0 10 11Unless required by applicable law or agreed to in writing, software 12distributed under the License is distributed on an "AS IS" BASIS, 13WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14See the License for the specific language governing permissions and 15limitations under the License. 16""" 17 18from jinja2 import Environment, FileSystemLoader 19import logging 20import os 21import shutil 22import naarad.utils 23import naarad.naarad_constants as CONSTANTS 24import naarad.resources 25 26logger = logging.getLogger('naarad.reporting.Report') 27 28 29class Report(object): 30 31 def __init__(self, report_name, output_directory, resource_directory, resource_path, metric_list, correlated_plots, **other_options): 32 if report_name == '': 33 self.report_name = CONSTANTS.DEFAULT_REPORT_TITLE 34 else: 35 self.report_name = report_name 36 self.output_directory = output_directory 37 self.resource_directory = resource_directory 38 self.resource_path = resource_path 39 self.metric_list = metric_list 40 self.correlated_plots = correlated_plots 41 self.stylesheet_includes = CONSTANTS.STYLESHEET_INCLUDES 42 self.javascript_includes = CONSTANTS.JAVASCRIPT_INCLUDES 43 self.header_template = CONSTANTS.TEMPLATE_HEADER 44 self.footer_template = CONSTANTS.TEMPLATE_FOOTER 45 self.summary_content_template = CONSTANTS.TEMPLATE_SUMMARY_CONTENT 46 self.summary_page_template = CONSTANTS.TEMPLATE_SUMMARY_PAGE 47 self.metric_page_template = CONSTANTS.TEMPLATE_METRIC_PAGE 48 self.client_charting_template = CONSTANTS.TEMPLATE_CLIENT_CHARTING 49 self.diff_client_charting_template = CONSTANTS.TEMPLATE_DIFF_CLIENT_CHARTING 50 self.diff_page_template = CONSTANTS.TEMPLATE_DIFF_PAGE 51 if other_options: 52 for (key, val) in other_options.iteritems(): 53 setattr(self, key, val) 54 55 def copy_local_includes(self): 56 resource_folder = self.get_resources_location() 57 for stylesheet in self.stylesheet_includes: 58 if ('http' not in stylesheet) and naarad.utils.is_valid_file(os.path.join(resource_folder, stylesheet)): 59 shutil.copy(os.path.join(resource_folder, stylesheet), self.resource_directory) 60 61 for javascript in self.javascript_includes: 62 if ('http' not in javascript) and naarad.utils.is_valid_file(os.path.join(resource_folder, javascript)): 63 shutil.copy(os.path.join(resource_folder, javascript), self.resource_directory) 64 65 return None 66 67 def get_summary_table(self, summary_stats_file): 68 summary_stats = [] 69 with open(summary_stats_file, 'r') as stats_file: 70 header = stats_file.readline().rstrip().rsplit(',') 71 summary_stats.append(header) 72 for line in stats_file: 73 summary_stats.append(line.rstrip().rsplit(',')) 74 return summary_stats 75 76 def is_correlated_image(self, image): 77 if os.path.basename(image) in self.correlated_plots: 78 return True 79 else: 80 return False 81 82 def validate_file_list(self, files_list): 83 for file_name in files_list: 84 if naarad.utils.is_valid_file(file_name): 85 return True 86 return False 87 88 def enable_summary_tab(self): 89 for metric in self.metric_list: 90 for stats_file in metric.important_stats_files: 91 if naarad.utils.is_valid_file(stats_file): 92 return True 93 if metric.status == CONSTANTS.SLA_FAILED: 94 return True 95 if self.validate_file_list(self.correlated_plots): 96 return True 97 return False 98 99 def generate_summary_page(self, template_environment, summary_html_content, coplot_html_content, header_template_data): 100 summary_html = template_environment.get_template(self.header_template).render(**header_template_data) + '\n' 101 summary_html += template_environment.get_template(self.summary_page_template).render(metric_list=sorted(self.metric_list), 102 summary_html_content=summary_html_content, 103 overlaid_plot_content=coplot_html_content) + '\n' 104 summary_html += template_environment.get_template(self.footer_template).render() 105 return summary_html 106 107 def strip_file_extension(self, file_name): 108 filename = file_name.split('.') 109 return '.'.join(filename[0:-1]) 110 111 def generate_client_charting_page(self, template_environment, timeseries_csv_list, percentiles_csv_list, summary_enabled, header_template_data): 112 client_charting_html = template_environment.get_template(self.header_template).render(**header_template_data) + '\n' 113 client_charting_html += template_environment.get_template(self.client_charting_template).render(metric_list=sorted(self.metric_list), 114 timeseries_data=sorted(timeseries_csv_list), 115 percentiles_data=sorted(percentiles_csv_list), 116 summary_enabled=summary_enabled, 117 resource_path=self.resource_path) + '\n' 118 client_charting_html += template_environment.get_template(self.footer_template).render() 119 with open(os.path.join(self.resource_directory, CONSTANTS.PLOTS_CSV_LIST_FILE), 'w') as FH: 120 FH.write(','.join(sorted(timeseries_csv_list))) 121 with open(os.path.join(self.resource_directory, CONSTANTS.CDF_PLOTS_CSV_LIST_FILE), 'w') as FH: 122 FH.write(','.join(sorted(percentiles_csv_list))) 123 return client_charting_html 124 125 def get_resources_location(self): 126 return naarad.resources.get_dir() 127 128 def generate(self): 129 template_loader = FileSystemLoader(self.get_resources_location()) 130 self.copy_local_includes() 131 template_environment = Environment(loader=template_loader) 132 template_environment.filters['sanitize_string'] = naarad.utils.sanitize_string 133 header_template_data = { 134 'custom_stylesheet_includes': CONSTANTS.STYLESHEET_INCLUDES, 135 'custom_javascript_includes': CONSTANTS.JAVASCRIPT_INCLUDES, 136 'resource_path': self.resource_path 137 } 138 summary_html_content = '' 139 coplot_html_content = '' 140 summary_enabled = self.enable_summary_tab() 141 timeseries_csv_list = [] 142 percentiles_csv_list = [] 143 stats_files = [] 144 metrics_in_error = [] 145 146 # if a metric has no csv_files associated with it, assume something failed upstream and skip metric from report. 147 for metric in self.metric_list: 148 if len(metric.csv_files) == 0: 149 metrics_in_error.append(metric) 150 self.metric_list = set(self.metric_list) - set(metrics_in_error) 151 152 for metric in self.metric_list: 153 timeseries_csv_list.extend(map(self.strip_file_extension, map(os.path.basename, metric.csv_files))) 154 percentiles_csv_list.extend(map(self.strip_file_extension, map(os.path.basename, metric.percentiles_files))) 155 metric_html = '' 156 div_html = '' 157 for plot_div in sorted(metric.plot_files): 158 with open(plot_div, 'r') as div_file: 159 div_html += '\n' + div_file.read() 160 if os.path.basename(plot_div) in metric.summary_charts: 161 div_file.seek(0) 162 coplot_html_content += '\n' + div_file.read() 163 164 if metric.summary_html_content_enabled: 165 for summary_stats_file in metric.important_stats_files: 166 if naarad.utils.is_valid_file(summary_stats_file): 167 summary_stats = self.get_summary_table(summary_stats_file) 168 summary_html_content += template_environment.get_template(self.summary_content_template).render(metric_stats=summary_stats, metric=metric) + '\n' 169 170 for metric_stats_file in metric.stats_files: 171 metric_template_data = {} 172 if naarad.utils.is_valid_file(metric_stats_file) or len(metric.plot_files) > 0: 173 stats_files.append(os.path.basename(metric_stats_file)) 174 metric_html = template_environment.get_template(self.header_template).render(**header_template_data) 175 metric_template_data = {'plot_div_content': div_html, 176 'metric': metric, 177 'metric_list': sorted(self.metric_list), 178 'summary_enabled': summary_enabled} 179 metric_html += template_environment.get_template(self.metric_page_template).render(**metric_template_data) 180 metric_html += template_environment.get_template(self.footer_template).render() 181 if metric_html != '': 182 with open(os.path.join(self.output_directory, metric.label + CONSTANTS.METRIC_REPORT_SUFFIX), 'w') as metric_report: 183 metric_report.write(metric_html) 184 185 for coplot in self.correlated_plots: 186 with open(coplot, 'r') as coplot_file: 187 coplot_html_content += coplot_file.read() 188 189 if summary_enabled: 190 with open(os.path.join(self.output_directory, CONSTANTS.SUMMARY_REPORT_FILE), 'w') as summary_report: 191 summary_report.write(self.generate_summary_page(template_environment, summary_html_content, coplot_html_content, header_template_data)) 192 193 with open(os.path.join(self.output_directory, CONSTANTS.CLIENT_CHARTING_FILE), 'w') as client_charting_report: 194 client_charting_report.write(self.generate_client_charting_page(template_environment, timeseries_csv_list, percentiles_csv_list, summary_enabled, 195 header_template_data)) 196 197 if len(stats_files) > 0: 198 with open(os.path.join(self.resource_directory, CONSTANTS.STATS_CSV_LIST_FILE), 'w') as stats_file: 199 stats_file.write(','.join(stats_files)) 200 201 return True 202