diff options
| -rw-r--r-- | scripts/logreport.py | 58 |
1 files changed, 20 insertions, 38 deletions
diff --git a/scripts/logreport.py b/scripts/logreport.py index cdde7c7..5fc1cef 100644 --- a/scripts/logreport.py +++ b/scripts/logreport.py @@ -6,6 +6,7 @@ import functools import itertools import math import operator +import os import psycopg2 import re import sys @@ -14,6 +15,7 @@ import time from writer import ConsoleWriter, HtmlWriter Nickset = None +MaxRequestTime = 10 SlowReqThreshold = 1000 WindowInterval = datetime.timedelta(minutes=5) ConnStr = "dbname=dumpfm user=postgres password=root" @@ -67,7 +69,6 @@ def median(pool): def mean(pool): return sum(pool) / len(pool) - def percentile(N, percent, key=lambda x:x): """ From http://code.activestate.com/recipes/511478-finding-the-percentile-of-the-values/ @@ -93,37 +94,13 @@ def percentile(N, percent, key=lambda x:x): def ninety_percentile(N): return percentile(N, .9) -def gather_by_key(rs, key): - d = collections.defaultdict(list) - for r in rs: - group = r[key] - d[group].append(r) - return d - -def group_by_time_interval(xs, interval, timekey): - if len(xs) == 0: - raise ValueError("Cannot group empty list") - start = xs[0][timekey] - end = xs[-1][timekey] - interval_seconds = int(interval.seconds) - span = int((end - start).seconds) / interval_seconds + 1 - res = [[] for i in range(span)] - - for x in xs: - idx = int((x[timekey] - start).seconds) / interval_seconds - res[idx].append(x) - return res - -def get_report_title(entries): - start_time = entries[0]['ts'] - end_time = entries[-1]['ts'] - return "Dump Report: " + format_time(start_time) + " to " + format_time(end_time) - -def build_js_array(pairs): +def build_js_array(pairs, max_val=None): s_list = [] for ts, val in pairs: unix_time = calendar.timegm(ts.timetuple()) * 1000 - s_list.append('[%s, %s]' % (unix_time, val)) + # Use max-val to ensure that + maxed_val = max(max_val, val) if max_val else val + s_list.append('[%s, %s]' % (unix_time, maxed_val)) return '[%s]' % ','.join(s_list) def summarize_window(ts, window, slow_threshold): @@ -250,6 +227,7 @@ class DumpReport(object): def _output_urlgroups(self, writer): + writer.section('URL Groups') headers = ['#', 'min', 'mean', 'max', '# slow', '% slow', 'bytes', 'bytes / req'] data = [] for group in sorted(self.urlgroup_stats.keys()): @@ -270,7 +248,7 @@ class DumpReport(object): def _output_usage_charts(self, writer): writer.section('Activity Rates') writer.graph('Reqs/sec', 'reqs_per_sec', - build_js_array((r['ts'], r['reqs'] / self.interval.seconds) for r in self.windows)) + build_js_array((r['ts'], r['images'] / self.interval.seconds) for r in self.windows)) writer.graph('Images/sec', 'images_per_sec', build_js_array((r['ts'], r['images'] / self.interval.seconds) for r in self.windows)) writer.graph('Dumps/sec', 'dumps_per_sec', @@ -280,18 +258,18 @@ class DumpReport(object): writer.graph('Chat users/sec', 'users_per_sec', build_js_array((r['ts'], r['chat_users'] / self.interval.seconds) for r in self.windows)) + resp_max_val = 10000 # resp_max_val keeps graphs readable even with large outliers writer.section('Response Rates') writer.graph('Mean response time', 'mean_resp', - build_js_array((r['ts'], r['mean_resp']) for r in self.windows)) + build_js_array(((r['ts'], r['mean_resp']) for r in self.windows), resp_max_val)) writer.graph('Median response time', 'median_resp', - build_js_array((r['ts'], r['median_resp']) for r in self.windows)) + build_js_array(((r['ts'], r['median_resp']) for r in self.windows), resp_max_val)) writer.graph('90% response time', 'ninety_resp', - build_js_array((r['ts'], r['ninety_resp']) for r in self.windows)) + build_js_array(((r['ts'], r['ninety_resp']) for r in self.windows), resp_max_val)) writer.graph('Max response time', 'max_resp', build_js_array((r['ts'], r['max_resp']) for r in self.windows)) - def _output_report(self, writer): writer.section('Overview') duration = self.end_time - self.start_time @@ -300,7 +278,7 @@ class DumpReport(object): format_time(self.end_time), duration), '%s requests (%.02f r/s)' % (comma_format(self.requests), - self.requests / duration.seconds), + float(self.requests) / total_seconds(duration)), '%s slow requests (%s)' % (comma_format(self.slow_requests), format_pct(float(self.slow_requests) / self.requests)), '%s bytes sent' % format_bytes(self.bytes_sent)] @@ -309,8 +287,12 @@ class DumpReport(object): self._output_urlgroups(writer) self._output_errors_and_unclassified(writer) - def write_report(self, path): - writer = HtmlWriter(path, 'Dump Report') + def write_report(self, outdir): + page_title = 'Dump Report: %s to %s' % (format_time(self.start_time), + format_time(self.end_time)) + filename = self.start_time.strftime('%Y-%m-%d__%H-%M.html') + path = os.path.join(outdir, filename) + writer = HtmlWriter(path, page_title) self._output_report(writer) writer.close() @@ -486,7 +468,7 @@ def analyze_dump_log(path, if __name__ == "__main__": if len(sys.argv) == 1: - print "usage: logreport.py logfile [outfile]" + print "usage: logreport.py logfile [outdir]" sys.exit(1) logpath = sys.argv[1] outpath = sys.argv[2] if len(sys.argv) > 2 else None |