[Tracker-discuss] [issue544] Tracker stats
Ezio Melotti
metatracker at psf.upfronthosting.co.za
Thu May 15 16:40:15 CEST 2014
Ezio Melotti added the comment:
Here is the patch.
I plan to commit it in the next 24 hours, so we can see if it works fine with tomorrow weekly summary.
If it does I'll add links to the stats page somewhere in the sidebar.
_______________________________________________________
PSF Meta Tracker <metatracker at psf.upfronthosting.co.za>
<http://psf.upfronthosting.co.za/roundup/meta/issue544>
_______________________________________________________
-------------- next part --------------
diff --git a/html/issue.stats.html b/html/issue.stats.html
new file mode 100644
--- /dev/null
+++ b/html/issue.stats.html
@@ -0,0 +1,107 @@
+<tal:block metal:use-macro="templates/page/macros/icing">
+<title metal:fill-slot="head_title" >
+ <span tal:omit-tag="true" i18n:translate="" >Issues stats</span>
+ <span tal:condition="request/dispname"
+ tal:replace="python:' - %s '%request.dispname"
+ /> - <span tal:replace="config/TRACKER_NAME" />
+</title>
+
+<metal:slot fill-slot="more-javascript">
+<script type="text/javascript" src="http://cdn.jsdelivr.net/jquery/2.1.1/jquery.min.js"></script>
+<script type="text/javascript" src="http://cdn.jsdelivr.net/jqplot/1.0.8/jquery.jqplot.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.dateAxisRenderer.min.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.barRenderer.min.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.canvasTextRenderer.min.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.canvasAxisTickRenderer.min.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.categoryAxisRenderer.min.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.pointLabels.min.js"></script>
+
+<script type="text/javascript">
+function make_chart(id, title, series, type, labels, ticks) {
+ if (labels == null)
+ var legend = {show: false};
+ else
+ var legend = {show: true, location: 'nw', labels: labels};
+ if (type == 'line') {
+ var series_renderer = $.jqplot.LineRenderer;
+ var axis_renderer = $.jqplot.DateAxisRenderer;
+ var show_label = false;
+ }
+ else {
+ var series_renderer = $.jqplot.BarRenderer;
+ var axis_renderer = $.jqplot.CategoryAxisRenderer;
+ var show_label = true;
+ }
+ var plot = $.jqplot(id, series, {
+ title: title,
+ seriesColors: ["#3771a0", "#fcd449"],
+ seriesDefaults: {renderer:series_renderer, showMarker:false,
+ pointLabels: { show: show_label },
+ rendererOptions: {fillToZero: true}},
+ axes: {xaxis:{renderer:axis_renderer, ticks:ticks,
+ tickRenderer: $.jqplot.CanvasAxisTickRenderer,
+ tickOptions: {angle: -30}}},
+ legend: legend,
+ });
+}
+
+function zip(first, second) {
+ var res = [];
+ // assume same length
+ for (var k = 0; k < first.length; k++) {
+ res.push([first[k], second[k]]);
+ }
+ return res;
+}
+
+$(document).ready(function(){
+ $.getJSON("@@file/issue.stats.json", function(j) {
+ var dates = [];
+ for (var k = 0; k < j.timespan.length; k++) {
+ dates.push(j.timespan[k][1]);
+ }
+
+ var num = -25; // show only last 25 weeks
+
+ make_chart('open_patches', 'Open issues (total)',
+ [zip(dates, j.open), zip(dates, j.patches)],
+ 'line', ['Open issues', 'Open issues with patches']);
+ make_chart('open_deltas', 'Open issues deltas (weekly)',
+ [j.open_delta.slice(num)], 'bar', null, dates.slice(num));
+ make_chart('open_closed_week', 'Opened and Closed (weekly)',
+ [j.total_delta.slice(num), j.closed_delta.slice(num)],
+ 'bar', ['Opened', 'Closed'], dates.slice(num));
+ make_chart('closed_total', 'Closed and Total issues (total)',
+ [zip(dates, j.closed), zip(dates, j.total)],
+ 'line', ['Closed', 'Total']);
+ });
+});
+</script>
+<link rel="stylesheet" type="text/css" href="http://cdn.jsdelivr.net/jqplot/1.0.8/jquery.jqplot.css">
+</metal:slot>
+
+<span metal:fill-slot="body_title" tal:omit-tag="true">
+ <span tal:omit-tag="true" i18n:translate="" >Issues stats</span>
+ <span tal:condition="request/dispname"
+ tal:replace="python:' - %s' % request.dispname" />
+</span>
+<tal:block metal:fill-slot="content">
+
+<p>These charts are updated weekly. JavaScript must be enabled to see the charts.</p>
+
+<p>Total number of open issues and open issues with patches:</p>
+<div id="open_patches"></div>
+
+<p>Delta of open issues compared with the previous week. If the delta is positive,
+the total number of open issues increased; if negative, the total number decreased.</p>
+<div id="open_deltas"></div>
+
+<p>Number of issues that have been opened and closed during each week.
+The difference between these two values is shown in the previous graph.</p>
+<div id="open_closed_week"></div>
+
+<p>The number of closed issues, and the number of issues regardless of their status:</p>
+<div id="closed_total"></div>
+
+</tal:block>
+</tal:block>
diff --git a/scripts/issuestats.py b/scripts/issuestats.py
new file mode 100644
--- /dev/null
+++ b/scripts/issuestats.py
@@ -0,0 +1,206 @@
+# Search for the weekly summary reports from bugs.python.org in the
+# python-dev archives and plot the result.
+#
+# $ issuestats.py collect
+#
+# Collects statistics from the mailing list and saves to
+# issue.stats.json
+#
+# $ issuestats.py plot
+#
+# Written by Ezio Melotti.
+# Based on the work of Petri Lehtinen (https://gist.github.com/akheron/2723809).
+#
+
+
+import os
+import re
+import sys
+import json
+import gzip
+import errno
+import argparse
+import datetime
+import tempfile
+import webbrowser
+import urllib.parse
+import urllib.request
+
+from collections import defaultdict
+
+MONTH_NAMES = [datetime.date(2012, n, 1).strftime('%B') for n in range(1, 13)]
+ARCHIVE_URL = 'http://mail.python.org/pipermail/python-dev/%s'
+
+STARTYEAR = 2011
+STARTMONTH = 1 # February
+
+NOW = datetime.date.today()
+ENDYEAR = NOW.year
+ENDMONTH = NOW.month
+
+STATISTICS_FILENAME = 'issue.stats.json'
+
+activity_re = re.compile('ACTIVITY SUMMARY \((\d{4}-\d\d-\d\d) - '
+ '(\d{4}-\d\d-\d\d)\)')
+count_re = re.compile('\s+(open|closed|total)\s+(\d+)\s+\(([^)]+)\)')
+patches_re = re.compile('Open issues with patches: (\d+)')
+
+def find_statistics(source):
+ print(source)
+ monthly_data = {}
+ with gzip.open(source) as file:
+ parsing = False
+ for line in file:
+ line = line.decode('utf-8')
+ if not parsing:
+ m = activity_re.match(line)
+ if not m:
+ continue
+ start_end = m.groups()
+ if start_end in monthly_data:
+ continue
+ monthly_data[start_end] = weekly_data = {}
+ parsing = True
+ continue
+ m = count_re.match(line)
+ if parsing and m:
+ type, count, delta = m.groups()
+ weekly_data[type] = int(count)
+ weekly_data[type + '_delta'] = int(delta)
+ m = patches_re.match(line)
+ if parsing and m:
+ weekly_data['patches'] = int(m.group(1))
+ parsing = False
+ print(' ', len(monthly_data), 'reports found')
+ return monthly_data
+
+
+
+def collect_data():
+ try:
+ os.mkdir('cache')
+ except OSError as exc:
+ if exc.errno != errno.EEXIST:
+ raise
+
+ statistics = {}
+
+ for year in range(STARTYEAR, ENDYEAR + 1):
+ # Assume STARTYEAR != ENDYEAR
+ if year == STARTYEAR:
+ month_range = range(STARTMONTH, 12)
+ elif year == ENDYEAR:
+ month_range = range(0, ENDMONTH)
+ else:
+ month_range = range(12)
+
+ for month in month_range:
+ prefix = '%04d-%s' % (year, MONTH_NAMES[month])
+
+ archive = prefix + '.txt.gz'
+ archive_path = os.path.join('cache', archive)
+
+ if not os.path.exists(archive_path):
+ print('Downloading %s' % archive)
+ url = ARCHIVE_URL % urllib.parse.quote(archive)
+ urllib.request.urlretrieve(url, archive_path)
+
+
+ print('Processing %s' % prefix)
+ statistics.update(find_statistics(archive_path))
+
+
+ statistics2 = defaultdict(list)
+ for key, val in sorted(statistics.items()):
+ statistics2['timespan'].append(key)
+ for k2, v2 in val.items():
+ statistics2[k2].append(v2)
+
+ with open(STATISTICS_FILENAME, 'w') as fobj:
+ json.dump(statistics2, fobj)
+
+ print('Now run "plot".')
+
+HTML = """<!DOCTYPE html>
+<html>
+<head>
+<script type="text/javascript" src="http://cdn.jsdelivr.net/jquery/2.1.1/jquery.min.js"></script>
+<script type="text/javascript" src="http://cdn.jsdelivr.net/jqplot/1.0.8/jquery.jqplot.js"></script>
+<script type="text/javascript" src="http://cdnjs.cloudflare.com/ajax/libs/jqPlot/1.0.8/plugins/jqplot.dateAxisRenderer.min.js"></script>
+<script type="text/javascript">
+function make_chart(id, title, dates, values) {
+ var data = [];
+ for (var k = 0; k < dates.length; k++) {
+ data.push([dates[k], values[k]]);
+ }
+ var plot = $.jqplot(id, [data], {
+ title: title,
+ series: [{showMarker:false}],
+ axes: {xaxis:{renderer:$.jqplot.DateAxisRenderer}},
+ });
+}
+$(document).ready(function(){
+ var j = %s;
+ var dates = [];
+ for (var k = 0; k < j.timespan.length; k++) {
+ console.log(j.timespan[k][1]);
+ dates.push(j.timespan[k][1]);
+ }
+ console.log(dates);
+ var open = [];
+ for (var k = 0; k < dates.length; k++) {
+ open.push([dates[k], j.open[k]]);
+ }
+ console.log(open);
+ make_chart('open', 'Open issues', dates, j.open);
+ make_chart('open_delta', 'Open issues (deltas)', dates, j.open_delta);
+ make_chart('open_week', 'Opened per week', dates, j.wopened);
+ make_chart('closed', 'Closed issues', dates, j.closed);
+ make_chart('closed_delta', 'Closed issues (deltas)', dates, j.closed_delta);
+ make_chart('closed_week', 'Closed per week', dates, j.wclosed);
+ make_chart('total', 'Total issues', dates, j.total);
+ make_chart('total_delta', 'Total issues (deltas)', dates, j.total_delta);
+ make_chart('patches', 'Open issues with patches', dates, j.patches);
+});
+</script>
+<link rel="stylesheet" type="text/css" href="http://cdn.jsdelivr.net/jqplot/1.0.8/jquery.jqplot.css">
+<style type="text/css">div { margin-bottom: 1em; }</style>
+</head>
+<body>
+<div id="open"></div>
+<div id="open_delta"></div>
+<div id="open_week"></div>
+<div id="closed"></div>
+<div id="closed_delta"></div>
+<div id="closed_week"></div>
+<div id="total"></div>
+<div id="total_delta"></div>
+<div id="patches"></div>
+</body>
+</html>
+"""
+def plot_statistics():
+ try:
+ with open(STATISTICS_FILENAME) as j:
+ json = j.read()
+ except FileNotFoundError:
+ sys.exit('You need to run "collect" first.')
+ with tempfile.NamedTemporaryFile('w', delete=False) as tf:
+ tf.write(HTML % json)
+ webbrowser.open(tf.name)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('command', choices=['collect', 'plot'])
+
+ args = parser.parse_args()
+
+ if args.command == 'collect':
+ collect_data()
+ elif args.command == 'plot':
+ plot_statistics()
+
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
diff --git a/scripts/roundup-summary b/scripts/roundup-summary
--- a/scripts/roundup-summary
+++ b/scripts/roundup-summary
@@ -32,6 +32,8 @@
Print journal for all the transactions in the given
date range.
-D, --DEBUG Print email content without sending it if -m is used.
+ --update-stats-file=FILENAME
+ Append tracker stats to JSON file FILENAME.
"""
# This script has a class (Report) that filters the issues and generates txt
@@ -44,6 +46,8 @@
#sys.path.insert(1, '/opt/tracker-roundup/lib/python2.6/site-packages/')
import cgi
+import json
+import os.path
import optparse
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
@@ -131,6 +135,10 @@
advanced.add_option(
'-D', '--DEBUG', dest='debug', action='store_true', default=False,
help='Print email content without sending it if -m is used.')
+ advanced.add_option(
+ '--update-stats-file', dest='update_stats',
+ metavar='FILENAME', default='issue.stats.json',
+ help='Append tracker stats to JSON file FILENAME.')
parser.add_option_group(advanced)
# Get the command line args:
@@ -712,6 +720,26 @@
smtp.sendmail(config.ADMIN_EMAIL, recipient, msg.as_string())
+def update_stats_file(stats, filename):
+ stats = dict(stats) # make a copy
+ stats_file = os.path.join(instance_home, 'html', OPTIONS.update_stats)
+ try:
+ with open(stats_file) as fi:
+ j = json.load(fi)
+ except IOError:
+ j = dict(open=[], closed=[], total=[], timespan=[], patches=[],
+ open_delta=[], closed_delta=[], total_delta=[])
+ timespan = stats.pop('timespan').split(' - ')
+ if timespan in j['timespan']:
+ return # we already updated the file today
+ j['timespan'].append(timespan)
+ for k, v in stats.items():
+ if k in j:
+ j[k].append(v)
+ with open(stats_file, 'w') as fo:
+ json.dump(j, fo)
+
+
def main():
"""Create the report and print or send it."""
issue_attrs = issues_map()
@@ -727,6 +755,8 @@
for recipient in OPTIONS.mailTo.split(','):
send_report(recipient, txt_report, html_report)
+ if OPTIONS.update_stats:
+ update_stats_file(report.header_content, OPTIONS.update_stats)
if __name__ == '__main__':
More information about the Tracker-discuss
mailing list