contrib/stats.py

#!/usr/bin/env python
import logging

from sqlalchemy import sql

import swiss.tabular as T

import pdw
import pdw.model as model
import pdw.stats

PREFIX = None
OUTDIR = None

def _fn(offset):
    path = os.path.join(OUTIDIR, '%s_%s' % (PREFIX, offset))
    return path

def num_works_with_items():
    q = model.work_2_item.count()
    out = q.execute().fetchall()[0][0]
    print out

def death_date_in_future():
    # ptab = model.person_table
    ptab = model.Person
    q = model.Person.query
    q = q.filter(sql.or_(
        ptab.death_date_ordered>2015,
        ptab.birth_date_ordered>2000,
        ))
    total = q.count()
    print '## Total: %s' % total
    q = q.limit(1000)
    out = q.all()
    for person in out:
        print person

def recent_pd():
    ptab = model.Person
    q = model.Person.query
    q = q.filter(ptab.death_date_ordered>1930).\
            filter(ptab.death_date_ordered<1938)
    total = q.count()
    print '## Total: %s' % total
    q = q.limit(50)
    out = q.all()
    for person in out:
        print person

def _entry(data, total):
    out ='%s (%i%%)' % (data, 100*float(data)/total)
    return out

def pdrel_by_decade():
    stats = pdw.stats.Stats.default()
    pdrel = stats.pdrel_for_item_by_decade()
    td = T.TabularData()
    td.header = [ u'Pub. Date', u'Total', u'No Author', u'Any Date', 'Death Date' ]
    for years in sorted(pdrel.keys()):
        data = pdrel[years]
        total = data['total']
        row = [ years, total,
                _entry(data['no_person'], total),
                _entry(data['any_date'], total),
                _entry(data['death_date'], total),
                ]
        td.data.append(row)
    writer = T.LatexWriter()
    # writer = T.HtmlWriter(pretty_print=True)
    out = writer.write_str(td)
    print out

def direct_pd():
    stats = pdw.stats.Stats.default()
    directpd = stats.fast_pd_for_item_by_decade()
    pdrel = stats.pdrel_for_item_by_decade()
    td = T.TabularData()
    td.header = [ u'Pub. Date', u'Total', u'PD', u'Not PD', '?',
        'Prop 1', 'Prop 2'
        ]
    for years in sorted(directpd.keys()):
        data = directpd[years]
        total = data['total']
        pd, not_pd = data['pd'], data['not_pd']
        row = [ years, total,
                _entry(pd, total),
                _entry(not_pd, total),
                _entry(data['undetermined'], total),
                '%i%%' % (100 * float(pd)/(pd+not_pd)),
                '%i%%' % (100 * float(pd) / pdrel[years]['any_date'])
                ]
        td.data.append(row)
    writer = T.LatexWriter()
    # writer = T.HtmlWriter(pretty_print=True)
    out = writer.write_str(td)
    print out

def work_item_counts():
    sqltext = '''SELECT work_id, count(*) from work_2_item
    group by work_id
    order by count(*)
    '''
    engine = model.metadata.bind
    out = engine.execute(sql.text(sqltext)).fetchall()
    print out[:10]

'''Can we afford to cache?
'''
def person_work_and_item_counts():
    cutoff = 500
    sqltext = '''SELECT count(*), person.name, person.id FROM item_2_person
    JOIN person ON item_2_person.person_id = person.id
    GROUP BY person.id, person.name
    ORDER BY count(*) DESC
    '''
    engine = model.metadata.bind
    out = engine.execute(sql.text(sqltext)).fetchall()
    td = T.TabularData(header=['Rank', 'No. of Items', 'Name'])
    bad_names = [ 'Great Britain. Parliament.',
            'Great Britain.',
            'England and Wales',
            'Church of England.',
            'England and Wales. Sovereign (1660-1685 : Charles II)',
            'Catholic Church.'
            ]
    # top 50 people
    td.data = [ list(x[:2]) for x in out[:200] ] # if x[1] not in bad_names ]
    td.data = [ [ii+1] + x for ii,x in enumerate(td.data) ]
    td.data = td.data[:50]
    # writer = T.TxtWriter()
    writer = T.HtmlWriter(pretty_print=True)
    print writer.write_str(td)

    counts = [ x[0] for x in out ]
    import matplotlib.pyplot as plt
    import scipy as S
    countslog = S.log(counts)
    params = { 'color': 'red', 'alpha': 0.7 }
    lparams = { 'color': 'black', 'linewidth': 2 }
    # plt.bar(range(cutoff), counts[:cutoff], **params)
    plt.plot(range(cutoff), counts[:cutoff], 'k-', **lparams)
    plt.xlabel('Rank')
    plt.ylabel('Counts')
    plt.savefig(_fn('person-item-by-rank.png'))
    plt.clf()

    plt.semilogy(range(cutoff), counts[:cutoff], **lparams)
    plt.xlabel('Rank')
    plt.ylabel('Counts')
    plt.savefig(_fn('person-item-by-rank-logy.png'))
    plt.clf()

    plt.semilogy(range(cutoff), countslog[:cutoff], **lparams)
    plt.ylabel('Log Counts (log)')
    plt.xlabel('Rank')
    plt.savefig(_fn('person-item-by-rank-2logy.png'))
    plt.clf()

    plt.semilogy(S.log(list(range(1,cutoff+1))), counts[:cutoff], **lparams)
    plt.ylabel('Counts (log)')
    plt.xlabel('Log Rank')
    plt.savefig(_fn('person-item-by-rank-logxlogy.png'))
    plt.clf()

    params = { 'fc': 'red', 'alpha': 0.7 }
    plt.hist(countslog, bins=100, **params)
    plt.xlabel('Log Counts')
    plt.ylabel('Freq')
    plt.savefig(_fn('person-item-hist-logx.png'))
    plt.clf()

    plt.hist(countslog, bins=100, log=True, **params)
    plt.xlabel('Log Counts')
    plt.ylabel('Freq (Log)')
    plt.savefig(_fn('person-item-hist-logxlogy.png'))

def person_item_counts(name):
    sqltext = '''SELECT count(*), person.name, person.id FROM item_2_person
    JOIN person ON item_2_person.person_id = person.id
    WHERE person.name LIKE '%s'
    GROUP BY person.id, person.name
    ORDER BY count(*) DESC
    ''' % ('%%'+name+'%%')
    engine = model.metadata.bind
    print sqltext
    out = engine.execute(sql.text(sqltext)).fetchall()
    for r in out:
        print r

def plot_pd_over_time():
    stats = pdw.stats.Stats.default()
    out = stats.fast_pd_year()
    totals = stats.by_year_all()['item']
    import matplotlib.pyplot as plt
    years, counts = zip(*out)
    tyears, tcounts = zip(*totals)
    plt.bar(tyears, tcounts, color='blue') # , edgecolor='blue')
    plt.bar(years, counts, color='red', edgecolor='red')
    plt.xlim(xmin=1600, xmax=1960)
    plt.savefig(_fn('pd-over-time.png'))


def into_pd():
    import pdw.model as model
    now = 2010
    # have to have died before jan 1st 70 years ago
    # so before now-70
    entering_pd = model.Person.query.\
            filter(model.Person.death_date_ordered<now-70).\
            filter(model.Person.death_date_ordered>=now-70-1)
    people = entering_pd.all()
    for person in people[:10]:
        print person.name, person.birth_date, person.death_date


import optparse
import os
import types
import sys
if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    local_methods = dict(locals())
    local_methods = [ x for x in local_methods if
            isinstance(local_methods[x], types.FunctionType) ]
    local_methods = filter(lambda x: not x.startswith('_'), local_methods)

    usage = '''%prog {action}

    '''
    usage += ('\n    '.join(local_methods))
    parser = optparse.OptionParser(usage)
    parser.add_option('-c', '--config')
    options, args = parser.parse_args()

    if not args or not options.config or not args[0] in local_methods:
        parser.print_help()
        sys.exit(1)

    method = args[0]
    cfg = os.path.abspath(options.config)
    pdw.load_config(cfg)
    PREFIX = pdw.default_prefix()
    OUTIDIR = pdw.default_outdir()

    locals()[method](*args[1:])