Changeset 169

Timestamp:

07/19/08 18:43:05 (12 months ago)

Author:

rgrp

Message:

[shakespeare/search][l]: create search package with SearchIndex? object + cli interface as first part of integration of xapian search support developed in Iain (next part is WUI).

* search.py (+ test): SearchIndex? class with add_item, search, default_index methods.

* paste_deploy_config.ini_tmpl: new config variable search_index_dir

* cli.py: search methods (search_add, search)

• also support for verbose option

* init.py: refactor main docs (in docstring)

• make clearer by having separate 'getting started' sections for users and developers plus remove mention
• remove mention of concordanc and put in todo about search index

Location:

trunk

Files:

• shakespeare.egg-info/paste_deploy_config.ini_tmpl (modified) (1 diff)
• shakespeare/__init__.py (modified) (3 diffs)
• shakespeare/cli.py (modified) (3 diffs)
• shakespeare/search.py (copied) (copied from milton/textsearch.py) (1 diff)
• shakespeare/tests/search_test.py (added)

trunk/shakespeare.egg-info/paste_deploy_config.ini_tmpl

@@ -17 +17 @@
 # 
 # At present should be different from the app's cache_dir
-cachedir = ./cache
+cachedir = cache
+
+# Directory for Xapian search index
+search_index_dir = searchindex
+
 
 [server:main]

trunk/shakespeare/__init__.py

@@ -56 +56 @@
 
 
-2. Setup Package
-================
+Getting Started
+***************
 
-Make a config file as follows::
+As a user:
+==========
 
-    paster make-config shakespeare config.ini
+1. Basic setup
+--------------
+
+To access most of the main features of Open Shakespeare you need a database.
+For this an other bits and bobs of configuration you will need a configuration
+file.
+
+You can make a config file as follows::
+
+    paster make-config shakespeare {your-config.ini}
 
 Tweak the config file as appropriate and then setup the application::
 
     paster setup-app config.ini
-
-
-3. Initialize the system
-========================
+ 
+[TODO: this should be part of setup-app]
 
 Run::
@@ -76 +84 @@
     $ shakespeare-admin db init
 
-If you want to build the concordance do::
+2. Extras
+---------
 
-    $ shakespeare-admin concordance
+1. Search index. [TODO]
 
-NB: This may take some time to run so be patient. TIP: using sqlite building
-the concordance really **does** seem to run forever so recommend using
-postgresql or mysql if you are going to build the concordance. 
-
-
-Getting Started
-***************
-
-As a user:
-==========
-
-Start up the web interface by running the webserver::
+2. You can start a web server to provide a easy-to-use web interface to the
+shakespeare material and facilities by doing::
 
     $ paster serve {your-config.ini}
@@ -102 +101 @@
 ===============
 
-0. Copy development.ini.tmpl to development.ini and edit to your taste.
+0. Setup
+--------
 
-1. Check out the administrative commands: $ bin/shakespeare-admin help.
+Follow the basic steps above put with an ini file named: development.ini
+
+NB: you'll probably want to change log levels to debug.
+
+1. Check out the administrative commands
+----------------------------------------
+
+    $ bin/shakespeare-admin help.
 
 2. Run the tests using either py.test of nosetests::
+----------------------------------------------------
 
     $ nosetests shakespeare

trunk/shakespeare/cli.py

@@ -9 +9 @@
     TODO: self.verbose option and associated self._print
     """
+
+    def __init__(self, verbose=False):
+        # cmd.Cmd is not a new style class
+        cmd.Cmd.__init__(self)
+        self.verbose = verbose
 
     prompt = 'The Bard > '
@@ -195 +200 @@
         print 'Information about this package.'
 
+    def do_search_add(self, line=None):
+        path = line.strip()
+        if not os.path.exists(path):
+            print '"%s" is not an existent path' % path
+            return 1
+        if os.path.isdir(path):
+            fns = os.listdir(path)
+            fns = filter(lambda x: x.endswith('.txt'), fns)
+            works = [ os.path.join(path, fn) for fn in fns ]
+        else:
+            works = [ path ]
+        import shakespeare.search
+        index = shakespeare.search.SearchIndex.default_index()
+        for work in works:
+            if self.verbose:
+                print 'Processing %s' % work
+            fileobj = open(work)
+            index.add_item(fileobj)
+
+    def help_search_add(self, line=None):
+        info = '''search_add {path}
+
+Add contents of {path} (file itself or all text files in directory if
+directory) to the search index.'''
+        print info
+
+    def do_search_add_all(self):
+        # TODO: automatically add all texts listed in index
+        pass
+
+    def do_search(self, line=None):
+        import shakespeare.search
+        index = shakespeare.search.SearchIndex.default_index()
+        query = line.strip()
+        if not query:
+            print 'No search term supplied.'
+            return 1
+        matches = index.search(query)
+        print "%i results found." % matches.get_matches_estimated()
+        print "Results 1-%i:" % matches.size()
+
+        for m in matches:
+            print
+            print '%i: %i%% docid=%i' % (m.rank + 1, m.percent, m.docid)
+            print m.document.get_data()
+
+    def help_search(self, line=None):
+        info = 'Supply a query with which to search the search index.'
+        print info
+
 def main():
     import optparse
@@ -210 +265 @@
         return 1
     else:
-        cmd = ShakespeareAdmin()
+        cmd = ShakespeareAdmin(verbose=options.verbose)
         args = ' '.join(args)
         args = args.replace('-','_')

trunk/shakespeare/search.py

@@ -1 @@
-#!/usr/bin/env python
-
+# Support for indexing and searching texts using xapian
+import os
 
 import xapian
 
-if len(sys.argv) < 2:
-    print >> sys.stderr, "Missing a search term" % sys.argv[0]
-    sys.exit(1)
+class SearchIndex(object):
+    def __init__(self, index_dir):
+        self.index_dir = index_dir
 
-try:
-    # Open the database for searching.
-    database = xapian.Database('./index')
+    @classmethod
+    def config_index_dir(self):
+        '''Get the search index directory specified in the config.'''
+        import shakespeare
+        conf = shakespeare.conf()
+        index_dir = conf['search_index_dir']
+        return index_dir
 
-        # Start an enquire session.
-    enquire = xapian.Enquire(database)
+    @classmethod
+    def default_index(self):
+        '''Return a SearchIndex instance initialized with the path specified in
+        the configuration file.
+        '''
+        index_dir = self.config_index_dir()
+        if not os.path.exists(index_dir):
+            os.makedirs(index_dir)
+        return SearchIndex(index_dir)
 
-        # Take the search argument and turn into a Xapian query
-    query_string = sys.argv[1]
-    for arg in sys.argv[2:]:
-        query_string += ' '
-        query_string += arg
+    def add_item(self, fileobj):
+        # TODO: remove this comment as no longer relevant (?)
+        #create the folder for a writable db: alter path
+        document = xapian.WritableDatabase (self.index_dir, xapian.DB_CREATE_OR_OPEN)
+        indexer = xapian.TermGenerator()
+        stemmer = xapian.Stem("english")
+        indexer.set_stemmer(stemmer)
+
+        para = ''
+        try:
+             for line in fileobj:
+                line = line.strip()
+                if line == '':
+                    if para != '':
+                        doc = xapian.Document()
+                        doc.set_data(para)
+
+                        indexer.set_document(doc)
+                        indexer.index_text(para)
+
+                        # Add the document to the database.
+                        document.add_document(doc)
+                        para = ''
+                else:
+                    if para != '':
+                        para += ' '
+                    para += line
+        except StopIteration:
+            # TODO: what is happening here?
+            pass
+            print Stopped
+
+    def search(self, query_string):
+        # Open the database for searching.
+        database = xapian.Database(self.index_dir)
+
+            # Start an enquire session.
+        enquire = xapian.Enquire(database)
 
         # Parse the query string to produce a Xapian::Query object.
-    qp = xapian.QueryParser()
-    stemmer = xapian.Stem("english")
-    qp.set_stemmer(stemmer)
-    qp.set_database(database)
-    qp.set_stemming_strategy(xapian.QueryParser.STEM_SOME)
-    query = qp.parse_query(query_string)
-    print "Parsed query is: %s" % query.get_description()
+        qp = xapian.QueryParser()
+        stemmer = xapian.Stem("english")
+        qp.set_stemmer(stemmer)
+        qp.set_database(database)
+        qp.set_stemming_strategy(xapian.QueryParser.STEM_SOME)
+        query = qp.parse_query(query_string)
+        print "Parsed query is: %s" % query.get_description()
 
-     # Find the top 10 results for the query.
-    enquire.set_query(query)
-    matches = enquire.get_mset(0, 10)
+         # Find the top 10 results for the query.
+        enquire.set_query(query)
+        matches = enquire.get_mset(0, 10)
+        return matches
 
+    @classmethod
+    def print_matches(self, matches):
+        # Display the results.
+        print "%i results found." % matches.get_matches_estimated()
+        print "Results 1-%i:" % matches.size()
 
-    # Display the results.
-    print "%i results found." % matches.get_matches_estimated()
-    print "Results 1-%i:" % matches.size()
+        for m in matches:
+            print "%i: %i%% docid=%i [%s]" % (m.rank + 1, m.percent, m.docid, m.document.get_data())
 
-    for m in matches:
-        print "%i: %i%% docid=%i [%s]" % (m.rank + 1, m.percent, m.docid, m.document.get_data())
-
-except Exception, e:
-    print >> sys.stderr, "Exception: %s" % str(e)
-    sys.exit(1)
-