Changeset 191

Timestamp:

08/16/08 23:51:24 (11 months ago)

Author:

rgrp

Message:

[shakespeare/search][m]: more support search in WUI.

Location:

trunk/shakespeare

Files:

• controllers/search.py (modified) (2 diffs)
• search.py (modified) (9 diffs)
• templates/search/index.html (modified) (1 diff)
• tests/functional/test_search.py (modified) (2 diffs)
• tests/test_search.py (modified) (3 diffs)

trunk/shakespeare/controllers/search.py

@@ -10 +10 @@
 
     def index(self):
-        query = request.params.get('query', '')
-        if query:
-            c.matches = self._get_results(query)
+        c.query = request.params.get('query', '')
+        if c.query:
+            c.matches = self._get_matches(c.query)
+            c.results = self._get_results(c.matches)
             c.total = c.matches.get_matches_estimated()
         else:
@@ -18 +19 @@
         return render('search/index')
     
-    def _get_results(self, query):
+    def _get_matches(self, query):
         index = shakespeare.search.SearchIndex.default_index()
-        matches = index.search(query)
+        matches = index.search(query, numresults=50)
         return matches
-        
 
+    def _get_results(self, matches):
+        results = []
+        for m in matches:
+            text, lineno = self._match_to_text(m)
+            if text:
+                # slight hack -- just attach direct to object
+                text._lineno = lineno
+                text._snippet = m.document.get_data()
+                results.append(text)
+            else:
+                # TODO: create a dummy text ...
+                pass
+        return results
+
+    def _match_to_text(self, m):
+        item_id = m.document.get_value(shakespeare.search.ITEM_ID)
+        text = model.Material.byName(item_id)
+        lineno = m.document.get_value(shakespeare.search.LINE_NO)
+        return (text, lineno)
+

trunk/shakespeare/search.py

@@ -36 +36 @@
 import xapian
 
+# keys for document values
+ITEM_ID = 0
+LINE_NO = 1
+
 class SearchIndex(object):
     def __init__(self, index_dir):
@@ -58 +62 @@
         return SearchIndex(index_dir)
 
-    def add_item(self, fileobj):
-        document = xapian.WritableDatabase (self.index_dir, xapian.DB_CREATE_OR_OPEN)
+    def add_item(self, fileobj, item_id=None):
+        database = xapian.WritableDatabase(self.index_dir, xapian.DB_CREATE_OR_OPEN)
         indexer = xapian.TermGenerator()
         stemmer = xapian.Stem("english")
@@ -66 +70 @@
         para = ''
         try:
+            count = -1
+            para_start = 0
             for line in fileobj:
+                count += 1
                 line = line.strip()
                 if line == '':
@@ -72 +79 @@
                         doc = xapian.Document()
                         doc.set_data(para)
+                        id_term = 'I' + str(item_id)
+                        doc.add_term(id_term)
+                        doc.add_value(ITEM_ID, str(item_id))
+                        doc.add_value(LINE_NO, str(para_start))
 
                         indexer.set_document(doc)
@@ -77 +88 @@
                         indexer.index_text(para)
 
-                        # Add the document to the database.
-                        document.add_document(doc)
+                        database.add_document(doc)
+                        # assume next para starts
                         para = ''
+                    # must come after
+                    para_start = count
                 else:
                     if para != '':
@@ -86 +99 @@
         except StopIteration:
             # TODO: what is happening here?
-            pass
+            raise
 
-    def search(self, query_string):
-        # Open the database for searching.
+    def get_database(self):
         database = xapian.Database(self.index_dir)
+        return database
 
-            # Start an enquire session.
+    def search(self, query_string, offset=0, numresults=10):
+        database = self.get_database()
         enquire = xapian.Enquire(database)
-
-        # Parse the query string to produce a Xapian::Query object.
         qp = xapian.QueryParser()
         stemmer = xapian.Stem("english")
@@ -102 +114 @@
         qp.set_stemming_strategy(xapian.QueryParser.STEM_SOME)
         query = qp.parse_query(query_string)
-        print "Parsed query is: %s" % query.get_description()
-
-         # Find the top 10 results for the query.
         enquire.set_query(query)
-        # get search results offset, offset+count
-        offset = 0
-        count = 10
-        matches = enquire.get_mset(offset, count)
+        matches = enquire.get_mset(offset, numresults)
         return matches
 
@@ -126 +132 @@
             works = [ path ]
         for work in works:
-            if self.verbose:
-                print 'Processing %s' % work
             fileobj = open(work)
-            self.index.add_item(fileobj)
+            self.add_item(fileobj)
 
     @classmethod
@@ -144 +148 @@
             msg += '\n'
         return msg
-
-
+    

trunk/shakespeare/templates/search/index.html

@@ -11 +11 @@
 
     <div class="search-results" py:if="c.matches is not None">
-      <h3>Search Results</h3>
+      <h3>Search Results For: ${c.query}</h3>
       There were ${c.total} results.
       <ul>
-        <li py:for="m in c.matches">
-        ${m.document.get_data()}
+        <li py:for="m in c.results">
+        Work: ${m.title}, Line: ${m._lineno}
+        <blockquote>
+          <pre>${m._snippet}</pre>
+        </blockquote>
         </li>
       </ul>

trunk/shakespeare/tests/functional/test_search.py

@@ -11 +11 @@
         text = make_fixture()
         sindex = shakespeare.search.SearchIndex.default_index()
-        sindex.add_item(StringIO.StringIO(text.content))
+        sindex.add_item(StringIO.StringIO(text.content), text.name)
 
     def test_index(self):
@@ -24 +24 @@
         form['query'] = 'summer'
         res = form.submit()
+        # print res
         assert 'Search Results' in res
         assert 'Shall I compare thee' in res

trunk/shakespeare/tests/test_search.py

@@ -5 +5 @@
 
 import shakespeare.search
+import shakespeare.tests
 
 class TestSearch:
-    # break up a little to make indexing more interesting
-    text = \
-'''
-Shall I compare thee to a summer's day?
-Thou art more lovely and more temperate:
-Rough winds do shake the darling buds of May,
-And summer's lease hath all too short a date:
-
-Sometime too hot the eye of heaven shines,
-And often is his gold complexion dimm'd,
-And every fair from fair sometime declines,
-By chance, or nature's changing course untrimm'd: 
-
-But thy eternal summer shall not fade,
-Nor lose possession of that fair thou ow'st,
-Nor shall death brag thou wander'st in his shade,
-When in eternal lines to time thou grow'st,
-
-  So long as men can breathe, or eyes can see,
-  So long lives this, and this gives life to thee.
-'''
-
     def setUp(self):
+        self.text = shakespeare.tests.make_fixture()
         basetmp = tempfile.gettempdir()
         self.tmpdir = os.path.join(basetmp, 'openshkspr-search')
@@ -37 +17 @@
         os.makedirs(self.tmpdir)
         self.index = shakespeare.search.SearchIndex(self.tmpdir)
+        self.index.add_item(StringIO.StringIO(self.text.content),
+                self.text.name)
+
+    # TODO: remove the document from the index
 
     def test_add_item(self):
-        self.index.add_item(StringIO.StringIO(self.text))
+        assert self.index.get_database().get_doccount() > 0
 
     def test_search(self):
-        self.index.add_item(StringIO.StringIO(self.text))
         out = self.index.search('summer')
         assert len(out) == 2
@@ -51 +34 @@
         out = self.index.search('rough')
         assert len(out) == 1
+    
+    def test_retrieve_lineno(self):
+        out = self.index.search('summer')
+        mset1 = out[1]
+        lineno = mset1.document.get_value(shakespeare.search.LINE_NO)
+        assert lineno == '9'
 
+    def test_retrieve_itemid(self):
+        out = self.index.search('summer')
+        mset1 = out[1]
+        name = mset1.document.get_value(shakespeare.search.ITEM_ID)
+        assert name == self.text.name
+