Changeset 194:76629f68bc4a

Timestamp:

09/14/08 18:39:10 (2 years ago)

Author:

rgrp

Branch:

default

convert_revision:

svn:10edda23-d834-0410-9182-b00384516d49/trunk@206

Message:

[factlet/new][l]: implement factlet creation from a wikipedia url in WUI.

• dbpedia:
  • put in tests for to_factlet and fix issues with that.
  • fix up verbose support in Describe
• factlet: support for factlet creation from a wikipedia url + test.

Location:

microfacts

Files:

• controllers/factlet.py (modified) (3 diffs)
• getdata/dbpedia.py (modified) (7 diffs)
• lib/cli.py (modified) (2 diffs)
• model/__init__.py (modified) (1 diff)
• templates/factlet/layout.html (modified) (1 diff)
• templates/factlet/new.html (modified) (2 diffs)
• tests/functional/test_factlet.py (modified) (2 diffs)
• tests/getdata/test_dbpedia.py (modified) (3 diffs)

microfacts/controllers/factlet.py

@@ +1 @@
+import logging
 import simplejson as sj
 import genshi
+
+logging.getLogger(__name__)
 
 from microfacts.lib.base import *
 from microfacts.modes import *
 import microfacts.lib.json
+
+# not everyone will have the libraries installed for working with dbpedia
+dbpedia_enabled = True
+try:
+    from microfacts.getdata.dbpedia import *
+except:
+    dbpedia_enabled = False
 
 class FactletController(BaseController):
@@ -34 +44 @@
 
     def new(self):
-        return render('factlet/new')
+        c.dbpedia_enabled = dbpedia_enabled
+        if not 'commit' in request.params: # no form submitted
+            return render('factlet/new')
 
-    def create(self):
-        if 'title' in request.params:
+        title = request.params.getone('title').strip()
+        url = request.params.getone('url').strip()
+        if title:
+            logger.debug('Creating new factlet with title: %s' % title)
             registry_path = '/factlet'
             entity_data = {
@@ -50 +64 @@
                 h.redirect_to(controller='factlet', action='update',
                         id=c.factlet.id)
-            else:
+            else: # this should really tell the user what was wrong
                 abort(self.mode.response_code)
+        elif url:
+            logger.debug('Creating new factlet from url: %s' % url)
+            # TODO: check dbpedia_enabled is True?
+            d = microfacts.getdata.dbpedia.Describe()
+            d.execute(url)
+            try:
+                c.factlet = d.to_factlet()
+            except Exception, inst:
+                c.error = str(inst)
+                return render('factlet/new')
+            h.redirect_to(controller='factlet', action='update',
+                    id=c.factlet.id)
         else:
+            # Error?
+            # c.error = 'No title or url supplied'
             return render('factlet/new')
+
 
     def read_core(self, id):

microfacts/getdata/dbpedia.py

@@ -75 +75 @@
 class Describe:
 
-    def __init__(self):
+    def __init__(self, verbose=False):
         self.results = []
-        self.verbose = False
+        self.verbose = verbose
         self.language = 'en'
 
     def execute(self, uri):
+        if 'wikipedia.org' in uri:
+            uri = self._convert_wikipedia_url(uri)
         self.uri = uri
         query2 = PREFIXES + '''
@@ -93 +95 @@
         # print self.results.all_nodes()
 
-    def to_str(self, verbose=False):
+    def _convert_wikipedia_url(self, url):
+        import urlparse
+        path = urlparse.urlparse(url)[2]
+        name = path.split('/')[-1]
+        dbpedia_uri = 'http://dbpedia.org/resource/' + name
+        return dbpedia_uri
+
+    def to_str(self):
         output = self.uri + '\n\n'
-        if verbose:
+        if self.verbose:
             for s,p,o in self.results:
                 output += u'%s, %s, %s\n' % (s,p,o)
@@ -109 +118 @@
         kwds = { 'title' : None,
                 'description' : None,
-                'start' : None,
-                'end' : None,
+                'start' : '',
+                'end' : '',
                 'source': None,
                 'place': [],
                 'long': None,
                 'lat': None,
-                'location' : {'type': 'Point', 'coordinates': [None, None]},
-                'image': [],
-                'license':  'GFDL'
+                'image_urls': [],
+                'license':  u'GFDL'
                 }
         for s,p,o in self.results:
@@ -159 +167 @@
                 kwds['source'] = unicode(o)
             elif str(p) == 'http://xmlns.com/foaf/0.1/img':
-                kwds['image'] = kwds['image'] + [unicode(o)]
+                kwds['image_urls'] = kwds['image_urls'] + [unicode(o)]
 
         # get lat/long indirectly ...
@@ -177 +185 @@
                     # once we have one set we don't need any more ...
                     break
-        kwds['location']['coordinates'][0] = kwds['long']
-        kwds['location']['coordinates'][1] = kwds['lat']
         return kwds 
 
@@ -214 +220 @@
                     raise ValueError('%s is not a useable date object' % o)
         except Exception, inst:
-            print 'Problem with extracting date from: %s' % o
-            try:
-                print inst
-            except:
-                pass
+            if self.verbose:
+                print 'Problem with extracting date from: %s' % o
+                try:
+                    print inst
+                except:
+                    pass
         return dates
 
@@ -228 +235 @@
         from microfacts.lib.json import FactletConverter
         conv = FactletConverter()
-        conv.to_domain_object(kwds)
-        model.Session.flush()
+        # convert kwds dates back to strings ...
+        kwds['start'] = str(kwds['start'])
+        kwds['end'] = str(kwds['end'])
+        # convert lists to non-lists (very crudely)
+        if kwds['image_urls']:
+            kwds['image'] = kwds['image_urls'][0]
+        if kwds['long']:
+            kwds['location'] = { 'type': 'Point', coordinates:None }
+            kwds['location']['coordinates'] = [ kwds['long'], kwds['lat'] ]
+        fct = conv.to_domain_object(kwds)
+        return fct 
 
 
 def describe(uri, verbose=False):
-    d = Describe()
+    d = Describe(verbose)
     d.execute(uri)
-    return d.to_str(verbose=verbose)
+    return d.to_str()
 
 

microfacts/lib/cli.py

@@ -111 +111 @@
     describe <uri>
     search <category-name>
-    wp <wikipedia-url>
     
 Examples:
 
     describe http://dbpedia.org/resource/Admiralty_Islands_campaign
+    describe http://en.wikipedia.org/wiki/Napoleon_I_of_France
     search Category:Battles_and_operations_of_World_War_II
-    wp http://en.wikipedia.org/wiki/Napoleon_I_of_France
     '''
     summary = __doc__.split('\n')[0]
@@ -141 +140 @@
             category = self.args[1]
             result = dbp.category_search(category)
-        elif cmd == 'wp':
-            import urlparse
-            url = self.args[1]
-            path = urlparse.urlparse(url)[2]
-            name = path.split('/')[-1]
-            dbpedia_uri = 'http://dbpedia.org/resource/' + name
-            result = dbp.describe(dbpedia_uri, verbose=self.verbose)
         else:
             msg = 'Command %s not recognized' % cmd

microfacts/model/__init__.py

@@ -186 +186 @@
 
 mapper(User, user_table, properties={
-    'factlets':orm.relation(Factlet, secondary=user_2_factlet, backref='owners'),
-    'threads':orm.relation(Thread, secondary=user_2_thread, backref='owners'),
+    'factlets':orm.relation(
+        Factlet,
+        secondary=user_2_factlet,
+        backref='owners'),
+    'threads':orm.relation(Thread,
+        secondary=user_2_thread,
+        backref='owners'),
     })

microfacts/templates/factlet/layout.html

@@ -23 +23 @@
           <li>${h.link_to('Home', h.url_for(controller='factlet', action='index', id=None))}</li>
           <li>${h.link_to('List', h.url_for(controller='factlet', action='list', id=None))}</li>
-          <li>${h.link_to('New', h.url_for(controller='factlet', action='create', id=None))}</li>
+          <li>${h.link_to('Create', h.url_for(controller='factlet', action='new', id=None))}</li>
         </ul>
       </aside>

microfacts/templates/factlet/new.html

@@ -1 @@
-<!DOCTYPE html>
 <html
   xmlns="http://www.w3.org/1999/xhtml"
@@ -10 +9 @@
 
   <head>
-    <title>New</title>
+    <title>Create</title>
   </head>
 
   <body>
-    <h2>Register New Factlet</h2>
-    <p>Please choose a title for the factlet.</p>
-    <form action="${h.url_for(controller='factlet', action='create')}" method="post">
+    <h2>Create a New Factlet</h2>
+    <p py:if="c.error"><strong>There was an error: ${c.error}</strong></p>
+    <form action="" method="post">
+      <p>Please choose a title for the factlet.</p>
       <label for="title">Title:</label>
       ${XML(h.text_field('title', size=30))}
       <br /><br />
+
+      <div py:strip="True" py:if="c.dbpedia_enabled">
+
+      <p><strong>OR</strong> enter a Wikipedia (or DBPedia) url:</p>
+      <label for="url">Url:</label>
+      ${XML(h.text_field('url', size=30))}
+
+      <p class="desc">The factlet will be generated from the information in that
+      article.</p>
+      </div>
+
       ${XML(h.submit('Create new factlet'))}
     </form>

microfacts/tests/functional/test_factlet.py

@@ -166 +166 @@
         form = response.forms[0]
         form['title'] = self.title
-        response = form.submit(status=[302])
+        # even though only one button if you do not specify name 'commit' is
+        # not in post params (which messes up controller function)
+        response = form.submit('commit', status=[302])
         response = response.follow()
         assert 'Factlets - Edit' in response, response
@@ -172 +174 @@
         assert self.ft
 
+import microfacts.controllers.factlet
+class TestFactletCreateViaWikipedia(TestController):
+
+    # nose attributes
+    __test__ = microfacts.controllers.factlet.dbpedia_enabled
+    external = True
+    tags = [ 'dbpedia' ]
+
+    title = u'Philip II of Spain'
+    url = u'http://en.wikipedia.org/wiki/Philip_II_of_Spain'
+
+    def tearDown(self):
+        ft = model.Factlet.query.filter_by(source=self.url).one()
+        ft.delete()
+        model.Session.flush()
+        model.Session.remove()
+
+    def test_create(self):
+        path = url_for(controller='factlet', action='new')
+        response = self.app.get(path)
+        assert 'enter a Wikipedia (or DBPedia resouce) url:'
+        form = response.forms[0]
+        form['url'] = self.url
+        response = form.submit('commit', status=[302])
+        response = response.follow()
+        assert 'Factlets - Edit' in response, response
+        self.ft = model.Factlet.query.filter_by(source=self.url).one()
+        assert self.ft.title == self.title
+        # May 21, 1527 (1527-05-21)
+        assert self.ft.start.year == 1527
+

microfacts/tests/getdata/test_dbpedia.py

@@ -1 +1 @@
 # not everyone will have the libraries installed for working with dbpedia
+dotest = True
 try:
     from microfacts.getdata.dbpedia import *
 except:
-    pass
+    dotest = False
 
-class TestDescribe:
-    __test__ = False
+class DbpediaBase(object):
+    __test__ = dotest
+    external = True
+    tags = [ 'dbpedia' ]
+
+class TestDescribe(DbpediaBase):
 
     def test_extract_1(self):
@@ -30 +35 @@
         assert kwds['title'] == u'Invasion of Normandy'
 
-class TestDescribe2(object):
-    __test__ = False
+class TestDescribe2(DbpediaBase):
 
     @classmethod
     def setup_class(self):
         uri3 = 'http://dbpedia.org/resource/Battle_of_Austerlitz'
-        d = Describe()
-        d.execute(uri3)
-        self.kwds = d.extract(recurse_for_location=True)
+        self.d = Describe()
+        self.d.execute(uri3)
+        self.kwds = self.d.extract(recurse_for_location=True)
+        self.fct = None
+        self.start = datetime.datetime(1805, 12, 2)
+
+    @classmethod
+    def teardown_class(self):
+        import microfacts.model as model
+        if self.fct:
+            self.fct.delete()
+        model.Session.flush()
+        model.Session.remove()
 
     def test_date(self):
-        assert self.kwds['start'] == datetime.datetime(1805, 12, 2)
+        assert self.kwds['start'] == self.start 
 
     def test_source(self):
@@ -50 +64 @@
         assert self.kwds['long'] == 16.76361083984375
 
+    def test_to_factlet(self):
+        self.fct = self.d.to_factlet()
+        assert self.fct.id
+        long = round(self.fct.location.x,1)
+        assert long == 16.8, long
+        assert self.fct.start == self.start
+
+
+class TestDescribeWikipedia(DbpediaBase):
+
+    uri = 'http://en.wikipedia.org/wiki/Battle_of_Austerlitz'
+
+    def test__convert_wikipedia_url(self):
+        d = Describe()
+        out = d._convert_wikipedia_url(self.uri)
+        assert out == 'http://dbpedia.org/resource/Battle_of_Austerlitz', out
+    
+    def test_1(self):
+        d = Describe()
+        d.execute(self.uri)
+        self.kwds = d.extract()
+        assert self.kwds['title'] == 'Battle of Austerlitz'
+        assert self.kwds['source'] == self.uri
+