Changeset 169:30df1ae8d73a

Timestamp:

01/06/09 18:47:18 (14 months ago)

Author:

rgrp

Branch:

default

convert_revision:

svn:0ead1229-0713-0410-96cd-f668dbfad531/trunk@247

Message:

[shksprdata,moby][m]: update getdata/moby.py and start work on xsl transforms of moby texts.

• shakespeare/cache: minor
  • minor refactoring
  • support for not downloading to nested directories (e.g. abc.com/abc.txt goes on disk to abc.txt not abc.com/abc.txt).
• shksprdata/getdata/moby.py:
  • was broken so fix up (making simpler at same time)
  • create metadata file at same time as downloading files (and rename files)
  • start work on xsl transforms using iain's xslt file
• shakespeare/init.py: fix to documentation and bump version to 0.7
• MANIFEST.in: fix incorrect recursive includes (comment out in fact as not needed)

Files:

• MANIFEST.in (modified) (1 diff)
• shakespeare/__init__.py (modified) (4 diffs)
• shakespeare/cache.py (modified) (4 diffs)
• shakespeare/gutenberg.py (modified) (1 diff)
• shakespeare/tests/test_cache.py (modified) (2 diffs)

MANIFEST.in

@@ -1 +1 @@
 recursive-include shakespeare/public *
 recursive-include shakespeare/templates *
-recursive-include shksprdata *
+# seem to be automatically included ...
+# recursive-include shksprdata *.txt
+# recursive-include shksprdata *.xml
+# recursive-include miltondata *.txt
+# recursive-include miltondata *.xml

shakespeare/__init__.py

@@ -4 +4 @@
 
 The Open Shakespeare package provides a full open set of shakespeare's works
-(often in multiple versions) along with ancillary material, a variety of tools
-and a python API.
-
-Specifically in addition to the works themselves (often in multiple versions)
-there is an introduction, a chronology, explanatory notes, a concordance and
-search facilities.
+(often in multiple versions) along with ancillary material, a variety of tools,
+a python API and a web interface that provides access to many (but not all) of
+these facilities from the comfort of your web browser (see
+http://www.openshakespeare.org/ for a demo).
 
 All material is open source/open knowledge so that anyone can use, redistribute
@@ -89 +87 @@
 
 2. To load the author packages, change into the miltondata or shksprdata directories
-and run the command load-milton (or load-shakespeare) -c <path to your development.ini>
-This will load the metadata text into the database. 
+   and run the command load-milton (or load-shakespeare) -c <path to your
+   development.ini> This will load the metadata text into the database. 
 
 3. You can start a web server to provide a easy-to-use web interface to the
-shakespeare material and facilities by doing::
+   shakespeare material and facilities by doing::
 
-    $ paster serve {your-config.ini}
+        $ paster serve {your-config.ini}
 
-NB: {your-config.ini} should be replaced with the name of the config file you
+   NB: {your-config.ini} should be replaced with the name of the config file you
 created earlier.
 
@@ -107 +105 @@
 --------
 
-Follow the basic steps above put with an ini file named: development.ini
+Follow the basic steps above but with an ini file named: development.ini
 
 NB: you'll probably want to change log levels to debug.
@@ -121 +119 @@
     $ nosetests shakespeare
 '''
-__version__ = '0.6a'
+__version__ = '0.7a'
 __application_name__ = 'shakespeare'
 

shakespeare/cache.py

@@ -1 +1 @@
 import os
+import urlparse
 import urllib
-
-import shakespeare
-conf = shakespeare.conf()
 
 class Cache(object):
@@ -9 +7 @@
     """
 
-    def __init__(self, cache_path):
+    def __init__(self, cache_path, fullpath=True):
+        '''
+        @param fullpath: save to local path corresponding to full url path
+            (creating directories as necessary) when retrieving.
+        '''
         self.cache_path = cache_path
+        self.fullpath = fullpath
 
     def path(self, remote_url, version=''):
@@ -16 +19 @@
         @type: string giving version of text (''|'cleaned')
         """
-        protocolEnd = remote_url.index(':') + 3  # add 3 for ://
-        path = remote_url[protocolEnd:]
-        base, name = os.path.split(path)
+        urlparts = urlparse.urlparse(remote_url)
+        base = urlparts[1]
+        pathparts = urlparts[2].split('/')
+        if len(pathparts) > 1:
+            base = os.path.join(base, *pathparts[:-1])
+        name = pathparts[-1]
         name = version + name
-        offset = os.path.join(base, name)
-        localPath = self.path_from_offset(offset)
-        return localPath
+        if self.fullpath:
+            offset = os.path.join(base, name)
+        else:
+            offset = name
+        local_path = self.path_from_offset(offset)
+        return local_path
+
+    def path_from_offset(self, offset):
+        "Get full path of file in cache given by offset."
+        return os.path.join(self.cache_path, offset)
 
     def download_url(self, url, overwrite=False):
@@ -42 +55 @@
             urllib.urlretrieve(url, localPath)
 
-    def path_from_offset(self, offset):
-        "Get full path of file in cache given by offset."
-        return os.path.join(self.cache_path, offset)
 
+try:
+    import shakespeare
+    conf = shakespeare.conf()
 
-default_path = shakespeare.conf()['cachedir']
-default = Cache(default_path)
+    default_path = shakespeare.conf()['cachedir']
+    default = Cache(default_path)
+except:
+    pass
 

shakespeare/gutenberg.py

@@ -144 +144 @@
         self.cache = cache
         self._index = works
+
+    def vprint(self, info, force=True):
+        if self.verbose or force:
+            print(info)
      
     def _filter_index(self, line):

shakespeare/tests/test_cache.py

@@ -21 +21 @@
         exp = os.path.join(self.cache_path, self.url[7:])
         out = self.cache.path(self.url)
-        assert out == exp
+        assert out == exp, (out, exp)
 
     def test_path_2(self):
@@ -29 +29 @@
         assert exp == out
 
+    def test_path_without_fullpath(self):
+        cache2 = shakespeare.cache.Cache('', fullpath=False)
+        out = cache2.path(self.url)
+        assert out == 'GUTINDEX.ALL'
+
     def test_download_url(self):
         exp = os.path.join(self.cache_path, self.url2[7:])