Migrate from bsddb3 to berkeleydb

README.rst

@@ -167,14 +167,14 @@ Tests
 
 Tests are run using the `nose2 <https://docs.nose2.io/en/latest/index.html>`_ framework.
 
-Run all the tests using the following command:
+Start off all the tests using the following command:
 
 .. sourcecode :: sh
 
     nose2 --verbose
 
 You can also run single tests. This example runs the tests in the :code:`TestLoadMetadata`
-class in :doc:`tests/test_acquire.py`:
+class in :download:`test_acquire.py </tests/test_acquire.py>`:
 
 .. sourcecode :: sh
 
@@ -244,7 +244,7 @@ To populate the cache:
 If you need more fine-grained control over the cache (e.g. where it's stored or
 which backend is used), you can use the :code:`set_metadata_cache` function to switch
 out the backend of the cache before you populate it. For example, to use the
-Sqlite cache backend instead of the default Sleepycat backend and store the
+Sqlite cache backend instead of the default Berkeley DB backend and store the
 cache at a custom location, you'd do the following:
 
 .. sourcecode :: python

gutenberg/acquire/metadata.py

@@ -41,6 +41,7 @@ class MetadataCache(metaclass=abc.ABCMeta):
     """Super-class for all metadata cache implementations.
 
     """
+
     def __init__(self, store, cache_uri):
         self.store = store
         self.cache_uri = cache_uri
@@ -164,23 +165,26 @@ def _iter_metadata_triples(cls, metadata_archive_path):
                 if pg_rdf_regex.search(item.name):
                     with disable_logging():
                         extracted = metadata_archive.extractfile(item)
-                        graph = Graph().parse(extracted)
+                        graph = Graph().parse(extracted,
+                                              format='application/rdf+xml')
                     for fact in graph:
                         if cls._metadata_is_invalid(fact):
                             logging.info('skipping invalid triple %s', fact)
                         else:
                             yield fact
 
 
-class SleepycatMetadataCache(MetadataCache):
-    """Default cache manager implementation, based on Sleepycat/Berkeley DB.
-    Sleepycat is natively supported by RDFlib so this cache is reasonably fast.
+class BerkeleyDBMetadataCache(MetadataCache):
+    """Default cache manager implementation, based on
+    BerkeleyDB plugin/Berkeley DB. BerkeleyDB is natively
+    supported by RDFlib so this cache is reasonably fast.
 
     """
+
     def __init__(self, cache_location):
         self._check_can_be_instantiated()
         cache_uri = cache_location
-        store = 'Sleepycat'
+        store = 'BerkeleyDB'
         MetadataCache.__init__(self, store, cache_uri)
 
     def _populate_setup(self):
@@ -190,11 +194,11 @@ def _populate_setup(self):
     @classmethod
     def _check_can_be_instantiated(cls):
         try:
-            from bsddb3 import db
+            from berkeleydb import db
         except ImportError:
             db = None
         if db is None:
-            raise InvalidCacheException('no install of bsddb3 found')
+            raise InvalidCacheException('No install of berkeleydb found.')
         del db
 
 
@@ -207,7 +211,8 @@ def __init__(self, cache_location, cache_url, user=None, password=None):
         MetadataCache.__init__(self, store, cache_url)
         user = user or os.getenv('GUTENBERG_FUSEKI_USER')
         password = password or os.getenv('GUTENBERG_FUSEKI_PASSWORD')
-        self.graph.store.setCredentials(user, password)
+        # self.graph.store.setCredentials(user, password)
+        self.graph.store.auth = (user, password)
         self._cache_marker = cache_location
 
     def _populate_setup(self):
@@ -347,7 +352,7 @@ def _create_metadata_cache(cache_location):
         return FusekiMetadataCache(cache_location, cache_url)
 
     try:
-        return SleepycatMetadataCache(cache_location)
+        return BerkeleyDBMetadataCache(cache_location)
     except InvalidCacheException:
         logging.warning('Unable to create cache based on BSD-DB. '
                         'Falling back to SQLite backend. '

setup.py

@@ -7,9 +7,9 @@
 
 
 install_requires = [
-    'bsddb3>=6.1.0',
+    'berkeleydb>=18.1.5',
     'future>=0.15.2',
-    'rdflib>=4.2.0,<5.0.0',
+    'rdflib>=6.0.0',
     'requests>=2.5.1',
     'setuptools>=18.5',
     'rdflib-sqlalchemy>=0.3.8',

tests/_util.py

@@ -13,7 +13,7 @@
 import requests
 
 import gutenberg.acquire.text
-from gutenberg.acquire.metadata import (SleepycatMetadataCache,
+from gutenberg.acquire.metadata import (BerkeleyDBMetadataCache,
                                         set_metadata_cache)
 
 INTEGRATION_TESTS_ENABLED = bool(os.getenv('GUTENBERG_RUN_INTEGRATION_TESTS'))
@@ -36,7 +36,7 @@ def sample_data(self):
         raise NotImplementedError  # pragma: no cover
 
     def setUp(self):
-        self.cache = _SleepycatMetadataCacheForTesting(self.sample_data, 'nt')
+        self.cache = _BerkeleyDBMetadataCacheForTesting(self.sample_data, 'nt')
         self.cache.populate()
         set_metadata_cache(self.cache)
 
@@ -45,14 +45,14 @@ def tearDown(self):
         self.cache.delete()
 
 
-class _SleepycatMetadataCacheForTesting(SleepycatMetadataCache):
+class _BerkeleyDBMetadataCacheForTesting(BerkeleyDBMetadataCache):
     def __init__(self, sample_data_factory, data_format):
-        SleepycatMetadataCache.__init__(self, tempfile.mktemp())
+        BerkeleyDBMetadataCache.__init__(self, tempfile.mktemp())
         self.sample_data_factory = sample_data_factory
         self.data_format = data_format
 
     def populate(self):
-        SleepycatMetadataCache.populate(self)
+        BerkeleyDBMetadataCache.populate(self)
 
         data = '\n'.join(item.rdf() for item in self.sample_data_factory())
 

tests/test_metadata_cache.py

@@ -11,7 +11,7 @@
 from gutenberg.acquire.metadata import CacheAlreadyExistsException
 from gutenberg.acquire.metadata import InvalidCacheException
 from gutenberg.acquire.metadata import FusekiMetadataCache
-from gutenberg.acquire.metadata import SleepycatMetadataCache
+from gutenberg.acquire.metadata import BerkeleyDBMetadataCache
 from gutenberg.acquire.metadata import SqliteMetadataCache
 from gutenberg.acquire.metadata import set_metadata_cache
 from gutenberg.query import get_metadata
@@ -97,10 +97,10 @@ def setUp(self):
         self.cache.catalog_source = _sample_metadata_catalog_source()
 
 
-class TestSleepycat(MetadataCache, unittest.TestCase):
+class TestBerkeleyDB(MetadataCache, unittest.TestCase):
     def setUp(self):
         self.local_storage = tempfile.mktemp()
-        self.cache = SleepycatMetadataCache(self.local_storage)
+        self.cache = BerkeleyDBMetadataCache(self.local_storage)
         self.cache.catalog_source = _sample_metadata_catalog_source()