From d1104c47c2435d4e4d9fe9717a7e3f364cded814 Mon Sep 17 00:00:00 2001 From: chrysle Date: Wed, 31 May 2023 16:47:13 +0200 Subject: [PATCH 1/3] Migrate from `bsddb3` to `berkeleydb` Fixes c-w/gutenberg#144 --- README.rst | 6 +++--- gutenberg/acquire/metadata.py | 18 ++++++++++-------- setup.py | 4 ++-- tests/_util.py | 10 +++++----- tests/test_metadata_cache.py | 6 +++--- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/README.rst b/README.rst index e840f83..e5b43ff 100644 --- a/README.rst +++ b/README.rst @@ -167,14 +167,14 @@ Tests Tests are run using the `nose2 `_ framework. -Run all the tests using the following command: +Start off all the tests using the following command: .. sourcecode :: sh nose2 --verbose You can also run single tests. This example runs the tests in the :code:`TestLoadMetadata` -class in :doc:`tests/test_acquire.py`: +class in :download:`test_acquire.py `: .. sourcecode :: sh @@ -244,7 +244,7 @@ To populate the cache: If you need more fine-grained control over the cache (e.g. where it's stored or which backend is used), you can use the :code:`set_metadata_cache` function to switch out the backend of the cache before you populate it. For example, to use the -Sqlite cache backend instead of the default Sleepycat backend and store the +Sqlite cache backend instead of the default Berkeley DB backend and store the cache at a custom location, you'd do the following: .. sourcecode :: python diff --git a/gutenberg/acquire/metadata.py b/gutenberg/acquire/metadata.py index b10f245..704fc21 100644 --- a/gutenberg/acquire/metadata.py +++ b/gutenberg/acquire/metadata.py @@ -41,6 +41,7 @@ class MetadataCache(metaclass=abc.ABCMeta): """Super-class for all metadata cache implementations. """ + def __init__(self, store, cache_uri): self.store = store self.cache_uri = cache_uri @@ -164,7 +165,7 @@ def _iter_metadata_triples(cls, metadata_archive_path): if pg_rdf_regex.search(item.name): with disable_logging(): extracted = metadata_archive.extractfile(item) - graph = Graph().parse(extracted) + graph = Graph().parse(extracted, format='application/rdf+xml') for fact in graph: if cls._metadata_is_invalid(fact): logging.info('skipping invalid triple %s', fact) @@ -172,15 +173,16 @@ def _iter_metadata_triples(cls, metadata_archive_path): yield fact -class SleepycatMetadataCache(MetadataCache): - """Default cache manager implementation, based on Sleepycat/Berkeley DB. - Sleepycat is natively supported by RDFlib so this cache is reasonably fast. +class BerkeleyDBMetadataCache(MetadataCache): + """Default cache manager implementation, based on BerkeleyDB plugin/Berkeley DB. + BerkeleyDB is natively supported by RDFlib so this cache is reasonably fast. """ + def __init__(self, cache_location): self._check_can_be_instantiated() cache_uri = cache_location - store = 'Sleepycat' + store = 'BerkeleyDB' MetadataCache.__init__(self, store, cache_uri) def _populate_setup(self): @@ -190,11 +192,11 @@ def _populate_setup(self): @classmethod def _check_can_be_instantiated(cls): try: - from bsddb3 import db + from berkeleydb import db except ImportError: db = None if db is None: - raise InvalidCacheException('no install of bsddb3 found') + raise InvalidCacheException('No install of berkeleydb found.') del db @@ -347,7 +349,7 @@ def _create_metadata_cache(cache_location): return FusekiMetadataCache(cache_location, cache_url) try: - return SleepycatMetadataCache(cache_location) + return BerkeleyDBMetadataCache(cache_location) except InvalidCacheException: logging.warning('Unable to create cache based on BSD-DB. ' 'Falling back to SQLite backend. ' diff --git a/setup.py b/setup.py index 25b8612..9253cd8 100644 --- a/setup.py +++ b/setup.py @@ -7,9 +7,9 @@ install_requires = [ - 'bsddb3>=6.1.0', + 'berkeleydb>=18.1.5', 'future>=0.15.2', - 'rdflib>=4.2.0,<5.0.0', + 'rdflib>=6.0.0', 'requests>=2.5.1', 'setuptools>=18.5', 'rdflib-sqlalchemy>=0.3.8', diff --git a/tests/_util.py b/tests/_util.py index b92435c..3c2417a 100644 --- a/tests/_util.py +++ b/tests/_util.py @@ -13,7 +13,7 @@ import requests import gutenberg.acquire.text -from gutenberg.acquire.metadata import (SleepycatMetadataCache, +from gutenberg.acquire.metadata import (BerkeleyDBMetadataCache, set_metadata_cache) INTEGRATION_TESTS_ENABLED = bool(os.getenv('GUTENBERG_RUN_INTEGRATION_TESTS')) @@ -36,7 +36,7 @@ def sample_data(self): raise NotImplementedError # pragma: no cover def setUp(self): - self.cache = _SleepycatMetadataCacheForTesting(self.sample_data, 'nt') + self.cache = _BerkeleyDBMetadataCacheForTesting(self.sample_data, 'nt') self.cache.populate() set_metadata_cache(self.cache) @@ -45,14 +45,14 @@ def tearDown(self): self.cache.delete() -class _SleepycatMetadataCacheForTesting(SleepycatMetadataCache): +class _BerkeleyDBMetadataCacheForTesting(BerkeleyDBMetadataCache): def __init__(self, sample_data_factory, data_format): - SleepycatMetadataCache.__init__(self, tempfile.mktemp()) + BerkeleyDBMetadataCache.__init__(self, tempfile.mktemp()) self.sample_data_factory = sample_data_factory self.data_format = data_format def populate(self): - SleepycatMetadataCache.populate(self) + BerkeleyDBMetadataCache.populate(self) data = '\n'.join(item.rdf() for item in self.sample_data_factory()) diff --git a/tests/test_metadata_cache.py b/tests/test_metadata_cache.py index 4f3098e..1071f80 100644 --- a/tests/test_metadata_cache.py +++ b/tests/test_metadata_cache.py @@ -11,7 +11,7 @@ from gutenberg.acquire.metadata import CacheAlreadyExistsException from gutenberg.acquire.metadata import InvalidCacheException from gutenberg.acquire.metadata import FusekiMetadataCache -from gutenberg.acquire.metadata import SleepycatMetadataCache +from gutenberg.acquire.metadata import BerkeleyDBMetadataCache from gutenberg.acquire.metadata import SqliteMetadataCache from gutenberg.acquire.metadata import set_metadata_cache from gutenberg.query import get_metadata @@ -97,10 +97,10 @@ def setUp(self): self.cache.catalog_source = _sample_metadata_catalog_source() -class TestSleepycat(MetadataCache, unittest.TestCase): +class TestBerkeleyDB(MetadataCache, unittest.TestCase): def setUp(self): self.local_storage = tempfile.mktemp() - self.cache = SleepycatMetadataCache(self.local_storage) + self.cache = BerkeleyDBMetadataCache(self.local_storage) self.cache.catalog_source = _sample_metadata_catalog_source() From 847348c55f6b65386a5f670b870ce77ad1ae841c Mon Sep 17 00:00:00 2001 From: chrysle Date: Wed, 31 May 2023 16:58:35 +0200 Subject: [PATCH 2/3] Fix lint --- gutenberg/acquire/metadata.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gutenberg/acquire/metadata.py b/gutenberg/acquire/metadata.py index 704fc21..4ed5a15 100644 --- a/gutenberg/acquire/metadata.py +++ b/gutenberg/acquire/metadata.py @@ -165,7 +165,8 @@ def _iter_metadata_triples(cls, metadata_archive_path): if pg_rdf_regex.search(item.name): with disable_logging(): extracted = metadata_archive.extractfile(item) - graph = Graph().parse(extracted, format='application/rdf+xml') + graph = Graph().parse(extracted, + format='application/rdf+xml') for fact in graph: if cls._metadata_is_invalid(fact): logging.info('skipping invalid triple %s', fact) @@ -174,8 +175,9 @@ def _iter_metadata_triples(cls, metadata_archive_path): class BerkeleyDBMetadataCache(MetadataCache): - """Default cache manager implementation, based on BerkeleyDB plugin/Berkeley DB. - BerkeleyDB is natively supported by RDFlib so this cache is reasonably fast. + """Default cache manager implementation, based on + BerkeleyDB plugin/Berkeley DB. BerkeleyDB is natively + supported by RDFlib so this cache is reasonably fast. """ From a02b3ff7e75efecebde2ffb3bcdd321d988facca Mon Sep 17 00:00:00 2001 From: chrysle Date: Thu, 1 Jun 2023 07:12:06 +0200 Subject: [PATCH 3/3] Fix passing Fuseki credentials to SPARQLStore --- gutenberg/acquire/metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gutenberg/acquire/metadata.py b/gutenberg/acquire/metadata.py index 4ed5a15..4d8340b 100644 --- a/gutenberg/acquire/metadata.py +++ b/gutenberg/acquire/metadata.py @@ -211,7 +211,8 @@ def __init__(self, cache_location, cache_url, user=None, password=None): MetadataCache.__init__(self, store, cache_url) user = user or os.getenv('GUTENBERG_FUSEKI_USER') password = password or os.getenv('GUTENBERG_FUSEKI_PASSWORD') - self.graph.store.setCredentials(user, password) + # self.graph.store.setCredentials(user, password) + self.graph.store.auth = (user, password) self._cache_marker = cache_location def _populate_setup(self):