diff --git a/Readme.md b/Readme.md index 4a40f99..6c1ca9f 100644 --- a/Readme.md +++ b/Readme.md @@ -298,6 +298,14 @@ settings. > Since version 0.0.12, the config file is optional. The client will use default localhost settings if no configuration > is provided. +> [!WARNING] +> **Citation consolidation and the `timeout` setting.** When `--consolidate_citations` (or `consolidate_citations=True`) +> is enabled, GROBID queries external services (e.g. CrossRef) to enrich the extracted references. This is considerably +> slower than a plain extraction, and a low `timeout` frequently causes `HTTP 408 (Request Timeout)` errors. +> Set the `timeout` to at least **120 seconds (2-3 minutes recommended)** when consolidating citations. The client emits +> a warning when consolidation is requested with a timeout below 120 seconds. +> See [issue #54](https://github.com/grobidOrg/grobid-client-python/issues/54). + ### Logging Configuration The client provides configurable logging with different verbosity levels. By default, only essential statistics and warnings are shown. diff --git a/grobid_client/grobid_client.py b/grobid_client/grobid_client.py index 1af79d1..387e23b 100644 --- a/grobid_client/grobid_client.py +++ b/grobid_client/grobid_client.py @@ -39,6 +39,11 @@ def __init__(self, message="GROBID server is not available"): class GrobidClient(ApiClient): + # Below this client-side timeout (in seconds) citation consolidation is + # likely to trigger HTTP 408 (Request Timeout) errors, so we warn the user. + # See https://github.com/grobidOrg/grobid-client-python/issues/54 + CONSOLIDATE_CITATIONS_MIN_TIMEOUT = 120 + # Default configuration values DEFAULT_CONFIG = { 'grobid_server': 'http://localhost:8070', @@ -112,6 +117,27 @@ def _set_config_params(self, params): if value is not None: self.config[key] = value + def _warn_on_consolidation_timeout(self, consolidate_citations): + """Warn when citation consolidation is enabled with a low client timeout. + + Consolidating citations makes GROBID query external services and can be + much slower than a plain extraction. A short client-side timeout often + leads to HTTP 408 errors, so we recommend at least a couple of minutes. + See https://github.com/grobidOrg/grobid-client-python/issues/54 + """ + if not consolidate_citations: + return + + timeout = self.config.get("timeout", self.DEFAULT_CONFIG["timeout"]) + if timeout < self.CONSOLIDATE_CITATIONS_MIN_TIMEOUT: + self.logger.warning( + f"Citation consolidation is enabled but the timeout is only {timeout}s. " + f"Consolidation queries external services and can be slow; a low timeout " + f"frequently causes HTTP 408 (Request Timeout) errors. Consider increasing " + f"the 'timeout' setting to at least {self.CONSOLIDATE_CITATIONS_MIN_TIMEOUT}s " + f"(2-3 minutes is recommended)." + ) + def _handle_server_busy_retry(self, file_path, retry_func, *args, **kwargs): """Handle server busy (503) retry logic.""" self.logger.warning(f"Server busy (503), retrying {file_path} after {self.config['sleep_time']} seconds") @@ -345,6 +371,13 @@ def process( start_time = time.time() batch_size_pdf = self.config["batch_size"] + # Warn if citation consolidation is requested with a short timeout: the + # consolidation step queries external services (e.g. CrossRef) and can + # be significantly slower, frequently resulting in HTTP 408 errors when + # the client-side timeout is too low. + # See https://github.com/grobidOrg/grobid-client-python/issues/54 + self._warn_on_consolidation_timeout(consolidate_citations) + # First pass: count all eligible files all_input_files = [] for (dirpath, dirnames, filenames) in os.walk(input_path):