From 2b0300c232cec91c457ae6e8df535fc5c0b94768 Mon Sep 17 00:00:00 2001 From: Reggie Cushing Date: Fri, 28 Nov 2025 14:08:13 +0100 Subject: [PATCH 1/4] Added support for simple JSON paths in DatasetDownloader's `_get_json_value` method to optimize performance. --- datahugger/base.py | 49 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/datahugger/base.py b/datahugger/base.py index a8b4994..9d7410e 100644 --- a/datahugger/base.py +++ b/datahugger/base.py @@ -6,6 +6,7 @@ import re import time import zipfile +from functools import lru_cache from pathlib import Path from typing import Union from urllib.parse import urlparse @@ -68,12 +69,46 @@ def __init__( self.print_only = print_only self.params = params + # Helper function to check if the json path is simple. + # i.e., does not contain wildcards, recursive descent, or filter expressions. + # This allows us to optimize simple paths by manually traversing the JSON structure. + # The check resulr is cached for performance. + # The method is static so self is not cached. + @staticmethod + @lru_cache(maxsize=128) + def _is_simple_path(json_path): + # Check for wildcard, recursive descent, or filter expressions + result = all(c.isalnum() or c in "._" for c in json_path) and "*" not in json_path and ".." not in json_path and "[" not in json_path and "]" not in json_path + return result + + # Helper function to get value from JSON using either manual traversal or JSONPath. + # find() is slow for simple paths, so we optimize those cases. + # This can result in speedups of 30x e.g. for doi "10.17026/DANS-XGB-TW5U" + def _get_json_value(self, record, json_path): + if DatasetDownloader._is_simple_path(json_path): + # Manual walk + try: + current = record + for part in json_path.split('.'): + # handle numeric indices for arrays + if part.isdigit(): + current = current[int(part)] + else: + current = current[part] + return current + except (KeyError, IndexError, TypeError): + return None + else: + # JSONPath find + try: + expr = parse(json_path) + matches = expr.find(record) + return matches[0].value if matches else None + except Exception: + return None + def _get_attr_attr(self, record, jsonp): - try: - jsonpath_expression = parse(jsonp) - return jsonpath_expression.find(record)[0].value - except Exception: - return None + return self._get_json_value(record, jsonp) def _get_attr_link(self, record, **kwargs): # get the link to the folder @@ -326,9 +361,7 @@ def _get_files_recursive(self, url, folder_name=None, base_url=None): ) if hasattr(self, "PAGINATION_JSONPATH"): - jsonpath_expression = parse(self.PAGINATION_JSONPATH) - next_url = jsonpath_expression.find(response)[0].value - + next_url = self._get_json_value(response, self.PAGINATION_JSONPATH) if next_url: result.extend( self._get_files_recursive(next_url, folder_name=folder_name) From 3006ecb5b6a2a769319c44a41fcc328522cba85a Mon Sep 17 00:00:00 2001 From: Reggie Cushing Date: Fri, 28 Nov 2025 14:26:40 +0100 Subject: [PATCH 2/4] Removed unnecessary result variable declation. --- datahugger/base.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datahugger/base.py b/datahugger/base.py index 9d7410e..e916add 100644 --- a/datahugger/base.py +++ b/datahugger/base.py @@ -78,8 +78,7 @@ def __init__( @lru_cache(maxsize=128) def _is_simple_path(json_path): # Check for wildcard, recursive descent, or filter expressions - result = all(c.isalnum() or c in "._" for c in json_path) and "*" not in json_path and ".." not in json_path and "[" not in json_path and "]" not in json_path - return result + return all(c.isalnum() or c in "._" for c in json_path) and "*" not in json_path and ".." not in json_path and "[" not in json_path and "]" not in json_path # Helper function to get value from JSON using either manual traversal or JSONPath. # find() is slow for simple paths, so we optimize those cases. From 8a1d5a4d9ec74b86a4c0409508463e13be3e3765 Mon Sep 17 00:00:00 2001 From: Reggie Cushing Date: Fri, 28 Nov 2025 15:11:39 +0100 Subject: [PATCH 3/4] Update comment. --- datahugger/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datahugger/base.py b/datahugger/base.py index e916add..23a61ba 100644 --- a/datahugger/base.py +++ b/datahugger/base.py @@ -82,7 +82,8 @@ def _is_simple_path(json_path): # Helper function to get value from JSON using either manual traversal or JSONPath. # find() is slow for simple paths, so we optimize those cases. - # This can result in speedups of 30x e.g. for doi "10.17026/DANS-XGB-TW5U" + # This results in speedups of 30x in some cases + # e.g. for doi "10.17026/DANS-XGB-TW5U" def _get_json_value(self, record, json_path): if DatasetDownloader._is_simple_path(json_path): # Manual walk From cb699c61b5f2f2ee1c7221604659106ae0b07873 Mon Sep 17 00:00:00 2001 From: Reggie Cushing Date: Mon, 1 Dec 2025 12:12:26 +0100 Subject: [PATCH 4/4] Fix linting: line too long --- datahugger/base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/datahugger/base.py b/datahugger/base.py index 23a61ba..71f4060 100644 --- a/datahugger/base.py +++ b/datahugger/base.py @@ -78,7 +78,13 @@ def __init__( @lru_cache(maxsize=128) def _is_simple_path(json_path): # Check for wildcard, recursive descent, or filter expressions - return all(c.isalnum() or c in "._" for c in json_path) and "*" not in json_path and ".." not in json_path and "[" not in json_path and "]" not in json_path + return ( + all(c.isalnum() or c in "._" for c in json_path) + and "*" not in json_path + and ".." not in json_path + and "[" not in json_path + and "]" not in json_path + ) # Helper function to get value from JSON using either manual traversal or JSONPath. # find() is slow for simple paths, so we optimize those cases.