[Sis integration] params for import/export

.github/workflows/pytest-verify.yml

@@ -0,0 +1,30 @@
+name: Run pytest on supported python versions
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Test with pytest
+      run: |
+        pytest

Dockerfile

@@ -0,0 +1,18 @@
+ARG PYTHON_VERSION=3.11
+
+
+FROM python:$PYTHON_VERSION
+
+WORKDIR /app
+
+COPY pyproject.toml .
+COPY README.md .
+COPY requirements* .
+
+RUN pip install -r requirements.txt
+RUN pip install .
+
+
+ENTRYPOINT ["python", "-m", "pytest"]
+
+

README.md

@@ -3,6 +3,12 @@
 A Collection of python utility functions
 Primarily focused on integration with Funidata APIs
 
+Install dependencies in local virtual environment using the pyproject.toml :
+`pip install .`
+
+Include dev dependencies for testing etc:
+`pip install -r requirements.txt`
+
 ### TODO
 
 Finish readme documentation

docker-compose.yaml

@@ -0,0 +1,27 @@
+x-tests-common: &tests-common
+    volumes:
+      - ./tests:/app/tests
+      - ./funidata_utils:/app/funidata_utils
+
+
+services:
+  pytest_311:
+    build:
+      context: .
+      args:
+        - PYTHON_VERSION=3.11
+    <<: *tests-common
+
+  pytest_312:
+    build:
+      context: .
+      args:
+        - PYTHON_VERSION=3.12
+    <<: *tests-common
+
+  pytest_313:
+    build:
+      context: .
+      args:
+        - PYTHON_VERSION=3.13
+    <<: *tests-common

funidata_utils/compat/utils_312.py

@@ -0,0 +1,10 @@
+from collections import defaultdict
+from functools import reduce
+from typing import Callable, Any
+
+
+def group_by[T](
+    seq: list[T],
+    key: Callable
+) -> dict[Any, list[T]]:
+    return reduce(lambda grp, val: grp[key(val)].append(val) or grp, seq, defaultdict(list))

funidata_utils/compat/utils_legacy.py

@@ -0,0 +1,13 @@
+from collections import defaultdict
+from functools import reduce
+from typing import Callable, Any, TypeVar
+
+
+T = TypeVar('T')
+
+
+def group_by(
+    seq: list[T],
+    key: Callable
+) -> dict[Any, list[T]]:
+    return reduce(lambda grp, val: grp[key(val)].append(val) or grp, seq, defaultdict(list))

funidata_utils/schemas/common_serializers.py

@@ -1,12 +1,7 @@
-from typing import Iterable
+import sys
 
 
-def serialize_as_list[typevar](v: set[typevar] | list[typevar] | None) -> list[typevar] | None:
-    if v is None:
-        return None
-
-    try:
-        return list(set(v))
-    except Exception as e:
-        # If it can't be hashed to set, just return as list
-        return list(v)
+if sys.version_info >= (3, 12):
+    from .compat.common_serializers_312 import serialize_as_list  # noqa: F401 ("Unused import")
+else:
+    from .compat.common_serializers_legacy import serialize_as_list  # noqa: F401 ("Unused import")

funidata_utils/schemas/compat/common_serializers_312.py

@@ -0,0 +1,12 @@
+# Python > 3.12 implementations
+
+
+def serialize_as_list[typevar](v: set[typevar] | list[typevar] | None) -> list[typevar] | None:
+    if v is None:
+        return None
+
+    try:
+        return list(set(v))
+    except Exception as e:
+        # If it can't be hashed to set, just return as list
+        return list(v)

funidata_utils/schemas/compat/common_serializers_legacy.py

@@ -0,0 +1,15 @@
+from typing import TypeVar
+
+
+T = TypeVar('T')
+
+
+def serialize_as_list(v: set[T] | list[T] | None) -> list[T] | None:
+    if v is None:
+        return None
+
+    try:
+        return list(set(v))
+    except Exception as e:
+        # If it can't be hashed to set, just return as list
+        return list(v)

funidata_utils/sis_integration/async_imports.py

@@ -22,11 +22,12 @@ async def import_to_sisu(
     resource: SisImportable,
     use_legacy_import: Literal[False],
     fp: IO,
-    batch_size: int | None,
-    binary_search_max_depth: int | None,
-    group_by_key: str | None,
-    binary_err_search_sublists: bool,
-    max_parallel_requests: int,
+    batch_size: int | None = UNSET_BATCH_SIZE,
+    binary_search_max_depth: int | None = 0,
+    group_by_key: str | None = None,
+    binary_err_search_sublists: bool = False,
+    max_parallel_requests: int = 1,
+    params: dict | None = None,
 ) -> list[httpx.Response]:
     ...
 
@@ -37,11 +38,12 @@ async def import_to_sisu(
     resource: SisLegacyImportable,
     use_legacy_import: Literal[True],
     fp: IO,
-    batch_size: int | None,
-    binary_search_max_depth: int | None,
-    group_by_key: str | None,
-    binary_err_search_sublists: bool,
-    max_parallel_requests: int,
+    batch_size: int | None = UNSET_BATCH_SIZE,
+    binary_search_max_depth: int | None = 0,
+    group_by_key: str | None = None,
+    binary_err_search_sublists: bool = False,
+    max_parallel_requests: int = 1,
+    params: dict | None = None,
 ) -> list[httpx.Response]:
     ...
 
@@ -52,11 +54,12 @@ async def import_to_sisu(
     resource: SisImportable,
     use_legacy_import: Literal[False],
     data: list[dict],
-    batch_size: int | None,
-    binary_search_max_depth: int | None,
-    group_by_key: str | None,
-    binary_err_search_sublists: bool,
-    max_parallel_requests: int,
+    batch_size: int | None = UNSET_BATCH_SIZE,
+    binary_search_max_depth: int | None = 0,
+    group_by_key: str | None = None,
+    binary_err_search_sublists: bool = False,
+    max_parallel_requests: int = 1,
+    params: dict | None = None,
 ) -> list[httpx.Response]:
     ...
 
@@ -67,11 +70,12 @@ async def import_to_sisu(
     resource: SisLegacyImportable,
     use_legacy_import: Literal[True],
     data: list[dict],
-    batch_size: int | None,
-    binary_search_max_depth: int | None,
-    group_by_key: str | None,
-    binary_err_search_sublists: bool,
-    max_parallel_requests: int,
+    batch_size: int | None = UNSET_BATCH_SIZE,
+    binary_search_max_depth: int | None = 0,
+    group_by_key: str | None = None,
+    binary_err_search_sublists: bool = False,
+    max_parallel_requests: int = 1,
+    params: dict | None = None,
 ) -> list[httpx.Response]:
     ...
 
@@ -86,7 +90,8 @@ async def import_to_sisu(
     binary_search_max_depth: int | None = 0,
     group_by_key: str | None = None,
     binary_err_search_sublists: bool = False,
-    max_parallel_requests: int = 1
+    max_parallel_requests: int = 1,
+    params: dict | None = None,
 ) -> list[httpx.Response]:
     if fp:
         raise NotImplementedError("Not yet implemented")
@@ -112,6 +117,7 @@ async def import_to_sisu(
         group_by_key=group_by_key,
         method='POST',
         max_parallel_requests=max_parallel_requests,
+        params=params,
     )
 
     return responses
@@ -127,7 +133,8 @@ async def patch_to_sisu(
     binary_search_max_depth: int | None = 0,
     group_by_key: str | None = None,
     binary_err_search_sublists: bool = False,
-    max_parallel_requests: int = 1
+    max_parallel_requests: int = 1,
+    params: dict | None = None,
 ) -> list[httpx.Response]:
     if fp:
         raise NotImplementedError("Not yet implemented")
@@ -153,6 +160,7 @@ async def patch_to_sisu(
         group_by_key=group_by_key,
         method='PATCH',
         max_parallel_requests=max_parallel_requests,
+        params=params,
     )
 
     return responses