diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 8da1ff8c..643f930f 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -40,12 +40,12 @@ jobs: - name: Install dependencies run: | - uv sync --frozen --group docs + uv sync --frozen --group mddocs - name: Check changelog entry exists run: | - if [ ! -s docs/changelog/next_release/${{ github.event.pull_request.number }}.*.rst ]; then - echo "Please add corresponding file 'docs/changelog/next_release/..rst' with changes description" + if [ ! -s mddocs/docs/changelog/next_release/${{ github.event.pull_request.number }}.*.md ]; then + echo "Please add corresponding file 'mddocs/docs/changelog/next_release/..md' with changes description" exit 1 fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5db3d404..bf88fdb2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: release: name: Release package runs-on: ubuntu-latest - if: github.repository == 'MTSWebServices/syncmaster' # prevent running on forks + if: github.repository == 'MTSWebServices/syncmaster' # prevent running on forks environment: name: pypi @@ -44,7 +44,7 @@ jobs: - name: Fix logo in Readme run: | - sed -i "s#image:: docs/#image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/$GITHUB_SHA/docs/#g" README.rst + sed -i "s#image:: mddocs/docs/#image:: https://raw.githubusercontent.com/MTSWebServices/syncmaster/$GITHUB_SHA/mddocs/docs/#g" README.rst sed -i "s#logo_wide_red_text.svg#logo_wide.svg#g" README.rst - name: Patch version template @@ -62,48 +62,16 @@ jobs: uv pip install cyclonedx-bom uv export --all-extras --format requirements.txt | uv run cyclonedx-py requirements - > sbom.cyclonedx.json - - name: Get changelog - run: | - cat docs/changelog/$GITHUB_REF_NAME.rst > changelog.rst - - - name: Prepare rST syntax for conversion to Markdown - run: | - # Replace Github links from Sphinx syntax with Markdown - sed -i -E 's/:github:issue:`(.*)`/#\1/g' changelog.rst - sed -i -E 's/:github:pull:`(.*)`/#\1/g' changelog.rst - sed -i -E 's/:github:user:`(.*)`/@\1/g' changelog.rst - sed -i -E 's/:github:org:`(.*)`/@\1/g' changelog.rst - - - name: Convert rST to Markdown - uses: docker://pandoc/core:2.9 - with: - args: >- - --output=changelog.md - --from=rst - --to=gfm - --wrap=none - changelog.rst - - - name: Fixing Markdown syntax after conversion - run: | - # Replace ``` {.python caption="abc"} with ```python caption="abc" - sed -i -E 's/``` \{\.(.*)\}/```\1/g' changelog.md - - # Replace ``` python with ```python - sed -i -E 's/``` (\w+)/```\1/g' changelog.md - - # Replace \# with # - sed -i -E 's/\\#/#/g' changelog.md - - - name: Get release name + - name: Prepare changelog id: release-name run: | - # Release name looks like: 0.7.0 (2023-05-15) + cat mddocs/docs/changelog/${GITHUB_REF_NAME}.md > changelog.md + # Remove anchors like "{ #syncmaster-changelog-0-3-4 }" + sed -i -E 's/\{.*\}//g' changelog.md + # Header looks like "## [0.3.4](...) - 2026-05-26" + # Release name looks like "0.3.4 - 2026-05-26" echo -n name= > "$GITHUB_OUTPUT" cat changelog.md | head -1 | sed -E "s/#+\s*//g" >> "$GITHUB_OUTPUT" - - - name: Fix headers - run: | # Remove header with release name sed -i -e '1,2d' changelog.md @@ -119,5 +87,5 @@ jobs: files: | dist/*.tar.gz dist/*.whl - docs/_static/openapi.json + mddocs/docs/_static/openapi.json sbom.cyclonedx.json diff --git a/.readthedocs.yaml b/.readthedocs.yaml index ba5c43b9..3dc6dd45 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,22 +1,19 @@ version: 2 +mkdocs: + configuration: mddocs/mkdocs.yml + build: - os: ubuntu-22.04 - apt_packages: - - make + os: ubuntu-24.04 tools: - python: '3.13' + python: '3.11' jobs: - post_checkout: - - git fetch --unshallow || true post_create_environment: - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH python -m pip install uv post_install: - # avoid instaling worker dependencies - - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --no-install-project --extra server --extra scheduler --group docs - - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv pip install --no-deps sphinx-plantuml # remove after https://github.com/zqmillet/sphinx-plantuml/pull/4 + - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --no-install-project --extra server --group mddocs - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH python -m pip list -v - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH make docs-openapi - -sphinx: - configuration: docs/conf.py + pre_build: + - python mddocs/scripts/fix_refs.py + - python mddocs/scripts/links2anchors.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 99177288..a5fa3d3b 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -230,17 +230,13 @@ Settings are stored in ``config.yml`` file. Build documentation ~~~~~~~~~~~~~~~~~~~ -Build documentation using Sphinx & open it: +Build documentation using mkdocs: .. code:: bash - make docs + make docs-serve -If documentation should be build cleanly instead of reusing existing build result: - -.. code:: bash - - make docs-fresh +Then open in browser ``http://localhost:8000/``. Create pull request ~~~~~~~~~~~~~~~~~~~ @@ -264,7 +260,7 @@ Write release notes for changelog management. To submit a change note about your PR, add a text file into the -`docs/changelog/next_release <./next_release>`_ folder. It should contain an +`mddocs/docs/changelog/next_release <./next_release>`_ folder. It should contain an explanation of what applying this PR will change in the way end-users interact with the project. One sentence is usually enough but feel free to add as many details as you feel necessary @@ -275,22 +271,15 @@ combined with others, it will be a part of the "news digest" telling the readers **what changed** in a specific version of the library *since the previous version*. -You should also use -reStructuredText syntax for highlighting code (inline or block), -linking parts of the docs or external sites. -If you wish to sign your change, feel free to add ``-- by -:user:`github-username``` at the end (replace ``github-username`` -with your own!). - Finally, name your file following the convention that Towncrier understands: it should start with the number of an issue or a PR followed by a dot, then add a patch type, like ``feature``, -``doc``, ``misc`` etc., and add ``.rst`` as a suffix. If you +``doc``, ``misc`` etc., and add ``.md`` as a suffix. If you need to add more than one fragment, you may add an optional sequence number (delimited with another period) between the type and the suffix. -In general the name will follow ``..rst`` pattern, +In general the name will follow ``..md`` pattern, where the categories are: - ``feature``: Any new feature. Adding new functionality that has not yet existed. @@ -312,21 +301,15 @@ changes accompanying the relevant code changes. Examples for adding changelog entries to your Pull Requests ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. code-block:: rst - :caption: docs/changelog/next_release/1234.doc.1.rst - - Added a ``:github:user:`` role to Sphinx config -- by :github:user:`someuser` - -.. code-block:: rst - :caption: docs/changelog/next_release/2345.bugfix.rst +.. code-block:: markdown + :caption: mddocs/docs/changelog/next_release/2345.bugfix.md - Fixed behavior of ``server`` -- by :github:user:`someuser` + Fixed behavior of `server` -.. code-block:: rst - :caption: docs/changelog/next_release/3456.feature.rst +.. code-block:: markdown + :caption: mddocs/docs/changelog/next_release/3456.feature.md - Added support of ``timeout`` in ``LDAP`` - -- by :github:user:`someuser`, :github:user:`anotheruser` and :github:user:`otheruser` + Added support of `timeout` in `LDAP` .. tip:: @@ -350,51 +333,26 @@ Release Process Before making a release from the ``develop`` branch, follow these steps: -0. Checkout to ``develop`` branch and update it to the actual state +1. Checkout to ``develop`` branch and update it to the actual state .. code:: bash git checkout develop git pull -p -1. Backup ``NEXT_RELEASE.rst`` - -.. code:: bash - - cp "docs/changelog/NEXT_RELEASE.rst" "docs/changelog/temp_NEXT_RELEASE.rst" - -2. Build the Release notes with Towncrier +2. Get current release version .. code:: bash VERSION=$(cat syncmaster/VERSION) - towncrier build "--version=${VERSION}" --yes - -3. Change file with changelog to release version number - -.. code:: bash - - mv docs/changelog/NEXT_RELEASE.rst "docs/changelog/${VERSION}.rst" - -4. Remove content above the version number heading in the ``${VERSION}.rst`` file - -.. code:: bash - - awk '!/^.*towncrier release notes start/' "docs/changelog/${VERSION}.rst" > temp && mv temp "docs/changelog/${VERSION}.rst" - -5. Update Changelog Index - -.. code:: bash - - awk -v version=${VERSION} '/DRAFT/{print;print " " version;next}1' docs/changelog/index.rst > temp && mv temp docs/changelog/index.rst -6. Restore ``NEXT_RELEASE.rst`` file from backup +3. Build changelog for current release .. code:: bash - mv "docs/changelog/temp_NEXT_RELEASE.rst" "docs/changelog/NEXT_RELEASE.rst" + make docs-generate-changelog -7. Commit and push changes to ``develop`` branch +4. Commit and push changes to ``develop`` branch .. code:: bash @@ -402,7 +360,7 @@ Before making a release from the ``develop`` branch, follow these steps: git commit -m "Prepare for release ${VERSION}" git push -8. Merge ``develop`` branch to ``master``, **WITHOUT** squashing +5. Merge ``develop`` branch to ``master``, **WITHOUT** squashing .. code:: bash @@ -411,14 +369,14 @@ Before making a release from the ``develop`` branch, follow these steps: git merge develop git push -9. Add git tag to the latest commit in ``master`` branch +6. Add git tag to the latest commit in ``master`` branch .. code:: bash git tag "$VERSION" git push origin "$VERSION" -10. Update version in ``develop`` branch **after release**: +7. Update version in ``develop`` branch **after release**: .. code:: bash diff --git a/Makefile b/Makefile index f32a2a39..2b0c3df6 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,6 @@ venv-cleanup: ##@Env Cleanup venv venv-install: ##@Env Install requirements to venv ${UV} sync --inexact --frozen --extra "server" --extra "scheduler" --extra "worker" --all-groups $(ARGS) - ${UV} pip install --no-deps sphinx-plantuml @@ -175,21 +174,27 @@ prod-cleanup: ##@Application Stop production containers docs: docs-build docs-open ##@Docs Generate & open docs docs-build: ##@Docs Generate docs - ${UV} run $(MAKE) -C docs html + PYTHONPATH=. DISABLE_MKDOCS_2_WARNING=true ${VIRTUAL_ENV}/bin/mkdocs build --config-file mddocs/mkdocs.yml docs-open: ##@Docs Open docs - xdg-open docs/_build/html/index.html + xdg-open mddocs/generated/index.html docs-cleanup: ##@Docs Cleanup docs - $(MAKE) -C docs clean + rm -rf mddocs/generated/ docs-fresh: docs-cleanup docs-build ##@Docs Cleanup & build docs -docs-openapi: ##@Docs Generate OpenAPI schema - ${PYTHON} -m syncmaster.server.scripts.export_openapi_schema docs/_static/openapi.json +docs-serve: ##@Docs Run docs server + PYTHONPATH=. DISABLE_MKDOCS_2_WARNING=true ${VIRTUAL_ENV}/bin/mkdocs serve --config-file mddocs/mkdocs.yml -mddocs-build: mddocs-openapi ##@Docs Generate mkdocs documentation - PYTHONPATH=. DISABLE_MKDOCS_2_WARNING=true ${VIRTUAL_ENV}/bin/mkdocs build --strict --config-file mddocs/mkdocs.yml +docs-generate-changelog: ##@Docs Generate changelog + cp "mddocs/docs/changelog/RELEASE_TEMPLATE.md" "mddocs/docs/changelog/temp_RELEASE_TEMPLATE.md" + ${UV} run towncrier build "--version=$(shell cat syncmaster/VERSION)" --yes + mv "mddocs/docs/changelog/RELEASE_TEMPLATE.md" "mddocs/docs/changelog/$(shell cat syncmaster/VERSION).md" + mv "mddocs/docs/changelog/temp_RELEASE_TEMPLATE.md" "mddocs/docs/changelog/RELEASE_TEMPLATE.md" + awk '/##/,0' "mddocs/docs/changelog/$(shell cat syncmaster/VERSION).md" > temp && mv temp "mddocs/docs/changelog/$(shell cat syncmaster/VERSION).md" + sed "s#\(.*NEXT_RELEASE.*\)#\1\n- [$(shell cat syncmaster/VERSION) ($(shell date --rfc-3339=date))][$(shell cat syncmaster/VERSION | tr '.' '-')]#" "mddocs/docs/changelog/index.md" > temp && mv temp "mddocs/docs/changelog/index.md" + sed "s#\(.*NEXT_RELEASE.*\)#\1\n * [$(shell cat syncmaster/VERSION)](changelog/$(shell cat syncmaster/VERSION).md)#" "mddocs/docs/nav.md" > temp && mv temp "mddocs/docs/nav.md" -mddocs-openapi: ##@Docs Generate OpenAPI schema for mkdocs documentation +docs-openapi: ##@Docs Generate OpenAPI schema ${PYTHON} -m syncmaster.server.scripts.export_openapi_schema mddocs/docs/_static/openapi.json diff --git a/README.rst b/README.rst index b0c21684..fc890c4f 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ |Logo| -.. |Logo| image:: docs/_static/logo_wide_red_text.svg +.. |Logo| image:: mddocs/docs/_static/logo_wide_red_text.svg :alt: Data.SyncMaster logo :target: https://github.com/MTSWebServices/data-syncmaster diff --git a/docker-compose.test.yml b/docker-compose.test.yml index f134809c..2c6cc2b3 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -63,7 +63,7 @@ services: volumes: - ./config.docker.yml:/app/config.yml - ./syncmaster:/app/syncmaster - - ./docs/_static:/app/docs/_static + - ./docs/_static:/app/mddocs/docs/_static - ./reports:/app/reports - ./tests:/app/tests - ./pyproject.toml:/app/pyproject.toml @@ -105,7 +105,7 @@ services: dockerfile: docker/Dockerfile.worker context: . target: test - command: --queues 123-test_queue # Queue slug + command: --queues 123-test_queue # Queue slug entrypoint: [coverage, run, -m, celery, -A, tests.test_integration.celery_test, worker, --max-tasks-per-child=1] env_file: .env.docker.test environment: @@ -320,18 +320,18 @@ services: metastore-hive: condition: service_healthy ports: - - 9820:9820 # HDFS IPC - - 9870:9870 # HDFS WebHDFS - - 8088:8088 # Yarn UI - - 8042:8042 # NodeManager UI - - 10000:10000 # Hive server - - 10002:10002 # Hive server Admin UI - - 19888:19888 # MapReduce JobServer History UI - - 9083:9083 # Hive Metastore server + - 9820:9820 # HDFS IPC + - 9870:9870 # HDFS WebHDFS + - 8088:8088 # Yarn UI + - 8042:8042 # NodeManager UI + - 10000:10000 # Hive server + - 10002:10002 # Hive server Admin UI + - 19888:19888 # MapReduce JobServer History UI + - 9083:9083 # Hive Metastore server - 9864:9864 # Datanode UI environment: TZ: UTC - WITH_HIVE_SERVER: 'false' # We leave only the metastore server, we don’t need Hive itself, we don’t waste resources on it + WITH_HIVE_SERVER: 'false' # We leave only the metastore server, we don’t need Hive itself, we don’t waste resources on it HIVE_METASTORE_DB_URL: jdbc:postgresql://metastore-hive:5432/metastore HIVE_METASTORE_DB_DRIVER: org.postgresql.Driver HIVE_METASTORE_DB_USER: test_hive diff --git a/docker/Dockerfile.server b/docker/Dockerfile.server index d56e2f03..1b835ba6 100644 --- a/docker/Dockerfile.server +++ b/docker/Dockerfile.server @@ -13,7 +13,7 @@ WORKDIR /app ENV PYTHONPATH=/app \ PATH="/app/.venv/bin:$PATH" -COPY ./docs/_static/*.svg /app/syncmaster/server/static/ +COPY ./mddocs/docs/_static/*.svg /app/syncmaster/server/static/ # Swagger UI ADD --chmod=755 https://cdn.jsdelivr.net/npm/swagger-ui-dist@latest/swagger-ui-bundle.js https://cdn.jsdelivr.net/npm/swagger-ui-dist@latest/swagger-ui.css \ diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index d4bb2cbb..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -SPHINXBUILD ?= sphinx-build -SOURCEDIR = . -BUILDDIR = _build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_static/custom.css b/docs/_static/custom.css deleted file mode 100644 index dcf2652d..00000000 --- a/docs/_static/custom.css +++ /dev/null @@ -1,25 +0,0 @@ -@media not print { - @media (min-width: 1552px) { - img.readme-wide-logo { - padding-top: 3em; - } - } - - @media (prefers-color-scheme: dark) { - body:not([data-theme="light"]) img.sidebar-logo { - content: url(logo_red_text.svg); - } - - body:not([data-theme="light"]) img.readme-wide-logo { - content: url(logo_wide_red_text.svg); - } - } - - body[data-theme="dark"] img.sidebar-logo { - content: url(logo_red_text.svg); - } - - body[data-theme="dark"] img.readme-wide-logo { - content: url(logo_wide_red_text.svg); - } -} \ No newline at end of file diff --git a/docs/_static/icon.svg b/docs/_static/icon.svg deleted file mode 100644 index c3b69244..00000000 --- a/docs/_static/icon.svg +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/docs/_static/logo.svg b/docs/_static/logo.svg deleted file mode 100644 index 21c04ce5..00000000 --- a/docs/_static/logo.svg +++ /dev/null @@ -1,110 +0,0 @@ - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - diff --git a/docs/_static/logo_no_text.svg b/docs/_static/logo_no_text.svg deleted file mode 100644 index e442d8ed..00000000 --- a/docs/_static/logo_no_text.svg +++ /dev/null @@ -1,80 +0,0 @@ - - - - - - - - - - - - - - diff --git a/docs/_static/logo_original.svg b/docs/_static/logo_original.svg deleted file mode 100644 index c4114240..00000000 --- a/docs/_static/logo_original.svg +++ /dev/null @@ -1,120 +0,0 @@ - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - SyncMaster - - diff --git a/docs/_static/logo_red_text.svg b/docs/_static/logo_red_text.svg deleted file mode 100644 index 3684903f..00000000 --- a/docs/_static/logo_red_text.svg +++ /dev/null @@ -1,110 +0,0 @@ - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - diff --git a/docs/_static/logo_wide.svg b/docs/_static/logo_wide.svg deleted file mode 100644 index 0b8c8e0c..00000000 --- a/docs/_static/logo_wide.svg +++ /dev/null @@ -1,112 +0,0 @@ - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - diff --git a/docs/_static/logo_wide_original.svg b/docs/_static/logo_wide_original.svg deleted file mode 100644 index db962124..00000000 --- a/docs/_static/logo_wide_original.svg +++ /dev/null @@ -1,122 +0,0 @@ - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - yncMaster - - diff --git a/docs/_static/logo_wide_red_text.svg b/docs/_static/logo_wide_red_text.svg deleted file mode 100644 index 92762a58..00000000 --- a/docs/_static/logo_wide_red_text.svg +++ /dev/null @@ -1,112 +0,0 @@ - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - diff --git a/docs/_static/openapi.json b/docs/_static/openapi.json deleted file mode 100644 index 7b3bf552..00000000 --- a/docs/_static/openapi.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "openapi": "3.1.0", - "version": "unknown", - "info": { - "title": "Generated in CI", - "version": "unknown" - }, - "paths": {} -} \ No newline at end of file diff --git a/docs/_static/redoc.html b/docs/_static/redoc.html deleted file mode 100644 index b9f59a07..00000000 --- a/docs/_static/redoc.html +++ /dev/null @@ -1,28 +0,0 @@ - - - - - SyncMaster - ReDoc - - - - - - - - - - - - - - diff --git a/docs/_static/swagger.html b/docs/_static/swagger.html deleted file mode 100644 index ef34cf30..00000000 --- a/docs/_static/swagger.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - SwaggerUI - - - - -
- - - - diff --git a/docs/changelog.rst b/docs/changelog.rst deleted file mode 100644 index 4d895f92..00000000 --- a/docs/changelog.rst +++ /dev/null @@ -1,9 +0,0 @@ -========= -Changelog -========= - -.. toctree:: - :maxdepth: 1 - :caption: Changelog - - changelog/index diff --git a/docs/changelog/0.1.1.rst b/docs/changelog/0.1.1.rst deleted file mode 100644 index a98a5fd7..00000000 --- a/docs/changelog/0.1.1.rst +++ /dev/null @@ -1,3 +0,0 @@ -0.1.1 (2024-03-29) -================== -SyncMaster is now open source! diff --git a/docs/changelog/0.1.2.rst b/docs/changelog/0.1.2.rst deleted file mode 100644 index d551c010..00000000 --- a/docs/changelog/0.1.2.rst +++ /dev/null @@ -1,16 +0,0 @@ -0.1.2 (2024-04-15) -================== - -Features --------- - -- Rename ``syncmaster.backend.main`` module to ``syncmaster.backend``. It now also accepts the same args as ``uvicorn``. (:issue:`35`) -- Add ``syncmaster.db.migrations`` module to run ``alembic`` with proper config. (:issue:`35`) -- Change backend and worker images to use entrypoint. (:issue:`35`) - - -Improvements ------------- - -- Run database migrations in the entrypoint of backend image, before starting the backend server. (:issue:`35`) -- Add healthchecks to example ``docker-compose.yml``. (:issue:`35`) diff --git a/docs/changelog/0.1.3.rst b/docs/changelog/0.1.3.rst deleted file mode 100644 index 73390c74..00000000 --- a/docs/changelog/0.1.3.rst +++ /dev/null @@ -1,7 +0,0 @@ -0.1.3 (2024-04-15) -================== - -Bug Fixes ----------- - -Fix backend image entrypoint. diff --git a/docs/changelog/0.1.4.rst b/docs/changelog/0.1.4.rst deleted file mode 100644 index 6c3e528b..00000000 --- a/docs/changelog/0.1.4.rst +++ /dev/null @@ -1,8 +0,0 @@ -0.1.4 (2024-04-15) -================== - -Bug Fixes ----------- - -- Fix missing backend factory for uvicorn. -- Fix missing ``kinit`` executable in worker image. diff --git a/docs/changelog/0.1.5.rst b/docs/changelog/0.1.5.rst deleted file mode 100644 index 7df1abb8..00000000 --- a/docs/changelog/0.1.5.rst +++ /dev/null @@ -1,23 +0,0 @@ -0.1.5 (2024-04-22) -================== - -Breaking Changes ----------------- - -- Pass current ``Run`` to ``CREATE_SPARK_SESSION_FUNCTION``. This allows using run/transfer/group information for Spark session options, - like ``appName`` or custom ones. (:issue:`38`) - - -Improvements ------------- - -- Reduce backend image size (:issue:`44`) - - -Bug Fixes ---------- - -- Fix 500 error in case of ``PATCH v1/connections/:id`` request with passed ``auth_data.password`` field value (:issue:`39`) -- Do not use ``asyncio.gather`` with SQLAlchemy requests (:issue:`40`) -- Fix 500 error while creating HDFS connection (:issue:`41`) -- Fix missing ``options`` field from Transfer params with ``hdfs`` and ``s3`` type diff --git a/docs/changelog/0.2.0.rst b/docs/changelog/0.2.0.rst deleted file mode 100644 index cd268645..00000000 --- a/docs/changelog/0.2.0.rst +++ /dev/null @@ -1,75 +0,0 @@ -0.2.0 (2025-04-04) -================== - -TL;DR ------ - -- Completely new UI. -- Add support for FileSystem connections. -- Add support for simple transformations within transfer - filter files, filter rows, change columns. -- Add support for incremental read strategy. -- Add support for running transfers by schedule. -- Add support for changing SparkSession resource limits. - -Huge thanks to :user:`Dmitry Pedchenko `, :user:`Maxim Lixakov `, :user:`Ilyas Gasanov `, :user:`Kirill Chernikov `. - -Breaking Changes ----------------- - -- Implement a single error handling format to improve consistency (:issue:`95`) -- Change response format for ``GET /v1/groups`` - add **current user role** for each group (:issue:`97`) -- Change response format for ``GET /v1/groups/:id`` - add **current user role** for group (:issue:`109`) -- Now migrations are executed in a dedicated one-off container, instead of being run as a part of ``backend`` container. (:issue:`163`) -- Delete **transfers**, **queues** and **groups** records instead of marking them as deleted (:issue:`168`) -- Move the ``type`` field from nested ``connection_data`` field to the root level of the ``Connection`` response. (:issue:`169`) -- Decouple ``auth_data.type`` from the connection type (e.g. ``postgres``) and link it to the authentication type (e.g. ``basic``). (:issue:`169`) -- Add deletion of **connections** records instead of marking them as deleted (:issue:`170`) -- Use PUT instead of PATCH for ``Connection`` and ``Transfer`` models (:issue:`215`) -- Return new connection and transfer object in ``POST /v1/transfer/copy`` and ``POST /v1/connection/copy`` endpoints. -- Change response status from 200 to 204 for all ``DELETE`` endpoints. - -Features --------- - -- Add hive to known types (:issue:`67`) -- Allow search for **groups**, **users**, **connections**, **transfers**, **queues** (:issue:`92`, :issue:`94`, :issue:`99`, :issue:`100`, :issue:`101`, :issue:`103`) -- Add filters for **connections**, **transfers** and **runs** (:issue:`94`, :issue:`102`, :issue:`106`) -- Implement a scheduler to run celery tasks on a schedule. This can be done by setting ``Transfer.is_scheduled=True`` and ``Transfer.schedule="..."`` (cron-like expression). - The Run model now has a ``type`` field with options ``MANUAL`` and ``SCHEDULED``. (:issue:`114`) -- Add GET ``v1/monitoring/metrics`` endpoint to provide basic HTTP server metrics in Prometheus format (:issue:`121`) -- Implemented ``KeycloakAuthProvider`` for Single Sign-On (SSO) authentication.(:issue:`123`) -- Implemented ``DummyAuthProvider`` for development and testing environments. (:issue:`123`) -- Add API schemas for new DB sources - Clickhouse, MSSQL, MySQL (:issue:`124`, :issue:`125`, :issue:`126`, :issue:`160`) -- Add logic for handling FTP, FTPS, SFTP, Samba, WebDAV transfers (:issue:`189`, :issue:`191`, :issue:`192`, :issue:`194`) -- Add API schemas for file sources - SFTP, FTP, FTPS, WebDAV, Samba (:issue:`187`) -- Add API schemas for file formats - Excel, XML, ORC, Parquet (:issue:`140`, :issue:`142`, :issue:`143`, :issue:`144`) -- Add compression options to file formats CSV, JSON, JSONLine, Excel, ORC, Parquet, XML (:issue:`159`, :issue:`161`) -- Add transformations for **Transfers** with dataframe row filtering (:issue:`184`) -- Add transformations for **Transfers** with dataframe column filtering (:issue:`186`) -- Add transformations for **Transfers** with file filtering (:issue:`198`) -- Add ``increment_by`` field to ``strategy_params`` (:issue:`202`) -- Implement increment strategy for transfers with file sources (:issue:`209`) -- Implement increment strategy for transfers with database sources (:issue:`211`) -- Add ``resources`` field to ``Transfer``. (:issue:`214`) -- Add ``file_name_template`` field to ``target_params`` (:issue:`196`, :issue:`201`) - -Improvements ------------- - -- Updated ``User`` model to include ``email``, ``first_name``, ``middle_name``, and ``last_name`` fields, all optional. (:issue:`123`) -- Read env variable ``SYNCMASTER__ENTRYPOINT__SUPERUSERS`` to promote users to ``SUPERUSER`` role during server startup. (:issue:`137`) -- Enabled dynamic selection of authentication provider via environment variable ``SYNCMASTER__AUTH__PROVIDER``. (:issue:`123`) -- Enable parallel reading from JDBC sources. (:issue:`219`) -- Reset HWM when changing strategy from ``incremental`` to ``full``. (:issue:`217`) -- Grant read-only permissions for the previous group owner when ownership is transferred (:issue:`135`) - -Bug Fixes ---------- - -- Use Hadoop AWS ``magic`` committer only if transfer *target* is S3. (:issue:`46`) -- Check that ``service_name`` and ``sid`` are mutually exclusive when editing Oracle connection. (:issue:`52`) -- Queue name is unique within a group, new field ``slug`` is globally-unique. (:issue:`54`, :issue:`119`) -- Prohibit updating connection type it if there is a transfer associated with this connection. (:issue:`55`) -- Fix error when ``is_scheduled`` field value was ignored. (:issue:`57`) -- Group without any users assigned was missing in groups list. (:issue:`62`) -- Dump connection credentials while starting a transfer. (:issue:`63`) diff --git a/docs/changelog/0.2.1.rst b/docs/changelog/0.2.1.rst deleted file mode 100644 index 820491ee..00000000 --- a/docs/changelog/0.2.1.rst +++ /dev/null @@ -1,14 +0,0 @@ -0.2.1 (2025-04-07) -================== - -Improvements ------------- - -- Change docker image user from ``root`` to ``syncmaster``, to improve security. -- Move server healthcheck to Docker image. -- SBOM file is generated on release. - -Bug fixes ---------- - -- Fix missing Swagger docs in prod image. diff --git a/docs/changelog/0.2.2.rst b/docs/changelog/0.2.2.rst deleted file mode 100644 index 8440bd98..00000000 --- a/docs/changelog/0.2.2.rst +++ /dev/null @@ -1,23 +0,0 @@ -0.2.2 (2025-04-11) -================== - -Breaking --------- - -- Use ``PUT /v1/qroups/:id`` instead of ``PATCH /v1/qroups/:id``. -- Use ``PUT /v1/qroups/:id/users/:id`` instead of ``PATCH /v1/qroups/:id/users/:id``. -- Use ``PUT /v1/queues/:id`` instead of ``PATCH /v1/queues/:id``. -- Now allowed names length should be in 3..128 symbols range, not 1..inf. - -Improvements ------------- - -- Now queue name can include any ASCII printable characters. -- Queue slug is always lowercase. Spaces, hyphens and underscores are replaced with ``-`` symbol. - -Bug fixes ---------- - -- Call ``kinit`` before starting Spark session connecting to ``Hive`` cluster. (:issue:`225`) -- Fix ``HDFS`` connection was trying to use anonymous auth instead of user/password. (:issue:`225`) -- Fix updating queue ignored name and didn't reset description. diff --git a/docs/changelog/0.2.3.rst b/docs/changelog/0.2.3.rst deleted file mode 100644 index 77138713..00000000 --- a/docs/changelog/0.2.3.rst +++ /dev/null @@ -1,12 +0,0 @@ -0.2.3 (2025-04-11) -================== - -Bug fixes ---------- - -- Fix Worker not updating Run ``status`` and ``ended_at`` fields after executing a very long ETL process. - -Improvements ------------- - -- Change Celery log level from DEBUG to INFO. diff --git a/docs/changelog/0.2.4.rst b/docs/changelog/0.2.4.rst deleted file mode 100644 index 8d90fb24..00000000 --- a/docs/changelog/0.2.4.rst +++ /dev/null @@ -1,8 +0,0 @@ -0.2.4 (2025-07-09) -================== - -Improvements ------------- - -Include all required jars from Maven to worker image. This increases image size, but drastically reduces time of Spark session startup. - diff --git a/docs/changelog/0.2.5.rst b/docs/changelog/0.2.5.rst deleted file mode 100644 index e10f2113..00000000 --- a/docs/changelog/0.2.5.rst +++ /dev/null @@ -1,21 +0,0 @@ -0.2.5 (2025-10-10) -================== - -Features --------- - -- Implement Keycloak login page on frontend. (`#128 `_) -- Implement ``GET /v1/auth/logout`` endpoint for ``KeycloakAuthProvider``. (:issue:`275`) - -Improvements ------------- - -- Improved full-text search for technical fields such as hostnames, table names, and directory paths. (:issue:`255`) -- Replace 307 redirect to Keycloak auth page with 401 response, due to browser restrictions for redirect + CORS + localhost. (:issue:`274`) - -Bug Fixes ---------- - -- Replace sync methods of Keycloak client with async ones. (:issue:`177`) - - Previously interaction with Keycloak could block asyncio event loop. diff --git a/docs/changelog/0.3.0.rst b/docs/changelog/0.3.0.rst deleted file mode 100644 index 5253631f..00000000 --- a/docs/changelog/0.3.0.rst +++ /dev/null @@ -1,105 +0,0 @@ -0.3.0 (2025-12-18) -================== - -Release of Data.SyncMaster 0.3.0 brings up support for Iceberg, Spark-on-K8s and Spark-on-Yarn. - -.. note:: - - Currently Spark-on-K8s and Spark-on-Yarn do not support FTP, FTPS, SFTP, Samba and WebDAV. - -Breaking Changes ----------------- - -- Worker container command should be changed from ``--queues 123-myqueue`` to ``worker --queues 123-myqueue`` (:issue:`295`). - -- Application should be configured via ``config.yml`` file (:issue:`289`). - - It's still possible to use environment variables instead. But it is not recommended for security reasons, as docker/k8s envs can be read by other users. - - Other notable changes: - * Environment variable ``SYNCMASTER__ENTRYPOINT__SUPERUSERS`` is renamed to ``SYNCMASTER__SUPERUSERS``. - * Logging format is configured explicitly via ``config.yml`` instead of having few predefined configuration files. - -- Moved ``server.session`` middleware settings to ``auth`` block (:issue:`304`). - Also rename some fields in ``auth.keycloak`` settings block. - - .. dropdown:: Before vs after - - Before: - - .. code:: yaml - - auth: - provider: ... - keycloak: - server_url: ... - redirect_url: ... - - server: - session: - enabled: true - secret_key: ... - - Now: - - .. code:: yaml - - auth: - provider: - keycloak: - api_url: ... - ui_callback_url: ... - cookie: - secret_key: ... - - -Features --------- - -- Added Iceberg support (:issue:`282`, :issue:`284`, :issue:`294`, :issue:`297`). - - Iceberg connection currently supports only Iceberg REST Catalog with S3 warehouse. - -- Allow using SyncMaster worker image as ``spark.kubernetes.container.image``. (:issue:`295`) - -- Allow passing default Spark session config via worker settings (:issue:`291`): - - .. dropdown:: Example - - .. code-block:: yaml - :caption: config.yml - - worker: - spark_session_default_config: - spark.master: local - spark.driver.host: 127.0.0.1 - spark.driver.bindAddress: 0.0.0.0 - spark.sql.pyspark.jvmStacktrace.enabled: true - spark.ui.enabled: false - -- Added OAuth2GatewayProvider (:issue:`283`). - - This allows using Data.SyncMaster under OAuth2 Gateway. Implementation is similar to DummyAuthProvider. - -- Allow disabling ``SessionMiddleware``, as it only required by ``KeycloakAuthProvider``. - -- Add hooks support to worker classes (TransferController, Handler) (:issue:`279`). -- Pass transfer name and group name to Handlers (:issue:`308`). - -Improvements ------------- - -- Make S3 connection ``region`` a mandatory option, to prevent possible errors. -- Hide ``database_name`` from Clickhouse and MySQL connection pages. -- Frontend: add placeholders to connection params, like host, port and so on. -- Sync frontend and backend checks for some field patterns, e.g. table name should be in format ``schema.table``. -- Improve OpenAPI schema fields description. - -Bug Fixes ---------- - -Fix some file format options were ignored by SyncMaster worker: - * XML: ``root_tag``, ``row_tag`` - * Excel ``start_cell``, ``include_header`` - * CSV ``include_header``, ``line_sep`` - * JSON, JSONLine: ``line_sep`` diff --git a/docs/changelog/0.3.1.rst b/docs/changelog/0.3.1.rst deleted file mode 100644 index 286a2724..00000000 --- a/docs/changelog/0.3.1.rst +++ /dev/null @@ -1,17 +0,0 @@ -0.3.1 (2026-01-15) -================== - -Improvements ------------- - -- Added table name validation - must be in format ``schema.table`` or ``namespace1.namespace2.table``. -- Cut 400MB of worker images size by removing copies of ``.jar`` files. -- Explicitly log transfer exception instead of ``raised unexpected: UnpickleableExceptionWrapper`` by celery. - -Misc ----- - -- Switch from ``poetry`` to ``uv`` for dependency management. -- Switch from ``black`` to ``ruff format``. -- Switch from ``flake8+wemake-python-styleguide`` to ``ruff check``, and fix found code smells. -- Dependency updates. diff --git a/docs/changelog/0.3.2.rst b/docs/changelog/0.3.2.rst deleted file mode 100644 index 1c59a46f..00000000 --- a/docs/changelog/0.3.2.rst +++ /dev/null @@ -1,7 +0,0 @@ -0.3.2 (2026-03-05) -================== - -Features --------- - -Implement SQL transformations for transfer (:issue:`327`, :issue:`330`) diff --git a/docs/changelog/0.3.3.rst b/docs/changelog/0.3.3.rst deleted file mode 100644 index 45905f6b..00000000 --- a/docs/changelog/0.3.3.rst +++ /dev/null @@ -1,9 +0,0 @@ -0.3.3 (2026-05-26) -================== - -Dependencies ------------- - -Updated ClickHouse ``spark-dialect-extension`` to `0.0.4 `_. (:issue:`371`) - -This brings support for writing nullable DataFrame columns to Clickhouse table. diff --git a/docs/changelog/0.3.4.rst b/docs/changelog/0.3.4.rst deleted file mode 100644 index 67fc4a46..00000000 --- a/docs/changelog/0.3.4.rst +++ /dev/null @@ -1,8 +0,0 @@ -0.3.4 (2026-05-26) -================== - -Bug fixes ---------- - -- ``GET /v1/groups`` returned ``role: SUPERUSER`` for superuser, but ``GET /v1/groups/:id`` didn't. -- Fix UI permission checks were not compliant with documentation for user roles. diff --git a/docs/changelog/DRAFT.rst b/docs/changelog/DRAFT.rst deleted file mode 100644 index d396f605..00000000 --- a/docs/changelog/DRAFT.rst +++ /dev/null @@ -1 +0,0 @@ -.. towncrier-draft-entries:: |release| [UNRELEASED] diff --git a/docs/changelog/NEXT_RELEASE.rst b/docs/changelog/NEXT_RELEASE.rst deleted file mode 100644 index d366073a..00000000 --- a/docs/changelog/NEXT_RELEASE.rst +++ /dev/null @@ -1 +0,0 @@ -.. towncrier release notes start \ No newline at end of file diff --git a/docs/changelog/index.rst b/docs/changelog/index.rst deleted file mode 100644 index 5959c688..00000000 --- a/docs/changelog/index.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. toctree:: - :maxdepth: 1 - :caption: Changelog - - DRAFT - 0.3.4 - 0.3.3 - 0.3.2 - 0.3.1 - 0.3.0 - 0.2.5 - 0.2.4 - 0.2.3 - 0.2.2 - 0.2.1 - 0.2.0 - 0.1.5 - 0.1.4 - 0.1.3 - 0.1.2 - 0.1.1 diff --git a/docs/changelog/next_release/.keep b/docs/changelog/next_release/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/conf.py b/docs/conf.py deleted file mode 100644 index 9d2ee8fe..00000000 --- a/docs/conf.py +++ /dev/null @@ -1,174 +0,0 @@ -# SPDX-FileCopyrightText: 2023-present MTS PJSC -# SPDX-License-Identifier: Apache-2.0 -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - -# -- Path setup -------------------------------------------------------------- - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. - - -import os -import sys -from pathlib import Path - -from packaging import version as Version - -PROJECT_ROOT_DIR = Path(__file__).parent.parent.resolve() - -sys.path.insert(0, os.fspath(PROJECT_ROOT_DIR)) - -# -- Project information ----------------------------------------------------- - -project = "syncmaster" -copyright = "2023-2025 MTS PJSC" -author = "MWS Data Bridge" - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. - -VERSION_FILE = PROJECT_ROOT_DIR / "syncmaster" / "VERSION" -ver = Version.parse(VERSION_FILE.read_text()) - -version = ver.base_version -# The full version, including alpha/beta/rc tags. -release = ver.public - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "numpydoc", - "sphinx_copybutton", - "sphinx.ext.doctest", - "sphinx.ext.autodoc", - "sphinx.ext.autosummary", - "sphinx.ext.intersphinx", - "sphinxcontrib.autodoc_pydantic", - "sphinxcontrib.towncrier", # provides `towncrier-draft-entries` directive - "sphinx_issues", - "sphinx_design", # provides `dropdown` directive - "sphinxcontrib.plantuml", - "sphinx_favicon", - "sphinxarg.ext", - "sphinx_last_updated_by_git", -] - -swagger = [ - { - "name": "SyncMaster REST API", - "page": "openapi", - "id": "syncmaster-api", - "options": { - "url": "_static/openapi.json", - }, - }, -] - -numpydoc_show_class_members = True -autodoc_pydantic_model_show_config = False -autodoc_pydantic_model_show_config_summary = False -autodoc_pydantic_model_show_config_member = False -autodoc_pydantic_model_show_json = False -autodoc_pydantic_model_show_validator_summary = False -autodoc_pydantic_model_show_validator_members = False -autodoc_pydantic_model_member_order = "bysource" -autodoc_pydantic_settings_show_config = False -autodoc_pydantic_settings_show_config_summary = True -autodoc_pydantic_settings_show_config_member = False -autodoc_pydantic_settings_show_json = False -autodoc_pydantic_settings_show_validator_summary = False -autodoc_pydantic_settings_show_validator_members = False -autodoc_pydantic_settings_member_order = "bysource" -autodoc_pydantic_field_list_validators = False -sphinx_tabs_disable_tab_closing = True - -# prevent >>>, ... and doctest outputs from copying -copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: " -copybutton_prompt_is_regexp = True -copybutton_copy_empty_lines = False -copybutton_only_copy_prompt_lines = True - -towncrier_draft_autoversion_mode = "draft" -towncrier_draft_include_empty = False -towncrier_draft_working_directory = PROJECT_ROOT_DIR - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. - -html_theme = "furo" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -html_theme_options = { - "sidebar_hide_name": True, - "top_of_page_buttons": [], -} - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] -html_extra_path = ["robots.txt"] - -html_css_files = [ - "custom.css", -] - -html_logo = "./_static/logo.svg" -favicons = [ - {"rel": "icon", "href": "icon.svg", "type": "image/svg+xml"}, -] -html_copy_source = False -html_show_sourcelink = False -git_exclude_patterns = ["docs/_static/*.svg"] - -# The master toctree document. -master_doc = "index" - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = "en" - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = False - - -# -- Options for HTMLHelp output ------------------------------------------ - -# Output file base name for HTML help builder. -htmlhelp_basename = "syncmaster-doc" - - -# which is the equivalent to: -issues_uri = "https://github.com/MTSWebServices/syncmaster/issues/{issue}" -issues_pr_uri = "https://github.com/MTSWebServices/syncmaster/pulls/{pr}" -issues_commit_uri = "https://github.com/MTSWebServices/syncmaster/commit/{commit}" -issues_user_uri = "https://github.com/{user}" diff --git a/docs/contributing.rst b/docs/contributing.rst deleted file mode 100644 index e582053e..00000000 --- a/docs/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../CONTRIBUTING.rst diff --git a/docs/design/entities/connection_info_db.png b/docs/design/entities/connection_info_db.png deleted file mode 100644 index 02d800d2..00000000 Binary files a/docs/design/entities/connection_info_db.png and /dev/null differ diff --git a/docs/design/entities/connection_info_fs.png b/docs/design/entities/connection_info_fs.png deleted file mode 100644 index 229f0cc0..00000000 Binary files a/docs/design/entities/connection_info_fs.png and /dev/null differ diff --git a/docs/design/entities/connection_list.png b/docs/design/entities/connection_list.png deleted file mode 100644 index 5a6f9d3c..00000000 Binary files a/docs/design/entities/connection_list.png and /dev/null differ diff --git a/docs/design/entities/create_transfer_advanced.png b/docs/design/entities/create_transfer_advanced.png deleted file mode 100644 index 9b111190..00000000 Binary files a/docs/design/entities/create_transfer_advanced.png and /dev/null differ diff --git a/docs/design/entities/create_transfer_advanced_filter_columns.png b/docs/design/entities/create_transfer_advanced_filter_columns.png deleted file mode 100644 index 97acdc16..00000000 Binary files a/docs/design/entities/create_transfer_advanced_filter_columns.png and /dev/null differ diff --git a/docs/design/entities/create_transfer_advanced_filter_files.png b/docs/design/entities/create_transfer_advanced_filter_files.png deleted file mode 100644 index 73f2dc43..00000000 Binary files a/docs/design/entities/create_transfer_advanced_filter_files.png and /dev/null differ diff --git a/docs/design/entities/create_transfer_advanced_filter_rows.png b/docs/design/entities/create_transfer_advanced_filter_rows.png deleted file mode 100644 index 7aad8d60..00000000 Binary files a/docs/design/entities/create_transfer_advanced_filter_rows.png and /dev/null differ diff --git a/docs/design/entities/create_transfer_footer.png b/docs/design/entities/create_transfer_footer.png deleted file mode 100644 index a051b066..00000000 Binary files a/docs/design/entities/create_transfer_footer.png and /dev/null differ diff --git a/docs/design/entities/create_transfer_head.png b/docs/design/entities/create_transfer_head.png deleted file mode 100644 index 7c6bf5cf..00000000 Binary files a/docs/design/entities/create_transfer_head.png and /dev/null differ diff --git a/docs/design/entities/create_transfer_source_target.png b/docs/design/entities/create_transfer_source_target.png deleted file mode 100644 index b8b19504..00000000 Binary files a/docs/design/entities/create_transfer_source_target.png and /dev/null differ diff --git a/docs/design/entities/group_add_member.png b/docs/design/entities/group_add_member.png deleted file mode 100644 index 9d88c8e2..00000000 Binary files a/docs/design/entities/group_add_member.png and /dev/null differ diff --git a/docs/design/entities/group_info.png b/docs/design/entities/group_info.png deleted file mode 100644 index f502fbbc..00000000 Binary files a/docs/design/entities/group_info.png and /dev/null differ diff --git a/docs/design/entities/group_list.png b/docs/design/entities/group_list.png deleted file mode 100644 index b4493e25..00000000 Binary files a/docs/design/entities/group_list.png and /dev/null differ diff --git a/docs/design/entities/index.rst b/docs/design/entities/index.rst deleted file mode 100644 index 79e168be..00000000 --- a/docs/design/entities/index.rst +++ /dev/null @@ -1,182 +0,0 @@ -.. _entities: - -Entities -======== - -User ----- - -SyncMaster is designed with multitenancy support and role-based access (see :ref:`role-permissions`). -All nteraction requires user authentication, there is no anonymous access allowed. - -Users are automatically after successful login, there is no special registration step. - -Group ------ - -All entity types (Connection, Transfer, Run, Queue) can be created only within some group. -Groups are independent from each other, and have globally unique name. - -.. image:: ./group_list.png -.. image:: ./group_info.png - -Group can be created by any user, which automatically get ``OWNER`` role. -This role allows adding members to the group, and assign them specific roles: - -.. image:: ./group_add_member.png - -Connection ----------- - -Connection describes how SyncMaster can access specific database or filesystem. It has a type (e.g. ``s3``, ``hive``, ``postgres``), -connection parameters (e.g. ``host``, ``port``, ``protocol``) and auth data (``user`` / ``password`` combination). - -Connections have unique name within the group. - -.. image:: ./connection_list.png -.. image:: ./connection_info_db.png -.. image:: ./connection_info_fs.png - -Transfer --------- - -Transfer is the heart of SyncMaster. It describes what some data should be fetched from a source (DB connection + table name, FileSystem connection + directory path), -and what the target is (DB or FileSystem). - -Transfers have unique name within a group. - -.. image:: ./transfer_list.png -.. image:: ./create_transfer_head.png -.. image:: ./create_transfer_source_target.png -.. image:: ./create_transfer_advanced.png - -It is possible to add transformations between reading and writing steps: - -.. image:: ./create_transfer_advanced_filter_files.png -.. image:: ./create_transfer_advanced_filter_rows.png -.. image:: ./create_transfer_advanced_filter_columns.png - -Other transfer features are: - * Choose different read strategies (``full``, ``incremental``) - * Execute transfer on schedule (hourly, daily, weekly and so on) - * Set specific resources (CPU, RAM) for each transfer run - -.. image:: ./create_transfer_footer.png - -Run ---- - -Each time transfer is started (manually or at some schedule), SyncMaster creates dedicated Run -which tracks the ETL process status, URL to worker logs and so on. - -.. image:: ./run_list.png -.. image:: ./run_info.png - -Queue ------ - -Queue allows to bind specific transfer to a set of SyncMaster :ref:`worker`. - -Queue have unique name within a group, and globally unique ``slug`` field which is generated during queue creation. - -.. image:: ./queue_list.png -.. image:: ./queue_info.png - -Transfers cannot be created without queue. If there are no workers bound to a queue, created runs will not be executed. - - -Entity Diagram --------------- - -.. plantuml:: - - @startuml - title Entity Diagram - - left to right direction - - entity User { - * id - ---- - username - is_active - is_superuser - created_at - updated_at - } - - entity Group { - * id - ---- - name - description - owner_id - created_at - updated_at - } - - entity Connection { - * id - ---- - group_id - type - name - description - data - created_at - updated_at - } - - entity Queue { - * id - ---- - name - slug - group_id - description - created_at - updated_at - } - - entity Transfer { - * id - ---- - group_id - name - source_connection_id - target_connection_id - strategy_params - target_params - transformations - resources - is_scheduled - schedule - queue_id - created_at - updated_at - } - - entity Run { - * id - ---- - transfer_id - started_at - ended_at - status - type - log_url - transfer_dump - created_at - updated_at - } - - Run ||--o{ Transfer - Transfer ||--o{ Queue - Transfer ||--o{ Connection - Transfer ||--o{ Group - Connection ||--o{ Group - Queue ||--o{ Group - Group }o--o{ User - Group "owner_id" ||--o{ User - - @enduml diff --git a/docs/design/entities/queue_info.png b/docs/design/entities/queue_info.png deleted file mode 100644 index 9f1d844b..00000000 Binary files a/docs/design/entities/queue_info.png and /dev/null differ diff --git a/docs/design/entities/queue_list.png b/docs/design/entities/queue_list.png deleted file mode 100644 index 29e44036..00000000 Binary files a/docs/design/entities/queue_list.png and /dev/null differ diff --git a/docs/design/entities/run_info.png b/docs/design/entities/run_info.png deleted file mode 100644 index 4f9c44fc..00000000 Binary files a/docs/design/entities/run_info.png and /dev/null differ diff --git a/docs/design/entities/run_list.png b/docs/design/entities/run_list.png deleted file mode 100644 index 0dd37194..00000000 Binary files a/docs/design/entities/run_list.png and /dev/null differ diff --git a/docs/design/entities/transfer_list.png b/docs/design/entities/transfer_list.png deleted file mode 100644 index 94264f8c..00000000 Binary files a/docs/design/entities/transfer_list.png and /dev/null differ diff --git a/docs/design/permissions.rst b/docs/design/permissions.rst deleted file mode 100644 index 6736307c..00000000 --- a/docs/design/permissions.rst +++ /dev/null @@ -1,168 +0,0 @@ -.. _role-permissions: - -Roles and permissions -===================== - - -Object within the group can be seen/interacted with only by users which are members of the group. -Permissions are limited by role assigned to user within specific group. - -Roles are: - -* ``GUEST`` - Read-only access to objects within a group. -* ``DEVELOPER`` - Read-write (manage) connections, transfers and runs. Read-only for queues. -* ``MAINTAINER`` (DevOps): - Manage connections, transfers, runs and queues. -* ``OWNER`` (Product Owner) - Manage connections, transfers, runs, queues and user-group membership. Group can have only one owner. -* ``SUPERUSER`` (Admin) - Meta role assigned to specific users, NOT within group. All permissions, including ability to create/delete groups. - Superusers are created by :ref:`manage-superusers-cli`. - -Groups -------- - -.. list-table:: Rights to work with the groups repository. - :header-rows: 1 - - * - Rule \ Role - - Guest - - Developer - - Maintainer - - Owner - - Superuser - * - READ - - x - - x - - x - - x - - x - * - UPDATE - - - - - - - - x - - x - * - CREATE - - x - - x - - x - - x - - x - * - DELETE - - - - - - - - - - x - -Add user to the group and delete ---------------------------------- -Each user has the right to remove himself from a group, regardless of his role in the group. - -.. list-table:: Rights to add/delete users to a group - :header-rows: 1 - - * - Rule \ Role - - Guest - - Developer - - Maintainer - - Owner - - Superuser - * - READ - - x - - x - - x - - x - - x - * - ADD, UPDATE - - - - - - - - x - - x - -Transfers, and Connections --------------------------- - -.. list-table:: Right to work with Transfers and Connections within a group - :header-rows: 1 - - - * - Rule \ Role - - Guest - - Developer - - Maintainer - - Owner - - Superuser - * - READ - - x - - x - - x - - x - - x - * - UPDATE, CREATE - - - - x - - x - - x - - x - * - DELETE - - - - - - x - - x - - x - -Runs ----- - -.. list-table:: Right to work with Runs within a group - :header-rows: 1 - - - * - Rule \ Role - - Guest - - Developer - - Maintainer - - Owner - - Superuser - * - READ - - x - - x - - x - - x - - x - * - CREATE (START), STOP - - - - x - - x - - x - - x - -Queues ------- - -.. list-table:: Rights to work with Queues within a namespace - :header-rows: 1 - - * - Rule \ Role - - Guest - - Developer - - Maintainer - - Owner - - Superuser - * - READ - - x - - x - - x - - x - - x - * - UPDATE, DELETE, CREATE - - - - - - x - - x - - x diff --git a/docs/index.rst b/docs/index.rst deleted file mode 100644 index 546a81bc..00000000 --- a/docs/index.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. toctree:: - :maxdepth: 2 - :caption: Data.SyncMaster - :hidden: - - self - -.. toctree:: - :maxdepth: 2 - :caption: High-level design - :hidden: - - design/entities/index - design/permissions - -.. toctree:: - :maxdepth: 2 - :caption: Reference - :hidden: - - reference/architecture - reference/database/index - reference/broker/index - reference/server/index - reference/frontend/index - reference/worker/index - reference/scheduler/index - -.. toctree:: - :maxdepth: 2 - :caption: Development - :hidden: - - changelog - contributing - security - -.. include:: ../README.rst - :end-before: |Logo| - -.. image:: _static/logo_wide.svg - :alt: Data.SyncMaster logo - :target: https://github.com/MTSWebServices/data-syncmaster - :class: readme-wide-logo - -.. include:: ../README.rst - :start-after: |Logo| - :end-before: documentation diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 53ad1e82..00000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=. -set BUILDDIR=_build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.https://www.sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/reference/architecture.png b/docs/reference/architecture.png deleted file mode 100644 index 4bea0bc5..00000000 Binary files a/docs/reference/architecture.png and /dev/null differ diff --git a/docs/reference/architecture.rst b/docs/reference/architecture.rst deleted file mode 100644 index c19befa8..00000000 --- a/docs/reference/architecture.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _architecture: - -Architecture -============ - -Components ----------- - -SyncMaster contains the following components: - -* :ref:`frontend`, main user interface. -* :ref:`server`, providing REST API for fetching and manipulating entities. -* :ref:`worker`, performing actual transfer work (ETL processes). -* :ref:`scheduler`, scheduling transfers to be executed in future. -* :ref:`database` for storing internal data. -* :ref:`message-broker` for communications between Server/Scheduler and Worker. - -Architecture diagram --------------------- - -.. image:: ./architecture.png diff --git a/docs/reference/broker/index.rst b/docs/reference/broker/index.rst deleted file mode 100644 index 60669608..00000000 --- a/docs/reference/broker/index.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. _message-broker: - -Message Broker -============== - -Message broker is componen used by :ref:`server`/:ref:`scheduler` to communicate with :ref:`worker`. - -SyncMaster can work virtually with any broker supported by `Celery `_. -But the only broker we tested is `RabbitMQ `_. - -Requirements ------------- - -* RabbitMQ 4.x. It is recommended to use latest RabbitMQ version. - -Setup -~~~~~ - -With Docker -^^^^^^^^^^^ - -* Install `Docker `_ -* Install `docker-compose `_ -* Run the following command: - - .. code:: console - - $ docker compose --profile broker up -d --wait - - ``docker-compose`` will download RabbitMQ image, create container and volume, and then start container. - Image entrypoint will create database if volume is empty. - - .. dropdown:: ``docker-compose.yml`` - - .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 34-46,134 - - Options can be set via ``config.yml`` file: - - .. dropdown:: ``config.yml`` - - .. literalinclude:: ../../../config.yml - :emphasize-lines: 4-5 - -Without Docker -^^^^^^^^^^^^^^ - -Please follow `RabbitMQ installation instruction `_. diff --git a/docs/reference/database/configuration.rst b/docs/reference/database/configuration.rst deleted file mode 100644 index 17898a3a..00000000 --- a/docs/reference/database/configuration.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _configuration-database: - -Database settings -================= - -.. autopydantic_model:: syncmaster.settings.DatabaseSettings diff --git a/docs/reference/database/credentials_encryption.rst b/docs/reference/database/credentials_encryption.rst deleted file mode 100644 index 33de600d..00000000 --- a/docs/reference/database/credentials_encryption.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _configuration-credentials-encryption: - -Credentials encryption -====================== - -.. autopydantic_model:: syncmaster.settings.credentials.CredentialsEncryptionSettings diff --git a/docs/reference/database/index.rst b/docs/reference/database/index.rst deleted file mode 100644 index 78f0d031..00000000 --- a/docs/reference/database/index.rst +++ /dev/null @@ -1,111 +0,0 @@ -.. _database: - -Relation Database -================= - -SyncMaster requires relational database for storing internal data. - -Currently, SyncMaster supports only `PostgreSQL `_. - -Migrations ------------- - -After a database is started, it is required to run migration script. -For empty database, it creates all the required tables and indexes. -For non-empty database, it will perform database structure upgrade, using `Alembic `_. - -.. warning:: - - Other containers (server, scheduler, worker) should be stopped while running migrations, to prevent interference. - -Requirements ------------- - -* PostgreSQL 12 or higher. It is recommended to use latest Postgres version. - -Install & run -------------- - -With Docker -~~~~~~~~~~~ - -* Install `Docker `_ -* Install `docker-compose `_ - -* Run the following command: - - .. code:: console - - $ docker compose up -d db db-migrations - - ``docker-compose`` will download PostgreSQL image, create container and volume, and then start container. - Image entrypoint will create database if volume is empty. - - After that, one-off container with migrations script will run. - - .. dropdown:: ``docker-compose.yml`` - - .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 1-32,133 - - Options can be set via ``config.yml`` file: - - .. dropdown:: ``config.yml`` - - .. literalinclude:: ../../../config.yml - :emphasize-lines: 1-2 - -Without Docker -~~~~~~~~~~~~~~ - -* For installing PostgreSQL, please follow `installation instruction `_. -* Install Python 3.11 or above -* Create virtual environment - - .. code-block:: console - - $ python -m venv /some/.venv - $ source /some/.venv/activate - -* Install ``syncmaster`` package with following *extra* dependencies: - - .. code-block:: console - - $ pip install data-syncmaster[postgres] - -* Configure :ref:`Database connection ` by creating config file: - - .. code-block:: yaml - :caption: /some/config.yml - - database: - url: postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster - - File should be created in current directory. If not, you can override its location via environment variable: - - .. code:: console - - $ export SYNCMASTER_CONFIG_FILE=/some/config.yml - -* Run migrations: - - .. code:: console - - $ python -m syncmaster.db.migrations upgrade head - - This is a thin wrapper around `alembic cli `_, - options and commands are just the same. - - .. note:: - - This command should be executed after each upgrade to new Data.Rentgen version. - -See also --------- - -.. toctree:: - :maxdepth: 1 - - configuration - credentials_encryption - structure diff --git a/docs/reference/database/structure.rst b/docs/reference/database/structure.rst deleted file mode 100644 index c725afa9..00000000 --- a/docs/reference/database/structure.rst +++ /dev/null @@ -1,125 +0,0 @@ -.. _database-structure: - -Database structure -================== - -.. https://plantuml.com/en/ie-diagram - -.. plantuml:: - - @startuml - title Database structure - - entity user { - * id: bigint - ---- - * username: varchar(256) - email: varchar(256) null - first_name: varchar(256) null - last_name: varchar(256) null - middle_name: varchar(256) null - is_superuser: boolean - is_active: boolean - created_at: timestamp - updated_at: timestamp - } - - entity group { - * id: bigint - ---- - name: varchar(256) - description: varchar(512) - owner_id: bigint - created_at: timestamptz - updated_at: timestamptz - search_vector: tsquery - } - - entity user_group { - * user_id: bigint - * group_id: bigint - ---- - role_id: varchar(255) - } - - entity connection { - * id: bigint - ---- - group_id: bigint - type: varchar(32) - name: varchar(123) - description: varchar(512) - data: json - created_at: timestamptz - updated_at: timestamptz - search_vector: tsquery - } - - entity auth_data { - * connection_id: bigint - ---- - value: text - created_at: timestamptz - updated_at: timestamptz - } - - entity queue { - * id: bigint - ---- - name: varchar(128) - slug: varchar(256) - group_id: bigint - description: varchar(512) - created_at: timestamptz - updated_at: timestamptz - } - - entity transfer { - * id: bigint - ---- - group_id: bigint - name: varchar(128) - source_connection_id: bigint - target_connection_id: bigint - strategy_params: json - target_params: json - transformations: json - resources: json - is_scheduled: boolean - schedule: varchar(32) - queue_id: bigint - created_at: timestamptz - updated_at: timestamptz - } - - entity run { - * id: bigint - ---- - transfer_id - started_at: timestamptz - ended_at: timestamptz - status: varchar(255) - type: varchar(64) - log_url: varchar(512) - transfer_dump: json - created_at: timestamptz - updated_at: timestamptz - } - - user_group ||--o{ user - user_group ||--o{ group - - group "owner_id" ||--o{ user - - queue ||--o{ group - - connection ||--o{ group - auth_data ||--o{ connection - - transfer ||--o{ queue - transfer ||--o{ connection - transfer ||--o{ group - - run ||--o{ transfer - - @enduml diff --git a/docs/reference/frontend/configuration.rst b/docs/reference/frontend/configuration.rst deleted file mode 100644 index 000c1e2e..00000000 --- a/docs/reference/frontend/configuration.rst +++ /dev/null @@ -1,41 +0,0 @@ -.. _configuration-frontend: - -Frontend configuration -====================== - -API URL -------- - -SyncMaster UI requires REST API to be accessible from browser. API url is set up using config file: - -.. code-block:: yaml - :caption: config.yml - - ui: - api_browser_url: http://localhost:8000 - -If both REST API and frontend are served on the same domain (e.g. through Nginx reverse proxy), for example: - -- REST API → ``/api`` -- Frontend → ``/`` - -Then you can use relative path: - -.. code-block:: yaml - :caption: config.yml - - ui: - api_browser_url: /api - -Auth provider -------------- - -By default, SyncMaster UI shows login page with username & password fields, designed for :ref:`server-auth-dummy`. -To show a login page for :ref:`keycloak-auth-provider`, you should set config option: - -.. code-block:: yaml - :caption: config.yml - - ui: - auth_provider: keycloakAuthProvider - # auth_provider: dummyAuthProvider diff --git a/docs/reference/frontend/index.rst b/docs/reference/frontend/index.rst deleted file mode 100644 index 7f0626d2..00000000 --- a/docs/reference/frontend/index.rst +++ /dev/null @@ -1,46 +0,0 @@ -.. _frontend: - -Frontend -======== - -SyncMaster provides a `Frontend (UI) `_ based on `React `_, -providing users the ability to create, update, delete entities. - -Install & run -------------- - -With Docker -~~~~~~~~~~~ - -* Install `Docker `_ -* Install `docker-compose `_ - -* Run the following command: - - .. code:: console - - $ docker compose --profile frontend up -d --wait - - ``docker-compose`` will download SyncMaster UI image, create containers, and then start them. - - .. dropdown:: ``docker-compose.yml`` - - .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 118-130 - - Options can be set via ``config.yml`` file: - - .. dropdown:: ``config.yml`` - - .. literalinclude:: ../../../config.yml - :emphasize-lines: 34-37 - -* After frontend is started and ready, open http://localhost:3000. - -See also --------- - -.. toctree:: - :maxdepth: 1 - - configuration diff --git a/docs/reference/scheduler/configuration/broker.rst b/docs/reference/scheduler/configuration/broker.rst deleted file mode 100644 index f6c84dd3..00000000 --- a/docs/reference/scheduler/configuration/broker.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _scheduler-configuration-broker: - -Broker settings -================= - -.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings diff --git a/docs/reference/scheduler/configuration/credentials.rst b/docs/reference/scheduler/configuration/credentials.rst deleted file mode 100644 index 22045414..00000000 --- a/docs/reference/scheduler/configuration/credentials.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _scheduler-configuration-credentials: - -Credentials settings -==================== - -.. autopydantic_model:: syncmaster.settings.credentials.CredentialsEncryptionSettings diff --git a/docs/reference/scheduler/configuration/database.rst b/docs/reference/scheduler/configuration/database.rst deleted file mode 100644 index 9084ae2b..00000000 --- a/docs/reference/scheduler/configuration/database.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _scheduler-configuration-database: - -Database settings -================= - -.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings diff --git a/docs/reference/scheduler/configuration/index.rst b/docs/reference/scheduler/configuration/index.rst deleted file mode 100644 index 457c4a73..00000000 --- a/docs/reference/scheduler/configuration/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. _scheduler-configuration: - -Configuration -============= - -.. toctree:: - :maxdepth: 1 - :caption: Configuration - :hidden: - - database - broker - credentials - logging - -.. autopydantic_settings:: syncmaster.scheduler.settings.SchedulerAppSettings -.. autopydantic_settings:: syncmaster.scheduler.settings.SchedulerSettings diff --git a/docs/reference/scheduler/configuration/logging.rst b/docs/reference/scheduler/configuration/logging.rst deleted file mode 100644 index eb841752..00000000 --- a/docs/reference/scheduler/configuration/logging.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. _scheduler-configuration-logging: - -Logging settings -================ - - -.. autopydantic_model:: syncmaster.settings.logging.LoggingSettings diff --git a/docs/reference/scheduler/index.rst b/docs/reference/scheduler/index.rst deleted file mode 100644 index c06646a5..00000000 --- a/docs/reference/scheduler/index.rst +++ /dev/null @@ -1,73 +0,0 @@ -.. _scheduler: - -Scheduler -========= - -SyncMaster scheduler is a dedicated process which periodically checks scheduler Transfers in :ref:`database`, -and creates corresponding Runs in :ref:`message-broker`. - -Implemented using `APScheduler `_. - -Install & run -------------- - -With docker -^^^^^^^^^^^ - -* Install `Docker `_ -* Install `docker-compose `_ -* Run the following command: - - .. code:: console - - $ docker compose --profile scheduler up -d --wait - - ``docker-compose`` will download all necessary images, create containers, and then start the scheduler. - - .. dropdown:: ``docker-compose.yml`` - - .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 98-116 - - Options can be set via ``config.yml`` file: - - .. dropdown:: ``config.yml`` - - .. literalinclude:: ../../../config.yml - :emphasize-lines: 1-10,57-58 - -Without docker -^^^^^^^^^^^^^^ - -* Install Python 3.11 or above -* Setup :ref:`database`, run migrations -* Setup :ref:`message-broker` -* Create virtual environment - - .. code-block:: console - - $ python -m venv /some/.venv - $ source /some/.venv/activate - -* Install ``syncmaster`` package with following *extra* dependencies: - - .. code-block:: console - - $ pip install data-syncmaster[scheduler] - -* Run scheduler process: - - .. code-block:: console - - $ python -m syncmaster.Scheduler - - Scheduler currently don't have any command line arguments. - -See also --------- - -.. toctree:: - :maxdepth: 1 - - configuration/index - diff --git a/docs/reference/server/auth/custom.rst b/docs/reference/server/auth/custom.rst deleted file mode 100644 index 899a493c..00000000 --- a/docs/reference/server/auth/custom.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. _server-auth-custom: - -Custom Auth provider -==================== - -You can implement custom auth provider by inheriting from class below and implementing necessary methods. - -.. autoclass:: syncmaster.server.providers.auth.AuthProvider - :members: - :member-order: bysource diff --git a/docs/reference/server/auth/dummy.rst b/docs/reference/server/auth/dummy.rst deleted file mode 100644 index cc5db6eb..00000000 --- a/docs/reference/server/auth/dummy.rst +++ /dev/null @@ -1,82 +0,0 @@ -.. _server-auth-dummy: - -Dummy Auth provider -=================== - -Description ------------ - -This auth provider allows to sign-in with any username and password, and and then issues an access token. - -After successful auth, username is saved to server database. - -Interaction schema ------------------- - -.. dropdown:: Interaction schema - - .. plantuml:: - - @startuml - title DummyAuthProvider - participant "Client" - participant "Server" - - == POST v1/auth/token == - - activate "Client" - alt Successful case - "Client" -> "Server" ++ : login + password - "Server" --> "Server" : Password is completely ignored - "Server" --> "Server" : Check user in internal server database - "Server" -> "Server" : Create user if not exist - "Server" -[#green]> "Client" -- : Generate and return access_token - - else User is blocked - "Client" -> "Server" ++ : login + password - "Server" --> "Server" : Password is completely ignored - "Server" --> "Server" : Check user in internal server database - "Server" x-[#red]> "Client" -- : 401 Unauthorized - - else User is deleted - "Client" -> "Server" ++ : login + password - "Server" --> "Server" : Password is completely ignored - "Server" --> "Server" : Check user in internal server database - "Server" x-[#red]> "Client" -- : 404 Not found - end - - == GET v1/users/me == - - alt Successful case - "Client" -> "Server" ++ : access_token - "Server" --> "Server" : Validate token - "Server" --> "Server" : Check user in internal server database - "Server" -> "Server" : Get data - "Server" -[#green]> "Client" -- : Return data - - else Token is expired - "Client" -> "Server" ++ : access_token - "Server" --> "Server" : Validate token - "Server" x-[#red]> "Client" -- : 401 Unauthorized - - else User is blocked - "Client" -> "Server" ++ : access_token - "Server" --> "Server" : Validate token - "Server" --> "Server" : Check user in internal server database - "Server" x-[#red]> "Client" -- : 401 Unauthorized - - else User is deleted - "Client" -> "Server" ++ : access_token - "Server" --> "Server" : Validate token - "Server" --> "Server" : Check user in internal server database - "Server" x-[#red]> "Client" -- : 404 Not found - end - - deactivate "Client" - @enduml - -Configuration -------------- - -.. autopydantic_model:: syncmaster.server.settings.auth.dummy.DummyAuthProviderSettings -.. autopydantic_model:: syncmaster.server.settings.auth.jwt.JWTSettings diff --git a/docs/reference/server/auth/index.rst b/docs/reference/server/auth/index.rst deleted file mode 100644 index edee6c99..00000000 --- a/docs/reference/server/auth/index.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _server-auth-providers: - -Auth Providers -============== - -Syncmaster supports different auth provider implementations. You can change implementation via settings: - -.. autopydantic_model:: syncmaster.server.settings.auth.AuthSettings - -.. toctree:: - :maxdepth: 2 - :caption: Auth providers - - dummy - keycloak/index - -.. toctree:: - :maxdepth: 2 - :caption: For developers - - custom diff --git a/docs/reference/server/auth/keycloak/images/keycloak-client-authentication.png b/docs/reference/server/auth/keycloak/images/keycloak-client-authentication.png deleted file mode 100644 index 126067a3..00000000 Binary files a/docs/reference/server/auth/keycloak/images/keycloak-client-authentication.png and /dev/null differ diff --git a/docs/reference/server/auth/keycloak/images/keycloak-client-redirect_uri.png b/docs/reference/server/auth/keycloak/images/keycloak-client-redirect_uri.png deleted file mode 100644 index a68b812c..00000000 Binary files a/docs/reference/server/auth/keycloak/images/keycloak-client-redirect_uri.png and /dev/null differ diff --git a/docs/reference/server/auth/keycloak/images/keycloak-client-secret.png b/docs/reference/server/auth/keycloak/images/keycloak-client-secret.png deleted file mode 100644 index 2a5f7edf..00000000 Binary files a/docs/reference/server/auth/keycloak/images/keycloak-client-secret.png and /dev/null differ diff --git a/docs/reference/server/auth/keycloak/images/keycloak-login.png b/docs/reference/server/auth/keycloak/images/keycloak-login.png deleted file mode 100644 index f6c0b2b8..00000000 Binary files a/docs/reference/server/auth/keycloak/images/keycloak-login.png and /dev/null differ diff --git a/docs/reference/server/auth/keycloak/images/keycloak-new-client.png b/docs/reference/server/auth/keycloak/images/keycloak-new-client.png deleted file mode 100644 index 7eb31977..00000000 Binary files a/docs/reference/server/auth/keycloak/images/keycloak-new-client.png and /dev/null differ diff --git a/docs/reference/server/auth/keycloak/images/keycloak-new-client_name.png b/docs/reference/server/auth/keycloak/images/keycloak-new-client_name.png deleted file mode 100644 index 58cd16ce..00000000 Binary files a/docs/reference/server/auth/keycloak/images/keycloak-new-client_name.png and /dev/null differ diff --git a/docs/reference/server/auth/keycloak/images/keycloak-new-realm.png b/docs/reference/server/auth/keycloak/images/keycloak-new-realm.png deleted file mode 100644 index 0819822b..00000000 Binary files a/docs/reference/server/auth/keycloak/images/keycloak-new-realm.png and /dev/null differ diff --git a/docs/reference/server/auth/keycloak/images/keycloak-new-realm_name.png b/docs/reference/server/auth/keycloak/images/keycloak-new-realm_name.png deleted file mode 100644 index 5d152016..00000000 Binary files a/docs/reference/server/auth/keycloak/images/keycloak-new-realm_name.png and /dev/null differ diff --git a/docs/reference/server/auth/keycloak/images/ui-login-page.png b/docs/reference/server/auth/keycloak/images/ui-login-page.png deleted file mode 100644 index e7e5da7c..00000000 Binary files a/docs/reference/server/auth/keycloak/images/ui-login-page.png and /dev/null differ diff --git a/docs/reference/server/auth/keycloak/index.rst b/docs/reference/server/auth/keycloak/index.rst deleted file mode 100644 index 67839cd7..00000000 --- a/docs/reference/server/auth/keycloak/index.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. _keycloak-auth-provider: - -KeyCloak Auth provider -====================== - -Description ------------ -Keycloak auth provider uses `python-keycloak `_ library to interact with Keycloak server. During the authentication process, -KeycloakAuthProvider redirects user to Keycloak authentication page. - -After successful authentication, Keycloak redirects user back to Syncmaster with authorization code. -Then KeycloakAuthProvider exchanges authorization code for an access token and uses it to get user information from Keycloak server. -If user is not found in Syncmaster database, KeycloakAuthProvider creates it. Finally, KeycloakAuthProvider returns user with access token. - -You can follow interaction schema below. - -Interaction schema ------------------- - -.. dropdown:: Interaction schema - - .. plantuml:: - - @startuml - title Keycloak Authorization Flow - participant "Client (User from Browser)" as Client - participant "Syncmaster" - participant "Keycloak" - - == Client Authentication at Keycloak == - Client -> Syncmaster : Request endpoint that requires authentication (/v1/users) - - Syncmaster x-[#red]> Client : Redirect to Keycloak login URL (if no access token) - - Client -> Keycloak : Callback redirect to Keycloak login page - - alt Successful login - Client --> Keycloak : Log in with login and password - else Login failed - Keycloak x-[#red]> Client -- : Display error (401 Unauthorized) - end - - Keycloak -> Client : Redirect to Syncmaster to callback endpoint with code - Client -> Syncmaster : Callback request to /v1/auth/callback with code - Syncmaster-> Keycloak : Exchange code for access token - Keycloak --> Syncmaster : Return JWT token - Syncmaster --> Client : Set JWT token in user's browser in cookies and redirect /v1/users - - Client --> Syncmaster : Redirect to /v1/users - Syncmaster -> Syncmaster : Get user info from JWT token and check user in internal server database - Syncmaster -> Syncmaster : Create user in internal server database if not exist - Syncmaster -[#green]> Client -- : Return requested data - - - - == GET v1/users == - alt Successful case - Client -> Syncmaster : Request data with JWT token - Syncmaster --> Syncmaster : Get user info from JWT token and check user in internal server database - Syncmaster -> Syncmaster : Create user in internal server database if not exist - Syncmaster -[#green]> Client -- : Return requested data - - else Access token is expired - Syncmaster -> Keycloak : Get new JWT token via refresh token - Keycloak --> Syncmaster : Return new JWT token - Syncmaster --> Syncmaster : Get user info from JWT token and check user in internal server database - Syncmaster -> Syncmaster : Create user in internal server database if not exist - Syncmaster -[#green]> Client -- : Return requested data and set new JWT token in user's browser in cookies - - else Refresh token is expired - Syncmaster x-[#red]> Client -- : Redirect to Keycloak login URL - end - - deactivate Client - @enduml - -Basic configuration -------------------- - -.. autopydantic_model:: syncmaster.server.settings.auth.keycloak.KeycloakAuthProviderSettings -.. autopydantic_model:: syncmaster.server.settings.auth.keycloak.KeycloakSettings -.. autopydantic_model:: syncmaster.server.settings.auth.keycloak.KeycloakCookieSettings - - -OAuth2 Gateway Provider ------------------------ - -In case of using an OAuth2 Gateway, all API requests will come with an Authorization: Bearer header. For this scenario, Syncmaster provides an alternative authentication provider called OAuth2GatewayProvider. This provider works as follows: - -- It extracts the access token from the Authorization header. -- It inspects the token in Keycloak. -- It searches for the user in the Syncmaster database and creates it if not found. - -This provider ensures integration with OAuth2 Gateway and maintains the standard authorization flow as described in the Keycloak Auth Provider section. It also uses the `python-keycloak `_ library for interactions with the Keycloak server and handles the token exchange process similarly. - -**Configuration** - -OAuth2GatewayProvider uses almost the same configuration models as KeycloakAuthProvider — namely: - -.. autopydantic_model:: syncmaster.server.settings.auth.oauth2_gateway.OAuth2GatewayProviderSettings -.. autopydantic_model:: syncmaster.server.settings.auth.oauth2_gateway.OAuth2GatewayKeycloakSettings - -.. toctree:: - :maxdepth: 1 - :caption: Keycloak - :hidden: - - local_installation diff --git a/docs/reference/server/auth/keycloak/local_installation.rst b/docs/reference/server/auth/keycloak/local_installation.rst deleted file mode 100644 index 0873432f..00000000 --- a/docs/reference/server/auth/keycloak/local_installation.rst +++ /dev/null @@ -1,163 +0,0 @@ -Local installation and testing ------------------------------- - -You can test Keycloak auth locally with docker compose: - - -.. code-block:: console - - $ docker compose -f docker-compose.test.yml up keycloak -d - - -Go to Keycloak Admin -~~~~~~~~~~~~~~~~~~~~~ - -Go to `Keycloak admin page `_ and sign in using login: ``admin``, password: ``admin`` (by default): - -.. image:: images/keycloak-login.png - :width: 400px - :align: center - - -Create new Realm -~~~~~~~~~~~~~~~~ - -Create new Realm with some name: - -.. image:: images/keycloak-new-realm.png - :width: 400px - :align: center - -.. image:: images/keycloak-new-realm_name.png - :width: 400px - :align: center - -Save realm name to config file: - -.. code-block:: yaml - :caption: config.yml - - auth: - keycloak: - realm_name: fastapi_realm - # ... - -Create new Client -~~~~~~~~~~~~~~~~~ - -In created realm create new client and save its name to config: - -.. image:: images/keycloak-new-client.png - :width: 400px - :align: center - -.. image:: images/keycloak-new-client_name.png - :width: 400px - :align: center - -.. code-block:: yaml - :caption: config.yml - - auth: - keycloak: - client_id: fastapi_client - # ... - - -Set ``client_authentication=ON`` to generate client_secret -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. image:: images/keycloak-client-authentication.png - :width: 400px - :align: center - -Configure Redirect URI -~~~~~~~~~~~~~~~~~~~~~~ - -Set URI to redirect from Keycloak login page for exchanging the code for an access token: - -.. image:: images/keycloak-client-redirect_uri.png - :width: 400px - :align: center - -.. code-block:: yaml - :caption: config.yml - - auth: - keycloak: - # Set here URL of SyncMaster UI page handling callback redirects - ui_callback_url: http://localhost:3000/auth/callback - # ... - -Configure the client secret -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Now go to **Credentials** tab and generate a client secret: - -.. image:: images/keycloak-client-secret.png - :width: 400px - :align: center - -.. code-block:: yaml - :caption: config.yml - - auth: - keycloak: - client_id: fastapi_client - client_secret: 6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez - -Now you can use create users in this realm, check `Keycloak documentation `_ on how to manage users creation. - -Cookie encryption secret -~~~~~~~~~~~~~~~~~~~~~~~~ - -Keycloak access & refresh tokens are stored in cookie with server-side encryption. -So we need to generate random string to use as encryption key: - -.. code-block:: yaml - :caption: config.yml - - auth: - cookie: - secret_key: secret_key_for_session_cookie - -Replace login page with Keycloak redirect button -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. image:: images/ui-login-page.png - :width: 400px - :align: center - -.. code-block:: yaml - :caption: config.yml - - ui: - auth_provider: keycloakAuthProvider - -Final configuration -~~~~~~~~~~~~~~~~~~~ - -After this you can use ``KeycloakAuthProvider`` in your application: - -.. code-block:: yaml - :caption: config.yml - - auth: - provider: syncmaster.server.providers.auth.keycloak_provider.KeycloakAuthProvider - keycloak: - # Keycloak URL accessible from both SyncMaster server and from browser - api_url: http://keycloak:8080 - # Set here URL of SyncMaster UI page handling callback redirects - ui_callback_url: http://localhost:3000/auth/callback - realm_name: fastapi_realm - client_id: fastapi_client - client_secret: 6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez - scope: email - verify_ssl: false - cookie: - secret_key: secret_key_for_session_cookie - - ui: - auth_provider: keycloakAuthProvider - # SyncMaster API URL, accessible from browser - api_browser_url: http://localhost:8000 diff --git a/docs/reference/server/configuration/broker.rst b/docs/reference/server/configuration/broker.rst deleted file mode 100644 index 3ff9cf1b..00000000 --- a/docs/reference/server/configuration/broker.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _server-configuration-broker: - -Broker settings -================= - -.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings diff --git a/docs/reference/server/configuration/cors.rst b/docs/reference/server/configuration/cors.rst deleted file mode 100644 index a14239b3..00000000 --- a/docs/reference/server/configuration/cors.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _server-configuration-cors: - -CORS settings -============= - -These settings used to control `CORS `_ options. - -.. autopydantic_model:: syncmaster.server.settings.server.cors.CORSSettings \ No newline at end of file diff --git a/docs/reference/server/configuration/credentials.rst b/docs/reference/server/configuration/credentials.rst deleted file mode 100644 index 94f4cc35..00000000 --- a/docs/reference/server/configuration/credentials.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _server-configuration-credentials: - -Credentials settings -==================== - -.. autopydantic_model:: syncmaster.settings.credentials.CredentialsEncryptionSettings diff --git a/docs/reference/server/configuration/database.rst b/docs/reference/server/configuration/database.rst deleted file mode 100644 index d89501ee..00000000 --- a/docs/reference/server/configuration/database.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _server-configuration-database: - -Database settings -================= - -.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings diff --git a/docs/reference/server/configuration/debug.rst b/docs/reference/server/configuration/debug.rst deleted file mode 100644 index 393f2b48..00000000 --- a/docs/reference/server/configuration/debug.rst +++ /dev/null @@ -1,88 +0,0 @@ -.. _server-configuration-debug: - -Enabling debug -=============== - -Return debug info in REST API responses ---------------------------------------- - -By default, server does not add error details to response bodies, -to avoid exposing instance-specific information to end users. - -With debug enabled: - -.. code-block:: yaml - :caption: config.yml - - server: - debug: true - -.. code-block:: console - - $ curl -XPOST http://localhost:8000/failing/endpoint ... - { - "error": { - "code": "unknown", - "message": "Got unhandled exception. Please contact support", - "details": null, - }, - } - -With debug disabled: - -.. code-block:: yaml - :caption: config.yml - - server: - debug: false - -.. code-block:: console - - $ curl -XPOST http://localhost:8000/failing/endpoint ... - Traceback (most recent call last): - File ".../uvicorn/protocols/http/h11_impl.py", line 408, in run_asgi - result = await app( # type: ignore[func-returns-value] - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File ".../site-packages/uvicorn/middleware/proxy_headers.py", line 84, in __call__ - return await self.app(scope, receive, send) - -.. DANGER:: - - This is only for development environment only. Do **NOT** use on production! - -Print debug logs on backend ---------------------------- - -See :ref:`server-configuration-logging`, but replace log level ``INFO`` with ``DEBUG``. - -Fill up ``X-Request-ID`` header on backend ------------------------------------------- - -Server can add ``X-Request-ID`` header to responses, which allows to match request on client with backend response. - -This is done by ``request_id`` middleware, which is enabled by default and can configured as described below: - -.. autopydantic_model:: syncmaster.server.settings.server.request_id.RequestIDSettings - -Print request ID to backend logs ---------------------------------- - -This is done by adding a specific filter to logging settings: - -.. dropdown:: ``logging.yml`` - - .. code-block:: yaml - :caption: config.yml - - logging: - filters: - correlation_id: - class: asgi_correlation_id.CorrelationIdFilter - uuid_length: 32 - default_value: '-' - -Resulting logs look like: - -.. code-block:: text - - 2023-12-18 17:14:11.711 uvicorn.access:498 [INFO] 018c15e97a068ae09484f8c25e2799dd 127.0.0.1:34884 - "GET /monitoring/ping HTTP/1.1" 200 diff --git a/docs/reference/server/configuration/index.rst b/docs/reference/server/configuration/index.rst deleted file mode 100644 index e1bbf44f..00000000 --- a/docs/reference/server/configuration/index.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. _server-configuration: - -Configuration -============= - -.. toctree:: - :maxdepth: 1 - :caption: Configuration - :hidden: - - database - broker - credentials - logging - cors - debug - monitoring - static_files - openapi - debug - -.. autopydantic_settings:: syncmaster.server.settings.ServerAppSettings -.. autopydantic_settings:: syncmaster.server.settings.server.ServerSettings diff --git a/docs/reference/server/configuration/logging.rst b/docs/reference/server/configuration/logging.rst deleted file mode 100644 index 5940bae9..00000000 --- a/docs/reference/server/configuration/logging.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. _server-configuration-logging: - -Logging settings -================ - - -.. autopydantic_model:: syncmaster.settings.logging.LoggingSettings diff --git a/docs/reference/server/configuration/monitoring.rst b/docs/reference/server/configuration/monitoring.rst deleted file mode 100644 index 05e3f6f6..00000000 --- a/docs/reference/server/configuration/monitoring.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. _server-configuration-monitoring: - -Server monitoring -================= - -REST API server provides the following endpoints with Prometheus compatible metrics: - -* ``GET /monitoring/metrics`` - server metrics, like number of requests per path and response status, CPU and RAM usage, and so on. - -These endpoints are enabled and configured using settings below: - -.. autopydantic_model:: syncmaster.server.settings.server.monitoring.MonitoringSettings diff --git a/docs/reference/server/configuration/openapi.rst b/docs/reference/server/configuration/openapi.rst deleted file mode 100644 index d4c235a9..00000000 --- a/docs/reference/server/configuration/openapi.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. _server-configuration-openapi: - -OpenAPI settings -================ - -These settings used to control exposing OpenAPI.json and SwaggerUI/ReDoc endpoints. - -.. autopydantic_model:: syncmaster.server.settings.server.openapi.OpenAPISettings -.. autopydantic_model:: syncmaster.server.settings.server.openapi.SwaggerSettings -.. autopydantic_model:: syncmaster.server.settings.server.openapi.RedocSettings -.. autopydantic_model:: syncmaster.server.settings.server.openapi.LogoSettings -.. autopydantic_model:: syncmaster.server.settings.server.openapi.FaviconSettings diff --git a/docs/reference/server/configuration/static_files.rst b/docs/reference/server/configuration/static_files.rst deleted file mode 100644 index fb6449b6..00000000 --- a/docs/reference/server/configuration/static_files.rst +++ /dev/null @@ -1,8 +0,0 @@ -.. _server-configuration-static-files: - -Serving static files -==================== - -These settings used to control serving static files by a server. - -.. autopydantic_model:: syncmaster.server.settings.server.static_files.StaticFilesSettings diff --git a/docs/reference/server/index.rst b/docs/reference/server/index.rst deleted file mode 100644 index 21756cdb..00000000 --- a/docs/reference/server/index.rst +++ /dev/null @@ -1,79 +0,0 @@ -.. _server: - -REST API Server -=============== - -SyncMaster server provides simple REST API for accessing entities stored in :ref:`database`. -Implemented using `FastAPI `_. - -Install & run -------------- - -With docker -^^^^^^^^^^^ - -* Install `Docker `_ -* Install `docker-compose `_ - -* Run the following command: - - .. code:: console - - $ docker compose --profile server up -d --wait - - ``docker-compose`` will download all necessary images, create containers, and then start the server. - - .. dropdown:: ``docker-compose.yml`` - - .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 48-75 - - Options can be set via ``config.yml`` file: - - .. dropdown:: ``config.yml`` - - .. literalinclude:: ../../../config.yml - :emphasize-lines: 1-31, 40-49, 70-71 - -* After server is started and ready, open http://localhost:8000/docs. - -Without docker -^^^^^^^^^^^^^^ - -* Install Python 3.11 or above -* Setup :ref:`database`, run migrations -* Setup :ref:`message-broker` -* Create virtual environment - - .. code-block:: console - - $ python -m venv /some/.venv - $ source /some/.venv/activate - -* Install ``syncmaster`` package with following *extra* dependencies: - - .. code-block:: console - - $ pip install data-syncmaster[server] - -* Run server process - - .. code-block:: console - - $ python -m syncmaster.server --host 0.0.0.0 --port 8000 - - This is a thin wrapper around `uvicorn `_ cli, - options and commands are just the same. - -* After server is started and ready, open http://localhost:8000/docs. - -See also --------- - -.. toctree:: - :maxdepth: 1 - - auth/index - configuration/index - manage_superusers_cli - openapi diff --git a/docs/reference/server/manage_superusers_cli.rst b/docs/reference/server/manage_superusers_cli.rst deleted file mode 100644 index 8ed3a0ae..00000000 --- a/docs/reference/server/manage_superusers_cli.rst +++ /dev/null @@ -1,33 +0,0 @@ -.. _manage-superusers-cli: - -CLI for managing superusers -=========================== - -There are two ways to manage users: - -* automatic - - :ref:`server` Docker container entrypoint will automatically create users with ``is_superuser=True`` in database - during startup. - - Usernames can be passed via config file: - - .. code-block:: yaml - :caption: config.yml - - superusers: - - user1 - - user2 - - Or via environment variable: - - .. code-block:: bash - - export 'SYNCMASTER__SUPERUSERS=["user1", "user2"]' - -* manual via CLI: - -.. argparse:: - :module: syncmaster.server.scripts.manage_superusers - :func: create_parser - :prog: python -m syncmaster.server.scripts.manage_superusers diff --git a/docs/reference/server/openapi.rst b/docs/reference/server/openapi.rst deleted file mode 100644 index e3fced36..00000000 --- a/docs/reference/server/openapi.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _server-openapi: - -OpenAPI specification -===================== - -.. this page cannot be properly rendered in local environment, it should be build in CI first - -.. raw:: html - :file: ../../_static/swagger.html \ No newline at end of file diff --git a/docs/reference/worker/configuration/broker.rst b/docs/reference/worker/configuration/broker.rst deleted file mode 100644 index 935b920c..00000000 --- a/docs/reference/worker/configuration/broker.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _worker-configuration-broker: - -Broker settings -================= - -.. autopydantic_model:: syncmaster.settings.broker.RabbitMQSettings diff --git a/docs/reference/worker/configuration/credentials.rst b/docs/reference/worker/configuration/credentials.rst deleted file mode 100644 index 3fcdba64..00000000 --- a/docs/reference/worker/configuration/credentials.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _worker-configuration-credentials: - -Credentials settings -==================== - -.. autopydantic_model:: syncmaster.settings.credentials.CredentialsEncryptionSettings diff --git a/docs/reference/worker/configuration/database.rst b/docs/reference/worker/configuration/database.rst deleted file mode 100644 index d89501ee..00000000 --- a/docs/reference/worker/configuration/database.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _server-configuration-database: - -Database settings -================= - -.. autopydantic_model:: syncmaster.settings.database.DatabaseSettings diff --git a/docs/reference/worker/configuration/hwm_store.rst b/docs/reference/worker/configuration/hwm_store.rst deleted file mode 100644 index 65de92c4..00000000 --- a/docs/reference/worker/configuration/hwm_store.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. _worker-configuration-hwm-store: - -HWM Store settings -================== - -.. autopydantic_model:: syncmaster.worker.settings.hwm_store.HWMStoreSettings diff --git a/docs/reference/worker/configuration/index.rst b/docs/reference/worker/configuration/index.rst deleted file mode 100644 index 590391ba..00000000 --- a/docs/reference/worker/configuration/index.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _worker-configuration: - -Configuration -============= - -.. toctree:: - :maxdepth: 1 - :caption: Configuration - :hidden: - - database - broker - credentials - logging - hwm_store - -.. autopydantic_settings:: syncmaster.worker.settings.WorkerAppSettings -.. autopydantic_settings:: syncmaster.worker.settings.WorkerSettings diff --git a/docs/reference/worker/configuration/logging.rst b/docs/reference/worker/configuration/logging.rst deleted file mode 100644 index 7176685e..00000000 --- a/docs/reference/worker/configuration/logging.rst +++ /dev/null @@ -1,7 +0,0 @@ -.. _worker-configuration-logging: - -Logging settings -================ - - -.. autopydantic_model:: syncmaster.settings.logging.LoggingSettings diff --git a/docs/reference/worker/create_spark_session.rst b/docs/reference/worker/create_spark_session.rst deleted file mode 100644 index 50daeee2..00000000 --- a/docs/reference/worker/create_spark_session.rst +++ /dev/null @@ -1,77 +0,0 @@ -.. _worker-create-spark-session: - -Configuring Spark session -========================= - -SyncMaster Worker creates `SparkSession `_ for each Run. - -By default, SparkSession is created with ``spark.master: local``, including all required .jar packages for DB/FileSystem types, and limited by transfer resources. - -Custom Spark session configuration ----------------------------------- - -It is possible to alter default `Spark Session configuration `_ worker settings: - -.. code-block:: yaml - :caption: config.yml - - worker: - spark_session_default_config: - spark.master: local - spark.driver.host: 127.0.0.1 - spark.driver.bindAddress: 0.0.0.0 - spark.sql.pyspark.jvmStacktrace.enabled: true - spark.ui.enabled: false - -For example, to use SyncMaster on Spark-on-K8s, you can use worker image for Spark executor containers: - -.. code-block:: yaml - :caption: config.yml - - worker: - spark_session_default_config: - spark.master: k8s://https://kubernetes.default.svc - spark.driver.host: service-for-spark-driver - spark.driver.bindAddress: 0.0.0.0 - spark.driver.port: 10000 - spark.blockManager.port: 10001 - spark.kubernetes.authenticate.driver.serviceAccountName: spark - spark.sql.pyspark.jvmStacktrace.enabled: true - spark.kubernetes.container.image: mtsrus/syncmaster-worker:{TAG} - -.. note:: - - Currently Spark-on-K8s and Spark-on-Yarn do not support interaction FTP, FTPS, SFTP, Samba and WebDAV. - This requires ``sparm.master: local``. - -Custom Spark session factory ----------------------------- - -It is also possible to use custom function which returns ``SparkSession`` object: - -.. code-block:: yaml - :caption: config.yml - - worker: - create_spark_session_function: my_worker.spark.create_custom_spark_session - -Here is a function example: - -.. code-block:: python - :caption: my_workers/spark.py - - from syncmaster.db.models import Run - from syncmaster.dto.connections import ConnectionDTO - from syncmaster.worker.settings import WorkerSettings - from pyspark.sql import SparkSession - - def create_custom_spark_session( - run: Run, - source: ConnectionDTO, - target: ConnectionDTO, - settings: WorkerSettings, - ) -> SparkSession: - # any custom code returning SparkSession object - return SparkSession.builde.config(...).getOrCreate() - -Module with custom function should be placed into the same Docker image or Python virtual environment used by SyncMaster worker. diff --git a/docs/reference/worker/index.rst b/docs/reference/worker/index.rst deleted file mode 100644 index 86cf8a76..00000000 --- a/docs/reference/worker/index.rst +++ /dev/null @@ -1,107 +0,0 @@ -.. _worker: - -Worker -====== - -SyncMaster worker is a dedicated process which receives new transfer Runs from :ref:`message-broker`, -executes them and updates status & log url in :ref:`database`. Implemented using `Celery `_. - -.. note:: - - Each worker process is bound to one or more Queues. You have to created it before starting a worker. - This can be done via :ref:`frontend` or via :ref:`server` REST API. - - Queue field ``slug`` value is then should be passed to Celery worker argument ``--queues``. - For example, for slug ``123-test_queue`` this should be ``--queues 123-test_queue``. - - Worker can listen multiple queues at the same time, you can pass a list with ``,`` as delimiter. - -Install & run -------------- - -With docker -^^^^^^^^^^^ - -* Install `Docker `_ -* Install `docker-compose `_ -* Go to `frontend ` -* Create new Group -* Create Queue in this group, and then get **Queue.slug** (e.g. ``123-test_queue``) -* Pass queue slug as ``--queues $slug`` into ```worker:command`` section of ``docker-compose.yml`` (see below) -* Run the following command: - - .. code:: console - - $ docker compose --profile worker up -d --wait - - ``docker-compose`` will download all necessary images, create containers, and then start the worker. - - .. dropdown:: ``docker-compose.yml`` - - .. literalinclude:: ../../../docker-compose.yml - :emphasize-lines: 77-96 - - Options can be set via ``config.yml`` file: - - .. dropdown:: ``config.yml`` - - .. literalinclude:: ../../../config.yml - :emphasize-lines: 1-10,57-67 - -Without docker -^^^^^^^^^^^^^^ - -* Install Python 3.11 or above -* Install Java 8 or above - - .. code-block:: console - - $ yum install java-1.8.0-openjdk-devel # CentOS 7 - $ dnf install java-11-openjdk-devel # CentOS 8 - $ apt-get install openjdk-11-jdk # Debian-based - -* Setup :ref:`database`, run migrations -* Setup :ref:`message-broker` -* Create virtual environment - - .. code-block:: console - - $ python -m venv /some/.venv - $ source /some/.venv/activate - -* Install ``syncmaster`` package with following *extra* dependencies: - - .. code-block:: console - - $ pip install data-syncmaster[server,worker] - -* Start :ref:`server` and :ref:`frontend` -* Create new Group -* Create Queue in this group, and then get **Queue.slug** (e.g. ``123-test_queue``) -* Run worker process: - - .. code-block:: console - - $ python -m celery -A syncmaster.worker.celery worker --queues 123-test_queue --max-tasks-per-child=1 - - You can specify options like concurrency and queues by adding additional flags: - - .. code-block:: bash - - $ python -m celery -A syncmaster.worker.celery worker --queues 123-test_queue --max-tasks-per-child=1 --concurrency=4 - - Refer to the `Celery `_ documentation for more advanced start options. - - .. note:: - - ``--max-tasks-per-child=1`` flag is important for Spark to properly work! - -See also --------- - -.. toctree:: - :maxdepth: 1 - - configuration/index - create_spark_session - log_url diff --git a/docs/reference/worker/log_url.rst b/docs/reference/worker/log_url.rst deleted file mode 100644 index 5ceceefe..00000000 --- a/docs/reference/worker/log_url.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _worker-log-url: - -Setting the `Run.log_url` value -=============================== - -Each run in the system is linked to a log URL where the Celery worker logs are available. -This log URL might point to an Elastic instance or another logging tool such as Grafana. - -The log URL is generated based on a template configured in the configuration. -The configuration parameter is: - -.. code-block:: yaml - :caption: config.yml - - worker: - log_url_template: https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }} - -In this example, run logs can be retrieved by either its correlation id ``x-request-id`` in http headers, or by ``Run.Id`` field value. diff --git a/docs/robots.txt b/docs/robots.txt deleted file mode 100644 index 029a5351..00000000 --- a/docs/robots.txt +++ /dev/null @@ -1,5 +0,0 @@ -User-agent: * -Allow: /*/stable/ -Allow: /en/stable/ # Fallback for bots that don't understand wildcards -Disallow: / -Sitemap: https://data-syncmaster.readthedocs.io/sitemap.xml \ No newline at end of file diff --git a/docs/security.rst b/docs/security.rst deleted file mode 100644 index 2f604339..00000000 --- a/docs/security.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../SECURITY.rst diff --git a/mddocs/docs/changelog/DRAFT.md b/mddocs/docs/changelog/DRAFT.md deleted file mode 100644 index 8a6b3a31..00000000 --- a/mddocs/docs/changelog/DRAFT.md +++ /dev/null @@ -1 +0,0 @@ -# DRAFT { #DRAFT } diff --git a/mddocs/docs/changelog/NEXT_RELEASE.md b/mddocs/docs/changelog/NEXT_RELEASE.md index a9831f9d..6d2714f1 100644 --- a/mddocs/docs/changelog/NEXT_RELEASE.md +++ b/mddocs/docs/changelog/NEXT_RELEASE.md @@ -1 +1 @@ -% towncrier release notes start +:: towncrier-draft Next release diff --git a/mddocs/docs/changelog/RELEASE_TEMPLATE.md b/mddocs/docs/changelog/RELEASE_TEMPLATE.md new file mode 100644 index 00000000..8cf66aa3 --- /dev/null +++ b/mddocs/docs/changelog/RELEASE_TEMPLATE.md @@ -0,0 +1 @@ + diff --git a/mddocs/docs/changelog/index.md b/mddocs/docs/changelog/index.md index ac9a48de..12081949 100644 --- a/mddocs/docs/changelog/index.md +++ b/mddocs/docs/changelog/index.md @@ -1,6 +1,6 @@ # Changelog { #changelog } -* [0.3.5 [UNRELEASED]][DRAFT] +* [Next release](./NEXT_RELEASE) * [0.3.4 (2026-05-26)][0.3.4] * [0.3.3 (2026-05-26)][0.3.3] * [0.3.2 (2026-03-05)][0.3.2] diff --git a/mddocs/docs/contributing.md b/mddocs/docs/contributing.md index 0f589b72..ae192cbb 100644 --- a/mddocs/docs/contributing.md +++ b/mddocs/docs/contributing.md @@ -43,7 +43,7 @@ If you already have venv, but need to install dependencies required for developm make venv-install ``` -We are using [poetry](https://python-poetry.org/docs/managing-dependencies/) for managing dependencies and building the package. +We are using [uv](https://docs.astral.sh/uv/) for managing dependencies and building the package. It allows to keep development environment the same for all developers due to using lock file with fixed dependency versions. There are *extra* dependencies (included into package as optional): @@ -55,7 +55,7 @@ And *groups* (not included into package, used locally and in CI): * `test` - for running tests * `dev` - for development, like linters, formatters, mypy, pre-commit and so on -* `docs` - for building documentation +* `mddocs` - for building documentation ### Enable pre-commit hooks @@ -208,17 +208,13 @@ Settings are stored in `.env.docker` file. ### Build documentation -Build documentation using Sphinx & open it: +Build documentation using mkdocs: ```bash -make docs +make docs-serve ``` -If documentation should be build cleanly instead of reusing existing build result: - -```bash -make docs-fresh -``` +Then open in browser http://localhost:8000/ ## Review process @@ -254,7 +250,7 @@ Data.SyncMaster uses [towncrier](https://pypi.org/project/towncrier/) for changelog management. To submit a change note about your PR, add a text file into the -`docs/changelog/next_release` folder. It should contain an +`mddocs/docs/changelog/next_release` folder. It should contain an explanation of what applying this PR will change in the way end-users interact with the project. One sentence is usually enough but feel free to add as many details as you feel necessary @@ -265,21 +261,15 @@ combined with others, it will be a part of the “news digest” telling the readers **what changed** in a specific version of the library *since the previous version*. -reStructuredText syntax for highlighting code (inline or block), -linking parts of the docs or external sites. -If you wish to sign your change, feel free to add `-- by -:user:`github-username`` at the end (replace `github-username` -with your own!). - Finally, name your file following the convention that Towncrier understands: it should start with the number of an issue or a PR followed by a dot, then add a patch type, like `feature`, -`doc`, `misc` etc., and add `.rst` as a suffix. If you +`doc`, `misc` etc., and add `.md` as a suffix. If you need to add more than one fragment, you may add an optional sequence number (delimited with another period) between the type and the suffix. -In general the name will follow `..rst` pattern, +In general the name will follow `..md` pattern, where the categories are: * `feature`: Any new feature. Adding new functionality that has not yet existed. @@ -301,17 +291,12 @@ changes accompanying the relevant code changes. #### Examples for adding changelog entries to your Pull Requests -```rst -Added a ``:github:user:`` role to Sphinx config -- by :github:user:`someuser` -``` - -```rst -Fixed behavior of ``server`` -- by :github:user:`someuser` +```markdown title="mddocs/docs/changelog/next_release/2345.bugfix.md" +Fixed behavior of `server` ``` -```rst -Added support of ``timeout`` in ``LDAP`` --- by :github:user:`someuser`, :github:user:`anotheruser` and :github:user:`otheruser` +```markdown title="mddocs/docs/changelog/next_release/3456.feature.md" +Added support of `timeout` in `LDAP` ``` #### How to skip change notes check? @@ -329,44 +314,19 @@ Before making a release from the `develop` branch, follow these steps: git pull -p ``` -2. Backup `NEXT_RELEASE.rst` - - ```bash - cp "docs/changelog/NEXT_RELEASE.rst" "docs/changelog/temp_NEXT_RELEASE.rst" - ``` - -3. Build the Release notes with Towncrier - - ```bash - VERSION=$(poetry version -s) - towncrier build "--version=${VERSION}" --yes - ``` - -4. Change file with changelog to release version number - - ```bash - mv docs/changelog/NEXT_RELEASE.rst "docs/changelog/${VERSION}.rst" - ``` - -5. Remove content above the version number heading in the `${VERSION}.rst` file - - ```bash - awk '!/^.*towncrier release notes start/' "docs/changelog/${VERSION}.rst" > temp && mv temp "docs/changelog/${VERSION}.rst" - ``` - -6. Update Changelog Index +2. Get current release version ```bash - awk -v version=${VERSION} '/DRAFT/{print;print " " version;next}1' docs/changelog/index.rst > temp && mv temp docs/changelog/index.rst + VERSION=$(cat syncmaster/VERSION) ``` -7. Restore `NEXT_RELEASE.rst` file from backup +3. Build changelog for current release ```bash - mv "docs/changelog/temp_NEXT_RELEASE.rst" "docs/changelog/NEXT_RELEASE.rst" + make docs-generate-changelog ``` -8. Commit and push changes to `develop` branch +4. Commit and push changes to `develop` branch ```bash git add . @@ -374,7 +334,7 @@ Before making a release from the `develop` branch, follow these steps: git push ``` -9. Merge `develop` branch to `master`, **WITHOUT** squashing +5. Merge `develop` branch to `master`, **WITHOUT** squashing ```bash git checkout master @@ -383,20 +343,20 @@ Before making a release from the `develop` branch, follow these steps: git push ``` -10. Add git tag to the latest commit in `master` branch +6. Add git tag to the latest commit in `master` branch ```bash git tag "$VERSION" git push origin "$VERSION" ``` -11. Update version in `develop` branch **after release**: +7. Update version in `develop` branch **after release**: ```bash git checkout develop NEXT_VERSION=$(echo "$VERSION" | awk -F. '/[0-9]+\./{$NF++;print}' OFS=.) - poetry version "$NEXT_VERSION" + echo $NEXT_VERSION > syncmaster/VERSION git add . git commit -m "Bump version" diff --git a/mddocs/docs/nav.md b/mddocs/docs/nav.md index 40d0b474..3cd9ad62 100644 --- a/mddocs/docs/nav.md +++ b/mddocs/docs/nav.md @@ -48,7 +48,7 @@ * [Contributing Guide](contributing.md) * [Security](security.md) * [Changelog](changelog/index.md) - * [0.3.5 [UNRELEASED]](changelog/DRAFT.md) + * [Next release](changelog/NEXT_RELEASE.md) * [0.3.4](changelog/0.3.4.md) * [0.3.3](changelog/0.3.3.md) * [0.3.2](changelog/0.3.2.md) diff --git a/mddocs/mkdocs.yml b/mddocs/mkdocs.yml index 04262b76..f6ab97e3 100644 --- a/mddocs/mkdocs.yml +++ b/mddocs/mkdocs.yml @@ -34,6 +34,7 @@ extra_javascript: - _static/javascripts/mermaid_zoom.js plugins: + - towncrier - autorefs: resolve_closest: true - literate-nav: diff --git a/pyproject.toml b/pyproject.toml index e44bb57f..d39fcfe2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dynamic = ["version"] [tool.setuptools.packages.find] include = ["syncmaster*"] -exclude = ["docs", "tests"] +exclude = ["mddocs", "tests"] [tool.setuptools.package-data] "*" = ["py.typed", "*.yml", "*.ini", "VERSION"] @@ -114,23 +114,6 @@ dev = [ "sqlalchemy[mypy]~=2.0.50", "types-jwcrypto~=1.5.0", ] -docs = [ - "autodoc-pydantic~=2.2.0", - "numpydoc~=1.10.0", - "sphinx~=9.0.4", - "furo~=2025.12.19", - "sphinx-copybutton~=0.5.2", - "sphinxcontrib-towncrier~=0.5.0a0", - "towncrier~=25.8.0", - "sphinx-issues~=6.0.0", - "sphinx-design~=0.7.0", - "sphinx-favicon~=1.1.0", - "sphinx-argparse~=0.5.2", - # https://github.com/mgeier/sphinx-last-updated-by-git/pull/77 - "sphinx-last-updated-by-git~=0.3.8", - # uncomment after https://github.com/zqmillet/sphinx-plantuml/pull/4 - # sphinx-plantuml~=1.0.0", -] mddocs = [ "mkdocs~=1.6.1", "mkdocs-material~=9.7.6", @@ -140,6 +123,9 @@ mddocs = [ "griffe-inherited-docstrings~=1.1.3", "griffe-pydantic~=1.3.1", "mkdocs-literate-nav~=0.6.1", + "mkdocs-towncrier>=0.1.5,<1", + "towncrier~=25.8.0", + "setuptools<82", ] [tool.uv] @@ -148,7 +134,7 @@ default-groups = [] [tool.ruff] target-version = "py311" line-length = 120 -extend-exclude = ["docs/", "mddocs/scripts/", "Makefile"] +extend-exclude = ["mddocs/", "Makefile"] [tool.ruff.lint] select = ["ALL"] @@ -287,10 +273,12 @@ partial_also = [ [tool.towncrier] name = "Syncmaster" package = "syncmaster" -filename = "docs/changelog/NEXT_RELEASE.rst" -directory = "docs/changelog/next_release/" -title_format = "{version} ({project_date})" -issue_format = ":issue:`{issue}`" +filename = "mddocs/docs/changelog/RELEASE_TEMPLATE.md" +start_string = "\n" +underlines = ["", "", ""] +directory = "mddocs/docs/changelog/next_release/" +title_format = "## [{version}](https://github.com/MTSWebServices/syncmaster/tree/{version}) - {project_date}" +issue_format = "[#{issue}](https://github.com/MTSWebServices/syncmaster/issues/{issue})" [[tool.towncrier.type]] directory = "breaking" diff --git a/syncmaster/server/settings/server/static_files.py b/syncmaster/server/settings/server/static_files.py index b2c7e93d..9b0f86fd 100644 --- a/syncmaster/server/settings/server/static_files.py +++ b/syncmaster/server/settings/server/static_files.py @@ -23,7 +23,7 @@ class StaticFilesSettings(BaseModel): enabled: bool = Field(default=True, description="Set to `True` to enable static file serving") directory: Path = Field( - default=Path("docs/_static"), + default=Path("mddocs/docs/_static"), description="Directory containing static files", ) diff --git a/uv.lock b/uv.lock index 7ffda1fc..ca140e42 100644 --- a/uv.lock +++ b/uv.lock @@ -7,18 +7,6 @@ resolution-markers = [ "python_full_version < '3.14'", ] -[[package]] -name = "accessible-pygments" -version = "0.0.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pygments" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/bc/c1/bbac6a50d02774f91572938964c582fff4270eee73ab822a4aeea4d8b11b/accessible_pygments-0.0.5.tar.gz", hash = "sha256:40918d3e6a2b619ad424cb91e556bd3bd8865443d9f22f1dcdf79e33c8046872", size = 1377899, upload-time = "2024-05-10T11:23:10.216Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl", hash = "sha256:88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7", size = 1395903, upload-time = "2024-05-10T11:23:08.421Z" }, -] - [[package]] name = "aiofiles" version = "25.1.0" @@ -28,15 +16,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/8a/340a1555ae33d7354dbca4faa54948d76d89a27ceef032c8c3bc661d003e/aiofiles-25.1.0-py3-none-any.whl", hash = "sha256:abe311e527c862958650f9438e859c1fa7568a141b22abcd015e120e86a85695", size = 14668, upload-time = "2025-10-09T20:51:03.174Z" }, ] -[[package]] -name = "alabaster" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210, upload-time = "2024-07-26T18:15:03.762Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, -] - [[package]] name = "alembic" version = "1.18.4" @@ -262,19 +241,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/95/adcb68e20c34162e9135f370d6e31737719c2b6f94bc953fe7ed1f10fe21/authlib-1.7.2-py2.py3-none-any.whl", hash = "sha256:3e1faedc9d87e7d56a164eca3ccb6ace0d61b94abe83e92242f8dc8bba9b4a9f", size = 259548, upload-time = "2026-05-06T08:10:21.436Z" }, ] -[[package]] -name = "autodoc-pydantic" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "sphinx" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/df/87120e2195f08d760bc5cf8a31cfa2381a6887517aa89453b23f1ae3354f/autodoc_pydantic-2.2.0-py3-none-any.whl", hash = "sha256:8c6a36fbf6ed2700ea9c6d21ea76ad541b621fbdf16b5a80ee04673548af4d95", size = 34001, upload-time = "2024-04-27T10:57:00.542Z" }, -] - [[package]] name = "babel" version = "2.18.0" @@ -367,19 +333,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/f8/972c96f5a2b6c4b3deca57009d93e946bbdbe2241dca9806d502f29dd3ee/bcrypt-5.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:6b8f520b61e8781efee73cba14e3e8c9556ccfb375623f4f97429544734545b4", size = 273375, upload-time = "2025-09-25T19:50:45.43Z" }, ] -[[package]] -name = "beautifulsoup4" -version = "4.15.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "soupsieve" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, -] - [[package]] name = "bidict" version = "0.23.1" @@ -870,20 +823,6 @@ dev = [ { name = "sqlalchemy", extra = ["mypy"] }, { name = "types-jwcrypto" }, ] -docs = [ - { name = "autodoc-pydantic" }, - { name = "furo" }, - { name = "numpydoc" }, - { name = "sphinx" }, - { name = "sphinx-argparse" }, - { name = "sphinx-copybutton" }, - { name = "sphinx-design" }, - { name = "sphinx-favicon" }, - { name = "sphinx-issues" }, - { name = "sphinx-last-updated-by-git" }, - { name = "sphinxcontrib-towncrier" }, - { name = "towncrier" }, -] mddocs = [ { name = "griffe-inherited-docstrings" }, { name = "griffe-pydantic" }, @@ -891,8 +830,11 @@ mddocs = [ { name = "mkdocs-autorefs" }, { name = "mkdocs-literate-nav" }, { name = "mkdocs-material" }, + { name = "mkdocs-towncrier" }, { name = "mkdocstrings", extra = ["python"] }, { name = "pymdown-extensions" }, + { name = "setuptools" }, + { name = "towncrier" }, ] test = [ { name = "coverage" }, @@ -952,20 +894,6 @@ dev = [ { name = "sqlalchemy", extras = ["mypy"], specifier = "~=2.0.50" }, { name = "types-jwcrypto", specifier = "~=1.5.0" }, ] -docs = [ - { name = "autodoc-pydantic", specifier = "~=2.2.0" }, - { name = "furo", specifier = "~=2025.12.19" }, - { name = "numpydoc", specifier = "~=1.10.0" }, - { name = "sphinx", specifier = "~=9.0.4" }, - { name = "sphinx-argparse", specifier = "~=0.5.2" }, - { name = "sphinx-copybutton", specifier = "~=0.5.2" }, - { name = "sphinx-design", specifier = "~=0.7.0" }, - { name = "sphinx-favicon", specifier = "~=1.1.0" }, - { name = "sphinx-issues", specifier = "~=6.0.0" }, - { name = "sphinx-last-updated-by-git", specifier = "~=0.3.8" }, - { name = "sphinxcontrib-towncrier", specifier = "~=0.5.0a0" }, - { name = "towncrier", specifier = "~=25.8.0" }, -] mddocs = [ { name = "griffe-inherited-docstrings", specifier = "~=1.1.3" }, { name = "griffe-pydantic", specifier = "~=1.3.1" }, @@ -973,8 +901,11 @@ mddocs = [ { name = "mkdocs-autorefs", specifier = "~=1.4.4" }, { name = "mkdocs-literate-nav", specifier = "~=0.6.1" }, { name = "mkdocs-material", specifier = "~=9.7.6" }, + { name = "mkdocs-towncrier", specifier = ">=0.1.5,<1" }, { name = "mkdocstrings", extras = ["python"], specifier = "~=1.0.3" }, { name = "pymdown-extensions", specifier = ">=10.21.2,<11.1.0" }, + { name = "setuptools", specifier = "<82" }, + { name = "towncrier", specifier = "~=25.8.0" }, ] test = [ { name = "coverage", specifier = "~=7.14.3" }, @@ -1028,15 +959,6 @@ version = "0.6.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/a2/55/8f8cab2afd404cf578136ef2cc5dfb50baa1761b68c9da1fb1e4eed343c9/docopt-0.6.2.tar.gz", hash = "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491", size = 25901, upload-time = "2014-06-16T11:18:57.406Z" } -[[package]] -name = "docutils" -version = "0.22.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ae/b6/03bb70946330e88ffec97aefd3ea75ba575cb2e762061e0e62a213befee8/docutils-0.22.4.tar.gz", hash = "sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968", size = 2291750, upload-time = "2025-12-18T19:00:26.443Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, -] - [[package]] name = "etl-entities" version = "2.6.0" @@ -1109,22 +1031,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/c1/b821b0677baf86020957359256d12fc05e9eb8b35b995bda069cdae262a9/ftputil-5.2.0-py3-none-any.whl", hash = "sha256:dcf34691161318bef64315d3a942d09f2475e66f653b5391a8c7303606acc70e", size = 51938, upload-time = "2026-04-25T17:23:43.277Z" }, ] -[[package]] -name = "furo" -version = "2025.12.19" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "accessible-pygments" }, - { name = "beautifulsoup4" }, - { name = "pygments" }, - { name = "sphinx" }, - { name = "sphinx-basic-ng" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ec/20/5f5ad4da6a5a27c80f2ed2ee9aee3f9e36c66e56e21c00fde467b2f8f88f/furo-2025.12.19.tar.gz", hash = "sha256:188d1f942037d8b37cd3985b955839fea62baa1730087dc29d157677c857e2a7", size = 1661473, upload-time = "2025-12-19T17:34:40.889Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/b2/50e9b292b5cac13e9e81272c7171301abc753a60460d21505b606e15cf21/furo-2025.12.19-py3-none-any.whl", hash = "sha256:bb0ead5309f9500130665a26bee87693c41ce4dbdff864dbfb6b0dae4673d24f", size = 339262, upload-time = "2025-12-19T17:34:38.905Z" }, -] - [[package]] name = "gevent" version = "26.5.0" @@ -1411,15 +1317,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] -[[package]] -name = "imagesize" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/e6/7bf14eeb8f8b7251141944835abd42eb20a658d89084b7e1f3e5fe394090/imagesize-2.0.0.tar.gz", hash = "sha256:8e8358c4a05c304f1fccf7ff96f036e7243a189e9e42e90851993c558cfe9ee3", size = 1773045, upload-time = "2026-03-03T14:18:29.941Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl", hash = "sha256:5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96", size = 9441, upload-time = "2026-03-03T14:18:27.892Z" }, -] - [[package]] name = "importlib-metadata" version = "9.0.0" @@ -1914,6 +1811,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/54/662a4743aa81d9582ee9339d4ffa3c8fd40a4965e033d77b9da9774d3960/mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31", size = 8728, upload-time = "2023-11-22T19:09:43.465Z" }, ] +[[package]] +name = "mkdocs-towncrier" +version = "0.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mkdocs" }, + { name = "towncrier" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/02/21/2b9ed48625a771310e9123b4f53c480bcea54731a40bee6af641b7ce2f04/mkdocs_towncrier-0.1.5.tar.gz", hash = "sha256:081dc80f3bc84035b9ee6994e0d8fb88acf3ccc123bcefb4cccabe0f515892de", size = 20321, upload-time = "2025-08-31T16:29:29.294Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/c1/1b180400e8912d7ff477ce569ae1f2c0d80890d2558ab8fe18b2f07e95c1/mkdocs_towncrier-0.1.5-py3-none-any.whl", hash = "sha256:ea6819a5600912a4238890cf9b921d2933eea1a2d0e7e862861caf091ef4e4b4", size = 4855, upload-time = "2025-08-31T16:29:28.254Z" }, +] + [[package]] name = "mkdocstrings" version = "1.0.4" @@ -2010,18 +1920,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, ] -[[package]] -name = "numpydoc" -version = "1.10.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e9/3c/dfccc9e7dee357fb2aa13c3890d952a370dd0ed071e0f7ed62ed0df567c1/numpydoc-1.10.0.tar.gz", hash = "sha256:3f7970f6eee30912260a6b31ac72bba2432830cd6722569ec17ee8d3ef5ffa01", size = 94027, upload-time = "2025-12-02T16:39:12.937Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/5e/3a6a3e90f35cea3853c45e5d5fb9b7192ce4384616f932cf7591298ab6e1/numpydoc-1.10.0-py3-none-any.whl", hash = "sha256:3149da9874af890bcc2a82ef7aae5484e5aa81cb2778f08e3c307ba6d963721b", size = 69255, upload-time = "2025-12-02T16:39:11.561Z" }, -] - [[package]] name = "onetl" version = "0.16.0" @@ -2855,15 +2753,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1d/4a/221da6ca167db45693d8d26c7dc79ccfc978a440251bf6721c9aaf251ac0/respx-0.23.1-py2.py3-none-any.whl", hash = "sha256:b18004b029935384bccfa6d7d9d74b4ec9af73a081cc28600fffc0447f4b8c1a", size = 25557, upload-time = "2026-04-08T14:37:14.613Z" }, ] -[[package]] -name = "roman-numerals" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, -] - [[package]] name = "ruff" version = "0.15.18" @@ -2890,212 +2779,21 @@ wheels = [ ] [[package]] -name = "six" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, -] - -[[package]] -name = "snowballstemmer" -version = "3.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/f8/0a71edf031f03c40db17503cb8ca78a69a171254e568e7db241b0ab57ea1/snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260", size = 123314, upload-time = "2026-06-03T00:56:40.194Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, -] - -[[package]] -name = "soupsieve" -version = "2.8.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/47/2c/0a5f6f8ee0d5589e48c7640213ed5175d52cf540a06725b628cc1a45d6ce/soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e", size = 121110, upload-time = "2026-05-24T13:55:57.154Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, -] - -[[package]] -name = "sphinx" -version = "9.0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "alabaster" }, - { name = "babel" }, - { name = "colorama", marker = "sys_platform == 'win32'" }, - { name = "docutils" }, - { name = "imagesize" }, - { name = "jinja2" }, - { name = "packaging" }, - { name = "pygments" }, - { name = "requests" }, - { name = "roman-numerals" }, - { name = "snowballstemmer" }, - { name = "sphinxcontrib-applehelp" }, - { name = "sphinxcontrib-devhelp" }, - { name = "sphinxcontrib-htmlhelp" }, - { name = "sphinxcontrib-jsmath" }, - { name = "sphinxcontrib-qthelp" }, - { name = "sphinxcontrib-serializinghtml" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/50/a8c6ccc36d5eacdfd7913ddccd15a9cee03ecafc5ee2bc40e1f168d85022/sphinx-9.0.4.tar.gz", hash = "sha256:594ef59d042972abbc581d8baa577404abe4e6c3b04ef61bd7fc2acbd51f3fa3", size = 8710502, upload-time = "2025-12-04T07:45:27.343Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/3f/4bbd76424c393caead2e1eb89777f575dee5c8653e2d4b6afd7a564f5974/sphinx-9.0.4-py3-none-any.whl", hash = "sha256:5bebc595a5e943ea248b99c13814c1c5e10b3ece718976824ffa7959ff95fffb", size = 3917713, upload-time = "2025-12-04T07:45:24.944Z" }, -] - -[[package]] -name = "sphinx-argparse" -version = "0.5.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "docutils" }, - { name = "sphinx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/21/a8c64e6633652111e6e4f89703182a53cbc3ed67233523e47472101358b6/sphinx_argparse-0.5.2.tar.gz", hash = "sha256:e5352f8fa894b6fb6fda0498ba28a9f8d435971ef4bbc1a6c9c6414e7644f032", size = 27838, upload-time = "2024-07-17T12:08:08.219Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/43/9f0e9bfb3ce02cbf7747aa2185c48a9d6e42ba95736a5e8f511a5054d976/sphinx_argparse-0.5.2-py3-none-any.whl", hash = "sha256:d771b906c36d26dee669dbdbb5605c558d9440247a5608b810f7fa6e26ab1fd3", size = 12547, upload-time = "2024-07-17T12:08:06.307Z" }, -] - -[[package]] -name = "sphinx-basic-ng" -version = "1.0.0b2" +name = "setuptools" +version = "81.0.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/98/0b/a866924ded68efec7a1759587a4e478aec7559d8165fac8b2ad1c0e774d6/sphinx_basic_ng-1.0.0b2.tar.gz", hash = "sha256:9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9", size = 20736, upload-time = "2023-07-08T18:40:54.166Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/1c/73e719955c59b8e424d015ab450f51c0af856ae46ea2da83eba51cc88de1/setuptools-81.0.0.tar.gz", hash = "sha256:487b53915f52501f0a79ccfd0c02c165ffe06631443a886740b91af4b7a5845a", size = 1198299, upload-time = "2026-02-06T21:10:39.601Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/dd/018ce05c532a22007ac58d4f45232514cd9d6dd0ee1dc374e309db830983/sphinx_basic_ng-1.0.0b2-py3-none-any.whl", hash = "sha256:eb09aedbabfb650607e9b4b68c9d240b90b1e1be221d6ad71d61c52e29f7932b", size = 22496, upload-time = "2023-07-08T18:40:52.659Z" }, + { url = "https://files.pythonhosted.org/packages/e1/e3/c164c88b2e5ce7b24d667b9bd83589cf4f3520d97cad01534cd3c4f55fdb/setuptools-81.0.0-py3-none-any.whl", hash = "sha256:fdd925d5c5d9f62e4b74b30d6dd7828ce236fd6ed998a08d81de62ce5a6310d6", size = 1062021, upload-time = "2026-02-06T21:10:37.175Z" }, ] [[package]] -name = "sphinx-copybutton" -version = "0.5.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fc/2b/a964715e7f5295f77509e59309959f4125122d648f86b4fe7d70ca1d882c/sphinx-copybutton-0.5.2.tar.gz", hash = "sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd", size = 23039, upload-time = "2023-04-14T08:10:22.998Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9e/48/1ea60e74949eecb12cdd6ac43987f9fd331156388dcc2319b45e2ebb81bf/sphinx_copybutton-0.5.2-py3-none-any.whl", hash = "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e", size = 13343, upload-time = "2023-04-14T08:10:20.844Z" }, -] - -[[package]] -name = "sphinx-design" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/13/7b/804f311da4663a4aecc6cf7abd83443f3d4ded970826d0c958edc77d4527/sphinx_design-0.7.0.tar.gz", hash = "sha256:d2a3f5b19c24b916adb52f97c5f00efab4009ca337812001109084a740ec9b7a", size = 2203582, upload-time = "2026-01-19T13:12:53.297Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/30/cf/45dd359f6ca0c3762ce0490f681da242f0530c49c81050c035c016bfdd3a/sphinx_design-0.7.0-py3-none-any.whl", hash = "sha256:f82bf179951d58f55dca78ab3706aeafa496b741a91b1911d371441127d64282", size = 2220350, upload-time = "2026-01-19T13:12:51.077Z" }, -] - -[[package]] -name = "sphinx-favicon" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "imagesize" }, - { name = "requests" }, - { name = "sphinx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/2c/26/e7ca2321e6286d6ed6a2e824a0ee35ae660ec9a45a4719e33a627ce9e4d2/sphinx_favicon-1.1.0.tar.gz", hash = "sha256:6f65939fc2a6ac4259c88b09169f0b72681cd4c03dd1d0cf91c57a1fa314e50b", size = 8744, upload-time = "2026-02-12T20:55:41.294Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/99/c85bc52d785557abddc4d8fdacfbcbda56fb3e715f76b9675fb9c2018aa5/sphinx_favicon-1.1.0-py3-none-any.whl", hash = "sha256:3ca71506fbb4d9a30bddc60a29e3fb8854f3e237ad95abad5a5c15f857d987b2", size = 7241, upload-time = "2026-02-12T20:55:40.312Z" }, -] - -[[package]] -name = "sphinx-issues" -version = "6.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/95/4e/c9a46b19c90d4d3cdcbcaf0e4d392b09e5b41ec75b134bc5a1aa192be94f/sphinx_issues-6.0.0.tar.gz", hash = "sha256:f40f2c71cecb2068c7f06a53825778b674d58d8395b4ea60551a36164d2988e3", size = 15230, upload-time = "2026-03-13T17:23:32.501Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/29/8a/28333d222ac31539aa0d818db0790077e5a6a408f933b4bf64f7d3440ffc/sphinx_issues-6.0.0-py3-none-any.whl", hash = "sha256:c7ed2915059526d02022733985a7712d4b2b64a707e16b98294ed9758e64df4f", size = 8362, upload-time = "2026-03-13T17:23:31.031Z" }, -] - -[[package]] -name = "sphinx-last-updated-by-git" -version = "0.3.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/03/fd/de1685b6dab173dff31da24e0d3b29f02873fc24a1cdbb7678721ddc8581/sphinx_last_updated_by_git-0.3.8.tar.gz", hash = "sha256:c145011f4609d841805b69a9300099fc02fed8f5bb9e5bcef77d97aea97b7761", size = 10785, upload-time = "2024-08-11T07:15:54.601Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/fb/e496f16fa11fbe2dbdd0b5e306ede153dfed050aae4766fc89d500720dc7/sphinx_last_updated_by_git-0.3.8-py3-none-any.whl", hash = "sha256:6382c8285ac1f222483a58569b78c0371af5e55f7fbf9c01e5e8a72d6fdfa499", size = 8580, upload-time = "2024-08-11T07:15:53.244Z" }, -] - -[[package]] -name = "sphinxcontrib-applehelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, -] - -[[package]] -name = "sphinxcontrib-devhelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, -] - -[[package]] -name = "sphinxcontrib-htmlhelp" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, -] - -[[package]] -name = "sphinxcontrib-jsmath" -version = "1.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, -] - -[[package]] -name = "sphinxcontrib-qthelp" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, -] - -[[package]] -name = "sphinxcontrib-serializinghtml" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, -] - -[[package]] -name = "sphinxcontrib-towncrier" -version = "0.5.0a0" +name = "six" +version = "1.17.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "sphinx" }, - { name = "towncrier" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/21/fe/72ed57093e28af10595c50839b183c5fdf0952482e9ef0ca6eb90eb85c5d/sphinxcontrib_towncrier-0.5.0a0.tar.gz", hash = "sha256:294e69df6e275e7a86df7ea6a927cc7c28c2c370a884cd5c45de6ec989858f27", size = 62453, upload-time = "2025-02-28T01:59:16.894Z" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/5c/f7e39f243636a5e1894f2f5a72579977bf3968922afdb75175ee45062066/sphinxcontrib_towncrier-0.5.0a0-py3-none-any.whl", hash = "sha256:11d130c3ad5e4649821d543c4ea7ab64bbe78df4d859ef94f4298e7845dc0f59", size = 12609, upload-time = "2025-02-28T01:59:15.178Z" }, + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] [[package]]