From fa208a3c3aa6ff09228269c142b41be04c83132a Mon Sep 17 00:00:00 2001 From: gary Date: Fri, 25 Oct 2024 11:25:02 +0800 Subject: [PATCH 001/236] chore: update workflow (#128) * fix: fix pre annoation exceptions (#120) * chore(release): 1.0.7 [skip ci] ## [1.0.7](https://github.com/opendatalab/labelU/compare/v1.0.6...v1.0.7) (2024-08-26) ### Bug Fixes * fix pre annoation exceptions ([#120](https://github.com/opendatalab/labelU/issues/120)) ([bd61682](https://github.com/opendatalab/labelU/commit/bd616821eb6dea47840df29efe340c3b96c2446f)) * docs: Update README.md * docs: Update README.md * docs: Update README_zh-CN.md (#126) * docs: Update README_zh-CN.md * docs: Update README_zh-CN.md * chore: update workflow --------- Co-authored-by: semantic-release-bot --- .github/workflows/trigger_from_labelu_kit.yml | 2 +- README.md | 6 +++--- README_zh-CN.md | 6 +++--- pyproject.toml | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/trigger_from_labelu_kit.yml b/.github/workflows/trigger_from_labelu_kit.yml index 32335ea1..fe4c6dde 100644 --- a/.github/workflows/trigger_from_labelu_kit.yml +++ b/.github/workflows/trigger_from_labelu_kit.yml @@ -71,7 +71,7 @@ jobs: - name: Prepare changelog id: set_changelog run: | - echo "CHANGE_LOG=$(echo "${{ inputs.changelog }}" | sed ':a;N;$!ba;s/\n/\\n/g')" >> $GITHUB_ENV + echo "CHANGE_LOG=$(echo '${{ inputs.changelog }}' | sed ':a;N;$!ba;s/\n/\\n/g')" >> $GITHUB_ENV - name: Trigger release workflow diff --git a/README.md b/README.md index 120e7907..26386dd4 100644 --- a/README.md +++ b/README.md @@ -104,11 +104,11 @@ conda activate labelu # Install all package dependencies poetry install +# Download the frontend statics from labelu-kit repo +sh ./scripts/resolve_frontend.sh true + # Start labelu, server: http://localhost:8000 uvicorn labelu.main:app --reload - -# Update submodule -git submodule update --remote --merge ``` diff --git a/README_zh-CN.md b/README_zh-CN.md index 0eaa6bbd..b1481488 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -101,11 +101,11 @@ conda activate labelu # 安装所有依赖包 poetry install +# 从 LabelU-kit 下载前端资源 +sh ./scripts/resolve_frontend.sh true + # 启动labelu, 默认访问地址: http://localhost:8000 uvicorn labelu.main:app --reload - -# 更新submodule -git submodule update --remote --merge ``` ## 快速上手 diff --git a/pyproject.toml b/pyproject.toml index ff3e356f..68001f9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.1' +version = '1.0.7' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 01fc3b53b570de4d05ad771f163d8da03236b141 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 25 Oct 2024 04:07:55 +0000 Subject: [PATCH 002/236] fix: Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 4e600081..18791dbf 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.3/frontend.zip -alpha_version: v5.3.0-alpha.3 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.4/frontend.zip +alpha_version: v5.3.0-alpha.4 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0/frontend.zip release_version: v5.3.0 From 2fc382b68365d1bcc6cb593c152df2be9cdd80d6 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 25 Oct 2024 04:09:59 +0000 Subject: [PATCH 003/236] chore(release): 1.0.7-alpha.2 [skip ci] ## [1.0.7-alpha.2](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.1...v1.0.7-alpha.2) (2024-10-25) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.3) ([e8721d6](https://github.com/opendatalab/labelU/commit/e8721d6ab4fdb288ef16263ffb09c73ea12ed52f)) * Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) ([01fc3b5](https://github.com/opendatalab/labelU/commit/01fc3b53b570de4d05ad771f163d8da03236b141)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 68001f9f..4996936a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7' +version = '1.0.7-alpha.2' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From f97601d0c4286c633ca2ce3a1f5ef3d8eec5a540 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 25 Oct 2024 12:03:18 +0000 Subject: [PATCH 004/236] fix: Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 18791dbf..c72c0efc 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.4/frontend.zip -alpha_version: v5.3.0-alpha.4 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.5/frontend.zip +alpha_version: v5.3.0-alpha.5 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0/frontend.zip release_version: v5.3.0 From 2b270ea8ae24d818977f91040d881e754c9e87fe Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 25 Oct 2024 12:05:28 +0000 Subject: [PATCH 005/236] chore(release): 1.0.7-alpha.3 [skip ci] ## [1.0.7-alpha.3](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.2...v1.0.7-alpha.3) (2024-10-25) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) ([f97601d](https://github.com/opendatalab/labelU/commit/f97601d0c4286c633ca2ce3a1f5ef3d8eec5a540)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4996936a..66d74a4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.2' +version = '1.0.7-alpha.3' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From b6abe172f7a8bb1a5547acb2bc8948bb7c0dae7d Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 11:05:14 +0800 Subject: [PATCH 006/236] fix: update notification action --- .github/workflows/release.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2e8a29a1..32563f41 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -177,11 +177,24 @@ jobs: # ====================== post release ====================== - - name: Send webhook message - uses: joelwmale/webhook-action@master + - name: Send http post request env: CHANGELOG: ${{ github.event.inputs.changelog != '' && format('## Frontend changelog👇🏻 \n\n{0}\n\n\n', github.event.inputs.changelog) || '' }} - with: - url: ${{ secrets.WEBHOOK_URL }} - headers: '{"Content-Type": "application/json"}' - body: '{"msg_type":"interactive","card":{"elements":[{"tag":"div","text":{"tag":"lark_md","content":"${{ env.RELEASE_NOTES }}${{ env.CHANGELOG }}Check it out now \ud83d\udc49\ud83c\udffb [v${{ env.NEXT_VERSION }}](${{ env.PYPI_URL }})"}}]}}' + WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} + run: | + curl -X POST $WEBHOOK_URL \ + -H "Content-Type: application/json" \ + -d '{ + "msg_type": "interactive", + "card": { + "elements": [ + { + "tag": "div", + "text": { + "tag": "lark_md", + "content": "'"${{ env.RELEASE_NOTES }}${{ env.CHANGELOG }}Check it out now \ud83d\udc49\ud83c\udffb [v${{ env.NEXT_VERSION }}](${{ env.PYPI_URL }})"'" + } + } + ] + } + }' \ No newline at end of file From 5ae2b8eaf9f038c03e5c75cfa7db2a90d068d3a8 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 28 Oct 2024 03:07:42 +0000 Subject: [PATCH 007/236] chore(release): 1.0.7-alpha.4 [skip ci] ## [1.0.7-alpha.4](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.3...v1.0.7-alpha.4) (2024-10-28) ### Bug Fixes * update notification action ([b6abe17](https://github.com/opendatalab/labelU/commit/b6abe172f7a8bb1a5547acb2bc8948bb7c0dae7d)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 66d74a4f..4123d0f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.3' +version = '1.0.7-alpha.4' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 04e0388e3cd98a2aacf12dc28c638c172eb6f0ff Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 11:26:31 +0800 Subject: [PATCH 008/236] fix: update notification action --- .github/workflows/release.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 32563f41..28260373 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,11 +71,15 @@ jobs: - name: Set test pip url if: ${{ github.ref_name == 'alpha' }} + env: + NEXT_VERSION: ${{ env.NEXT_VERSION }} run: | echo "PYPI_URL=https://test.pypi.org/project/labelu/${{ env.NEXT_VERSION }}" >> $GITHUB_ENV - name: Set release pip url if: ${{ github.ref_name == 'main' }} + env: + NEXT_VERSION: ${{ env.NEXT_VERSION }} run: | echo "PYPI_URL=https://pypi.org/project/labelu/${{ env.NEXT_VERSION }}" >> $GITHUB_ENV @@ -181,6 +185,9 @@ jobs: env: CHANGELOG: ${{ github.event.inputs.changelog != '' && format('## Frontend changelog👇🏻 \n\n{0}\n\n\n', github.event.inputs.changelog) || '' }} WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} + RELEASE_NOTES: ${{ env.RELEASE_NOTES }} + NEXT_VERSION: ${{ env.NEXT_VERSION }} + PYPI_URL: ${{ env.PYPI_URL }} run: | curl -X POST $WEBHOOK_URL \ -H "Content-Type: application/json" \ From 5afaf218d687ee38e4153ca3d8223692d20a7bfc Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 28 Oct 2024 03:28:35 +0000 Subject: [PATCH 009/236] chore(release): 1.0.7-alpha.5 [skip ci] ## [1.0.7-alpha.5](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.4...v1.0.7-alpha.5) (2024-10-28) ### Bug Fixes * update notification action ([04e0388](https://github.com/opendatalab/labelU/commit/04e0388e3cd98a2aacf12dc28c638c172eb6f0ff)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4123d0f6..a6ae2355 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.4' +version = '1.0.7-alpha.5' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From e9152c98869d70e500fd9ab458d8ba179270d9c9 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 11:33:32 +0800 Subject: [PATCH 010/236] fix: update notification action --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 28260373..14416ff9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -189,7 +189,7 @@ jobs: NEXT_VERSION: ${{ env.NEXT_VERSION }} PYPI_URL: ${{ env.PYPI_URL }} run: | - curl -X POST $WEBHOOK_URL \ + curl -v -X POST $WEBHOOK_URL \ -H "Content-Type: application/json" \ -d '{ "msg_type": "interactive", From 9076388b108820921cb8cf92079acf46bead2316 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 28 Oct 2024 03:35:48 +0000 Subject: [PATCH 011/236] chore(release): 1.0.7-alpha.6 [skip ci] ## [1.0.7-alpha.6](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.5...v1.0.7-alpha.6) (2024-10-28) ### Bug Fixes * update notification action ([e9152c9](https://github.com/opendatalab/labelU/commit/e9152c98869d70e500fd9ab458d8ba179270d9c9)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a6ae2355..595fb9b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.5' +version = '1.0.7-alpha.6' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 046f47a3f4053e6934f04c3c0ede5518b9f1850b Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 11:41:58 +0800 Subject: [PATCH 012/236] fix: update notification action --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 14416ff9..279d8dd1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -189,7 +189,7 @@ jobs: NEXT_VERSION: ${{ env.NEXT_VERSION }} PYPI_URL: ${{ env.PYPI_URL }} run: | - curl -v -X POST $WEBHOOK_URL \ + curl -v POST $WEBHOOK_URL \ -H "Content-Type: application/json" \ -d '{ "msg_type": "interactive", @@ -199,7 +199,7 @@ jobs: "tag": "div", "text": { "tag": "lark_md", - "content": "'"${{ env.RELEASE_NOTES }}${{ env.CHANGELOG }}Check it out now \ud83d\udc49\ud83c\udffb [v${{ env.NEXT_VERSION }}](${{ env.PYPI_URL }})"'" + "content": "'"${RELEASE_NOTES}${CHANGELOG}Check it out now \ud83d\udc49\ud83c\udffb [v${NEXT_VERSION}](${PYPI_URL})"'" } } ] From c040ef4e4e48dd363ff9f04423d3e555726f6e17 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 28 Oct 2024 03:44:13 +0000 Subject: [PATCH 013/236] chore(release): 1.0.7-alpha.7 [skip ci] ## [1.0.7-alpha.7](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.6...v1.0.7-alpha.7) (2024-10-28) ### Bug Fixes * update notification action ([046f47a](https://github.com/opendatalab/labelU/commit/046f47a3f4053e6934f04c3c0ede5518b9f1850b)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 595fb9b7..0ae67876 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.6' +version = '1.0.7-alpha.7' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 8fed078a0d77a8204430fb6c24d91de1250a8bc7 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 11:46:48 +0800 Subject: [PATCH 014/236] fix: test notification action --- .github/workflows/release.yml | 44 +++++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 279d8dd1..0f1397f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -189,19 +189,33 @@ jobs: NEXT_VERSION: ${{ env.NEXT_VERSION }} PYPI_URL: ${{ env.PYPI_URL }} run: | - curl -v POST $WEBHOOK_URL \ - -H "Content-Type: application/json" \ - -d '{ - "msg_type": "interactive", - "card": { - "elements": [ - { - "tag": "div", - "text": { - "tag": "lark_md", - "content": "'"${RELEASE_NOTES}${CHANGELOG}Check it out now \ud83d\udc49\ud83c\udffb [v${NEXT_VERSION}](${PYPI_URL})"'" - } - } - ] + echo '{ + "msg_type": "interactive", + "card": { + "elements": [ + { + "tag": "div", + "text": { + "tag": "lark_md", + "content": "'"${RELEASE_NOTES}${CHANGELOG}Check it out now \ud83d\udc49\ud83c\udffb [v${NEXT_VERSION}](${PYPI_URL})"'" + } } - }' \ No newline at end of file + ] + } + }' + # curl -v POST $WEBHOOK_URL \ + # -H "Content-Type: application/json" \ + # -d '{ + # "msg_type": "interactive", + # "card": { + # "elements": [ + # { + # "tag": "div", + # "text": { + # "tag": "lark_md", + # "content": "'"${RELEASE_NOTES}${CHANGELOG}Check it out now \ud83d\udc49\ud83c\udffb [v${NEXT_VERSION}](${PYPI_URL})"'" + # } + # } + # ] + # } + # }' \ No newline at end of file From f96d234a1c8a5793e5a1b6f6fdc3986f57ea1bcc Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 28 Oct 2024 03:48:45 +0000 Subject: [PATCH 015/236] chore(release): 1.0.7-alpha.8 [skip ci] ## [1.0.7-alpha.8](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.7...v1.0.7-alpha.8) (2024-10-28) ### Bug Fixes * test notification action ([8fed078](https://github.com/opendatalab/labelU/commit/8fed078a0d77a8204430fb6c24d91de1250a8bc7)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0ae67876..1f1b60a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.7' +version = '1.0.7-alpha.8' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 851818e912cf39690be6e689c1926ec29c148a66 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 11:54:08 +0800 Subject: [PATCH 016/236] fix: test notification action --- .github/workflows/release.yml | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0f1397f6..e4769b31 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -189,7 +189,8 @@ jobs: NEXT_VERSION: ${{ env.NEXT_VERSION }} PYPI_URL: ${{ env.PYPI_URL }} run: | - echo '{ + REQUEST_BODY=$(cat < Date: Mon, 28 Oct 2024 03:56:11 +0000 Subject: [PATCH 017/236] chore(release): 1.0.7-alpha.9 [skip ci] ## [1.0.7-alpha.9](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.8...v1.0.7-alpha.9) (2024-10-28) ### Bug Fixes * test notification action ([851818e](https://github.com/opendatalab/labelU/commit/851818e912cf39690be6e689c1926ec29c148a66)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1f1b60a0..1557d29e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.8' +version = '1.0.7-alpha.9' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From bc63ee59da0d079208b1385d33184267251a2bb5 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 12:01:33 +0800 Subject: [PATCH 018/236] fix: test notification action --- .github/workflows/release.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e4769b31..2e58b6f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -190,22 +190,22 @@ jobs: PYPI_URL: ${{ env.PYPI_URL }} run: | REQUEST_BODY=$(cat < Date: Mon, 28 Oct 2024 04:03:35 +0000 Subject: [PATCH 019/236] chore(release): 1.0.7-alpha.10 [skip ci] ## [1.0.7-alpha.10](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.9...v1.0.7-alpha.10) (2024-10-28) ### Bug Fixes * test notification action ([bc63ee5](https://github.com/opendatalab/labelU/commit/bc63ee59da0d079208b1385d33184267251a2bb5)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1557d29e..4a276274 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.9' +version = '1.0.7-alpha.10' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 83f43eb9d03390686a41dc22666a058190bfb63c Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 12:47:24 +0800 Subject: [PATCH 020/236] fix: test notification action --- .github/workflows/release.yml | 26 ++------ scripts/release-notification.js | 63 +++++++++++++++++++ scripts/utils.js | 107 ++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 22 deletions(-) create mode 100644 scripts/release-notification.js create mode 100644 scripts/utils.js diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2e58b6f5..5934d2da 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,6 +46,7 @@ jobs: python-version: [3.11] poetry-version: ['1.2.2'] os: [ubuntu-20.04] + node-version: [20.8.1] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -181,7 +182,7 @@ jobs: # ====================== post release ====================== - - name: Send http post request + - name: Send release notification env: CHANGELOG: ${{ github.event.inputs.changelog != '' && format('## Frontend changelog👇🏻 \n\n{0}\n\n\n', github.event.inputs.changelog) || '' }} WEBHOOK_URL: ${{ secrets.WEBHOOK_URL }} @@ -189,24 +190,5 @@ jobs: NEXT_VERSION: ${{ env.NEXT_VERSION }} PYPI_URL: ${{ env.PYPI_URL }} run: | - REQUEST_BODY=$(cat < 0) { + throw new Error(`POST 请求失败: ${responseBody.msg}`) + } + + console.log('Release 信息已成功发送') + } + catch (error) { + console.error('发生错误:', error) + process.exit(1) + } +} + +main() diff --git a/scripts/utils.js b/scripts/utils.js new file mode 100644 index 00000000..a1adbf46 --- /dev/null +++ b/scripts/utils.js @@ -0,0 +1,107 @@ +const https = require('node:https') + +function httpGet(url, options) { + return new Promise((resolve, reject) => { + const req = https.get(url, options, (res) => { + let data = '' + + res.on('data', (chunk) => { + data += chunk + }) + res.on('end', () => { + resolve(data) + }) + + if (res.statusCode && res.statusCode >= 400) { + reject(res) + } + }) + + req.on('error', (err) => { + reject(err) + }) + }) +} + +function httpPost(url, options, payload) { + const mergedOptions = { + ...(options || {}), + method: 'POST', + } + + return new Promise((resolve, reject) => { + const req = https.request(url, mergedOptions, (res) => { + res.setEncoding('utf8') + + let responseBody = '' + + res.on('data', (chunk) => { + if (typeof chunk === 'string') { + responseBody += chunk + } + }) + + res.on('end', () => { + resolve(responseBody) + }) + + if (res.statusCode && res.statusCode >= 400) { + reject(res) + } + }) + + req.on('error', (err) => { + reject(err) + }) + + let finalPayload = payload + + if (mergedOptions?.headers?.['Content-Type'] === 'application/json' && typeof payload === 'object') { + finalPayload = JSON.stringify(payload) + } + + req.write(finalPayload) + req.end() + }) +} + +function makeLarkMessage(title, content, footer, link) { + return { + msg_type: 'interactive', + card: { + header: { + title: { + tag: 'plain_text', + content: title, + }, + }, + elements: [{ + tag: 'div', + text: { + tag: 'lark_md', + content, + }, + }, { + tag: 'action', + actions: [ + { + tag: 'button', + text: { + tag: 'lark_md', + content: footer, + }, + url: link, + type: 'default', + value: {}, + }, + ], + }], + }, + } +} + +module.exports = { + httpGet, + httpPost, + makeLarkMessage, +} From 8d372fd703fac162d33fc9478014f67f0e2e8071 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 28 Oct 2024 04:49:38 +0000 Subject: [PATCH 021/236] chore(release): 1.0.7-alpha.11 [skip ci] ## [1.0.7-alpha.11](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.10...v1.0.7-alpha.11) (2024-10-28) ### Bug Fixes * test notification action ([83f43eb](https://github.com/opendatalab/labelU/commit/83f43eb9d03390686a41dc22666a058190bfb63c)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4a276274..c466413b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.10' +version = '1.0.7-alpha.11' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 63286cb6958e20ab3c80495ab3b51a1ddd31dfb7 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 12:51:17 +0800 Subject: [PATCH 022/236] fix: update notification action --- scripts/release-notification.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/release-notification.js b/scripts/release-notification.js index f0057f7c..b4c310c6 100644 --- a/scripts/release-notification.js +++ b/scripts/release-notification.js @@ -27,11 +27,12 @@ async function main() { const pypiUrl = process.env.PYPI_URL console.log('url', botUrl) + const frontendContent = frontendChangelog ? `\n\n**Frontend Changelog**\n${format(frontendChangelog)}` : '' try { const payload = makeLarkMessage( `LabelU@v${releaseTag}`, - `${format(releaseNotes)}\n\n**Frontend Changelog**\n${format(frontendChangelog)}`, + `${format(releaseNotes)}${frontendContent}`, '点击访问', pypiUrl, ) From 60b515121b3a2ffdb536a742c2e55fc342afdd8c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 28 Oct 2024 04:53:19 +0000 Subject: [PATCH 023/236] chore(release): 1.0.7-alpha.12 [skip ci] ## [1.0.7-alpha.12](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.11...v1.0.7-alpha.12) (2024-10-28) ### Bug Fixes * update notification action ([63286cb](https://github.com/opendatalab/labelU/commit/63286cb6958e20ab3c80495ab3b51a1ddd31dfb7)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c466413b..c40f947c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.7-alpha.11' +version = '1.0.7-alpha.12' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 84ac337e450f9405cf80c39c2897ec33b8966aa6 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 28 Oct 2024 05:45:50 +0000 Subject: [PATCH 024/236] chore(release): 1.0.9-alpha.1 [skip ci] ## [1.0.9-alpha.1](https://github.com/opendatalab/labelU/compare/v1.0.8...v1.0.9-alpha.1) (2024-10-28) ### Bug Fixes * test notification action ([83f43eb](https://github.com/opendatalab/labelU/commit/83f43eb9d03390686a41dc22666a058190bfb63c)) * test notification action ([bc63ee5](https://github.com/opendatalab/labelU/commit/bc63ee59da0d079208b1385d33184267251a2bb5)) * test notification action ([851818e](https://github.com/opendatalab/labelU/commit/851818e912cf39690be6e689c1926ec29c148a66)) * test notification action ([8fed078](https://github.com/opendatalab/labelU/commit/8fed078a0d77a8204430fb6c24d91de1250a8bc7)) * update notification action ([63286cb](https://github.com/opendatalab/labelU/commit/63286cb6958e20ab3c80495ab3b51a1ddd31dfb7)) * update notification action ([046f47a](https://github.com/opendatalab/labelU/commit/046f47a3f4053e6934f04c3c0ede5518b9f1850b)) * update notification action ([e9152c9](https://github.com/opendatalab/labelU/commit/e9152c98869d70e500fd9ab458d8ba179270d9c9)) * update notification action ([04e0388](https://github.com/opendatalab/labelU/commit/04e0388e3cd98a2aacf12dc28c638c172eb6f0ff)) * update notification action ([b6abe17](https://github.com/opendatalab/labelU/commit/b6abe172f7a8bb1a5547acb2bc8948bb7c0dae7d)) * Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) ([01fc3b5](https://github.com/opendatalab/labelU/commit/01fc3b53b570de4d05ad771f163d8da03236b141)) * Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) ([f97601d](https://github.com/opendatalab/labelU/commit/f97601d0c4286c633ca2ce3a1f5ef3d8eec5a540)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4e599d3e..6a15336a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.8' +version = '1.0.9-alpha.1' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 9f5e698c8a0476be0bb034bb12e226b9db211328 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 08:30:21 +0000 Subject: [PATCH 025/236] fix: Upgrade frontend to v5.3.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.6) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index c72c0efc..74425a3c 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.5/frontend.zip -alpha_version: v5.3.0-alpha.5 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.6/frontend.zip +alpha_version: v5.3.0-alpha.6 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0/frontend.zip release_version: v5.3.0 From 760b4e86d25bdb5987af970b9423e5d500953982 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 28 Oct 2024 08:32:34 +0000 Subject: [PATCH 026/236] chore(release): 1.0.9-alpha.2 [skip ci] ## [1.0.9-alpha.2](https://github.com/opendatalab/labelU/compare/v1.0.9-alpha.1...v1.0.9-alpha.2) (2024-10-28) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.6) ([9f5e698](https://github.com/opendatalab/labelU/commit/9f5e698c8a0476be0bb034bb12e226b9db211328)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6a15336a..ddeb00e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.9-alpha.1' +version = '1.0.9-alpha.2' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 7cb455eaf06b8408f2cacfe3cc39c8c2e3bde013 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 28 Oct 2024 11:52:04 +0000 Subject: [PATCH 027/236] fix: Upgrade frontend to v5.3.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.7) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 74425a3c..455bb710 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.6/frontend.zip -alpha_version: v5.3.0-alpha.6 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.7/frontend.zip +alpha_version: v5.3.0-alpha.7 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0/frontend.zip release_version: v5.3.0 From 40152c143c06a0bb54991a3bb5db1eb4b8dbe67d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 28 Oct 2024 11:54:21 +0000 Subject: [PATCH 028/236] chore(release): 1.0.9-alpha.3 [skip ci] ## [1.0.9-alpha.3](https://github.com/opendatalab/labelU/compare/v1.0.9-alpha.2...v1.0.9-alpha.3) (2024-10-28) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.7) ([7cb455e](https://github.com/opendatalab/labelU/commit/7cb455eaf06b8408f2cacfe3cc39c8c2e3bde013)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ddeb00e5..a7f66cc9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.9-alpha.2' +version = '1.0.9-alpha.3' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 7b046706c7c1ff7e9fa94fd912f4263d33181e7b Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 29 Oct 2024 08:15:59 +0000 Subject: [PATCH 029/236] fix: Upgrade frontend to v5.5.0-alpha.1. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.1) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 455bb710..bfc3756f 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.7/frontend.zip -alpha_version: v5.3.0-alpha.7 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.1/frontend.zip +alpha_version: v5.5.0-alpha.1 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0/frontend.zip release_version: v5.3.0 From 491cf335a0f10c1565536bc0c4cea254d0b6b23b Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 29 Oct 2024 08:30:50 +0000 Subject: [PATCH 030/236] fix: Upgrade frontend to v5.5.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.2) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index bfc3756f..bda0d7c6 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.1/frontend.zip -alpha_version: v5.5.0-alpha.1 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.2/frontend.zip +alpha_version: v5.5.0-alpha.2 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0/frontend.zip release_version: v5.3.0 From ea9baed25182dac6e0ea806c5054dcd12856ab2d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 29 Oct 2024 08:33:02 +0000 Subject: [PATCH 031/236] chore(release): 1.0.9-alpha.4 [skip ci] ## [1.0.9-alpha.4](https://github.com/opendatalab/labelU/compare/v1.0.9-alpha.3...v1.0.9-alpha.4) (2024-10-29) ### Bug Fixes * Upgrade frontend to v5.5.0-alpha.1. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.1) ([7b04670](https://github.com/opendatalab/labelU/commit/7b046706c7c1ff7e9fa94fd912f4263d33181e7b)) * Upgrade frontend to v5.5.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.2) ([491cf33](https://github.com/opendatalab/labelU/commit/491cf335a0f10c1565536bc0c4cea254d0b6b23b)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a7f66cc9..f8334972 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.9-alpha.3' +version = '1.0.9-alpha.4' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From dc022fd8da66f496afa8ee5a52c46e65892e7c51 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 30 Oct 2024 08:44:00 +0000 Subject: [PATCH 032/236] fix: Upgrade frontend to v5.5.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.3) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index bda0d7c6..e9085ed9 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.2/frontend.zip -alpha_version: v5.5.0-alpha.2 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.3/frontend.zip +alpha_version: v5.5.0-alpha.3 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0/frontend.zip release_version: v5.3.0 From 694629c43184bd799247791840c560539c59f6e1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 30 Oct 2024 08:46:14 +0000 Subject: [PATCH 033/236] chore(release): 1.0.9-alpha.5 [skip ci] ## [1.0.9-alpha.5](https://github.com/opendatalab/labelU/compare/v1.0.9-alpha.4...v1.0.9-alpha.5) (2024-10-30) ### Bug Fixes * Upgrade frontend to v5.5.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.3) ([dc022fd](https://github.com/opendatalab/labelU/commit/dc022fd8da66f496afa8ee5a52c46e65892e7c51)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f8334972..46b1ba08 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.9-alpha.4' +version = '1.0.9-alpha.5' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 7622524e6f36abe2a663614c679d4d9b6c02a0f0 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 5 Nov 2024 05:02:31 +0000 Subject: [PATCH 034/236] fix: Upgrade frontend to v5.5.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.4) --- .VERSION | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.VERSION b/.VERSION index 6c269fee..8276a195 100644 --- a/.VERSION +++ b/.VERSION @@ -1,5 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.7/frontend.zip -alpha_version: v5.3.0-alpha.7 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.4/frontend.zip +alpha_version: v5.5.0-alpha.4 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 - From 516a88df35a8cf7f8c01902840cc7129c87b79a7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 5 Nov 2024 05:07:39 +0000 Subject: [PATCH 035/236] chore(release): 1.1.0-alpha.1 [skip ci] # [1.1.0-alpha.1](https://github.com/opendatalab/labelU/compare/v1.0.9-alpha.5...v1.1.0-alpha.1) (2024-11-05) ### Bug Fixes * Upgrade frontend to v5.5.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.4) ([7622524](https://github.com/opendatalab/labelU/commit/7622524e6f36abe2a663614c679d4d9b6c02a0f0)) ### Features * supports exporting "csv" format files ([d35a93a](https://github.com/opendatalab/labelU/commit/d35a93ae5053e84053d12b4dac3e24c4c7ca9644)) * supports exporting "labelme" format files ([8d4ea8e](https://github.com/opendatalab/labelU/commit/8d4ea8e6bd4dc28245de825c590bd2f963ad2984)) * supports exporting "pascal voc" format files ([4344f99](https://github.com/opendatalab/labelU/commit/4344f99799a95f278dd3e1cf008ea66d0d9d6a0f)) * supports exporting "tfrecord" format files ([f728fd6](https://github.com/opendatalab/labelU/commit/f728fd61b9f8060936ec5f68d239fcaeea4a9acd)) * supports exporting "xml" format files ([b165cce](https://github.com/opendatalab/labelU/commit/b165cce71a1f62690d33e156580f36d0ccee187f)) * supports exporting "yolo" format files ([b3d07b4](https://github.com/opendatalab/labelU/commit/b3d07b4d162fd4c7a2afc1281a96d9f2c06ec343)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 46435d4d..d98ba5e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.9' +version = '1.1.0-alpha.1' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 22afaf025f6c2f85af0e333dee34db6cd095a91e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 5 Nov 2024 11:00:26 +0000 Subject: [PATCH 036/236] chore(release): 1.1.0-alpha.2 [skip ci] # [1.1.0-alpha.2](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.1...v1.1.0-alpha.2) (2024-11-05) ### Bug Fixes * fix category name in pascal_voc and tfrecord format files ([b089074](https://github.com/opendatalab/labelU/commit/b08907442aadf95b3ded49c6dedab519ffedd839)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d98ba5e6..e30f4080 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.1' +version = '1.1.0-alpha.2' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 27f27be399f99b9032c50ab000614b06d4c608cb Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 6 Nov 2024 09:34:34 +0000 Subject: [PATCH 037/236] fix: Upgrade frontend to v5.5.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.5) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 8276a195..a00ebf1c 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.4/frontend.zip -alpha_version: v5.5.0-alpha.4 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.5/frontend.zip +alpha_version: v5.5.0-alpha.5 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 7a4a4e1031b7c3724b267dab5e04b7083f298240 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 6 Nov 2024 09:39:56 +0000 Subject: [PATCH 038/236] chore(release): 1.1.0-alpha.3 [skip ci] # [1.1.0-alpha.3](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.2...v1.1.0-alpha.3) (2024-11-06) ### Bug Fixes * supports rotated image in yolo and tfrecord files ([4d5afdf](https://github.com/opendatalab/labelU/commit/4d5afdf3200d54b7ff7f2cbbae82f7c1a574f75a)) * Upgrade frontend to v5.5.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.5) ([27f27be](https://github.com/opendatalab/labelU/commit/27f27be399f99b9032c50ab000614b06d4c608cb)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e30f4080..d4deb07e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.2' +version = '1.1.0-alpha.3' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From fb4746fd588e560f77e7699e408ede9bf9a570c9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 11 Nov 2024 07:47:41 +0000 Subject: [PATCH 039/236] chore(release): 1.1.0-alpha.4 [skip ci] # [1.1.0-alpha.4](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.3...v1.1.0-alpha.4) (2024-11-11) ### Bug Fixes * update attributes in xml and pascal_voc ([0affd60](https://github.com/opendatalab/labelU/commit/0affd60cf6688bdc5d0636c34d87cf2d99b353c4)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d4deb07e..3030a896 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.3' +version = '1.1.0-alpha.4' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 2d986b8435d45741f09fb095d9aafe4df0c1d424 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 11 Nov 2024 08:59:27 +0000 Subject: [PATCH 040/236] fix: Upgrade frontend to v5.5.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.6) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index a00ebf1c..f093569a 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.5/frontend.zip -alpha_version: v5.5.0-alpha.5 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.6/frontend.zip +alpha_version: v5.5.0-alpha.6 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 672a1cf86adb2f165e2053b648e4dbf1e9cd2083 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 11 Nov 2024 09:04:48 +0000 Subject: [PATCH 041/236] chore(release): 1.1.0-alpha.5 [skip ci] # [1.1.0-alpha.5](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.4...v1.1.0-alpha.5) (2024-11-11) ### Bug Fixes * Upgrade frontend to v5.5.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.6) ([2d986b8](https://github.com/opendatalab/labelU/commit/2d986b8435d45741f09fb095d9aafe4df0c1d424)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3030a896..8068e204 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.4' +version = '1.1.0-alpha.5' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 7dc220ad9d490cc8196c1cab4e72973f3692506f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 11 Nov 2024 10:43:33 +0000 Subject: [PATCH 042/236] chore(release): 1.1.0-alpha.6 [skip ci] # [1.1.0-alpha.6](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.5...v1.1.0-alpha.6) (2024-11-11) ### Bug Fixes * fix file format in tfrecord ([77cba31](https://github.com/opendatalab/labelU/commit/77cba31fd50f255561d4181eed67064efd148f65)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8068e204..f61725fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.5' +version = '1.1.0-alpha.6' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 658f1a4a8c406c4d05abc6b5f1f7da1a40a69513 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 14 Nov 2024 06:41:28 +0000 Subject: [PATCH 043/236] fix: Upgrade frontend to v5.5.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.7) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index f093569a..e990c8c6 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.6/frontend.zip -alpha_version: v5.5.0-alpha.6 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.7/frontend.zip +alpha_version: v5.5.0-alpha.7 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 1df8fbb1d384c91fbc13e40c9158fb431eb2e45a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 14 Nov 2024 07:10:00 +0000 Subject: [PATCH 044/236] chore(release): 1.1.0-alpha.7 [skip ci] # [1.1.0-alpha.7](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.6...v1.1.0-alpha.7) (2024-11-14) ### Bug Fixes * optimize the task_pre_annotation table and service ([d38ed7b](https://github.com/opendatalab/labelU/commit/d38ed7bf3adab1684c3109f677eb9445de50d86f)) * Upgrade frontend to v5.5.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.7) ([658f1a4](https://github.com/opendatalab/labelU/commit/658f1a4a8c406c4d05abc6b5f1f7da1a40a69513)) ### Features * supports labelu json file as the pre-annotation file ([2b4a9bd](https://github.com/opendatalab/labelU/commit/2b4a9bd333caa85e60a838293ddbaa0b356c59ef)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f61725fd..7da67b00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.6' +version = '1.1.0-alpha.7' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From f2ab8e04d4dc4fd733ee7663181fea3cf871216a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 15 Nov 2024 08:16:55 +0000 Subject: [PATCH 045/236] chore(release): 1.1.0-alpha.8 [skip ci] # [1.1.0-alpha.8](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.7...v1.1.0-alpha.8) (2024-11-15) ### Bug Fixes * fix [#69](https://github.com/opendatalab/labelU/issues/69) ([0baa13c](https://github.com/opendatalab/labelU/commit/0baa13c50aecc1144821a50b40223ba0daf52b1b)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7da67b00..9d2c6bc3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.7' +version = '1.1.0-alpha.8' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From b2ea80a79a4aee7ec3ce87c2c4abbf472297ded9 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 18 Nov 2024 06:01:23 +0000 Subject: [PATCH 046/236] fix: Upgrade frontend to v5.5.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.8) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index e990c8c6..32048b13 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.7/frontend.zip -alpha_version: v5.5.0-alpha.7 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.8/frontend.zip +alpha_version: v5.5.0-alpha.8 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 6ef75be38cf77248a0906fdb7034a6dd81a2dd9f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 18 Nov 2024 06:06:54 +0000 Subject: [PATCH 047/236] chore(release): 1.1.0-alpha.9 [skip ci] # [1.1.0-alpha.9](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.8...v1.1.0-alpha.9) (2024-11-18) ### Bug Fixes * Upgrade frontend to v5.5.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.8) ([b2ea80a](https://github.com/opendatalab/labelU/commit/b2ea80a79a4aee7ec3ce87c2c4abbf472297ded9)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9d2c6bc3..8553df0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.8' +version = '1.1.0-alpha.9' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 16d66fd40aee2d94c7ef0ab7396f5e859f9d88a3 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 18 Nov 2024 06:23:16 +0000 Subject: [PATCH 048/236] fix: Upgrade frontend to v5.5.0-alpha.9. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.9) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 32048b13..a3f1b31c 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.8/frontend.zip -alpha_version: v5.5.0-alpha.8 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.9/frontend.zip +alpha_version: v5.5.0-alpha.9 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 456ab2c42831179c65f0eab44de72848e2144334 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 18 Nov 2024 07:47:12 +0000 Subject: [PATCH 049/236] chore(release): 1.1.0-alpha.10 [skip ci] # [1.1.0-alpha.10](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.9...v1.1.0-alpha.10) (2024-11-18) ### Bug Fixes * add folder field to the json output ([3589d22](https://github.com/opendatalab/labelU/commit/3589d22d10762d0c39b86035aab3a9fca523c449)) * remove uuid prefix in attachment creation ([f0820ff](https://github.com/opendatalab/labelU/commit/f0820ff6907c4125bef17523c5df9ca05b8aa6b2)) * Upgrade frontend to v5.5.0-alpha.9. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.9) ([16d66fd](https://github.com/opendatalab/labelU/commit/16d66fd40aee2d94c7ef0ab7396f5e859f9d88a3)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8553df0b..cc01e2b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.9' +version = '1.1.0-alpha.10' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From e6919bd91f36fc347fd977e339e02c552afd683d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 19 Nov 2024 06:17:35 +0000 Subject: [PATCH 050/236] chore(release): 1.1.0-alpha.11 [skip ci] # [1.1.0-alpha.11](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.10...v1.1.0-alpha.11) (2024-11-19) ### Bug Fixes * update alembic script ([97d6c08](https://github.com/opendatalab/labelU/commit/97d6c088abff6ab9e6c14c24874b0ea4e545ae00)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cc01e2b4..18602100 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.10' +version = '1.1.0-alpha.11' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 2fd776543db8ca728832110ba0c38af16b57e097 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 19 Nov 2024 08:26:03 +0000 Subject: [PATCH 051/236] fix: Upgrade frontend to v5.5.0-alpha.10. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.10) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index a3f1b31c..756ae351 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.9/frontend.zip -alpha_version: v5.5.0-alpha.9 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.10/frontend.zip +alpha_version: v5.5.0-alpha.10 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 18079cc0ed73c294fa00f75ba4d7cb03faa02bdd Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 19 Nov 2024 08:31:18 +0000 Subject: [PATCH 052/236] chore(release): 1.1.0-alpha.12 [skip ci] # [1.1.0-alpha.12](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.11...v1.1.0-alpha.12) (2024-11-19) ### Bug Fixes * Upgrade frontend to v5.5.0-alpha.10. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.10) ([2fd7765](https://github.com/opendatalab/labelU/commit/2fd776543db8ca728832110ba0c38af16b57e097)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 18602100..0b0ca38e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.11' +version = '1.1.0-alpha.12' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 39bbff793248a255d3c1016a2033f682b113b878 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 19 Nov 2024 09:35:04 +0000 Subject: [PATCH 053/236] fix: Upgrade frontend to v5.5.0-alpha.11. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.11) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 756ae351..d3134556 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.10/frontend.zip -alpha_version: v5.5.0-alpha.10 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.11/frontend.zip +alpha_version: v5.5.0-alpha.11 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 06388e5494096b05d92a2eeb0e82152b345fe2c7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 19 Nov 2024 09:40:23 +0000 Subject: [PATCH 054/236] chore(release): 1.1.0-alpha.13 [skip ci] # [1.1.0-alpha.13](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.12...v1.1.0-alpha.13) (2024-11-19) ### Bug Fixes * Upgrade frontend to v5.5.0-alpha.11. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.11) ([39bbff7](https://github.com/opendatalab/labelU/commit/39bbff793248a255d3c1016a2033f682b113b878)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0b0ca38e..38aec48d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.12' +version = '1.1.0-alpha.13' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 95232c8c8fee9bb5c83bb01bd2271d5bd613d89e Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 20 Nov 2024 01:32:50 +0000 Subject: [PATCH 055/236] fix: Upgrade frontend to v5.5.0-alpha.12. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.12) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index d3134556..e712c66b 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.11/frontend.zip -alpha_version: v5.5.0-alpha.11 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.12/frontend.zip +alpha_version: v5.5.0-alpha.12 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 12f0e5743d4e314a4ff1dfb048739f1bc0d03096 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 20 Nov 2024 01:38:12 +0000 Subject: [PATCH 056/236] chore(release): 1.1.0-alpha.14 [skip ci] # [1.1.0-alpha.14](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.13...v1.1.0-alpha.14) (2024-11-20) ### Bug Fixes * Upgrade frontend to v5.5.0-alpha.12. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.12) ([95232c8](https://github.com/opendatalab/labelU/commit/95232c8c8fee9bb5c83bb01bd2271d5bd613d89e)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 38aec48d..15e059eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.13' +version = '1.1.0-alpha.14' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 9326d0601cebcb9c1c2dd8dd94275d0ddffa04b0 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 25 Dec 2024 11:55:08 +0000 Subject: [PATCH 057/236] fix: Upgrade frontend to v5.6.0-alpha.1. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.1) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index e712c66b..04563dd3 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.5.0-alpha.12/frontend.zip -alpha_version: v5.5.0-alpha.12 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.1/frontend.zip +alpha_version: v5.6.0-alpha.1 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From a2fba633cf7f6229c467a104fa8156a7d28597f9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 25 Dec 2024 12:01:46 +0000 Subject: [PATCH 058/236] chore(release): 1.1.0-alpha.15 [skip ci] # [1.1.0-alpha.15](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.14...v1.1.0-alpha.15) (2024-12-25) ### Bug Fixes * Upgrade frontend to v5.6.0-alpha.1. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.1) ([9326d06](https://github.com/opendatalab/labelU/commit/9326d0601cebcb9c1c2dd8dd94275d0ddffa04b0)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 15e059eb..00d73b30 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.14' +version = '1.1.0-alpha.15' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 39b718217da55c2de92fc71401ee7c205799bec2 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 26 Dec 2024 03:48:38 +0000 Subject: [PATCH 059/236] fix: Upgrade frontend to v5.6.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.2) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 04563dd3..e923b981 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.1/frontend.zip -alpha_version: v5.6.0-alpha.1 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.2/frontend.zip +alpha_version: v5.6.0-alpha.2 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From d305df53faa16015b872514132b2eafd91f7a7e0 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 26 Dec 2024 03:54:45 +0000 Subject: [PATCH 060/236] chore(release): 1.1.0-alpha.16 [skip ci] # [1.1.0-alpha.16](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.15...v1.1.0-alpha.16) (2024-12-26) ### Bug Fixes * Upgrade frontend to v5.6.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.2) ([39b7182](https://github.com/opendatalab/labelU/commit/39b718217da55c2de92fc71401ee7c205799bec2)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 00d73b30..b0514f00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.15' +version = '1.1.0-alpha.16' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 8ef74dd851c6c8271f24d97d111df67c91f064f9 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 26 Dec 2024 04:11:27 +0000 Subject: [PATCH 061/236] fix: Upgrade frontend to v5.6.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.3) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index e923b981..819d2183 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.2/frontend.zip -alpha_version: v5.6.0-alpha.2 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.3/frontend.zip +alpha_version: v5.6.0-alpha.3 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 3f8b71ca667849732c7abe256b3d4eeee6c59624 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 26 Dec 2024 04:16:37 +0000 Subject: [PATCH 062/236] chore(release): 1.1.0-alpha.17 [skip ci] # [1.1.0-alpha.17](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.16...v1.1.0-alpha.17) (2024-12-26) ### Bug Fixes * Upgrade frontend to v5.6.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.3) ([8ef74dd](https://github.com/opendatalab/labelU/commit/8ef74dd851c6c8271f24d97d111df67c91f064f9)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b0514f00..3526398f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.16' +version = '1.1.0-alpha.17' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 77e2913a0905e077e95549e1e54320206816fc1f Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 26 Dec 2024 04:30:41 +0000 Subject: [PATCH 063/236] fix: Upgrade frontend to v5.6.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.4) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 819d2183..57ae537a 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.3/frontend.zip -alpha_version: v5.6.0-alpha.3 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.4/frontend.zip +alpha_version: v5.6.0-alpha.4 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 1c5ede3e1d51784c9e16292d028be3479c8e4eda Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 26 Dec 2024 04:36:01 +0000 Subject: [PATCH 064/236] chore(release): 1.1.0-alpha.18 [skip ci] # [1.1.0-alpha.18](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.17...v1.1.0-alpha.18) (2024-12-26) ### Bug Fixes * Upgrade frontend to v5.6.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.4) ([77e2913](https://github.com/opendatalab/labelU/commit/77e2913a0905e077e95549e1e54320206816fc1f)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3526398f..12cdc388 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.17' +version = '1.1.0-alpha.18' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 60707c548a3b4d8e5ac099e3eaeb39da859796ec Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 26 Dec 2024 07:14:47 +0000 Subject: [PATCH 065/236] fix: Upgrade frontend to v5.6.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.5) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 57ae537a..24b08b30 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.4/frontend.zip -alpha_version: v5.6.0-alpha.4 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.5/frontend.zip +alpha_version: v5.6.0-alpha.5 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 3331a2db8f0747366501a1717441dc22758ee1cc Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 26 Dec 2024 07:19:44 +0000 Subject: [PATCH 066/236] chore(release): 1.1.0-alpha.19 [skip ci] # [1.1.0-alpha.19](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.18...v1.1.0-alpha.19) (2024-12-26) ### Bug Fixes * Upgrade frontend to v5.6.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.5) ([60707c5](https://github.com/opendatalab/labelU/commit/60707c548a3b4d8e5ac099e3eaeb39da859796ec)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 12cdc388..65d89e1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.18' +version = '1.1.0-alpha.19' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 632ec09bbed951a2877f7f51da627065ed64b75c Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 27 Dec 2024 06:36:07 +0000 Subject: [PATCH 067/236] fix: Upgrade frontend to v5.6.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.6) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 24b08b30..85f185af 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.5/frontend.zip -alpha_version: v5.6.0-alpha.5 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.6/frontend.zip +alpha_version: v5.6.0-alpha.6 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From eb2404ff5347f94d5429ca3a6fb44f9ea8d30ac8 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 27 Dec 2024 06:41:06 +0000 Subject: [PATCH 068/236] chore(release): 1.1.0-alpha.20 [skip ci] # [1.1.0-alpha.20](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.19...v1.1.0-alpha.20) (2024-12-27) ### Bug Fixes * Upgrade frontend to v5.6.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.6) ([632ec09](https://github.com/opendatalab/labelU/commit/632ec09bbed951a2877f7f51da627065ed64b75c)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 65d89e1e..3f37ca81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.19' +version = '1.1.0-alpha.20' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From b3c4f00c7ae3ead62e7c8de89ca27d38efaa7b0c Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 27 Dec 2024 09:46:41 +0000 Subject: [PATCH 069/236] fix: Upgrade frontend to v5.6.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.7) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 85f185af..4b8aeabb 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.6/frontend.zip -alpha_version: v5.6.0-alpha.6 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.7/frontend.zip +alpha_version: v5.6.0-alpha.7 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From a5e0bd02a3d81f96b82f75438f5baecda455cb3d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 27 Dec 2024 09:51:42 +0000 Subject: [PATCH 070/236] chore(release): 1.1.0-alpha.21 [skip ci] # [1.1.0-alpha.21](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.20...v1.1.0-alpha.21) (2024-12-27) ### Bug Fixes * Upgrade frontend to v5.6.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.7) ([b3c4f00](https://github.com/opendatalab/labelU/commit/b3c4f00c7ae3ead62e7c8de89ca27d38efaa7b0c)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3f37ca81..e9986c39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.20' +version = '1.1.0-alpha.21' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 2c544f6e000ddea9a91896c5a69c514c417261c9 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 6 Jan 2025 10:00:05 +0000 Subject: [PATCH 071/236] fix: Upgrade frontend to v5.6.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.8) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 4b8aeabb..9d4d64da 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.7/frontend.zip -alpha_version: v5.6.0-alpha.7 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.0-alpha.8/frontend.zip +alpha_version: v5.6.0-alpha.8 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.4.1/frontend.zip release_version: v5.4.1 From 58dbb5abf638c56257d418206442c83d35cca2b2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 6 Jan 2025 10:05:31 +0000 Subject: [PATCH 072/236] chore(release): 1.1.0-alpha.22 [skip ci] # [1.1.0-alpha.22](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.21...v1.1.0-alpha.22) (2025-01-06) ### Bug Fixes * Upgrade frontend to v5.6.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.8) ([2c544f6](https://github.com/opendatalab/labelU/commit/2c544f6e000ddea9a91896c5a69c514c417261c9)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e9986c39..0ffe1746 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.21' +version = '1.1.0-alpha.22' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 752a473149fc65ccbf84b109edbd64c6b0d7ced3 Mon Sep 17 00:00:00 2001 From: gary Date: Tue, 29 Oct 2024 11:46:43 +0800 Subject: [PATCH 073/236] chore: update scripts (#130) * chore: update workflow (#128) * fix: fix pre annoation exceptions (#120) * chore(release): 1.0.7 [skip ci] * fix pre annoation exceptions ([#120](https://github.com/opendatalab/labelU/issues/120)) ([bd61682](https://github.com/opendatalab/labelU/commit/bd616821eb6dea47840df29efe340c3b96c2446f)) * docs: Update README.md * docs: Update README.md * docs: Update README_zh-CN.md (#126) * docs: Update README_zh-CN.md * docs: Update README_zh-CN.md * chore: update workflow --------- Co-authored-by: semantic-release-bot * fix: Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) * chore(release): 1.0.7-alpha.2 [skip ci] * Upgrade frontend to v5.3.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.3) ([e8721d6](https://github.com/opendatalab/labelU/commit/e8721d6ab4fdb288ef16263ffb09c73ea12ed52f)) * Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) ([01fc3b5](https://github.com/opendatalab/labelU/commit/01fc3b53b570de4d05ad771f163d8da03236b141)) * fix: Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) * chore(release): 1.0.7-alpha.3 [skip ci] * Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) ([f97601d](https://github.com/opendatalab/labelU/commit/f97601d0c4286c633ca2ce3a1f5ef3d8eec5a540)) * fix: update notification action * chore(release): 1.0.7-alpha.4 [skip ci] * update notification action ([b6abe17](https://github.com/opendatalab/labelU/commit/b6abe172f7a8bb1a5547acb2bc8948bb7c0dae7d)) * fix: update notification action * chore(release): 1.0.7-alpha.5 [skip ci] * update notification action ([04e0388](https://github.com/opendatalab/labelU/commit/04e0388e3cd98a2aacf12dc28c638c172eb6f0ff)) * fix: update notification action * chore(release): 1.0.7-alpha.6 [skip ci] * update notification action ([e9152c9](https://github.com/opendatalab/labelU/commit/e9152c98869d70e500fd9ab458d8ba179270d9c9)) * fix: update notification action * chore(release): 1.0.7-alpha.7 [skip ci] * update notification action ([046f47a](https://github.com/opendatalab/labelU/commit/046f47a3f4053e6934f04c3c0ede5518b9f1850b)) * fix: test notification action * chore(release): 1.0.7-alpha.8 [skip ci] * test notification action ([8fed078](https://github.com/opendatalab/labelU/commit/8fed078a0d77a8204430fb6c24d91de1250a8bc7)) * fix: test notification action * chore(release): 1.0.7-alpha.9 [skip ci] * test notification action ([851818e](https://github.com/opendatalab/labelU/commit/851818e912cf39690be6e689c1926ec29c148a66)) * fix: test notification action * chore(release): 1.0.7-alpha.10 [skip ci] * test notification action ([bc63ee5](https://github.com/opendatalab/labelU/commit/bc63ee59da0d079208b1385d33184267251a2bb5)) * fix: test notification action * chore(release): 1.0.7-alpha.11 [skip ci] * test notification action ([83f43eb](https://github.com/opendatalab/labelU/commit/83f43eb9d03390686a41dc22666a058190bfb63c)) * fix: update notification action * chore(release): 1.0.7-alpha.12 [skip ci] * update notification action ([63286cb](https://github.com/opendatalab/labelU/commit/63286cb6958e20ab3c80495ab3b51a1ddd31dfb7)) * chore(release): 1.0.9-alpha.1 [skip ci] * test notification action ([83f43eb](https://github.com/opendatalab/labelU/commit/83f43eb9d03390686a41dc22666a058190bfb63c)) * test notification action ([bc63ee5](https://github.com/opendatalab/labelU/commit/bc63ee59da0d079208b1385d33184267251a2bb5)) * test notification action ([851818e](https://github.com/opendatalab/labelU/commit/851818e912cf39690be6e689c1926ec29c148a66)) * test notification action ([8fed078](https://github.com/opendatalab/labelU/commit/8fed078a0d77a8204430fb6c24d91de1250a8bc7)) * update notification action ([63286cb](https://github.com/opendatalab/labelU/commit/63286cb6958e20ab3c80495ab3b51a1ddd31dfb7)) * update notification action ([046f47a](https://github.com/opendatalab/labelU/commit/046f47a3f4053e6934f04c3c0ede5518b9f1850b)) * update notification action ([e9152c9](https://github.com/opendatalab/labelU/commit/e9152c98869d70e500fd9ab458d8ba179270d9c9)) * update notification action ([04e0388](https://github.com/opendatalab/labelU/commit/04e0388e3cd98a2aacf12dc28c638c172eb6f0ff)) * update notification action ([b6abe17](https://github.com/opendatalab/labelU/commit/b6abe172f7a8bb1a5547acb2bc8948bb7c0dae7d)) * Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) ([01fc3b5](https://github.com/opendatalab/labelU/commit/01fc3b53b570de4d05ad771f163d8da03236b141)) * Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) ([f97601d](https://github.com/opendatalab/labelU/commit/f97601d0c4286c633ca2ce3a1f5ef3d8eec5a540)) * fix: Upgrade frontend to v5.3.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.6) * chore(release): 1.0.9-alpha.2 [skip ci] * Upgrade frontend to v5.3.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.6) ([9f5e698](https://github.com/opendatalab/labelU/commit/9f5e698c8a0476be0bb034bb12e226b9db211328)) * fix: Upgrade frontend to v5.3.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.7) * chore(release): 1.0.9-alpha.3 [skip ci] * Upgrade frontend to v5.3.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.7) ([7cb455e](https://github.com/opendatalab/labelU/commit/7cb455eaf06b8408f2cacfe3cc39c8c2e3bde013)) --------- Co-authored-by: semantic-release-bot fix: upgrade frontend to 5.4.0 #106 chore: delete test script chore(release): 1.0.9 [skip ci] * upgrade frontend to 5.4.0 [#106](https://github.com/opendatalab/labelU/issues/106) ([33b71be](https://github.com/opendatalab/labelU/commit/33b71beb99a0e56b96ab56bb2cffe18c95624c37)) fix: upgrade frontend to 5.4.1 feat: supports exporting "labelme" format files feat: supports exporting "yolo" format files feat: supports exporting "csv" format files feat: supports exporting "xml" format files feat: supports exporting "tfrecord" format files update: supports exporting "tfrecord" format files feat: supports exporting "pascal voc" format files fix: fix category name in pascal_voc and tfrecord format files fix: supports rotated image in yolo and tfrecord files fix: update attributes in xml and pascal_voc fix: fix file format in tfrecord fix: optimize the task_pre_annotation table and service update: optimize the task_pre_annotation table and service feat: supports labelu json file as the pre-annotation file update: fix duplicated sample name in pre-annotation file fix: fix #69 fix: add folder field to the json output fix: remove uuid prefix in attachment creation chore: update test case fix: update alembic script chore: update scripts (#130) * chore: update workflow (#128) * fix: fix pre annoation exceptions (#120) * chore(release): 1.0.7 [skip ci] ## [1.0.7](https://github.com/opendatalab/labelU/compare/v1.0.6...v1.0.7) (2024-08-26) ### Bug Fixes * fix pre annoation exceptions ([#120](https://github.com/opendatalab/labelU/issues/120)) ([bd61682](https://github.com/opendatalab/labelU/commit/bd616821eb6dea47840df29efe340c3b96c2446f)) * docs: Update README.md * docs: Update README.md * docs: Update README_zh-CN.md (#126) * docs: Update README_zh-CN.md * docs: Update README_zh-CN.md * chore: update workflow --------- Co-authored-by: semantic-release-bot * fix: Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) * chore(release): 1.0.7-alpha.2 [skip ci] ## [1.0.7-alpha.2](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.1...v1.0.7-alpha.2) (2024-10-25) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.3) ([e8721d6](https://github.com/opendatalab/labelU/commit/e8721d6ab4fdb288ef16263ffb09c73ea12ed52f)) * Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) ([01fc3b5](https://github.com/opendatalab/labelU/commit/01fc3b53b570de4d05ad771f163d8da03236b141)) * fix: Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) * chore(release): 1.0.7-alpha.3 [skip ci] ## [1.0.7-alpha.3](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.2...v1.0.7-alpha.3) (2024-10-25) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) ([f97601d](https://github.com/opendatalab/labelU/commit/f97601d0c4286c633ca2ce3a1f5ef3d8eec5a540)) * fix: update notification action * chore(release): 1.0.7-alpha.4 [skip ci] ## [1.0.7-alpha.4](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.3...v1.0.7-alpha.4) (2024-10-28) ### Bug Fixes * update notification action ([b6abe17](https://github.com/opendatalab/labelU/commit/b6abe172f7a8bb1a5547acb2bc8948bb7c0dae7d)) * fix: update notification action * chore(release): 1.0.7-alpha.5 [skip ci] ## [1.0.7-alpha.5](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.4...v1.0.7-alpha.5) (2024-10-28) ### Bug Fixes * update notification action ([04e0388](https://github.com/opendatalab/labelU/commit/04e0388e3cd98a2aacf12dc28c638c172eb6f0ff)) * fix: update notification action * chore(release): 1.0.7-alpha.6 [skip ci] ## [1.0.7-alpha.6](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.5...v1.0.7-alpha.6) (2024-10-28) ### Bug Fixes * update notification action ([e9152c9](https://github.com/opendatalab/labelU/commit/e9152c98869d70e500fd9ab458d8ba179270d9c9)) * fix: update notification action * chore(release): 1.0.7-alpha.7 [skip ci] ## [1.0.7-alpha.7](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.6...v1.0.7-alpha.7) (2024-10-28) ### Bug Fixes * update notification action ([046f47a](https://github.com/opendatalab/labelU/commit/046f47a3f4053e6934f04c3c0ede5518b9f1850b)) * fix: test notification action * chore(release): 1.0.7-alpha.8 [skip ci] ## [1.0.7-alpha.8](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.7...v1.0.7-alpha.8) (2024-10-28) ### Bug Fixes * test notification action ([8fed078](https://github.com/opendatalab/labelU/commit/8fed078a0d77a8204430fb6c24d91de1250a8bc7)) * fix: test notification action * chore(release): 1.0.7-alpha.9 [skip ci] ## [1.0.7-alpha.9](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.8...v1.0.7-alpha.9) (2024-10-28) ### Bug Fixes * test notification action ([851818e](https://github.com/opendatalab/labelU/commit/851818e912cf39690be6e689c1926ec29c148a66)) * fix: test notification action * chore(release): 1.0.7-alpha.10 [skip ci] ## [1.0.7-alpha.10](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.9...v1.0.7-alpha.10) (2024-10-28) ### Bug Fixes * test notification action ([bc63ee5](https://github.com/opendatalab/labelU/commit/bc63ee59da0d079208b1385d33184267251a2bb5)) * fix: test notification action * chore(release): 1.0.7-alpha.11 [skip ci] ## [1.0.7-alpha.11](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.10...v1.0.7-alpha.11) (2024-10-28) ### Bug Fixes * test notification action ([83f43eb](https://github.com/opendatalab/labelU/commit/83f43eb9d03390686a41dc22666a058190bfb63c)) * fix: update notification action * chore(release): 1.0.7-alpha.12 [skip ci] ## [1.0.7-alpha.12](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.11...v1.0.7-alpha.12) (2024-10-28) ### Bug Fixes * update notification action ([63286cb](https://github.com/opendatalab/labelU/commit/63286cb6958e20ab3c80495ab3b51a1ddd31dfb7)) * chore(release): 1.0.9-alpha.1 [skip ci] ## [1.0.9-alpha.1](https://github.com/opendatalab/labelU/compare/v1.0.8...v1.0.9-alpha.1) (2024-10-28) ### Bug Fixes * test notification action ([83f43eb](https://github.com/opendatalab/labelU/commit/83f43eb9d03390686a41dc22666a058190bfb63c)) * test notification action ([bc63ee5](https://github.com/opendatalab/labelU/commit/bc63ee59da0d079208b1385d33184267251a2bb5)) * test notification action ([851818e](https://github.com/opendatalab/labelU/commit/851818e912cf39690be6e689c1926ec29c148a66)) * test notification action ([8fed078](https://github.com/opendatalab/labelU/commit/8fed078a0d77a8204430fb6c24d91de1250a8bc7)) * update notification action ([63286cb](https://github.com/opendatalab/labelU/commit/63286cb6958e20ab3c80495ab3b51a1ddd31dfb7)) * update notification action ([046f47a](https://github.com/opendatalab/labelU/commit/046f47a3f4053e6934f04c3c0ede5518b9f1850b)) * update notification action ([e9152c9](https://github.com/opendatalab/labelU/commit/e9152c98869d70e500fd9ab458d8ba179270d9c9)) * update notification action ([04e0388](https://github.com/opendatalab/labelU/commit/04e0388e3cd98a2aacf12dc28c638c172eb6f0ff)) * update notification action ([b6abe17](https://github.com/opendatalab/labelU/commit/b6abe172f7a8bb1a5547acb2bc8948bb7c0dae7d)) * Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) ([01fc3b5](https://github.com/opendatalab/labelU/commit/01fc3b53b570de4d05ad771f163d8da03236b141)) * Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) ([f97601d](https://github.com/opendatalab/labelU/commit/f97601d0c4286c633ca2ce3a1f5ef3d8eec5a540)) * fix: Upgrade frontend to v5.3.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.6) * chore(release): 1.0.9-alpha.2 [skip ci] ## [1.0.9-alpha.2](https://github.com/opendatalab/labelU/compare/v1.0.9-alpha.1...v1.0.9-alpha.2) (2024-10-28) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.6) ([9f5e698](https://github.com/opendatalab/labelU/commit/9f5e698c8a0476be0bb034bb12e226b9db211328)) * fix: Upgrade frontend to v5.3.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.7) * chore(release): 1.0.9-alpha.3 [skip ci] ## [1.0.9-alpha.3](https://github.com/opendatalab/labelU/compare/v1.0.9-alpha.2...v1.0.9-alpha.3) (2024-10-28) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.7) ([7cb455e](https://github.com/opendatalab/labelU/commit/7cb455eaf06b8408f2cacfe3cc39c8c2e3bde013)) --------- Co-authored-by: semantic-release-bot fix: upgrade frontend to 5.4.0 #106 chore: delete test script chore(release): 1.0.9 [skip ci] ## [1.0.9](https://github.com/opendatalab/labelU/compare/v1.0.8...v1.0.9) (2024-10-29) ### Bug Fixes * upgrade frontend to 5.4.0 [#106](https://github.com/opendatalab/labelU/issues/106) ([33b71be](https://github.com/opendatalab/labelU/commit/33b71beb99a0e56b96ab56bb2cffe18c95624c37)) fix: upgrade frontend to 5.4.1 chore(release): 1.0.10 [skip ci] ## [1.0.10](https://github.com/opendatalab/labelU/compare/v1.0.9...v1.0.10) (2024-10-29) ### Bug Fixes * upgrade frontend to 5.4.1 ([73323b9](https://github.com/opendatalab/labelU/commit/73323b9b51dc1b96c85d1bf9fe7f582e4d758c51)) docs: Create LICENSE fix: Update .VERSION chore(release): 1.1.0 [skip ci] # [1.1.0](https://github.com/opendatalab/labelU/compare/v1.0.10...v1.1.0) (2024-11-20) ### Bug Fixes * add folder field to the json output ([3589d22](https://github.com/opendatalab/labelU/commit/3589d22d10762d0c39b86035aab3a9fca523c449)) * fix [#69](https://github.com/opendatalab/labelU/issues/69) ([0baa13c](https://github.com/opendatalab/labelU/commit/0baa13c50aecc1144821a50b40223ba0daf52b1b)) * fix category name in pascal_voc and tfrecord format files ([b089074](https://github.com/opendatalab/labelU/commit/b08907442aadf95b3ded49c6dedab519ffedd839)) * fix file format in tfrecord ([77cba31](https://github.com/opendatalab/labelU/commit/77cba31fd50f255561d4181eed67064efd148f65)) * optimize the task_pre_annotation table and service ([d38ed7b](https://github.com/opendatalab/labelU/commit/d38ed7bf3adab1684c3109f677eb9445de50d86f)) * remove uuid prefix in attachment creation ([f0820ff](https://github.com/opendatalab/labelU/commit/f0820ff6907c4125bef17523c5df9ca05b8aa6b2)) * supports rotated image in yolo and tfrecord files ([4d5afdf](https://github.com/opendatalab/labelU/commit/4d5afdf3200d54b7ff7f2cbbae82f7c1a574f75a)) * Update .VERSION ([dae830a](https://github.com/opendatalab/labelU/commit/dae830a9bf2b968c98adc840fc3aecb4c3371e57)) * update alembic script ([97d6c08](https://github.com/opendatalab/labelU/commit/97d6c088abff6ab9e6c14c24874b0ea4e545ae00)) * update attributes in xml and pascal_voc ([0affd60](https://github.com/opendatalab/labelU/commit/0affd60cf6688bdc5d0636c34d87cf2d99b353c4)) ### Features * supports exporting "csv" format files ([d35a93a](https://github.com/opendatalab/labelU/commit/d35a93ae5053e84053d12b4dac3e24c4c7ca9644)) * supports exporting "labelme" format files ([8d4ea8e](https://github.com/opendatalab/labelU/commit/8d4ea8e6bd4dc28245de825c590bd2f963ad2984)) * supports exporting "pascal voc" format files ([4344f99](https://github.com/opendatalab/labelU/commit/4344f99799a95f278dd3e1cf008ea66d0d9d6a0f)) * supports exporting "tfrecord" format files ([f728fd6](https://github.com/opendatalab/labelU/commit/f728fd61b9f8060936ec5f68d239fcaeea4a9acd)) * supports exporting "xml" format files ([b165cce](https://github.com/opendatalab/labelU/commit/b165cce71a1f62690d33e156580f36d0ccee187f)) * supports exporting "yolo" format files ([b3d07b4](https://github.com/opendatalab/labelU/commit/b3d07b4d162fd4c7a2afc1281a96d9f2c06ec343)) * supports labelu json file as the pre-annotation file ([2b4a9bd](https://github.com/opendatalab/labelU/commit/2b4a9bd333caa85e60a838293ddbaa0b356c59ef)) fix: wrong indent in yolo conventer Fixed an issue where multiple redundant files with the same name were generated when exporting YOLO format annotation files due to accidental indentation. chore(release): 1.1.1 [skip ci] ## [1.1.1](https://github.com/opendatalab/labelU/compare/v1.1.0...v1.1.1) (2024-11-25) ### Bug Fixes * wrong indent in yolo conventer ([5ebf616](https://github.com/opendatalab/labelU/commit/5ebf6165e53bf7d5b7776491e5fc9d291048c2e5)) fix: add full i18n supports chore(release): 1.1.2 [skip ci] ## [1.1.2](https://github.com/opendatalab/labelU/compare/v1.1.1...v1.1.2) (2025-01-06) ### Bug Fixes * add full i18n supports ([fdf9b76](https://github.com/opendatalab/labelU/commit/fdf9b769a3dd84a7656addda64d29af6bca319a4)) Update .VERSION fix: langOptions error chore(release): 1.1.3 [skip ci] ## [1.1.3](https://github.com/opendatalab/labelU/compare/v1.1.2...v1.1.3) (2025-01-06) ### Bug Fixes * langOptions error ([b2c9af5](https://github.com/opendatalab/labelU/commit/b2c9af55e467f233780abed3e219226faf472439)) fix: no cache for html file --- .VERSION | 8 +- .gitmodules | 5 - LICENSE | 203 +++++++ ...b_add_sample_name_and_data_to_task_pre_.py | 170 ++++++ .../adapter/persistence/crud_attachment.py | 52 +- .../persistence/crud_pre_annotation.py | 59 +- .../adapter/routers/pre_annotation.py | 62 +- labelu/internal/application/command/sample.py | 6 + .../application/response/pre_annotation.py | 15 + .../application/service/attachment.py | 94 ++-- .../application/service/pre_annotation.py | 220 ++++---- labelu/internal/common/converter.py | 530 +++++++++++++++++- labelu/internal/common/error_code.py | 4 + labelu/internal/common/tf_record_converter.py | 225 ++++++++ labelu/internal/common/xml_converter.py | 290 ++++++++++ .../internal/domain/models/pre_annotation.py | 8 +- labelu/main.py | 10 +- .../adapter/routers/test_attachment.py | 40 +- .../adapter/routers/test_pre_annotation.py | 14 +- labelu/tests/utils/utils.py | 24 +- pyproject.toml | 3 +- 21 files changed, 1853 insertions(+), 189 deletions(-) delete mode 100644 .gitmodules create mode 100644 LICENSE create mode 100644 labelu/alembic_labelu/versions/eb9c5b98168b_add_sample_name_and_data_to_task_pre_.py create mode 100644 labelu/internal/common/tf_record_converter.py create mode 100644 labelu/internal/common/xml_converter.py diff --git a/.VERSION b/.VERSION index c72c0efc..6e913bea 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.5/frontend.zip -alpha_version: v5.3.0-alpha.5 -release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0/frontend.zip -release_version: v5.3.0 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.7/frontend.zip +alpha_version: v5.3.0-alpha.7 +release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.1/frontend.zip +release_version: v5.6.1 diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index ac23d03a..00000000 --- a/.gitmodules +++ /dev/null @@ -1,5 +0,0 @@ -[submodule "frontend"] - path = frontend - url = https://github.com/opendatalab/labelU-frontend.git - branch = main - ignore = dirty \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..1bfc23e4 --- /dev/null +++ b/LICENSE @@ -0,0 +1,203 @@ +Copyright 2018-2023 OpenMMLab. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018-2023 OpenMMLab. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/labelu/alembic_labelu/versions/eb9c5b98168b_add_sample_name_and_data_to_task_pre_.py b/labelu/alembic_labelu/versions/eb9c5b98168b_add_sample_name_and_data_to_task_pre_.py new file mode 100644 index 00000000..3976060a --- /dev/null +++ b/labelu/alembic_labelu/versions/eb9c5b98168b_add_sample_name_and_data_to_task_pre_.py @@ -0,0 +1,170 @@ +"""add_sample_name_and_data_to_task_pre_annotation + +1. Add sample_name column to task_pre_annotation table. +2. Read jsonl file content and store it in data column of task_pre_annotation table. + +Revision ID: eb9c5b98168b +Revises: bc8fcb35b66b +Create Date: 2024-11-13 14:08:09.374271 + +""" +import json +from typing import List +from alembic import op, context +import sqlalchemy as sa +from sqlalchemy.orm import sessionmaker, Session +from sqlalchemy.ext.automap import automap_base +from sqlalchemy import select +from labelu.internal.common.config import settings + +from labelu.internal.domain.models.pre_annotation import TaskPreAnnotation +from labelu.internal.adapter.persistence import crud_attachment + +Base = automap_base() + + +# revision identifiers, used by Alembic. +revision = 'eb9c5b98168b' +down_revision = 'bc8fcb35b66b' +branch_labels = None +depends_on = None + +def index_exists(bind, table_name, index_name): + inspector = sa.inspect(bind) + indexes = inspector.get_indexes(table_name) + for index in indexes: + if index['name'] == index_name: + return True + return False + +def read_jsonl_file(db: Session, file_id: int) -> List[dict]: + attachment = crud_attachment.get(db, file_id) + if attachment is None: + return [] + + attachment_path = attachment.path + file_full_path = settings.MEDIA_ROOT.joinpath(attachment_path.lstrip("/")) + + # check if the file exists + if not file_full_path.exists() or not attachment.filename.endswith('.jsonl'): + return [] + + + with open(file_full_path, "r", encoding="utf-8") as f: + data = f.readlines() + + parsed_data = [json.loads(line) for line in data] + return parsed_data + +def upgrade() -> None: + bind = op.get_bind() + Base.prepare(autoload_with=bind, reflect=True) + # make a session + SessionMade = sessionmaker(bind=bind) + session = SessionMade() + + try: + with context.begin_transaction(): + # add column sample_name, check if the column exists + if not hasattr(Base.classes.task_pre_annotation, "sample_name"): + op.add_column( + "task_pre_annotation", + sa.Column("sample_name", sa.String(255), index=True, comment="One of the sample names of the task"), + ) + + # create index, check if the index exists + if not index_exists(bind, "task_pre_annotation", "idx_pre_annotation_sample_name"): + op.create_index("idx_pre_annotation_sample_name", "task_pre_annotation", ["sample_name"]) + + # create task_pre_annotation + exist_task_pre_annotations = session.execute( + select([Base.classes.task_pre_annotation]) + ).scalars().all() + + for task_pre_annotation in exist_task_pre_annotations: + file_id = task_pre_annotation.file_id + jsonl_contents = read_jsonl_file(session, file_id) + + # create new task_pre_annotation + for jsonl_content in jsonl_contents: + sample_name = jsonl_content.get("sample_name") + query = select(Base.classes.task_attachment).where( + Base.classes.task_attachment.task_id == task_pre_annotation.task_id, + # filename include sample_name, full name is xxxxxxxxx-sample_name.png, shot name is sample_name.png + Base.classes.task_attachment.filename.contains(sample_name), + ) + sample_file = session.execute(query).scalars().first() + new_task_pre_annotation = TaskPreAnnotation( + task_id=task_pre_annotation.task_id, + # full file name + sample_name=sample_file.filename if sample_file else None, + file_id=file_id, + created_by=task_pre_annotation.created_by, + updated_by=task_pre_annotation.updated_by, + data=json.dumps(jsonl_content), + ) + session.add(new_task_pre_annotation) + + # remove existing task_pre_annotation + for task_pre_annotation in exist_task_pre_annotations: + session.delete(task_pre_annotation) + + # commit + session.commit() + + except Exception as e: + session.rollback() + raise e + + finally: + session.close() + + +def downgrade() -> None: + bind = op.get_bind() + Base.prepare(autoload_with=bind, reflect=True) + SessionMade = sessionmaker(bind=bind) + session = SessionMade() + + try: + with context.begin_transaction(): + # remove new task_pre_annotation + new_task_pre_annotations = session.query(Base.classes.task_pre_annotation).filter( + Base.classes.task_pre_annotation.sample_name.isnot(None) + ).all() + + for task_pre_annotation in new_task_pre_annotations: + session.delete(task_pre_annotation) + + # restore old task_pre_annotation + old_task_pre_annotations = session.query(TaskPreAnnotation).filter( + TaskPreAnnotation.sample_name.isnot(None) + ).all() + + for task_pre_annotation in old_task_pre_annotations: + restored_task_pre_annotation = Base.classes.task_pre_annotation( + task_id=task_pre_annotation.task_id, + file_id=task_pre_annotation.file_id, + created_by=task_pre_annotation.created_by, + updated_by=task_pre_annotation.updated_by, + data=task_pre_annotation.data, + ) + session.add(restored_task_pre_annotation) + + # drop index + if index_exists(bind, "task_pre_annotation", "idx_pre_annotation_sample_name"): + op.drop_index("idx_pre_annotation_sample_name", table_name="task_pre_annotation") + + # drop column + if hasattr(Base.classes.task_pre_annotation, "sample_name"): + op.drop_column("task_pre_annotation", "sample_name") + + # commit + session.commit() + + except Exception as e: + session.rollback() + raise e + + finally: + session.close() \ No newline at end of file diff --git a/labelu/internal/adapter/persistence/crud_attachment.py b/labelu/internal/adapter/persistence/crud_attachment.py index 84cb20ec..efa6a466 100644 --- a/labelu/internal/adapter/persistence/crud_attachment.py +++ b/labelu/internal/adapter/persistence/crud_attachment.py @@ -1,10 +1,60 @@ from datetime import datetime -from typing import List +from typing import List, Optional, Tuple from sqlalchemy.orm import Session from labelu.internal.domain.models.attachment import TaskAttachment +def list_by( + db: Session, + pageSize: int, + ids: List[int] | None = [], + task_id: Optional[int] = None, + owner_id: Optional[int] = None, + after: Optional[int] = None, + before: Optional[int] = None, + pageNo: Optional[int] = None, + sorting: Optional[str] = None, +) -> Tuple[List[TaskAttachment], int]: +# query filter + query_filter = [TaskAttachment.deleted_at == None, TaskAttachment.id.in_(ids)] + if owner_id: + query_filter.append(TaskAttachment.created_by == owner_id) + + if before: + query_filter.append(TaskAttachment.id < before) + if after: + query_filter.append(TaskAttachment.id > after) + if task_id: + query_filter.append(TaskAttachment.task_id == task_id) + + query = db.query(TaskAttachment).filter(*query_filter) + + # default order by id, before need select last items + if before: + query = query.order_by(TaskAttachment.id.desc()) + else: + query = query.order_by(TaskAttachment.id.asc()) + + count = query.count() + + results = ( + query.offset(offset=pageNo * pageSize if pageNo else 0) + .limit(limit=pageSize) + .all() + ) + + if sorting: + field, order = sorting.split(":") + if order == "desc": + results = sorted(results, key=lambda x: getattr(x, field), reverse=True) + else: + results = sorted(results, key=lambda x: getattr(x, field)) + + if before: + results.reverse() + + return results, count def create(db: Session, attachment: TaskAttachment) -> TaskAttachment: db.add(attachment) diff --git a/labelu/internal/adapter/persistence/crud_pre_annotation.py b/labelu/internal/adapter/persistence/crud_pre_annotation.py index 71017c6f..7b3ddcb6 100644 --- a/labelu/internal/adapter/persistence/crud_pre_annotation.py +++ b/labelu/internal/adapter/persistence/crud_pre_annotation.py @@ -1,14 +1,11 @@ from datetime import datetime -import json -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Union, Tuple - -from sqlalchemy import case, text from sqlalchemy.orm import Session +from sqlalchemy import or_ from fastapi.encoders import jsonable_encoder from labelu.internal.domain.models.pre_annotation import TaskPreAnnotation -from labelu.internal.adapter.persistence import crud_attachment def batch(db: Session, pre_annotations: List[TaskPreAnnotation]) -> List[TaskPreAnnotation]: @@ -20,12 +17,13 @@ def list_by( db: Session, task_id: Union[int, None], owner_id: int, + sample_name: str | None, after: Union[int, None], before: Union[int, None], pageNo: Union[int, None], pageSize: int, sorting: Union[str, None], -) -> List[TaskPreAnnotation]: +) -> Tuple[List[TaskPreAnnotation], int]: # query filter query_filter = [TaskPreAnnotation.created_by == owner_id, TaskPreAnnotation.deleted_at == None] @@ -35,6 +33,10 @@ def list_by( query_filter.append(TaskPreAnnotation.id > after) if task_id: query_filter.append(TaskPreAnnotation.task_id == task_id) + + if sample_name: + query_filter.append(or_(TaskPreAnnotation.sample_name == sample_name, TaskPreAnnotation.sample_name == sample_name[9:])) + query = db.query(TaskPreAnnotation).filter(*query_filter) # default order by id, before need select last items @@ -42,18 +44,26 @@ def list_by( query = query.order_by(TaskPreAnnotation.id.desc()) else: query = query.order_by(TaskPreAnnotation.id.asc()) + + count = query.count() + results = ( query.offset(offset=pageNo * pageSize if pageNo else 0) .limit(limit=pageSize) .all() ) - # No sorting + if sorting: + field, order = sorting.split(":") + if order == "desc": + results = sorted(results, key=lambda x: getattr(x, field), reverse=True) + else: + results = sorted(results, key=lambda x: getattr(x, field)) if before: results.reverse() - return results + return results, count def list_by_task_id_and_owner_id(db: Session, task_id: int, owner_id: int) -> Dict[str, List[TaskPreAnnotation]]: pre_annotations = db.query(TaskPreAnnotation).filter( @@ -64,6 +74,33 @@ def list_by_task_id_and_owner_id(db: Session, task_id: int, owner_id: int) -> Di return pre_annotations +def list_by_task_id_and_file_id(db: Session, task_id: int, file_id: int, owner_id: int) -> List[TaskPreAnnotation]: + return db.query(TaskPreAnnotation).filter( + TaskPreAnnotation.task_id == task_id, + TaskPreAnnotation.created_by == owner_id, + TaskPreAnnotation.deleted_at == None, + TaskPreAnnotation.file_id == file_id + ).all() + +def list_by_task_id_and_owner_id_and_sample_name(db: Session, task_id: int, owner_id: int, sample_name: str) -> List[TaskPreAnnotation]: + """list pre annotations by task_id, owner_id and sample_name without pagination + + Args: + db (Session): _description_ + task_id (int): _description_ + owner_id (int): _description_ + sample_name (str): _description_ + + Returns: + List[TaskPreAnnotation]: _description_ + """ + return db.query(TaskPreAnnotation).filter( + TaskPreAnnotation.task_id == task_id, + TaskPreAnnotation.deleted_at == None, + TaskPreAnnotation.created_by == owner_id, + TaskPreAnnotation.sample_name == sample_name + ).all() + def get(db: Session, pre_annotation_id: int) -> TaskPreAnnotation: return ( db.query(TaskPreAnnotation) @@ -97,8 +134,12 @@ def delete(db: Session, pre_annotation_ids: List[int]) -> None: ) -def count(db: Session, task_id: int, owner_id: int) -> int: +def count(db: Session, task_id: int, owner_id: int, sample_name: str | None) -> int: query_filter = [TaskPreAnnotation.created_by == owner_id, TaskPreAnnotation.deleted_at == None] if task_id: query_filter.append(TaskPreAnnotation.task_id == task_id) + + if sample_name: + query_filter.append(TaskPreAnnotation.sample_name == sample_name) + return db.query(TaskPreAnnotation).filter(*query_filter).count() diff --git a/labelu/internal/adapter/routers/pre_annotation.py b/labelu/internal/adapter/routers/pre_annotation.py index 9231aaa5..83ed5113 100644 --- a/labelu/internal/adapter/routers/pre_annotation.py +++ b/labelu/internal/adapter/routers/pre_annotation.py @@ -2,7 +2,6 @@ from sqlalchemy.orm import Session from fastapi import APIRouter, Depends, Query, status, Security -from fastapi.responses import FileResponse from fastapi.security import HTTPAuthorizationCredentials from labelu.internal.common import db @@ -18,7 +17,7 @@ from labelu.internal.application.response.base import MetaData from labelu.internal.application.response.base import CommonDataResp from labelu.internal.application.response.base import OkRespWithMeta -from labelu.internal.application.response.pre_annotation import PreAnnotationResponse +from labelu.internal.application.response.pre_annotation import PreAnnotationFileResponse, PreAnnotationResponse from labelu.internal.application.response.pre_annotation import CreatePreAnnotationResponse @@ -49,6 +48,65 @@ async def create( # response return OkResp[CreatePreAnnotationResponse](data=data) +@router.get( + "/{task_id}/pre_annotations/files", + response_model=OkRespWithMeta[List[PreAnnotationFileResponse]], + status_code=status.HTTP_200_OK, +) +async def list_pre_annotation_files_request( + task_id: int, + after: Union[int, None] = Query(default=None, gt=0), + before: Union[int, None] = Query(default=None, gt=0), + pageNo: Union[int, None] = Query(default=None, ge=0), + pageSize: Union[int, None] = 100, + sort: Union[str, None] = Query( + default=None, regex="(created_at):(desc|asc)" + ), + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + Get a list of pre annotation files(json or jsonl). + """ + + data, total = await service.list_pre_annotation_files( + db=db, + task_id=task_id, + current_user=current_user, + after=after, + before=before, + pageNo=pageNo, + pageSize=pageSize, + sorting=sort, + ) + + # response + meta_data = MetaData(total=total, page=pageNo, size=len(data)) + return OkRespWithMeta[List[PreAnnotationFileResponse]](meta_data=meta_data, data=data) + +@router.delete( + "/{task_id}/pre_annotations/files/{file_id}", + response_model=OkResp[CommonDataResp], + status_code=status.HTTP_200_OK, +) +async def delete_pre_annotation_file_request( + task_id: int, + file_id: int, + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + Delete a pre annotation file(json or jsonl). + """ + + data = await service.delete_pre_annotation_file( + db=db, task_id=task_id, file_id=file_id, current_user=current_user + ) + + # response + return OkResp[CommonDataResp](data=data) @router.get( "/{task_id}/pre_annotations", diff --git a/labelu/internal/application/command/sample.py b/labelu/internal/application/command/sample.py index 22fa801a..5a537859 100644 --- a/labelu/internal/application/command/sample.py +++ b/labelu/internal/application/command/sample.py @@ -13,6 +13,12 @@ class ExportType(str, Enum): JSON = "JSON" MASK = "MASK" COCO = "COCO" + YOLO = "YOLO" + CSV = "CSV" + XML = "XML" + LABEL_ME = "LABEL_ME" + TF_RECORD = "TF_RECORD" + PASCAL_VOC = "PASCAL_VOC" class CreateSampleCommand(BaseModel): diff --git a/labelu/internal/application/response/pre_annotation.py b/labelu/internal/application/response/pre_annotation.py index 674bd025..99eaf1c1 100644 --- a/labelu/internal/application/response/pre_annotation.py +++ b/labelu/internal/application/response/pre_annotation.py @@ -10,6 +10,21 @@ class CreatePreAnnotationResponse(BaseModel): ids: Union[List[int], None] = Field( default=None, description="description: attachment ids" ) + +class PreAnnotationFileResponse(BaseModel): + id: int = Field( + gt=0, + description="description: attachment file id", + ) + filename: str = Field( + description="description: attachment file name", + ) + url: str = Field( + description="description: attachment file url", + ) + sample_names: List[str] = Field( + description="description: sample names", + ) class PreAnnotationResponse(BaseModel): diff --git a/labelu/internal/application/service/attachment.py b/labelu/internal/application/service/attachment.py index a9bd4878..7b0e366f 100644 --- a/labelu/internal/application/service/attachment.py +++ b/labelu/internal/application/service/attachment.py @@ -1,6 +1,5 @@ import aiofiles import os -import uuid from PIL import Image from pathlib import Path @@ -12,7 +11,6 @@ from labelu.internal.common.error_code import ErrorCode from labelu.internal.common.error_code import LabelUException from labelu.internal.domain.models.user import User -from labelu.internal.domain.models.task import TaskStatus from labelu.internal.domain.models.attachment import TaskAttachment from labelu.internal.adapter.persistence import crud_task from labelu.internal.adapter.persistence import crud_attachment @@ -34,56 +32,58 @@ async def create( status_code=status.HTTP_404_NOT_FOUND, ) - # save file - try: # file relative path - path_filename = cmd.file.filename.split("/") - filename = str(uuid.uuid4())[0:8] + "-" + path_filename[-1] - path = "/".join(path_filename[:-1]) - attachment_relative_base_dir = Path(settings.UPLOAD_DIR).joinpath( - str(task_id), path - ) - attachment_relative_path = str(attachment_relative_base_dir.joinpath(filename)) + path_filename = cmd.file.filename.split("/") + # filename = str(uuid.uuid4())[0:8] + "-" + path_filename[-1] + filename = path_filename[-1] + path = "/".join(path_filename[:-1]) + attachment_relative_base_dir = Path(settings.UPLOAD_DIR).joinpath( + str(task_id), path + ) + attachment_relative_path = str(attachment_relative_base_dir.joinpath(filename)) - # file full path - attachment_full_base_dir = Path(settings.MEDIA_ROOT).joinpath( - attachment_relative_base_dir - ) - attachment_full_path = Path(settings.MEDIA_ROOT).joinpath( - attachment_relative_path - ) + # file full path + attachment_full_base_dir = Path(settings.MEDIA_ROOT).joinpath( + attachment_relative_base_dir + ) + attachment_full_path = Path(settings.MEDIA_ROOT).joinpath( + attachment_relative_path + ) - # create dicreatory - attachment_full_base_dir.mkdir(parents=True, exist_ok=True) - - # save image - logger.info(attachment_full_path) - async with aiofiles.open(attachment_full_path, "wb") as out_file: - content = await cmd.file.read() # async read - await out_file.write(content) # async write - - # create thumbnail for image - if cmd.file.content_type.startswith("image/"): - tumbnail_full_path = Path( - f"{attachment_full_path.parent}/{attachment_full_path.stem}-thumbnail{attachment_full_path.suffix}" - ) - logger.info(tumbnail_full_path) - image = Image.open(attachment_full_path) - image.thumbnail( - ( - round(image.width / image.height * settings.THUMBNAIL_HEIGH_PIXEL), - settings.THUMBNAIL_HEIGH_PIXEL, - ), - ) - if image.mode != "RGB": - image = image.convert("RGB") - image.save(tumbnail_full_path) - except Exception as e: - logger.error(e) + # check file exist + if attachment_full_path.exists(): + logger.error("file already exists:{}", attachment_full_path) raise LabelUException( - code=ErrorCode.CODE_51000_CREATE_ATTACHMENT_ERROR, - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + code=ErrorCode.CODE_51002_TASK_ATTACHMENT_ALREADY_EXISTS, + status_code=status.HTTP_400_BAD_REQUEST, + ) + + # create dicreatory + attachment_full_base_dir.mkdir(parents=True, exist_ok=True) + + + # save image + logger.info(attachment_full_path) + async with aiofiles.open(attachment_full_path, "wb") as out_file: + content = await cmd.file.read() # async read + await out_file.write(content) # async write + + # create thumbnail for image + if cmd.file.content_type.startswith("image/"): + tumbnail_full_path = Path( + f"{attachment_full_path.parent}/{attachment_full_path.stem}-thumbnail{attachment_full_path.suffix}" + ) + logger.info(tumbnail_full_path) + image = Image.open(attachment_full_path) + image.thumbnail( + ( + round(image.width / image.height * settings.THUMBNAIL_HEIGH_PIXEL), + settings.THUMBNAIL_HEIGH_PIXEL, + ), ) + if image.mode != "RGB": + image = image.convert("RGB") + image.save(tumbnail_full_path) # check file already saved if not attachment_full_path.exists() or ( diff --git a/labelu/internal/application/service/pre_annotation.py b/labelu/internal/application/service/pre_annotation.py index 1e766e6f..76fec6c4 100644 --- a/labelu/internal/application/service/pre_annotation.py +++ b/labelu/internal/application/service/pre_annotation.py @@ -1,8 +1,6 @@ import json -from datetime import datetime -from typing import Dict, List, Tuple, Optional +from typing import List, Tuple, Optional -from pathlib import Path from loguru import logger from fastapi import status from sqlalchemy.orm import Session @@ -13,17 +11,17 @@ from labelu.internal.adapter.persistence import crud_task from labelu.internal.adapter.persistence import crud_pre_annotation from labelu.internal.adapter.persistence import crud_attachment +from labelu.internal.domain.models.attachment import TaskAttachment from labelu.internal.domain.models.user import User from labelu.internal.domain.models.pre_annotation import TaskPreAnnotation from labelu.internal.application.command.pre_annotation import CreatePreAnnotationCommand from labelu.internal.application.response.base import UserResp from labelu.internal.application.response.base import CommonDataResp -from labelu.internal.application.response.pre_annotation import CreatePreAnnotationResponse +from labelu.internal.application.response.pre_annotation import CreatePreAnnotationResponse, PreAnnotationFileResponse from labelu.internal.application.response.pre_annotation import PreAnnotationResponse from labelu.internal.application.response.attachment import AttachmentResponse -def read_jsonl_file(db: Session, file_id: int) -> List[dict]: - attachment = crud_attachment.get(db, file_id) +def read_pre_annotation_file(attachment: TaskAttachment) -> List[dict]: if attachment is None: return [] @@ -31,18 +29,24 @@ def read_jsonl_file(db: Session, file_id: int) -> List[dict]: file_full_path = settings.MEDIA_ROOT.joinpath(attachment_path.lstrip("/")) # check if the file exists - if not file_full_path.exists() or not attachment.filename.endswith('.jsonl'): + if not file_full_path.exists() or (not attachment.filename.endswith('.jsonl') and not attachment.filename.endswith('.json')): return [] try: - with open(file_full_path, "r", encoding="utf-8") as f: - data = f.readlines() + if attachment.filename.endswith('.jsonl'): + with open(file_full_path, "r", encoding="utf-8") as f: + data = f.readlines() + return [json.loads(line) for line in data] + else: + with open(file_full_path, "r", encoding="utf-8") as f: + # parse result + parsed_data = json.load(f) + + return [{**item, "result": json.loads(item["result"])} for item in parsed_data] + except FileNotFoundError: raise LabelUException(status_code=404, code=ErrorCode.CODE_51001_TASK_ATTACHMENT_NOT_FOUND) - parsed_data = [json.loads(line) for line in data] - return parsed_data - async def create( db: Session, task_id: int, cmd: List[CreatePreAnnotationCommand], current_user: User ) -> CreatePreAnnotationResponse: @@ -55,49 +59,33 @@ async def create( code=ErrorCode.CODE_50002_TASK_NOT_FOUND, status_code=status.HTTP_404_NOT_FOUND, ) - - def validate_sample_name_exists(file_id: int, pre_annotations_dict: Dict[str, List[TaskPreAnnotation]]) -> List[dict]: - jsonl_content = read_jsonl_file(db, file_id) + + pre_annotations = [] + for pre_annotation in cmd: + pre_annotation_file = crud_attachment.get(db, pre_annotation.file_id) + pre_annotation_contents = read_pre_annotation_file(pre_annotation_file) - for item in jsonl_content: - sample_name = item.get("sample_name") - pre_annotations = pre_annotations_dict.get(sample_name, []) + for _item in pre_annotation_contents: + sample_name = _item.get("sample_name") if pre_annotation_file.filename.endswith(".jsonl") else _item.get("fileName") + exist_pre_annotations = crud_pre_annotation.list_by_task_id_and_owner_id_and_sample_name(db=db, task_id=task_id, owner_id=current_user.id, sample_name=sample_name) - if len(pre_annotations) > 0: + if len(exist_pre_annotations) > 0: raise LabelUException( code=ErrorCode.CODE_55002_SAMPLE_NAME_EXISTS, status_code=status.HTTP_400_BAD_REQUEST, ) + + pre_annotations.append( + TaskPreAnnotation( + task_id=task_id, + file_id=pre_annotation.file_id, + sample_name=sample_name, + data=json.dumps(_item, ensure_ascii=False), + created_by=current_user.id, + updated_by=current_user.id, + ) + ) - # Get all pre_annotations in one query - query_pre_annotations = crud_pre_annotation.list_by_task_id_and_owner_id(db=db, task_id=task_id, owner_id=current_user.id) - - pre_annotations_dict = {} - for pre_annotation in query_pre_annotations: - jsonl_content = read_jsonl_file(db, pre_annotation.file_id) - - for item in jsonl_content: - sample_name = item.get("sample_name") - - # 如果字典中已经有这个 sample_name，就添加到列表中 - if sample_name in pre_annotations_dict: - pre_annotations_dict[sample_name].append(pre_annotation) - # 否则，创建一个新的列表 - else: - pre_annotations_dict[sample_name] = [pre_annotation] - - for pre_annotation in cmd: - validate_sample_name_exists(pre_annotation.file_id, pre_annotations_dict) - - pre_annotations = [ - TaskPreAnnotation( - task_id=task_id, - file_id=pre_annotation.file_id, - created_by=current_user.id, - updated_by=current_user.id, - ) - for pre_annotation in cmd - ] new_pre_annotations = crud_pre_annotation.batch(db=db, pre_annotations=pre_annotations) # response @@ -117,64 +105,77 @@ async def list_by( current_user: User, ) -> Tuple[List[PreAnnotationResponse], int]: - pre_annotations = [] - - # 指定 sample_name 查询时，需要对所有的 pre_annotation 进行查询 - # TODO 优化查询逻辑：将上传的jsonl中的每个预标注提取并存储到单独的表（PreAnnotationDetail）中 - if sample_name: - pre_annotations = db.query(TaskPreAnnotation).all() - else: - pre_annotations = crud_pre_annotation.list_by( - db=db, - task_id=task_id, - owner_id=current_user.id, - after=after, - before=before, - pageNo=pageNo, - pageSize=None if sample_name else pageSize, - sorting=sorting, - ) - - - total = crud_pre_annotation.count(db=db, task_id=task_id, owner_id=current_user.id) - - def parse_jsonl_file(file_id: int, sample_name: str) -> List[dict]: - jsonl_content = read_jsonl_file(db, file_id) - # Filter by sample_name - result = [] - for item in jsonl_content: - if sample_name is None or item.get("sample_name") == sample_name: - result.append(item) - return result + pre_annotations, total = crud_pre_annotation.list_by( + db=db, + task_id=task_id, + owner_id=current_user.id, + after=after, + before=before, + pageNo=pageNo, + sample_name=sample_name, + pageSize=None if sample_name else pageSize, + sorting=sorting, + ) - real_sample_name = sample_name[9:] if sample_name else None - parsed_data_list = [parse_jsonl_file(pre_annotation.file_id, real_sample_name) for pre_annotation in pre_annotations] - - def is_contains_sample_name(inputs: List[dict], sample_name: str) -> bool: - return any(data.get("sample_name") == sample_name for data in inputs) - - filtered_pre_annotations = [] - for i, pre_annotation in enumerate(pre_annotations): - if real_sample_name is None or is_contains_sample_name(parsed_data_list[i], real_sample_name): - filtered_pre_annotations.append( - PreAnnotationResponse( - id=pre_annotation.id, - file=AttachmentResponse(id=pre_annotation.file.id, filename=pre_annotation.file.filename, url=pre_annotation.file.url) if pre_annotation.file else None, - created_at=pre_annotation.created_at, - data=parsed_data_list[i], - created_by=UserResp( - id=pre_annotation.owner.id, - username=pre_annotation.owner.username, - ), - updated_at=pre_annotation.updated_at, - updated_by=UserResp( - id=pre_annotation.updater.id, - username=pre_annotation.updater.username, - ), - ) + return [ + PreAnnotationResponse( + id=pre_annotation.id, + data=pre_annotation.data, + file=AttachmentResponse(id=pre_annotation.file.id, filename=pre_annotation.file.filename, url=pre_annotation.file.url) if pre_annotation.file else None, + created_at=pre_annotation.created_at, + created_by=UserResp( + id=pre_annotation.owner.id, + username=pre_annotation.owner.username, + ), + updated_at=pre_annotation.updated_at, + updated_by=UserResp( + id=pre_annotation.updater.id, + username=pre_annotation.updater.username, + ), + ) for pre_annotation in pre_annotations + ], total + +async def list_pre_annotation_files( + db: Session, + task_id: Optional[int], + after: Optional[int], + before: Optional[int], + pageNo: Optional[int], + pageSize: int, + sorting: Optional[str], + current_user: User, +) -> Tuple[List[TaskAttachment], int]: + try: + pre_annotations = crud_pre_annotation.list_by_task_id_and_owner_id(db=db, task_id=task_id, owner_id=current_user.id) + file_ids = [pre_annotation.file_id for pre_annotation in pre_annotations] + + attachments, total = crud_attachment.list_by(db=db, ids=file_ids, after=after, before=before, pageNo=pageNo, pageSize=pageSize, sorting=sorting) + + _attachment_ids = [attachment.id for attachment in attachments] + def get_sample_names(): + _names = [] + for pre_annotation in pre_annotations: + if pre_annotation.file_id in _attachment_ids and pre_annotation.sample_name is not None: + _names.append(pre_annotation.sample_name) + + return _names + + return [ + PreAnnotationFileResponse( + id=attachment.id, + url=attachment.url, + filename=attachment.filename, + sample_names=get_sample_names(), ) - - return filtered_pre_annotations, total + for attachment in attachments + ], total + + except Exception as e: + logger.error("list pre annotation files error: {}", e) + raise LabelUException( + code=ErrorCode.CODE_51001_TASK_ATTACHMENT_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) async def get( @@ -207,6 +208,17 @@ async def get( username=pre_annotation.updater.username, ), ) + +async def delete_pre_annotation_file( + db: Session, task_id: int, file_id: int, current_user: User +) -> CommonDataResp: + with db.begin(): + pre_annotations = crud_pre_annotation.list_by_task_id_and_file_id(db=db, task_id=task_id, owner_id=current_user.id, file_id=file_id) + pre_annotation_ids = [pre_annotation.id for pre_annotation in pre_annotations] + crud_pre_annotation.delete(db=db, pre_annotation_ids=pre_annotation_ids) + crud_attachment.delete(db=db, attachment_ids=[file_id]) + # response + return CommonDataResp(ok=True) async def delete( db: Session, pre_annotation_ids: List[int], current_user: User diff --git a/labelu/internal/common/converter.py b/labelu/internal/common/converter.py index de7cc516..4336c7e1 100644 --- a/labelu/internal/common/converter.py +++ b/labelu/internal/common/converter.py @@ -1,18 +1,32 @@ +import base64 +import csv import json +import os from zipfile import ZipFile from PIL import Image, ImageDraw from enum import Enum +import xml.etree.ElementTree as ET from typing import List from loguru import logger -from labelu.internal.common.color import colors +from .xml_converter import XML_converter +from .tf_record_converter import TF_record_converter +from .color import colors +from .config import settings class Format(str, Enum): JSON = "JSON" COCO = "COCO" MASK = "MASK" + YOLO = "YOLO" + CSV = "CSV" + XML = "XML" + VOC = "VOC" + TF_RECORD = "TF_RECORD" + LABEL_ME = "LABEL_ME" + PASCAL_VOC = "PASCAL_VOC" class Converter: @@ -43,7 +57,49 @@ def convert( out_data_dir=out_data_dir, out_data_file_name_prefix=out_data_file_name_prefix, ) - + elif format == Format.LABEL_ME.value: + return self.convert_to_labelme( + config=config, + input_data=input_data, + out_data_file_name_prefix=out_data_file_name_prefix, + out_data_dir=out_data_dir, + ) + elif format == Format.YOLO.value: + return self.convert_to_yolo( + config=config, + input_data=input_data, + out_data_file_name_prefix=out_data_file_name_prefix, + out_data_dir=out_data_dir, + ) + elif format == Format.CSV.value: + return self.convert_to_csv( + config=config, + input_data=input_data, + out_data_file_name_prefix=out_data_file_name_prefix, + out_data_dir=out_data_dir, + ) + elif format == Format.XML.value: + return self.convert_to_xml( + config=config, + input_data=input_data, + out_data_file_name_prefix=out_data_file_name_prefix, + out_data_dir=out_data_dir, + ) + elif format == Format.TF_RECORD.value: + return self.convert_to_tf_record( + config=config, + input_data=input_data, + out_data_file_name_prefix=out_data_file_name_prefix, + out_data_dir=out_data_dir, + ) + elif format == Format.PASCAL_VOC.value: + return self.convert_to_pascal_voc( + config=config, + input_data=input_data, + out_data_file_name_prefix=out_data_file_name_prefix, + out_data_dir=out_data_dir, + ) + def convert_to_json( self, input_data: List[dict], @@ -92,8 +148,9 @@ def convert_to_json( { "id": sample.get("id"), "result": annotated_result_str, + "folder": settings.MEDIA_ROOT, "url": file.get("url"), - "fileName": file.get("filename", "")[9:], + "fileName": file.get("filename", ""), } ) @@ -166,7 +223,7 @@ def convert_to_coco( # coco image image = { "id": sample.get("id"), - "fileName": file.get("filename", "")[9:], + "fileName": file.get("filename", ""), "width": annotation_result.get("width", 0), "height": annotation_result.get("height", 0), "valid": False @@ -263,7 +320,7 @@ def convert_to_mask( logger.info("data is: {}", sample) filename = file.get("filename") if filename and filename.split("/")[-1]: - file_relative_path_base_name = filename.split("/")[-1].split(".")[0][9:] + file_relative_path_base_name = filename.split("/")[-1].split(".")[0] else: file_relative_path_base_name = "result" @@ -292,8 +349,8 @@ def convert_to_mask( file_relative_path_model_l = f"{file_relative_path_base_name}-trainIds.png" file_full_path_model_l = out_data_dir.joinpath(file_relative_path_model_l) img_model_l = Image.new("L", (width, height), 0) - for p in polygons: - ImageDraw.Draw(img_model_l).polygon(p, outline=1, fill=1) + for _index, p in enumerate(polygons): + ImageDraw.Draw(img_model_l).polygon(p, outline=1, fill=_index + 1) img_model_l.save(file_full_path_model_l, "PNG") export_files.append(file_full_path_model_l) @@ -336,7 +393,465 @@ def convert_to_mask( logger.info("Export file path: {}", file_full_path_zip) return file_full_path_zip + def convert_to_labelme(self, config: dict, input_data: List[dict], out_data_file_name_prefix: str, out_data_dir: str): + out_data_dir.mkdir(parents=True, exist_ok=True) + export_files = [] + result = [] + # does not support cuboid / spline + shape_dict = { + "polygonTool": "polygon", + "rectTool": "rectangle", + "lineTool": "linestrip", + "pointTool": "point", + } + + label_text_dict = {} + common_attributes = { attr.get("value"): attr.get("key") for attr in config.get("attributes", [])} + + for tool in config.get("tools", []): + label_text_dict[tool.get("tool")] = { attr.get("value"): attr.get("key") for attr in tool.get("config", {}).get("attributes", [])} + + def get_label(_tool: str, _input_label: str): + _label = label_text_dict.get(_tool, {}).get(_input_label, "") + + if not _label: + _label = common_attributes.get(_input_label, "") + + return _label + + def convert_points(points: List[dict]): + return [[point.get("x"), point.get("y")] for point in points] + + def image_to_base64(file_path: str): + file_full_path = settings.MEDIA_ROOT.joinpath(file_path.lstrip("/")) + with open(file_full_path, "rb") as image_file: + return base64.b64encode(image_file.read()).decode('utf-8') + + for sample in input_data: + labelme_item = { + "version": "5.5.0", + "flags": {}, + "shapes": [], + "imagePath": "", + "imageData": "", + "imageHeight": 0, + "imageWidth": 0, + } + data = json.loads(sample.get("data")) + file = sample.get("file", {}) + + # skip invalid data + annotated_result = json.loads(data.get("result")) + if sample.get("state") == "SKIPPED": + continue + + labelme_item["imagePath"] = file.get("filename", "") + labelme_item["imageData"] = image_to_base64(file.get("path")) + + if annotated_result: + labelme_item["imageWidth"] = annotated_result.get("width", 0) + labelme_item["imageHeight"] = annotated_result.get("height", 0) + + for tool in annotated_result.copy().keys(): + if tool.endswith("Tool") and tool in shape_dict: + # polygon + if tool == "polygonTool": + tool_results = annotated_result.pop(tool) + for tool_result in tool_results.get("result", []): + attributes = tool_result.get("attributes", {}) + shape = { + "label": get_label(tool, tool_result.get("label", "")), + "points": convert_points(tool_result.get("points", [])), + "group_id": "", + # Get description from attributes + "description": attributes.get("description", ""), + "shape_type": shape_dict.get(tool, "polygon"), + "flags": {}, + "mask": "", + } + labelme_item["shapes"].append(shape) + + # rect + if tool == "rectTool": + tool_results = annotated_result.pop(tool) + for tool_result in tool_results.get("result", []): + attributes = tool_result.get("attributes", {}) + x = tool_result.get("x", 0) + y = tool_result.get("y", 0) + width = tool_result.get("width", 0) + height = tool_result.get("height", 0) + shape = { + "label": get_label(tool, tool_result.get("label", "")), + "points": [[x, y], [x + width, y + height]], + "group_id": "", + "description": attributes.get("description", ""), + "shape_type": shape_dict.get(tool, "rectangle"), + "flags": {}, + "mask": "", + } + labelme_item["shapes"].append(shape) + + if tool == "lineTool": + tool_results = annotated_result.pop(tool) + for tool_result in tool_results.get("result", []): + attributes = tool_result.get("attributes", {}) + shape = { + "label": get_label(tool, tool_result.get("label", "")), + "points": convert_points(tool_result.get("points", [])), + "group_id": "", + "description": attributes.get("description", ""), + "shape_type": shape_dict.get(tool, "linestrip"), + "flags": {}, + "mask": "", + } + labelme_item["shapes"].append(shape) + + if tool == "pointTool": + tool_results = annotated_result.pop(tool) + for tool_result in tool_results.get("result", []): + attributes = tool_result.get("attributes", {}) + shape = { + "label": get_label(tool, tool_result.get("label", "")), + "points": [[tool_result.get("x", 0), tool_result.get("y", 0)]], + "group_id": "", + "description": attributes.get("description", ""), + "shape_type": shape_dict.get(tool, "point"), + "flags": {}, + "mask": "", + } + labelme_item["shapes"].append(shape) + result.append(labelme_item) + file_basename = os.path.splitext(file.get("filename", ""))[0] + file_name = out_data_dir.joinpath(f"{file_basename}.json") + with file_name.open("w") as outfile: + # 格式化json，两个空格缩进 + json.dump(labelme_item, outfile, indent=2, ensure_ascii=False) + + export_files.append(file_name) + + + file_relative_path_zip = f"task-{out_data_file_name_prefix}-label-me.zip" + file_full_path_zip = out_data_dir.joinpath(file_relative_path_zip) + with ZipFile(file_full_path_zip, "w") as zipf: + for f in export_files: + zipf.write(str(f), arcname=f.name) + logger.info("Export file path: {}", file_full_path_zip) + return file_full_path_zip + + def convert_to_yolo(self, config: dict, input_data: List[dict], out_data_file_name_prefix: str, out_data_dir: str): + out_data_dir.mkdir(parents=True, exist_ok=True) + export_files = [] + classes = [] + + # make classes + for tool in config.get("tools", []): + for attr in tool.get("config", {}).get("attributes", []): + classes.append(attr.get("value")) + for attr in config.get("attributes", []): + classes.append(attr.get("value")) + + # make classes.txt + classes_file = out_data_dir.joinpath("classes.txt") + with classes_file.open("w") as outfile: + for c in classes: + outfile.write(f"{c}\n") + export_files.append(classes_file) + + for sample in input_data: + data = json.loads(sample.get("data")) + file = sample.get("file", {}) + + # skip invalid data + annotated_result = json.loads(data.get("result")) + if sample.get("state") == "SKIPPED" or not annotated_result: + continue + + image_path = settings.MEDIA_ROOT.joinpath(file.get("path").lstrip("/")) + file_basename = os.path.splitext(file.get("filename", ""))[0] + file_name = out_data_dir.joinpath(f"{file_basename}.txt") + image_width = annotated_result.get("width", 0) + image_height = annotated_result.get("height", 0) + rotate = annotated_result.get("rotate", 0) + + with Image.open(image_path) as img: + if rotate: + img = img.rotate(rotate, expand=True) + image_width, image_height = img.size + + with file_name.open("a") as outfile: + if rotate: + outfile.write(f"# rotate: {rotate}\n") + + for tool in annotated_result.copy().keys(): + if tool == 'rectTool': + tool_results = annotated_result.pop(tool) + for tool_result in tool_results.get("result", []): + x = tool_result.get("x", 0) + y = tool_result.get("y", 0) + width = tool_result.get("width", 0) + height = tool_result.get("height", 0) + label = tool_result.get("label", "") + x_center = x + width / 2 + y_center = y + height / 2 + x_center /= image_width + y_center /= image_height + width /= image_width + height /= image_height + + outfile.write(f"{classes.index(label)} {x_center} {y_center} {width} {height}\n") + export_files.append(file_name) + + + file_relative_path_zip = f"task-{out_data_file_name_prefix}-yolo.zip" + file_full_path_zip = out_data_dir.joinpath(file_relative_path_zip) + with ZipFile(file_full_path_zip, "w") as zipf: + for f in export_files: + zipf.write(str(f), arcname=f.name) + logger.info("Export file path: {}", file_full_path_zip) + return file_full_path_zip + + def convert_to_csv(self, config: dict, input_data: List[dict], out_data_file_name_prefix: str, out_data_dir: str): + out_data_dir.mkdir(parents=True, exist_ok=True) + export_files = [] + + label_text_dict = {} + common_attributes = { attr.get("value"): attr.get("key") for attr in config.get("attributes", [])} + + def get_label(_tool: str, _input_label: str): + _label = label_text_dict.get(_tool, {}).get(_input_label, "") + + if not _label: + _label = common_attributes.get(_input_label, "") + + return _label + + def get_attributes(attributes: dict): + result = [] + + for value in attributes.values(): + result.append(", ".join(value) if isinstance(value, list) else value) + + return ", ".join(result) + + def get_points(direction: dict): + return [ + ( + direction.get("tl").get("x"), + direction.get("tl").get("y"), + ), + ( + direction.get("tr").get("x"), + direction.get("tr").get("y"), + ), + ( + direction.get("br").get("x"), + direction.get("br").get("y"), + ), + ( + direction.get("bl").get("x"), + direction.get("bl").get("y"), + ), + ] + + + for tool in config.get("tools", []): + label_text_dict[tool.get("tool")] = { attr.get("value"): attr.get("key") for attr in tool.get("config", {}).get("attributes", [])} + + for sample in input_data: + data = json.loads(sample.get("data")) + file = sample.get("file", {}) + # tool_name, label, x, y, width, height etc. + rows = [] + + # skip invalid data + annotated_result = json.loads(data.get("result")) + if sample.get("state") == "SKIPPED" or not annotated_result: + continue + + for tool in annotated_result.copy().keys(): + if tool == 'rectTool': + tool_results = annotated_result.pop(tool) + rows.append(["tool_name", "label", "label_text", "x", "y", "width", "height", "attributes", "order"]) + for tool_result in tool_results.get("result", []): + x = tool_result.get("x", 0) + y = tool_result.get("y", 0) + width = tool_result.get("width", 0) + height = tool_result.get("height", 0) + label = tool_result.get("label", "") + order = tool_result.get("order", 0) + label_text = get_label(tool, label) + rows.append([tool, label, label_text, x, y, width, height, get_attributes(tool_result.get('attributes', {})), order]) + + if tool == 'lineTool': + tool_results = annotated_result.pop(tool) + rows.append(["tool_name", "label", "label_text", "points", "control_points", "attributes", "order"]) + for tool_result in tool_results.get("result", []): + points = tool_result.get("points", []) + control_points = tool_result.get("controlPoints", []) + label = tool_result.get("label", "") + order = tool_result.get("order", 0) + label_text = get_label(tool, label) + rows.append([tool, label, label_text, points, control_points, get_attributes(tool_result.get('attributes', {})), order]) + + if tool == 'pointTool': + tool_results = annotated_result.pop(tool) + rows.append(["tool_name", "label", "label_text", "x", "y", "order"]) + for tool_result in tool_results.get("result", []): + x = tool_result.get("x", 0) + y = tool_result.get("y", 0) + label = tool_result.get("label", "") + order = tool_result.get("order", 0) + label_text = get_label(tool, label) + rows.append([tool, label, label_text, x, y, order]) + + if tool == 'polygonTool': + tool_results = annotated_result.pop(tool) + rows.append(["tool_name", "label", "label_text", "points", "attributes", "order"]) + for tool_result in tool_results.get("result", []): + points = tool_result.get("points", []) + control_points = tool_result.get("controlPoints", []) + label = tool_result.get("label", "") + order = tool_result.get("order", 0) + label_text = get_label(tool, label) + rows.append([tool, label, label_text, points, control_points, get_attributes(tool_result.get('attributes', {})), order]) + + if tool == 'cuboidTool': + tool_results = annotated_result.pop(tool) + rows.append(["tool_name", "label", "label_text", "direction", "front", "back", "attributes", "order"]) + for tool_result in tool_results.get("result", []): + direction = tool_result.get("direction") + # [[x,y], ...] + front = get_points(tool_result.get("front")) + back = get_points(tool_result.get("back")) + width = tool_result.get("width", 0) + height = tool_result.get("height", 0) + label = tool_result.get("label", "") + order = tool_result.get("order", 0) + label_text = get_label(tool, label) + rows.append([tool, label, label_text, direction, front, back, get_attributes(tool_result.get('attributes', {})), order]) + + file_basename = os.path.splitext(file.get("filename", ""))[0] + file_name = out_data_dir.joinpath(f"{file_basename}.csv") + with file_name.open("w") as outfile: + writer = csv.writer(outfile) + writer.writerows(rows) + export_files.append(file_name) + + file_relative_path_zip = f"task-{out_data_file_name_prefix}-csv.zip" + file_full_path_zip = out_data_dir.joinpath(file_relative_path_zip) + with ZipFile(file_full_path_zip, "w") as zipf: + for f in export_files: + zipf.write(str(f), arcname=f.name) + logger.info("Export file path: {}", file_full_path_zip) + return file_full_path_zip + + + def convert_to_xml(self, config: dict, input_data: List[dict], out_data_file_name_prefix: str, out_data_dir: str): + out_data_dir.mkdir(parents=True, exist_ok=True) + file_full_path = out_data_dir.joinpath("result.xml") + + # result struct + xml_converter = XML_converter() + root = ET.Element("root") + sample_item = ET.SubElement(root, "sample") + + for sample in input_data: + data = json.loads(sample.get("data")) + file = sample.get("file", {}) + + # skip invalid data + annotated_result = json.loads(data.get("result")) + result = ET.SubElement(sample_item, "result") + if annotated_result and sample.get("state") == "SKIPPED": + ET.SubElement(result, "valid").text = "False" + + ET.SubElement(sample_item, "id").text = str(sample.get("id")) + + ET.SubElement(sample_item, "folder").text = str(settings.MEDIA_ROOT) + ET.SubElement(sample_item, "path").text = file.get("path") + ET.SubElement(sample_item, "fileName").text = file.get("filename", "") + + ET.SubElement(result, "width").text = str(annotated_result.get("width", 0)) + ET.SubElement(result, "height").text = str(annotated_result.get("height", 0)) + ET.SubElement(result, "rotate").text = str(annotated_result.get("rotate", 0)) + + # change result struct + if annotated_result: + annotations = ET.SubElement(result, "annotations") + for tool in annotated_result.copy().keys(): + tool_results = annotated_result.pop(tool) + if tool.endswith("Tool"): + tool_annotations = xml_converter.convert_tool_results(tool, tool_results) + + for annotation in tool_annotations: + annotations.append(annotation) + + tree = ET.ElementTree(root) + tree.write(file_full_path, encoding="utf-8", xml_declaration=True) + logger.info("Export file path: {}", file_full_path) + return file_full_path + + def convert_to_tf_record(self, config: dict, input_data: List[dict], out_data_file_name_prefix: str, out_data_dir: str): + out_data_dir.mkdir(parents=True, exist_ok=True) + export_files = [] + + # result struct + tf_record_examples = TF_record_converter().create_tf_examples(input_data, config) + + for sample in input_data: + file = sample.get("file", {}) + example = tf_record_examples.pop(0) + file_basename = os.path.splitext(file.get("filename", ""))[0] + tf_record = f"{file_basename}.tfrecord" + file_full_path = out_data_dir.joinpath(tf_record) + + with file_full_path.open("wb") as outfile: + outfile.write(example.SerializeToString()) + + export_files.append(file_full_path) + + file_relative_path_zip = f"task-{out_data_file_name_prefix}-tfrecord.zip" + file_full_path_zip = out_data_dir.joinpath(file_relative_path_zip) + with ZipFile(file_full_path_zip, "w") as zipf: + for f in export_files: + zipf.write(str(f), arcname=f.name) + logger.info("Export file path: {}", file_full_path_zip) + return file_full_path_zip + + def convert_to_pascal_voc(self, config: dict, input_data: List[dict], out_data_file_name_prefix: str, out_data_dir: str): + out_data_dir.mkdir(parents=True, exist_ok=True) + export_files = [] + + xml_converter = XML_converter() + + for sample in input_data: + data = json.loads(sample.get("data")) + file = sample.get("file", {}) + + # skip invalid data + annotated_result = json.loads(data.get("result")) + if sample.get("state") == "SKIPPED" or not annotated_result: + continue + + voc_xml = xml_converter.create_pascal_voc_xml(config, file, annotated_result) + file_basename = os.path.splitext(file.get("filename", ""))[0] + file_name = out_data_dir.joinpath(f"{file_basename}.xml") + + tree = ET.ElementTree(voc_xml) + tree.write(file_name, encoding="utf-8", xml_declaration=True) + + export_files.append(file_name) + + file_relative_path_zip = f"task-{out_data_file_name_prefix}-pascal-voc.zip" + file_full_path_zip = out_data_dir.joinpath(file_relative_path_zip) + + with ZipFile(file_full_path_zip, "w") as zipf: + for f in export_files: + zipf.write(str(f), arcname=f.name) + logger.info("Export file path: {}", file_full_path_zip) + return file_full_path_zip + def _polygonArea(X, Y): # Initialize area @@ -352,5 +867,4 @@ def _polygonArea(X, Y): # Return absolute value return abs(area) / 2.0 - converter = Converter() diff --git a/labelu/internal/common/error_code.py b/labelu/internal/common/error_code.py index 1505feef..d47e6f5b 100644 --- a/labelu/internal/common/error_code.py +++ b/labelu/internal/common/error_code.py @@ -72,6 +72,10 @@ class ErrorCode(Enum): "Attachment file not found", ) + CODE_51002_TASK_ATTACHMENT_ALREADY_EXISTS =( + TASK_INIT_CODE + 1002, + "Attachment file already exists", + ) # task sample error code CODE_55000_SAMPLE_LIST_PARAMETERS_ERROR = ( TASK_INIT_CODE + 5000, diff --git a/labelu/internal/common/tf_record_converter.py b/labelu/internal/common/tf_record_converter.py new file mode 100644 index 00000000..a4590d95 --- /dev/null +++ b/labelu/internal/common/tf_record_converter.py @@ -0,0 +1,225 @@ +import json +import os +from PIL import Image +from typing import List +from tfrecord import example_pb2 +from labelu.internal.common.config import settings + +class TF_record_converter: + def __init__(self): + self.label_map = {} + + def _get_label_id(self, label: str): + if label not in self.label_map: + self.label_map[label] = len(self.label_map) + 1 + + return self.label_map[label] + + def create_tf_examples(self, sample_results: List[dict], config: dict): + examples = [] + label_text_dict = {} + + for tool in config.get("tools", []): + label_text_dict[tool.get("tool")] = { attr.get("value"): attr.get("key") for attr in tool.get("config", {}).get("attributes", [])} + + common_attributes = { attr.get("value"): attr.get("key") for attr in config.get("attributes", [])} + + def get_label(_tool: str, _input_label: str): + _label = label_text_dict.get(_tool, {}).get(_input_label, "") + + if not _label: + _label = common_attributes.get(_input_label, "") + + return _label + + classes_text = [] + classes = [] + + for sample in sample_results: + data = json.loads(sample.get("data")) + file = sample.get("file", {}) + + # skip invalid data + annotated_result = json.loads(data.get("result")) + if sample.get("state") == "SKIPPED" or not annotated_result: + continue + + image_width = annotated_result.get("width", 0) + image_height = annotated_result.get("height", 0) + rotate = annotated_result.get("rotate", 0) + + _, file_extension = os.path.splitext(file.get("filename", "")) + + file_full_path = settings.MEDIA_ROOT.joinpath(file.get("path").lstrip("/")) + + with Image.open(file_full_path) as img: + if rotate: + img = img.rotate(rotate) + + image_width, image_height = img.size + encoded_image_data = img.tobytes() + + xmins = [] + xmaxs = [] + ymins = [] + ymaxs = [] + line_x = [] + line_y = [] + segmentation = [] + keypoints_x = [] + keypoints_y = [] + keypoints_visibility = [] + cuboid_front = [] + cuboid_back = [] + tags = [] + texts = [] + + base_info = { + "image/width": self._int64_feature(image_width), + "image/height": self._int64_feature(image_height), + "image/filename": self._bytes_feature(file['filename'].encode('utf8')), + "image/source_id": self._bytes_feature(file['filename'].encode('utf8')), + "image/encoded": self._bytes_feature(encoded_image_data), + "image/format": self._bytes_feature(file_extension.encode('utf8')), + } + + for tool in annotated_result.copy().keys(): + tool_results = annotated_result.pop(tool) + if tool == "rectTool": + for tool_result in tool_results.get("result", []): + label_text = get_label(tool, tool_result.get("label", "")) + classes_text.append(label_text.encode('utf8')) + classes.append(self._get_label_id(label_text)) + xmins.append(tool_result.get("x", 0) / image_width) + xmaxs.append((tool_result.get("x", 0) + tool_result.get("width", 0)) / image_width) + ymins.append(tool_result.get("y", 0) / image_height) + ymaxs.append((tool_result.get("y", 0) + tool_result.get("height", 0)) / image_height) + + base_info.update( + { + "image/object/bbox/xmin": self._float_list_feature(xmins), + "image/object/bbox/xmax": self._float_list_feature(xmaxs), + "image/object/bbox/ymin": self._float_list_feature(ymins), + "image/object/bbox/ymax": self._float_list_feature(ymaxs), + } + ) + if tool == "lineTool": + for tool_result in tool_results.get("result", []): + points = tool_result.get("points", []) + + label_text = get_label(tool, tool_result.get("label", "")) + classes.append(label_text.encode('utf8')) + classes.append(self._get_label_id(label_text)) + + for point in points: + line_x.append(point.get("x", 0) / image_width) + line_y.append(point.get("y", 0) / image_height) + + base_info.update( + { + "image/object/line/x": self._float_list_feature(line_x), + "image/object/line/y": self._float_list_feature(line_y), + } + ) + + if tool == "polygonTool": + for tool_result in tool_results.get("result", []): + points = tool_result.get("points", []) + segmentation = [] + for point in points: + segmentation.append(point.get("x", 0) / image_width) + segmentation.append(point.get("y", 0) / image_height) + label = tool_result.get("label", "") + classes_text.append(label.encode('utf8')) + classes.append(self._get_label_id(label)) + + base_info.update( + { + "image/object/segmentation": self._float_list_feature(segmentation), + } + ) + + if tool == "pointTool": + for tool_result in tool_results.get("result", []): + points = tool_result.get("points", []) + for point in points: + keypoints_x.append(point.get("x", 0) / image_width) + keypoints_y.append(point.get("y", 0) / image_height) + # TODO: visibility + keypoints_visibility.append(2) + + label = tool_result.get("label", "") + classes_text.append(label.encode('utf8')) + classes.append(self._get_label_id(label)) + + base_info.update( + { + "image/object/keypoints/x": self._float_list_feature(keypoints_x), + "image/object/keypoints/y": self._float_list_feature(keypoints_y), + "image/object/keypoints/visibility": self._int64_feature(keypoints_visibility), + } + ) + + if tool == "cuboidTool": + for tool_result in tool_results.get("result", []): + front = tool_result.get("front", {}) + back = tool_result.get("back", {}) + for key in ["tl", "tr", "br", "bl"]: + cuboid_front.append(front.get(key, {}).get("x", 0) / image_width) + cuboid_front.append(front.get(key, {}).get("y", 0) / image_height) + cuboid_back.append(back.get(key, {}).get("x", 0) / image_width) + cuboid_back.append(back.get(key, {}).get("y", 0) / image_height) + label = tool_result.get("label", "") + classes_text.append(label.encode('utf8')) + classes.append(self._get_label_id(label)) + + base_info.update( + { + "image/object/cuboid/front": self._float_list_feature(cuboid_front), + "image/object/cuboid/back": self._float_list_feature(cuboid_back), + } + ) + + if tool == "tagTool": + for tool_result in tool_results.get("result", []): + for _, tag in tool_result.get("value", {}).items(): + for value in tag: + tags.append(value.encode('utf8')) + + base_info.update( + { + "image/object/tag": self._bytes_feature(tags), + } + ) + + if tool == "textTool": + for tool_result in tool_results.get("result", []): + for _, text in tool_result.get("value", {}).items(): + texts.append(text.encode('utf8')) + + base_info.update( + { + "image/object/text": self._bytes_feature(texts), + } + ) + + base_info.update( + { + "image/object/class/text": self._bytes_feature(classes_text), + "image/object/class/label": self._int64_feature(classes), + } + ) + + examples.append(example_pb2.Example(features=example_pb2.Features(feature=base_info))) + + return examples + + + def _bytes_feature(self, value: bytes | List[bytes]): + return example_pb2.Feature(bytes_list=example_pb2.BytesList(value=[value] if isinstance(value, bytes) else value)) + + def _float_list_feature(self, value: List[float]): + return example_pb2.Feature(float_list=example_pb2.FloatList(value=value)) + + def _int64_feature(self, value: int | List[int]): + return example_pb2.Feature(int64_list=example_pb2.Int64List(value=[value] if isinstance(value, int) else value)) diff --git a/labelu/internal/common/xml_converter.py b/labelu/internal/common/xml_converter.py new file mode 100644 index 00000000..c5879715 --- /dev/null +++ b/labelu/internal/common/xml_converter.py @@ -0,0 +1,290 @@ +import xml.etree.ElementTree as ET +from labelu.internal.common.config import settings + +class XML_converter: + def create_pascal_voc_xml(self, config: dict, file: dict, sample_result: dict): + annotation = ET.Element("annotation") + + folder = ET.SubElement(annotation, "folder") + folder.text = str(settings.MEDIA_ROOT) + + filename_elem = ET.SubElement(annotation, "filename") + filename_elem.text = file.get("filename", "") + + path_elem = ET.SubElement(annotation, "path") + path_elem.text = file.get("path") + + image_width = sample_result.get("width", 0) + image_height = sample_result.get("height", 0) + + size_elem = ET.SubElement(annotation, "size") + width = ET.SubElement(size_elem, "width") + width.text = str(image_width) + height = ET.SubElement(size_elem, "height") + height.text = str(image_height) + depth = ET.SubElement(size_elem, "depth") + depth.text = "3" + rotate = ET.SubElement(size_elem, "rotate") + rotate.text = str(sample_result.get("rotate", 0)) + + label_text_dict = {} + + for tool in config.get("tools", []): + label_text_dict[tool.get("tool")] = { attr.get("value"): attr.get("key") for attr in tool.get("config", {}).get("attributes", [])} + + common_attributes = { attr.get("value"): attr.get("key") for attr in config.get("attributes", [])} + + def get_label(_tool: str, _input_label: str): + _label = label_text_dict.get(_tool, {}).get(_input_label, "") + + if not _label: + _label = common_attributes.get(_input_label, "") + + return _label + + for tool in sample_result.copy().keys(): + tool_results = sample_result.pop(tool) + + if tool == "rectTool": + for tool_result in tool_results.get("result", []): + + obj_elem = ET.SubElement(annotation, "object") + name = ET.SubElement(obj_elem, "name") + name.text = get_label(tool, tool_result.get("label", "")) + # get value from attributes + truncated = ET.SubElement(obj_elem, "truncated") + truncated_value = tool_result.get("attributes", {}).get("truncated", [0]) + truncated.text = str(truncated_value[0]) + difficult = ET.SubElement(obj_elem, "difficult") + difficult_value = tool_result.get("attributes", {}).get("difficult", [0]) + difficult.text = str(difficult_value[0]) + + bndbox = ET.SubElement(obj_elem, "bndbox") + xmin = ET.SubElement(bndbox, "xmin") + xmin_value = tool_result.get("x", 0) + xmin.text = str(xmin_value) + ymin = ET.SubElement(bndbox, "ymin") + ymin_value = tool_result.get("y", 0) + ymin.text = str(ymin_value) + xmax = ET.SubElement(bndbox, "xmax") + xmax.text = str(xmin_value + tool_result.get("width", 0)) + ymax = ET.SubElement(bndbox, "ymax") + ymax.text = str(ymin_value + tool_result.get("height", 0)) + + + return annotation + + def convert_tool_results(self, tool_name, tool_results): + annotations = [] + for tool_result in tool_results.get("result", []): + if tool_name == 'rectTool': + annotation = self.create_rect(tool_result) + elif tool_name == 'polygonTool': + annotation = self.create_polygon(tool_result) + elif tool_name == 'pointTool': + annotation = self.create_point(tool_result) + elif tool_name == 'lineTool': + annotation = self.create_line(tool_result) + elif tool_name == 'cuboidTool': + annotation = self.create_cuboid(tool_result) + elif tool_name == 'tagTool': + annotation = self.create_tag(tool_result) + elif tool_name == 'textTool': + annotation = self.create_text(tool_result) + else: + continue + annotations.append(annotation) + return annotations + + def create_attributes(self, attributes: dict): + result = ET.Element("attributes") + + for key, value in attributes.items(): + attribute = ET.SubElement(result, "attribute") + ET.SubElement(attribute, "key").text = key + ET.SubElement(attribute, "value").text = value if isinstance(value, str) else ", ".join(value) + + return result + + def create_rect(self, anno: dict): + annotation = ET.Element("object") + + ET.SubElement(annotation, "toolName").text = "rectTool" + + result = ET.Element("result") + + ET.SubElement(result, "x").text = str(anno.get("x", 0)) + ET.SubElement(result, "y").text = str(anno.get("y", 0)) + ET.SubElement(result, "width").text = str(anno.get("width", 0)) + ET.SubElement(result, "height").text = str(anno.get("height", 0)) + ET.SubElement(result, "label").text = anno.get("label", "") + ET.SubElement(result, "order").text = str(anno.get("order", 0)) + + result.append(self.create_attributes(anno.get("attributes", {}))) + + annotation.append(result) + + return annotation + + def create_polygon(self, anno: dict): + annotation = ET.Element("object") + + ET.SubElement(annotation, "toolName").text = "polygonTool" + + result = ET.Element("result") + + ET.SubElement(result, "label").text = anno.get("label", "") + ET.SubElement(result, "order").text = str(anno.get("order", 0)) + ET.SubElement(result, "type").text = str(anno.get("type", "line")) + + points = ET.Element("points") + for point in anno.get("points", []): + p = ET.Element("point") + + ET.SubElement(p, "x").text = str(point.get("x", 0)) + ET.SubElement(p, "y").text = str(point.get("y", 0)) + + points.append(p) + + if anno.get("controlPoints"): + control_points = ET.Element("controlPoints") + + for cp in anno.get("controlPoints", []): + item = ET.Element("controlPoint") + + ET.SubElement(item, "x").text = str(cp.get("x", 0)) + ET.SubElement(item, "y").text = str(cp.get("y", 0)) + + control_points.append(item) + + result.append(control_points) + + result.append(self.create_attributes(anno.get("attributes", {}))) + result.append(points) + annotation.append(result) + + return annotation + + def create_point(self, anno: dict): + annotation = ET.Element("object") + + ET.SubElement(annotation, "toolName").text = "pointTool" + + result = ET.Element("result") + + ET.SubElement(result, "x").text = str(anno.get("x", 0)) + ET.SubElement(result, "y").text = str(anno.get("y", 0)) + ET.SubElement(result, "label").text = anno.get("label", "") + ET.SubElement(result, "order").text = str(anno.get("order", 0)) + + result.append(self.create_attributes(anno.get("attributes", {}))) + annotation.append(result) + + return annotation + + def create_line(self, anno: dict): + annotation = ET.Element("object") + + ET.SubElement(annotation, "toolName").text = "lineTool" + + result = ET.Element("result") + + ET.SubElement(result, "label").text = anno.get("label", "") + ET.SubElement(result, "order").text = str(anno.get("order", 0)) + ET.SubElement(result, "type").text = str(anno.get("type", "line")) + + points = ET.Element("points") + for point in anno.get("points", []): + p = ET.Element("point") + + ET.SubElement(p, "x").text = str(point.get("x", 0)) + ET.SubElement(p, "y").text = str(point.get("y", 0)) + + points.append(p) + + if anno.get("controlPoints"): + control_points = ET.Element("controlPoints") + + for cp in anno.get("controlPoints", []): + item = ET.Element("controlPoint") + + ET.SubElement(item, "x").text = str(cp.get("x", 0)) + ET.SubElement(item, "y").text = str(cp.get("y", 0)) + + control_points.append(item) + + result.append(control_points) + + result.append(self.create_attributes(anno.get("attributes", {}))) + result.append(points) + annotation.append(result) + + return annotation + + def create_cuboid(self, anno: dict): + annotation = ET.Element("object") + + ET.SubElement(annotation, "toolName").text = "cuboidTool" + + result = ET.Element("result") + + ET.SubElement(result, "label").text = str(anno.get("label", "")) + ET.SubElement(result, "order").text = str(anno.get("order", 0)) + ET.SubElement(result, "direction").text = str(anno.get("direction", "front")) + + front = ET.Element("front") + + for pos, point in anno.get("front", {}).items(): + position = ET.Element(pos) + ET.SubElement(position, "x").text = str(point.get("x", 0)) + ET.SubElement(position, "y").text = str(point.get("y", 0)) + front.append(position) + + back = ET.Element("back") + for pos, point in anno.get("back", {}).items(): + position = ET.Element(pos) + ET.SubElement(position, "x").text = str(point.get("x", 0)) + ET.SubElement(position, "y").text = str(point.get("y", 0)) + back.append(position) + + result.append(self.create_attributes(anno.get("attributes", {}))) + result.append(front) + result.append(back) + + annotation.append(result) + + return annotation + + def create_tag(self, anno: dict): + annotation = ET.Element("object") + result = ET.SubElement(annotation, "result") + ET.SubElement(annotation, "toolName").text = "tagTool" + + + ET.SubElement(result, "order").text = str(anno.get("order", 0)) + + ET.SubElement(result, "id").text = anno.get("id", "") + + values = ET.SubElement(result, "values") + + for key, value in anno.get("value", {}).items(): + ET.SubElement(values, key).text = ", ".join(value) if value else "" + + return annotation + + def create_text(self, anno: dict): + annotation = ET.Element("object") + + result = ET.SubElement(annotation, "result") + ET.SubElement(annotation, "toolName").text = "textTool" + + ET.SubElement(result, "order").text = str(anno.get("order", 0)) + ET.SubElement(result, "id").text = anno.get("id", "") + + values = ET.SubElement(result, "values") + + for key, value in anno.get("value", {}).items(): + ET.SubElement(values, key).text = value + + return annotation + \ No newline at end of file diff --git a/labelu/internal/domain/models/pre_annotation.py b/labelu/internal/domain/models/pre_annotation.py index c77ddf9a..c0444698 100644 --- a/labelu/internal/domain/models/pre_annotation.py +++ b/labelu/internal/domain/models/pre_annotation.py @@ -9,9 +9,12 @@ class TaskPreAnnotation(Base): __tablename__ = "task_pre_annotation" + """Task pre-annotations for a sample of a task. + """ id = Column(Integer, primary_key=True, autoincrement=True, index=True) - task_id = Column(Integer, ForeignKey("task.id"), index=True) + task_id = Column(Integer, ForeignKey("task.id"), index=True, comment="Task ID") + sample_name = Column(String(255), index=True, comment="One of the sample names of the task") file_id = Column(Integer, ForeignKey("task_attachment.id"), index=True) created_by = Column(Integer, ForeignKey("user.id"), index=True) updated_by = Column(Integer, ForeignKey("user.id")) @@ -25,12 +28,13 @@ class TaskPreAnnotation(Base): comment="Last time a task pre annotation result was updated", ) data = Column(Text, comment="task sample pre annotation result") - deleted_at = Column(DateTime, index=True, comment="Task delete time") + deleted_at = Column(DateTime, index=True, comment="Task pre-annotation delete time") file = relationship("TaskAttachment", foreign_keys=[file_id]) task = relationship("Task", foreign_keys=[task_id]) owner = relationship("User", foreign_keys=[created_by]) updater = relationship("User", foreign_keys=[updated_by]) + Index("idx_pre_annotation_sample_name", sample_name) Index("idx_pre_annotation_id_deleted_at", id, deleted_at) diff --git a/labelu/main.py b/labelu/main.py index 4d460e44..518c7a40 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -103,9 +103,13 @@ def __init__(self, *args: Any, **kwargs: Any): def file_response(self, *args: Any, **kwargs: Any) -> Response: resp = super().file_response(*args, **kwargs) - resp.headers.setdefault("Cache-Control", self.cachecontrol) - resp.headers.setdefault("Pragma", self.pragma) - resp.headers.setdefault("Expires", self.expires) + + # No cache for html files + if resp.media_type == "text/html": + resp.headers.setdefault("Cache-Control", self.cachecontrol) + resp.headers.setdefault("Pragma", self.pragma) + resp.headers.setdefault("Expires", self.expires) + return resp app.mount("", NoCacheStaticFiles(packages=["labelu.internal"], html=True)) diff --git a/labelu/tests/internal/adapter/routers/test_attachment.py b/labelu/tests/internal/adapter/routers/test_attachment.py index 1f19f907..0f7cdef9 100644 --- a/labelu/tests/internal/adapter/routers/test_attachment.py +++ b/labelu/tests/internal/adapter/routers/test_attachment.py @@ -8,6 +8,7 @@ from labelu.internal.adapter.persistence import crud_user from labelu.internal.adapter.persistence import crud_task from labelu.internal.adapter.persistence import crud_attachment +from labelu.tests.utils.utils import empty_task_upload class TestClassTaskAttachmentRouter: @@ -21,10 +22,13 @@ def test_upload_file_successful( "description": "task description", "tips": "task tips", } + task = client.post( f"{settings.API_V1_STR}/tasks", headers=testuser_token_headers, json=data ) task_id = task.json()["data"]["id"] + + empty_task_upload(task_id, "test.png") # run with Path("labelu/tests/data/test.png").open(mode="rb") as f: @@ -39,8 +43,10 @@ def test_upload_file_successful( parts = json["data"]["url"].split("/")[-3:] assert Path(f"{settings.MEDIA_ROOT}").joinpath("/".join(parts)).exists() - parts[-1] = parts[-1][:8] + "-test-thumbnail.png" + parts[-1] = "test-thumbnail.png" assert Path(f"{settings.MEDIA_ROOT}").joinpath("/".join(parts)).exists() + + empty_task_upload(task_id, "test.jsonl") with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: jsonl_res = client.post( @@ -54,6 +60,30 @@ def test_upload_file_successful( parts = json["data"]["url"].split("/")[-3:] assert Path(f"{settings.MEDIA_ROOT}").joinpath("/".join(parts)).exists() + + def test_file_exists( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + # prepare data + data = { + "name": "task name", + "description": "task description", + "tips": "task tips", + } + task = client.post( + f"{settings.API_V1_STR}/tasks", headers=testuser_token_headers, json=data + ) + task_id = task.json()["data"]["id"] + + # upload file + with Path("labelu/tests/data/test.png").open(mode="rb") as f: + attachment = client.post( + f"{settings.API_V1_STR}/tasks/{task_id}/attachments", + headers=testuser_token_headers, + files={"file": f}, + ) + + assert attachment.status_code == 400 def test_upload_file_when_task_finished( self, client: TestClient, testuser_token_headers: dict, db: Session @@ -72,6 +102,7 @@ def test_upload_file_when_task_finished( ), ) + empty_task_upload(task.id, "test.png") # run with Path("labelu/tests/data/test.png").open(mode="rb") as f: new_res = client.post( @@ -97,6 +128,8 @@ def test_upload_file_not_image( f"{settings.API_V1_STR}/tasks", headers=testuser_token_headers, json=data ) task_id = task.json()["data"]["id"] + + empty_task_upload(task_id, "test.txt") # run with Path("labelu/tests/data/test.txt").open(mode="rb") as f: @@ -147,6 +180,8 @@ def test_download_file_successful( f"{settings.API_V1_STR}/tasks", headers=testuser_token_headers, json=data ) task_id = task.json()["data"]["id"] + + empty_task_upload(task_id, "test.png") # upload file with Path("labelu/tests/data/test.png").open(mode="rb") as f: @@ -209,6 +244,9 @@ def test_task_delete_task_file( updated_by=current_user.id, ), ) + + empty_task_upload(task.id, "test.png") + # upload file with Path("labelu/tests/data/test.png").open(mode="rb") as f: attachment = client.post( diff --git a/labelu/tests/internal/adapter/routers/test_pre_annotation.py b/labelu/tests/internal/adapter/routers/test_pre_annotation.py index bd7701d3..a2117986 100644 --- a/labelu/tests/internal/adapter/routers/test_pre_annotation.py +++ b/labelu/tests/internal/adapter/routers/test_pre_annotation.py @@ -12,6 +12,7 @@ from labelu.internal.domain.models.attachment import TaskAttachment from labelu.internal.domain.models.pre_annotation import TaskPreAnnotation from labelu.internal.domain.models.sample import TaskSample +from labelu.tests.utils.utils import empty_task_upload class TestClassTaskPreAnnotationRouter: @@ -31,6 +32,7 @@ def test_create_pre_annotation_successful( ) # prepare data + empty_task_upload(task.id, "test.jsonl") with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: jsonl = client.post( f"{settings.API_V1_STR}/tasks/{task.id}/attachments", @@ -66,6 +68,8 @@ def test_create_pre_annotation_sample_exists( ), ) + empty_task_upload(task.id, "test.jsonl") + # prepare data with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: res = client.post( @@ -83,6 +87,8 @@ def test_create_pre_annotation_sample_exists( json=data, ) + empty_task_upload(task.id, "test.jsonl") + with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: res = client.post( f"{settings.API_V1_STR}/tasks/{task.id}/attachments", @@ -139,6 +145,8 @@ def test_pre_annotation_list_by_page( ), ) + empty_task_upload(task.id, "test.jsonl") + with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: jsonl = client.post( f"{settings.API_V1_STR}/tasks/{task.id}/attachments", @@ -194,6 +202,8 @@ def test_pre_annotation_list_with_sample_name( ), ) + empty_task_upload(task.id, "test.png") + empty_task_upload(task.id, "test.jsonl") # upload sample file with Path("labelu/tests/data/test.png").open(mode="rb") as f: img = client.post( @@ -227,6 +237,7 @@ def test_pre_annotation_list_with_sample_name( TaskPreAnnotation( task_id=task.id, file_id=jsonl.json()["data"]["id"], + sample_name=img.json()["data"]["filename"], created_by=current_user.id, updated_by=current_user.id, ) @@ -267,7 +278,7 @@ def test_pre_annotation_list_with_sample_name_not_found( updated_by=0, ), ) - + empty_task_upload(task.id, "test.png") # upload sample file with Path("labelu/tests/data/test.png").open(mode="rb") as f: img = client.post( @@ -276,6 +287,7 @@ def test_pre_annotation_list_with_sample_name_not_found( files={"file": f}, ) # upload jsonl file + empty_task_upload(task.id, "test.jsonl") with Path("labelu/tests/data/test.jsonl").open(mode="rb") as f: jsonl = client.post( f"{settings.API_V1_STR}/tasks/{task.id}/attachments", diff --git a/labelu/tests/utils/utils.py b/labelu/tests/utils/utils.py index caf2b9f0..b2c90565 100644 --- a/labelu/tests/utils/utils.py +++ b/labelu/tests/utils/utils.py @@ -1,6 +1,7 @@ +from pathlib import Path import random +from labelu.internal.common.config import settings import string -from typing import Dict def random_lower_string() -> str: @@ -9,3 +10,24 @@ def random_lower_string() -> str: def random_username() -> str: return f"{random_lower_string()}@{random_lower_string()}.com" + +def empty_task_upload(task_id: int, input_filename: str): + path_filename = input_filename.split("/") + # filename = str(uuid.uuid4())[0:8] + "-" + path_filename[-1] + filename = path_filename[-1] + path = "/".join(path_filename[:-1]) + # delete task 1 folder + attachment_relative_base_dir = Path(settings.UPLOAD_DIR).joinpath( + str(task_id), path + ) + attachment_relative_path = str(attachment_relative_base_dir.joinpath(filename)) + + attachment_full_path = Path(settings.MEDIA_ROOT).joinpath( + attachment_relative_path + ) + + # remove folder + if attachment_full_path.exists(): + attachment_full_path.unlink() + + return attachment_full_path \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6a15336a..5d387e1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.0.9-alpha.1' +version = '1.1.3' description = "" license = "Apache-2.0" authors = ["shenguanlin "] @@ -28,6 +28,7 @@ aiofiles = "^22.1.0" pillow = "^9.3.0" alembic = "^1.9.4" httpx = "^0.27.0" +tfrecord = "^1.14.5" [[tool.poetry.source]] From 1bc7b2196c6bdcb3c3df0702eef55a410e31e0b2 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 7 Jan 2025 16:16:55 +0800 Subject: [PATCH 074/236] fix: delete pre-annotation records after pre-annotation files deleted #159 --- .../internal/application/service/pre_annotation.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/labelu/internal/application/service/pre_annotation.py b/labelu/internal/application/service/pre_annotation.py index 76fec6c4..76dc860e 100644 --- a/labelu/internal/application/service/pre_annotation.py +++ b/labelu/internal/application/service/pre_annotation.py @@ -1,4 +1,6 @@ import json +import os +from pathlib import Path from typing import List, Tuple, Optional from loguru import logger @@ -213,10 +215,20 @@ async def delete_pre_annotation_file( db: Session, task_id: int, file_id: int, current_user: User ) -> CommonDataResp: with db.begin(): + attachments = crud_attachment.get_by_ids( + db=db, attachment_ids=[file_id] + ) + + for attachment in attachments: + file_full_path = Path(settings.MEDIA_ROOT).joinpath(attachment.path) + os.remove(file_full_path) + pre_annotations = crud_pre_annotation.list_by_task_id_and_file_id(db=db, task_id=task_id, owner_id=current_user.id, file_id=file_id) pre_annotation_ids = [pre_annotation.id for pre_annotation in pre_annotations] + crud_pre_annotation.delete(db=db, pre_annotation_ids=pre_annotation_ids) crud_attachment.delete(db=db, attachment_ids=[file_id]) + # response return CommonDataResp(ok=True) From be7db25faa629b723b7bd1a584f474badd2c3a7f Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 7 Jan 2025 18:34:13 +0800 Subject: [PATCH 075/236] fix: optimize sample_names query --- .../application/service/pre_annotation.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/labelu/internal/application/service/pre_annotation.py b/labelu/internal/application/service/pre_annotation.py index 76dc860e..bf5642a1 100644 --- a/labelu/internal/application/service/pre_annotation.py +++ b/labelu/internal/application/service/pre_annotation.py @@ -152,22 +152,19 @@ async def list_pre_annotation_files( file_ids = [pre_annotation.file_id for pre_annotation in pre_annotations] attachments, total = crud_attachment.list_by(db=db, ids=file_ids, after=after, before=before, pageNo=pageNo, pageSize=pageSize, sorting=sorting) - _attachment_ids = [attachment.id for attachment in attachments] - def get_sample_names(): - _names = [] - for pre_annotation in pre_annotations: - if pre_annotation.file_id in _attachment_ids and pre_annotation.sample_name is not None: - _names.append(pre_annotation.sample_name) - - return _names + sample_names_those_has_pre_annotations = [] + + for pre_annotation in pre_annotations: + if pre_annotation.file_id in _attachment_ids and pre_annotation.sample_name is not None: + sample_names_those_has_pre_annotations.append(pre_annotation.sample_name) return [ PreAnnotationFileResponse( id=attachment.id, url=attachment.url, filename=attachment.filename, - sample_names=get_sample_names(), + sample_names=sample_names_those_has_pre_annotations, ) for attachment in attachments ], total From 02a1aabeadbc312b991995d48c8d713c7f3bd84d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 7 Jan 2025 10:47:16 +0000 Subject: [PATCH 076/236] chore(release): 1.1.0-alpha.23 [skip ci] # [1.1.0-alpha.23](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.22...v1.1.0-alpha.23) (2025-01-07) ### Bug Fixes * delete pre-annotation records after pre-annotation files deleted [#159](https://github.com/opendatalab/labelU/issues/159) ([1bc7b21](https://github.com/opendatalab/labelU/commit/1bc7b2196c6bdcb3c3df0702eef55a410e31e0b2)) * optimize sample_names query ([be7db25](https://github.com/opendatalab/labelU/commit/be7db25faa629b723b7bd1a584f474badd2c3a7f)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5d387e1b..0bb8b9db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.3' +version = '1.1.0-alpha.23' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 1c001500ec4919f325e094a7bb8c98d7e99e1172 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 7 Jan 2025 11:11:38 +0000 Subject: [PATCH 077/236] fix: Upgrade frontend to v5.7.0-alpha.1. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.1) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 6e913bea..518f33a7 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.3.0-alpha.7/frontend.zip -alpha_version: v5.3.0-alpha.7 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.1/frontend.zip +alpha_version: v5.7.0-alpha.1 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.1/frontend.zip release_version: v5.6.1 From cdf6e6b4e4cef201ba74187b06d9f90092dfd350 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 7 Jan 2025 11:18:45 +0000 Subject: [PATCH 078/236] chore(release): 1.1.0-alpha.24 [skip ci] # [1.1.0-alpha.24](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.23...v1.1.0-alpha.24) (2025-01-07) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.1. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.1) ([1c00150](https://github.com/opendatalab/labelU/commit/1c001500ec4919f325e094a7bb8c98d7e99e1172)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0bb8b9db..2bb9c18f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.23' +version = '1.1.0-alpha.24' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 8606988f00e34f489252b16d8f89a514b6a79905 Mon Sep 17 00:00:00 2001 From: gary Date: Wed, 8 Jan 2025 12:49:19 +0800 Subject: [PATCH 079/236] fix: add is_pre_annotated field to the smaple response --- .../persistence/crud_pre_annotation.py | 14 +-- .../internal/application/response/sample.py | 3 + labelu/internal/application/service/sample.py | 89 ++++++++++++------- pyproject.toml | 2 +- 4 files changed, 66 insertions(+), 42 deletions(-) diff --git a/labelu/internal/adapter/persistence/crud_pre_annotation.py b/labelu/internal/adapter/persistence/crud_pre_annotation.py index 7b3ddcb6..ff7fa4c6 100644 --- a/labelu/internal/adapter/persistence/crud_pre_annotation.py +++ b/labelu/internal/adapter/persistence/crud_pre_annotation.py @@ -15,14 +15,14 @@ def batch(db: Session, pre_annotations: List[TaskPreAnnotation]) -> List[TaskPre def list_by( db: Session, - task_id: Union[int, None], owner_id: int, - sample_name: str | None, - after: Union[int, None], - before: Union[int, None], - pageNo: Union[int, None], - pageSize: int, - sorting: Union[str, None], + task_id: int | None = None, + sample_name: str | None = None, + after: int | None = None, + before: int | None = None, + pageNo: int | None = None, + sorting: str | None = None, + pageSize: int | None = 10, ) -> Tuple[List[TaskPreAnnotation], int]: # query filter diff --git a/labelu/internal/application/response/sample.py b/labelu/internal/application/response/sample.py index 63def954..391bafb2 100644 --- a/labelu/internal/application/response/sample.py +++ b/labelu/internal/application/response/sample.py @@ -26,6 +26,9 @@ class SampleResponse(BaseModel): file: Union[object, None] = Field( default=None, description="description: media attachment file" ) + is_pre_annotated: Union[bool, None] = Field( + default=False, description="description: is pre annotated" + ) annotated_count: Union[int, None] = Field( default=0, description="description: annotate result count" ) diff --git a/labelu/internal/application/service/sample.py b/labelu/internal/application/service/sample.py index 65df5195..695d9438 100644 --- a/labelu/internal/application/service/sample.py +++ b/labelu/internal/application/service/sample.py @@ -12,8 +12,9 @@ from labelu.internal.common.converter import converter from labelu.internal.common.error_code import ErrorCode from labelu.internal.common.error_code import LabelUException -from labelu.internal.adapter.persistence import crud_task +from labelu.internal.adapter.persistence import crud_pre_annotation, crud_task from labelu.internal.adapter.persistence import crud_sample +from labelu.internal.domain.models.pre_annotation import TaskPreAnnotation from labelu.internal.domain.models.user import User from labelu.internal.domain.models.task import Task from labelu.internal.domain.models.task import TaskStatus @@ -28,6 +29,17 @@ from labelu.internal.application.response.sample import SampleResponse from labelu.internal.application.response.attachment import AttachmentResponse +def is_sample_pre_annotated(db: Session, task_id: int, current_user: User, sample_name: str = None) -> Tuple[List[TaskPreAnnotation], int]: + _, total = crud_pre_annotation.list_by( + db=db, + task_id=task_id, + owner_id=current_user.id, + sample_name=sample_name, + pageSize=1, + ) + + return total > 0 + async def create( db: Session, task_id: int, cmd: List[CreateSampleCommand], current_user: User ) -> CreateSampleResponse: @@ -74,42 +86,49 @@ async def list_by( sorting: Union[str, None], current_user: User, ) -> Tuple[List[SampleResponse], int]: + try: + samples = crud_sample.list_by( + db=db, + task_id=task_id, + owner_id=current_user.id, + after=after, + before=before, + pageNo=pageNo, + pageSize=pageSize, + sorting=sorting, + ) - samples = crud_sample.list_by( - db=db, - task_id=task_id, - owner_id=current_user.id, - after=after, - before=before, - pageNo=pageNo, - pageSize=pageSize, - sorting=sorting, - ) - - total = crud_sample.count(db=db, task_id=task_id, owner_id=current_user.id) + total = crud_sample.count(db=db, task_id=task_id, owner_id=current_user.id) - # response - return [ - SampleResponse( - id=sample.id, - inner_id=sample.inner_id, - state=sample.state, - data=json.loads(sample.data), - annotated_count=sample.annotated_count, - file=AttachmentResponse(id=sample.file.id, filename=sample.file.filename, url=sample.file.url) if sample.file else None, - created_at=sample.created_at, - created_by=UserResp( - id=sample.owner.id, - username=sample.owner.username, - ), - updated_at=sample.updated_at, - updated_by=UserResp( - id=sample.updater.id, - username=sample.updater.username, - ), + # response + return [ + SampleResponse( + id=sample.id, + inner_id=sample.inner_id, + state=sample.state, + data=json.loads(sample.data), + annotated_count=sample.annotated_count, + is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename), + file=AttachmentResponse(id=sample.file.id, filename=sample.file.filename, url=sample.file.url) if sample.file else None, + created_at=sample.created_at, + created_by=UserResp( + id=sample.owner.id, + username=sample.owner.username, + ), + updated_at=sample.updated_at, + updated_by=UserResp( + id=sample.updater.id, + username=sample.updater.username, + ), + ) + for sample in samples + ], total + except Exception as e: + logger.error(e) + raise LabelUException( + code=ErrorCode.CODE_55000_SAMPLE_LIST_PARAMETERS_ERROR, + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, ) - for sample in samples - ], total async def get( @@ -133,6 +152,7 @@ async def get( inner_id=sample.inner_id, state=sample.state, data=json.loads(sample.data), + is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename), file=AttachmentResponse(id=sample.file.id, filename=sample.file.filename, url=sample.file.url) if sample.file else None, annotated_count=sample.annotated_count, created_at=sample.created_at, @@ -210,6 +230,7 @@ async def patch( inner_id=updated_sample.inner_id, state=updated_sample.state, data=json.loads(updated_sample.data), + is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename), annotated_count=updated_sample.annotated_count, created_at=updated_sample.created_at, created_by=UserResp( diff --git a/pyproject.toml b/pyproject.toml index 2bb9c18f..0f3b469a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.24' +version = '1.1.4' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From ee8fcd1c2b5de08725f3b883450ac000742bf7bd Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 8 Jan 2025 04:53:14 +0000 Subject: [PATCH 080/236] fix: Upgrade frontend to v5.7.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.2) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 518f33a7..b91683c6 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.1/frontend.zip -alpha_version: v5.7.0-alpha.1 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.2/frontend.zip +alpha_version: v5.7.0-alpha.2 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.1/frontend.zip release_version: v5.6.1 From 1f9002b619bf2608227afddaf533af12b06fc59a Mon Sep 17 00:00:00 2001 From: gary Date: Wed, 8 Jan 2025 13:02:29 +0800 Subject: [PATCH 081/236] update: add is_pre_annotated field to the sample response * chore: update scripts (#130) * chore: update workflow (#128) * fix: fix pre annoation exceptions (#120) * chore(release): 1.0.7 [skip ci] ## [1.0.7](https://github.com/opendatalab/labelU/compare/v1.0.6...v1.0.7) (2024-08-26) ### Bug Fixes * fix pre annoation exceptions ([#120](https://github.com/opendatalab/labelU/issues/120)) ([bd61682](https://github.com/opendatalab/labelU/commit/bd616821eb6dea47840df29efe340c3b96c2446f)) * docs: Update README.md * docs: Update README.md * docs: Update README_zh-CN.md (#126) * docs: Update README_zh-CN.md * docs: Update README_zh-CN.md * chore: update workflow --------- Co-authored-by: semantic-release-bot * fix: Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) * chore(release): 1.0.7-alpha.2 [skip ci] ## [1.0.7-alpha.2](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.1...v1.0.7-alpha.2) (2024-10-25) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.3) ([e8721d6](https://github.com/opendatalab/labelU/commit/e8721d6ab4fdb288ef16263ffb09c73ea12ed52f)) * Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) ([01fc3b5](https://github.com/opendatalab/labelU/commit/01fc3b53b570de4d05ad771f163d8da03236b141)) * fix: Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) * chore(release): 1.0.7-alpha.3 [skip ci] ## [1.0.7-alpha.3](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.2...v1.0.7-alpha.3) (2024-10-25) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) ([f97601d](https://github.com/opendatalab/labelU/commit/f97601d0c4286c633ca2ce3a1f5ef3d8eec5a540)) * fix: update notification action * chore(release): 1.0.7-alpha.4 [skip ci] ## [1.0.7-alpha.4](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.3...v1.0.7-alpha.4) (2024-10-28) ### Bug Fixes * update notification action ([b6abe17](https://github.com/opendatalab/labelU/commit/b6abe172f7a8bb1a5547acb2bc8948bb7c0dae7d)) * fix: update notification action * chore(release): 1.0.7-alpha.5 [skip ci] ## [1.0.7-alpha.5](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.4...v1.0.7-alpha.5) (2024-10-28) ### Bug Fixes * update notification action ([04e0388](https://github.com/opendatalab/labelU/commit/04e0388e3cd98a2aacf12dc28c638c172eb6f0ff)) * fix: update notification action * chore(release): 1.0.7-alpha.6 [skip ci] ## [1.0.7-alpha.6](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.5...v1.0.7-alpha.6) (2024-10-28) ### Bug Fixes * update notification action ([e9152c9](https://github.com/opendatalab/labelU/commit/e9152c98869d70e500fd9ab458d8ba179270d9c9)) * fix: update notification action * chore(release): 1.0.7-alpha.7 [skip ci] ## [1.0.7-alpha.7](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.6...v1.0.7-alpha.7) (2024-10-28) ### Bug Fixes * update notification action ([046f47a](https://github.com/opendatalab/labelU/commit/046f47a3f4053e6934f04c3c0ede5518b9f1850b)) * fix: test notification action * chore(release): 1.0.7-alpha.8 [skip ci] ## [1.0.7-alpha.8](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.7...v1.0.7-alpha.8) (2024-10-28) ### Bug Fixes * test notification action ([8fed078](https://github.com/opendatalab/labelU/commit/8fed078a0d77a8204430fb6c24d91de1250a8bc7)) * fix: test notification action * chore(release): 1.0.7-alpha.9 [skip ci] ## [1.0.7-alpha.9](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.8...v1.0.7-alpha.9) (2024-10-28) ### Bug Fixes * test notification action ([851818e](https://github.com/opendatalab/labelU/commit/851818e912cf39690be6e689c1926ec29c148a66)) * fix: test notification action * chore(release): 1.0.7-alpha.10 [skip ci] ## [1.0.7-alpha.10](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.9...v1.0.7-alpha.10) (2024-10-28) ### Bug Fixes * test notification action ([bc63ee5](https://github.com/opendatalab/labelU/commit/bc63ee59da0d079208b1385d33184267251a2bb5)) * fix: test notification action * chore(release): 1.0.7-alpha.11 [skip ci] ## [1.0.7-alpha.11](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.10...v1.0.7-alpha.11) (2024-10-28) ### Bug Fixes * test notification action ([83f43eb](https://github.com/opendatalab/labelU/commit/83f43eb9d03390686a41dc22666a058190bfb63c)) * fix: update notification action * chore(release): 1.0.7-alpha.12 [skip ci] ## [1.0.7-alpha.12](https://github.com/opendatalab/labelU/compare/v1.0.7-alpha.11...v1.0.7-alpha.12) (2024-10-28) ### Bug Fixes * update notification action ([63286cb](https://github.com/opendatalab/labelU/commit/63286cb6958e20ab3c80495ab3b51a1ddd31dfb7)) * chore(release): 1.0.9-alpha.1 [skip ci] ## [1.0.9-alpha.1](https://github.com/opendatalab/labelU/compare/v1.0.8...v1.0.9-alpha.1) (2024-10-28) ### Bug Fixes * test notification action ([83f43eb](https://github.com/opendatalab/labelU/commit/83f43eb9d03390686a41dc22666a058190bfb63c)) * test notification action ([bc63ee5](https://github.com/opendatalab/labelU/commit/bc63ee59da0d079208b1385d33184267251a2bb5)) * test notification action ([851818e](https://github.com/opendatalab/labelU/commit/851818e912cf39690be6e689c1926ec29c148a66)) * test notification action ([8fed078](https://github.com/opendatalab/labelU/commit/8fed078a0d77a8204430fb6c24d91de1250a8bc7)) * update notification action ([63286cb](https://github.com/opendatalab/labelU/commit/63286cb6958e20ab3c80495ab3b51a1ddd31dfb7)) * update notification action ([046f47a](https://github.com/opendatalab/labelU/commit/046f47a3f4053e6934f04c3c0ede5518b9f1850b)) * update notification action ([e9152c9](https://github.com/opendatalab/labelU/commit/e9152c98869d70e500fd9ab458d8ba179270d9c9)) * update notification action ([04e0388](https://github.com/opendatalab/labelU/commit/04e0388e3cd98a2aacf12dc28c638c172eb6f0ff)) * update notification action ([b6abe17](https://github.com/opendatalab/labelU/commit/b6abe172f7a8bb1a5547acb2bc8948bb7c0dae7d)) * Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) ([01fc3b5](https://github.com/opendatalab/labelU/commit/01fc3b53b570de4d05ad771f163d8da03236b141)) * Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) ([f97601d](https://github.com/opendatalab/labelU/commit/f97601d0c4286c633ca2ce3a1f5ef3d8eec5a540)) * fix: Upgrade frontend to v5.3.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.6) * chore(release): 1.0.9-alpha.2 [skip ci] ## [1.0.9-alpha.2](https://github.com/opendatalab/labelU/compare/v1.0.9-alpha.1...v1.0.9-alpha.2) (2024-10-28) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.6) ([9f5e698](https://github.com/opendatalab/labelU/commit/9f5e698c8a0476be0bb034bb12e226b9db211328)) * fix: Upgrade frontend to v5.3.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.7) * chore(release): 1.0.9-alpha.3 [skip ci] ## [1.0.9-alpha.3](https://github.com/opendatalab/labelU/compare/v1.0.9-alpha.2...v1.0.9-alpha.3) (2024-10-28) ### Bug Fixes * Upgrade frontend to v5.3.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.7) ([7cb455e](https://github.com/opendatalab/labelU/commit/7cb455eaf06b8408f2cacfe3cc39c8c2e3bde013)) --------- Co-authored-by: semantic-release-bot * fix: upgrade frontend to 5.4.0 #106 * chore(release): 1.0.9 [skip ci] ## [1.0.9](https://github.com/opendatalab/labelU/compare/v1.0.8...v1.0.9) (2024-10-29) ### Bug Fixes * upgrade frontend to 5.4.0 [#106](https://github.com/opendatalab/labelU/issues/106) ([33b71be](https://github.com/opendatalab/labelU/commit/33b71beb99a0e56b96ab56bb2cffe18c95624c37)) * chore: delete test script * fix: upgrade frontend to 5.4.1 * chore(release): 1.0.10 [skip ci] ## [1.0.10](https://github.com/opendatalab/labelU/compare/v1.0.9...v1.0.10) (2024-10-29) ### Bug Fixes * upgrade frontend to 5.4.1 ([73323b9](https://github.com/opendatalab/labelU/commit/73323b9b51dc1b96c85d1bf9fe7f582e4d758c51)) * docs: Create LICENSE * fix: Update .VERSION * chore(release): 1.1.0 [skip ci] # [1.1.0](https://github.com/opendatalab/labelU/compare/v1.0.10...v1.1.0) (2024-11-20) ### Bug Fixes * add folder field to the json output ([3589d22](https://github.com/opendatalab/labelU/commit/3589d22d10762d0c39b86035aab3a9fca523c449)) * fix [#69](https://github.com/opendatalab/labelU/issues/69) ([0baa13c](https://github.com/opendatalab/labelU/commit/0baa13c50aecc1144821a50b40223ba0daf52b1b)) * fix category name in pascal_voc and tfrecord format files ([b089074](https://github.com/opendatalab/labelU/commit/b08907442aadf95b3ded49c6dedab519ffedd839)) * fix file format in tfrecord ([77cba31](https://github.com/opendatalab/labelU/commit/77cba31fd50f255561d4181eed67064efd148f65)) * optimize the task_pre_annotation table and service ([d38ed7b](https://github.com/opendatalab/labelU/commit/d38ed7bf3adab1684c3109f677eb9445de50d86f)) * remove uuid prefix in attachment creation ([f0820ff](https://github.com/opendatalab/labelU/commit/f0820ff6907c4125bef17523c5df9ca05b8aa6b2)) * supports rotated image in yolo and tfrecord files ([4d5afdf](https://github.com/opendatalab/labelU/commit/4d5afdf3200d54b7ff7f2cbbae82f7c1a574f75a)) * Update .VERSION ([dae830a](https://github.com/opendatalab/labelU/commit/dae830a9bf2b968c98adc840fc3aecb4c3371e57)) * update alembic script ([97d6c08](https://github.com/opendatalab/labelU/commit/97d6c088abff6ab9e6c14c24874b0ea4e545ae00)) * update attributes in xml and pascal_voc ([0affd60](https://github.com/opendatalab/labelU/commit/0affd60cf6688bdc5d0636c34d87cf2d99b353c4)) ### Features * supports exporting "csv" format files ([d35a93a](https://github.com/opendatalab/labelU/commit/d35a93ae5053e84053d12b4dac3e24c4c7ca9644)) * supports exporting "labelme" format files ([8d4ea8e](https://github.com/opendatalab/labelU/commit/8d4ea8e6bd4dc28245de825c590bd2f963ad2984)) * supports exporting "pascal voc" format files ([4344f99](https://github.com/opendatalab/labelU/commit/4344f99799a95f278dd3e1cf008ea66d0d9d6a0f)) * supports exporting "tfrecord" format files ([f728fd6](https://github.com/opendatalab/labelU/commit/f728fd61b9f8060936ec5f68d239fcaeea4a9acd)) * supports exporting "xml" format files ([b165cce](https://github.com/opendatalab/labelU/commit/b165cce71a1f62690d33e156580f36d0ccee187f)) * supports exporting "yolo" format files ([b3d07b4](https://github.com/opendatalab/labelU/commit/b3d07b4d162fd4c7a2afc1281a96d9f2c06ec343)) * supports labelu json file as the pre-annotation file ([2b4a9bd](https://github.com/opendatalab/labelU/commit/2b4a9bd333caa85e60a838293ddbaa0b356c59ef)) * fix: wrong indent in yolo conventer Fixed an issue where multiple redundant files with the same name were generated when exporting YOLO format annotation files due to accidental indentation. * chore(release): 1.1.1 [skip ci] ## [1.1.1](https://github.com/opendatalab/labelU/compare/v1.1.0...v1.1.1) (2024-11-25) ### Bug Fixes * wrong indent in yolo conventer ([5ebf616](https://github.com/opendatalab/labelU/commit/5ebf6165e53bf7d5b7776491e5fc9d291048c2e5)) * fix: add full i18n supports * chore(release): 1.1.2 [skip ci] ## [1.1.2](https://github.com/opendatalab/labelU/compare/v1.1.1...v1.1.2) (2025-01-06) ### Bug Fixes * add full i18n supports ([fdf9b76](https://github.com/opendatalab/labelU/commit/fdf9b769a3dd84a7656addda64d29af6bca319a4)) * Update .VERSION * fix: langOptions error * chore(release): 1.1.3 [skip ci] ## [1.1.3](https://github.com/opendatalab/labelU/compare/v1.1.2...v1.1.3) (2025-01-06) ### Bug Fixes * langOptions error ([b2c9af5](https://github.com/opendatalab/labelU/commit/b2c9af55e467f233780abed3e219226faf472439)) * fix: no cache for html file * chore(release): 1.1.4 [skip ci] ## [1.1.4](https://github.com/opendatalab/labelU/compare/v1.1.3...v1.1.4) (2025-01-07) ### Bug Fixes * no cache for html file ([9258822](https://github.com/opendatalab/labelU/commit/92588223eedf6a8984788b4fb525ee02888a4dfd)) * fix: add is_pre_annotated field to the smaple response * update: add is_pre_annotated field to the smaple response --------- Co-authored-by: semantic-release-bot Co-authored-by: Yonas --- labelu/internal/application/service/sample.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/labelu/internal/application/service/sample.py b/labelu/internal/application/service/sample.py index 695d9438..77b07c98 100644 --- a/labelu/internal/application/service/sample.py +++ b/labelu/internal/application/service/sample.py @@ -29,7 +29,10 @@ from labelu.internal.application.response.sample import SampleResponse from labelu.internal.application.response.attachment import AttachmentResponse -def is_sample_pre_annotated(db: Session, task_id: int, current_user: User, sample_name: str = None) -> Tuple[List[TaskPreAnnotation], int]: +def is_sample_pre_annotated(db: Session, task_id: int, current_user: User, sample_name: str | None = None) -> Tuple[List[TaskPreAnnotation], int]: + if sample_name is None: + return False + _, total = crud_pre_annotation.list_by( db=db, task_id=task_id, @@ -108,7 +111,7 @@ async def list_by( state=sample.state, data=json.loads(sample.data), annotated_count=sample.annotated_count, - is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename), + is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename if sample.file else None), file=AttachmentResponse(id=sample.file.id, filename=sample.file.filename, url=sample.file.url) if sample.file else None, created_at=sample.created_at, created_by=UserResp( @@ -152,7 +155,7 @@ async def get( inner_id=sample.inner_id, state=sample.state, data=json.loads(sample.data), - is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename), + is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename if sample.file else None), file=AttachmentResponse(id=sample.file.id, filename=sample.file.filename, url=sample.file.url) if sample.file else None, annotated_count=sample.annotated_count, created_at=sample.created_at, @@ -230,7 +233,7 @@ async def patch( inner_id=updated_sample.inner_id, state=updated_sample.state, data=json.loads(updated_sample.data), - is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename), + is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename if sample.file else None), annotated_count=updated_sample.annotated_count, created_at=updated_sample.created_at, created_by=UserResp( From 3aae0cc9f71241ab491ee2094d0a82be051a6672 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 8 Jan 2025 05:08:41 +0000 Subject: [PATCH 082/236] chore(release): 1.1.0-alpha.25 [skip ci] # [1.1.0-alpha.25](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.24...v1.1.0-alpha.25) (2025-01-08) ### Bug Fixes * add is_pre_annotated field to the smaple response ([8606988](https://github.com/opendatalab/labelU/commit/8606988f00e34f489252b16d8f89a514b6a79905)) * Upgrade frontend to v5.7.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.2) ([ee8fcd1](https://github.com/opendatalab/labelU/commit/ee8fcd1c2b5de08725f3b883450ac000742bf7bd)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0f3b469a..ebcdff1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.4' +version = '1.1.0-alpha.25' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 582eb07457bdf827f66fa66a0cbec14a5a205327 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 9 Jan 2025 09:26:08 +0000 Subject: [PATCH 083/236] fix: Upgrade frontend to v5.7.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.3) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index b91683c6..cbbdf321 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.2/frontend.zip -alpha_version: v5.7.0-alpha.2 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.3/frontend.zip +alpha_version: v5.7.0-alpha.3 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.1/frontend.zip release_version: v5.6.1 From 4656e5c6e5c5df2814cf2a7194f9cf524c6a26a2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 9 Jan 2025 09:31:44 +0000 Subject: [PATCH 084/236] chore(release): 1.1.0-alpha.26 [skip ci] # [1.1.0-alpha.26](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.25...v1.1.0-alpha.26) (2025-01-09) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.3) ([582eb07](https://github.com/opendatalab/labelU/commit/582eb07457bdf827f66fa66a0cbec14a5a205327)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ebcdff1e..4d1d9040 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.25' +version = '1.1.0-alpha.26' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 9b5879ef82069895e04f259f117cdaaf1e64e959 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 14 Jan 2025 07:37:31 +0000 Subject: [PATCH 085/236] fix: Upgrade frontend to v5.7.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.4) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index cbbdf321..c5681f2c 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.3/frontend.zip -alpha_version: v5.7.0-alpha.3 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.4/frontend.zip +alpha_version: v5.7.0-alpha.4 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.1/frontend.zip release_version: v5.6.1 From eef80f099d96384cdad9e702dc7012dba2d06a26 Mon Sep 17 00:00:00 2001 From: gary Date: Tue, 14 Jan 2025 15:40:17 +0800 Subject: [PATCH 086/236] fix: no data to export --- .VERSION | 4 ++-- labelu/internal/application/service/sample.py | 22 ++++++------------- labelu/internal/common/converter.py | 11 ++++++++++ labelu/internal/common/error_code.py | 7 ++++++ pyproject.toml | 2 +- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/.VERSION b/.VERSION index c5681f2c..fff63bb7 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.4/frontend.zip alpha_version: v5.7.0-alpha.4 -release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.1/frontend.zip -release_version: v5.6.1 +release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.3/frontend.zip +release_version: v5.6.3 \ No newline at end of file diff --git a/labelu/internal/application/service/sample.py b/labelu/internal/application/service/sample.py index 77b07c98..fe9a0219 100644 --- a/labelu/internal/application/service/sample.py +++ b/labelu/internal/application/service/sample.py @@ -284,21 +284,13 @@ async def export( ) # converter to export_type - try: - file_full_path = converter.convert( - config=json.loads(task.config), - input_data=data, - out_data_dir=out_data_dir, - out_data_file_name_prefix=task_id, - format=export_type.value, - ) - except Exception as e: - logger.error(data) - logger.error(e) - raise LabelUException( - code=ErrorCode.CODE_55002_SAMPLE_FORMAT_ERROR, - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) + file_full_path = converter.convert( + config=json.loads(task.config), + input_data=data, + out_data_dir=out_data_dir, + out_data_file_name_prefix=task_id, + format=export_type.value, + ) # response return file_full_path diff --git a/labelu/internal/common/converter.py b/labelu/internal/common/converter.py index 4336c7e1..e818a04e 100644 --- a/labelu/internal/common/converter.py +++ b/labelu/internal/common/converter.py @@ -8,8 +8,11 @@ import xml.etree.ElementTree as ET from typing import List +from fastapi import status from loguru import logger +from labelu.internal.common.error_code import ErrorCode, LabelUException + from .xml_converter import XML_converter from .tf_record_converter import TF_record_converter from .color import colors @@ -799,8 +802,16 @@ def convert_to_tf_record(self, config: dict, input_data: List[dict], out_data_fi # result struct tf_record_examples = TF_record_converter().create_tf_examples(input_data, config) + if len(tf_record_examples) == 0: + raise LabelUException( + code=ErrorCode.CODE_61000_NO_DATA, + status_code=status.HTTP_400_BAD_REQUEST, + ) + for sample in input_data: file = sample.get("file", {}) + if len(tf_record_examples) == 0: + continue example = tf_record_examples.pop(0) file_basename = os.path.splitext(file.get("filename", ""))[0] tf_record = f"{file_basename}.tfrecord" diff --git a/labelu/internal/common/error_code.py b/labelu/internal/common/error_code.py index d47e6f5b..3b6b1f04 100644 --- a/labelu/internal/common/error_code.py +++ b/labelu/internal/common/error_code.py @@ -20,6 +20,9 @@ # task init error code TASK_INIT_CODE = 50000 +# export +EXPORT_INIT_CODE = 60000 + class ErrorCode(Enum): """ @@ -90,6 +93,10 @@ class ErrorCode(Enum): TASK_INIT_CODE + 5003, "Sample name exists", ) + CODE_61000_NO_DATA = ( + EXPORT_INIT_CODE + 1000, + "No data", + ) class LabelUException(HTTPException): diff --git a/pyproject.toml b/pyproject.toml index 4d1d9040..e6768fa5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.26' +version = '1.1.6' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 599e0a0bd56673742248f10abea38982b9fde564 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 14 Jan 2025 07:45:43 +0000 Subject: [PATCH 087/236] chore(release): 1.1.0-alpha.27 [skip ci] # [1.1.0-alpha.27](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.26...v1.1.0-alpha.27) (2025-01-14) ### Bug Fixes * no data to export ([eef80f0](https://github.com/opendatalab/labelU/commit/eef80f099d96384cdad9e702dc7012dba2d06a26)) * Upgrade frontend to v5.7.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.4) ([9b5879e](https://github.com/opendatalab/labelU/commit/9b5879ef82069895e04f259f117cdaaf1e64e959)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e6768fa5..e7a6c761 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.6' +version = '1.1.0-alpha.27' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 70f67898e82666d6c4c6a8dbfa481db3c10a2132 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 15 Jan 2025 03:25:10 +0000 Subject: [PATCH 088/236] fix: Upgrade frontend to v5.7.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.5) --- .VERSION | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.VERSION b/.VERSION index fff63bb7..1d211f81 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.4/frontend.zip -alpha_version: v5.7.0-alpha.4 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.5/frontend.zip +alpha_version: v5.7.0-alpha.5 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.3/frontend.zip -release_version: v5.6.3 \ No newline at end of file +release_version: v5.6.3 From 57ee4cbd3f3b8101e2674d407619f029dae4b8ae Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 15 Jan 2025 03:30:40 +0000 Subject: [PATCH 089/236] chore(release): 1.1.0-alpha.28 [skip ci] # [1.1.0-alpha.28](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.27...v1.1.0-alpha.28) (2025-01-15) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.5) ([70f6789](https://github.com/opendatalab/labelU/commit/70f67898e82666d6c4c6a8dbfa481db3c10a2132)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e7a6c761..d5b2577d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.27' +version = '1.1.0-alpha.28' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From d57db71df1e98553a05325d1ee1f6f5e75edefe4 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 16 Jan 2025 02:57:42 +0000 Subject: [PATCH 090/236] fix: Upgrade frontend to v5.7.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.6) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 1d211f81..e9882fee 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.5/frontend.zip -alpha_version: v5.7.0-alpha.5 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.6/frontend.zip +alpha_version: v5.7.0-alpha.6 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.3/frontend.zip release_version: v5.6.3 From a00b3b96f06d99b26005e9a6ea606aa8f574d1a2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 16 Jan 2025 03:03:13 +0000 Subject: [PATCH 091/236] chore(release): 1.1.0-alpha.29 [skip ci] # [1.1.0-alpha.29](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.28...v1.1.0-alpha.29) (2025-01-16) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.6) ([d57db71](https://github.com/opendatalab/labelU/commit/d57db71df1e98553a05325d1ee1f6f5e75edefe4)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d5b2577d..0606fbcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.28' +version = '1.1.0-alpha.29' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From b9b509b402f521d4a586fde001f4d98dacf50fca Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 11 Feb 2025 08:57:20 +0000 Subject: [PATCH 092/236] fix: Upgrade frontend to v5.7.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.7) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index e9882fee..c3f69d5f 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.6/frontend.zip -alpha_version: v5.7.0-alpha.6 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.7/frontend.zip +alpha_version: v5.7.0-alpha.7 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.3/frontend.zip release_version: v5.6.3 From c3a4da55758af349f2d9fa1da7106fdf8ae93bea Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 11 Feb 2025 09:02:27 +0000 Subject: [PATCH 093/236] chore(release): 1.1.0-alpha.30 [skip ci] # [1.1.0-alpha.30](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.29...v1.1.0-alpha.30) (2025-02-11) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.7) ([b9b509b](https://github.com/opendatalab/labelU/commit/b9b509b402f521d4a586fde001f4d98dacf50fca)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0606fbcb..a5dba878 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.29' +version = '1.1.0-alpha.30' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 4255b53837d4da446374e02cade16b28a3670172 Mon Sep 17 00:00:00 2001 From: shenguanlin Date: Wed, 12 Feb 2025 10:00:08 +0800 Subject: [PATCH 094/236] fix: delete attachment when delete sample --- labelu/internal/application/service/sample.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/labelu/internal/application/service/sample.py b/labelu/internal/application/service/sample.py index fe9a0219..2d5e3859 100644 --- a/labelu/internal/application/service/sample.py +++ b/labelu/internal/application/service/sample.py @@ -1,4 +1,5 @@ import json +import os import uuid from datetime import datetime from typing import List, Tuple, Union @@ -12,7 +13,7 @@ from labelu.internal.common.converter import converter from labelu.internal.common.error_code import ErrorCode from labelu.internal.common.error_code import LabelUException -from labelu.internal.adapter.persistence import crud_pre_annotation, crud_task +from labelu.internal.adapter.persistence import crud_attachment, crud_pre_annotation, crud_task from labelu.internal.adapter.persistence import crud_sample from labelu.internal.domain.models.pre_annotation import TaskPreAnnotation from labelu.internal.domain.models.user import User @@ -253,6 +254,18 @@ async def delete( ) -> CommonDataResp: with db.begin(): + # delete media + samples = crud_sample.get_by_ids(db=db, sample_ids=sample_ids) + attachment_ids = [sample.file_id for sample in samples if sample.file_id] + attachments = crud_attachment.get_by_ids(db=db, attachment_ids=attachment_ids) + + attachments = crud_attachment.get_by_ids( + db=db, attachment_ids=attachment_ids + ) + for attachment in attachments: + file_full_path = Path(settings.MEDIA_ROOT).joinpath(attachment.path) + os.remove(file_full_path) + crud_sample.delete(db=db, sample_ids=sample_ids) # response return CommonDataResp(ok=True) From 55bc1f7a87074f3cdcdf6de7fa644ffee8e6e33a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 12 Feb 2025 02:05:34 +0000 Subject: [PATCH 095/236] chore(release): 1.1.0-alpha.31 [skip ci] # [1.1.0-alpha.31](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.30...v1.1.0-alpha.31) (2025-02-12) ### Bug Fixes * delete attachment when delete sample ([4255b53](https://github.com/opendatalab/labelU/commit/4255b53837d4da446374e02cade16b28a3670172)) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a5dba878..cb538b05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.30' +version = '1.1.0-alpha.31' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 576333d2db2ba61a3ba37f0667dc4d191ede46d0 Mon Sep 17 00:00:00 2001 From: gary Date: Wed, 12 Feb 2025 16:13:16 +0800 Subject: [PATCH 096/236] fix: set current version on http response headers (#183) --- .VERSION | 8 ++++---- .github/workflows/release.yml | 1 + labelu/internal/application/service/sample.py | 2 +- labelu/internal/common/config.py | 20 +++++++++---------- labelu/internal/common/converter.py | 5 +++-- labelu/main.py | 7 +++++-- labelu/version.py | 1 + pyproject.toml | 8 +------- 8 files changed, 26 insertions(+), 26 deletions(-) create mode 100644 labelu/version.py diff --git a/.VERSION b/.VERSION index c3f69d5f..f2f6043e 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.7/frontend.zip -alpha_version: v5.7.0-alpha.7 -release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.3/frontend.zip -release_version: v5.6.3 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.4/frontend.zip +alpha_version: v5.7.0-alpha.4 +release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.10/frontend.zip +release_version: v5.6.10 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5934d2da..b473746f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -152,6 +152,7 @@ jobs: - name: Manage version run: | sed -i "s/^version[ ]*=.*/version = '${NEXT_VERSION}'/" pyproject.toml + sed -i "s/^version[ ]*=.*/version='${NEXT_VERSION}'/" labelu/version.py - name: Publish to TestPyPi if: ${{ github.ref_name == 'alpha'}} diff --git a/labelu/internal/application/service/sample.py b/labelu/internal/application/service/sample.py index 2d5e3859..ae5f374b 100644 --- a/labelu/internal/application/service/sample.py +++ b/labelu/internal/application/service/sample.py @@ -306,4 +306,4 @@ async def export( ) # response - return file_full_path + return file_full_path \ No newline at end of file diff --git a/labelu/internal/common/config.py b/labelu/internal/common/config.py index 24bc509c..90b03a9a 100644 --- a/labelu/internal/common/config.py +++ b/labelu/internal/common/config.py @@ -14,24 +14,24 @@ class Settings(BaseSettings): API_V1_STR: str = "/api/v1" MEDIA_HOST: str = f"{SCHEME}://{HOST}:{PORT}" - BASE_DATA_DIR = get_data_dir() - MEDIA_ROOT = Path(BASE_DATA_DIR).joinpath("media") - UPLOAD_DIR = "upload" - EXPORT_DIR = "export" + BASE_DATA_DIR: str = get_data_dir() + MEDIA_ROOT: Path = Path(BASE_DATA_DIR).joinpath("media") + UPLOAD_DIR: str = "upload" + EXPORT_DIR: str = "export" os.makedirs(MEDIA_ROOT, exist_ok=True) logger.info("Database and media directory: {}", BASE_DATA_DIR) - UPLOAD_FILE_MAX_SIZE = 200_000_000 # ~200MB - THUMBNAIL_HEIGH_PIXEL = 120 + UPLOAD_FILE_MAX_SIZE: int = 200_000_000 # ~200MB + THUMBNAIL_HEIGH_PIXEL: int = 120 DATABASE_URL: str = f"sqlite:///{BASE_DATA_DIR}/labelu.sqlite" - PASSWORD_SECRET_KEY = ( + PASSWORD_SECRET_KEY: str = ( "e5b7d00a59aaa2a5ea86a7c4d72f856b20bafa1b8d0e66124082ada81f6340bd" ) - TOKEN_GENERATE_ALGORITHM = "HS256" - TOKEN_ACCESS_EXPIRE_MINUTES = 30 - TOKEN_TYPE = "Bearer" + TOKEN_GENERATE_ALGORITHM: str = "HS256" + TOKEN_ACCESS_EXPIRE_MINUTES: int = 30 + TOKEN_TYPE: str = "Bearer" class Config: env_prefix = "" diff --git a/labelu/internal/common/converter.py b/labelu/internal/common/converter.py index e818a04e..f365e4e7 100644 --- a/labelu/internal/common/converter.py +++ b/labelu/internal/common/converter.py @@ -757,11 +757,11 @@ def convert_to_xml(self, config: dict, input_data: List[dict], out_data_file_nam # result struct xml_converter = XML_converter() root = ET.Element("root") - sample_item = ET.SubElement(root, "sample") for sample in input_data: data = json.loads(sample.get("data")) file = sample.get("file", {}) + sample_item = ET.SubElement(root, "sample") # skip invalid data annotated_result = json.loads(data.get("result")) @@ -791,7 +791,8 @@ def convert_to_xml(self, config: dict, input_data: List[dict], out_data_file_nam annotations.append(annotation) tree = ET.ElementTree(root) - tree.write(file_full_path, encoding="utf-8", xml_declaration=True) + with open(file_full_path, 'wb') as f: + tree.write(f, encoding='utf-8', xml_declaration=True) logger.info("Export file path: {}", file_full_path) return file_full_path diff --git a/labelu/main.py b/labelu/main.py index 518c7a40..8137f0ea 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -12,6 +12,8 @@ from labelu.internal.common.error_code import add_exception_handler from labelu.alembic_labelu.run_migrate import run_sqlite_migrations +from .version import version as labelu_version + description = """ labelU backend. @@ -69,11 +71,10 @@ }, ] - app = FastAPI( title="labelU", description=description, - version="0.1.0", + version=labelu_version, terms_of_service="", contact={ "name": "labelu", @@ -119,6 +120,8 @@ def file_response(self, *args: Any, **kwargs: Any) -> Response: async def add_correct_content_type(request: Request, call_next): response = await call_next(request) + response.headers["LabelU-Version"] = labelu_version + if request.url.path.endswith(".js"): response.headers["content-type"] = "application/javascript" return response diff --git a/labelu/version.py b/labelu/version.py new file mode 100644 index 00000000..b12f6382 --- /dev/null +++ b/labelu/version.py @@ -0,0 +1 @@ +version='1.1.5' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cb538b05..e0fddcab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.31' +version = '1.1.14' description = "" license = "Apache-2.0" authors = ["shenguanlin "] @@ -31,12 +31,6 @@ httpx = "^0.27.0" tfrecord = "^1.14.5" -[[tool.poetry.source]] -name = "testpypi" -url = "https://test.pypi.org/legacy/" -default = false -secondary = false - [tool.poetry.group.dev.dependencies] black = "^22.10.0" flake8 = "^5.0.4" From 086a179889864c3ac6f3194f73acc363066c1048 Mon Sep 17 00:00:00 2001 From: gary Date: Wed, 12 Feb 2025 16:20:24 +0800 Subject: [PATCH 097/236] Fix/version (#184) From a2d3cda77cb2c1479b6f1913e09165a44afe3ae1 Mon Sep 17 00:00:00 2001 From: gary Date: Wed, 12 Feb 2025 16:24:44 +0800 Subject: [PATCH 098/236] update: poetry config (#185) --- .releaserc.json | 2 +- pyproject.toml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.releaserc.json b/.releaserc.json index 282c0014..bd14ba7f 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -20,7 +20,7 @@ [ "@semantic-release/git", { - "assets": ["pyproject.toml"], + "assets": ["pyproject.toml", "labelu/version.py"], "message": "chore(release): ${nextRelease.version} [skip ci] \n\n${nextRelease.notes}" } ], diff --git a/pyproject.toml b/pyproject.toml index e0fddcab..2fd94c27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,12 @@ alembic = "^1.9.4" httpx = "^0.27.0" tfrecord = "^1.14.5" +[[tool.poetry.source]] +name = "testpypi" +url = "https://test.pypi.org/legacy/" +default = false +secondary = false + [tool.poetry.group.dev.dependencies] black = "^22.10.0" From 1c0f590542d2c5139af88c7fc17474dd2a4567db Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 12 Feb 2025 08:29:43 +0000 Subject: [PATCH 099/236] chore(release): 1.1.0-alpha.32 [skip ci] # [1.1.0-alpha.32](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.31...v1.1.0-alpha.32) (2025-02-12) ### Bug Fixes * set current version on http response headers ([#183](https://github.com/opendatalab/labelU/issues/183)) ([576333d](https://github.com/opendatalab/labelU/commit/576333d2db2ba61a3ba37f0667dc4d191ede46d0)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index b12f6382..e445f6c4 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.5' \ No newline at end of file +version='1.1.0-alpha.32' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 2fd94c27..98ed005e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.14' +version = '1.1.0-alpha.32' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 0f7253e5be4c43562302cae9e814bde3398eee71 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 12 Feb 2025 08:58:54 +0000 Subject: [PATCH 100/236] fix: Upgrade frontend to v5.7.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.8) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index f2f6043e..200e95ed 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.4/frontend.zip -alpha_version: v5.7.0-alpha.4 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.8/frontend.zip +alpha_version: v5.7.0-alpha.8 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.10/frontend.zip release_version: v5.6.10 From f8c4245a056bd8e6c1c84cfc60e77890a5b06b98 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 12 Feb 2025 09:03:58 +0000 Subject: [PATCH 101/236] chore(release): 1.1.0-alpha.33 [skip ci] # [1.1.0-alpha.33](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.32...v1.1.0-alpha.33) (2025-02-12) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.8) ([0f7253e](https://github.com/opendatalab/labelU/commit/0f7253e5be4c43562302cae9e814bde3398eee71)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index e445f6c4..74436303 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.32' \ No newline at end of file +version='1.1.0-alpha.33' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 98ed005e..c4a4523e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.32' +version = '1.1.0-alpha.33' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From d512215ee90e599847e2a1e30e35d4aee868d44f Mon Sep 17 00:00:00 2001 From: gary Date: Wed, 12 Feb 2025 17:16:03 +0800 Subject: [PATCH 102/236] fix: upgrade frontend to v5.6.11 --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 200e95ed..e15b75f8 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.8/frontend.zip alpha_version: v5.7.0-alpha.8 -release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.10/frontend.zip -release_version: v5.6.10 +release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip +release_version: v5.6.11 From 2734618d97826c7cb0bf6c77821e50880b89ac7f Mon Sep 17 00:00:00 2001 From: gary Date: Mon, 17 Feb 2025 14:07:20 +0800 Subject: [PATCH 103/236] feat: supports mysql database (#187) --- .github/workflows/release.yml | 11 +- README_zh-CN.md | 9 +- docker/conf.d/labelu.cnf | 20 +++ labelu/alembic_labelu/alembic.ini | 67 +------- labelu/alembic_labelu/env.py | 2 +- labelu/alembic_labelu/run_migrate.py | 2 +- labelu/internal/common/config.py | 17 +- labelu/internal/common/db.py | 21 ++- labelu/internal/domain/models/attachment.py | 6 +- .../internal/domain/models/pre_annotation.py | 6 +- labelu/internal/domain/models/sample.py | 7 +- labelu/internal/domain/models/task.py | 6 +- labelu/internal/domain/models/user.py | 8 +- labelu/main.py | 23 ++- labelu/scripts/migrate_to_mysql.py | 162 ++++++++++++++++++ labelu/version.py | 2 +- pyproject.toml | 10 +- 17 files changed, 269 insertions(+), 110 deletions(-) create mode 100644 docker/conf.d/labelu.cnf create mode 100644 labelu/scripts/migrate_to_mysql.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b473746f..ae93f18b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -154,16 +154,9 @@ jobs: sed -i "s/^version[ ]*=.*/version = '${NEXT_VERSION}'/" pyproject.toml sed -i "s/^version[ ]*=.*/version='${NEXT_VERSION}'/" labelu/version.py - - name: Publish to TestPyPi - if: ${{ github.ref_name == 'alpha'}} - env: - TEST_PYPI_TOKEN: ${{ secrets.TEST_PYPI_TOKEN }} - run: | - poetry config pypi-token.testpypi $TEST_PYPI_TOKEN - poetry publish --build --skip-existing -r testpypi - - name: Publish to PyPi - if: ${{ github.ref_name == 'main'}} + # main or alpha + if: ${{ github.ref_name == 'main' || github.ref_name == 'alpha' }} env: PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} run: | diff --git a/README_zh-CN.md b/README_zh-CN.md index b1481488..2d40f510 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -69,13 +69,20 @@ conda activate labelu 4. 安装 LabelU： +安装基础版本（使用SQLite） ```bash pip install labelu ``` +安装使用 MySQL 版本： + +```bash +pip install labelu[mysql] +``` + > 安装测试版本：`pip install labelu==<测试版本号> --pre` -5. 运行： +1. 运行： ```bash labelu diff --git a/docker/conf.d/labelu.cnf b/docker/conf.d/labelu.cnf new file mode 100644 index 00000000..8707cdbf --- /dev/null +++ b/docker/conf.d/labelu.cnf @@ -0,0 +1,20 @@ +# Perf +[mysqld] +innodb_buffer_pool_size=1G +innodb_log_file_size=256M +max_connections=1000 + +# Charset +character-set-server=utf8mb4 +collation-server=utf8mb4_unicode_ci + +# Log +slow_query_log=1 +slow_query_log_file=/var/log/mysql/slow.log +long_query_time=2 + +[mysql] +default-character-set=utf8mb4 + +[client] +default-character-set=utf8mb4 diff --git a/labelu/alembic_labelu/alembic.ini b/labelu/alembic_labelu/alembic.ini index 659fce2f..551ca784 100644 --- a/labelu/alembic_labelu/alembic.ini +++ b/labelu/alembic_labelu/alembic.ini @@ -1,75 +1,14 @@ -# A generic, single database configuration. - [alembic] -# path to migration scripts script_location = . -# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s -# Uncomment the line below if you want the files to be prepended with date and time -# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file -# for all available tokens -# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s - -# sys.path path, will be prepended to sys.path if present. -# defaults to the current working directory. prepend_sys_path = . -# timezone to use when rendering the date within the migration file -# as well as the filename. -# If specified, requires the python-dateutil library that can be -# installed by adding `alembic[tz]` to the pip requirements -# string value is passed to dateutil.tz.gettz() -# leave blank for localtime -# timezone = - -# max length of characters to apply to the -# "slug" field -# truncate_slug_length = 40 - -# set to 'true' to run the environment during -# the 'revision' command, regardless of autogenerate -# revision_environment = false - -# set to 'true' to allow .pyc and .pyo files without -# a source .py file to be detected as revisions in the -# versions/ directory -# sourceless = false +version_path_separator = os -# version location specification; This defaults -# to alembic_labelu/versions. When using multiple version -# directories, initial revisions must be specified with --version-path. -# The path separator used here should be the separator specified by "version_path_separator" below. -# version_locations = %(here)s/bar:%(here)s/bat:alembic_labelu/versions - -# version path separator; As mentioned above, this is the character used to split -# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. -# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. -# Valid values for version_path_separator are: -# -# version_path_separator = : -# version_path_separator = ; -# version_path_separator = space -version_path_separator = os # Use os.pathsep. Default configuration used for new projects. - -# the output encoding used when revision files -# are written from script.py.mako -# output_encoding = utf-8 +output_encoding = utf-8 sqlalchemy.url = - -[post_write_hooks] -# post_write_hooks defines scripts or Python functions that are run -# on newly generated revision scripts. See the documentation for further -# detail and examples - -# format using "black" - use the console_scripts runner, against the "black" entrypoint -# hooks = black -# black.type = console_scripts -# black.entrypoint = black -# black.options = -l 79 REVISION_SCRIPT_FILENAME - -# Logging configuration [loggers] keys = root,sqlalchemy,alembic @@ -102,4 +41,4 @@ formatter = generic [formatter_generic] format = %(levelname)-5.5s [%(name)s] %(message)s -datefmt = %H:%M:%S +datefmt = %H:%M:%S \ No newline at end of file diff --git a/labelu/alembic_labelu/env.py b/labelu/alembic_labelu/env.py index 2889efa0..52bb441c 100644 --- a/labelu/alembic_labelu/env.py +++ b/labelu/alembic_labelu/env.py @@ -74,7 +74,7 @@ def run_migrations_online() -> None: ) with connectable.connect() as connection: - context.configure(connection=connection, target_metadata=target_metadata) + context.configure(connection=connection, target_metadata=target_metadata, render_as_batch=not str(connection.engine.url).startswith('mysql')) with context.begin_transaction(): context.run_migrations() diff --git a/labelu/alembic_labelu/run_migrate.py b/labelu/alembic_labelu/run_migrate.py index 18c64350..d952357c 100644 --- a/labelu/alembic_labelu/run_migrate.py +++ b/labelu/alembic_labelu/run_migrate.py @@ -7,7 +7,7 @@ from labelu.internal.common.config import settings -def run_sqlite_migrations(): +def run_db_migrations(): migrations_dir = os.path.dirname(os.path.realpath(__file__)) # assumes the alembic.ini is also contained in the current directory of this file config_file = os.path.join(migrations_dir, "alembic.ini") diff --git a/labelu/internal/common/config.py b/labelu/internal/common/config.py index 90b03a9a..7a2d4e6d 100644 --- a/labelu/internal/common/config.py +++ b/labelu/internal/common/config.py @@ -2,7 +2,7 @@ from pathlib import Path from loguru import logger -from pydantic import BaseSettings +from pydantic import BaseSettings, Field from labelu.internal.common.io import get_data_dir @@ -23,7 +23,12 @@ class Settings(BaseSettings): UPLOAD_FILE_MAX_SIZE: int = 200_000_000 # ~200MB THUMBNAIL_HEIGH_PIXEL: int = 120 - DATABASE_URL: str = f"sqlite:///{BASE_DATA_DIR}/labelu.sqlite" + DATABASE_URL: str = Field( + default="mysql://labelu:labelupass@localhost/labeludb", + # default=f"sqlite:///{BASE_DATA_DIR}/labelu.sqlite", + description="Database connection URL. Supports SQLite and MySQL." + ) + # or using DATABASE_URL=mysql://labelu:labelupass@10.1.52.75/labeludb PASSWORD_SECRET_KEY: str = ( "e5b7d00a59aaa2a5ea86a7c4d72f856b20bafa1b8d0e66124082ada81f6340bd" @@ -33,6 +38,14 @@ class Settings(BaseSettings): TOKEN_ACCESS_EXPIRE_MINUTES: int = 30 TOKEN_TYPE: str = "Bearer" + @property + def need_migration_to_mysql(self) -> bool: + sqlite_path = Path(self.BASE_DATA_DIR) / "labelu.sqlite" + return ( + self.DATABASE_URL.startswith('mysql') and + sqlite_path.exists() + ) + class Config: env_prefix = "" env_file_encoding = "utf-8" diff --git a/labelu/internal/common/db.py b/labelu/internal/common/db.py index cba66ce4..a100ecee 100644 --- a/labelu/internal/common/db.py +++ b/labelu/internal/common/db.py @@ -4,10 +4,23 @@ from sqlalchemy.ext.declarative import declarative_base from labelu.internal.common.config import settings -# connect_args is needed only for SQLite. It's not needed for other databases -engine = create_engine( - settings.DATABASE_URL, connect_args={"check_same_thread": False}, echo=True -) +engine = None +database_url = settings.DATABASE_URL + +if settings.DATABASE_URL.startswith('mysql'): + engine = create_engine( + database_url, + pool_pre_ping=True, + pool_recycle=3600, + ) +else: + # connect_args is needed only for SQLite. It's not needed for other databases + engine = create_engine( + database_url, + connect_args={"check_same_thread": False}, + echo=True, + ) + SessionLocal = sessionmaker(autocommit=True, autoflush=False, bind=engine) Base = declarative_base() diff --git a/labelu/internal/domain/models/attachment.py b/labelu/internal/domain/models/attachment.py index a87255a0..35f99625 100644 --- a/labelu/internal/domain/models/attachment.py +++ b/labelu/internal/domain/models/attachment.py @@ -1,7 +1,7 @@ from datetime import datetime from sqlalchemy.schema import Index -from sqlalchemy import Column, DateTime, ForeignKey, Integer, SmallInteger, String +from sqlalchemy import Column, DateTime, ForeignKey, Integer, String from labelu.internal.common.db import Base @@ -17,10 +17,10 @@ class TaskAttachment(Base): created_by = Column(Integer, ForeignKey("user.id"), index=True) updated_by = Column(Integer, ForeignKey("user.id"), index=True) created_at = Column( - DateTime, default=datetime.now, comment="Time a task was created" + DateTime(timezone=True), default=datetime.now, comment="Time a task was created" ) updated_at = Column( - DateTime, + DateTime(timezone=True), default=datetime.now, onupdate=datetime.now, comment="Last time a task was updated", diff --git a/labelu/internal/domain/models/pre_annotation.py b/labelu/internal/domain/models/pre_annotation.py index c0444698..2fb34c00 100644 --- a/labelu/internal/domain/models/pre_annotation.py +++ b/labelu/internal/domain/models/pre_annotation.py @@ -19,16 +19,16 @@ class TaskPreAnnotation(Base): created_by = Column(Integer, ForeignKey("user.id"), index=True) updated_by = Column(Integer, ForeignKey("user.id")) created_at = Column( - DateTime, default=datetime.now, comment="Time a task sample result was created" + DateTime(timezone=True), default=datetime.now, comment="Time a task sample result was created" ) updated_at = Column( - DateTime, + DateTime(timezone=True), default=datetime.now, onupdate=datetime.now, comment="Last time a task pre annotation result was updated", ) data = Column(Text, comment="task sample pre annotation result") - deleted_at = Column(DateTime, index=True, comment="Task pre-annotation delete time") + deleted_at = Column(DateTime(timezone=True), index=True, comment="Task pre-annotation delete time") file = relationship("TaskAttachment", foreign_keys=[file_id]) task = relationship("Task", foreign_keys=[task_id]) diff --git a/labelu/internal/domain/models/sample.py b/labelu/internal/domain/models/sample.py index 7004ebdf..feeb8b87 100644 --- a/labelu/internal/domain/models/sample.py +++ b/labelu/internal/domain/models/sample.py @@ -28,10 +28,10 @@ class TaskSample(Base): created_by = Column(Integer, ForeignKey("user.id"), index=True) updated_by = Column(Integer, ForeignKey("user.id")) created_at = Column( - DateTime, default=datetime.now, comment="Time a task sample result was created" + DateTime(timezone=True), default=datetime.now, comment="Time a task sample result was created" ) updated_at = Column( - DateTime, + DateTime(timezone=True), default=datetime.now, onupdate=datetime.now, comment="Last time a task sample result was updated", @@ -47,7 +47,8 @@ class TaskSample(Base): default=SampleState.NEW.value, comment="NEW is has not start yet, DONE is completed, SKIPPED is skipped", ) - deleted_at = Column(DateTime, index=True, comment="Task delete time") + task_attachment_ids = Column(Text, comment="task sample file ids") + deleted_at = Column(DateTime(timezone=True), index=True, comment="Task delete time") # 由旧的data里的fileNames和urls中的唯一一个，迁移到media中 file = relationship("TaskAttachment", foreign_keys=[file_id]) diff --git a/labelu/internal/domain/models/task.py b/labelu/internal/domain/models/task.py index 4ea08541..bf954387 100644 --- a/labelu/internal/domain/models/task.py +++ b/labelu/internal/domain/models/task.py @@ -51,15 +51,15 @@ class Task(Base): Integer, ForeignKey(column="user.id"), comment="Last time a task was updated by" ) created_at = Column( - DateTime, default=datetime.now, comment="Time a task was created" + DateTime(timezone=True), default=datetime.now, comment="Time a task was created" ) updated_at = Column( - DateTime, + DateTime(timezone=True), default=datetime.now, onupdate=datetime.now, comment="Last time a task was updated", ) - deleted_at = Column(DateTime, index=True, comment="Task delete time") + deleted_at = Column(DateTime(timezone=True), index=True, comment="Task delete time") owner = relationship("User", foreign_keys=[created_by]) updater = relationship("User", foreign_keys=[updated_by]) diff --git a/labelu/internal/domain/models/user.py b/labelu/internal/domain/models/user.py index 8fcc8dbe..cbb93a63 100644 --- a/labelu/internal/domain/models/user.py +++ b/labelu/internal/domain/models/user.py @@ -9,11 +9,11 @@ class User(Base): __tablename__ = "user" id = Column(Integer, primary_key=True, autoincrement=True, index=True) - username = Column(String, unique=True, index=True) - hashed_password = Column(String) + username = Column(String(50), unique=True, index=True) + hashed_password = Column(String(255)) created_at = Column( - DateTime, default=datetime.utcnow, comment="Time a task was created" + DateTime(timezone=True), default=datetime.now, comment="Time a task was created" ) updated_at = Column( - DateTime, default=datetime.utcnow, comment="Last time a task was updated" + DateTime(timezone=True), default=datetime.now, comment="Last time a task was updated" ) diff --git a/labelu/main.py b/labelu/main.py index 8137f0ea..fcfb3c78 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -1,4 +1,5 @@ from typing import Any +from loguru import logger import uvicorn from typer import Typer from fastapi import FastAPI, Request, Response @@ -10,7 +11,10 @@ from labelu.internal.common.db import init_tables from labelu.internal.common.config import settings from labelu.internal.common.error_code import add_exception_handler -from labelu.alembic_labelu.run_migrate import run_sqlite_migrations +from labelu.alembic_labelu.run_migrate import run_db_migrations +from labelu.scripts.migrate_to_mysql import migrate_to_mysql + +from .version import version as labelu_version from .version import version as labelu_version @@ -90,11 +94,18 @@ init_logging() init_tables() -run_sqlite_migrations() +run_db_migrations() add_exception_handler(app=app) add_router(app=app) add_middleware(app=app) +def startup(): + if settings.need_migration_to_mysql: + logger.info("Migrating database to MySQL") + migrate_to_mysql() + +app.add_event_handler("startup", startup) + class NoCacheStaticFiles(StaticFiles): def __init__(self, *args: Any, **kwargs: Any): self.cachecontrol = "max-age=0, no-cache, no-store, , must-revalidate" @@ -129,7 +140,11 @@ async def add_correct_content_type(request: Request, call_next): cli = Typer() - +@cli.command() +def to_mysql(): + """Migrate database to MySQL""" + migrate_to_mysql() + @cli.command() def main( host: str = "localhost", port: int = 8000, media_host: str = "http://localhost:8000" @@ -144,4 +159,4 @@ def main( if __name__ == "__main__": - cli() + cli() \ No newline at end of file diff --git a/labelu/scripts/migrate_to_mysql.py b/labelu/scripts/migrate_to_mysql.py new file mode 100644 index 00000000..41da1119 --- /dev/null +++ b/labelu/scripts/migrate_to_mysql.py @@ -0,0 +1,162 @@ +from loguru import logger +from sqlalchemy import create_engine, MetaData, Table, inspect, text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.exc import SQLAlchemyError, IntegrityError +from contextlib import contextmanager +from typing import List, Dict, Any +import time +from datetime import datetime +from labelu.internal.common.config import settings + +class MigrationError(Exception): + """自定义迁移异常""" + pass + +@contextmanager +def timer(operation: str): + """计时器上下文管理器""" + start = time.time() + yield + duration = time.time() - start + logger.info(f"{operation} took {duration:.2f} seconds") + +class DatabaseMigrator: + BATCH_SIZE = 1000 # 批量插入的大小 + + def __init__(self, sqlite_url: str, mysql_url: str): + self.sqlite_engine = create_engine(sqlite_url) + self.mysql_engine = create_engine(mysql_url) + self.metadata = MetaData() + + @contextmanager + def create_sessions(self): + """创建数据库会话的上下文管理器""" + sqlite_session = sessionmaker(bind=self.sqlite_engine)() + mysql_session = sessionmaker(bind=self.mysql_engine)() + try: + yield sqlite_session, mysql_session + finally: + sqlite_session.close() + mysql_session.close() + + def get_tables_in_order(self) -> List[str]: + """获取考虑外键依赖的表排序""" + inspector = inspect(self.sqlite_engine) + tables = [] + table_names = inspector.get_table_names() + + dependencies = {name: set() for name in table_names} + for name in table_names: + for fk in inspector.get_foreign_keys(name): + dependencies[name].add(fk['referred_table']) + + while dependencies: + independent_tables = [ + name for name, deps in dependencies.items() + if not deps + ] + if not independent_tables: + raise MigrationError("Circular dependency detected in tables") + + tables.extend(independent_tables) + for name in independent_tables: + del dependencies[name] + for deps in dependencies.values(): + deps.discard(name) + + return tables + + def _process_row_data(self, row: Dict[str, Any]) -> Dict[str, Any]: + """处理行数据，确保数据类型兼容性""" + processed = {} + for key, value in row.items(): + if isinstance(value, datetime): + processed[key] = value.isoformat() + elif isinstance(value, bytes): + processed[key] = value.decode('utf-8') + else: + processed[key] = value + return processed + + def _batch_insert(self, session, table: Table, rows: List[Dict[str, Any]]) -> None: + """批量插入数据""" + try: + processed_rows = [self._process_row_data(row._asdict()) for row in rows] + session.execute(table.insert(), processed_rows) + session.commit() + except IntegrityError as e: + session.rollback() + logger.error(f"Integrity error during batch insert: {e}") + # 回退到逐行插入以处理问题数据 + for row in rows: + try: + processed_row = self._process_row_data(row._asdict()) + session.execute(table.insert(), [processed_row]) + session.commit() + except Exception as e: + session.rollback() + logger.error(f"Failed to insert row: {e}") + except Exception as e: + session.rollback() + raise MigrationError(f"Batch insert failed: {e}") + + def check_migration_status(self, mysql_session) -> bool: + """检查是否已经迁移""" + try: + result = mysql_session.execute(text("SELECT COUNT(*) FROM user")).scalar() + return result > 0 + except SQLAlchemyError: + return False + + def migrate_table(self, table_name: str, sqlite_session, mysql_session) -> None: + """迁移单个表""" + if table_name == "alembic_version": + return + + with timer(f"Migrating table {table_name}"): + table = Table(table_name, self.metadata) + query = sqlite_session.query(table) + total_rows = query.count() + + if total_rows == 0: + logger.info(f"Table {table_name} is empty, skipping...") + return + + batches = range(0, total_rows, self.BATCH_SIZE) + for batch_start in batches: + rows = query.offset(batch_start).limit(self.BATCH_SIZE).all() + self._batch_insert(mysql_session, table, rows) + logger.info(f"Migrated {min(batch_start + self.BATCH_SIZE, total_rows)}/{total_rows} rows in {table_name}") + + def migrate(self) -> None: + """执行完整的数据迁移""" + with self.create_sessions() as (sqlite_session, mysql_session): + if self.check_migration_status(mysql_session): + logger.info("Database has already been migrated.") + return + + try: + self.metadata.reflect(bind=self.sqlite_engine) + ordered_tables = self.get_tables_in_order() + + with timer("Complete migration"): + for table_name in ordered_tables: + self.migrate_table(table_name, sqlite_session, mysql_session) + + logger.info("Migration completed successfully.") + + except Exception as e: + logger.error(f"Migration failed: {e}") + mysql_session.rollback() + raise MigrationError(f"Migration failed: {e}") + +def migrate_to_mysql(): + """主迁移函数""" + sqlite_url = f"sqlite:///{settings.BASE_DATA_DIR}/labelu.sqlite" + + try: + migrator = DatabaseMigrator(sqlite_url, settings.DATABASE_URL) + migrator.migrate() + except MigrationError as e: + logger.error(f"Migration failed: {e}") + raise e \ No newline at end of file diff --git a/labelu/version.py b/labelu/version.py index 74436303..94764219 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.33' \ No newline at end of file +version='1.1.15' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index c4a4523e..6d737eaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.33' +version = '1.1.15' description = "" license = "Apache-2.0" authors = ["shenguanlin "] @@ -30,12 +30,8 @@ alembic = "^1.9.4" httpx = "^0.27.0" tfrecord = "^1.14.5" -[[tool.poetry.source]] -name = "testpypi" -url = "https://test.pypi.org/legacy/" -default = false -secondary = false - +[tool.poetry.extras] +mysql = ["mysqlclient"] [tool.poetry.group.dev.dependencies] black = "^22.10.0" From 35e8858c4689b7756549b676396d771e306c122c Mon Sep 17 00:00:00 2001 From: gary Date: Mon, 17 Feb 2025 14:11:52 +0800 Subject: [PATCH 104/236] Feat/collaboration (#188) --- labelu/internal/common/config.py | 4 ++-- labelu/main.py | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/labelu/internal/common/config.py b/labelu/internal/common/config.py index 7a2d4e6d..b628a546 100644 --- a/labelu/internal/common/config.py +++ b/labelu/internal/common/config.py @@ -24,8 +24,8 @@ class Settings(BaseSettings): THUMBNAIL_HEIGH_PIXEL: int = 120 DATABASE_URL: str = Field( - default="mysql://labelu:labelupass@localhost/labeludb", - # default=f"sqlite:///{BASE_DATA_DIR}/labelu.sqlite", + # default="mysql://labelu:labelupass@localhost/labeludb", + default=f"sqlite:///{BASE_DATA_DIR}/labelu.sqlite", description="Database connection URL. Supports SQLite and MySQL." ) # or using DATABASE_URL=mysql://labelu:labelupass@10.1.52.75/labeludb diff --git a/labelu/main.py b/labelu/main.py index fcfb3c78..64c2a67f 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -16,9 +16,6 @@ from .version import version as labelu_version -from .version import version as labelu_version - - description = """ labelU backend. From 840a313227d766a96d70d05151977be3412cc9bc Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 17 Feb 2025 06:16:51 +0000 Subject: [PATCH 105/236] chore(release): 1.1.0-alpha.34 [skip ci] # [1.1.0-alpha.34](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.33...v1.1.0-alpha.34) (2025-02-17) ### Bug Fixes * upgrade frontend to v5.6.11 ([d512215](https://github.com/opendatalab/labelU/commit/d512215ee90e599847e2a1e30e35d4aee868d44f)) ### Features * supports mysql database ([#187](https://github.com/opendatalab/labelU/issues/187)) ([2734618](https://github.com/opendatalab/labelU/commit/2734618d97826c7cb0bf6c77821e50880b89ac7f)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 94764219..589a9746 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.15' \ No newline at end of file +version='1.1.0-alpha.34' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6d737eaa..fab8eafd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.15' +version = '1.1.0-alpha.34' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From b159c634286dd446c548e70ca0b7367d0221a760 Mon Sep 17 00:00:00 2001 From: gary Date: Fri, 21 Feb 2025 13:25:29 +0800 Subject: [PATCH 106/236] feat: supports task collaborators (#189) --- .github/workflows/release.yml | 9 +- docker/conf.d/labelu.cnf | 20 -- labelu/alembic_labelu/env.py | 7 - ...83c9a254_add_collaborators_and_updaters.py | 135 ++++++++++++++ .../adapter/persistence/crud_attachment.py | 8 +- .../persistence/crud_pre_annotation.py | 22 +-- .../adapter/persistence/crud_sample.py | 18 +- .../internal/adapter/persistence/crud_task.py | 27 ++- .../internal/adapter/persistence/crud_user.py | 22 ++- labelu/internal/adapter/routers/attachment.py | 1 - .../adapter/routers/pre_annotation.py | 23 ++- labelu/internal/adapter/routers/sample.py | 15 +- labelu/internal/adapter/routers/task.py | 113 +++++++++++- labelu/internal/adapter/routers/user.py | 44 ++++- labelu/internal/adapter/ws/__init__.py | 6 + labelu/internal/adapter/ws/sample.py | 74 ++++++++ .../application/command/attachment.py | 5 +- labelu/internal/application/command/task.py | 7 +- .../internal/application/response/sample.py | 4 +- labelu/internal/application/response/user.py | 3 +- .../application/service/pre_annotation.py | 22 +-- labelu/internal/application/service/sample.py | 109 ++++++----- labelu/internal/application/service/task.py | 171 +++++++++++++++++- labelu/internal/application/service/user.py | 16 +- labelu/internal/clients/__init__.py | 0 labelu/internal/clients/ws.py | 4 + labelu/internal/common/config.py | 4 +- labelu/internal/common/error_code.py | 21 ++- labelu/internal/common/websocket.py | 50 +++++ labelu/internal/dependencies/user.py | 33 +++- labelu/internal/domain/models/__init__.py | 7 + labelu/internal/domain/models/sample.py | 2 +- labelu/internal/domain/models/task.py | 3 +- .../domain/models/task_collaborator.py | 24 +++ .../domain/models/task_sample_updater.py | 26 +++ labelu/internal/domain/models/user.py | 2 + labelu/internal/middleware/__init__.py | 4 +- labelu/internal/middleware/content_type.py | 16 ++ labelu/main.py | 26 +-- labelu/scripts/migrate_to_mysql.py | 15 +- .../adapter/routers/test_pre_annotation.py | 10 +- .../internal/adapter/routers/test_sample.py | 12 +- pyproject.toml | 1 + 43 files changed, 906 insertions(+), 235 deletions(-) delete mode 100644 docker/conf.d/labelu.cnf create mode 100644 labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py create mode 100644 labelu/internal/adapter/ws/__init__.py create mode 100644 labelu/internal/adapter/ws/sample.py create mode 100644 labelu/internal/clients/__init__.py create mode 100644 labelu/internal/clients/ws.py create mode 100644 labelu/internal/common/websocket.py create mode 100644 labelu/internal/domain/models/__init__.py create mode 100644 labelu/internal/domain/models/task_collaborator.py create mode 100644 labelu/internal/domain/models/task_sample_updater.py create mode 100644 labelu/internal/middleware/content_type.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ae93f18b..49cbb84e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -70,15 +70,8 @@ jobs: # ====================== release ====================== - - name: Set test pip url - if: ${{ github.ref_name == 'alpha' }} - env: - NEXT_VERSION: ${{ env.NEXT_VERSION }} - run: | - echo "PYPI_URL=https://test.pypi.org/project/labelu/${{ env.NEXT_VERSION }}" >> $GITHUB_ENV - - name: Set release pip url - if: ${{ github.ref_name == 'main' }} + if: ${{ github.ref_name == 'main' || github.ref_name == 'alpha' }} env: NEXT_VERSION: ${{ env.NEXT_VERSION }} run: | diff --git a/docker/conf.d/labelu.cnf b/docker/conf.d/labelu.cnf deleted file mode 100644 index 8707cdbf..00000000 --- a/docker/conf.d/labelu.cnf +++ /dev/null @@ -1,20 +0,0 @@ -# Perf -[mysqld] -innodb_buffer_pool_size=1G -innodb_log_file_size=256M -max_connections=1000 - -# Charset -character-set-server=utf8mb4 -collation-server=utf8mb4_unicode_ci - -# Log -slow_query_log=1 -slow_query_log_file=/var/log/mysql/slow.log -long_query_time=2 - -[mysql] -default-character-set=utf8mb4 - -[client] -default-character-set=utf8mb4 diff --git a/labelu/alembic_labelu/env.py b/labelu/alembic_labelu/env.py index 52bb441c..4428f6e1 100644 --- a/labelu/alembic_labelu/env.py +++ b/labelu/alembic_labelu/env.py @@ -1,5 +1,3 @@ -import os -import sys from logging.config import fileConfig from sqlalchemy import engine_from_config @@ -8,11 +6,6 @@ from alembic import context from labelu.internal.common.db import Base -from labelu.internal.domain.models.task import Task -from labelu.internal.domain.models.user import User -from labelu.internal.domain.models.sample import TaskSample -from labelu.internal.domain.models.attachment import TaskAttachment -from labelu.internal.common.config import settings # this is the Alembic Config object, which provides diff --git a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py new file mode 100644 index 00000000..076ae798 --- /dev/null +++ b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py @@ -0,0 +1,135 @@ +"""add_collaborators_and_updaters + +Revision ID: 2eb983c9a254 +Revises: eb9c5b98168b +Create Date: 2025-02-19 16:16:39.259779 + +""" +from datetime import datetime +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql import table, column + + +# revision identifiers, used by Alembic. +revision = '2eb983c9a254' +down_revision = 'eb9c5b98168b' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Create task_collaborator table + op.create_table( + 'task_collaborator', + sa.Column('task_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column( + 'created_at', + sa.DateTime(timezone=True), + server_default=sa.text('CURRENT_TIMESTAMP'), + nullable=False + ), + sa.ForeignKeyConstraint( + ['task_id'], + ['task.id'], + ondelete='CASCADE' + ), + sa.ForeignKeyConstraint( + ['user_id'], + ['user.id'], + ondelete='CASCADE' + ), + sa.PrimaryKeyConstraint('task_id', 'user_id') + ) + + # Performances index + op.create_index( + 'ix_task_collaborator_task_id', + 'task_collaborator', + ['task_id'] + ) + op.create_index( + 'ix_task_collaborator_user_id', + 'task_collaborator', + ['user_id'] + ) + op.create_index( + 'ix_task_created_by_deleted_at', + 'task', + ['created_by', 'deleted_at'] + ) + + # Task sample: updater -> updaters; create a new table task_sample_updater + op.create_table( + 'task_sample_updater', + sa.Column('sample_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column( + 'created_at', + sa.DateTime(timezone=True), + server_default=sa.text('CURRENT_TIMESTAMP'), + nullable=False + ), + sa.ForeignKeyConstraint( + ['sample_id'], + ['task_sample.id'], + ondelete='CASCADE' + ), + sa.ForeignKeyConstraint( + ['user_id'], + ['user.id'], + ondelete='CASCADE' + ), + sa.PrimaryKeyConstraint('sample_id', 'user_id') + ) + + # Performances index + op.create_index( + 'ix_task_sample_updater_sample_id', + 'task_sample_updater', + ['sample_id'] + ) + op.create_index( + 'ix_task_sample_updater_user_id', + 'task_sample_updater', + ['user_id'] + ) + + # Migrate data from task_sample.updated_by to task_sample_updater + task_sample = table( + 'task_sample', + column('id', sa.Integer), + column('updated_by', sa.Integer), + column('updated_at', sa.DateTime) + ) + + task_sample_updater = table( + 'task_sample_updater', + column('sample_id', sa.Integer), + column('user_id', sa.Integer), + column('created_at', sa.DateTime) + ) + + conn = op.get_bind() + for row in conn.execute(sa.select([task_sample.c.id, task_sample.c.updated_by, task_sample.c.updated_at])): + if row.updated_by: + conn.execute( + task_sample_updater.insert().values( + sample_id=row.id, + user_id=row.updated_by, + created_at=row.updated_at or datetime.now() + ) + ) + + +def downgrade() -> None: + op.drop_index('ix_task_collaborator_user_id') + op.drop_index('ix_task_collaborator_task_id') + + op.drop_table('task_collaborator') + + op.drop_index('ix_task_sample_updater_user_id') + op.drop_index('ix_task_sample_updater_task_sample_id') + + op.drop_table('task_sample_updater') diff --git a/labelu/internal/adapter/persistence/crud_attachment.py b/labelu/internal/adapter/persistence/crud_attachment.py index efa6a466..9b376540 100644 --- a/labelu/internal/adapter/persistence/crud_attachment.py +++ b/labelu/internal/adapter/persistence/crud_attachment.py @@ -7,13 +7,13 @@ def list_by( db: Session, - pageSize: int, + size: int, ids: List[int] | None = [], task_id: Optional[int] = None, owner_id: Optional[int] = None, after: Optional[int] = None, before: Optional[int] = None, - pageNo: Optional[int] = None, + page: Optional[int] = None, sorting: Optional[str] = None, ) -> Tuple[List[TaskAttachment], int]: # query filter @@ -39,8 +39,8 @@ def list_by( count = query.count() results = ( - query.offset(offset=pageNo * pageSize if pageNo else 0) - .limit(limit=pageSize) + query.offset(offset=page * size if page else 0) + .limit(limit=size) .all() ) diff --git a/labelu/internal/adapter/persistence/crud_pre_annotation.py b/labelu/internal/adapter/persistence/crud_pre_annotation.py index ff7fa4c6..3a4cfae6 100644 --- a/labelu/internal/adapter/persistence/crud_pre_annotation.py +++ b/labelu/internal/adapter/persistence/crud_pre_annotation.py @@ -15,18 +15,17 @@ def batch(db: Session, pre_annotations: List[TaskPreAnnotation]) -> List[TaskPre def list_by( db: Session, - owner_id: int, task_id: int | None = None, sample_name: str | None = None, after: int | None = None, before: int | None = None, - pageNo: int | None = None, + page: int | None = None, sorting: str | None = None, - pageSize: int | None = 10, + size: int | None = 10, ) -> Tuple[List[TaskPreAnnotation], int]: # query filter - query_filter = [TaskPreAnnotation.created_by == owner_id, TaskPreAnnotation.deleted_at == None] + query_filter = [TaskPreAnnotation.deleted_at == None] if before: query_filter.append(TaskPreAnnotation.id < before) if after: @@ -48,8 +47,8 @@ def list_by( count = query.count() results = ( - query.offset(offset=pageNo * pageSize if pageNo else 0) - .limit(limit=pageSize) + query.offset(offset=page * size if page else 0) + .limit(limit=size) .all() ) @@ -65,19 +64,17 @@ def list_by( return results, count -def list_by_task_id_and_owner_id(db: Session, task_id: int, owner_id: int) -> Dict[str, List[TaskPreAnnotation]]: +def list_by_task_id_and_owner_id(db: Session, task_id: int) -> Dict[str, List[TaskPreAnnotation]]: pre_annotations = db.query(TaskPreAnnotation).filter( TaskPreAnnotation.task_id == task_id, TaskPreAnnotation.deleted_at == None, - TaskPreAnnotation.created_by == owner_id ).all() return pre_annotations -def list_by_task_id_and_file_id(db: Session, task_id: int, file_id: int, owner_id: int) -> List[TaskPreAnnotation]: +def list_by_task_id_and_file_id(db: Session, task_id: int, file_id: int) -> List[TaskPreAnnotation]: return db.query(TaskPreAnnotation).filter( TaskPreAnnotation.task_id == task_id, - TaskPreAnnotation.created_by == owner_id, TaskPreAnnotation.deleted_at == None, TaskPreAnnotation.file_id == file_id ).all() @@ -97,7 +94,6 @@ def list_by_task_id_and_owner_id_and_sample_name(db: Session, task_id: int, owne return db.query(TaskPreAnnotation).filter( TaskPreAnnotation.task_id == task_id, TaskPreAnnotation.deleted_at == None, - TaskPreAnnotation.created_by == owner_id, TaskPreAnnotation.sample_name == sample_name ).all() @@ -134,8 +130,8 @@ def delete(db: Session, pre_annotation_ids: List[int]) -> None: ) -def count(db: Session, task_id: int, owner_id: int, sample_name: str | None) -> int: - query_filter = [TaskPreAnnotation.created_by == owner_id, TaskPreAnnotation.deleted_at == None] +def count(db: Session, task_id: int, sample_name: str | None) -> int: + query_filter = [TaskPreAnnotation.deleted_at == None] if task_id: query_filter.append(TaskPreAnnotation.task_id == task_id) diff --git a/labelu/internal/adapter/persistence/crud_sample.py b/labelu/internal/adapter/persistence/crud_sample.py index db76b886..7a0a5fad 100644 --- a/labelu/internal/adapter/persistence/crud_sample.py +++ b/labelu/internal/adapter/persistence/crud_sample.py @@ -18,16 +18,15 @@ def batch(db: Session, samples: List[TaskSample]) -> List[TaskSample]: def list_by( db: Session, task_id: Union[int, None], - owner_id: int, after: Union[int, None], before: Union[int, None], - pageNo: Union[int, None], - pageSize: int, + page: Union[int, None], + size: int, sorting: Union[str, None], ) -> List[TaskSample]: # query filter - query_filter = [TaskSample.created_by == owner_id, TaskSample.deleted_at == None] + query_filter = [TaskSample.deleted_at == None] if before: query_filter.append(TaskSample.id < before) if after: @@ -58,8 +57,8 @@ def list_by( else: query = query.order_by(TaskSample.id.asc()) results = ( - query.offset(offset=pageNo * pageSize if pageNo else 0) - .limit(limit=pageSize) + query.offset(offset=page * size if page else 0) + .limit(limit=size) .all() ) if before: @@ -100,8 +99,8 @@ def delete(db: Session, sample_ids: List[int]) -> None: ) -def count(db: Session, task_id: int, owner_id: int) -> int: - query_filter = [TaskSample.created_by == owner_id, TaskSample.deleted_at == None] +def count(db: Session, task_id: int) -> int: + query_filter = [TaskSample.deleted_at == None] if task_id: query_filter.append(TaskSample.task_id == task_id) return db.query(TaskSample).filter(*query_filter).count() @@ -109,10 +108,9 @@ def count(db: Session, task_id: int, owner_id: int) -> int: def statics( db: Session, - owner_id: int, task_ids: List[int], ) -> dict: - query_filter = [TaskSample.created_by == owner_id, TaskSample.deleted_at == None] + query_filter = [TaskSample.deleted_at == None] if task_ids: query_filter.append(TaskSample.task_id.in_(task_ids)) diff --git a/labelu/internal/adapter/persistence/crud_task.py b/labelu/internal/adapter/persistence/crud_task.py index 5e53285c..94bf359a 100644 --- a/labelu/internal/adapter/persistence/crud_task.py +++ b/labelu/internal/adapter/persistence/crud_task.py @@ -1,10 +1,12 @@ from datetime import datetime -from typing import Any, Dict, List +from sqlalchemy import or_, exists +from typing import Any, Dict, List, Tuple from sqlalchemy.orm import Session from fastapi.encoders import jsonable_encoder from labelu.internal.domain.models.task import Task +from labelu.internal.domain.models.task_collaborator import TaskCollaborator def create(db: Session, task: Task) -> Task: @@ -14,15 +16,30 @@ def create(db: Session, task: Task) -> Task: return task -def list_by(db: Session, owner_id: int, page: int = 0, size: int = 100) -> List[Task]: - return ( +def list_by(db: Session, owner_id: int, page: int = 0, size: int = 100) -> Tuple[List[Task], int]: + collaborator_exists = exists().where( + TaskCollaborator.task_id == Task.id, + TaskCollaborator.user_id == owner_id + ) + query = ( db.query(Task) - .filter(Task.created_by == owner_id, Task.deleted_at == None) + .filter( + Task.deleted_at == None, + or_( + Task.created_by == owner_id, + collaborator_exists + ) + ) + ) + total = query.count() + + return ( + query .order_by(Task.id.desc()) .offset(offset=page * size) .limit(limit=size) .all() - ) + ), total def get(db: Session, task_id: int, lock: bool = False) -> Task: diff --git a/labelu/internal/adapter/persistence/crud_user.py b/labelu/internal/adapter/persistence/crud_user.py index 926e0c47..8f740931 100644 --- a/labelu/internal/adapter/persistence/crud_user.py +++ b/labelu/internal/adapter/persistence/crud_user.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Tuple from sqlalchemy.orm import Session from labelu.internal.domain.models.user import User @@ -18,3 +18,23 @@ def get(db: Session, id: int) -> User: def get_user_by_username(db: Session, username: str) -> User: return db.query(User).filter(User.username == username).first() + +def list_by(db: Session, page: int, size: int, username: str = None) -> Tuple[List[User], int]: + query_filter = [] + + if username: + query_filter.append(User.username == username) + + query = db.query(User).filter(*query_filter) + result = ( + query + .order_by(User.id.desc()) + .offset(offset=page * size) + .limit(limit=size) + .all() + ) + total = query.count() + return result, total + +def list_by_ids(db: Session, ids: List[int]) -> List[User]: + return db.query(User).filter(User.id.in_(ids)).all() \ No newline at end of file diff --git a/labelu/internal/adapter/routers/attachment.py b/labelu/internal/adapter/routers/attachment.py index 57e60ed1..75f1beb7 100644 --- a/labelu/internal/adapter/routers/attachment.py +++ b/labelu/internal/adapter/routers/attachment.py @@ -7,7 +7,6 @@ import os from labelu.internal.common import db -from labelu.internal.common.config import settings from labelu.internal.common.security import security from labelu.internal.domain.models.user import User from labelu.internal.dependencies.user import get_current_user diff --git a/labelu/internal/adapter/routers/pre_annotation.py b/labelu/internal/adapter/routers/pre_annotation.py index 83ed5113..a03fa4ef 100644 --- a/labelu/internal/adapter/routers/pre_annotation.py +++ b/labelu/internal/adapter/routers/pre_annotation.py @@ -57,8 +57,8 @@ async def list_pre_annotation_files_request( task_id: int, after: Union[int, None] = Query(default=None, gt=0), before: Union[int, None] = Query(default=None, gt=0), - pageNo: Union[int, None] = Query(default=None, ge=0), - pageSize: Union[int, None] = 100, + page: Union[int, None] = Query(default=None, ge=0), + size: Union[int, None] = 100, sort: Union[str, None] = Query( default=None, regex="(created_at):(desc|asc)" ), @@ -73,16 +73,15 @@ async def list_pre_annotation_files_request( data, total = await service.list_pre_annotation_files( db=db, task_id=task_id, - current_user=current_user, after=after, before=before, - pageNo=pageNo, - pageSize=pageSize, + page=page, + size=size, sorting=sort, ) # response - meta_data = MetaData(total=total, page=pageNo, size=len(data)) + meta_data = MetaData(total=total, page=page, size=len(data)) return OkRespWithMeta[List[PreAnnotationFileResponse]](meta_data=meta_data, data=data) @router.delete( @@ -118,8 +117,8 @@ async def list_by( sample_name: str = Query(default=None, min_length=1, max_length=255), after: Union[int, None] = Query(default=None, gt=0), before: Union[int, None] = Query(default=None, gt=0), - pageNo: Union[int, None] = Query(default=None, ge=0), - pageSize: Union[int, None] = 100, + page: Union[int, None] = Query(default=None, ge=0), + size: Union[int, None] = 100, sort: Union[str, None] = Query( default=None, regex="(annotated_count|state):(desc|asc)" ), @@ -131,7 +130,7 @@ async def list_by( Get a annotation result. """ - if len([i for i in (after, before, pageNo) if i != None]) != 1: + if len([i for i in (after, before, page) if i != None]) != 1: raise LabelUException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, code=ErrorCode.CODE_55000_SAMPLE_LIST_PARAMETERS_ERROR, @@ -144,14 +143,14 @@ async def list_by( sample_name=sample_name, after=after, before=before, - pageNo=pageNo, - pageSize=pageSize, + page=page, + size=size, sorting=sort, current_user=current_user, ) # response - meta_data = MetaData(total=total, page=pageNo, size=len(data)) + meta_data = MetaData(total=total, page=page, size=len(data)) return OkRespWithMeta[List[PreAnnotationResponse]](meta_data=meta_data, data=data) diff --git a/labelu/internal/adapter/routers/sample.py b/labelu/internal/adapter/routers/sample.py index 66141336..02f97b90 100644 --- a/labelu/internal/adapter/routers/sample.py +++ b/labelu/internal/adapter/routers/sample.py @@ -62,8 +62,8 @@ async def list_by( task_id: int, after: Union[int, None] = Query(default=None, gt=0), before: Union[int, None] = Query(default=None, gt=0), - pageNo: Union[int, None] = Query(default=None, ge=0), - pageSize: Union[int, None] = 100, + page: Union[int, None] = Query(default=None, ge=0), + size: Union[int, None] = 100, sort: Union[str, None] = Query( default=None, regex="(annotated_count|state|inner_id|updated_at):(desc|asc)" ), @@ -75,7 +75,7 @@ async def list_by( Get a annotation result. """ - if len([i for i in (after, before, pageNo) if i != None]) != 1: + if len([i for i in (after, before, page) if i != None]) != 1: raise LabelUException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, code=ErrorCode.CODE_55000_SAMPLE_LIST_PARAMETERS_ERROR, @@ -87,14 +87,13 @@ async def list_by( task_id=task_id, after=after, before=before, - pageNo=pageNo, - pageSize=pageSize, + page=page, + size=size, sorting=sort, - current_user=current_user, ) # response - meta_data = MetaData(total=total, page=pageNo, size=len(data)) + meta_data = MetaData(total=total, page=page, size=len(data)) return OkRespWithMeta[List[SampleResponse]](meta_data=meta_data, data=data) @@ -116,7 +115,7 @@ async def get( # business logic data = await service.get( - db=db, task_id=task_id, sample_id=sample_id, current_user=current_user + db=db, task_id=task_id, sample_id=sample_id ) # response diff --git a/labelu/internal/adapter/routers/task.py b/labelu/internal/adapter/routers/task.py index aa2b6c38..25693db3 100644 --- a/labelu/internal/adapter/routers/task.py +++ b/labelu/internal/adapter/routers/task.py @@ -4,12 +4,13 @@ from fastapi import APIRouter, status, Depends, Security from fastapi.security import HTTPAuthorizationCredentials +from labelu.internal.application.response.user import UserResponse from labelu.internal.common import db from labelu.internal.common.security import security from labelu.internal.domain.models.user import User from labelu.internal.dependencies.user import get_current_user from labelu.internal.application.service import task as service -from labelu.internal.application.command.task import BasicConfigCommand +from labelu.internal.application.command.task import BasicConfigCommand, CollaboratorBatchCommand, CollaboratorUpdateCommand from labelu.internal.application.command.task import UpdateCommand from labelu.internal.application.response.base import OkResp from labelu.internal.application.response.base import MetaData @@ -91,6 +92,116 @@ async def get( # response return OkResp[TaskResponseWithStatics](data=data) +@router.get( + "/{task_id}/collaborators", + response_model=OkResp[List[UserResponse]], + status_code=status.HTTP_200_OK, +) +async def get_collaborators( + task_id: int, + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + Get task detail. + """ + + # business logic + data = await service.get_collaborators(db=db, task_id=task_id, current_user=current_user) + + # response + return OkResp[List[UserResponse]](data=data) + +@router.post( + "/{task_id}/collaborators", + response_model=OkResp[UserResponse], + status_code=status.HTTP_201_CREATED, +) +async def add_collaborator( + task_id: int, + cmd: CollaboratorUpdateCommand, + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + Add collaborator to task. + """ + + # business logic + data = await service.add_collaborator(db=db, task_id=task_id, user_id=cmd.user_id, current_user=current_user) + + # response + return OkResp[UserResponse](data=data) + +@router.post( + "/{task_id}/collaborators/batch_add", + response_model=OkResp[List[UserResponse]], + status_code=status.HTTP_201_CREATED, +) +async def batch_add_collaborator_request( + task_id: int, + cmd: CollaboratorBatchCommand, + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + Add collaborators to task. + """ + + # business logic + data = await service.batch_add_collaborators(db=db, task_id=task_id, user_ids=cmd.user_ids, current_user=current_user) + + # response + return OkResp[List[UserResponse]](data=data) + +@router.post( + "/{task_id}/collaborators/batch_remove", + response_model=OkResp[CommonDataResp], + status_code=status.HTTP_201_CREATED, +) +async def batch_remove_collaborator_request( + task_id: int, + cmd: CollaboratorBatchCommand, + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + Add collaborators to task. + """ + + # business logic + data = await service.batch_remove_collaborators(db=db, task_id=task_id, user_ids=cmd.user_ids, current_user=current_user) + + # response + return OkResp[CommonDataResp](data=data) + + +@router.delete( + "/{task_id}/collaborators/{user_id}", + response_model=OkResp[CommonDataResp], + status_code=status.HTTP_200_OK, +) +async def remove_collaborator( + task_id: int, + user_id: int, + authorization: HTTPAuthorizationCredentials = Security(security), + db: Session = Depends(db.get_db), + current_user: User = Depends(get_current_user), +): + """ + Remove collaborator from task. + """ + + # business logic + data = await service.remove_collaborator(db=db, task_id=task_id, user_id=user_id, current_user=current_user) + + # response + return OkResp[CommonDataResp](data=data) + @router.patch( "/{task_id}", diff --git a/labelu/internal/adapter/routers/user.py b/labelu/internal/adapter/routers/user.py index 91eefd17..462198b4 100644 --- a/labelu/internal/adapter/routers/user.py +++ b/labelu/internal/adapter/routers/user.py @@ -1,3 +1,4 @@ +from typing import List from sqlalchemy.orm import Session from fastapi import APIRouter, status, Depends, Security from fastapi.security import HTTPAuthorizationCredentials @@ -6,20 +7,55 @@ from labelu.internal.domain.models.user import User from labelu.internal.common.security import security from labelu.internal.dependencies.user import get_current_user -from labelu.internal.application.response.base import OkResp +from labelu.internal.application.response.base import MetaData, OkResp, OkRespWithMeta from labelu.internal.application.service import user as service from labelu.internal.application.command.user import LoginCommand from labelu.internal.application.command.user import SignupCommand from labelu.internal.application.response.user import LoginResponse from labelu.internal.application.response.user import LogoutResponse -from labelu.internal.application.response.user import SignupResponse +from labelu.internal.application.response.user import UserResponse router = APIRouter(prefix="/users", tags=["users"]) +@router.get('', response_model=OkRespWithMeta[List[UserResponse]], status_code=status.HTTP_200_OK) +async def list_users( + authorization: HTTPAuthorizationCredentials = Security(security), + current_user: User = Depends(get_current_user), + db: Session = Depends(db.get_db), + page: int | None = 0, + size: int | None = 10, + username: str | None = None, +): + """ + List all users + """ + + # business logic + data, total = await service.list_by(db=db, page=page, size=size, username=username) + + meta_data = MetaData(total=total, page=page, size=len(data)) + + # response + return OkRespWithMeta[List[UserResponse]](meta_data=meta_data, data=data) + +@router.get('/me', response_model=OkResp[UserResponse], status_code=status.HTTP_200_OK) +async def get_me( + authorization: HTTPAuthorizationCredentials = Security(security), + current_user: User = Depends(get_current_user), +): + """ + Get current user information + """ + + # response + return OkResp[UserResponse](data=UserResponse( + id=current_user.id, + username=current_user.username, + )) @router.post( "/signup", - response_model=OkResp[SignupResponse], + response_model=OkResp[UserResponse], status_code=status.HTTP_201_CREATED, ) async def signup(cmd: SignupCommand, db: Session = Depends(db.get_db)): @@ -31,7 +67,7 @@ async def signup(cmd: SignupCommand, db: Session = Depends(db.get_db)): data = await service.signup(db=db, cmd=cmd) # response - return OkResp[SignupResponse](data=data) + return OkResp[UserResponse](data=data) @router.post( diff --git a/labelu/internal/adapter/ws/__init__.py b/labelu/internal/adapter/ws/__init__.py new file mode 100644 index 00000000..50d1e4bb --- /dev/null +++ b/labelu/internal/adapter/ws/__init__.py @@ -0,0 +1,6 @@ +from fastapi import FastAPI + +from labelu.internal.adapter.ws.sample import task_sample_ws_endpoint + +def add_ws_router(app: FastAPI): + app.add_api_websocket_route("/ws/task/{task_id}/{sample_id}", task_sample_ws_endpoint) \ No newline at end of file diff --git a/labelu/internal/adapter/ws/sample.py b/labelu/internal/adapter/ws/sample.py new file mode 100644 index 00000000..4a390b3b --- /dev/null +++ b/labelu/internal/adapter/ws/sample.py @@ -0,0 +1,74 @@ +from fastapi import Depends, WebSocket, WebSocketDisconnect +from loguru import logger +from pydantic import BaseModel + +from labelu.internal.clients.ws import sampleConnections +from labelu.internal.common.websocket import ConnectionData +from labelu.internal.dependencies.user import verify_ws_token + +class TaskWsPayload(BaseModel): + task_id: int + user_id: int + username: str + +class TaskSampleWsPayload(TaskWsPayload): + sample_id: int + +def get_task_sample_connection_payloads(conns: ConnectionData): + result = [] + + if not conns: + return { + "type": "active_connections", + "data": [] + } + + for conn in conns: + result.append({ + "task_id": conn.data.task_id, + "user_id": conn.data.user_id, + "username": conn.data.username, + "sample_id": conn.data.sample_id, + }) + + return { + "type": "active_connections", + "data": result + } + +async def task_sample_ws_endpoint(websocket: WebSocket, task_id: int, sample_id: int, user=Depends(verify_ws_token)): + if not user: + return + + client_id = f"{task_id}_{sample_id}" + + async def tell_others(): + connections = sampleConnections.active_connections + current_connection = connections.get(client_id, []) + sample_payload = get_task_sample_connection_payloads(current_connection) + + await sampleConnections.send_message(sample_payload, client_id) + + try: + await sampleConnections.connect( + websocket, + client_id, + data=TaskSampleWsPayload( + task_id=task_id, + sample_id=sample_id, + user_id=user.id, + username=user.username + ) + ) + await tell_others() + + while True: + await websocket.receive_text() + + except WebSocketDisconnect: + await sampleConnections.disconnect(websocket, client_id) + await tell_others() + except Exception as e: + logger.error(f"WebSocket error for user {client_id}: {e}") + await sampleConnections.disconnect(websocket, client_id) + await tell_others() \ No newline at end of file diff --git a/labelu/internal/application/command/attachment.py b/labelu/internal/application/command/attachment.py index 998dbf8e..db187a91 100644 --- a/labelu/internal/application/command/attachment.py +++ b/labelu/internal/application/command/attachment.py @@ -1,10 +1,7 @@ -from typing import List, Union +from typing import List from pydantic import BaseModel, Field from fastapi import UploadFile, File -from labelu.internal.domain.models.task import MediaType - - class AttachmentCommand(BaseModel): file: UploadFile = File() diff --git a/labelu/internal/application/command/task.py b/labelu/internal/application/command/task.py index 9ae41e41..7d2972c5 100644 --- a/labelu/internal/application/command/task.py +++ b/labelu/internal/application/command/task.py @@ -1,4 +1,4 @@ -from typing import Union +from typing import List, Union from pydantic import BaseModel, Field from labelu.internal.domain.models.task import MediaType @@ -16,6 +16,11 @@ class BasicConfigCommand(BaseModel): default=None, description="description: task tips", max_length=1000 ) +class CollaboratorUpdateCommand(BaseModel): + user_id: int = Field(description="description: user id") + +class CollaboratorBatchCommand(BaseModel): + user_ids: List[int] = Field(description="description: user ids") class UpdateCommand(BaseModel): name: Union[str, None] = Field( diff --git a/labelu/internal/application/response/sample.py b/labelu/internal/application/response/sample.py index 391bafb2..38206f00 100644 --- a/labelu/internal/application/response/sample.py +++ b/labelu/internal/application/response/sample.py @@ -41,6 +41,6 @@ class SampleResponse(BaseModel): updated_at: Union[datetime, None] = Field( default=None, description="description: task updated at time" ) - updated_by: Union[UserResp, None] = Field( - default=None, description="description: task updated by" + updaters: List[UserResp] | None = Field( + default=[], description="description: sample updaters" ) diff --git a/labelu/internal/application/response/user.py b/labelu/internal/application/response/user.py index 10647b77..34fd9cac 100644 --- a/labelu/internal/application/response/user.py +++ b/labelu/internal/application/response/user.py @@ -1,7 +1,6 @@ from pydantic import BaseModel, EmailStr, Field - -class SignupResponse(BaseModel): +class UserResponse(BaseModel): id: int username: EmailStr diff --git a/labelu/internal/application/service/pre_annotation.py b/labelu/internal/application/service/pre_annotation.py index bf5642a1..64000566 100644 --- a/labelu/internal/application/service/pre_annotation.py +++ b/labelu/internal/application/service/pre_annotation.py @@ -69,7 +69,7 @@ async def create( for _item in pre_annotation_contents: sample_name = _item.get("sample_name") if pre_annotation_file.filename.endswith(".jsonl") else _item.get("fileName") - exist_pre_annotations = crud_pre_annotation.list_by_task_id_and_owner_id_and_sample_name(db=db, task_id=task_id, owner_id=current_user.id, sample_name=sample_name) + exist_pre_annotations = crud_pre_annotation.list_by_task_id_and_owner_id_and_sample_name(db=db, task_id=task_id, sample_name=sample_name) if len(exist_pre_annotations) > 0: raise LabelUException( @@ -101,8 +101,8 @@ async def list_by( sample_name: Optional[str], after: Optional[int], before: Optional[int], - pageNo: Optional[int], - pageSize: int, + page: Optional[int], + size: int, sorting: Optional[str], current_user: User, ) -> Tuple[List[PreAnnotationResponse], int]: @@ -110,12 +110,11 @@ async def list_by( pre_annotations, total = crud_pre_annotation.list_by( db=db, task_id=task_id, - owner_id=current_user.id, after=after, before=before, - pageNo=pageNo, + page=page, sample_name=sample_name, - pageSize=None if sample_name else pageSize, + size=None if sample_name else size, sorting=sorting, ) @@ -142,16 +141,15 @@ async def list_pre_annotation_files( task_id: Optional[int], after: Optional[int], before: Optional[int], - pageNo: Optional[int], - pageSize: int, + page: Optional[int], + size: int, sorting: Optional[str], - current_user: User, ) -> Tuple[List[TaskAttachment], int]: try: - pre_annotations = crud_pre_annotation.list_by_task_id_and_owner_id(db=db, task_id=task_id, owner_id=current_user.id) + pre_annotations = crud_pre_annotation.list_by_task_id_and_owner_id(db=db, task_id=task_id) file_ids = [pre_annotation.file_id for pre_annotation in pre_annotations] - attachments, total = crud_attachment.list_by(db=db, ids=file_ids, after=after, before=before, pageNo=pageNo, pageSize=pageSize, sorting=sorting) + attachments, total = crud_attachment.list_by(db=db, ids=file_ids, after=after, before=before, page=page, size=size, sorting=sorting) _attachment_ids = [attachment.id for attachment in attachments] sample_names_those_has_pre_annotations = [] @@ -220,7 +218,7 @@ async def delete_pre_annotation_file( file_full_path = Path(settings.MEDIA_ROOT).joinpath(attachment.path) os.remove(file_full_path) - pre_annotations = crud_pre_annotation.list_by_task_id_and_file_id(db=db, task_id=task_id, owner_id=current_user.id, file_id=file_id) + pre_annotations = crud_pre_annotation.list_by_task_id_and_file_id(db=db, task_id=task_id, file_id=file_id) pre_annotation_ids = [pre_annotation.id for pre_annotation in pre_annotations] crud_pre_annotation.delete(db=db, pre_annotation_ids=pre_annotation_ids) diff --git a/labelu/internal/application/service/sample.py b/labelu/internal/application/service/sample.py index ae5f374b..54cd076e 100644 --- a/labelu/internal/application/service/sample.py +++ b/labelu/internal/application/service/sample.py @@ -30,16 +30,15 @@ from labelu.internal.application.response.sample import SampleResponse from labelu.internal.application.response.attachment import AttachmentResponse -def is_sample_pre_annotated(db: Session, task_id: int, current_user: User, sample_name: str | None = None) -> Tuple[List[TaskPreAnnotation], int]: +def is_sample_pre_annotated(db: Session, task_id: int, sample_name: str | None = None) -> Tuple[List[TaskPreAnnotation], int]: if sample_name is None: return False _, total = crud_pre_annotation.list_by( db=db, task_id=task_id, - owner_id=current_user.id, sample_name=sample_name, - pageSize=1, + size=1, ) return total > 0 @@ -85,58 +84,49 @@ async def list_by( task_id: Union[int, None], after: Union[int, None], before: Union[int, None], - pageNo: Union[int, None], - pageSize: int, + page: Union[int, None], + size: int, sorting: Union[str, None], - current_user: User, ) -> Tuple[List[SampleResponse], int]: - try: - samples = crud_sample.list_by( - db=db, - task_id=task_id, - owner_id=current_user.id, - after=after, - before=before, - pageNo=pageNo, - pageSize=pageSize, - sorting=sorting, - ) + samples = crud_sample.list_by( + db=db, + task_id=task_id, + after=after, + before=before, + page=page, + size=size, + sorting=sorting, + ) - total = crud_sample.count(db=db, task_id=task_id, owner_id=current_user.id) + total = crud_sample.count(db=db, task_id=task_id) - # response - return [ - SampleResponse( - id=sample.id, - inner_id=sample.inner_id, - state=sample.state, - data=json.loads(sample.data), - annotated_count=sample.annotated_count, - is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename if sample.file else None), - file=AttachmentResponse(id=sample.file.id, filename=sample.file.filename, url=sample.file.url) if sample.file else None, - created_at=sample.created_at, - created_by=UserResp( - id=sample.owner.id, - username=sample.owner.username, - ), - updated_at=sample.updated_at, - updated_by=UserResp( - id=sample.updater.id, - username=sample.updater.username, - ), - ) - for sample in samples - ], total - except Exception as e: - logger.error(e) - raise LabelUException( - code=ErrorCode.CODE_55000_SAMPLE_LIST_PARAMETERS_ERROR, - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + # response + return [ + SampleResponse( + id=sample.id, + inner_id=sample.inner_id, + state=sample.state, + data=json.loads(sample.data), + annotated_count=sample.annotated_count, + is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, sample_name=sample.file.filename if sample.file else None), + file=AttachmentResponse(id=sample.file.id, filename=sample.file.filename, url=sample.file.url) if sample.file else None, + created_at=sample.created_at, + created_by=UserResp( + id=sample.owner.id, + username=sample.owner.username, + ), + updated_at=sample.updated_at, + updaters=[UserResp( + id=updater.id, + username=updater.username, + ) for updater in sample.updaters], ) + for sample in samples + ], total async def get( - db: Session, task_id: int, sample_id: int, current_user: User + db: Session, task_id: int, sample_id: int ) -> SampleResponse: sample = crud_sample.get( db=db, @@ -156,7 +146,7 @@ async def get( inner_id=sample.inner_id, state=sample.state, data=json.loads(sample.data), - is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename if sample.file else None), + is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, sample_name=sample.file.filename if sample.file else None), file=AttachmentResponse(id=sample.file.id, filename=sample.file.filename, url=sample.file.url) if sample.file else None, annotated_count=sample.annotated_count, created_at=sample.created_at, @@ -165,10 +155,10 @@ async def get( username=sample.owner.username, ), updated_at=sample.updated_at, - updated_by=UserResp( - id=sample.updater.id, - username=sample.updater.username, - ), + updaters=[UserResp( + id=updater.id, + username=updater.username, + ) for updater in sample.updaters], ) @@ -215,7 +205,7 @@ async def patch( # update task status if task.status != TaskStatus.FINISHED.value: statics = crud_sample.statics( - db=db, owner_id=current_user.id, task_ids=[task_id] + db=db, task_ids=[task_id] ) task_obj_in = {Task.status.key: TaskStatus.INPROGRESS.value} new_sample_cnt = statics.get(f"{task.id}_{SampleState.NEW.value}", 0) @@ -225,6 +215,9 @@ async def patch( task_obj_in[Task.status.key] = TaskStatus.FINISHED.value if task.status != task_obj_in[Task.status.key]: crud_task.update(db=db, db_obj=task, obj_in=task_obj_in) + # updaters + if current_user not in sample.updaters: + sample.updaters.append(current_user) # update task sample result updated_sample = crud_sample.update(db=db, db_obj=sample, obj_in=sample_obj_in) @@ -234,7 +227,7 @@ async def patch( inner_id=updated_sample.inner_id, state=updated_sample.state, data=json.loads(updated_sample.data), - is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, current_user=current_user, sample_name=sample.file.filename if sample.file else None), + is_pre_annotated=is_sample_pre_annotated(db=db, task_id=task_id, sample_name=sample.file.filename if sample.file else None), annotated_count=updated_sample.annotated_count, created_at=updated_sample.created_at, created_by=UserResp( @@ -242,10 +235,10 @@ async def patch( username=updated_sample.owner.username, ), updated_at=updated_sample.updated_at, - updated_by=UserResp( - id=updated_sample.updater.id, - username=updated_sample.updater.username, - ), + updaters=[UserResp( + id=updater.id, + username=updater.username, + ) for updater in updated_sample.updaters], ) diff --git a/labelu/internal/application/service/task.py b/labelu/internal/application/service/task.py index 4e6a8c06..974c315c 100644 --- a/labelu/internal/application/service/task.py +++ b/labelu/internal/application/service/task.py @@ -13,7 +13,7 @@ from labelu.internal.domain.models.task import Task from labelu.internal.domain.models.task import TaskStatus from labelu.internal.domain.models.sample import SampleState -from labelu.internal.adapter.persistence import crud_task +from labelu.internal.adapter.persistence import crud_task, crud_user from labelu.internal.adapter.persistence import crud_sample from labelu.internal.application.command.task import BasicConfigCommand from labelu.internal.application.command.task import UpdateCommand @@ -65,16 +65,12 @@ async def list_by( page: int, size: int, ) -> Tuple[List[TaskResponseWithStatics], int]: - - # get task total count - total_task = crud_task.count(db=db, owner_id=current_user.id) - # get task list - tasks = crud_task.list_by(db=db, owner_id=current_user.id, page=page, size=size) + tasks, total_task = crud_task.list_by(db=db, owner_id=current_user.id, page=page, size=size) # get progress task_ids = [task.id for task in tasks] - statics = crud_sample.statics(db=db, owner_id=current_user.id, task_ids=task_ids) + statics = crud_sample.statics(db=db, task_ids=task_ids) # response tasks_with_statics = [ @@ -116,7 +112,6 @@ async def get(db: Session, task_id: int, current_user: User) -> TaskResponseWith # get progress statics = crud_sample.statics( db=db, - owner_id=current_user.id, task_ids=[task.id], ) @@ -141,6 +136,166 @@ async def get(db: Session, task_id: int, current_user: User) -> TaskResponseWith ), ) +async def get_collaborators(db: Session, task_id: int, current_user: User): + task = crud_task.get(db=db, task_id=task_id) + if not task: + logger.error("cannot find task:{}", task_id) + raise LabelUException( + code=ErrorCode.CODE_50002_TASK_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + return [UserResp(id=user.id, username=user.username) for user in task.collaborators] + +async def add_collaborator(db: Session, task_id: int, user_id: int, current_user: User): + task = crud_task.get(db=db, task_id=task_id) + if not task: + logger.error("cannot find task:{}", task_id) + raise LabelUException( + code=ErrorCode.CODE_50002_TASK_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + if task.created_by != current_user.id: + logger.error( + "cannot add collaborator, the task owner is:{}, the add operator is:{}", + task.created_by, + current_user.id, + ) + raise LabelUException( + code=ErrorCode.CODE_30001_NO_PERMISSION, + status_code=status.HTTP_403_FORBIDDEN, + ) + + user = crud_user.get(db=db, id=user_id) + if not user: + logger.error("cannot find user:{}", user_id) + raise LabelUException( + code=ErrorCode.CODE_50001_USER_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + # collaborator exists + if user in task.collaborators: + logger.error("collaborator already exists:{}", user_id) + raise LabelUException( + code=ErrorCode.CODE_50003_COLLABORATOR_ALREADY_EXISTS, + status_code=status.HTTP_400_BAD_REQUEST, + ) + + with db.begin(): + task.collaborators.append(user) + + return UserResp(id=user.id, username=user.username) + +async def batch_add_collaborators(db: Session, task_id: int, user_ids: List[int], current_user: User): + task = crud_task.get(db=db, task_id=task_id) + if not task: + logger.error("cannot find task:{}", task_id) + raise LabelUException( + code=ErrorCode.CODE_50002_TASK_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + if task.created_by != current_user.id: + logger.error( + "cannot add collaborator, the task owner is:{}, the add operator is:{}", + task.created_by, + current_user.id, + ) + raise LabelUException( + code=ErrorCode.CODE_30001_NO_PERMISSION, + status_code=status.HTTP_403_FORBIDDEN, + ) + + user_ids = set(user_ids) - set([user.id for user in task.collaborators]) - set([task.created_by]) + users = crud_user.list_by_ids(db=db, ids=user_ids) + if len(users) != len(user_ids): + logger.error("cannot find users:{}", user_ids) + raise LabelUException( + code=ErrorCode.CODE_40002_USER_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + with db.begin(): + task.collaborators.extend(users) + + return [UserResp(id=user.id, username=user.username) for user in users] + +async def remove_collaborator(db: Session, task_id: int, user_id: int, current_user: User): + task = crud_task.get(db=db, task_id=task_id) + if not task: + logger.error("cannot find task:{}", task_id) + raise LabelUException( + code=ErrorCode.CODE_50002_TASK_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + if task.created_by != current_user.id: + logger.error( + "cannot remove collaborator, the task owner is:{}, the remove operator is:{}", + task.created_by, + current_user.id, + ) + raise LabelUException( + code=ErrorCode.CODE_30001_NO_PERMISSION, + status_code=status.HTTP_403_FORBIDDEN, + ) + + user = crud_user.get(db=db, id=user_id) + if not user: + logger.error("cannot find user:{}", user_id) + raise LabelUException( + code=ErrorCode.CODE_40002_USER_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + if user not in task.collaborators: + logger.error("cannot find collaborator:{}", user_id) + raise LabelUException( + code=ErrorCode.CODE_50004_COLLABORATOR_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + with db.begin(): + task.collaborators.remove(user) + + return CommonDataResp(ok=True) + +async def batch_remove_collaborators(db: Session, task_id: int, user_ids: List[int], current_user: User): + task = crud_task.get(db=db, task_id=task_id) + if not task: + logger.error("cannot find task:{}", task_id) + raise LabelUException( + code=ErrorCode.CODE_50002_TASK_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + if task.created_by != current_user.id: + logger.error( + "cannot remove collaborator, the task owner is:{}, the remove operator is:{}", + task.created_by, + current_user.id, + ) + raise LabelUException( + code=ErrorCode.CODE_30001_NO_PERMISSION, + status_code=status.HTTP_403_FORBIDDEN, + ) + + users = crud_user.list_by_ids(db=db, ids=user_ids) + if len(users) != len(user_ids): + logger.error("cannot find users:{}", user_ids) + raise LabelUException( + code=ErrorCode.CODE_40002_USER_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + with db.begin(): + for user in users: + if user in task.collaborators: + task.collaborators.remove(user) + + return CommonDataResp(ok=True) async def update(db: Session, task_id: int, cmd: UpdateCommand) -> TaskResponse: diff --git a/labelu/internal/application/service/user.py b/labelu/internal/application/service/user.py index 7944a2d2..d683c039 100644 --- a/labelu/internal/application/service/user.py +++ b/labelu/internal/application/service/user.py @@ -1,4 +1,5 @@ from datetime import timedelta +from typing import List, Tuple from loguru import logger from fastapi import status @@ -15,11 +16,11 @@ from labelu.internal.common.security import create_access_token from labelu.internal.application.command.user import LoginCommand from labelu.internal.application.command.user import SignupCommand -from labelu.internal.application.response.user import SignupResponse +from labelu.internal.application.response.user import UserResponse from labelu.internal.application.response.user import LoginResponse -async def signup(db: Session, cmd: SignupCommand) -> SignupResponse: +async def signup(db: Session, cmd: SignupCommand) -> UserResponse: # check user alredy exists user = crud_user.get_user_by_username(db, username=cmd.username) if user: @@ -40,8 +41,17 @@ async def signup(db: Session, cmd: SignupCommand) -> SignupResponse: ) # response - return SignupResponse(id=user.id, username=user.username) + return UserResponse(id=user.id, username=user.username) +async def list_by(db: Session, page: int, size: int, username: str = None) -> Tuple[List[UserResponse], int]: + # get all users + users, total = crud_user.list_by(db, page, size, username) + + # response + return [ + UserResponse(id=user.id, username=user.username) + for user in users + ], total async def login(db: Session, cmd: LoginCommand) -> LoginResponse: diff --git a/labelu/internal/clients/__init__.py b/labelu/internal/clients/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/labelu/internal/clients/ws.py b/labelu/internal/clients/ws.py new file mode 100644 index 00000000..7315b4ff --- /dev/null +++ b/labelu/internal/clients/ws.py @@ -0,0 +1,4 @@ +from labelu.internal.common.websocket import ConnectionManager + + +sampleConnections = ConnectionManager() \ No newline at end of file diff --git a/labelu/internal/common/config.py b/labelu/internal/common/config.py index b628a546..1164d569 100644 --- a/labelu/internal/common/config.py +++ b/labelu/internal/common/config.py @@ -28,7 +28,7 @@ class Settings(BaseSettings): default=f"sqlite:///{BASE_DATA_DIR}/labelu.sqlite", description="Database connection URL. Supports SQLite and MySQL." ) - # or using DATABASE_URL=mysql://labelu:labelupass@10.1.52.75/labeludb + # or using MySQL DATABASE_URL=mysql://labelu:labelupass@localhost/labeludb PASSWORD_SECRET_KEY: str = ( "e5b7d00a59aaa2a5ea86a7c4d72f856b20bafa1b8d0e66124082ada81f6340bd" @@ -40,7 +40,7 @@ class Settings(BaseSettings): @property def need_migration_to_mysql(self) -> bool: - sqlite_path = Path(self.BASE_DATA_DIR) / "labelu.sqlite" + sqlite_path = Path(self.BASE_DATA_DIR) / "labelu-test-1.sqlite" return ( self.DATABASE_URL.startswith('mysql') and sqlite_path.exists() diff --git a/labelu/internal/common/error_code.py b/labelu/internal/common/error_code.py index 3b6b1f04..fc0707c6 100644 --- a/labelu/internal/common/error_code.py +++ b/labelu/internal/common/error_code.py @@ -54,7 +54,7 @@ class ErrorCode(Enum): ) CODE_40001_USERNAME_ALREADY_EXISTS = ( USER_INIT_CODE + 1, - "Username Aready exists in the system", + "Username Already exists in the system", ) CODE_40002_USER_NOT_FOUND = (USER_INIT_CODE + 2, "User not found") CODE_40003_CREDENTIAL_ERROR = (USER_INIT_CODE + 3, "Could not validate credentials") @@ -64,7 +64,14 @@ class ErrorCode(Enum): CODE_50000_TASK_ERROR = (TASK_INIT_CODE, "Internal Error") CODE_50001_TASK_FINISHED_ERROR = (TASK_INIT_CODE + 1, "Task is finished") CODE_50002_TASK_NOT_FOUND = (TASK_INIT_CODE + 2, "Task not found") + CODE_50003_COLLABORATOR_ALREADY_EXISTS = ( + TASK_INIT_CODE + 3, + "Collaborator already exists") + CODE_50004_COLLABORATOR_NOT_FOUND = ( + TASK_INIT_CODE + 4, + "Collaborator not found", + ) # task attachment error code CODE_51000_CREATE_ATTACHMENT_ERROR = ( TASK_INIT_CODE + 1000, @@ -82,7 +89,7 @@ class ErrorCode(Enum): # task sample error code CODE_55000_SAMPLE_LIST_PARAMETERS_ERROR = ( TASK_INIT_CODE + 5000, - "Paramenters error: 'after', 'before', 'pageNo' only one must be Ture, pageNo can be 0", + "Paramenters error: 'after', 'before', 'page' only one must be Ture, page can be 0", ) CODE_55001_SAMPLE_NOT_FOUND = (TASK_INIT_CODE + 5001, "Sample not found") CODE_55002_SAMPLE_FORMAT_ERROR = ( @@ -171,9 +178,19 @@ async def validation_exception_handler(request, exc): }, ) +async def unexpected_exception_handler(request: Request, exc: Exception): + logger.error(exc) + return JSONResponse( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + content={ + "msg": str(exc), + "err_code": ErrorCode.CODE_30000_SQL_ERROR.value[0], + }, + ) def add_exception_handler(app: FastAPI): app.add_exception_handler(RequestValidationError, validation_exception_handler) app.add_exception_handler(LabelUException, labelu_exception_handler) app.add_exception_handler(StarletteHTTPException, http_exception_handler) app.add_exception_handler(SQLAlchemyError, sqlalchemy_exception_handler) + app.add_exception_handler(Exception, unexpected_exception_handler) diff --git a/labelu/internal/common/websocket.py b/labelu/internal/common/websocket.py new file mode 100644 index 00000000..518e594f --- /dev/null +++ b/labelu/internal/common/websocket.py @@ -0,0 +1,50 @@ +from typing import Any, Dict, List +from fastapi import WebSocket +from loguru import logger +from dataclasses import dataclass + +@dataclass +class ConnectionData: + ws: WebSocket + data: Any = None + +class ConnectionManager: + def __init__(self): + self.active_connections: Dict[str, List[ConnectionData]] = {} + + async def connect(self, websocket: WebSocket, client_id: str, data: Any = None): + await websocket.accept() + + if client_id not in self.active_connections: + self.active_connections[client_id] = [] + + logger.info(f"Client {client_id} connected") + self.active_connections[client_id].append(ConnectionData(ws=websocket, data=data)) + + async def disconnect(self, websocket: WebSocket, client_id: str): + if client_id in self.active_connections: + self.active_connections[client_id] = [ + connection for connection in self.active_connections[client_id] + if connection.ws != websocket + ] + + if not self.active_connections[client_id]: + del self.active_connections[client_id] + + logger.info(f"Client {client_id} disconnected") + + async def send_message(self, message: dict, client_id: str): + if client_id in self.active_connections: + for connection in self.active_connections[client_id]: + await connection.ws.send_json(message) + + async def broadcast(self, message: dict, exclude: str = None): + for client_id, connections in self.active_connections.items(): + if client_id != exclude: + for connection in connections: + await connection.ws.send_json(message) + + async def send_personal_message(self, message: dict, client_id: str): + if client_id in self.active_connections: + for connection in self.active_connections[client_id]: + await connection.ws.send_json(message) \ No newline at end of file diff --git a/labelu/internal/dependencies/user.py b/labelu/internal/dependencies/user.py index 2f094ff7..509cdb3d 100644 --- a/labelu/internal/dependencies/user.py +++ b/labelu/internal/dependencies/user.py @@ -1,9 +1,10 @@ from typing import Optional from jose import jwt +from loguru import logger from sqlalchemy.orm import Session from pydantic import ValidationError -from fastapi import status, Depends, Request +from fastapi import HTTPException, WebSocket, status, Depends, Request from labelu.internal.domain.models.user import User from labelu.internal.common import db @@ -44,3 +45,33 @@ def get_current_user( if not user: raise LabelUException(code=ErrorCode.CODE_40002_USER_NOT_FOUND, status_code=404) return user + + +async def verify_ws_token(websocket: WebSocket, db: Session = Depends(db.get_db)): + try: + token = websocket.query_params.get('token') + + if not token: + await websocket.close(code=4001, reason="Missing authentication token") + raise HTTPException(status_code=401, detail="Missing authentication token") + + payload = jwt.decode( + token, + settings.PASSWORD_SECRET_KEY, + algorithms=[settings.TOKEN_GENERATE_ALGORITHM], + options={"verify_exp": False}, + ) + + token_data = AccessToken(**payload) + user = crud_user.get(db, id=token_data.id) + + if not user: + await websocket.close(code=4002, reason="User not found") + raise HTTPException(status_code=401, detail="User not found") + + return user + + except jwt.JWTError as e: + logger.error(f"WebSocket authentication error: {str(e)}") + await websocket.close(code=4003, reason="Invalid token") + raise HTTPException(status_code=401, detail="Invalid token") \ No newline at end of file diff --git a/labelu/internal/domain/models/__init__.py b/labelu/internal/domain/models/__init__.py new file mode 100644 index 00000000..3dc80b6f --- /dev/null +++ b/labelu/internal/domain/models/__init__.py @@ -0,0 +1,7 @@ +from .attachment import TaskAttachment +from .sample import TaskSample +from .task import Task +from .user import User +from .task_collaborator import TaskCollaborator +from .task_sample_updater import TaskSampleUpdater +from .pre_annotation import TaskPreAnnotation \ No newline at end of file diff --git a/labelu/internal/domain/models/sample.py b/labelu/internal/domain/models/sample.py index feeb8b87..3cc496b8 100644 --- a/labelu/internal/domain/models/sample.py +++ b/labelu/internal/domain/models/sample.py @@ -54,6 +54,6 @@ class TaskSample(Base): file = relationship("TaskAttachment", foreign_keys=[file_id]) task = relationship("Task", foreign_keys=[task_id]) owner = relationship("User", foreign_keys=[created_by]) - updater = relationship("User", foreign_keys=[updated_by]) + updaters = relationship("User", secondary="task_sample_updater") Index("idx_sample_id_deleted_at", id, deleted_at) diff --git a/labelu/internal/domain/models/task.py b/labelu/internal/domain/models/task.py index bf954387..b24dbab3 100644 --- a/labelu/internal/domain/models/task.py +++ b/labelu/internal/domain/models/task.py @@ -60,7 +60,8 @@ class Task(Base): comment="Last time a task was updated", ) deleted_at = Column(DateTime(timezone=True), index=True, comment="Task delete time") - + + collaborators = relationship("User", secondary="task_collaborator", back_populates="tasks") owner = relationship("User", foreign_keys=[created_by]) updater = relationship("User", foreign_keys=[updated_by]) diff --git a/labelu/internal/domain/models/task_collaborator.py b/labelu/internal/domain/models/task_collaborator.py new file mode 100644 index 00000000..c1557bee --- /dev/null +++ b/labelu/internal/domain/models/task_collaborator.py @@ -0,0 +1,24 @@ +from datetime import datetime +from sqlalchemy import Column, DateTime, ForeignKey, Integer +from labelu.internal.common.db import Base + +class TaskCollaborator(Base): + __tablename__ = "task_collaborator" + + task_id = Column( + Integer, + ForeignKey("task.id", ondelete="CASCADE"), + primary_key=True, + index=True + ) + user_id = Column( + Integer, + ForeignKey("user.id", ondelete="CASCADE"), + primary_key=True, + index=True + ) + created_at = Column( + DateTime(timezone=True), + default=datetime.now, + comment="Time a task collaborator was created" + ) \ No newline at end of file diff --git a/labelu/internal/domain/models/task_sample_updater.py b/labelu/internal/domain/models/task_sample_updater.py new file mode 100644 index 00000000..8055023d --- /dev/null +++ b/labelu/internal/domain/models/task_sample_updater.py @@ -0,0 +1,26 @@ +from datetime import datetime +from sqlalchemy import Column, DateTime, ForeignKey, Integer +from labelu.internal.common.db import Base + +class TaskSampleUpdater(Base): + __tablename__ = "task_sample_updater" + + sample_id = Column( + Integer, + ForeignKey("task_sample.id", ondelete="CASCADE"), + primary_key=True, + index=True + ) + user_id = Column( + Integer, + ForeignKey("user.id", ondelete="CASCADE"), + primary_key=True, + index=True + ) + created_at = Column( + DateTime(timezone=True), + default=datetime.now, + onupdate=datetime.now, + nullable=False, + comment="created time" + ) \ No newline at end of file diff --git a/labelu/internal/domain/models/user.py b/labelu/internal/domain/models/user.py index cbb93a63..07c18eeb 100644 --- a/labelu/internal/domain/models/user.py +++ b/labelu/internal/domain/models/user.py @@ -1,6 +1,7 @@ from datetime import datetime from sqlalchemy import Column, DateTime, Integer, String +from sqlalchemy.orm import relationship from labelu.internal.common.db import Base @@ -17,3 +18,4 @@ class User(Base): updated_at = Column( DateTime(timezone=True), default=datetime.now, comment="Last time a task was updated" ) + tasks = relationship("Task", secondary="task_collaborator", back_populates="collaborators") diff --git a/labelu/internal/middleware/__init__.py b/labelu/internal/middleware/__init__.py index e6a10a86..7cc21224 100644 --- a/labelu/internal/middleware/__init__.py +++ b/labelu/internal/middleware/__init__.py @@ -1,9 +1,9 @@ from fastapi import FastAPI -from labelu.internal.common.config import settings +from labelu.internal.middleware.content_type import ContentTypeMiddleware from labelu.internal.middleware.tracing import TracingMiddleWare -from fastapi.middleware.trustedhost import TrustedHostMiddleware def add_middleware(app: FastAPI): app.add_middleware(TracingMiddleWare) + app.add_middleware(ContentTypeMiddleware) diff --git a/labelu/internal/middleware/content_type.py b/labelu/internal/middleware/content_type.py new file mode 100644 index 00000000..c53fc84e --- /dev/null +++ b/labelu/internal/middleware/content_type.py @@ -0,0 +1,16 @@ +from fastapi import Request +from starlette.middleware.base import BaseHTTPMiddleware +from labelu.version import version as labelu_version + +class ContentTypeMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + response = await call_next(request) + + # add LabelU-Version header + response.headers["LabelU-Version"] = labelu_version + + # set content-type for javascript files + if request.url.path.endswith(".js"): + response.headers["content-type"] = "application/javascript" + + return response \ No newline at end of file diff --git a/labelu/main.py b/labelu/main.py index 64c2a67f..75107210 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -2,10 +2,11 @@ from loguru import logger import uvicorn from typer import Typer -from fastapi import FastAPI, Request, Response +from fastapi import FastAPI, Response from fastapi.staticfiles import StaticFiles from labelu.internal.adapter.routers import add_router +from labelu.internal.adapter.ws import add_ws_router from labelu.internal.middleware import add_middleware from labelu.internal.common.logger import init_logging from labelu.internal.common.db import init_tables @@ -25,7 +26,7 @@ * **Signup** * **Login** -* **Logout** (_not implemented_). +* **Logout**. ## Tasks @@ -94,6 +95,7 @@ run_db_migrations() add_exception_handler(app=app) add_router(app=app) +add_ws_router(app=app) add_middleware(app=app) def startup(): @@ -105,7 +107,7 @@ def startup(): class NoCacheStaticFiles(StaticFiles): def __init__(self, *args: Any, **kwargs: Any): - self.cachecontrol = "max-age=0, no-cache, no-store, , must-revalidate" + self.cachecontrol = "max-age=0, no-cache, no-store, must-revalidate" self.pragma = "no-cache" self.expires = "0" super().__init__(*args, **kwargs) @@ -123,18 +125,6 @@ def file_response(self, *args: Any, **kwargs: Any) -> Response: app.mount("", NoCacheStaticFiles(packages=["labelu.internal"], html=True)) - -@app.middleware("http") -async def add_correct_content_type(request: Request, call_next): - response = await call_next(request) - - response.headers["LabelU-Version"] = labelu_version - - if request.url.path.endswith(".js"): - response.headers["content-type"] = "application/javascript" - return response - - cli = Typer() @cli.command() @@ -152,8 +142,8 @@ def main( settings.HOST = host if media_host: settings.MEDIA_HOST = media_host - uvicorn.run(app=app, host=settings.HOST, port=settings.PORT) - - + + uvicorn.run(app=app, host=settings.HOST, port=settings.PORT, ws="websockets") + if __name__ == "__main__": cli() \ No newline at end of file diff --git a/labelu/scripts/migrate_to_mysql.py b/labelu/scripts/migrate_to_mysql.py index 41da1119..301bcf7a 100644 --- a/labelu/scripts/migrate_to_mysql.py +++ b/labelu/scripts/migrate_to_mysql.py @@ -9,19 +9,17 @@ from labelu.internal.common.config import settings class MigrationError(Exception): - """自定义迁移异常""" pass @contextmanager def timer(operation: str): - """计时器上下文管理器""" start = time.time() yield duration = time.time() - start logger.info(f"{operation} took {duration:.2f} seconds") class DatabaseMigrator: - BATCH_SIZE = 1000 # 批量插入的大小 + BATCH_SIZE = 1000 def __init__(self, sqlite_url: str, mysql_url: str): self.sqlite_engine = create_engine(sqlite_url) @@ -30,7 +28,6 @@ def __init__(self, sqlite_url: str, mysql_url: str): @contextmanager def create_sessions(self): - """创建数据库会话的上下文管理器""" sqlite_session = sessionmaker(bind=self.sqlite_engine)() mysql_session = sessionmaker(bind=self.mysql_engine)() try: @@ -40,7 +37,6 @@ def create_sessions(self): mysql_session.close() def get_tables_in_order(self) -> List[str]: - """获取考虑外键依赖的表排序""" inspector = inspect(self.sqlite_engine) tables = [] table_names = inspector.get_table_names() @@ -67,7 +63,6 @@ def get_tables_in_order(self) -> List[str]: return tables def _process_row_data(self, row: Dict[str, Any]) -> Dict[str, Any]: - """处理行数据，确保数据类型兼容性""" processed = {} for key, value in row.items(): if isinstance(value, datetime): @@ -79,7 +74,6 @@ def _process_row_data(self, row: Dict[str, Any]) -> Dict[str, Any]: return processed def _batch_insert(self, session, table: Table, rows: List[Dict[str, Any]]) -> None: - """批量插入数据""" try: processed_rows = [self._process_row_data(row._asdict()) for row in rows] session.execute(table.insert(), processed_rows) @@ -87,7 +81,6 @@ def _batch_insert(self, session, table: Table, rows: List[Dict[str, Any]]) -> No except IntegrityError as e: session.rollback() logger.error(f"Integrity error during batch insert: {e}") - # 回退到逐行插入以处理问题数据 for row in rows: try: processed_row = self._process_row_data(row._asdict()) @@ -101,7 +94,6 @@ def _batch_insert(self, session, table: Table, rows: List[Dict[str, Any]]) -> No raise MigrationError(f"Batch insert failed: {e}") def check_migration_status(self, mysql_session) -> bool: - """检查是否已经迁移""" try: result = mysql_session.execute(text("SELECT COUNT(*) FROM user")).scalar() return result > 0 @@ -109,7 +101,6 @@ def check_migration_status(self, mysql_session) -> bool: return False def migrate_table(self, table_name: str, sqlite_session, mysql_session) -> None: - """迁移单个表""" if table_name == "alembic_version": return @@ -129,7 +120,6 @@ def migrate_table(self, table_name: str, sqlite_session, mysql_session) -> None: logger.info(f"Migrated {min(batch_start + self.BATCH_SIZE, total_rows)}/{total_rows} rows in {table_name}") def migrate(self) -> None: - """执行完整的数据迁移""" with self.create_sessions() as (sqlite_session, mysql_session): if self.check_migration_status(mysql_session): logger.info("Database has already been migrated.") @@ -151,8 +141,7 @@ def migrate(self) -> None: raise MigrationError(f"Migration failed: {e}") def migrate_to_mysql(): - """主迁移函数""" - sqlite_url = f"sqlite:///{settings.BASE_DATA_DIR}/labelu.sqlite" + sqlite_url = f"sqlite:///{settings.BASE_DATA_DIR}/labelu-test-1.sqlite" try: migrator = DatabaseMigrator(sqlite_url, settings.DATABASE_URL) diff --git a/labelu/tests/internal/adapter/routers/test_pre_annotation.py b/labelu/tests/internal/adapter/routers/test_pre_annotation.py index a2117986..f6c293ef 100644 --- a/labelu/tests/internal/adapter/routers/test_pre_annotation.py +++ b/labelu/tests/internal/adapter/routers/test_pre_annotation.py @@ -173,7 +173,7 @@ def test_pre_annotation_list_by_page( r = client.get( f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations", headers=testuser_token_headers, - params={"pageNo": 0, "pageSize": 10}, + params={"page": 0, "size": 10}, ) # check @@ -251,7 +251,7 @@ def test_pre_annotation_list_with_sample_name( r = client.get( f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations?sample_name={img.json()['data']['filename']}", headers=testuser_token_headers, - params={"pageNo": 0}, + params={"page": 0}, ) # check @@ -326,7 +326,7 @@ def test_pre_annotation_list_with_sample_name_not_found( r = client.get( f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations?sample_name=asdasd.png", headers=testuser_token_headers, - params={"pageNo": 0}, + params={"page": 0}, ) # check @@ -344,7 +344,7 @@ def test_pre_annotations_list_by_params_error( r = client.get( f"{settings.API_V1_STR}/tasks/{1}/pre_annotations", headers=testuser_token_headers, - params={"after": 1, "before": 1, "pageNo": 1, "pageSize": 10}, + params={"after": 1, "before": 1, "page": 1, "size": 10}, ) # check @@ -372,7 +372,7 @@ def test_sample_list_by_sample_name_error( r = client.get( f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations?sample_name=", headers=testuser_token_headers, - params={"pageNo": 0, "pageSize": 10}, + params={"page": 0, "size": 10}, ) # check diff --git a/labelu/tests/internal/adapter/routers/test_sample.py b/labelu/tests/internal/adapter/routers/test_sample.py index 7b37588b..4a777d62 100644 --- a/labelu/tests/internal/adapter/routers/test_sample.py +++ b/labelu/tests/internal/adapter/routers/test_sample.py @@ -131,7 +131,7 @@ def test_sample_list_by_page( r = client.get( f"{settings.API_V1_STR}/tasks/{task.id}/samples", headers=testuser_token_headers, - params={"pageNo": 0, "pageSize": 10}, + params={"page": 0, "size": 10}, ) # check @@ -178,7 +178,7 @@ def test_sample_list_by_before( r = client.get( f"{settings.API_V1_STR}/tasks/{task.id}/samples", headers=testuser_token_headers, - params={"before": 13, "pageSize": 10}, + params={"before": 13, "size": 10}, ) # check @@ -226,7 +226,7 @@ def test_sample_list_by_after( r = client.get( f"{settings.API_V1_STR}/tasks/{task.id}/samples", headers=testuser_token_headers, - params={"after": 5, "pageSize": 10}, + params={"after": 5, "size": 10}, ) # check @@ -275,7 +275,7 @@ def test_sample_list_with_sort( r = client.get( f"{settings.API_V1_STR}/tasks/{task.id}/samples?sort=annotated_count:desc", headers=testuser_token_headers, - params={"after": 5, "pageSize": 10}, + params={"after": 5, "size": 10}, ) # check @@ -295,7 +295,7 @@ def test_sample_list_by_params_error( r = client.get( f"{settings.API_V1_STR}/tasks/{1}/samples", headers=testuser_token_headers, - params={"after": 1, "before": 1, "pageNo": 1, "pageSize": 10}, + params={"after": 1, "before": 1, "page": 1, "size": 10}, ) # check @@ -323,7 +323,7 @@ def test_sample_list_by_sort_error( r = client.get( f"{settings.API_V1_STR}/tasks/{task.id}/samples?sort", headers=testuser_token_headers, - params={"pageNo": 0, "pageSize": 10}, + params={"page": 0, "size": 10}, ) # check diff --git a/pyproject.toml b/pyproject.toml index fab8eafd..ef14156f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ pillow = "^9.3.0" alembic = "^1.9.4" httpx = "^0.27.0" tfrecord = "^1.14.5" +websockets = "^15.0" [tool.poetry.extras] mysql = ["mysqlclient"] From 27fcf677cc4ac85a73207c4ea771d32024719efb Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 21 Feb 2025 05:41:40 +0000 Subject: [PATCH 107/236] fix: Upgrade frontend to v5.7.0-alpha.9. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.9) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index e15b75f8..4eb816c7 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.8/frontend.zip -alpha_version: v5.7.0-alpha.8 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.9/frontend.zip +alpha_version: v5.7.0-alpha.9 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 1a4b46e353f0645721d481e3574d114b2627211a Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 21 Feb 2025 06:04:02 +0000 Subject: [PATCH 108/236] fix: Upgrade frontend to v5.7.0-alpha.10. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.10) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 4eb816c7..296d52da 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.9/frontend.zip -alpha_version: v5.7.0-alpha.9 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.10/frontend.zip +alpha_version: v5.7.0-alpha.10 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From c436b0041189cafd51eaea58871bb336d9f8619f Mon Sep 17 00:00:00 2001 From: gary Date: Fri, 21 Feb 2025 14:16:27 +0800 Subject: [PATCH 109/236] fix: update migrate script From 3591bcf2ad3a4d8fc9e9f59e13441e9eb64df99b Mon Sep 17 00:00:00 2001 From: gary Date: Fri, 21 Feb 2025 14:31:57 +0800 Subject: [PATCH 110/236] update migrate script (#191) --- ...83c9a254_add_collaborators_and_updaters.py | 90 +++++++++++-------- .../domain/models/task_collaborator.py | 7 +- .../domain/models/task_sample_updater.py | 6 +- 3 files changed, 62 insertions(+), 41 deletions(-) diff --git a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py index 076ae798..706df3b7 100644 --- a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py +++ b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py @@ -20,47 +20,59 @@ def upgrade() -> None: # Create task_collaborator table - op.create_table( - 'task_collaborator', - sa.Column('task_id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column( - 'created_at', - sa.DateTime(timezone=True), - server_default=sa.text('CURRENT_TIMESTAMP'), - nullable=False - ), - sa.ForeignKeyConstraint( - ['task_id'], - ['task.id'], - ondelete='CASCADE' - ), - sa.ForeignKeyConstraint( - ['user_id'], - ['user.id'], - ondelete='CASCADE' - ), - sa.PrimaryKeyConstraint('task_id', 'user_id') - ) - - # Performances index - op.create_index( - 'ix_task_collaborator_task_id', - 'task_collaborator', - ['task_id'] - ) - op.create_index( - 'ix_task_collaborator_user_id', - 'task_collaborator', - ['user_id'] - ) - op.create_index( - 'ix_task_created_by_deleted_at', - 'task', - ['created_by', 'deleted_at'] - ) + # if the table is not exists then create it + is_table_exist = op.execute( + "SHOW TABLES LIKE 'task_collaborator';" + ).fetchone() + if not is_table_exist: + op.create_table( + 'task_collaborator', + sa.Column('task_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column( + 'created_at', + sa.DateTime(timezone=True), + server_default=sa.text('CURRENT_TIMESTAMP'), + nullable=False + ), + sa.ForeignKeyConstraint( + ['task_id'], + ['task.id'], + ondelete='CASCADE' + ), + sa.ForeignKeyConstraint( + ['user_id'], + ['user.id'], + ondelete='CASCADE' + ), + sa.PrimaryKeyConstraint('task_id', 'user_id') + ) + + # Performances index + op.create_index( + 'ix_task_collaborator_task_id', + 'task_collaborator', + ['task_id'] + ) + op.create_index( + 'ix_task_collaborator_user_id', + 'task_collaborator', + ['user_id'] + ) + op.create_index( + 'ix_task_created_by_deleted_at', + 'task', + ['created_by', 'deleted_at'] + ) # Task sample: updater -> updaters; create a new table task_sample_updater + is_task_sample_updater_table_exist = op.execute( + "SHOW TABLES LIKE 'task_sample_updater';" + ).fetchone() + + if is_task_sample_updater_table_exist: + return + op.create_table( 'task_sample_updater', sa.Column('sample_id', sa.Integer(), nullable=False), diff --git a/labelu/internal/domain/models/task_collaborator.py b/labelu/internal/domain/models/task_collaborator.py index c1557bee..66b3f10e 100644 --- a/labelu/internal/domain/models/task_collaborator.py +++ b/labelu/internal/domain/models/task_collaborator.py @@ -1,4 +1,5 @@ from datetime import datetime +from sqlalchemy.schema import Index from sqlalchemy import Column, DateTime, ForeignKey, Integer from labelu.internal.common.db import Base @@ -21,4 +22,8 @@ class TaskCollaborator(Base): DateTime(timezone=True), default=datetime.now, comment="Time a task collaborator was created" - ) \ No newline at end of file + ) + + Index("ix_task_collaborator_task_id", task_id) + Index("ix_task_collaborator_user_id", user_id) + Index("ix_task_created_by_deleted_at", task_id, user_id) diff --git a/labelu/internal/domain/models/task_sample_updater.py b/labelu/internal/domain/models/task_sample_updater.py index 8055023d..6bbb9251 100644 --- a/labelu/internal/domain/models/task_sample_updater.py +++ b/labelu/internal/domain/models/task_sample_updater.py @@ -1,4 +1,5 @@ from datetime import datetime +from sqlalchemy.schema import Index from sqlalchemy import Column, DateTime, ForeignKey, Integer from labelu.internal.common.db import Base @@ -23,4 +24,7 @@ class TaskSampleUpdater(Base): onupdate=datetime.now, nullable=False, comment="created time" - ) \ No newline at end of file + ) + + Index("ix_task_sample_updater_sample_id", sample_id) + Index("ix_task_sample_updater_user_id", user_id) From f917091792f734ada91d67c473dde1d5c34b9e81 Mon Sep 17 00:00:00 2001 From: gary Date: Fri, 21 Feb 2025 14:46:06 +0800 Subject: [PATCH 111/236] fix: remove owner_id param in list pre_annotations (#192) --- ...2eb983c9a254_add_collaborators_and_updaters.py | 15 +++++++++++++++ .../adapter/persistence/crud_pre_annotation.py | 2 +- .../internal/domain/models/task_collaborator.py | 4 ---- .../internal/domain/models/task_sample_updater.py | 3 --- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py index 706df3b7..ce761d5a 100644 --- a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py +++ b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py @@ -49,6 +49,13 @@ def upgrade() -> None: ) # Performances index + is_index_2_exist = op.execute( + "SHOW INDEX FROM task_collaborator WHERE Key_name = 'ix_task_collaborator_task_id';" + ).fetchone() + + if is_index_2_exist: + return + op.create_index( 'ix_task_collaborator_task_id', 'task_collaborator', @@ -97,6 +104,14 @@ def upgrade() -> None: ) # Performances index + # check if the index is already exists + is_index_exist = op.execute( + "SHOW INDEX FROM task_sample_updater WHERE Key_name = 'ix_task_sample_updater_sample_id';" + ).fetchone() + + if is_index_exist: + return + op.create_index( 'ix_task_sample_updater_sample_id', 'task_sample_updater', diff --git a/labelu/internal/adapter/persistence/crud_pre_annotation.py b/labelu/internal/adapter/persistence/crud_pre_annotation.py index 3a4cfae6..83fd79df 100644 --- a/labelu/internal/adapter/persistence/crud_pre_annotation.py +++ b/labelu/internal/adapter/persistence/crud_pre_annotation.py @@ -79,7 +79,7 @@ def list_by_task_id_and_file_id(db: Session, task_id: int, file_id: int) -> List TaskPreAnnotation.file_id == file_id ).all() -def list_by_task_id_and_owner_id_and_sample_name(db: Session, task_id: int, owner_id: int, sample_name: str) -> List[TaskPreAnnotation]: +def list_by_task_id_and_owner_id_and_sample_name(db: Session, task_id: int, sample_name: str) -> List[TaskPreAnnotation]: """list pre annotations by task_id, owner_id and sample_name without pagination Args: diff --git a/labelu/internal/domain/models/task_collaborator.py b/labelu/internal/domain/models/task_collaborator.py index 66b3f10e..349bd6ef 100644 --- a/labelu/internal/domain/models/task_collaborator.py +++ b/labelu/internal/domain/models/task_collaborator.py @@ -23,7 +23,3 @@ class TaskCollaborator(Base): default=datetime.now, comment="Time a task collaborator was created" ) - - Index("ix_task_collaborator_task_id", task_id) - Index("ix_task_collaborator_user_id", user_id) - Index("ix_task_created_by_deleted_at", task_id, user_id) diff --git a/labelu/internal/domain/models/task_sample_updater.py b/labelu/internal/domain/models/task_sample_updater.py index 6bbb9251..d414a161 100644 --- a/labelu/internal/domain/models/task_sample_updater.py +++ b/labelu/internal/domain/models/task_sample_updater.py @@ -25,6 +25,3 @@ class TaskSampleUpdater(Base): nullable=False, comment="created time" ) - - Index("ix_task_sample_updater_sample_id", sample_id) - Index("ix_task_sample_updater_user_id", user_id) From 047380a7b1696e46c7f2a69dc78a10b644928dcb Mon Sep 17 00:00:00 2001 From: gary Date: Fri, 21 Feb 2025 20:00:19 +0800 Subject: [PATCH 112/236] update: update migration script (#193) --- labelu/alembic_labelu/alembic_labelu_tools.py | 36 ++--- ...83c9a254_add_collaborators_and_updaters.py | 138 ++++++++---------- ..._replace_key_with_value_in_sample_table.py | 35 ++++- ...8fcb35b66b_add_media_and_pre_annotation.py | 19 +-- .../versions/e76c2ca5562e_add_inner_id.py | 18 +-- labelu/internal/application/service/task.py | 12 ++ labelu/internal/dependencies/user.py | 2 +- 7 files changed, 124 insertions(+), 136 deletions(-) diff --git a/labelu/alembic_labelu/alembic_labelu_tools.py b/labelu/alembic_labelu/alembic_labelu_tools.py index 0ed629e8..ff6a1aff 100644 --- a/labelu/alembic_labelu/alembic_labelu_tools.py +++ b/labelu/alembic_labelu/alembic_labelu_tools.py @@ -1,8 +1,7 @@ import json from alembic import op -from sqlalchemy import engine_from_config -from sqlalchemy.engine import reflection +import sqlalchemy as sa def table_exist(table_name): """check table is not exist @@ -13,20 +12,11 @@ def table_exist(table_name): Returns: bool: true or false, whether the table_name exists """ - config = op.get_context().config - engine = engine_from_config( - config.get_section(config.config_ini_section), prefix="sqlalchemy." - ) - insp = reflection.Inspector.from_engine(engine) - table_exist = False + conn = op.get_bind() + inspector = sa.inspect(conn) + tables = inspector.get_table_names() - for table in insp.get_table_names(): - if table_name not in table: - continue - table_exist = True - break - - return table_exist + return table_name in tables def column_exist_in_table(table_name, column_name): """check column is not exist in table @@ -38,17 +28,11 @@ def column_exist_in_table(table_name, column_name): Returns: bool: true or false, whether the column_name exists in the table_name """ - config = op.get_context().config - engine = engine_from_config( - config.get_section(config.config_ini_section), prefix="sqlalchemy." - ) - insp = reflection.Inspector.from_engine(engine) - column_exist = False - for col in insp.get_columns(table_name): - if column_name not in col["name"]: - continue - column_exist = True - return column_exist + conn = op.get_bind() + inspector = sa.inspect(conn) + columns = inspector.get_columns(table_name) + + return column_name in [column["name"] for column in columns] def get_tool_label_dict(task_config: dict) -> dict: diff --git a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py index ce761d5a..4d78c923 100644 --- a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py +++ b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py @@ -10,7 +10,6 @@ import sqlalchemy as sa from sqlalchemy.sql import table, column - # revision identifiers, used by Alembic. revision = '2eb983c9a254' down_revision = 'eb9c5b98168b' @@ -21,10 +20,11 @@ def upgrade() -> None: # Create task_collaborator table # if the table is not exists then create it - is_table_exist = op.execute( - "SHOW TABLES LIKE 'task_collaborator';" - ).fetchone() - if not is_table_exist: + conn = op.get_bind() + inspector = sa.inspect(conn) + tables = inspector.get_table_names() + + if 'task_collaborator' not in tables: op.create_table( 'task_collaborator', sa.Column('task_id', sa.Integer(), nullable=False), @@ -49,79 +49,67 @@ def upgrade() -> None: ) # Performances index - is_index_2_exist = op.execute( - "SHOW INDEX FROM task_collaborator WHERE Key_name = 'ix_task_collaborator_task_id';" - ).fetchone() + indices = inspector.get_indexes('task_collaborator') + existing_index_names = {idx['name'] for idx in indices} - if is_index_2_exist: - return - - op.create_index( - 'ix_task_collaborator_task_id', - 'task_collaborator', - ['task_id'] - ) - op.create_index( - 'ix_task_collaborator_user_id', - 'task_collaborator', - ['user_id'] - ) - op.create_index( - 'ix_task_created_by_deleted_at', - 'task', - ['created_by', 'deleted_at'] - ) + if 'ix_task_collaborator_task_id' not in existing_index_names: + op.create_index( + 'ix_task_collaborator_task_id', + 'task_collaborator', + ['task_id'] + ) + op.create_index( + 'ix_task_collaborator_user_id', + 'task_collaborator', + ['user_id'] + ) + op.create_index( + 'ix_task_created_by_deleted_at', + 'task', + ['created_by', 'deleted_at'] + ) # Task sample: updater -> updaters; create a new table task_sample_updater - is_task_sample_updater_table_exist = op.execute( - "SHOW TABLES LIKE 'task_sample_updater';" - ).fetchone() - - if is_task_sample_updater_table_exist: - return - - op.create_table( - 'task_sample_updater', - sa.Column('sample_id', sa.Integer(), nullable=False), - sa.Column('user_id', sa.Integer(), nullable=False), - sa.Column( - 'created_at', - sa.DateTime(timezone=True), - server_default=sa.text('CURRENT_TIMESTAMP'), - nullable=False - ), - sa.ForeignKeyConstraint( - ['sample_id'], - ['task_sample.id'], - ondelete='CASCADE' - ), - sa.ForeignKeyConstraint( - ['user_id'], - ['user.id'], - ondelete='CASCADE' - ), - sa.PrimaryKeyConstraint('sample_id', 'user_id') - ) - - # Performances index - # check if the index is already exists - is_index_exist = op.execute( - "SHOW INDEX FROM task_sample_updater WHERE Key_name = 'ix_task_sample_updater_sample_id';" - ).fetchone() - - if is_index_exist: - return - - op.create_index( - 'ix_task_sample_updater_sample_id', - 'task_sample_updater', - ['sample_id'] - ) - op.create_index( - 'ix_task_sample_updater_user_id', - 'task_sample_updater', - ['user_id'] - ) + if 'task_sample_updater' not in tables: + op.create_table( + 'task_sample_updater', + sa.Column('sample_id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column( + 'created_at', + sa.DateTime(timezone=True), + server_default=sa.text('CURRENT_TIMESTAMP'), + nullable=False + ), + sa.ForeignKeyConstraint( + ['sample_id'], + ['task_sample.id'], + ondelete='CASCADE' + ), + sa.ForeignKeyConstraint( + ['user_id'], + ['user.id'], + ondelete='CASCADE' + ), + sa.PrimaryKeyConstraint('sample_id', 'user_id') + ) + + # Performances index + # check if the index is already exists + indices = inspector.get_indexes('task_sample_updater') + existing_index_names = {idx['name'] for idx in indices} + + if 'ix_task_sample_updater_sample_id' not in existing_index_names: + op.create_index( + 'ix_task_sample_updater_sample_id', + 'task_sample_updater', + ['sample_id'] + ) + op.create_index( + 'ix_task_sample_updater_user_id', + 'task_sample_updater', + ['user_id'] + ) # Migrate data from task_sample.updated_by to task_sample_updater task_sample = table( diff --git a/labelu/alembic_labelu/versions/9d5da133bbe4_replace_key_with_value_in_sample_table.py b/labelu/alembic_labelu/versions/9d5da133bbe4_replace_key_with_value_in_sample_table.py index f229c4e4..632e2aeb 100644 --- a/labelu/alembic_labelu/versions/9d5da133bbe4_replace_key_with_value_in_sample_table.py +++ b/labelu/alembic_labelu/versions/9d5da133bbe4_replace_key_with_value_in_sample_table.py @@ -13,11 +13,38 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.ext.automap import automap_base -from labelu.alembic_labelu.alembic_labelu_tools import ( - get_tool_label_dict, - replace_key_with_value, -) +def get_tool_label_dict(task_config: dict) -> dict: + """get the key value of labels in a given task_id and task_config""" + label_dict = {"无标签": "noneAttribute"} + # obtain the labels in current task + # get the general labels + for normal_label in task_config.get("attribute", []): + if normal_label.get("key", ""): + label_dict[normal_label.get("key")] = normal_label.get("value") + # get the labels in configuration defined by user + for task_tool in task_config.get("tools", []): + if "config" not in task_tool.keys(): + continue + labels = task_tool.get("config").get("attributeList", []) + for label in labels: + if label.get("key", ""): + label_dict[label.get("key")] = label.get("value") + return label_dict + + +def replace_key_with_value(sample_data: dict, label_dict: dict) -> dict: + """replace the key with value in task_sample table to modify the error for the history version""" + + annotated_result = sample_data.get("result") + annotated_result = json.loads(annotated_result) + for sample_tool, sample_tool_results in annotated_result.items(): + if sample_tool.endswith("Tool"): + for sample_tool_result in sample_tool_results.get("result", []): + tool_label = sample_tool_result.get("attribute", "") + if tool_label in label_dict: + sample_tool_result["attribute"] = label_dict[tool_label] + return annotated_result # revision identifiers, used by Alembic. revision = "9d5da133bbe4" diff --git a/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py b/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py index 416c5c08..c2619b7e 100644 --- a/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py +++ b/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py @@ -5,7 +5,6 @@ Create Date: 2024-02-07 15:58:30.618151 """ -import imp import json import os @@ -14,6 +13,7 @@ from sqlalchemy.ext.automap import automap_base from sqlalchemy.orm import sessionmaker from labelu.internal.common.config import settings +from labelu.alembic_labelu.alembic_labelu_tools import table_exist, column_exist_in_table Base = automap_base() @@ -24,17 +24,6 @@ branch_labels = None depends_on = None -# import alembic_labelu_tools from the absolute path -alembic_labelu_tools = imp.load_source( - "alembic_labelu_tools", - ( - os.path.join( - os.path.abspath(os.path.dirname(os.path.dirname(__file__))), - "alembic_labelu_tools.py", - ) - ), -) - def upgrade() -> None: bind = op.get_bind() Base.prepare(autoload_with=bind, reflect=True) @@ -47,7 +36,7 @@ def upgrade() -> None: with context.begin_transaction(): # Create a new table task_pre_annotation - if not alembic_labelu_tools.table_exist("task_pre_annotation"): + if not table_exist("task_pre_annotation"): op.create_table( "task_pre_annotation", sa.Column("id", sa.Integer, primary_key=True, autoincrement=True, index=True), @@ -78,7 +67,7 @@ def upgrade() -> None: ), ) # Update the task_sample table - if not alembic_labelu_tools.column_exist_in_table( + if not column_exist_in_table( "task_sample", "file_id" ): with op.batch_alter_table('task_sample', recreate="always") as batch_op: @@ -93,7 +82,7 @@ def upgrade() -> None: ) # Update the task_attachment table - if not alembic_labelu_tools.column_exist_in_table("task_attachment", "filename"): + if not column_exist_in_table("task_attachment", "filename"): with op.batch_alter_table("task_attachment", recreate="always") as batch_op_task_attachment: batch_op_task_attachment.add_column( sa.Column( diff --git a/labelu/alembic_labelu/versions/e76c2ca5562e_add_inner_id.py b/labelu/alembic_labelu/versions/e76c2ca5562e_add_inner_id.py index c1d32569..1ade91a3 100644 --- a/labelu/alembic_labelu/versions/e76c2ca5562e_add_inner_id.py +++ b/labelu/alembic_labelu/versions/e76c2ca5562e_add_inner_id.py @@ -5,24 +5,12 @@ Create Date: 2023-02-28 22:29:31.595257 """ -import imp -import os - from alembic import op from alembic import context import sqlalchemy as sa from sqlalchemy.sql import text +from labelu.alembic_labelu.alembic_labelu_tools import column_exist_in_table -# import alembic_labelu_tools from the absolute path -alembic_labelu_tools = imp.load_source( - "alembic_labelu_tools", - ( - os.path.join( - os.path.abspath(os.path.dirname(os.path.dirname(__file__))), - "alembic_labelu_tools.py", - ) - ), -) # revision identifiers, used by Alembic. revision = "e76c2ca5562e" @@ -36,7 +24,7 @@ def upgrade() -> None: add inner_id and last_sample_inner_id columns in the task and sample tables """ with context.begin_transaction(): - if not alembic_labelu_tools.column_exist_in_table( + if not column_exist_in_table( "task", "last_sample_inner_id" ): with op.batch_alter_table("task", recreate="always") as batch_op_task: @@ -50,7 +38,7 @@ def upgrade() -> None: ), insert_before="config", ) - if not alembic_labelu_tools.column_exist_in_table("task_sample", "inner_id"): + if not column_exist_in_table("task_sample", "inner_id"): with op.batch_alter_table( "task_sample", recreate="always" ) as batch_op_task_sample: diff --git a/labelu/internal/application/service/task.py b/labelu/internal/application/service/task.py index 974c315c..5bab8317 100644 --- a/labelu/internal/application/service/task.py +++ b/labelu/internal/application/service/task.py @@ -102,6 +102,18 @@ async def get(db: Session, task_id: int, current_user: User) -> TaskResponseWith # get task detail task = crud_task.get(db=db, task_id=task_id) + + # not the collaborators + if task.created_by != current_user.id and current_user not in task.collaborators: + logger.error( + "cannot get task, the task owner is:{}, the get operator is:{}", + task.created_by, + current_user.id, + ) + raise LabelUException( + code=ErrorCode.CODE_30001_NO_PERMISSION, + status_code=status.HTTP_403_FORBIDDEN, + ) if not task: logger.error("cannot find task:{}", task_id) raise LabelUException( diff --git a/labelu/internal/dependencies/user.py b/labelu/internal/dependencies/user.py index 509cdb3d..2d9ae870 100644 --- a/labelu/internal/dependencies/user.py +++ b/labelu/internal/dependencies/user.py @@ -43,7 +43,7 @@ def get_current_user( ) user = crud_user.get(db, id=token_data.id) if not user: - raise LabelUException(code=ErrorCode.CODE_40002_USER_NOT_FOUND, status_code=404) + raise LabelUException(code=ErrorCode.CODE_40002_USER_NOT_FOUND, status_code=401) return user From 10ae011fc19ff1d5e8bea26c546eda7573663d06 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 21 Feb 2025 12:03:39 +0000 Subject: [PATCH 113/236] fix: Upgrade frontend to v5.7.0-alpha.11. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.11) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 296d52da..98788d9d 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.10/frontend.zip -alpha_version: v5.7.0-alpha.10 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.11/frontend.zip +alpha_version: v5.7.0-alpha.11 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 5bae632c808d2f84ed2b6f00808f15d677a4c58d Mon Sep 17 00:00:00 2001 From: gary Date: Fri, 21 Feb 2025 20:27:42 +0800 Subject: [PATCH 114/236] update: update test case --- labelu/tests/internal/adapter/routers/test_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labelu/tests/internal/adapter/routers/test_user.py b/labelu/tests/internal/adapter/routers/test_user.py index 56c0d6a7..3c6c10cc 100644 --- a/labelu/tests/internal/adapter/routers/test_user.py +++ b/labelu/tests/internal/adapter/routers/test_user.py @@ -127,5 +127,5 @@ def test_user_not_found(self, client: TestClient, db: Session) -> None: r = client.post(f"{settings.API_V1_STR}/users/logout", headers=headers) # check - assert r.status_code == 404 + assert r.status_code == 401 assert r.json()["err_code"] == 40002 From b22092b1c8b85535bafca862f112c5a3331bf267 Mon Sep 17 00:00:00 2001 From: gary Date: Mon, 24 Feb 2025 15:15:50 +0800 Subject: [PATCH 115/236] update: update get task --- labelu/internal/application/service/task.py | 13 +++++++------ labelu/internal/common/error_code.py | 8 +++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/labelu/internal/application/service/task.py b/labelu/internal/application/service/task.py index 5bab8317..ac966c61 100644 --- a/labelu/internal/application/service/task.py +++ b/labelu/internal/application/service/task.py @@ -103,6 +103,13 @@ async def get(db: Session, task_id: int, current_user: User) -> TaskResponseWith # get task detail task = crud_task.get(db=db, task_id=task_id) + if not task: + logger.error("cannot find task:{}", task_id) + raise LabelUException( + code=ErrorCode.CODE_50002_TASK_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + # not the collaborators if task.created_by != current_user.id and current_user not in task.collaborators: logger.error( @@ -114,12 +121,6 @@ async def get(db: Session, task_id: int, current_user: User) -> TaskResponseWith code=ErrorCode.CODE_30001_NO_PERMISSION, status_code=status.HTTP_403_FORBIDDEN, ) - if not task: - logger.error("cannot find task:{}", task_id) - raise LabelUException( - code=ErrorCode.CODE_50002_TASK_NOT_FOUND, - status_code=status.HTTP_404_NOT_FOUND, - ) # get progress statics = crud_sample.statics( diff --git a/labelu/internal/common/error_code.py b/labelu/internal/common/error_code.py index fc0707c6..185449b8 100644 --- a/labelu/internal/common/error_code.py +++ b/labelu/internal/common/error_code.py @@ -23,11 +23,17 @@ # export EXPORT_INIT_CODE = 60000 +UNEXPECTED_ERROR_CODE = 99999 + class ErrorCode(Enum): """ business error code """ + UNEXPECTED_ERROR = ( + UNEXPECTED_ERROR_CODE, + "Internal Error", + ) # common init error code CODE_30000_SQL_ERROR = ( @@ -184,7 +190,7 @@ async def unexpected_exception_handler(request: Request, exc: Exception): status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, content={ "msg": str(exc), - "err_code": ErrorCode.CODE_30000_SQL_ERROR.value[0], + "err_code": ErrorCode.UNEXPECTED_ERROR.value[0], }, ) From 973ba7283aedf6f4d64364fa6722c3aa63538bf1 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 24 Feb 2025 07:21:15 +0000 Subject: [PATCH 116/236] chore(release): 1.1.0-alpha.35 [skip ci] # [1.1.0-alpha.35](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.34...v1.1.0-alpha.35) (2025-02-24) ### Bug Fixes * remove owner_id param in list pre_annotations ([#192](https://github.com/opendatalab/labelU/issues/192)) ([f917091](https://github.com/opendatalab/labelU/commit/f917091792f734ada91d67c473dde1d5c34b9e81)) * update migrate script ([c436b00](https://github.com/opendatalab/labelU/commit/c436b0041189cafd51eaea58871bb336d9f8619f)) * Upgrade frontend to v5.7.0-alpha.10. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.10) ([1a4b46e](https://github.com/opendatalab/labelU/commit/1a4b46e353f0645721d481e3574d114b2627211a)) * Upgrade frontend to v5.7.0-alpha.11. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.11) ([10ae011](https://github.com/opendatalab/labelU/commit/10ae011fc19ff1d5e8bea26c546eda7573663d06)) * Upgrade frontend to v5.7.0-alpha.9. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.9) ([27fcf67](https://github.com/opendatalab/labelU/commit/27fcf677cc4ac85a73207c4ea771d32024719efb)) ### Features * supports task collaborators ([#189](https://github.com/opendatalab/labelU/issues/189)) ([b159c63](https://github.com/opendatalab/labelU/commit/b159c634286dd446c548e70ca0b7367d0221a760)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 589a9746..269881af 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.34' \ No newline at end of file +version='1.1.0-alpha.35' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ef14156f..e2c45574 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.34' +version = '1.1.0-alpha.35' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 2f90701cc9cfa6976beba10df9532114127ca1f2 Mon Sep 17 00:00:00 2001 From: gary Date: Fri, 28 Feb 2025 15:26:24 +0800 Subject: [PATCH 117/236] feat: add task ws api router (#196) --- labelu/internal/adapter/ws/__init__.py | 3 +- labelu/internal/adapter/ws/sample.py | 104 +++++++++++++++++++++++-- labelu/internal/clients/ws.py | 1 + 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/labelu/internal/adapter/ws/__init__.py b/labelu/internal/adapter/ws/__init__.py index 50d1e4bb..4c5fe049 100644 --- a/labelu/internal/adapter/ws/__init__.py +++ b/labelu/internal/adapter/ws/__init__.py @@ -1,6 +1,7 @@ from fastapi import FastAPI -from labelu.internal.adapter.ws.sample import task_sample_ws_endpoint +from labelu.internal.adapter.ws.sample import task_ws_endpoint, task_sample_ws_endpoint def add_ws_router(app: FastAPI): + app.add_api_websocket_route("/ws/task/{task_id}", task_ws_endpoint) app.add_api_websocket_route("/ws/task/{task_id}/{sample_id}", task_sample_ws_endpoint) \ No newline at end of file diff --git a/labelu/internal/adapter/ws/sample.py b/labelu/internal/adapter/ws/sample.py index 4a390b3b..9394275a 100644 --- a/labelu/internal/adapter/ws/sample.py +++ b/labelu/internal/adapter/ws/sample.py @@ -1,25 +1,73 @@ +from typing import List from fastapi import Depends, WebSocket, WebSocketDisconnect from loguru import logger from pydantic import BaseModel -from labelu.internal.clients.ws import sampleConnections +from labelu.internal.clients.ws import sampleConnections, taskConnections from labelu.internal.common.websocket import ConnectionData from labelu.internal.dependencies.user import verify_ws_token + +class CollaboratorWsPayload(BaseModel): + user_id: int + username: str + sample_id: int class TaskWsPayload(BaseModel): task_id: int user_id: int username: str + collaborators: List[CollaboratorWsPayload] = [] -class TaskSampleWsPayload(TaskWsPayload): +class TaskSampleWsPayload(BaseModel): + task_id: int + user_id: int + username: str sample_id: int +def get_task_connection_payloads(conns: ConnectionData): + if not conns: + return { + "type": "connected", + "data": { + "collaborators": [], + "connections": [] + } + } + + connections = [] + collaborators = [] + + for conn in conns: + connections.append({ + "task_id": conn.data.task_id, + "user_id": conn.data.user_id, + "username": conn.data.username + }) + + for _, sample_conns in sampleConnections.active_connections.items(): + if sample_conns[0].data.task_id == conns[0].data.task_id: + for sample_conn in sample_conns: + # The first connector in the sample connection is the annotator, and the others can not annotate + collaborators.append({ + "user_id": sample_conn.data.user_id, + "username": sample_conn.data.username, + "sample_id": sample_conn.data.sample_id + }) + + return { + "type": "connected", + "data": { + "collaborators": collaborators, + "connections": connections + } + } + def get_task_sample_connection_payloads(conns: ConnectionData): result = [] if not conns: return { - "type": "active_connections", + "type": "connected", "data": [] } @@ -32,9 +80,45 @@ def get_task_sample_connection_payloads(conns: ConnectionData): }) return { - "type": "active_connections", + "type": "connected", "data": result } + +async def task_ws_endpoint(websocket: WebSocket, task_id: int, user=Depends(verify_ws_token)): + if not user: + return + + client_id = f"{task_id}" + + async def tell_others(): + connections = taskConnections.active_connections + current_connection = connections.get(client_id, []) + task_payload = get_task_connection_payloads(current_connection) + + await taskConnections.send_message(task_payload, client_id) + + try: + await taskConnections.connect( + websocket, + client_id, + data=TaskWsPayload( + task_id=task_id, + user_id=user.id, + username=user.username + ) + ) + await tell_others() + + while True: + await websocket.receive_text() + + except WebSocketDisconnect: + await taskConnections.disconnect(websocket, client_id) + await tell_others() + except Exception as e: + logger.error(f"WebSocket error for user {client_id}: {e}") + await taskConnections.disconnect(websocket, client_id) + await tell_others() async def task_sample_ws_endpoint(websocket: WebSocket, task_id: int, sample_id: int, user=Depends(verify_ws_token)): if not user: @@ -49,6 +133,13 @@ async def tell_others(): await sampleConnections.send_message(sample_payload, client_id) + async def tell_other_task_connections(): + connections = taskConnections.active_connections + current_connection = connections.get(f"{task_id}", []) + task_payload = get_task_connection_payloads(current_connection) + + await taskConnections.send_message(task_payload, f"{task_id}") + try: await sampleConnections.connect( websocket, @@ -61,6 +152,7 @@ async def tell_others(): ) ) await tell_others() + await tell_other_task_connections() while True: await websocket.receive_text() @@ -68,7 +160,9 @@ async def tell_others(): except WebSocketDisconnect: await sampleConnections.disconnect(websocket, client_id) await tell_others() + await tell_other_task_connections() except Exception as e: logger.error(f"WebSocket error for user {client_id}: {e}") await sampleConnections.disconnect(websocket, client_id) - await tell_others() \ No newline at end of file + await tell_others() + await tell_other_task_connections() \ No newline at end of file diff --git a/labelu/internal/clients/ws.py b/labelu/internal/clients/ws.py index 7315b4ff..2badf9bb 100644 --- a/labelu/internal/clients/ws.py +++ b/labelu/internal/clients/ws.py @@ -1,4 +1,5 @@ from labelu.internal.common.websocket import ConnectionManager +taskConnections = ConnectionManager() sampleConnections = ConnectionManager() \ No newline at end of file From 8f88debf369eece83c9666619b38a44c1fc30729 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 28 Feb 2025 09:38:51 +0000 Subject: [PATCH 118/236] fix: Upgrade frontend to v5.7.0-alpha.12. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.12) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 98788d9d..96fd7b29 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.11/frontend.zip -alpha_version: v5.7.0-alpha.11 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.12/frontend.zip +alpha_version: v5.7.0-alpha.12 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 7624fd989afe97ae27b272f4c6890b56b0d7e896 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 28 Feb 2025 09:43:44 +0000 Subject: [PATCH 119/236] chore(release): 1.1.0-alpha.36 [skip ci] # [1.1.0-alpha.36](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.35...v1.1.0-alpha.36) (2025-02-28) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.12. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.12) ([8f88deb](https://github.com/opendatalab/labelU/commit/8f88debf369eece83c9666619b38a44c1fc30729)) ### Features * add task ws api router ([#196](https://github.com/opendatalab/labelU/issues/196)) ([2f90701](https://github.com/opendatalab/labelU/commit/2f90701cc9cfa6976beba10df9532114127ca1f2)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 269881af..1313b1fb 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.35' \ No newline at end of file +version='1.1.0-alpha.36' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index e2c45574..4bc54fdc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.35' +version = '1.1.0-alpha.36' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 6bae992b2024edb6f4fe32f03de950d88db933eb Mon Sep 17 00:00:00 2001 From: gary Date: Mon, 3 Mar 2025 19:09:34 +0800 Subject: [PATCH 120/236] fix: update default command (#197) --- labelu/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/labelu/main.py b/labelu/main.py index 75107210..7219e7b9 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -145,5 +145,7 @@ def main( uvicorn.run(app=app, host=settings.HOST, port=settings.PORT, ws="websockets") +cli.default_command = "main" + if __name__ == "__main__": cli() \ No newline at end of file From 2a7b9695b922dc2bd678d1f001f13b831a899d1e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 3 Mar 2025 11:14:39 +0000 Subject: [PATCH 121/236] chore(release): 1.1.0-alpha.37 [skip ci] # [1.1.0-alpha.37](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.36...v1.1.0-alpha.37) (2025-03-03) ### Bug Fixes * update default command ([#197](https://github.com/opendatalab/labelU/issues/197)) ([6bae992](https://github.com/opendatalab/labelU/commit/6bae992b2024edb6f4fe32f03de950d88db933eb)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 1313b1fb..43993ad6 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.36' \ No newline at end of file +version='1.1.0-alpha.37' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4bc54fdc..116777d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.36' +version = '1.1.0-alpha.37' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From d5a8545e76582fd07c3b376c9fb9d2343fe152fc Mon Sep 17 00:00:00 2001 From: gary Date: Tue, 4 Mar 2025 11:21:08 +0800 Subject: [PATCH 122/236] fix: update default command (#198) --- labelu/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labelu/main.py b/labelu/main.py index 7219e7b9..22168269 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -145,7 +145,7 @@ def main( uvicorn.run(app=app, host=settings.HOST, port=settings.PORT, ws="websockets") -cli.default_command = "main" +cli.callback()(main) if __name__ == "__main__": cli() \ No newline at end of file From 5221fd8f67b0cdb5be9a4b3c94af046edaae1a1b Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 4 Mar 2025 03:27:06 +0000 Subject: [PATCH 123/236] fix: Upgrade frontend to v5.7.0-alpha.13. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.13) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 96fd7b29..fdf62124 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.12/frontend.zip -alpha_version: v5.7.0-alpha.12 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.13/frontend.zip +alpha_version: v5.7.0-alpha.13 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 4ea57f93edfff9225342be1b5c5d38b308469cb5 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 4 Mar 2025 03:32:23 +0000 Subject: [PATCH 124/236] chore(release): 1.1.0-alpha.38 [skip ci] # [1.1.0-alpha.38](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.37...v1.1.0-alpha.38) (2025-03-04) ### Bug Fixes * update default command ([#198](https://github.com/opendatalab/labelU/issues/198)) ([d5a8545](https://github.com/opendatalab/labelU/commit/d5a8545e76582fd07c3b376c9fb9d2343fe152fc)) * Upgrade frontend to v5.7.0-alpha.13. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.13) ([5221fd8](https://github.com/opendatalab/labelU/commit/5221fd8f67b0cdb5be9a4b3c94af046edaae1a1b)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 43993ad6..dd4eafb4 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.37' \ No newline at end of file +version='1.1.0-alpha.38' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 116777d8..fd39c2e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.37' +version = '1.1.0-alpha.38' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 5e413ab86f7bc0ca18e5e2fce82ed2e22f4d9cc2 Mon Sep 17 00:00:00 2001 From: gary Date: Tue, 4 Mar 2025 11:54:16 +0800 Subject: [PATCH 125/236] fix: main.py -> labelu.py (#199) --- labelu/{main.py => labelu.py} | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) rename labelu/{main.py => labelu.py} (98%) diff --git a/labelu/main.py b/labelu/labelu.py similarity index 98% rename from labelu/main.py rename to labelu/labelu.py index 22168269..85ae1c07 100644 --- a/labelu/main.py +++ b/labelu/labelu.py @@ -131,9 +131,9 @@ def file_response(self, *args: Any, **kwargs: Any) -> Response: def to_mysql(): """Migrate database to MySQL""" migrate_to_mysql() - -@cli.command() -def main( + +@app.command() +def labbelu( host: str = "localhost", port: int = 8000, media_host: str = "http://localhost:8000" ): if port: @@ -145,7 +145,6 @@ def main( uvicorn.run(app=app, host=settings.HOST, port=settings.PORT, ws="websockets") -cli.callback()(main) if __name__ == "__main__": cli() \ No newline at end of file From ac5ae358b70661cea2932d252cfa05c3a24ac2f2 Mon Sep 17 00:00:00 2001 From: gary Date: Tue, 4 Mar 2025 12:04:41 +0800 Subject: [PATCH 126/236] fix: update command (#200) --- labelu/labelu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labelu/labelu.py b/labelu/labelu.py index 85ae1c07..2397f428 100644 --- a/labelu/labelu.py +++ b/labelu/labelu.py @@ -127,7 +127,7 @@ def file_response(self, *args: Any, **kwargs: Any) -> Response: cli = Typer() -@cli.command() +@cli.command('migrate_to_mysql') def to_mysql(): """Migrate database to MySQL""" migrate_to_mysql() From fb6c3e1bac3fb714c91139cae82d4d79f13b255d Mon Sep 17 00:00:00 2001 From: gary Date: Tue, 4 Mar 2025 12:06:38 +0800 Subject: [PATCH 127/236] fix: labelu.py -> main.py --- labelu/{labelu.py => main.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename labelu/{labelu.py => main.py} (100%) diff --git a/labelu/labelu.py b/labelu/main.py similarity index 100% rename from labelu/labelu.py rename to labelu/main.py From 634ce0c9a808f336f34be6843aeb14bd8f366911 Mon Sep 17 00:00:00 2001 From: gary Date: Tue, 4 Mar 2025 12:15:45 +0800 Subject: [PATCH 128/236] update: app -> cli --- labelu/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labelu/main.py b/labelu/main.py index 2397f428..19a6df0a 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -132,7 +132,7 @@ def to_mysql(): """Migrate database to MySQL""" migrate_to_mysql() -@app.command() +@cli.command() def labbelu( host: str = "localhost", port: int = 8000, media_host: str = "http://localhost:8000" ): From a42e8ef7b0e8aae345adcb978ff41115cfe52efb Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 4 Mar 2025 04:20:59 +0000 Subject: [PATCH 129/236] chore(release): 1.1.0-alpha.39 [skip ci] # [1.1.0-alpha.39](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.38...v1.1.0-alpha.39) (2025-03-04) ### Bug Fixes * labelu.py -> main.py ([fb6c3e1](https://github.com/opendatalab/labelU/commit/fb6c3e1bac3fb714c91139cae82d4d79f13b255d)) * main.py -> labelu.py ([#199](https://github.com/opendatalab/labelU/issues/199)) ([5e413ab](https://github.com/opendatalab/labelU/commit/5e413ab86f7bc0ca18e5e2fce82ed2e22f4d9cc2)) * update command ([#200](https://github.com/opendatalab/labelU/issues/200)) ([ac5ae35](https://github.com/opendatalab/labelU/commit/ac5ae358b70661cea2932d252cfa05c3a24ac2f2)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index dd4eafb4..dab95662 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.38' \ No newline at end of file +version='1.1.0-alpha.39' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index fd39c2e4..ac438291 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.38' +version = '1.1.0-alpha.39' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 6b345b9eae82b4e1408cdd05e63bdf8771ccad8b Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 4 Mar 2025 12:23:29 +0800 Subject: [PATCH 130/236] fix: update default command --- labelu/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labelu/main.py b/labelu/main.py index 19a6df0a..4deb2f95 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -133,7 +133,7 @@ def to_mysql(): migrate_to_mysql() @cli.command() -def labbelu( +def main( host: str = "localhost", port: int = 8000, media_host: str = "http://localhost:8000" ): if port: From 6787477d7ea16a3b5bb9298792fcf7088c9aecb5 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 4 Mar 2025 04:28:53 +0000 Subject: [PATCH 131/236] chore(release): 1.1.0-alpha.40 [skip ci] # [1.1.0-alpha.40](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.39...v1.1.0-alpha.40) (2025-03-04) ### Bug Fixes * update default command ([6b345b9](https://github.com/opendatalab/labelU/commit/6b345b9eae82b4e1408cdd05e63bdf8771ccad8b)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index dab95662..0e8e13b0 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.39' \ No newline at end of file +version='1.1.0-alpha.40' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ac438291..80eb0fe1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.39' +version = '1.1.0-alpha.40' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 948d9afcee4b86b1a0f373af0333df832a7e7753 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 4 Mar 2025 12:34:59 +0800 Subject: [PATCH 132/236] fix: update default command --- labelu/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labelu/main.py b/labelu/main.py index 4deb2f95..399c9237 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -132,7 +132,7 @@ def to_mysql(): """Migrate database to MySQL""" migrate_to_mysql() -@cli.command() +@cli.callback(invoke_without_command=True) def main( host: str = "localhost", port: int = 8000, media_host: str = "http://localhost:8000" ): From 9dad646d37246ba9743d45762adab04c82145f6e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 4 Mar 2025 04:40:29 +0000 Subject: [PATCH 133/236] chore(release): 1.1.0-alpha.41 [skip ci] # [1.1.0-alpha.41](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.40...v1.1.0-alpha.41) (2025-03-04) ### Bug Fixes * update default command ([948d9af](https://github.com/opendatalab/labelU/commit/948d9afcee4b86b1a0f373af0333df832a7e7753)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 0e8e13b0..d299ec73 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.40' \ No newline at end of file +version='1.1.0-alpha.41' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 80eb0fe1..a010fe8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.40' +version = '1.1.0-alpha.41' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 54329c548a7b4d9bf9c81de5febddbb08c2ee176 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 4 Mar 2025 11:08:04 +0000 Subject: [PATCH 134/236] fix: Upgrade frontend to v5.7.0-alpha.14. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.14) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index fdf62124..d4f838d5 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.13/frontend.zip -alpha_version: v5.7.0-alpha.13 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.14/frontend.zip +alpha_version: v5.7.0-alpha.14 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From ea9fdd69bb55f0d9169b2a3c1942c1e406beb9ca Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 4 Mar 2025 11:13:11 +0000 Subject: [PATCH 135/236] chore(release): 1.1.0-alpha.42 [skip ci] # [1.1.0-alpha.42](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.41...v1.1.0-alpha.42) (2025-03-04) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.14. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.14) ([54329c5](https://github.com/opendatalab/labelU/commit/54329c548a7b4d9bf9c81de5febddbb08c2ee176)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index d299ec73..80acef29 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.41' \ No newline at end of file +version='1.1.0-alpha.42' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index a010fe8a..c53e5637 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.41' +version = '1.1.0-alpha.42' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 20ef7b36f36924183a5a4315beef21271389afbb Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 5 Mar 2025 08:36:35 +0000 Subject: [PATCH 136/236] fix: Upgrade frontend to v5.7.0-alpha.15. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.15) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index d4f838d5..b1c1fd6f 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.14/frontend.zip -alpha_version: v5.7.0-alpha.14 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.15/frontend.zip +alpha_version: v5.7.0-alpha.15 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From ab51a8e2fbe94ab4b2674fecf32f5a34bc40d5f3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 5 Mar 2025 08:41:23 +0000 Subject: [PATCH 137/236] chore(release): 1.1.0-alpha.43 [skip ci] # [1.1.0-alpha.43](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.42...v1.1.0-alpha.43) (2025-03-05) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.15. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.15) ([20ef7b3](https://github.com/opendatalab/labelU/commit/20ef7b36f36924183a5a4315beef21271389afbb)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 80acef29..06e8baa0 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.42' \ No newline at end of file +version='1.1.0-alpha.43' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index c53e5637..32d7b3fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.42' +version = '1.1.0-alpha.43' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 39f519ee10cac74ddee429f0988a3090e0412b1e Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 7 Mar 2025 15:34:03 +0800 Subject: [PATCH 138/236] fix: connections -> peers --- labelu/internal/adapter/ws/sample.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/labelu/internal/adapter/ws/sample.py b/labelu/internal/adapter/ws/sample.py index 9394275a..67bf4a1a 100644 --- a/labelu/internal/adapter/ws/sample.py +++ b/labelu/internal/adapter/ws/sample.py @@ -27,7 +27,7 @@ class TaskSampleWsPayload(BaseModel): def get_task_connection_payloads(conns: ConnectionData): if not conns: return { - "type": "connected", + "type": "peers", "data": { "collaborators": [], "connections": [] @@ -55,7 +55,7 @@ def get_task_connection_payloads(conns: ConnectionData): }) return { - "type": "connected", + "type": "peers", "data": { "collaborators": collaborators, "connections": connections @@ -67,7 +67,7 @@ def get_task_sample_connection_payloads(conns: ConnectionData): if not conns: return { - "type": "connected", + "type": "peers", "data": [] } @@ -80,7 +80,7 @@ def get_task_sample_connection_payloads(conns: ConnectionData): }) return { - "type": "connected", + "type": "peers", "data": result } @@ -133,6 +133,17 @@ async def tell_others(): await sampleConnections.send_message(sample_payload, client_id) + async def tell_others_someone_leaved(): + await sampleConnections.send_message({ + "type": "leave", + "data": { + "task_id": task_id, + "user_id": user.id, + "username": user.username, + "sample_id": sample_id + }, + }, client_id) + async def tell_other_task_connections(): connections = taskConnections.active_connections current_connection = connections.get(f"{task_id}", []) @@ -160,9 +171,12 @@ async def tell_other_task_connections(): except WebSocketDisconnect: await sampleConnections.disconnect(websocket, client_id) await tell_others() + await tell_others_someone_leaved() await tell_other_task_connections() + except Exception as e: logger.error(f"WebSocket error for user {client_id}: {e}") await sampleConnections.disconnect(websocket, client_id) await tell_others() + await tell_others_someone_leaved() await tell_other_task_connections() \ No newline at end of file From 266013274172d95592e63775391250086e969300 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 7 Mar 2025 07:45:56 +0000 Subject: [PATCH 139/236] fix: Upgrade frontend to v5.7.0-alpha.16. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.16) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index b1c1fd6f..dad98b28 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.15/frontend.zip -alpha_version: v5.7.0-alpha.15 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.16/frontend.zip +alpha_version: v5.7.0-alpha.16 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 7c83c4f4f0ec0ffe25a2a8eef5e04a31a72bfc94 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 7 Mar 2025 07:51:18 +0000 Subject: [PATCH 140/236] chore(release): 1.1.0-alpha.44 [skip ci] # [1.1.0-alpha.44](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.43...v1.1.0-alpha.44) (2025-03-07) ### Bug Fixes * connections -> peers ([39f519e](https://github.com/opendatalab/labelU/commit/39f519ee10cac74ddee429f0988a3090e0412b1e)) * Upgrade frontend to v5.7.0-alpha.16. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.16) ([2660132](https://github.com/opendatalab/labelU/commit/266013274172d95592e63775391250086e969300)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 06e8baa0..2aa99ad8 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.43' \ No newline at end of file +version='1.1.0-alpha.44' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 32d7b3fc..cd67732d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.43' +version = '1.1.0-alpha.44' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From a874f0bdc84246e9df1831a23d3c17c40574b852 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 7 Mar 2025 19:41:20 +0800 Subject: [PATCH 141/236] fix: optimize ws connection --- labelu/internal/adapter/ws/__init__.py | 5 +- labelu/internal/adapter/ws/sample.py | 211 ++++++++----------------- labelu/internal/clients/ws.py | 3 +- labelu/internal/common/websocket.py | 117 ++++++++++---- 4 files changed, 156 insertions(+), 180 deletions(-) diff --git a/labelu/internal/adapter/ws/__init__.py b/labelu/internal/adapter/ws/__init__.py index 4c5fe049..1f54407f 100644 --- a/labelu/internal/adapter/ws/__init__.py +++ b/labelu/internal/adapter/ws/__init__.py @@ -1,7 +1,6 @@ from fastapi import FastAPI -from labelu.internal.adapter.ws.sample import task_ws_endpoint, task_sample_ws_endpoint +from labelu.internal.adapter.ws.sample import task_ws_endpoint def add_ws_router(app: FastAPI): - app.add_api_websocket_route("/ws/task/{task_id}", task_ws_endpoint) - app.add_api_websocket_route("/ws/task/{task_id}/{sample_id}", task_sample_ws_endpoint) \ No newline at end of file + app.add_api_websocket_route("/ws/task/{task_id}/{sample_id}", task_ws_endpoint) \ No newline at end of file diff --git a/labelu/internal/adapter/ws/sample.py b/labelu/internal/adapter/ws/sample.py index 67bf4a1a..b09678d1 100644 --- a/labelu/internal/adapter/ws/sample.py +++ b/labelu/internal/adapter/ws/sample.py @@ -1,22 +1,10 @@ -from typing import List from fastapi import Depends, WebSocket, WebSocketDisconnect from loguru import logger from pydantic import BaseModel -from labelu.internal.clients.ws import sampleConnections, taskConnections -from labelu.internal.common.websocket import ConnectionData +from labelu.internal.clients.ws import sampleConnectionManager +from labelu.internal.common.websocket import ConnectionData, Message, MessageType from labelu.internal.dependencies.user import verify_ws_token - -class CollaboratorWsPayload(BaseModel): - user_id: int - username: str - sample_id: int - -class TaskWsPayload(BaseModel): - task_id: int - user_id: int - username: str - collaborators: List[CollaboratorWsPayload] = [] class TaskSampleWsPayload(BaseModel): task_id: int @@ -24,159 +12,92 @@ class TaskSampleWsPayload(BaseModel): username: str sample_id: int -def get_task_connection_payloads(conns: ConnectionData): - if not conns: - return { - "type": "peers", - "data": { - "collaborators": [], - "connections": [] - } - } - - connections = [] - collaborators = [] - - for conn in conns: - connections.append({ - "task_id": conn.data.task_id, - "user_id": conn.data.user_id, - "username": conn.data.username - }) - - for _, sample_conns in sampleConnections.active_connections.items(): - if sample_conns[0].data.task_id == conns[0].data.task_id: - for sample_conn in sample_conns: - # The first connector in the sample connection is the annotator, and the others can not annotate - collaborators.append({ - "user_id": sample_conn.data.user_id, - "username": sample_conn.data.username, - "sample_id": sample_conn.data.sample_id - }) - - return { - "type": "peers", - "data": { - "collaborators": collaborators, - "connections": connections - } - } - def get_task_sample_connection_payloads(conns: ConnectionData): - result = [] - if not conns: - return { - "type": "peers", - "data": [] - } - - for conn in conns: - result.append({ - "task_id": conn.data.task_id, - "user_id": conn.data.user_id, - "username": conn.data.username, - "sample_id": conn.data.sample_id, - }) + return Message( + type=MessageType.PEERS, + data=[] + ) - return { - "type": "peers", - "data": result - } + return Message( + type=MessageType.PEERS, + data=[ + TaskSampleWsPayload( + task_id=conn.data.task_id, + user_id=conn.data.user_id, + username=conn.data.username, + sample_id=conn.data.sample_id + ) for conn in conns + ] + ) -async def task_ws_endpoint(websocket: WebSocket, task_id: int, user=Depends(verify_ws_token)): + +async def task_ws_endpoint(websocket: WebSocket, task_id: int, sample_id: int, user=Depends(verify_ws_token)): if not user: + await websocket.close(code=1008, reason="Unauthorized") return - client_id = f"{task_id}" + client_id = f"task_{task_id}" - async def tell_others(): - connections = taskConnections.active_connections - current_connection = connections.get(client_id, []) - task_payload = get_task_connection_payloads(current_connection) + async def sync_peers(): + connections = sampleConnectionManager.active_connections + current_connections = connections.get(client_id, {}) + sample_payload = get_task_sample_connection_payloads(current_connections.values()) + + await sampleConnectionManager.send_message(client_id=client_id, message=sample_payload) - await taskConnections.send_message(task_payload, client_id) - try: - await taskConnections.connect( - websocket, - client_id, - data=TaskWsPayload( - task_id=task_id, - user_id=user.id, - username=user.username + connection = None + + async def cleanup(): + await sampleConnectionManager.disconnect(client_id, websocket) + await sync_peers() + # Tell clients that someone has left the sample page and refresh the page data + await sampleConnectionManager.send_message( + client_id=client_id, + message=Message( + type=MessageType.LEAVE, + data=TaskSampleWsPayload( + task_id=task_id, + user_id=user.id, + username=user.username, + sample_id=sample_id, + ) ) ) - await tell_others() - while True: - await websocket.receive_text() - - except WebSocketDisconnect: - await taskConnections.disconnect(websocket, client_id) - await tell_others() - except Exception as e: - logger.error(f"WebSocket error for user {client_id}: {e}") - await taskConnections.disconnect(websocket, client_id) - await tell_others() - -async def task_sample_ws_endpoint(websocket: WebSocket, task_id: int, sample_id: int, user=Depends(verify_ws_token)): - if not user: - return - - client_id = f"{task_id}_{sample_id}" - - async def tell_others(): - connections = sampleConnections.active_connections - current_connection = connections.get(client_id, []) - sample_payload = get_task_sample_connection_payloads(current_connection) - - await sampleConnections.send_message(sample_payload, client_id) - - async def tell_others_someone_leaved(): - await sampleConnections.send_message({ - "type": "leave", - "data": { - "task_id": task_id, - "user_id": user.id, - "username": user.username, - "sample_id": sample_id - }, - }, client_id) - - async def tell_other_task_connections(): - connections = taskConnections.active_connections - current_connection = connections.get(f"{task_id}", []) - task_payload = get_task_connection_payloads(current_connection) - - await taskConnections.send_message(task_payload, f"{task_id}") + if connection: + connection.heartbeat_task.cancel() try: - await sampleConnections.connect( - websocket, + connection = await sampleConnectionManager.connect( client_id, + websocket, data=TaskSampleWsPayload( task_id=task_id, - sample_id=sample_id, user_id=user.id, - username=user.username + username=user.username, + sample_id=sample_id, ) ) - await tell_others() - await tell_other_task_connections() + + await sync_peers() while True: - await websocket.receive_text() + try: + data = await websocket.receive_text() + message = Message.parse_raw(data) + + if message.type == MessageType.PONG: + connection.update_heartbeat() + + except WebSocketDisconnect as e: + logger.info(f"WebSocket disconnected for client {client_id}: Code={e.code}") + break - except WebSocketDisconnect: - await sampleConnections.disconnect(websocket, client_id) - await tell_others() - await tell_others_someone_leaved() - await tell_other_task_connections() - + except WebSocketDisconnect as e: + logger.info(f"WebSocket disconnected during connection setup for client {client_id}: Code={e.code}") except Exception as e: - logger.error(f"WebSocket error for user {client_id}: {e}") - await sampleConnections.disconnect(websocket, client_id) - await tell_others() - await tell_others_someone_leaved() - await tell_other_task_connections() \ No newline at end of file + logger.error(f"WebSocket error for client {client_id}: {e}") + finally: + await cleanup() \ No newline at end of file diff --git a/labelu/internal/clients/ws.py b/labelu/internal/clients/ws.py index 2badf9bb..ba6860f0 100644 --- a/labelu/internal/clients/ws.py +++ b/labelu/internal/clients/ws.py @@ -1,5 +1,4 @@ from labelu.internal.common.websocket import ConnectionManager -taskConnections = ConnectionManager() -sampleConnections = ConnectionManager() \ No newline at end of file +sampleConnectionManager = ConnectionManager() \ No newline at end of file diff --git a/labelu/internal/common/websocket.py b/labelu/internal/common/websocket.py index 518e594f..6627ab33 100644 --- a/labelu/internal/common/websocket.py +++ b/labelu/internal/common/websocket.py @@ -1,50 +1,107 @@ +from enum import Enum +import time from typing import Any, Dict, List +import uuid from fastapi import WebSocket from loguru import logger +import asyncio from dataclasses import dataclass +from pydantic import BaseModel + +HEARTBEAT_INTERVAL = 30 +HEARTBEAT_TIMEOUT = 60 + +class MessageType(str, Enum): + PEERS = "peers" + PING = "ping" + PONG = "pong" + LEAVE = "leave" + +class Message(BaseModel): + type: MessageType + data: Any = None + @dataclass class ConnectionData: - ws: WebSocket + id: uuid.UUID = None + ws: WebSocket = None data: Any = None + heartbeat_task: asyncio.Task = None + _last_heartbeat: float = time.time() + + def update_heartbeat(self): + self._last_heartbeat = time.time() class ConnectionManager: def __init__(self): - self.active_connections: Dict[str, List[ConnectionData]] = {} - - async def connect(self, websocket: WebSocket, client_id: str, data: Any = None): + self.active_connections: Dict[str, Dict[str, ConnectionData]] = {} + + def _get_connection(self, client_id: str, connection_id: uuid.UUID) -> ConnectionData: + if client_id in self.active_connections: + return self.active_connections[client_id].get(connection_id) + + return None + + async def _heartbeat(self, client_id: str, connection_id: uuid.UUID): + while True: + try: + await asyncio.sleep(HEARTBEAT_INTERVAL) + + # Send ping message + connection = self._get_connection(client_id, connection_id) + await self.send_message(client_id, connection_id, Message(type=MessageType.PING)) + + # Check if heartbeat timeout + if connection and time.time() - connection._last_heartbeat > HEARTBEAT_TIMEOUT: + logger.error(f"Heartbeat timeout for client {client_id}") + await self.disconnect(client_id, connection.ws) + break + + except Exception as e: + logger.error(f"Error in heartbeat for client {client_id}: {e}") + break + + async def connect(self, client_id: str, websocket: WebSocket, data: Any = None) -> ConnectionData: await websocket.accept() if client_id not in self.active_connections: self.active_connections[client_id] = [] logger.info(f"Client {client_id} connected") - self.active_connections[client_id].append(ConnectionData(ws=websocket, data=data)) - - async def disconnect(self, websocket: WebSocket, client_id: str): - if client_id in self.active_connections: - self.active_connections[client_id] = [ - connection for connection in self.active_connections[client_id] - if connection.ws != websocket - ] + id = uuid.uuid4() + connection_data = ConnectionData( + id=id, + ws=websocket, + data=data, + heartbeat_task=asyncio.create_task(self._heartbeat(client_id, id)), + ) + + if not self.active_connections[client_id]: + self.active_connections[client_id] = {} - if not self.active_connections[client_id]: - del self.active_connections[client_id] - - logger.info(f"Client {client_id} disconnected") - - async def send_message(self, message: dict, client_id: str): - if client_id in self.active_connections: - for connection in self.active_connections[client_id]: - await connection.ws.send_json(message) + self.active_connections[client_id][id] = connection_data + + return connection_data - async def broadcast(self, message: dict, exclude: str = None): - for client_id, connections in self.active_connections.items(): - if client_id != exclude: - for connection in connections: - await connection.ws.send_json(message) - - async def send_personal_message(self, message: dict, client_id: str): + async def disconnect(self, client_id: str, websocket: WebSocket): if client_id in self.active_connections: - for connection in self.active_connections[client_id]: - await connection.ws.send_json(message) \ No newline at end of file + for connection in self.active_connections[client_id].values(): + if connection.ws == websocket: + connection.heartbeat_task.cancel() + del self.active_connections[client_id][connection.id] + logger.info(f"Client {client_id} disconnected") + break + + if not self.active_connections[client_id]: + del self.active_connections[client_id] + + async def send_message(self, client_id: str, conn_id: uuid.UUID | None = None, message: Message = None): + if not conn_id: + if client_id in self.active_connections: + for connection in self.active_connections[client_id].values(): + await connection.ws.send_json(message.dict()) + else: + connection = self._get_connection(client_id, conn_id) + if connection: + await connection.ws.send_json(message.dict()) \ No newline at end of file From 7a94abe6e5ee97d40c04d8b51e83ae52e9ec89b4 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 7 Mar 2025 11:48:06 +0000 Subject: [PATCH 142/236] fix: Upgrade frontend to v5.7.0-alpha.17. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.17) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index dad98b28..5bf870db 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.16/frontend.zip -alpha_version: v5.7.0-alpha.16 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.17/frontend.zip +alpha_version: v5.7.0-alpha.17 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 8db71fe07def7e02f1bc88f7df2f3d584cef0da3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 7 Mar 2025 11:53:08 +0000 Subject: [PATCH 143/236] chore(release): 1.1.0-alpha.45 [skip ci] # [1.1.0-alpha.45](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.44...v1.1.0-alpha.45) (2025-03-07) ### Bug Fixes * optimize ws connection ([a874f0b](https://github.com/opendatalab/labelU/commit/a874f0bdc84246e9df1831a23d3c17c40574b852)) * Upgrade frontend to v5.7.0-alpha.17. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.17) ([7a94abe](https://github.com/opendatalab/labelU/commit/7a94abe6e5ee97d40c04d8b51e83ae52e9ec89b4)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 2aa99ad8..0e3e0411 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.44' \ No newline at end of file +version='1.1.0-alpha.45' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cd67732d..980628b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.44' +version = '1.1.0-alpha.45' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 643cd5930b3bdc9f38ca114eb5ee15b8f23bfc5e Mon Sep 17 00:00:00 2001 From: shenguanlin Date: Sat, 8 Mar 2025 12:30:51 +0800 Subject: [PATCH 144/236] fix: missing task_attachment_ids column in task_sample table --- .../2eb983c9a254_add_collaborators_and_updaters.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py index 4d78c923..8000ab86 100644 --- a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py +++ b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py @@ -10,6 +10,8 @@ import sqlalchemy as sa from sqlalchemy.sql import table, column +from labelu.alembic_labelu.alembic_labelu_tools import column_exist_in_table + # revision identifiers, used by Alembic. revision = '2eb983c9a254' down_revision = 'eb9c5b98168b' @@ -137,6 +139,18 @@ def upgrade() -> None: ) ) + # Fix missing task_attachment_ids column in task_sample table + if not column_exist_in_table("task_sample", "task_attachment_id"): + with op.batch_alter_table('task_sample', recreate="always") as batch_op: + batch_op.add_column( + sa.Column( + "task_attachment_id", + sa.Integer(), + sa.ForeignKey("task_attachment.id", name="fk_task_attachment_id"), + index=True, + comment="task attachment id", + ), + ) def downgrade() -> None: op.drop_index('ix_task_collaborator_user_id') From 581a1988d627aeb384d2983f7adf546d8102f301 Mon Sep 17 00:00:00 2001 From: shenguanlin Date: Sat, 8 Mar 2025 19:12:30 +0800 Subject: [PATCH 145/236] update: missing column task_attachment_ids --- ...83c9a254_add_collaborators_and_updaters.py | 45 ++++++++++++++----- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py index 8000ab86..d9e48c93 100644 --- a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py +++ b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py @@ -140,25 +140,46 @@ def upgrade() -> None: ) # Fix missing task_attachment_ids column in task_sample table - if not column_exist_in_table("task_sample", "task_attachment_id"): + if not column_exist_in_table("task_sample", "task_attachment_ids"): with op.batch_alter_table('task_sample', recreate="always") as batch_op: batch_op.add_column( sa.Column( - "task_attachment_id", - sa.Integer(), - sa.ForeignKey("task_attachment.id", name="fk_task_attachment_id"), - index=True, - comment="task attachment id", + "task_attachment_ids", + sa.Text(), + comment="task attachment ids", ), ) def downgrade() -> None: - op.drop_index('ix_task_collaborator_user_id') - op.drop_index('ix_task_collaborator_task_id') + conn = op.get_bind() + inspector = sa.inspect(conn) + tables = inspector.get_table_names() - op.drop_table('task_collaborator') + if 'task_collaborator' in tables: + indices = inspector.get_indexes('task_collaborator') + existing_index_names = {idx['name'] for idx in indices} + + if 'ix_task_collaborator_user_id' in existing_index_names: + op.drop_index('ix_task_collaborator_user_id', table_name='task_collaborator') + if 'ix_task_collaborator_task_id' in existing_index_names: + op.drop_index('ix_task_collaborator_task_id', table_name='task_collaborator') + + op.drop_table('task_collaborator') - op.drop_index('ix_task_sample_updater_user_id') - op.drop_index('ix_task_sample_updater_task_sample_id') + if 'task' in tables: + indices = inspector.get_indexes('task') + existing_index_names = {idx['name'] for idx in indices} + + if 'ix_task_created_by_deleted_at' in existing_index_names: + op.drop_index('ix_task_created_by_deleted_at', table_name='task') - op.drop_table('task_sample_updater') + if 'task_sample_updater' in tables: + indices = inspector.get_indexes('task_sample_updater') + existing_index_names = {idx['name'] for idx in indices} + + if 'ix_task_sample_updater_user_id' in existing_index_names: + op.drop_index('ix_task_sample_updater_user_id', table_name='task_sample_updater') + if 'ix_task_sample_updater_sample_id' in existing_index_names: + op.drop_index('ix_task_sample_updater_sample_id', table_name='task_sample_updater') + + op.drop_table('task_sample_updater') From 81286215f7f6ee26aae68fdbe5564affd49c7d5a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Sat, 8 Mar 2025 11:18:39 +0000 Subject: [PATCH 146/236] chore(release): 1.1.0-alpha.46 [skip ci] # [1.1.0-alpha.46](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.45...v1.1.0-alpha.46) (2025-03-08) ### Bug Fixes * missing task_attachment_ids column in task_sample table ([643cd59](https://github.com/opendatalab/labelU/commit/643cd5930b3bdc9f38ca114eb5ee15b8f23bfc5e)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 0e3e0411..7552f752 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.45' \ No newline at end of file +version='1.1.0-alpha.46' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 980628b6..c6967920 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.45' +version = '1.1.0-alpha.46' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 71f4c4a6b169f19ded2167fc07818a494a4bac8c Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 10 Mar 2025 11:17:41 +0800 Subject: [PATCH 147/236] fix: update migration script --- ...8fcb35b66b_add_media_and_pre_annotation.py | 49 ++++++++++--------- labelu/internal/domain/models/sample.py | 1 - 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py b/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py index c2619b7e..64e8a587 100644 --- a/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py +++ b/labelu/alembic_labelu/versions/bc8fcb35b66b_add_media_and_pre_annotation.py @@ -120,32 +120,33 @@ def upgrade() -> None: f"UPDATE task_attachment SET filename='{filename}', url='{url}' WHERE id={attachment_id}" ) - for task_item in task_items: - task_id = task_item[0] - task_samples = session.execute( - f"SELECT id, task_attachment_ids FROM task_sample WHERE task_id={task_id}" - ) - - for task_sample in task_samples: - task_sample_id = task_sample[0] - attachment_ids = json.loads(task_sample[1]) - # attachment_ids 存储的是字符串[id1, id2, id3]，需要转换成数组 - file_id = attachment_ids[0] - - if not file_id: - continue - - attachment = session.execute( - f"SELECT id, path FROM task_attachment WHERE id={file_id}" + if column_exist_in_table("task_sample", "task_attachment_ids"): + for task_item in task_items: + task_id = task_item[0] + task_samples = session.execute( + f"SELECT id, task_attachment_ids FROM task_sample WHERE task_id={task_id}" ) - attachment_path = list(attachment)[0][1] - - if attachment_path: - # Update the task_sample table - session.execute( - f"UPDATE task_sample SET file_id={file_id} WHERE id={task_sample_id}" + + for task_sample in task_samples: + task_sample_id = task_sample[0] + attachment_ids = json.loads(task_sample[1]) + # attachment_ids 存储的是字符串[id1, id2, id3]，需要转换成数组 + file_id = attachment_ids[0] + + if not file_id: + continue + + attachment = session.execute( + f"SELECT id, path FROM task_attachment WHERE id={file_id}" ) - + attachment_path = list(attachment)[0][1] + + if attachment_path: + # Update the task_sample table + session.execute( + f"UPDATE task_sample SET file_id={file_id} WHERE id={task_sample_id}" + ) + session.commit() except Exception as e: diff --git a/labelu/internal/domain/models/sample.py b/labelu/internal/domain/models/sample.py index 3cc496b8..7e6c946b 100644 --- a/labelu/internal/domain/models/sample.py +++ b/labelu/internal/domain/models/sample.py @@ -47,7 +47,6 @@ class TaskSample(Base): default=SampleState.NEW.value, comment="NEW is has not start yet, DONE is completed, SKIPPED is skipped", ) - task_attachment_ids = Column(Text, comment="task sample file ids") deleted_at = Column(DateTime(timezone=True), index=True, comment="Task delete time") # 由旧的data里的fileNames和urls中的唯一一个，迁移到media中 From a9d878e867f04591175c8f6c4c3de0ffb9004131 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 10 Mar 2025 03:23:01 +0000 Subject: [PATCH 148/236] chore(release): 1.1.0-alpha.47 [skip ci] # [1.1.0-alpha.47](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.46...v1.1.0-alpha.47) (2025-03-10) ### Bug Fixes * update migration script ([71f4c4a](https://github.com/opendatalab/labelU/commit/71f4c4a6b169f19ded2167fc07818a494a4bac8c)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 7552f752..538aa48a 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.46' \ No newline at end of file +version='1.1.0-alpha.47' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index c6967920..dc5f5fd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.46' +version = '1.1.0-alpha.47' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 6d16f340f3e80f9739d4946fd34df3d71edb0a49 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 10 Mar 2025 16:04:59 +0800 Subject: [PATCH 149/236] fix: optimize ws connection --- labelu/internal/adapter/ws/sample.py | 22 ++++++---------- labelu/internal/common/websocket.py | 39 ++++++++++++++++++---------- 2 files changed, 33 insertions(+), 28 deletions(-) diff --git a/labelu/internal/adapter/ws/sample.py b/labelu/internal/adapter/ws/sample.py index b09678d1..5993eb2e 100644 --- a/labelu/internal/adapter/ws/sample.py +++ b/labelu/internal/adapter/ws/sample.py @@ -52,22 +52,16 @@ async def sync_peers(): async def cleanup(): await sampleConnectionManager.disconnect(client_id, websocket) await sync_peers() - # Tell clients that someone has left the sample page and refresh the page data - await sampleConnectionManager.send_message( - client_id=client_id, - message=Message( - type=MessageType.LEAVE, - data=TaskSampleWsPayload( - task_id=task_id, - user_id=user.id, - username=user.username, - sample_id=sample_id, - ) - ) - ) if connection: - connection.heartbeat_task.cancel() + # Tell clients that someone has left the sample page and refresh the page data + await sampleConnectionManager.send_message( + client_id=client_id, + message=Message( + type=MessageType.LEAVE, + data=connection.data + ) + ) try: connection = await sampleConnectionManager.connect( diff --git a/labelu/internal/common/websocket.py b/labelu/internal/common/websocket.py index 6627ab33..cdca62ec 100644 --- a/labelu/internal/common/websocket.py +++ b/labelu/internal/common/websocket.py @@ -24,14 +24,14 @@ class Message(BaseModel): @dataclass class ConnectionData: - id: uuid.UUID = None + id: str = None ws: WebSocket = None data: Any = None heartbeat_task: asyncio.Task = None - _last_heartbeat: float = time.time() + last_heartbeat: float = time.time() def update_heartbeat(self): - self._last_heartbeat = time.time() + self.last_heartbeat = time.time() class ConnectionManager: def __init__(self): @@ -50,11 +50,15 @@ async def _heartbeat(self, client_id: str, connection_id: uuid.UUID): # Send ping message connection = self._get_connection(client_id, connection_id) + + if not connection: + break + await self.send_message(client_id, connection_id, Message(type=MessageType.PING)) # Check if heartbeat timeout - if connection and time.time() - connection._last_heartbeat > HEARTBEAT_TIMEOUT: - logger.error(f"Heartbeat timeout for client {client_id}") + if connection and time.time() - connection.last_heartbeat > HEARTBEAT_TIMEOUT: + logger.error(f"Heartbeat timeout for client {connection.data.username}") await self.disconnect(client_id, connection.ws) break @@ -69,11 +73,12 @@ async def connect(self, client_id: str, websocket: WebSocket, data: Any = None) self.active_connections[client_id] = [] logger.info(f"Client {client_id} connected") - id = uuid.uuid4() + id = str(uuid.uuid4()) connection_data = ConnectionData( id=id, ws=websocket, data=data, + last_heartbeat=time.time(), heartbeat_task=asyncio.create_task(self._heartbeat(client_id, id)), ) @@ -85,15 +90,21 @@ async def connect(self, client_id: str, websocket: WebSocket, data: Any = None) return connection_data async def disconnect(self, client_id: str, websocket: WebSocket): - if client_id in self.active_connections: - for connection in self.active_connections[client_id].values(): - if connection.ws == websocket: - connection.heartbeat_task.cancel() - del self.active_connections[client_id][connection.id] - logger.info(f"Client {client_id} disconnected") - break + if client_id not in self.active_connections: + return - if not self.active_connections[client_id]: + connection_to_remove = None + for connection_id, connection in self.active_connections[client_id].items(): + if connection.ws == websocket: + connection.heartbeat_task.cancel() + connection_to_remove = connection + logger.info(f"Client {client_id} disconnected") + break + + if connection_to_remove is not None: + del self.active_connections[client_id][connection_to_remove.id] + + if client_id in self.active_connections and not self.active_connections[client_id]: del self.active_connections[client_id] async def send_message(self, client_id: str, conn_id: uuid.UUID | None = None, message: Message = None): From 64522357491caf554016c3f984035ba6742518b9 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 10 Mar 2025 08:10:14 +0000 Subject: [PATCH 150/236] chore(release): 1.1.0-alpha.48 [skip ci] # [1.1.0-alpha.48](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.47...v1.1.0-alpha.48) (2025-03-10) ### Bug Fixes * optimize ws connection ([6d16f34](https://github.com/opendatalab/labelU/commit/6d16f340f3e80f9739d4946fd34df3d71edb0a49)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 538aa48a..bf2894ec 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.47' \ No newline at end of file +version='1.1.0-alpha.48' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index dc5f5fd8..b1c169cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.47' +version = '1.1.0-alpha.48' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 61e23ee3893c778fc2506826b5dd0bead4840ed1 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 10 Mar 2025 12:46:44 +0000 Subject: [PATCH 151/236] fix: Upgrade frontend to v5.7.0-alpha.18. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.18) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 5bf870db..1974b555 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.17/frontend.zip -alpha_version: v5.7.0-alpha.17 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.18/frontend.zip +alpha_version: v5.7.0-alpha.18 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 0d07c0981da82ca25e3bda0e4df7c254ddfa96cc Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 10 Mar 2025 12:51:59 +0000 Subject: [PATCH 152/236] chore(release): 1.1.0-alpha.49 [skip ci] # [1.1.0-alpha.49](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.48...v1.1.0-alpha.49) (2025-03-10) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.18. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.18) ([61e23ee](https://github.com/opendatalab/labelU/commit/61e23ee3893c778fc2506826b5dd0bead4840ed1)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index bf2894ec..f7258a6e 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.48' \ No newline at end of file +version='1.1.0-alpha.49' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index b1c169cb..585ebeaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.48' +version = '1.1.0-alpha.49' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 43133dac5d5cd1863136b8481eab48c8ca508bed Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 11 Mar 2025 12:10:20 +0800 Subject: [PATCH 153/236] fix: ws message when sample is updated --- labelu/internal/adapter/ws/sample.py | 10 ---------- labelu/internal/application/service/sample.py | 17 +++++++++++++++++ labelu/internal/common/websocket.py | 2 +- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/labelu/internal/adapter/ws/sample.py b/labelu/internal/adapter/ws/sample.py index 5993eb2e..c47aa204 100644 --- a/labelu/internal/adapter/ws/sample.py +++ b/labelu/internal/adapter/ws/sample.py @@ -52,16 +52,6 @@ async def sync_peers(): async def cleanup(): await sampleConnectionManager.disconnect(client_id, websocket) await sync_peers() - - if connection: - # Tell clients that someone has left the sample page and refresh the page data - await sampleConnectionManager.send_message( - client_id=client_id, - message=Message( - type=MessageType.LEAVE, - data=connection.data - ) - ) try: connection = await sampleConnectionManager.connect( diff --git a/labelu/internal/application/service/sample.py b/labelu/internal/application/service/sample.py index 54cd076e..4b991d2a 100644 --- a/labelu/internal/application/service/sample.py +++ b/labelu/internal/application/service/sample.py @@ -29,6 +29,9 @@ from labelu.internal.application.response.sample import CreateSampleResponse from labelu.internal.application.response.sample import SampleResponse from labelu.internal.application.response.attachment import AttachmentResponse +from labelu.internal.clients.ws import sampleConnectionManager +from labelu.internal.common.websocket import Message, MessageType +from labelu.internal.adapter.ws.sample import TaskSampleWsPayload def is_sample_pre_annotated(db: Session, task_id: int, sample_name: str | None = None) -> Tuple[List[TaskPreAnnotation], int]: if sample_name is None: @@ -220,6 +223,20 @@ async def patch( sample.updaters.append(current_user) # update task sample result updated_sample = crud_sample.update(db=db, db_obj=sample, obj_in=sample_obj_in) + + # tell other clients in the same sample page to refresh data + await sampleConnectionManager.send_message( + client_id=f"task_{task_id}", + message=Message( + type=MessageType.UPDATE, + data=TaskSampleWsPayload( + task_id=task_id, + user_id=current_user.id, + username=current_user.username, + sample_id=sample_id, + ) + ) + ) # response return SampleResponse( diff --git a/labelu/internal/common/websocket.py b/labelu/internal/common/websocket.py index cdca62ec..00aafddc 100644 --- a/labelu/internal/common/websocket.py +++ b/labelu/internal/common/websocket.py @@ -16,7 +16,7 @@ class MessageType(str, Enum): PEERS = "peers" PING = "ping" PONG = "pong" - LEAVE = "leave" + UPDATE = "update" class Message(BaseModel): type: MessageType From fca41a3a60c1849245e67927bbd3c6ea45265f48 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 11 Mar 2025 04:17:01 +0000 Subject: [PATCH 154/236] fix: Upgrade frontend to v5.7.0-alpha.19. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.19) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 1974b555..5f2c2355 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.18/frontend.zip -alpha_version: v5.7.0-alpha.18 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.19/frontend.zip +alpha_version: v5.7.0-alpha.19 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 33c7b770e1bdad7db797dec92b306aa2a9c7599c Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 11 Mar 2025 04:22:15 +0000 Subject: [PATCH 155/236] chore(release): 1.1.0-alpha.50 [skip ci] # [1.1.0-alpha.50](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.49...v1.1.0-alpha.50) (2025-03-11) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.19. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.19) ([fca41a3](https://github.com/opendatalab/labelU/commit/fca41a3a60c1849245e67927bbd3c6ea45265f48)) * ws message when sample is updated ([43133da](https://github.com/opendatalab/labelU/commit/43133dac5d5cd1863136b8481eab48c8ca508bed)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index f7258a6e..3d845fbe 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.49' \ No newline at end of file +version='1.1.0-alpha.50' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 585ebeaf..f9ba531f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.49' +version = '1.1.0-alpha.50' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From d35101d737ddd0a7b1c9a3b6265266fa8a610807 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 11 Mar 2025 07:57:23 +0000 Subject: [PATCH 156/236] fix: Upgrade frontend to v5.7.0-alpha.20. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.20) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 5f2c2355..d9804ccf 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.19/frontend.zip -alpha_version: v5.7.0-alpha.19 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.20/frontend.zip +alpha_version: v5.7.0-alpha.20 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 4c546350d5b9c5979dfc1b0746e1aeec838ace62 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 11 Mar 2025 08:02:24 +0000 Subject: [PATCH 157/236] chore(release): 1.1.0-alpha.51 [skip ci] # [1.1.0-alpha.51](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.50...v1.1.0-alpha.51) (2025-03-11) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.20. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.20) ([d35101d](https://github.com/opendatalab/labelU/commit/d35101d737ddd0a7b1c9a3b6265266fa8a610807)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 3d845fbe..65de165c 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.50' \ No newline at end of file +version='1.1.0-alpha.51' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index f9ba531f..fe16b066 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.50' +version = '1.1.0-alpha.51' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 01b40b8304137f4170791bf59916f2bfeb98ddde Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 11 Mar 2025 09:27:51 +0000 Subject: [PATCH 158/236] fix: Upgrade frontend to v5.7.0-alpha.21. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.21) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index d9804ccf..7830095f 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.20/frontend.zip -alpha_version: v5.7.0-alpha.20 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.21/frontend.zip +alpha_version: v5.7.0-alpha.21 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 608619989186654d44dcd069673986af15b0f9a2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 11 Mar 2025 09:33:16 +0000 Subject: [PATCH 159/236] chore(release): 1.1.0-alpha.52 [skip ci] # [1.1.0-alpha.52](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.51...v1.1.0-alpha.52) (2025-03-11) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.21. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.21) ([01b40b8](https://github.com/opendatalab/labelU/commit/01b40b8304137f4170791bf59916f2bfeb98ddde)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 65de165c..aeaa7653 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.51' \ No newline at end of file +version='1.1.0-alpha.52' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index fe16b066..fa5b85d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.51' +version = '1.1.0-alpha.52' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From c1f9cca74deed0b6a5792011b43c22c2a7d928b0 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 11 Mar 2025 11:48:10 +0000 Subject: [PATCH 160/236] fix: Upgrade frontend to v5.7.0-alpha.22. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.22) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 7830095f..a8b99f53 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.21/frontend.zip -alpha_version: v5.7.0-alpha.21 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.22/frontend.zip +alpha_version: v5.7.0-alpha.22 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From ef5b69f7b12ce0e1cd2bb059f3d947f4402a7e1e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 11 Mar 2025 11:53:47 +0000 Subject: [PATCH 161/236] chore(release): 1.1.0-alpha.53 [skip ci] # [1.1.0-alpha.53](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.52...v1.1.0-alpha.53) (2025-03-11) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.22. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.22) ([c1f9cca](https://github.com/opendatalab/labelU/commit/c1f9cca74deed0b6a5792011b43c22c2a7d928b0)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index aeaa7653..6a1f5a72 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.52' \ No newline at end of file +version='1.1.0-alpha.53' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index fa5b85d7..66d28dfa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.52' +version = '1.1.0-alpha.53' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 1e88cb5d8e066fab6ea67d2a1ebf893b62077d0b Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 11 Mar 2025 12:17:16 +0000 Subject: [PATCH 162/236] fix: Upgrade frontend to v5.7.0-alpha.23. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.23) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index a8b99f53..e99c21a0 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.22/frontend.zip -alpha_version: v5.7.0-alpha.22 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.23/frontend.zip +alpha_version: v5.7.0-alpha.23 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 125d1e7a6f39fedc81d17b50cdf0ee86f7e42126 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 11 Mar 2025 12:22:40 +0000 Subject: [PATCH 163/236] chore(release): 1.1.0-alpha.54 [skip ci] # [1.1.0-alpha.54](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.53...v1.1.0-alpha.54) (2025-03-11) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.23. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.23) ([1e88cb5](https://github.com/opendatalab/labelU/commit/1e88cb5d8e066fab6ea67d2a1ebf893b62077d0b)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 6a1f5a72..d6fc2ef8 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.53' \ No newline at end of file +version='1.1.0-alpha.54' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 66d28dfa..2aa1ec18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.53' +version = '1.1.0-alpha.54' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From cc2cd41cbf6e9e745d6d45e7d7ffddfac6598c71 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 12 Mar 2025 04:00:52 +0000 Subject: [PATCH 164/236] fix: Upgrade frontend to v5.7.0-alpha.24. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.24) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index e99c21a0..f496f38e 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.23/frontend.zip -alpha_version: v5.7.0-alpha.23 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.24/frontend.zip +alpha_version: v5.7.0-alpha.24 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 9c33d6a9cfe5c69a563e58f9b4083250f5696772 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 12 Mar 2025 04:06:00 +0000 Subject: [PATCH 165/236] chore(release): 1.1.0-alpha.55 [skip ci] # [1.1.0-alpha.55](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.54...v1.1.0-alpha.55) (2025-03-12) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.24. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.24) ([cc2cd41](https://github.com/opendatalab/labelU/commit/cc2cd41cbf6e9e745d6d45e7d7ffddfac6598c71)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index d6fc2ef8..f4a56008 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.54' \ No newline at end of file +version='1.1.0-alpha.55' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 2aa1ec18..cbcf86e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.54' +version = '1.1.0-alpha.55' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 2ecf3a650c5e0aed792fe863dba1b6391aa8cff3 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 12 Mar 2025 06:47:49 +0000 Subject: [PATCH 166/236] fix: Upgrade frontend to v5.7.0-alpha.25. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.25) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index f496f38e..7be416f8 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.24/frontend.zip -alpha_version: v5.7.0-alpha.24 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.25/frontend.zip +alpha_version: v5.7.0-alpha.25 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 9255efc3afad2a68115f4ae391b77112a3f0cb6a Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 12 Mar 2025 06:52:42 +0000 Subject: [PATCH 167/236] chore(release): 1.1.0-alpha.56 [skip ci] # [1.1.0-alpha.56](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.55...v1.1.0-alpha.56) (2025-03-12) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.25. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.25) ([2ecf3a6](https://github.com/opendatalab/labelU/commit/2ecf3a650c5e0aed792fe863dba1b6391aa8cff3)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index f4a56008..b83793b3 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.55' \ No newline at end of file +version='1.1.0-alpha.56' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index cbcf86e4..f33c78bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.55' +version = '1.1.0-alpha.56' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From a00b4fa21c03d4340f6ac1f92d93e3d76be9284e Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 12 Mar 2025 07:22:36 +0000 Subject: [PATCH 168/236] fix: Upgrade frontend to v5.7.0-alpha.26. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.26) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 7be416f8..3270d3dc 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.25/frontend.zip -alpha_version: v5.7.0-alpha.25 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.26/frontend.zip +alpha_version: v5.7.0-alpha.26 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From fe2a902596d810c6e6f1e925aa2a07519b53a03f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 12 Mar 2025 07:27:30 +0000 Subject: [PATCH 169/236] chore(release): 1.1.0-alpha.57 [skip ci] # [1.1.0-alpha.57](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.56...v1.1.0-alpha.57) (2025-03-12) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.26. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.26) ([a00b4fa](https://github.com/opendatalab/labelU/commit/a00b4fa21c03d4340f6ac1f92d93e3d76be9284e)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index b83793b3..f5423fbc 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.56' \ No newline at end of file +version='1.1.0-alpha.57' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index f33c78bc..98cecdf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.56' +version = '1.1.0-alpha.57' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From accc1bd72d8d695cefd9c69376391ef29a61985a Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 12 Mar 2025 08:15:32 +0000 Subject: [PATCH 170/236] fix: Upgrade frontend to v5.7.0-alpha.27. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.27) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 3270d3dc..33a912e9 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.26/frontend.zip -alpha_version: v5.7.0-alpha.26 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.27/frontend.zip +alpha_version: v5.7.0-alpha.27 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.6.11/frontend.zip release_version: v5.6.11 From 05280b49c80419ea1abf728b8f8fc60eece0a409 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 12 Mar 2025 08:20:38 +0000 Subject: [PATCH 171/236] chore(release): 1.1.0-alpha.58 [skip ci] # [1.1.0-alpha.58](https://github.com/opendatalab/labelU/compare/v1.1.0-alpha.57...v1.1.0-alpha.58) (2025-03-12) ### Bug Fixes * Upgrade frontend to v5.7.0-alpha.27. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.27) ([accc1bd](https://github.com/opendatalab/labelU/commit/accc1bd72d8d695cefd9c69376391ef29a61985a)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index f5423fbc..4ef12b76 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.1.0-alpha.57' \ No newline at end of file +version='1.1.0-alpha.58' \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 98cecdf1..d9615c73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.1.0-alpha.57' +version = '1.1.0-alpha.58' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From fa1f8613e6c610b5a241a233bcbceb2ba70e0b5f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 13 Mar 2025 13:17:18 +0000 Subject: [PATCH 172/236] chore(release): 1.3.0-alpha.1 [skip ci] # [1.3.0-alpha.1](https://github.com/opendatalab/labelU/compare/v1.2.1...v1.3.0-alpha.1) (2025-03-13) ### Bug Fixes * add is_pre_annotated field to the smaple response ([8606988](https://github.com/opendatalab/labelU/commit/8606988f00e34f489252b16d8f89a514b6a79905)) * connections -> peers ([39f519e](https://github.com/opendatalab/labelU/commit/39f519ee10cac74ddee429f0988a3090e0412b1e)) * delete attachment when delete sample ([4255b53](https://github.com/opendatalab/labelU/commit/4255b53837d4da446374e02cade16b28a3670172)) * delete pre-annotation records after pre-annotation files deleted [#159](https://github.com/opendatalab/labelU/issues/159) ([1bc7b21](https://github.com/opendatalab/labelU/commit/1bc7b2196c6bdcb3c3df0702eef55a410e31e0b2)) * labelu.py -> main.py ([fb6c3e1](https://github.com/opendatalab/labelU/commit/fb6c3e1bac3fb714c91139cae82d4d79f13b255d)) * main.py -> labelu.py ([#199](https://github.com/opendatalab/labelU/issues/199)) ([5e413ab](https://github.com/opendatalab/labelU/commit/5e413ab86f7bc0ca18e5e2fce82ed2e22f4d9cc2)) * missing task_attachment_ids column in task_sample table ([643cd59](https://github.com/opendatalab/labelU/commit/643cd5930b3bdc9f38ca114eb5ee15b8f23bfc5e)) * no data to export ([eef80f0](https://github.com/opendatalab/labelU/commit/eef80f099d96384cdad9e702dc7012dba2d06a26)) * optimize sample_names query ([be7db25](https://github.com/opendatalab/labelU/commit/be7db25faa629b723b7bd1a584f474badd2c3a7f)) * optimize ws connection ([6d16f34](https://github.com/opendatalab/labelU/commit/6d16f340f3e80f9739d4946fd34df3d71edb0a49)) * optimize ws connection ([a874f0b](https://github.com/opendatalab/labelU/commit/a874f0bdc84246e9df1831a23d3c17c40574b852)) * remove owner_id param in list pre_annotations ([#192](https://github.com/opendatalab/labelU/issues/192)) ([f917091](https://github.com/opendatalab/labelU/commit/f917091792f734ada91d67c473dde1d5c34b9e81)) * set current version on http response headers ([#183](https://github.com/opendatalab/labelU/issues/183)) ([576333d](https://github.com/opendatalab/labelU/commit/576333d2db2ba61a3ba37f0667dc4d191ede46d0)) * test notification action ([83f43eb](https://github.com/opendatalab/labelU/commit/83f43eb9d03390686a41dc22666a058190bfb63c)) * test notification action ([bc63ee5](https://github.com/opendatalab/labelU/commit/bc63ee59da0d079208b1385d33184267251a2bb5)) * test notification action ([851818e](https://github.com/opendatalab/labelU/commit/851818e912cf39690be6e689c1926ec29c148a66)) * test notification action ([8fed078](https://github.com/opendatalab/labelU/commit/8fed078a0d77a8204430fb6c24d91de1250a8bc7)) * update command ([#200](https://github.com/opendatalab/labelU/issues/200)) ([ac5ae35](https://github.com/opendatalab/labelU/commit/ac5ae358b70661cea2932d252cfa05c3a24ac2f2)) * update default command ([948d9af](https://github.com/opendatalab/labelU/commit/948d9afcee4b86b1a0f373af0333df832a7e7753)) * update default command ([6b345b9](https://github.com/opendatalab/labelU/commit/6b345b9eae82b4e1408cdd05e63bdf8771ccad8b)) * update default command ([#197](https://github.com/opendatalab/labelU/issues/197)) ([6bae992](https://github.com/opendatalab/labelU/commit/6bae992b2024edb6f4fe32f03de950d88db933eb)) * update default command ([#198](https://github.com/opendatalab/labelU/issues/198)) ([d5a8545](https://github.com/opendatalab/labelU/commit/d5a8545e76582fd07c3b376c9fb9d2343fe152fc)) * update migrate script ([c436b00](https://github.com/opendatalab/labelU/commit/c436b0041189cafd51eaea58871bb336d9f8619f)) * update migration script ([71f4c4a](https://github.com/opendatalab/labelU/commit/71f4c4a6b169f19ded2167fc07818a494a4bac8c)) * update notification action ([63286cb](https://github.com/opendatalab/labelU/commit/63286cb6958e20ab3c80495ab3b51a1ddd31dfb7)) * update notification action ([046f47a](https://github.com/opendatalab/labelU/commit/046f47a3f4053e6934f04c3c0ede5518b9f1850b)) * update notification action ([e9152c9](https://github.com/opendatalab/labelU/commit/e9152c98869d70e500fd9ab458d8ba179270d9c9)) * update notification action ([04e0388](https://github.com/opendatalab/labelU/commit/04e0388e3cd98a2aacf12dc28c638c172eb6f0ff)) * update notification action ([b6abe17](https://github.com/opendatalab/labelU/commit/b6abe172f7a8bb1a5547acb2bc8948bb7c0dae7d)) * Upgrade frontend to v5.3.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.4) ([01fc3b5](https://github.com/opendatalab/labelU/commit/01fc3b53b570de4d05ad771f163d8da03236b141)) * Upgrade frontend to v5.3.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.5) ([f97601d](https://github.com/opendatalab/labelU/commit/f97601d0c4286c633ca2ce3a1f5ef3d8eec5a540)) * Upgrade frontend to v5.3.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.6) ([9f5e698](https://github.com/opendatalab/labelU/commit/9f5e698c8a0476be0bb034bb12e226b9db211328)) * Upgrade frontend to v5.3.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.3.0-alpha.7) ([7cb455e](https://github.com/opendatalab/labelU/commit/7cb455eaf06b8408f2cacfe3cc39c8c2e3bde013)) * Upgrade frontend to v5.5.0-alpha.1. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.1) ([7b04670](https://github.com/opendatalab/labelU/commit/7b046706c7c1ff7e9fa94fd912f4263d33181e7b)) * Upgrade frontend to v5.5.0-alpha.10. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.10) ([2fd7765](https://github.com/opendatalab/labelU/commit/2fd776543db8ca728832110ba0c38af16b57e097)) * Upgrade frontend to v5.5.0-alpha.11. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.11) ([39bbff7](https://github.com/opendatalab/labelU/commit/39bbff793248a255d3c1016a2033f682b113b878)) * Upgrade frontend to v5.5.0-alpha.12. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.12) ([95232c8](https://github.com/opendatalab/labelU/commit/95232c8c8fee9bb5c83bb01bd2271d5bd613d89e)) * Upgrade frontend to v5.5.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.2) ([491cf33](https://github.com/opendatalab/labelU/commit/491cf335a0f10c1565536bc0c4cea254d0b6b23b)) * Upgrade frontend to v5.5.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.3) ([dc022fd](https://github.com/opendatalab/labelU/commit/dc022fd8da66f496afa8ee5a52c46e65892e7c51)) * Upgrade frontend to v5.5.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.4) ([7622524](https://github.com/opendatalab/labelU/commit/7622524e6f36abe2a663614c679d4d9b6c02a0f0)) * Upgrade frontend to v5.5.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.5) ([27f27be](https://github.com/opendatalab/labelU/commit/27f27be399f99b9032c50ab000614b06d4c608cb)) * Upgrade frontend to v5.5.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.6) ([2d986b8](https://github.com/opendatalab/labelU/commit/2d986b8435d45741f09fb095d9aafe4df0c1d424)) * Upgrade frontend to v5.5.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.7) ([658f1a4](https://github.com/opendatalab/labelU/commit/658f1a4a8c406c4d05abc6b5f1f7da1a40a69513)) * Upgrade frontend to v5.5.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.8) ([b2ea80a](https://github.com/opendatalab/labelU/commit/b2ea80a79a4aee7ec3ce87c2c4abbf472297ded9)) * Upgrade frontend to v5.5.0-alpha.9. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.5.0-alpha.9) ([16d66fd](https://github.com/opendatalab/labelU/commit/16d66fd40aee2d94c7ef0ab7396f5e859f9d88a3)) * Upgrade frontend to v5.6.0-alpha.1. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.1) ([9326d06](https://github.com/opendatalab/labelU/commit/9326d0601cebcb9c1c2dd8dd94275d0ddffa04b0)) * Upgrade frontend to v5.6.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.2) ([39b7182](https://github.com/opendatalab/labelU/commit/39b718217da55c2de92fc71401ee7c205799bec2)) * Upgrade frontend to v5.6.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.3) ([8ef74dd](https://github.com/opendatalab/labelU/commit/8ef74dd851c6c8271f24d97d111df67c91f064f9)) * Upgrade frontend to v5.6.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.4) ([77e2913](https://github.com/opendatalab/labelU/commit/77e2913a0905e077e95549e1e54320206816fc1f)) * Upgrade frontend to v5.6.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.5) ([60707c5](https://github.com/opendatalab/labelU/commit/60707c548a3b4d8e5ac099e3eaeb39da859796ec)) * Upgrade frontend to v5.6.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.6) ([632ec09](https://github.com/opendatalab/labelU/commit/632ec09bbed951a2877f7f51da627065ed64b75c)) * Upgrade frontend to v5.6.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.7) ([b3c4f00](https://github.com/opendatalab/labelU/commit/b3c4f00c7ae3ead62e7c8de89ca27d38efaa7b0c)) * Upgrade frontend to v5.6.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.6.0-alpha.8) ([2c544f6](https://github.com/opendatalab/labelU/commit/2c544f6e000ddea9a91896c5a69c514c417261c9)) * upgrade frontend to v5.6.11 ([d512215](https://github.com/opendatalab/labelU/commit/d512215ee90e599847e2a1e30e35d4aee868d44f)) * Upgrade frontend to v5.7.0-alpha.1. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.1) ([1c00150](https://github.com/opendatalab/labelU/commit/1c001500ec4919f325e094a7bb8c98d7e99e1172)) * Upgrade frontend to v5.7.0-alpha.10. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.10) ([1a4b46e](https://github.com/opendatalab/labelU/commit/1a4b46e353f0645721d481e3574d114b2627211a)) * Upgrade frontend to v5.7.0-alpha.11. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.11) ([10ae011](https://github.com/opendatalab/labelU/commit/10ae011fc19ff1d5e8bea26c546eda7573663d06)) * Upgrade frontend to v5.7.0-alpha.12. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.12) ([8f88deb](https://github.com/opendatalab/labelU/commit/8f88debf369eece83c9666619b38a44c1fc30729)) * Upgrade frontend to v5.7.0-alpha.13. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.13) ([5221fd8](https://github.com/opendatalab/labelU/commit/5221fd8f67b0cdb5be9a4b3c94af046edaae1a1b)) * Upgrade frontend to v5.7.0-alpha.14. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.14) ([54329c5](https://github.com/opendatalab/labelU/commit/54329c548a7b4d9bf9c81de5febddbb08c2ee176)) * Upgrade frontend to v5.7.0-alpha.15. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.15) ([20ef7b3](https://github.com/opendatalab/labelU/commit/20ef7b36f36924183a5a4315beef21271389afbb)) * Upgrade frontend to v5.7.0-alpha.16. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.16) ([2660132](https://github.com/opendatalab/labelU/commit/266013274172d95592e63775391250086e969300)) * Upgrade frontend to v5.7.0-alpha.17. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.17) ([7a94abe](https://github.com/opendatalab/labelU/commit/7a94abe6e5ee97d40c04d8b51e83ae52e9ec89b4)) * Upgrade frontend to v5.7.0-alpha.18. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.18) ([61e23ee](https://github.com/opendatalab/labelU/commit/61e23ee3893c778fc2506826b5dd0bead4840ed1)) * Upgrade frontend to v5.7.0-alpha.19. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.19) ([fca41a3](https://github.com/opendatalab/labelU/commit/fca41a3a60c1849245e67927bbd3c6ea45265f48)) * Upgrade frontend to v5.7.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.2) ([ee8fcd1](https://github.com/opendatalab/labelU/commit/ee8fcd1c2b5de08725f3b883450ac000742bf7bd)) * Upgrade frontend to v5.7.0-alpha.20. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.20) ([d35101d](https://github.com/opendatalab/labelU/commit/d35101d737ddd0a7b1c9a3b6265266fa8a610807)) * Upgrade frontend to v5.7.0-alpha.21. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.21) ([01b40b8](https://github.com/opendatalab/labelU/commit/01b40b8304137f4170791bf59916f2bfeb98ddde)) * Upgrade frontend to v5.7.0-alpha.22. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.22) ([c1f9cca](https://github.com/opendatalab/labelU/commit/c1f9cca74deed0b6a5792011b43c22c2a7d928b0)) * Upgrade frontend to v5.7.0-alpha.23. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.23) ([1e88cb5](https://github.com/opendatalab/labelU/commit/1e88cb5d8e066fab6ea67d2a1ebf893b62077d0b)) * Upgrade frontend to v5.7.0-alpha.24. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.24) ([cc2cd41](https://github.com/opendatalab/labelU/commit/cc2cd41cbf6e9e745d6d45e7d7ffddfac6598c71)) * Upgrade frontend to v5.7.0-alpha.25. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.25) ([2ecf3a6](https://github.com/opendatalab/labelU/commit/2ecf3a650c5e0aed792fe863dba1b6391aa8cff3)) * Upgrade frontend to v5.7.0-alpha.26. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.26) ([a00b4fa](https://github.com/opendatalab/labelU/commit/a00b4fa21c03d4340f6ac1f92d93e3d76be9284e)) * Upgrade frontend to v5.7.0-alpha.27. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.27) ([accc1bd](https://github.com/opendatalab/labelU/commit/accc1bd72d8d695cefd9c69376391ef29a61985a)) * Upgrade frontend to v5.7.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.3) ([582eb07](https://github.com/opendatalab/labelU/commit/582eb07457bdf827f66fa66a0cbec14a5a205327)) * Upgrade frontend to v5.7.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.4) ([9b5879e](https://github.com/opendatalab/labelU/commit/9b5879ef82069895e04f259f117cdaaf1e64e959)) * Upgrade frontend to v5.7.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.5) ([70f6789](https://github.com/opendatalab/labelU/commit/70f67898e82666d6c4c6a8dbfa481db3c10a2132)) * Upgrade frontend to v5.7.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.6) ([d57db71](https://github.com/opendatalab/labelU/commit/d57db71df1e98553a05325d1ee1f6f5e75edefe4)) * Upgrade frontend to v5.7.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.7) ([b9b509b](https://github.com/opendatalab/labelU/commit/b9b509b402f521d4a586fde001f4d98dacf50fca)) * Upgrade frontend to v5.7.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.8) ([0f7253e](https://github.com/opendatalab/labelU/commit/0f7253e5be4c43562302cae9e814bde3398eee71)) * Upgrade frontend to v5.7.0-alpha.9. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.7.0-alpha.9) ([27fcf67](https://github.com/opendatalab/labelU/commit/27fcf677cc4ac85a73207c4ea771d32024719efb)) * ws message when sample is updated ([43133da](https://github.com/opendatalab/labelU/commit/43133dac5d5cd1863136b8481eab48c8ca508bed)) ### Features * add task ws api router ([#196](https://github.com/opendatalab/labelU/issues/196)) ([2f90701](https://github.com/opendatalab/labelU/commit/2f90701cc9cfa6976beba10df9532114127ca1f2)) * supports mysql database ([#187](https://github.com/opendatalab/labelU/issues/187)) ([2734618](https://github.com/opendatalab/labelU/commit/2734618d97826c7cb0bf6c77821e50880b89ac7f)) * supports task collaborators ([#189](https://github.com/opendatalab/labelU/issues/189)) ([b159c63](https://github.com/opendatalab/labelU/commit/b159c634286dd446c548e70ca0b7367d0221a760)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 72a00a11..84624249 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.2.1' +version='1.3.0-alpha.1' diff --git a/pyproject.toml b/pyproject.toml index 69978182..5874459a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.2.1' +version = '1.3.0-alpha.1' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From e8e02bde4c7ccbdf554489856f819cb0e4808566 Mon Sep 17 00:00:00 2001 From: shenguanlin Date: Thu, 13 Mar 2025 21:10:41 +0800 Subject: [PATCH 173/236] fix: remove unnecessary alembic script --- .../2eb983c9a254_add_collaborators_and_updaters.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py index d9e48c93..4a64e2b6 100644 --- a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py +++ b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py @@ -139,17 +139,6 @@ def upgrade() -> None: ) ) - # Fix missing task_attachment_ids column in task_sample table - if not column_exist_in_table("task_sample", "task_attachment_ids"): - with op.batch_alter_table('task_sample', recreate="always") as batch_op: - batch_op.add_column( - sa.Column( - "task_attachment_ids", - sa.Text(), - comment="task attachment ids", - ), - ) - def downgrade() -> None: conn = op.get_bind() inspector = sa.inspect(conn) From 281e7638a7a04be304f14bacd5fb3f03e7c8294e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 13 Mar 2025 13:22:35 +0000 Subject: [PATCH 174/236] chore(release): 1.3.0-alpha.2 [skip ci] # [1.3.0-alpha.2](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.1...v1.3.0-alpha.2) (2025-03-13) ### Bug Fixes * remove unnecessary alembic script ([e8e02bd](https://github.com/opendatalab/labelU/commit/e8e02bde4c7ccbdf554489856f819cb0e4808566)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 84624249..e306aeeb 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.1' +version='1.3.0-alpha.2' diff --git a/pyproject.toml b/pyproject.toml index 5874459a..713b7b3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.1' +version = '1.3.0-alpha.2' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 5c392c4889d7a93328f9002cc26733c23bf4e7e4 Mon Sep 17 00:00:00 2001 From: shenguanlin Date: Thu, 13 Mar 2025 21:38:50 +0800 Subject: [PATCH 175/236] fix: migration db name --- labelu/scripts/migrate_to_mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labelu/scripts/migrate_to_mysql.py b/labelu/scripts/migrate_to_mysql.py index 301bcf7a..31c78a85 100644 --- a/labelu/scripts/migrate_to_mysql.py +++ b/labelu/scripts/migrate_to_mysql.py @@ -141,7 +141,7 @@ def migrate(self) -> None: raise MigrationError(f"Migration failed: {e}") def migrate_to_mysql(): - sqlite_url = f"sqlite:///{settings.BASE_DATA_DIR}/labelu-test-1.sqlite" + sqlite_url = f"sqlite:///{settings.BASE_DATA_DIR}/labelu.sqlite" try: migrator = DatabaseMigrator(sqlite_url, settings.DATABASE_URL) From 278e408e0b69c18aab0e2c1e8d33e8c9e6f2bda2 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 13 Mar 2025 13:44:59 +0000 Subject: [PATCH 176/236] chore(release): 1.3.0-alpha.3 [skip ci] # [1.3.0-alpha.3](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.2...v1.3.0-alpha.3) (2025-03-13) ### Bug Fixes * migration db name ([5c392c4](https://github.com/opendatalab/labelU/commit/5c392c4889d7a93328f9002cc26733c23bf4e7e4)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index e306aeeb..229d9b18 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.2' +version='1.3.0-alpha.3' diff --git a/pyproject.toml b/pyproject.toml index 713b7b3e..df1f38ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.2' +version = '1.3.0-alpha.3' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 94e0761d10ced9ea9e8541427ebe6104dcd1a009 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 13 Mar 2025 22:46:28 +0800 Subject: [PATCH 177/236] update: add migration script --- ...034c7045b540_remove_task_attachment_ids.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 labelu/alembic_labelu/versions/034c7045b540_remove_task_attachment_ids.py diff --git a/labelu/alembic_labelu/versions/034c7045b540_remove_task_attachment_ids.py b/labelu/alembic_labelu/versions/034c7045b540_remove_task_attachment_ids.py new file mode 100644 index 00000000..626da52d --- /dev/null +++ b/labelu/alembic_labelu/versions/034c7045b540_remove_task_attachment_ids.py @@ -0,0 +1,24 @@ +"""remove_task_attachment_ids + +Revision ID: 034c7045b540 +Revises: 2eb983c9a254 +Create Date: 2025-03-13 22:43:30.564428 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '034c7045b540' +down_revision = '2eb983c9a254' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass From d8125be2012a8145db7ae3bf283d507c3401c05b Mon Sep 17 00:00:00 2001 From: shenguanlin Date: Thu, 13 Mar 2025 22:59:51 +0800 Subject: [PATCH 178/236] update: remove task_attachment_ids --- .../034c7045b540_remove_task_attachment_ids.py | 10 +++++++--- .../2eb983c9a254_add_collaborators_and_updaters.py | 2 -- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/labelu/alembic_labelu/versions/034c7045b540_remove_task_attachment_ids.py b/labelu/alembic_labelu/versions/034c7045b540_remove_task_attachment_ids.py index 626da52d..6ca50fe3 100644 --- a/labelu/alembic_labelu/versions/034c7045b540_remove_task_attachment_ids.py +++ b/labelu/alembic_labelu/versions/034c7045b540_remove_task_attachment_ids.py @@ -6,8 +6,8 @@ """ from alembic import op -import sqlalchemy as sa +from labelu.alembic_labelu.alembic_labelu_tools import column_exist_in_table # revision identifiers, used by Alembic. revision = '034c7045b540' @@ -17,8 +17,12 @@ def upgrade() -> None: - pass + # remove task_attachment_ids column from task sample table + # fix the unnecessary column from: https://github.com/opendatalab/labelU/blob/v1.2.1/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py#L143 + if column_exist_in_table('task_sample', 'task_attachment_ids'): + op.drop_column('task_sample', 'task_attachment_ids') def downgrade() -> None: - pass + if not column_exist_in_table('task_sample', 'task_attachment_ids'): + op.execute('ALTER TABLE task_sample ADD COLUMN task_attachment_ids TEXT COMMENT "task attachment ids"') diff --git a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py index 4a64e2b6..3dc74a71 100644 --- a/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py +++ b/labelu/alembic_labelu/versions/2eb983c9a254_add_collaborators_and_updaters.py @@ -10,8 +10,6 @@ import sqlalchemy as sa from sqlalchemy.sql import table, column -from labelu.alembic_labelu.alembic_labelu_tools import column_exist_in_table - # revision identifiers, used by Alembic. revision = '2eb983c9a254' down_revision = 'eb9c5b98168b' From fbba89ae87f4608631bd8ee8f27891a1f3fb1b0e Mon Sep 17 00:00:00 2001 From: gary Date: Thu, 13 Mar 2025 23:04:07 +0800 Subject: [PATCH 179/236] fix: release alpha From ed9ebb3d162339be81518fb99e3e5b926417d365 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 13 Mar 2025 15:09:14 +0000 Subject: [PATCH 180/236] chore(release): 1.3.0-alpha.4 [skip ci] # [1.3.0-alpha.4](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.3...v1.3.0-alpha.4) (2025-03-13) ### Bug Fixes * release alpha ([fbba89a](https://github.com/opendatalab/labelU/commit/fbba89ae87f4608631bd8ee8f27891a1f3fb1b0e)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 229d9b18..9de6c9f6 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.3' +version='1.3.0-alpha.4' diff --git a/pyproject.toml b/pyproject.toml index df1f38ad..bb6e02ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.3' +version = '1.3.0-alpha.4' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From ced5d8a1ff3aa316834c36e74d89d7544b46e5b5 Mon Sep 17 00:00:00 2001 From: gary Date: Thu, 13 Mar 2025 23:13:42 +0800 Subject: [PATCH 181/236] fix: update frontend --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 6df2b949..1a55e722 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.7.0-alpha.8/frontend.zip -alpha_version: v5.7.0-alpha.8 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.1/frontend.zip +alpha_version: v5.8.1 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.1/frontend.zip release_version: v5.8.1 From 988e72afee922a818994bb8348c2302eef1fd017 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 13 Mar 2025 15:18:49 +0000 Subject: [PATCH 182/236] chore(release): 1.3.0-alpha.5 [skip ci] # [1.3.0-alpha.5](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.4...v1.3.0-alpha.5) (2025-03-13) ### Bug Fixes * update frontend ([ced5d8a](https://github.com/opendatalab/labelU/commit/ced5d8a1ff3aa316834c36e74d89d7544b46e5b5)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 9de6c9f6..d524f1c2 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.4' +version='1.3.0-alpha.5' diff --git a/pyproject.toml b/pyproject.toml index bb6e02ac..510e00e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.4' +version = '1.3.0-alpha.5' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 51c7d367329e6d5cd37511d40d986b113b684aec Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 18 Mar 2025 13:44:55 +0800 Subject: [PATCH 183/236] fix: optimize attachment upload and download --- labelu/internal/adapter/routers/attachment.py | 101 ++++++++++++++---- .../application/service/attachment.py | 27 ++++- 2 files changed, 103 insertions(+), 25 deletions(-) diff --git a/labelu/internal/adapter/routers/attachment.py b/labelu/internal/adapter/routers/attachment.py index 75f1beb7..43ccdb27 100644 --- a/labelu/internal/adapter/routers/attachment.py +++ b/labelu/internal/adapter/routers/attachment.py @@ -1,10 +1,13 @@ +from pathlib import Path +import re +import aiofiles +from app.internal.common.error_code import ErrorCode, LabelToolException from sqlalchemy.orm import Session from fastapi import APIRouter, status, Depends, Security -from fastapi import File, Header, Response, UploadFile -from fastapi.responses import FileResponse +from fastapi import File, Header, UploadFile +from fastapi.responses import FileResponse, StreamingResponse from fastapi.security import HTTPAuthorizationCredentials import mimetypes -import os from labelu.internal.common import db from labelu.internal.common.security import security @@ -69,26 +72,84 @@ async def get_content(file_path: str, range: str = Header(None)): """ partial content """ + + try: + full_path = await service.download_attachment(file_path=file_path) + full_path = Path(full_path) + except Exception: + raise LabelToolException( + code=ErrorCode.CODE_51001_TASK_ATTACHMENT_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + + if not full_path.exists(): + raise LabelToolException( + code=ErrorCode.CODE_51001_TASK_ATTACHMENT_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) # Business logic - full_path = await service.download_attachment(file_path=file_path) - CHUNK_SIZE = 1024 * 1024 - full_size = os.path.getsize(full_path) - start, end = range.replace("bytes=", "").split("-") - start = int(start) - end = int(end) if end else min(start + CHUNK_SIZE, full_size) - 1 - media_type = mimetypes.guess_type(full_path)[0] - # TODO 不知为何到最后range阶段视频播放失效，先加上这个处理取消byte range暂时解决 - if full_size - end == 1: - return full_path + file_size = full_path.stat().st_size + media_type = mimetypes.guess_type(str(full_path))[0] or "application/octet-stream" + + if not range: + return FileResponse( + path=str(full_path), + media_type=media_type, + headers={"Accept-Ranges": "bytes"} + ) + + try: + range_match = re.match(r'bytes=(\d+)-(\d*)', range) + if not range_match: + raise ValueError("Invalid range format") + + start = int(range_match.group(1)) + end_str = range_match.group(2) + end = int(end_str) if end_str else min(start + 1024 * 1024, file_size - 1) + + # 验证范围 + if start < 0 or start >= file_size: + raise ValueError(f"Start position {start} out of bounds") + + if end >= file_size: + end = file_size - 1 + + content_length = end - start + 1 + + except ValueError: + return FileResponse( + path=str(full_path), + media_type=media_type, + headers={"Accept-Ranges": "bytes"} + ) + + + async def file_stream(): + async with aiofiles.open(str(full_path), 'rb') as f: + await f.seek(start) + remaining = content_length + chunk_size = min(64 * 1024, remaining) + + while remaining > 0: + chunk = await f.read(min(chunk_size, remaining)) + if not chunk: + break + remaining -= len(chunk) + yield chunk - with open(full_path, 'rb') as video: - video.seek(start) - data = video.read(end - start) - file_size = str(full_path.stat().st_size) - # 视频或音频标注时，需要支持快进等选定播放时间点，因此需要手动增加以下响应头部 - headers = {"Accept-Ranges": "bytes", "Content-Range": f"bytes {str(start)}-{str(end)}/{file_size}"} - return Response(data, headers=headers, media_type=media_type, status_code=206) + headers = { + "Accept-Ranges": "bytes", + "Content-Range": f"bytes {start}-{end}/{file_size}", + "Content-Length": str(content_length), + } + + return StreamingResponse( + file_stream(), + status_code=206, + media_type=media_type, + headers=headers + ) @router.delete( diff --git a/labelu/internal/application/service/attachment.py b/labelu/internal/application/service/attachment.py index 6e015b03..cc9cb24e 100644 --- a/labelu/internal/application/service/attachment.py +++ b/labelu/internal/application/service/attachment.py @@ -64,12 +64,29 @@ async def create( # create dicreatory attachment_full_base_dir.mkdir(parents=True, exist_ok=True) - - # save image + CHUNK_SIZE = 8 * 1024 * 1024 # 8MB logger.info(attachment_full_path) - async with aiofiles.open(attachment_full_path, "wb") as out_file: - content = await cmd.file.read() # async read - await out_file.write(content) # async write + try: + async with aiofiles.open(attachment_full_path, "wb") as out_file: + total_size = 0 + while True: + chunk = await cmd.file.read(CHUNK_SIZE) + if not chunk: + break + await out_file.write(chunk) + total_size += len(chunk) + logger.debug(f"{total_size} bytes written") + + logger.info(f"File saved: {attachment_full_path}, size: {total_size} bytes") + except Exception as e: + if attachment_full_path.exists(): + os.remove(attachment_full_path) + logger.error(f"Upload failed: {str(e)}") + raise LabelUException( + code=ErrorCode.CODE_51000_CREATE_ATTACHMENT_ERROR, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + message=f"Upload failed: {str(e)}" + ) # create thumbnail for image if cmd.file.content_type.startswith("image/"): From 8f91912cc72a0e182f90ee36f3d970b9982d0f87 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 18 Mar 2025 15:45:30 +0800 Subject: [PATCH 184/236] update: fix import --- labelu/internal/adapter/routers/attachment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/labelu/internal/adapter/routers/attachment.py b/labelu/internal/adapter/routers/attachment.py index 43ccdb27..e90be177 100644 --- a/labelu/internal/adapter/routers/attachment.py +++ b/labelu/internal/adapter/routers/attachment.py @@ -1,7 +1,6 @@ from pathlib import Path import re import aiofiles -from app.internal.common.error_code import ErrorCode, LabelToolException from sqlalchemy.orm import Session from fastapi import APIRouter, status, Depends, Security from fastapi import File, Header, UploadFile @@ -10,6 +9,7 @@ import mimetypes from labelu.internal.common import db +from labelu.internal.common.error_code import ErrorCode, LabelToolException from labelu.internal.common.security import security from labelu.internal.domain.models.user import User from labelu.internal.dependencies.user import get_current_user From 5e060536d9dc48138fecae765bdee8a4de9b4257 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 18 Mar 2025 16:02:30 +0800 Subject: [PATCH 185/236] update: fix import --- labelu/internal/adapter/routers/attachment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/labelu/internal/adapter/routers/attachment.py b/labelu/internal/adapter/routers/attachment.py index e90be177..6a5c7c82 100644 --- a/labelu/internal/adapter/routers/attachment.py +++ b/labelu/internal/adapter/routers/attachment.py @@ -9,7 +9,7 @@ import mimetypes from labelu.internal.common import db -from labelu.internal.common.error_code import ErrorCode, LabelToolException +from labelu.internal.common.error_code import ErrorCode, LabelUException from labelu.internal.common.security import security from labelu.internal.domain.models.user import User from labelu.internal.dependencies.user import get_current_user @@ -77,13 +77,13 @@ async def get_content(file_path: str, range: str = Header(None)): full_path = await service.download_attachment(file_path=file_path) full_path = Path(full_path) except Exception: - raise LabelToolException( + raise LabelUException( code=ErrorCode.CODE_51001_TASK_ATTACHMENT_NOT_FOUND, status_code=status.HTTP_404_NOT_FOUND, ) if not full_path.exists(): - raise LabelToolException( + raise LabelUException( code=ErrorCode.CODE_51001_TASK_ATTACHMENT_NOT_FOUND, status_code=status.HTTP_404_NOT_FOUND, ) From cdaf47bb14c9ede18895948bb5a3cc986e5c197b Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 18 Mar 2025 08:08:23 +0000 Subject: [PATCH 186/236] chore(release): 1.3.0-alpha.6 [skip ci] # [1.3.0-alpha.6](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.5...v1.3.0-alpha.6) (2025-03-18) ### Bug Fixes * add websockets dependence ([49c9b78](https://github.com/opendatalab/labelU/commit/49c9b7891acd1b7c32b8e38eee0d36f890eb6133)) * enable wss on https ([3bbdb2d](https://github.com/opendatalab/labelU/commit/3bbdb2d138d8e84e19f7fc77c674d5946d834206)) * migration db name ([4d594d4](https://github.com/opendatalab/labelU/commit/4d594d49d616a090e048e951e2a47c2def274d00)) * optimize attachment upload and download ([51c7d36](https://github.com/opendatalab/labelU/commit/51c7d367329e6d5cd37511d40d986b113b684aec)) * remove unnecessary alembic script ([5c83e14](https://github.com/opendatalab/labelU/commit/5c83e141ee13b4813b044fba030ffd6688f71233)) * sanitized filename ([3d18981](https://github.com/opendatalab/labelU/commit/3d1898129e8e000726a4471359eeb1bd3b1f214d)) * upgrade frontend to 5.8.4 ([aab395d](https://github.com/opendatalab/labelU/commit/aab395dc36e74c2a4fbfb0b9a8055d9d38ca2863)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index f13ba1cb..aba3feeb 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.2.6' +version='1.3.0-alpha.6' diff --git a/pyproject.toml b/pyproject.toml index fdca4404..366ddb4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.2.6' +version = '1.3.0-alpha.6' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 97e3bcabb6e54e02da0d4beebceff4445c8ecf74 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 3 Jul 2025 16:25:30 +0800 Subject: [PATCH 187/236] feat: add relation tool --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 3bc9542f..03b4e0cd 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip -alpha_version: v5.8.12 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.1/frontend.zip +alpha_version: v5.9.0-alpha.1 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip release_version: v5.8.12 From 2656ba815a3822b700a104d3c85adb7bd7e51e58 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 3 Jul 2025 08:31:25 +0000 Subject: [PATCH 188/236] chore(release): 1.3.0-alpha.7 [skip ci] # [1.3.0-alpha.7](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.6...v1.3.0-alpha.7) (2025-07-03) ### Bug Fixes * [#232](https://github.com/opendatalab/labelU/issues/232) [#221](https://github.com/opendatalab/labelU/issues/221) ([070611a](https://github.com/opendatalab/labelU/commit/070611a3c2a360a71f37aa0d077994c40a7f4667)) * optimize attachment upload and download ([72ca6b7](https://github.com/opendatalab/labelU/commit/72ca6b72a9aa3df3fc0679310a7e923abc170a96)) * sanitized filename with hash# ([4522016](https://github.com/opendatalab/labelU/commit/4522016443e28df0aa45675139faa3910d9daaf9)) * upgrade frontend to 5.8.10 ([e082195](https://github.com/opendatalab/labelU/commit/e0821950360353f4b3b1d75015a0dcc0d122baf4)) * upgrade frontend to 5.8.11 ([932a8ac](https://github.com/opendatalab/labelU/commit/932a8ac5624be7fd5b2f87d0ae0c9ed845ed622a)) * upgrade frontend to 5.8.8 ([4811af9](https://github.com/opendatalab/labelU/commit/4811af90ab339ab57f7ab693c83609feb31a79a1)) * upgrade frontend to 5.8.9 ([7f70c4a](https://github.com/opendatalab/labelU/commit/7f70c4a13b9fcedc9719fe26b033042ddaff0877)) * upgrade frontend to v5.8.12 ([6ef1fd7](https://github.com/opendatalab/labelU/commit/6ef1fd70e5ae9ce18973c9f124b1a22b6532b714)) ### Features * add relation tool ([97e3bca](https://github.com/opendatalab/labelU/commit/97e3bcabb6e54e02da0d4beebceff4445c8ecf74)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index e6fd5acb..57672a39 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.2.13' +version='1.3.0-alpha.7' diff --git a/pyproject.toml b/pyproject.toml index b8edffbf..9db37b39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.2.13' +version = '1.3.0-alpha.7' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 4ae1904037058153a61993861aab48047b6ba60a Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 3 Jul 2025 16:43:38 +0800 Subject: [PATCH 189/236] fix: update package contact info --- labelu/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/main.py b/labelu/main.py index 399c9237..aea0f6c6 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -80,8 +80,8 @@ terms_of_service="", contact={ "name": "labelu", - "url": "http://labelu.example.com/contact/", - "email": "labelu@example.com", + "url": "https://github.com/opendatalab/labelU/issues", + "email": "shenguanlin@pjlab.org.cn", }, license_info={ "name": "Apache 2.0", From 0311ac991be9a441d0654e32252c5c355e335b36 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 3 Jul 2025 08:49:23 +0000 Subject: [PATCH 190/236] chore(release): 1.3.0-alpha.8 [skip ci] # [1.3.0-alpha.8](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.7...v1.3.0-alpha.8) (2025-07-03) ### Bug Fixes * update package contact info ([4ae1904](https://github.com/opendatalab/labelU/commit/4ae1904037058153a61993861aab48047b6ba60a)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 57672a39..a1ee562e 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.7' +version='1.3.0-alpha.8' diff --git a/pyproject.toml b/pyproject.toml index 9db37b39..1cb131a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.7' +version = '1.3.0-alpha.8' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From d13fb61c41c8d00598c7d5a890ce500e9e1f97b9 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Thu, 3 Jul 2025 10:20:11 +0000 Subject: [PATCH 191/236] fix: Upgrade frontend to v5.9.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.2) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 03b4e0cd..91ba4efa 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.1/frontend.zip -alpha_version: v5.9.0-alpha.1 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.2/frontend.zip +alpha_version: v5.9.0-alpha.2 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip release_version: v5.8.12 From 1d7410431dff633ef44d2d94e3dc1b2441c4a7ed Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Thu, 3 Jul 2025 10:26:06 +0000 Subject: [PATCH 192/236] chore(release): 1.3.0-alpha.9 [skip ci] # [1.3.0-alpha.9](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.8...v1.3.0-alpha.9) (2025-07-03) ### Bug Fixes * Upgrade frontend to v5.9.0-alpha.2. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.2) ([d13fb61](https://github.com/opendatalab/labelU/commit/d13fb61c41c8d00598c7d5a890ce500e9e1f97b9)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index a1ee562e..fa16ab91 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.8' +version='1.3.0-alpha.9' diff --git a/pyproject.toml b/pyproject.toml index 1cb131a3..b89e9956 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.8' +version = '1.3.0-alpha.9' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 69beb62518bbfd94f4f638266b982e7e73f3fae2 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 4 Jul 2025 03:21:18 +0000 Subject: [PATCH 193/236] fix: Upgrade frontend to v5.9.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.3) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 91ba4efa..173923f9 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.2/frontend.zip -alpha_version: v5.9.0-alpha.2 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.3/frontend.zip +alpha_version: v5.9.0-alpha.3 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip release_version: v5.8.12 From a16ed71e46c1e66ece8406e9348f2eed5e8efd0b Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 4 Jul 2025 03:27:06 +0000 Subject: [PATCH 194/236] chore(release): 1.3.0-alpha.10 [skip ci] # [1.3.0-alpha.10](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.9...v1.3.0-alpha.10) (2025-07-04) ### Bug Fixes * Upgrade frontend to v5.9.0-alpha.3. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.3) ([69beb62](https://github.com/opendatalab/labelU/commit/69beb62518bbfd94f4f638266b982e7e73f3fae2)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index fa16ab91..ab3dee28 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.9' +version='1.3.0-alpha.10' diff --git a/pyproject.toml b/pyproject.toml index b89e9956..b1390c00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.9' +version = '1.3.0-alpha.10' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 178fc5a0519c3c587c76c47a932981694a2846f2 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Fri, 4 Jul 2025 08:07:24 +0000 Subject: [PATCH 195/236] fix: Upgrade frontend to v5.9.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.4) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 173923f9..bf56ff6a 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.3/frontend.zip -alpha_version: v5.9.0-alpha.3 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.4/frontend.zip +alpha_version: v5.9.0-alpha.4 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip release_version: v5.8.12 From 7e67b1201f2b6c61c064c79578bad60c9eddaa2d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 4 Jul 2025 08:13:12 +0000 Subject: [PATCH 196/236] chore(release): 1.3.0-alpha.11 [skip ci] # [1.3.0-alpha.11](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.10...v1.3.0-alpha.11) (2025-07-04) ### Bug Fixes * Upgrade frontend to v5.9.0-alpha.4. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.4) ([178fc5a](https://github.com/opendatalab/labelU/commit/178fc5a0519c3c587c76c47a932981694a2846f2)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index ab3dee28..c86c3c13 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.10' +version='1.3.0-alpha.11' diff --git a/pyproject.toml b/pyproject.toml index b1390c00..1e166c41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.10' +version = '1.3.0-alpha.11' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 0eecc4839792abb6dd1f7d7e9b209e7bf8207ea4 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 7 Jul 2025 07:57:12 +0000 Subject: [PATCH 197/236] fix: Upgrade frontend to v5.9.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.5) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index bf56ff6a..0bfbb3b6 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.4/frontend.zip -alpha_version: v5.9.0-alpha.4 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.5/frontend.zip +alpha_version: v5.9.0-alpha.5 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip release_version: v5.8.12 From 3e77359e9dfa77c8e7f6297a9f0ede14499147f7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 7 Jul 2025 08:02:55 +0000 Subject: [PATCH 198/236] chore(release): 1.3.0-alpha.12 [skip ci] # [1.3.0-alpha.12](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.11...v1.3.0-alpha.12) (2025-07-07) ### Bug Fixes * Upgrade frontend to v5.9.0-alpha.5. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.5) ([0eecc48](https://github.com/opendatalab/labelU/commit/0eecc4839792abb6dd1f7d7e9b209e7bf8207ea4)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index c86c3c13..68e441e2 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.11' +version='1.3.0-alpha.12' diff --git a/pyproject.toml b/pyproject.toml index 1e166c41..af4a3652 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.11' +version = '1.3.0-alpha.12' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 5231ea5de4bdcf7458416ff7f45d7730d4019983 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 7 Jul 2025 10:34:27 +0000 Subject: [PATCH 199/236] fix: Upgrade frontend to v5.9.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.6) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 0bfbb3b6..3b06ba90 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.5/frontend.zip -alpha_version: v5.9.0-alpha.5 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.6/frontend.zip +alpha_version: v5.9.0-alpha.6 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip release_version: v5.8.12 From 176df8903004e157e744c43f2d422a298291d44f Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 7 Jul 2025 10:40:22 +0000 Subject: [PATCH 200/236] chore(release): 1.3.0-alpha.13 [skip ci] # [1.3.0-alpha.13](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.12...v1.3.0-alpha.13) (2025-07-07) ### Bug Fixes * Upgrade frontend to v5.9.0-alpha.6. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.6) ([5231ea5](https://github.com/opendatalab/labelU/commit/5231ea5de4bdcf7458416ff7f45d7730d4019983)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 68e441e2..7e0276b7 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.12' +version='1.3.0-alpha.13' diff --git a/pyproject.toml b/pyproject.toml index af4a3652..16c98737 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.12' +version = '1.3.0-alpha.13' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 6bbaac56e925d0e7c8f27e517aef3b8b25e086ef Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Mon, 7 Jul 2025 12:00:53 +0000 Subject: [PATCH 201/236] fix: Upgrade frontend to v5.9.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.7) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 3b06ba90..ab83ecc2 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.6/frontend.zip -alpha_version: v5.9.0-alpha.6 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.7/frontend.zip +alpha_version: v5.9.0-alpha.7 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip release_version: v5.8.12 From 224e1fd70b27f1c85219fcfab3abd99db6d922f7 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Mon, 7 Jul 2025 12:06:54 +0000 Subject: [PATCH 202/236] chore(release): 1.3.0-alpha.14 [skip ci] # [1.3.0-alpha.14](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.13...v1.3.0-alpha.14) (2025-07-07) ### Bug Fixes * Upgrade frontend to v5.9.0-alpha.7. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.7) ([6bbaac5](https://github.com/opendatalab/labelU/commit/6bbaac56e925d0e7c8f27e517aef3b8b25e086ef)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 7e0276b7..e3ba42ff 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.13' +version='1.3.0-alpha.14' diff --git a/pyproject.toml b/pyproject.toml index 16c98737..afac2685 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.13' +version = '1.3.0-alpha.14' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 0182ce8b63a70c3cdbcb693e5b7803db9d269c22 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 8 Jul 2025 03:31:12 +0000 Subject: [PATCH 203/236] fix: Upgrade frontend to v5.9.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.8) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index ab83ecc2..492fb1d4 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.7/frontend.zip -alpha_version: v5.9.0-alpha.7 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.8/frontend.zip +alpha_version: v5.9.0-alpha.8 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip release_version: v5.8.12 From a2f8363bb522490ed71febd4f165eed37f6303b3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Tue, 8 Jul 2025 03:37:10 +0000 Subject: [PATCH 204/236] chore(release): 1.3.0-alpha.15 [skip ci] # [1.3.0-alpha.15](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.14...v1.3.0-alpha.15) (2025-07-08) ### Bug Fixes * Upgrade frontend to v5.9.0-alpha.8. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.8) ([0182ce8](https://github.com/opendatalab/labelU/commit/0182ce8b63a70c3cdbcb693e5b7803db9d269c22)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index e3ba42ff..d0179a69 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.14' +version='1.3.0-alpha.15' diff --git a/pyproject.toml b/pyproject.toml index afac2685..5ae70213 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.14' +version = '1.3.0-alpha.15' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From d1f5a5a25c3f9bc8326178faad29111690d2e582 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 9 Jul 2025 05:15:19 +0000 Subject: [PATCH 205/236] fix: Upgrade frontend to v5.9.0-alpha.9. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.9) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index 492fb1d4..b1eeb3d7 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.8/frontend.zip -alpha_version: v5.9.0-alpha.8 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.9/frontend.zip +alpha_version: v5.9.0-alpha.9 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip release_version: v5.8.12 From 66dc35639b71e86cdf1b72a1680ddd693c9695c3 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 9 Jul 2025 05:21:29 +0000 Subject: [PATCH 206/236] chore(release): 1.3.0-alpha.16 [skip ci] # [1.3.0-alpha.16](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.15...v1.3.0-alpha.16) (2025-07-09) ### Bug Fixes * Upgrade frontend to v5.9.0-alpha.9. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.9) ([d1f5a5a](https://github.com/opendatalab/labelU/commit/d1f5a5a25c3f9bc8326178faad29111690d2e582)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index d0179a69..53c14724 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.15' +version='1.3.0-alpha.16' diff --git a/pyproject.toml b/pyproject.toml index 5ae70213..7c5f4070 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.15' +version = '1.3.0-alpha.16' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From b6298afa25ad0c9bef587cda501c3d0b52e3da71 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 9 Jul 2025 08:35:27 +0000 Subject: [PATCH 207/236] fix: Upgrade frontend to v5.9.0-alpha.10. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.10) --- .VERSION | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.VERSION b/.VERSION index b1eeb3d7..9ea7ea3e 100644 --- a/.VERSION +++ b/.VERSION @@ -1,4 +1,4 @@ -alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.9/frontend.zip -alpha_version: v5.9.0-alpha.9 +alpha_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.9.0-alpha.10/frontend.zip +alpha_version: v5.9.0-alpha.10 release_assets_url: https://github.com/opendatalab/labelU-Kit/releases/download/v5.8.12/frontend.zip release_version: v5.8.12 From 39b8ced79fb3ebfd8afacf54262c63b9a76a2337 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 9 Jul 2025 08:41:38 +0000 Subject: [PATCH 208/236] chore(release): 1.3.0-alpha.17 [skip ci] # [1.3.0-alpha.17](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.16...v1.3.0-alpha.17) (2025-07-09) ### Bug Fixes * Upgrade frontend to v5.9.0-alpha.10. See the details in [release notes](https://github.com/opendatalab/labelU-Kit/releases/tag/v5.9.0-alpha.10) ([b6298af](https://github.com/opendatalab/labelU/commit/b6298afa25ad0c9bef587cda501c3d0b52e3da71)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 53c14724..4ec39562 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.16' +version='1.3.0-alpha.17' diff --git a/pyproject.toml b/pyproject.toml index 7c5f4070..c7ab2a6f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.16' +version = '1.3.0-alpha.17' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From 2c42b7e6f1b412ca78c33fa61ee1ca984a5d5e60 Mon Sep 17 00:00:00 2001 From: gary Date: Tue, 30 Sep 2025 17:48:23 +0800 Subject: [PATCH 209/236] fix: cli options --- labelu/main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/labelu/main.py b/labelu/main.py index aea0f6c6..4ba90527 100644 --- a/labelu/main.py +++ b/labelu/main.py @@ -1,5 +1,6 @@ from typing import Any from loguru import logger +import typer import uvicorn from typer import Typer from fastapi import FastAPI, Response @@ -80,8 +81,8 @@ terms_of_service="", contact={ "name": "labelu", - "url": "https://github.com/opendatalab/labelU/issues", - "email": "shenguanlin@pjlab.org.cn", + "url": "http://labelu.example.com/contact/", + "email": "labelu@example.com", }, license_info={ "name": "Apache 2.0", @@ -134,7 +135,9 @@ def to_mysql(): @cli.callback(invoke_without_command=True) def main( - host: str = "localhost", port: int = 8000, media_host: str = "http://localhost:8000" + host: str = typer.Option("localhost", "--host", help="Server host"), + port: int = typer.Option(8000, "--port", help="Server port"), + media_host: str = typer.Option("http://localhost:8000", "--media-host", help="Media URL") ): if port: settings.PORT = port @@ -147,4 +150,4 @@ def main( if __name__ == "__main__": - cli() \ No newline at end of file + cli() From ffcb1993ebf9d39528e6ca4a736f5c6e35e354fa Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 30 Sep 2025 19:15:21 +0800 Subject: [PATCH 210/236] chore: fix test --- labelu/tests/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index f7c5c61e..2ad4e3e1 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -45,6 +45,12 @@ def db() -> Generator: db.close() +@pytest.fixture(scope="session", autouse=True) +def ensure_test_user() -> Generator: + init_db() + yield + + @pytest.fixture(autouse=True) def run_around_tests(): # Code that will run before your test, for example: From 84956846faf53c719e4f6012ec31ca48c09266cf Mon Sep 17 00:00:00 2001 From: gary Date: Tue, 30 Sep 2025 19:16:23 +0800 Subject: [PATCH 211/236] chore: fix tests --- labelu/tests/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index f7c5c61e..2ad4e3e1 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -45,6 +45,12 @@ def db() -> Generator: db.close() +@pytest.fixture(scope="session", autouse=True) +def ensure_test_user() -> Generator: + init_db() + yield + + @pytest.fixture(autouse=True) def run_around_tests(): # Code that will run before your test, for example: From 385c0df71454e31e0f7e286be679a8b9107ae847 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 30 Sep 2025 22:54:07 +0800 Subject: [PATCH 212/236] chore: fix test --- labelu/internal/common/config.py | 4 ++++ labelu/tests/conftest.py | 11 ++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/labelu/internal/common/config.py b/labelu/internal/common/config.py index ed790af5..7d65fcf6 100644 --- a/labelu/internal/common/config.py +++ b/labelu/internal/common/config.py @@ -37,6 +37,10 @@ class Settings(BaseSettings): TOKEN_GENERATE_ALGORITHM: str = "HS256" TOKEN_ACCESS_EXPIRE_MINUTES: int = 30 TOKEN_TYPE: str = "Bearer" + TEST_USER_PASSWORD: str | None = Field( + default=None, + description="Optional override for the test user's password." + ) @property def need_migration_to_mysql(self) -> bool: diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index 2ad4e3e1..6a7e3cbb 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -14,7 +14,16 @@ from labelu.internal.adapter.persistence import crud_user TEST_USERNAME = "test@example.com" -TEST_USER_PASSWORD = "test@123" +DEFAULT_TEST_PASSWORD = "test@123" + + +def _resolve_test_password() -> str: + configured = getattr(settings, "TEST_USER_PASSWORD", None) + password = configured if configured else DEFAULT_TEST_PASSWORD + return password[:72] + + +TEST_USER_PASSWORD = _resolve_test_password() SQLALCHEMY_DATABASE_URL = "sqlite:///./test.db" From 5e5606096b62dd8bd815be6b4fd21c7cda2544ef Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 30 Sep 2025 23:20:43 +0800 Subject: [PATCH 213/236] chore: fix test --- labelu/internal/common/config.py | 8 +++++++- labelu/tests/conftest.py | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/labelu/internal/common/config.py b/labelu/internal/common/config.py index 7d65fcf6..1aa4b049 100644 --- a/labelu/internal/common/config.py +++ b/labelu/internal/common/config.py @@ -2,7 +2,7 @@ from pathlib import Path from loguru import logger -from pydantic import BaseSettings, Field +from pydantic import BaseSettings, Field, validator from labelu.internal.common.io import get_data_dir @@ -42,6 +42,12 @@ class Settings(BaseSettings): description="Optional override for the test user's password." ) + @validator("TEST_USER_PASSWORD", pre=True) + def _truncate_test_user_password(cls, value: str | None) -> str | None: + if value is None: + return None + return value[:72] + @property def need_migration_to_mysql(self) -> bool: sqlite_path = Path(self.BASE_DATA_DIR) / "labelu.sqlite" diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index 6a7e3cbb..db3a817e 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -20,7 +20,9 @@ def _resolve_test_password() -> str: configured = getattr(settings, "TEST_USER_PASSWORD", None) password = configured if configured else DEFAULT_TEST_PASSWORD - return password[:72] + raw_bytes = password.encode("utf-8") + safe_bytes = raw_bytes[:72] + return safe_bytes.decode("utf-8", errors="ignore") TEST_USER_PASSWORD = _resolve_test_password() From 8b0e96655fec4f6f7e0519e8d7cef7d5d53e42ed Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 30 Sep 2025 23:30:11 +0800 Subject: [PATCH 214/236] chore: fix test --- labelu/internal/common/security.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/labelu/internal/common/security.py b/labelu/internal/common/security.py index a73b25f0..7f1f28bc 100644 --- a/labelu/internal/common/security.py +++ b/labelu/internal/common/security.py @@ -20,10 +20,18 @@ class AccessToken(BaseModel): def verify_password(plain_password: str, hashed_password: str) -> bool: + # bcrypt has a 72 byte limit, truncate if necessary + password_bytes = plain_password.encode("utf-8") + if len(password_bytes) > 72: + plain_password = password_bytes[:72].decode("utf-8", errors="ignore") return pwd_context.verify(plain_password, hashed_password) def get_password_hash(password: str) -> str: + # bcrypt has a 72 byte limit, truncate if necessary + password_bytes = password.encode("utf-8") + if len(password_bytes) > 72: + password = password_bytes[:72].decode("utf-8", errors="ignore") return pwd_context.hash(password) From d4d656f8b0ee4d98f4e104c3ecc3656603426820 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 30 Sep 2025 23:37:11 +0800 Subject: [PATCH 215/236] chore: fix test --- labelu/tests/conftest.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index db3a817e..d9902a26 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -56,12 +56,6 @@ def db() -> Generator: db.close() -@pytest.fixture(scope="session", autouse=True) -def ensure_test_user() -> Generator: - init_db() - yield - - @pytest.fixture(autouse=True) def run_around_tests(): # Code that will run before your test, for example: From 5f20eba6490da711c393c7994b9462536ace9f36 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Tue, 30 Sep 2025 23:47:03 +0800 Subject: [PATCH 216/236] chore: fix test --- labelu/tests/conftest.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index d9902a26..3a62694f 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -56,11 +56,17 @@ def db() -> Generator: db.close() +@pytest.fixture(scope="session", autouse=True) +def ensure_test_user() -> Generator: + # Initialize test user once per test session + init_db() + yield + + @pytest.fixture(autouse=True) def run_around_tests(): # Code that will run before your test, for example: print("start up") - init_db() # A test function will be run at this point yield # Code that will run after your test, for example: @@ -93,7 +99,16 @@ def get_testuser_token_headers(client: TestClient) -> Dict[str, str]: "password": TEST_USER_PASSWORD, } r = client.post(f"{settings.API_V1_STR}/users/login", json=data) - token = r.json()["data"]["token"] + response_json = r.json() + + # Add debugging info for CI failures + if "data" not in response_json: + print(f"Login failed. Response: {response_json}") + print(f"Status code: {r.status_code}") + print(f"Password length (bytes): {len(TEST_USER_PASSWORD.encode('utf-8'))}") + raise ValueError(f"Login failed with response: {response_json}") + + token = response_json["data"]["token"] headers = {"Authorization": f"{token}"} return headers From 14510f55ebd8b092eb569ff3115414ceae32c148 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 1 Oct 2025 00:01:45 +0800 Subject: [PATCH 217/236] chore: fix test --- labelu/internal/common/security.py | 32 +++++++++++++----- labelu/tests/conftest.py | 14 ++++++-- test_bcrypt_fix.py | 53 ++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 11 deletions(-) create mode 100644 test_bcrypt_fix.py diff --git a/labelu/internal/common/security.py b/labelu/internal/common/security.py index 7f1f28bc..30923e7f 100644 --- a/labelu/internal/common/security.py +++ b/labelu/internal/common/security.py @@ -13,6 +13,28 @@ pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") +def _truncate_password_for_bcrypt(password: str) -> str: + """ + Truncate password to 72 bytes for bcrypt compatibility. + + bcrypt has a 72-byte limit. This function ensures the password + doesn't exceed this limit while maintaining valid UTF-8 encoding. + """ + password_bytes = password.encode("utf-8") + if len(password_bytes) <= 72: + return password + + # Truncate to 72 bytes and decode, handling incomplete UTF-8 sequences + truncated = password_bytes[:72].decode("utf-8", errors="ignore") + + # Re-encode to verify it's still within limit after decode + # (in case the decode added bytes or there were edge cases) + while truncated and len(truncated.encode("utf-8")) > 72: + truncated = truncated[:-1] + + return truncated + + class AccessToken(BaseModel): id: int username: str @@ -20,18 +42,12 @@ class AccessToken(BaseModel): def verify_password(plain_password: str, hashed_password: str) -> bool: - # bcrypt has a 72 byte limit, truncate if necessary - password_bytes = plain_password.encode("utf-8") - if len(password_bytes) > 72: - plain_password = password_bytes[:72].decode("utf-8", errors="ignore") + plain_password = _truncate_password_for_bcrypt(plain_password) return pwd_context.verify(plain_password, hashed_password) def get_password_hash(password: str) -> str: - # bcrypt has a 72 byte limit, truncate if necessary - password_bytes = password.encode("utf-8") - if len(password_bytes) > 72: - password = password_bytes[:72].decode("utf-8", errors="ignore") + password = _truncate_password_for_bcrypt(password) return pwd_context.hash(password) diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index 3a62694f..c6d343b5 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -20,9 +20,17 @@ def _resolve_test_password() -> str: configured = getattr(settings, "TEST_USER_PASSWORD", None) password = configured if configured else DEFAULT_TEST_PASSWORD - raw_bytes = password.encode("utf-8") - safe_bytes = raw_bytes[:72] - return safe_bytes.decode("utf-8", errors="ignore") + + # Ensure password is not longer than 72 bytes for bcrypt + password_bytes = password.encode("utf-8") + if len(password_bytes) > 72: + # Truncate to 72 bytes and decode, ensuring valid UTF-8 + truncated = password_bytes[:72].decode("utf-8", errors="ignore") + # Re-encode to verify it's still within limit after decode + while len(truncated.encode("utf-8")) > 72: + truncated = truncated[:-1] + return truncated + return password TEST_USER_PASSWORD = _resolve_test_password() diff --git a/test_bcrypt_fix.py b/test_bcrypt_fix.py new file mode 100644 index 00000000..fa95221d --- /dev/null +++ b/test_bcrypt_fix.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +""" +测试 bcrypt 密码截断修复 +""" + +from labelu.internal.common.security import get_password_hash, verify_password + +# 测试用例 1: 正常长度密码 +normal_password = "test@123" +print(f"测试 1: 正常密码 '{normal_password}'") +print(f" 长度: {len(normal_password.encode('utf-8'))} 字节") +hash1 = get_password_hash(normal_password) +print(f" 哈希成功: {hash1[:20]}...") +print(f" 验证成功: {verify_password(normal_password, hash1)}") +print() + +# 测试用例 2: 恰好 72 字节 +password_72 = "a" * 72 +print(f"测试 2: 72 字节密码") +print(f" 长度: {len(password_72.encode('utf-8'))} 字节") +hash2 = get_password_hash(password_72) +print(f" 哈希成功: {hash2[:20]}...") +print(f" 验证成功: {verify_password(password_72, hash2)}") +print() + +# 测试用例 3: 超过 72 字节（ASCII） +long_password = "0" * 200 +print(f"测试 3: 200 字节密码（全 ASCII）") +print(f" 原始长度: {len(long_password.encode('utf-8'))} 字节") +hash3 = get_password_hash(long_password) +print(f" 哈希成功: {hash3[:20]}...") +print(f" 验证成功: {verify_password(long_password, hash3)}") +print() + +# 测试用例 4: 超过 72 字节（包含多字节 UTF-8 字符） +long_password_utf8 = "测试密码" * 30 # 每个中文字符 3 字节 +print(f"测试 4: 超长密码（包含 UTF-8 字符）") +print(f" 原始长度: {len(long_password_utf8.encode('utf-8'))} 字节") +hash4 = get_password_hash(long_password_utf8) +print(f" 哈希成功: {hash4[:20]}...") +print(f" 验证成功: {verify_password(long_password_utf8, hash4)}") +print() + +# 测试用例 5: 模拟 GitHub Actions 环境中的超长密码 +github_actions_password = "0123456789" * 25 # 250 字节 +print(f"测试 5: GitHub Actions 超长密码") +print(f" 原始长度: {len(github_actions_password.encode('utf-8'))} 字节") +hash5 = get_password_hash(github_actions_password) +print(f" 哈希成功: {hash5[:20]}...") +print(f" 验证成功: {verify_password(github_actions_password, hash5)}") +print() + +print("✅ 所有测试通过！bcrypt 密码截断修复成功！") From fee13433bfd2bbd4270431aa5bbf26d28f6842a1 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 1 Oct 2025 11:39:34 +0800 Subject: [PATCH 218/236] chore: fix test --- labelu/internal/common/security.py | 10 +++--- test_bcrypt_fix.py | 53 ------------------------------ 2 files changed, 6 insertions(+), 57 deletions(-) delete mode 100644 test_bcrypt_fix.py diff --git a/labelu/internal/common/security.py b/labelu/internal/common/security.py index 30923e7f..59afb7e3 100644 --- a/labelu/internal/common/security.py +++ b/labelu/internal/common/security.py @@ -42,13 +42,15 @@ class AccessToken(BaseModel): def verify_password(plain_password: str, hashed_password: str) -> bool: - plain_password = _truncate_password_for_bcrypt(plain_password) - return pwd_context.verify(plain_password, hashed_password) + safe_password = plain_password[:72] + plain_password = _truncate_password_for_bcrypt(safe_password) + return pwd_context.verify(safe_password, hashed_password) def get_password_hash(password: str) -> str: - password = _truncate_password_for_bcrypt(password) - return pwd_context.hash(password) + safe_password = password[:72] + password = _truncate_password_for_bcrypt(safe_password) + return pwd_context.hash(safe_password) # create access token for user login diff --git a/test_bcrypt_fix.py b/test_bcrypt_fix.py deleted file mode 100644 index fa95221d..00000000 --- a/test_bcrypt_fix.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -""" -测试 bcrypt 密码截断修复 -""" - -from labelu.internal.common.security import get_password_hash, verify_password - -# 测试用例 1: 正常长度密码 -normal_password = "test@123" -print(f"测试 1: 正常密码 '{normal_password}'") -print(f" 长度: {len(normal_password.encode('utf-8'))} 字节") -hash1 = get_password_hash(normal_password) -print(f" 哈希成功: {hash1[:20]}...") -print(f" 验证成功: {verify_password(normal_password, hash1)}") -print() - -# 测试用例 2: 恰好 72 字节 -password_72 = "a" * 72 -print(f"测试 2: 72 字节密码") -print(f" 长度: {len(password_72.encode('utf-8'))} 字节") -hash2 = get_password_hash(password_72) -print(f" 哈希成功: {hash2[:20]}...") -print(f" 验证成功: {verify_password(password_72, hash2)}") -print() - -# 测试用例 3: 超过 72 字节（ASCII） -long_password = "0" * 200 -print(f"测试 3: 200 字节密码（全 ASCII）") -print(f" 原始长度: {len(long_password.encode('utf-8'))} 字节") -hash3 = get_password_hash(long_password) -print(f" 哈希成功: {hash3[:20]}...") -print(f" 验证成功: {verify_password(long_password, hash3)}") -print() - -# 测试用例 4: 超过 72 字节（包含多字节 UTF-8 字符） -long_password_utf8 = "测试密码" * 30 # 每个中文字符 3 字节 -print(f"测试 4: 超长密码（包含 UTF-8 字符）") -print(f" 原始长度: {len(long_password_utf8.encode('utf-8'))} 字节") -hash4 = get_password_hash(long_password_utf8) -print(f" 哈希成功: {hash4[:20]}...") -print(f" 验证成功: {verify_password(long_password_utf8, hash4)}") -print() - -# 测试用例 5: 模拟 GitHub Actions 环境中的超长密码 -github_actions_password = "0123456789" * 25 # 250 字节 -print(f"测试 5: GitHub Actions 超长密码") -print(f" 原始长度: {len(github_actions_password.encode('utf-8'))} 字节") -hash5 = get_password_hash(github_actions_password) -print(f" 哈希成功: {hash5[:20]}...") -print(f" 验证成功: {verify_password(github_actions_password, hash5)}") -print() - -print("✅ 所有测试通过！bcrypt 密码截断修复成功！") From 93523b545cc9b03e3a01f799270cab6ba1fa5aa2 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 1 Oct 2025 11:52:40 +0800 Subject: [PATCH 219/236] chore: fix test --- labelu/internal/common/security.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/labelu/internal/common/security.py b/labelu/internal/common/security.py index 59afb7e3..f55eccd6 100644 --- a/labelu/internal/common/security.py +++ b/labelu/internal/common/security.py @@ -42,14 +42,12 @@ class AccessToken(BaseModel): def verify_password(plain_password: str, hashed_password: str) -> bool: - safe_password = plain_password[:72] - plain_password = _truncate_password_for_bcrypt(safe_password) + safe_password = _truncate_password_for_bcrypt(plain_password[:72]) return pwd_context.verify(safe_password, hashed_password) def get_password_hash(password: str) -> str: - safe_password = password[:72] - password = _truncate_password_for_bcrypt(safe_password) + safe_password = _truncate_password_for_bcrypt(password[:72]) return pwd_context.hash(safe_password) From 1548312be8a50a1280768bcae1bac57bc45d466f Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 1 Oct 2025 12:04:56 +0800 Subject: [PATCH 220/236] chore: fix test --- labelu/internal/common/config.py | 10 ------- labelu/internal/common/security.py | 29 ++----------------- labelu/tests/conftest.py | 4 ++- .../adapter/persistence/test_crud_user.py | 1 - 4 files changed, 5 insertions(+), 39 deletions(-) diff --git a/labelu/internal/common/config.py b/labelu/internal/common/config.py index 1aa4b049..757c960f 100644 --- a/labelu/internal/common/config.py +++ b/labelu/internal/common/config.py @@ -37,16 +37,6 @@ class Settings(BaseSettings): TOKEN_GENERATE_ALGORITHM: str = "HS256" TOKEN_ACCESS_EXPIRE_MINUTES: int = 30 TOKEN_TYPE: str = "Bearer" - TEST_USER_PASSWORD: str | None = Field( - default=None, - description="Optional override for the test user's password." - ) - - @validator("TEST_USER_PASSWORD", pre=True) - def _truncate_test_user_password(cls, value: str | None) -> str | None: - if value is None: - return None - return value[:72] @property def need_migration_to_mysql(self) -> bool: diff --git a/labelu/internal/common/security.py b/labelu/internal/common/security.py index f55eccd6..b128b197 100644 --- a/labelu/internal/common/security.py +++ b/labelu/internal/common/security.py @@ -12,29 +12,6 @@ pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") - -def _truncate_password_for_bcrypt(password: str) -> str: - """ - Truncate password to 72 bytes for bcrypt compatibility. - - bcrypt has a 72-byte limit. This function ensures the password - doesn't exceed this limit while maintaining valid UTF-8 encoding. - """ - password_bytes = password.encode("utf-8") - if len(password_bytes) <= 72: - return password - - # Truncate to 72 bytes and decode, handling incomplete UTF-8 sequences - truncated = password_bytes[:72].decode("utf-8", errors="ignore") - - # Re-encode to verify it's still within limit after decode - # (in case the decode added bytes or there were edge cases) - while truncated and len(truncated.encode("utf-8")) > 72: - truncated = truncated[:-1] - - return truncated - - class AccessToken(BaseModel): id: int username: str @@ -42,13 +19,11 @@ class AccessToken(BaseModel): def verify_password(plain_password: str, hashed_password: str) -> bool: - safe_password = _truncate_password_for_bcrypt(plain_password[:72]) - return pwd_context.verify(safe_password, hashed_password) + return pwd_context.verify(plain_password, hashed_password) def get_password_hash(password: str) -> str: - safe_password = _truncate_password_for_bcrypt(password[:72]) - return pwd_context.hash(safe_password) + return pwd_context.hash(password) # create access token for user login diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index c6d343b5..19b98704 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -127,9 +127,11 @@ def init_db() -> None: with db.begin(): user = crud_user.get_user_by_username(db, username=TEST_USERNAME) if not user: + print("Creating test user:", TEST_USERNAME) + print("Test user password length (bytes):", len(TEST_USER_PASSWORD.encode('utf-8'))) user_in = User( username=TEST_USERNAME, - hashed_password=get_password_hash(TEST_USER_PASSWORD), + hashed_password=get_password_hash(TEST_USER_PASSWORD[:72]), ) user = crud_user.create(db=db, user=user_in) finally: diff --git a/labelu/tests/internal/adapter/persistence/test_crud_user.py b/labelu/tests/internal/adapter/persistence/test_crud_user.py index 62b55eca..45d78d3b 100644 --- a/labelu/tests/internal/adapter/persistence/test_crud_user.py +++ b/labelu/tests/internal/adapter/persistence/test_crud_user.py @@ -3,7 +3,6 @@ from labelu.internal.domain.models.user import User from labelu.internal.adapter.persistence import crud_user -from labelu.internal.common.security import verify_password from labelu.internal.common.security import get_password_hash from labelu.tests.utils.utils import random_username From 142b14df73338a60579c5c236e5afaba99d54051 Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 1 Oct 2025 12:28:28 +0800 Subject: [PATCH 221/236] chore: fix test --- labelu/internal/common/security.py | 22 +++++++++++++++++++--- labelu/tests/conftest.py | 17 ++--------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/labelu/internal/common/security.py b/labelu/internal/common/security.py index b128b197..a92cff20 100644 --- a/labelu/internal/common/security.py +++ b/labelu/internal/common/security.py @@ -8,9 +8,25 @@ from labelu.internal.common.config import settings +BCRYPT_MAX_BYTES = 72 + security = HTTPBearer() -pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") +pwd_context = CryptContext( + schemes=["bcrypt"], + deprecated="auto", + bcrypt__truncate_error=False, +) + + +def _normalize_password(password: str) -> str: + """Clamp password to the maximum length bcrypt accepts (72 bytes).""" + password_bytes = password.encode("utf-8") + if len(password_bytes) <= BCRYPT_MAX_BYTES: + return password + + truncated_bytes = password_bytes[:BCRYPT_MAX_BYTES] + return truncated_bytes.decode("utf-8", errors="ignore") class AccessToken(BaseModel): id: int @@ -19,11 +35,11 @@ class AccessToken(BaseModel): def verify_password(plain_password: str, hashed_password: str) -> bool: - return pwd_context.verify(plain_password, hashed_password) + return pwd_context.verify(_normalize_password(plain_password), hashed_password) def get_password_hash(password: str) -> str: - return pwd_context.hash(password) + return pwd_context.hash(_normalize_password(password)) # create access token for user login diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index 19b98704..50130ecd 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -19,18 +19,7 @@ def _resolve_test_password() -> str: configured = getattr(settings, "TEST_USER_PASSWORD", None) - password = configured if configured else DEFAULT_TEST_PASSWORD - - # Ensure password is not longer than 72 bytes for bcrypt - password_bytes = password.encode("utf-8") - if len(password_bytes) > 72: - # Truncate to 72 bytes and decode, ensuring valid UTF-8 - truncated = password_bytes[:72].decode("utf-8", errors="ignore") - # Re-encode to verify it's still within limit after decode - while len(truncated.encode("utf-8")) > 72: - truncated = truncated[:-1] - return truncated - return password + return configured if configured else DEFAULT_TEST_PASSWORD TEST_USER_PASSWORD = _resolve_test_password() @@ -127,11 +116,9 @@ def init_db() -> None: with db.begin(): user = crud_user.get_user_by_username(db, username=TEST_USERNAME) if not user: - print("Creating test user:", TEST_USERNAME) - print("Test user password length (bytes):", len(TEST_USER_PASSWORD.encode('utf-8'))) user_in = User( username=TEST_USERNAME, - hashed_password=get_password_hash(TEST_USER_PASSWORD[:72]), + hashed_password=get_password_hash(TEST_USER_PASSWORD), ) user = crud_user.create(db=db, user=user_in) finally: From b772c5f2e8f861ac351b975e460249e67d61469e Mon Sep 17 00:00:00 2001 From: gary-Shen Date: Wed, 1 Oct 2025 12:46:00 +0800 Subject: [PATCH 222/236] chore: fix test --- labelu/internal/common/security.py | 16 ++-------------- labelu/tests/conftest.py | 10 +--------- pyproject.toml | 3 ++- 3 files changed, 5 insertions(+), 24 deletions(-) diff --git a/labelu/internal/common/security.py b/labelu/internal/common/security.py index a92cff20..d678c5fa 100644 --- a/labelu/internal/common/security.py +++ b/labelu/internal/common/security.py @@ -8,8 +8,6 @@ from labelu.internal.common.config import settings -BCRYPT_MAX_BYTES = 72 - security = HTTPBearer() pwd_context = CryptContext( @@ -18,16 +16,6 @@ bcrypt__truncate_error=False, ) - -def _normalize_password(password: str) -> str: - """Clamp password to the maximum length bcrypt accepts (72 bytes).""" - password_bytes = password.encode("utf-8") - if len(password_bytes) <= BCRYPT_MAX_BYTES: - return password - - truncated_bytes = password_bytes[:BCRYPT_MAX_BYTES] - return truncated_bytes.decode("utf-8", errors="ignore") - class AccessToken(BaseModel): id: int username: str @@ -35,11 +23,11 @@ class AccessToken(BaseModel): def verify_password(plain_password: str, hashed_password: str) -> bool: - return pwd_context.verify(_normalize_password(plain_password), hashed_password) + return pwd_context.verify(plain_password, hashed_password) def get_password_hash(password: str) -> str: - return pwd_context.hash(_normalize_password(password)) + return pwd_context.hash(password) # create access token for user login diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index 50130ecd..b82524af 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -14,15 +14,7 @@ from labelu.internal.adapter.persistence import crud_user TEST_USERNAME = "test@example.com" -DEFAULT_TEST_PASSWORD = "test@123" - - -def _resolve_test_password() -> str: - configured = getattr(settings, "TEST_USER_PASSWORD", None) - return configured if configured else DEFAULT_TEST_PASSWORD - - -TEST_USER_PASSWORD = _resolve_test_password() +TEST_USER_PASSWORD = "test@123" SQLALCHEMY_DATABASE_URL = "sqlite:///./test.db" diff --git a/pyproject.toml b/pyproject.toml index c7ab2a6f..d713d258 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,10 +27,11 @@ appdirs = "^1.4.4" mysqlclient = { version = "^2.1.1", optional = true } aiofiles = "^22.1.0" pillow = "^9.3.0" -alembic = "^1.9.4" +alembic = "1.15.2" httpx = "^0.27.0" tfrecord = "^1.14.5" websockets = "^10.0.0" +bcrypt = "4.3.0" [tool.poetry.extras] mysql = ["mysqlclient"] From e36a9c6b0861c86064fdc0a2bffac9620bd321a0 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 1 Oct 2025 04:52:05 +0000 Subject: [PATCH 223/236] chore(release): 1.3.0-alpha.18 [skip ci] # [1.3.0-alpha.18](https://github.com/opendatalab/labelU/compare/v1.3.0-alpha.17...v1.3.0-alpha.18) (2025-10-01) ### Bug Fixes * cli options ([2c42b7e](https://github.com/opendatalab/labelU/commit/2c42b7e6f1b412ca78c33fa61ee1ca984a5d5e60)) --- labelu/version.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/labelu/version.py b/labelu/version.py index 4ec39562..817009f5 100644 --- a/labelu/version.py +++ b/labelu/version.py @@ -1 +1 @@ -version='1.3.0-alpha.17' +version='1.3.0-alpha.18' diff --git a/pyproject.toml b/pyproject.toml index d713d258..c548e7b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "labelu" -version = '1.3.0-alpha.17' +version = '1.3.0-alpha.18' description = "" license = "Apache-2.0" authors = ["shenguanlin "] From bed89a0cb2f7b8f3ee4c65f56d530cfe5f1daf08 Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 11:39:57 +0800 Subject: [PATCH 224/236] fix(security): prevent path traversal in LocalStorageBackend (unauthenticated arbitrary file read) _resolve only stripped leading slashes from the key; it neither handled ".." nor verified the resolved path stayed within MEDIA_ROOT. Because the /api/v1/tasks/attachment/{path} and /partial/{path} download endpoints are unauthenticated, an attacker could traverse with "../" to read any process-readable file (e.g. labelu.sqlite, /etc/passwd). _resolve now resolve()s both MEDIA_ROOT and the target and enforces containment, raising 404 on escape. This covers read/write/delete/exists. Co-Authored-By: Claude Fable 5 --- labelu/internal/common/storage.py | 14 ++++++++- labelu/tests/internal/common/test_storage.py | 32 ++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 labelu/tests/internal/common/test_storage.py diff --git a/labelu/internal/common/storage.py b/labelu/internal/common/storage.py index 4f1f0370..c6a25e04 100644 --- a/labelu/internal/common/storage.py +++ b/labelu/internal/common/storage.py @@ -6,9 +6,12 @@ from pathlib import Path from typing import Optional +from fastapi import status +from loguru import logger from PIL import Image from labelu.internal.common.config import settings +from labelu.internal.common.error_code import ErrorCode, LabelUException class StorageBackend(ABC): @@ -56,7 +59,16 @@ def backend_name(self) -> str: return "local" def _resolve(self, key: str) -> Path: - return settings.MEDIA_ROOT.joinpath(key.lstrip("/")) + root = settings.MEDIA_ROOT.resolve() + target = root.joinpath(key.lstrip("/")).resolve() + # Reject any key that escapes the media root via "../" traversal. + if target != root and root not in target.parents: + logger.error("rejected path traversal outside media root: {}", key) + raise LabelUException( + code=ErrorCode.CODE_51001_TASK_ATTACHMENT_NOT_FOUND, + status_code=status.HTTP_404_NOT_FOUND, + ) + return target def save_file(self, local_path: Path, key: str, content_type: Optional[str] = None) -> None: target_path = self._resolve(key) diff --git a/labelu/tests/internal/common/test_storage.py b/labelu/tests/internal/common/test_storage.py new file mode 100644 index 00000000..a8cf653c --- /dev/null +++ b/labelu/tests/internal/common/test_storage.py @@ -0,0 +1,32 @@ +import pytest + +from labelu.internal.common.config import settings +from labelu.internal.common.storage import LocalStorageBackend +from labelu.internal.common.error_code import LabelUException + + +class TestLocalStorageResolveTraversal: + """Regression tests for path traversal in LocalStorageBackend._resolve.""" + + def test_resolve_rejects_parent_traversal(self): + backend = LocalStorageBackend() + with pytest.raises(LabelUException): + backend._resolve("../../../../../../../../etc/passwd") + + def test_resolve_rejects_nested_traversal(self): + backend = LocalStorageBackend() + with pytest.raises(LabelUException): + backend._resolve("upload/1/../../../../../../../../etc/passwd") + + def test_resolve_rejects_absolute_escape(self): + # lstrip("/") turns "/etc/passwd" into "etc/passwd" (stays inside root), + # but a backslash-free absolute traversal must still be contained. + backend = LocalStorageBackend() + resolved = backend._resolve("/etc/passwd") + assert str(resolved).startswith(str(settings.MEDIA_ROOT.resolve())) + + def test_resolve_allows_normal_key(self): + backend = LocalStorageBackend() + resolved = backend._resolve("upload/1/test.png") + assert str(resolved).startswith(str(settings.MEDIA_ROOT.resolve())) + assert resolved.name == "test.png" From ecac9f5526a41c3508b024d4c77a2fd4239aa0e7 Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 11:46:15 +0800 Subject: [PATCH 225/236] fix(security): sanitize the directory part of uploaded filenames (arbitrary file write) Attachment upload only sanitized the last path segment; the directory part "/".join(path_filename[:-1]) was kept verbatim and could contain "..", letting save_file -> _resolve -> shutil.move create files anywhere the process can write (potential RCE). Each directory segment is now sanitized the same way and "", "." and ".." segments are dropped. Legitimate subdirectories still work while traversal is neutralized; uploads always land under upload//. Co-Authored-By: Claude Fable 5 --- .../application/service/attachment.py | 14 ++++++- .../adapter/routers/test_attachment.py | 42 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/labelu/internal/application/service/attachment.py b/labelu/internal/application/service/attachment.py index 52a42612..d25cff5a 100644 --- a/labelu/internal/application/service/attachment.py +++ b/labelu/internal/application/service/attachment.py @@ -82,11 +82,21 @@ async def create( # file relative path path_filename = cmd.file.filename.split("/") - # filename = str(uuid.uuid4())[0:8] + "-" + path_filename[-1] NOTE: If you want keep filename safe, you can use uuid as filename + # filename = str(uuid.uuid4())[0:8] + "-" + path_filename[-1] NOTE: If you want keep filename safe, you can use uuid as filename filename = path_filename[-1] sanitized = re.sub(r'%', '_pct_', filename) sanitized = re.sub(r'[\\/*?:"<>|#]', '_', sanitized) - path = "/".join(path_filename[:-1]) + # Sanitize the directory portion of the filename too: drop traversal + # components ("..", ".", empty) and apply the same character sanitization, + # so a crafted filename can never escape the task's upload directory. + safe_parts = [] + for part in path_filename[:-1]: + part = re.sub(r'%', '_pct_', part) + part = re.sub(r'[\\/*?:"<>|#]', '_', part) + if part in ("", ".", ".."): + continue + safe_parts.append(part) + path = "/".join(safe_parts) attachment_relative_base_dir = Path(settings.UPLOAD_DIR).joinpath( str(task_id), path ) diff --git a/labelu/tests/internal/adapter/routers/test_attachment.py b/labelu/tests/internal/adapter/routers/test_attachment.py index 1e49d1df..b5b2e454 100644 --- a/labelu/tests/internal/adapter/routers/test_attachment.py +++ b/labelu/tests/internal/adapter/routers/test_attachment.py @@ -406,6 +406,48 @@ def test_partial_content_file_not_found( assert r.status_code == 404 assert r.json()["err_code"] == 30003 + def test_upload_rejects_path_traversal_filename( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + # prepare data + data = { + "name": "traversal test", + "description": "test", + "tips": "test", + } + task = client.post( + f"{settings.API_V1_STR}/tasks", headers=testuser_token_headers, json=data + ) + task_id = task.json()["data"]["id"] + + # clean the location where a sanitized upload would actually land + empty_task_upload(task_id, "tmp/pwned_traversal_marker.txt") + + marker = "pwned_traversal_marker.txt" + escape_target = Path("/tmp").joinpath(marker) + if escape_target.exists(): + escape_target.unlink() + + # craft a filename whose directory part tries to escape the upload dir + malicious_name = "../../../../../../../../tmp/" + marker + with Path("labelu/tests/data/test.txt").open(mode="rb") as f: + res = client.post( + f"{settings.API_V1_STR}/tasks/{task_id}/attachments", + headers=testuser_token_headers, + files={"file": (malicious_name, f, "text/plain")}, + ) + + # the crafted file must NOT be written outside the media root + assert not escape_target.exists() + # the upload is handled safely and stored inside the task upload dir + assert res.status_code == 201 + assert f"upload/{task_id}/" in res.json()["data"]["url"] + # the stored key stays within the media root + parts = res.json()["data"]["url"].split("/attachment/")[-1] + assert Path(settings.MEDIA_ROOT).joinpath(parts).resolve().is_relative_to( + Path(settings.MEDIA_ROOT).resolve() + ) + def test_upload_file_with_hash_symbol( self, client: TestClient, testuser_token_headers: dict, db: Session ) -> None: From 6e8b12d7b1e18e74977005717d850168f995076e Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 11:50:15 +0800 Subject: [PATCH 226/236] fix(security): fix SQL injection in the sample-listing sort parameter (CWE-89) crud_sample.list_by interpolated the sort column/direction into an ORDER BY clause via a raw text() f-string, and the route pattern used an unanchored re.search that could be bypassed with "valid_col:dir,", letting an authenticated user blind-extract arbitrary data. Primary fix: the crud layer now maps allowed column names to real ORM columns, accepts only asc/desc, and skips anything else, removing the text() interpolation entirely. Defense in depth: the route pattern is anchored to ^(col:dir)(,col:dir)*$, rejecting injection at validation and closing the substring bypass while still allowing comma-separated multi-column sorts. Co-Authored-By: Claude Fable 5 --- .../adapter/persistence/crud_sample.py | 29 +++++-- labelu/internal/adapter/routers/sample.py | 4 +- .../adapter/persistence/test_crud_sample.py | 85 +++++++++++++++++++ 3 files changed, 110 insertions(+), 8 deletions(-) create mode 100644 labelu/tests/internal/adapter/persistence/test_crud_sample.py diff --git a/labelu/internal/adapter/persistence/crud_sample.py b/labelu/internal/adapter/persistence/crud_sample.py index 5a38443b..f070e157 100644 --- a/labelu/internal/adapter/persistence/crud_sample.py +++ b/labelu/internal/adapter/persistence/crud_sample.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Union -from sqlalchemy import case, func, text +from sqlalchemy import case, func from sqlalchemy.orm import Session from fastapi.encoders import jsonable_encoder @@ -39,17 +39,32 @@ def list_by( whens = {state.value: index for index, state in enumerate(SampleState)} sort_logic = case(whens, value=TaskSample.state).label(TaskSample.state.key) + # Allowlist of sortable columns mapped to real ORM columns. Only these + # identifiers may reach ORDER BY; anything else is rejected so request + # input can never be interpolated into raw SQL. + sortable_columns = { + "inner_id": TaskSample.inner_id, + "updated_at": TaskSample.updated_at, + "annotated_count": TaskSample.annotated_count, + } + if sorting: sort_strings = sorting.split(",") for item in sort_strings: sort_key = item.split(":") - if sort_key[0] == TaskSample.state.key: - if sort_key[1] == "asc": - query = query.order_by(sort_logic.asc()) - else: - query = query.order_by(sort_logic.desc()) + column = sort_key[0] + direction = sort_key[1] if len(sort_key) > 1 else "asc" + if direction not in ("asc", "desc"): + continue + if column == TaskSample.state.key: + order_target = sort_logic + elif column in sortable_columns: + order_target = sortable_columns[column] else: - query = query.order_by(text(f"{sort_key[0]} {sort_key[1]}")) + continue + query = query.order_by( + order_target.asc() if direction == "asc" else order_target.desc() + ) # default order by id, before need select last items if before: diff --git a/labelu/internal/adapter/routers/sample.py b/labelu/internal/adapter/routers/sample.py index ddd77483..02cddbe1 100644 --- a/labelu/internal/adapter/routers/sample.py +++ b/labelu/internal/adapter/routers/sample.py @@ -84,7 +84,9 @@ async def list_by( page: Union[int, None] = Query(default=None, ge=0), size: Union[int, None] = 100, sort: Union[str, None] = Query( - default=None, pattern="(annotated_count|state|inner_id|updated_at):(desc|asc)" + default=None, + pattern=r"^(annotated_count|state|inner_id|updated_at):(desc|asc)" + r"(,(annotated_count|state|inner_id|updated_at):(desc|asc))*$", ), authorization: HTTPAuthorizationCredentials = Security(security), db: Session = Depends(db_module.get_db), diff --git a/labelu/tests/internal/adapter/persistence/test_crud_sample.py b/labelu/tests/internal/adapter/persistence/test_crud_sample.py new file mode 100644 index 00000000..77d13830 --- /dev/null +++ b/labelu/tests/internal/adapter/persistence/test_crud_sample.py @@ -0,0 +1,85 @@ +from sqlalchemy.orm import Session + +from labelu.internal.common.db import begin_transaction +from labelu.internal.domain.models.task import Task +from labelu.internal.domain.models.sample import TaskSample +from labelu.internal.adapter.persistence import crud_task +from labelu.internal.adapter.persistence import crud_sample + + +def _make_task_with_samples(db: Session, n: int = 3): + with begin_transaction(db): + task = crud_task.create( + db=db, + task=Task(name="n", description="d", tips="t", created_by=1, updated_by=1), + ) + samples = [ + TaskSample( + task_id=task.id, + file_id=1, + created_by=1, + updated_by=1, + data="{}", + annotated_count=i, + ) + for i in range(n) + ] + with begin_transaction(db): + crud_sample.batch(db=db, samples=samples) + return task + + +class TestCrudSampleSortInjection: + """Regression tests for SQL injection via the sample-listing sort param.""" + + def test_error_based_injection_is_ignored(self, db: Session): + task = _make_task_with_samples(db) + # a non-existent column injected after a valid pair must NOT reach SQL + sorting = "updated_at:asc,SQLI_MARKER_no_such_col:asc" + results = crud_sample.list_by( + db=db, task_id=task.id, after=None, before=None, + page=0, size=10, sorting=sorting, + ) + assert len(results) == 3 + + def test_boolean_based_injection_has_no_effect(self, db: Session): + task = _make_task_with_samples(db) + payload_true = ( + "state:asc,(CASE WHEN 1=1 THEN task_sample.id " + "ELSE 0-task_sample.id END):desc" + ) + payload_false = ( + "state:asc,(CASE WHEN 1=0 THEN task_sample.id " + "ELSE 0-task_sample.id END):desc" + ) + ids_true = [ + s.id for s in crud_sample.list_by( + db=db, task_id=task.id, after=None, before=None, + page=0, size=10, sorting=payload_true, + ) + ] + ids_false = [ + s.id for s in crud_sample.list_by( + db=db, task_id=task.id, after=None, before=None, + page=0, size=10, sorting=payload_false, + ) + ] + # if the injected CASE executed, the two orderings would differ + assert ids_true == ids_false + + def test_valid_sort_still_works(self, db: Session): + task = _make_task_with_samples(db) + results = crud_sample.list_by( + db=db, task_id=task.id, after=None, before=None, + page=0, size=10, sorting="annotated_count:desc", + ) + counts = [s.annotated_count for s in results] + assert counts == sorted(counts, reverse=True) + + def test_state_sort_still_works(self, db: Session): + task = _make_task_with_samples(db) + results = crud_sample.list_by( + db=db, task_id=task.id, after=None, before=None, + page=0, size=10, sorting="state:asc", + ) + assert len(results) == 3 From 23bd33ad60a4a500d02620c1e94d2f83656b569a Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 11:53:10 +0800 Subject: [PATCH 227/236] fix(security): validate the data source S3 endpoint to prevent SSRF CreateDataSourceCommand/UpdateDataSourceCommand only bounded the length of endpoint, and _build_s3_client passed it straight to boto3 as endpoint_url, so an authenticated user could point it at http://169.254.169.254 and other internal/metadata addresses, triggering outbound requests via list_objects/import_from_s3 (cloud IAM credential theft, internal scanning). The endpoint is now validated: only http(s) is allowed, and the host is resolved and rejected if it maps to a loopback/private/link-local/reserved/ multicast/unspecified address. Note: connection-time DNS-rebinding re-check (TOCTOU) is left as a follow-up defense-in-depth item. Co-Authored-By: Claude Fable 5 --- .../application/command/datasource.py | 45 ++++++++++++++- labelu/tests/internal/application/__init__.py | 0 .../internal/application/command/__init__.py | 0 .../application/command/test_datasource.py | 55 +++++++++++++++++++ 4 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 labelu/tests/internal/application/__init__.py create mode 100644 labelu/tests/internal/application/command/__init__.py create mode 100644 labelu/tests/internal/application/command/test_datasource.py diff --git a/labelu/internal/application/command/datasource.py b/labelu/internal/application/command/datasource.py index ee4576b6..0776a081 100644 --- a/labelu/internal/application/command/datasource.py +++ b/labelu/internal/application/command/datasource.py @@ -1,12 +1,53 @@ +import ipaddress +import socket from typing import Union +from urllib.parse import urlparse -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, Field, field_validator, model_validator + + +def validate_s3_endpoint(endpoint: Union[str, None]) -> Union[str, None]: + """Reject S3 endpoints that could be abused for SSRF. + + Only http(s) URLs are allowed, and the host must not resolve to a + loopback/private/link-local/reserved address (e.g. the cloud metadata + endpoint 169.254.169.254 or internal services). + """ + if endpoint is None or endpoint == "": + return endpoint + + parsed = urlparse(endpoint) + if parsed.scheme not in ("http", "https"): + raise ValueError("endpoint must be an http(s) URL") + host = parsed.hostname + if not host: + raise ValueError("endpoint must include a host") + + try: + infos = socket.getaddrinfo(host, None) + except socket.gaierror: + raise ValueError("endpoint host cannot be resolved") + + for info in infos: + addr = ipaddress.ip_address(info[4][0]) + if ( + addr.is_private + or addr.is_loopback + or addr.is_link_local + or addr.is_reserved + or addr.is_multicast + or addr.is_unspecified + ): + raise ValueError("endpoint host resolves to a disallowed address") + return endpoint class CreateDataSourceCommand(BaseModel): name: str = Field(max_length=128, description="Display name") type: str = Field(default="S3", max_length=32) endpoint: Union[str, None] = Field(default=None, max_length=512) + + _validate_endpoint = field_validator("endpoint")(validate_s3_endpoint) region: Union[str, None] = Field(default=None, max_length=64) bucket: str = Field(max_length=256) prefix: str = Field(default="", max_length=512) @@ -20,6 +61,8 @@ class CreateDataSourceCommand(BaseModel): class UpdateDataSourceCommand(BaseModel): name: Union[str, None] = Field(default=None, max_length=128) endpoint: Union[str, None] = Field(default=None, max_length=512) + + _validate_endpoint = field_validator("endpoint")(validate_s3_endpoint) region: Union[str, None] = Field(default=None, max_length=64) bucket: Union[str, None] = Field(default=None, max_length=256) prefix: Union[str, None] = Field(default=None, max_length=512) diff --git a/labelu/tests/internal/application/__init__.py b/labelu/tests/internal/application/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/labelu/tests/internal/application/command/__init__.py b/labelu/tests/internal/application/command/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/labelu/tests/internal/application/command/test_datasource.py b/labelu/tests/internal/application/command/test_datasource.py new file mode 100644 index 00000000..5ce9116d --- /dev/null +++ b/labelu/tests/internal/application/command/test_datasource.py @@ -0,0 +1,55 @@ +import pytest +from pydantic import ValidationError + +from labelu.internal.application.command.datasource import ( + CreateDataSourceCommand, + UpdateDataSourceCommand, +) + + +def _create(endpoint): + return CreateDataSourceCommand( + name="ds", + bucket="b", + access_key_id="ak", + secret_access_key="sk", + endpoint=endpoint, + ) + + +class TestDataSourceEndpointSSRF: + """Regression tests: the S3 endpoint must not point at internal targets.""" + + @pytest.mark.parametrize( + "endpoint", + [ + "http://169.254.169.254", # cloud metadata (IMDS) + "http://169.254.169.254/latest/", # IMDS with path + "http://127.0.0.1:9000", # loopback + "http://localhost:9000", # loopback by name + "http://10.1.2.3", # private range + "http://192.168.0.1", # private range + "http://172.16.5.5", # private range + "http://[::1]:9000", # IPv6 loopback + "file:///etc/passwd", # non-http scheme + "ftp://example.com", # non-http scheme + "http://0.0.0.0", # unspecified + ], + ) + def test_rejects_internal_or_bad_scheme_endpoint(self, endpoint): + with pytest.raises(ValidationError): + _create(endpoint) + with pytest.raises(ValidationError): + UpdateDataSourceCommand(endpoint=endpoint) + + @pytest.mark.parametrize( + "endpoint", + [ + None, # optional -> AWS default + "https://8.8.8.8:9000", # public IP literal + "http://1.1.1.1", # public IP literal + ], + ) + def test_allows_public_endpoint(self, endpoint): + cmd = _create(endpoint) + assert cmd.endpoint == endpoint From ec871fac048606f0eb1b98d52213fb2bd9410d6e Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 12:02:47 +0800 Subject: [PATCH 228/236] fix(security): fix broken access control on samples and exports (IDOR) Sample create/import/list/get/patch/delete and export create/status/download did not verify the caller could access the corresponding task, so any authenticated user could enumerate auto-increment IDs to read, modify or delete other users' annotation data, or download their export results. Adds access.assert_task_access(task, current_user) (owner or collaborator) and enforces it across the sample service and export endpoints; get/patch additionally verify sample.task_id matches the URL task to prevent referencing another user's sample through a task you own. Co-Authored-By: Claude Fable 5 --- labelu/internal/adapter/routers/sample.py | 13 +- labelu/internal/application/service/access.py | 18 +++ labelu/internal/application/service/sample.py | 29 +++- .../internal/adapter/routers/test_sample.py | 132 +++++++++++++----- 4 files changed, 154 insertions(+), 38 deletions(-) create mode 100644 labelu/internal/application/service/access.py diff --git a/labelu/internal/adapter/routers/sample.py b/labelu/internal/adapter/routers/sample.py index 02cddbe1..6c1d8165 100644 --- a/labelu/internal/adapter/routers/sample.py +++ b/labelu/internal/adapter/routers/sample.py @@ -30,6 +30,8 @@ from labelu.internal.application.response.auto_label import AutoLabelResponse, AutoLabelJobResponse from labelu.internal.application.response.export import ExportJobResponse from labelu.internal.adapter.persistence import crud_export_job +from labelu.internal.adapter.persistence import crud_task +from labelu.internal.application.service.access import assert_task_access from labelu.internal.common.storage import get_storage_backend @@ -111,6 +113,7 @@ async def list_by( page=page, size=size, sorting=sort, + current_user=current_user, ) # response @@ -136,7 +139,7 @@ async def get( # business logic data = await service.get( - db=db, task_id=task_id, sample_id=sample_id + db=db, task_id=task_id, sample_id=sample_id, current_user=current_user ) # response @@ -331,6 +334,10 @@ async def get_export_status( status_code=status.HTTP_404_NOT_FOUND, ) + task = crud_task.get(db=db, task_id=job.task_id) + if task is not None: + assert_task_access(task, current_user) + skipped_count, warning_message = _export_progress( sample_count=job.sample_count, processed_count=job.processed_count, @@ -375,6 +382,10 @@ async def download_export( status_code=status.HTTP_404_NOT_FOUND, ) + task = crud_task.get(db=db, task_id=job.task_id) + if task is not None: + assert_task_access(task, current_user) + storage = get_storage_backend() if storage.is_remote: download_url = storage.get_read_url(job.file_path) diff --git a/labelu/internal/application/service/access.py b/labelu/internal/application/service/access.py new file mode 100644 index 00000000..33e91342 --- /dev/null +++ b/labelu/internal/application/service/access.py @@ -0,0 +1,18 @@ +from fastapi import status + +from labelu.internal.common.error_code import ErrorCode, LabelUException + + +def assert_task_access(task, current_user) -> None: + """Ensure current_user may access the given task's data. + + Access is granted to the task owner or any of its collaborators. Any other + authenticated user is rejected with 403, closing IDOR/BOLA on task-scoped + resources (samples, pre-annotations, exports, ...). + """ + collaborator_ids = {c.id for c in task.collaborators} + if task.created_by != current_user.id and current_user.id not in collaborator_ids: + raise LabelUException( + code=ErrorCode.CODE_30001_NO_PERMISSION, + status_code=status.HTTP_403_FORBIDDEN, + ) diff --git a/labelu/internal/application/service/sample.py b/labelu/internal/application/service/sample.py index 4e7bac68..358e7326 100644 --- a/labelu/internal/application/service/sample.py +++ b/labelu/internal/application/service/sample.py @@ -19,6 +19,7 @@ build_thumbnail_key, get_storage_backend, ) +from labelu.internal.application.service.access import assert_task_access from labelu.internal.adapter.persistence import crud_attachment, crud_pre_annotation, crud_task from labelu.internal.adapter.persistence import crud_sample from labelu.internal.adapter.persistence import crud_export_job @@ -71,6 +72,8 @@ async def create( status_code=status.HTTP_404_NOT_FOUND, ) + assert_task_access(task, current_user) + samples = [ TaskSample( inner_id=task.last_sample_inner_id + i + 1, @@ -167,6 +170,8 @@ async def import_from_s3( status_code=status.HTTP_404_NOT_FOUND, ) + assert_task_access(task, current_user) + ds = crud_datasource.get(db=db, ds_id=cmd.data_source_id) if not ds: raise LabelUException( @@ -229,7 +234,12 @@ async def list_by( page: Union[int, None], size: int, sorting: Union[str, None], + current_user: User, ) -> Tuple[List[SampleResponse], int]: + if task_id is not None: + task = crud_task.get(db=db, task_id=task_id) + if task is not None: + assert_task_access(task, current_user) samples = crud_sample.list_by( db=db, task_id=task_id, @@ -268,20 +278,24 @@ async def list_by( async def get( - db: Session, task_id: int, sample_id: int + db: Session, task_id: int, sample_id: int, current_user: User ) -> SampleResponse: sample = crud_sample.get( db=db, sample_id=sample_id, ) - if not sample: + if not sample or sample.task_id != task_id: logger.error("cannot find sample:{}", sample_id) raise LabelUException( code=ErrorCode.CODE_55001_SAMPLE_NOT_FOUND, status_code=status.HTTP_404_NOT_FOUND, ) + task = crud_task.get(db=db, task_id=sample.task_id) + if task is not None: + assert_task_access(task, current_user) + # response return SampleResponse( id=sample.id, @@ -321,9 +335,11 @@ async def patch( status_code=status.HTTP_404_NOT_FOUND, ) + assert_task_access(task, current_user) + # get sample sample = crud_sample.get(db=db, sample_id=sample_id) - if not sample: + if not sample or sample.task_id != task_id: logger.error("cannot find sample:{}", sample_id) raise LabelUException( code=ErrorCode.CODE_55001_SAMPLE_NOT_FOUND, @@ -407,6 +423,11 @@ async def delete( with begin_transaction(db): # delete media samples = crud_sample.get_by_ids(db=db, sample_ids=sample_ids) + # authorize: caller must have access to every sample's task + for task_id in {sample.task_id for sample in samples}: + task = crud_task.get(db=db, task_id=task_id) + if task is not None: + assert_task_access(task, current_user) attachment_ids = [sample.file_id for sample in samples if sample.file_id] attachments = crud_attachment.get_by_ids(db=db, attachment_ids=attachment_ids) @@ -440,6 +461,8 @@ async def create_export_job( status_code=status.HTTP_404_NOT_FOUND, ) + assert_task_access(task, current_user) + with begin_transaction(db): job = crud_export_job.create( db=db, diff --git a/labelu/tests/internal/adapter/routers/test_sample.py b/labelu/tests/internal/adapter/routers/test_sample.py index 5e349d7f..a3417f32 100644 --- a/labelu/tests/internal/adapter/routers/test_sample.py +++ b/labelu/tests/internal/adapter/routers/test_sample.py @@ -17,6 +17,9 @@ def test_create_sample_successful( ) -> None: # prepare data + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) with begin_transaction(db): task = crud_task.create( db=db, @@ -24,8 +27,8 @@ def test_create_sample_successful( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) data = [{"file_id": 1, "data": {}}] @@ -50,6 +53,9 @@ def test_create_sample_task_status_not_draft( ) -> None: # prepare data + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) with begin_transaction(db): task = crud_task.create( db=db, @@ -57,8 +63,8 @@ def test_create_sample_task_status_not_draft( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, status="CONFIGURED", ), ) @@ -113,8 +119,8 @@ def test_sample_list_by_page( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) samples = [ @@ -162,8 +168,8 @@ def test_sample_list_by_before( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) samples = [ @@ -212,8 +218,8 @@ def test_sample_list_by_after( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) samples = [ @@ -262,8 +268,8 @@ def test_sample_list_with_sort( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) samples = [ @@ -333,8 +339,8 @@ def test_sample_list_by_sort_error( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) # run @@ -347,6 +353,58 @@ def test_sample_list_by_sort_error( # check assert r.status_code == 422 + def test_sample_access_forbidden_for_non_member( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + # a task owned by someone else; the test user is neither owner nor collaborator + with begin_transaction(db): + task = crud_task.create( + db=db, + task=Task( + name="other", description="d", tips="t", + created_by=999, updated_by=999, + ), + ) + with begin_transaction(db): + samples = crud_sample.batch( + db=db, + samples=[ + TaskSample( + task_id=task.id, file_id=1, + created_by=999, updated_by=999, data="{}", + ) + ], + ) + sid = samples[0].id + + # list + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/samples", + headers=testuser_token_headers, params={"page": 0, "size": 10}, + ) + assert r.status_code == 403 + # get + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/samples/{sid}", + headers=testuser_token_headers, + ) + assert r.status_code == 403 + # patch + r = client.patch( + f"{settings.API_V1_STR}/tasks/{task.id}/samples/{sid}", + headers=testuser_token_headers, + json={"data": {}, "annotated_count": 0}, + ) + assert r.status_code == 403 + # delete + r = client.request( + "delete", + f"{settings.API_V1_STR}/tasks/{task.id}/samples", + headers=testuser_token_headers, + json={"sample_ids": [sid]}, + ) + assert r.status_code == 403 + def test_sample_get( self, client: TestClient, testuser_token_headers: dict, db: Session ) -> None: @@ -362,8 +420,8 @@ def test_sample_get( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) with begin_transaction(db): @@ -407,8 +465,8 @@ def test_sample_get_not_found( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) @@ -438,8 +496,8 @@ def test_sample_patch( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) with begin_transaction(db): @@ -487,8 +545,8 @@ def test_sample_patch_skip( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) with begin_transaction(db): @@ -541,6 +599,9 @@ def test_sample_patch_task_sample_not_found( ) -> None: # prepare data + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) with begin_transaction(db): task = crud_task.create( db=db, @@ -548,8 +609,8 @@ def test_sample_patch_task_sample_not_found( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) @@ -580,8 +641,8 @@ def test_sample_delete( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) with begin_transaction(db): @@ -614,6 +675,9 @@ def test_sample_delete_not_found( ) -> None: # prepare data + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) with begin_transaction(db): task = crud_task.create( db=db, @@ -621,8 +685,8 @@ def test_sample_delete_not_found( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) @@ -653,8 +717,8 @@ def test_export_sample( description="description", tips="tips", config='{"tools":[{"tool":"rectTool","config":{"isShowCursor":false,"showConfirm":false,"skipWhileNoDependencies":false,"drawOutsideTarget":false,"copyBackwardResult":false,"minWidth":1,"attributeConfigurable":true,"textConfigurable":true,"textCheckType":4,"customFormat":"","attributes":[{"key":"rectTool","value":"rectTool"}]}},{"tool":"pointTool","config":{"upperLimit":10,"isShowCursor":false,"attributeConfigurable":true,"copyBackwardResult":false,"textConfigurable":true,"textCheckType":0,"customFormat":"","attributes":[{"key":"pointTool","value":"pointTool"}]}},{"tool":"polygonTool","config":{"isShowCursor":false,"lineType":0,"lineColor":0,"drawOutsideTarget":false,"edgeAdsorption":true,"copyBackwardResult":false,"attributeConfigurable":true,"textConfigurable":true,"textCheckType":0,"customFormat":"","attributes":[{"key":"polygonTool","value":"polygonTool"}],"lowerLimitPointNum":"4","upperLimitPointNum":100}},{"tool":"lineTool","config":{"isShowCursor":false,"lineType":0,"lineColor":1,"edgeAdsorption":true,"outOfTarget":true,"copyBackwardResult":false,"attributeConfigurable":true,"textConfigurable":true,"textCheckType":4,"customFormat":"^[\s\S]{1,3}$","lowerLimitPointNum":4,"upperLimitPointNum":"","attributes":[{"key":"lineTool","value":"lineTool"}]}},{"tool":"tagTool"},{"tool":"textTool"}],"tagList":[{"key":"类别1","value":"class1","isMulti":true,"subSelected":[{"key":"选项1","value":"option1","isDefault":true},{"key":"选项2","value":"option2","isDefault":false}]},{"key":"类别2","value":"class2","isMulti":true,"subSelected":[{"key":"a选项1","value":"aoption1","isDefault":true},{"key":"a选项2","value":"aoption2","isDefault":false}]}],"attributes":[{"key":"RT","value":"RT"}],"textConfig":[{"label":"我的描述","key":"描述的关键","required":true,"default":"","maxLength":200},{"label":"我的描述1","key":"描述的关键1","required":true,"default":"","maxLength":200}],"fileInfo":{"type":"img","list":[{"id":1,"url":"/src/img/example/bear6.webp","result":"[]"}]},"commonAttributeConfigurable":true}', - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) samples = crud_sample.batch( @@ -704,8 +768,8 @@ def test_export_includes_all_states( description="description", tips="tips", config='{"tools":[{"tool":"rectTool","config":{"attributes":[{"key":"RT","value":"RT"}]}}],"tagList":[],"attributes":[],"textConfig":[],"fileInfo":{"type":"img","list":[]},"commonAttributeConfigurable":true}', - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) result_data = '{"result": "{\"width\":100,\"height\":100,\"valid\":true,\"rotate\":0,\"rectTool\":{\"toolName\":\"rectTool\",\"result\":[{\"x\":10,\"y\":10,\"width\":50,\"height\":50,\"label\":\"RT\",\"valid\":true,\"isVisible\":true,\"id\":\"test1\",\"order\":1}]}}"}' @@ -771,8 +835,8 @@ def test_export_status_reports_skipped_count( description="description", tips="tips", config='{"tools":[{"tool":"rectTool","config":{"attributes":[{"key":"RT","value":"RT"}]}}],"tagList":[],"attributes":[],"textConfig":[],"fileInfo":{"type":"img","list":[]},"commonAttributeConfigurable":true}', - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) job = crud_export_job.create( From 7f879f72ba0b190e3c654775ad9dcc686dca1118 Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 12:04:28 +0800 Subject: [PATCH 229/236] fix(security): fix broken access control on task update (IDOR) task.update neither received current_user nor checked ownership (inconsistent with get/delete/collaborator management in the same file), so any authenticated user could PATCH and overwrite another user's task config/name/description/media_type. update now receives current_user and calls assert_task_access (owner or collaborator). Co-Authored-By: Claude Fable 5 --- labelu/internal/adapter/routers/task.py | 2 +- labelu/internal/application/service/task.py | 5 ++++- .../internal/adapter/routers/test_task.py | 21 +++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/labelu/internal/adapter/routers/task.py b/labelu/internal/adapter/routers/task.py index a650ecbc..fe4d5999 100644 --- a/labelu/internal/adapter/routers/task.py +++ b/labelu/internal/adapter/routers/task.py @@ -220,7 +220,7 @@ async def update( """ # business logic - data = await service.update(db=db, task_id=task_id, cmd=cmd) + data = await service.update(db=db, task_id=task_id, cmd=cmd, current_user=current_user) # response return OkResp[TaskResponse](data=data) diff --git a/labelu/internal/application/service/task.py b/labelu/internal/application/service/task.py index 6a483e8a..eee0f9c1 100644 --- a/labelu/internal/application/service/task.py +++ b/labelu/internal/application/service/task.py @@ -14,6 +14,7 @@ from labelu.internal.domain.models.task import Task from labelu.internal.domain.models.task import TaskStatus from labelu.internal.domain.models.sample import SampleState +from labelu.internal.application.service.access import assert_task_access from labelu.internal.adapter.persistence import crud_task, crud_user from labelu.internal.adapter.persistence import crud_sample from labelu.internal.application.command.task import BasicConfigCommand @@ -312,7 +313,7 @@ async def batch_remove_collaborators(db: Session, task_id: int, user_ids: List[i return CommonDataResp(ok=True) -async def update(db: Session, task_id: int, cmd: UpdateCommand) -> TaskResponse: +async def update(db: Session, task_id: int, cmd: UpdateCommand, current_user: User) -> TaskResponse: # get task task = crud_task.get(db=db, task_id=task_id) @@ -323,6 +324,8 @@ async def update(db: Session, task_id: int, cmd: UpdateCommand) -> TaskResponse: status_code=status.HTTP_404_NOT_FOUND, ) + assert_task_access(task, current_user) + # update obj_in = cmd.dict(exclude_unset=True) if cmd.config and cmd.media_type: diff --git a/labelu/tests/internal/adapter/routers/test_task.py b/labelu/tests/internal/adapter/routers/test_task.py index 4688aa4c..a8e96da6 100644 --- a/labelu/tests/internal/adapter/routers/test_task.py +++ b/labelu/tests/internal/adapter/routers/test_task.py @@ -176,6 +176,27 @@ def test_task_update( assert json["data"]["config"] == "new config" assert json["data"]["media_type"] == "IMAGE" + def test_task_update_forbidden_for_non_owner( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + # task owned by another user + with begin_transaction(db): + task = crud_task.create( + db=db, + task=Task( + name="other", description="d", tips="t", + created_by=999, updated_by=999, + ), + ) + + r = client.patch( + f"{settings.API_V1_STR}/tasks/{task.id}", + headers=testuser_token_headers, + json={"name": "hijacked", "description": "x", "tips": "y", + "config": "c", "media_type": "IMAGE"}, + ) + assert r.status_code == 403 + def test_task_update_no_found_task( self, client: TestClient, testuser_token_headers: dict, db: Session ) -> None: From 9f964540d79ccf2eeeb28a937bdb9861d1c9c95c Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 12:07:37 +0800 Subject: [PATCH 230/236] fix(security): fix broken access control on data sources (IDOR) Data source get/update/delete/list_objects and S3 import did not check ownership, so any authenticated user could read others' S3 configuration, tamper with/delete data sources, or use the victim's decrypted S3 credentials to list their private bucket or import objects. Adds access.assert_owner(resource, current_user) (owner-only; data sources have no collaborator concept) and enforces it on the four data source endpoints and in sample.import_from_s3 before the credentials are used; list_objects checks access before building the S3 client, preventing credential abuse. Co-Authored-By: Claude Fable 5 --- labelu/internal/adapter/routers/datasource.py | 4 +- labelu/internal/application/service/access.py | 13 ++++ .../application/service/datasource.py | 8 ++- labelu/internal/application/service/sample.py | 3 +- .../adapter/routers/test_datasource.py | 66 +++++++++++++++++++ 5 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 labelu/tests/internal/adapter/routers/test_datasource.py diff --git a/labelu/internal/adapter/routers/datasource.py b/labelu/internal/adapter/routers/datasource.py index 51020583..f45253c6 100644 --- a/labelu/internal/adapter/routers/datasource.py +++ b/labelu/internal/adapter/routers/datasource.py @@ -72,7 +72,7 @@ async def get( db: Session = Depends(db_module.get_db), current_user: User = Depends(get_current_user), ): - data = await service.get(db=db, ds_id=ds_id) + data = await service.get(db=db, ds_id=ds_id, current_user=current_user) return OkResp[DataSourceResponse](data=data) @@ -123,7 +123,7 @@ async def list_objects( current_user: User = Depends(get_current_user), ): data = await service.list_objects( - db=db, ds_id=ds_id, prefix=prefix, extension=extension, + db=db, ds_id=ds_id, current_user=current_user, prefix=prefix, extension=extension, page_token=page_token, size=size, ) return OkResp[S3ObjectListResponse](data=data) diff --git a/labelu/internal/application/service/access.py b/labelu/internal/application/service/access.py index 33e91342..a78b25b0 100644 --- a/labelu/internal/application/service/access.py +++ b/labelu/internal/application/service/access.py @@ -16,3 +16,16 @@ def assert_task_access(task, current_user) -> None: code=ErrorCode.CODE_30001_NO_PERMISSION, status_code=status.HTTP_403_FORBIDDEN, ) + + +def assert_owner(resource, current_user) -> None: + """Ensure current_user owns the resource (created_by). + + Used for user-private resources without a collaborator concept + (e.g. data sources). + """ + if resource.created_by != current_user.id: + raise LabelUException( + code=ErrorCode.CODE_30001_NO_PERMISSION, + status_code=status.HTTP_403_FORBIDDEN, + ) diff --git a/labelu/internal/application/service/datasource.py b/labelu/internal/application/service/datasource.py index feef3f53..f14f7d02 100644 --- a/labelu/internal/application/service/datasource.py +++ b/labelu/internal/application/service/datasource.py @@ -9,6 +9,7 @@ from sqlalchemy.orm import Session from labelu.internal.adapter.persistence import crud_datasource +from labelu.internal.application.service.access import assert_owner from labelu.internal.application.command.datasource import ( CreateDataSourceCommand, UpdateDataSourceCommand, @@ -110,13 +111,14 @@ async def list_by( return [_to_response(ds) for ds in items], total -async def get(db: Session, ds_id: int) -> DataSourceResponse: +async def get(db: Session, ds_id: int, current_user: User) -> DataSourceResponse: ds = crud_datasource.get(db=db, ds_id=ds_id) if not ds: raise LabelUException( code=ErrorCode.CODE_61000_NO_DATA, status_code=status.HTTP_404_NOT_FOUND, ) + assert_owner(ds, current_user) return _to_response(ds) @@ -129,6 +131,7 @@ async def update( code=ErrorCode.CODE_61000_NO_DATA, status_code=status.HTTP_404_NOT_FOUND, ) + assert_owner(ds, current_user) obj_in = cmd.model_dump(exclude_unset=True) if "access_key_id" in obj_in and obj_in["access_key_id"] is not None: obj_in["access_key_id"] = encrypt_value(obj_in["access_key_id"]) @@ -147,6 +150,7 @@ async def delete(db: Session, ds_id: int, current_user: User) -> None: code=ErrorCode.CODE_61000_NO_DATA, status_code=status.HTTP_404_NOT_FOUND, ) + assert_owner(ds, current_user) with begin_transaction(db): crud_datasource.soft_delete(db=db, db_obj=ds) @@ -156,6 +160,7 @@ async def delete(db: Session, ds_id: int, current_user: User) -> None: async def list_objects( db: Session, ds_id: int, + current_user: User, prefix: Optional[str] = None, extension: Optional[str] = None, page_token: Optional[str] = None, @@ -167,6 +172,7 @@ async def list_objects( code=ErrorCode.CODE_61000_NO_DATA, status_code=status.HTTP_404_NOT_FOUND, ) + assert_owner(ds, current_user) client = _build_s3_client(ds) full_prefix = prefix if prefix is not None else (ds.prefix or "") diff --git a/labelu/internal/application/service/sample.py b/labelu/internal/application/service/sample.py index 358e7326..a4d03a77 100644 --- a/labelu/internal/application/service/sample.py +++ b/labelu/internal/application/service/sample.py @@ -19,7 +19,7 @@ build_thumbnail_key, get_storage_backend, ) -from labelu.internal.application.service.access import assert_task_access +from labelu.internal.application.service.access import assert_task_access, assert_owner from labelu.internal.adapter.persistence import crud_attachment, crud_pre_annotation, crud_task from labelu.internal.adapter.persistence import crud_sample from labelu.internal.adapter.persistence import crud_export_job @@ -178,6 +178,7 @@ async def import_from_s3( code=ErrorCode.CODE_61000_NO_DATA, status_code=status.HTTP_404_NOT_FOUND, ) + assert_owner(ds, current_user) # Resolve object keys: either from explicit list or by listing S3 prefix object_keys = cmd.object_keys diff --git a/labelu/tests/internal/adapter/routers/test_datasource.py b/labelu/tests/internal/adapter/routers/test_datasource.py new file mode 100644 index 00000000..cc83264d --- /dev/null +++ b/labelu/tests/internal/adapter/routers/test_datasource.py @@ -0,0 +1,66 @@ +from sqlalchemy.orm import Session +from fastapi.testclient import TestClient + +from labelu.internal.common.config import settings +from labelu.internal.common.db import begin_transaction +from labelu.internal.domain.models.data_source import DataSource +from labelu.internal.adapter.persistence import crud_datasource + + +class TestDataSourceIDOR: + def _other_users_datasource(self, db: Session) -> DataSource: + with begin_transaction(db): + return crud_datasource.create( + db=db, + data_source=DataSource( + name="victim-ds", + type="S3", + endpoint="https://8.8.8.8", + bucket="victim-bucket", + created_by=999, + updated_by=999, + ), + ) + + def test_get_others_datasource_forbidden( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + ds = self._other_users_datasource(db) + r = client.get( + f"{settings.API_V1_STR}/datasources/{ds.id}", + headers=testuser_token_headers, + ) + assert r.status_code == 403 + + def test_update_others_datasource_forbidden( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + ds = self._other_users_datasource(db) + r = client.patch( + f"{settings.API_V1_STR}/datasources/{ds.id}", + headers=testuser_token_headers, + json={"name": "hijacked"}, + ) + assert r.status_code == 403 + + def test_delete_others_datasource_forbidden( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + ds = self._other_users_datasource(db) + r = client.request( + "delete", + f"{settings.API_V1_STR}/datasources/{ds.id}", + headers=testuser_token_headers, + ) + assert r.status_code == 403 + + def test_list_objects_others_datasource_forbidden( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + ds = self._other_users_datasource(db) + # must be denied before any S3 request is made with the victim's creds + r = client.get( + f"{settings.API_V1_STR}/datasources/{ds.id}/objects", + headers=testuser_token_headers, + ) + assert r.status_code == 403 From ce8273fb2410c8dfa46a74e17f89bfb5bb26794a Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 12:11:47 +0800 Subject: [PATCH 231/236] fix(security): fix broken access control on pre-annotations (IDOR) Pre-annotation create/list/get/delete/delete_file already received current_user but never used it for authorization, so any authenticated user could read/inject/delete another task's pre-annotations (delete_file also physically removed the underlying files). All endpoints now call assert_task_access; get additionally verifies pre_annotation.task_id matches the URL task, and delete checks access for each pre-annotation's task. Co-Authored-By: Claude Fable 5 --- .../application/service/pre_annotation.py | 34 +++++- .../adapter/routers/test_pre_annotation.py | 104 ++++++++++++++---- 2 files changed, 111 insertions(+), 27 deletions(-) diff --git a/labelu/internal/application/service/pre_annotation.py b/labelu/internal/application/service/pre_annotation.py index 297cfae2..71f0dd88 100644 --- a/labelu/internal/application/service/pre_annotation.py +++ b/labelu/internal/application/service/pre_annotation.py @@ -16,6 +16,7 @@ build_thumbnail_key, get_storage_backend, ) +from labelu.internal.application.service.access import assert_task_access from labelu.internal.adapter.persistence import crud_task from labelu.internal.adapter.persistence import crud_pre_annotation from labelu.internal.adapter.persistence import crud_attachment @@ -63,7 +64,9 @@ async def create( code=ErrorCode.CODE_50002_TASK_NOT_FOUND, status_code=status.HTTP_404_NOT_FOUND, ) - + + assert_task_access(task, current_user) + pre_annotations = [] for pre_annotation in cmd: pre_annotation_file = crud_attachment.get(db, pre_annotation.file_id) @@ -108,7 +111,12 @@ async def list_by( sorting: Optional[str], current_user: User, ) -> Tuple[List[PreAnnotationResponse], int]: - + + if task_id is not None: + task = crud_task.get(db=db, task_id=task_id) + if task is not None: + assert_task_access(task, current_user) + pre_annotations, total = crud_pre_annotation.list_by( db=db, task_id=task_id, @@ -198,13 +206,17 @@ async def get( pre_annotation_id=pre_annotation_id, ) - if not pre_annotation: + if not pre_annotation or pre_annotation.task_id != task_id: logger.error("cannot find pre_annotation: {}", pre_annotation_id) raise LabelUException( code=ErrorCode.CODE_55001_SAMPLE_NOT_FOUND, status_code=status.HTTP_404_NOT_FOUND, ) - + + task = crud_task.get(db=db, task_id=pre_annotation.task_id) + if task is not None: + assert_task_access(task, current_user) + # response return PreAnnotationResponse( id=pre_annotation.id, @@ -233,7 +245,9 @@ async def delete_pre_annotation_file( code=ErrorCode.CODE_50002_TASK_NOT_FOUND, status_code=status.HTTP_404_NOT_FOUND, ) - + + assert_task_access(task, current_user) + attachments = crud_attachment.get_by_ids( db=db, attachment_ids=[file_id] ) @@ -264,6 +278,16 @@ async def delete( ) -> CommonDataResp: with begin_transaction(db): + # authorize: caller must have access to every pre-annotation's task + pre_annotations = [ + crud_pre_annotation.get(db=db, pre_annotation_id=pid) + for pid in pre_annotation_ids + ] + for task_id in {pa.task_id for pa in pre_annotations if pa}: + task = crud_task.get(db=db, task_id=task_id) + if task is not None: + assert_task_access(task, current_user) + crud_pre_annotation.delete(db=db, pre_annotation_ids=pre_annotation_ids) # response return CommonDataResp(ok=True) diff --git a/labelu/tests/internal/adapter/routers/test_pre_annotation.py b/labelu/tests/internal/adapter/routers/test_pre_annotation.py index 42f9b134..8f620200 100644 --- a/labelu/tests/internal/adapter/routers/test_pre_annotation.py +++ b/labelu/tests/internal/adapter/routers/test_pre_annotation.py @@ -20,6 +20,9 @@ class TestClassTaskPreAnnotationRouter: def test_create_pre_annotation_successful( self, client: TestClient, testuser_token_headers: dict, db: Session ) -> None: + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) with begin_transaction(db): task = crud_task.create( @@ -28,8 +31,8 @@ def test_create_pre_annotation_successful( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) @@ -55,9 +58,57 @@ def test_create_pre_annotation_successful( assert r.status_code == 201 assert len(json["data"]["ids"]) == 1 + def test_pre_annotation_access_forbidden_for_non_member( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + # task + pre-annotation owned by another user + with begin_transaction(db): + task = crud_task.create( + db=db, + task=Task( + name="other", description="d", tips="t", + created_by=999, updated_by=999, + ), + ) + with begin_transaction(db): + pre = crud_pre_annotation.batch( + db=db, + pre_annotations=[ + TaskPreAnnotation( + task_id=task.id, file_id=1, sample_name="x", + data="{}", created_by=999, updated_by=999, + ) + ], + ) + pid = pre[0].id + + # list + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations", + headers=testuser_token_headers, params={"page": 0, "size": 10}, + ) + assert r.status_code == 403 + # get + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations/{pid}", + headers=testuser_token_headers, + ) + assert r.status_code == 403 + # delete + r = client.request( + "delete", + f"{settings.API_V1_STR}/tasks/{task.id}/pre_annotations", + headers=testuser_token_headers, + json={"pre_annotation_ids": [pid]}, + ) + assert r.status_code == 403 + def test_create_pre_annotation_sample_exists( self, client: TestClient, testuser_token_headers: dict, db: Session ) -> None: + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) with begin_transaction(db): task = crud_task.create( @@ -66,8 +117,8 @@ def test_create_pre_annotation_sample_exists( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) @@ -144,8 +195,8 @@ def test_pre_annotation_list_by_page( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) @@ -203,8 +254,8 @@ def test_pre_annotation_list_with_sample_name( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) @@ -283,8 +334,8 @@ def test_pre_annotation_list_with_sample_name_not_found( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) empty_task_upload(task.id, "test.png") @@ -376,8 +427,8 @@ def test_sample_list_by_sample_name_error( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) # run @@ -405,8 +456,8 @@ def test_pre_annotation_get( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) with begin_transaction(db): @@ -448,8 +499,8 @@ def test_pre_annotation_delete( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) with begin_transaction(db): @@ -480,6 +531,9 @@ def test_pre_annotation_delete( def test_pre_annotations_delete_not_found( self, client: TestClient, testuser_token_headers: dict, db: Session ) -> None: + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) # prepare data with begin_transaction(db): @@ -489,8 +543,8 @@ def test_pre_annotations_delete_not_found( name="name", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) @@ -595,6 +649,9 @@ def test_pre_annotation_files_list( def test_pre_annotation_files_list_with_errors( self, client: TestClient, testuser_token_headers: dict, db: Session ) -> None: + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) # 测试不存在的任务 r = client.get( f"{settings.API_V1_STR}/tasks/9999/pre_annotations/files", @@ -613,8 +670,8 @@ def test_pre_annotation_files_list_with_errors( name="invalid sort task", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) @@ -687,6 +744,9 @@ def test_delete_pre_annotation_file( def test_delete_pre_annotation_file_errors( self, client: TestClient, testuser_token_headers: dict, db: Session ) -> None: + current_user = crud_user.get_user_by_username( + db=db, username="test@example.com" + ) # 测试不存在的任务 r1 = client.delete( f"{settings.API_V1_STR}/tasks/9999/pre_annotations/files/1", @@ -704,8 +764,8 @@ def test_delete_pre_annotation_file_errors( name="not found file task", description="description", tips="tips", - created_by=0, - updated_by=0, + created_by=current_user.id, + updated_by=current_user.id, ), ) From 5a548445aeb4e117d0a20503a112d7e46615187b Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 12:29:13 +0800 Subject: [PATCH 232/236] test: make test suite hermetic by isolating DB and media per run The test suite persisted state across runs in two places: test.db was pinned to the cwd (so sqlite_sequence accumulated and task_ids kept growing) and MEDIA_ROOT pointed at the real appdirs data directory (~/Library/Application Support/labelu/media). Fresh task_ids collided with leftover upload//... files, intermittently failing uploads with a 400 "file already exists" (e.g. test_upload_file_with_hash_symbol). - Redirect both the sqlite test DB and MEDIA_ROOT into a per-run temp directory created at import time (the engine is built at module import, before any tmp_path/monkeypatch fixture could run); remove it at session end via the _hermetic_storage fixture. - Force STORAGE_BACKEND=local and clear the get_storage_backend lru_cache so it resolves against the new MEDIA_ROOT. - Seed the test user at session scope so it exists before the module-scoped login fixture (previously it survived only because test.db persisted across runs). Co-Authored-By: Claude Opus 4.8 --- labelu/tests/conftest.py | 56 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/labelu/tests/conftest.py b/labelu/tests/conftest.py index e51ddb22..d386286d 100644 --- a/labelu/tests/conftest.py +++ b/labelu/tests/conftest.py @@ -1,13 +1,40 @@ -import pytest +import os +import shutil +import tempfile +from pathlib import Path from typing import Dict, Generator +import pytest from sqlalchemy import create_engine from sqlalchemy.orm import Session from sqlalchemy.orm import sessionmaker from fastapi.testclient import TestClient -from labelu.main import app +# Make the test suite hermetic: every run gets its own throwaway directory that +# holds BOTH the sqlite test DB and all uploaded media. Without this, tests +# wrote uploads into the developer's real appdirs data directory +# (~/Library/Application Support/labelu/media) and kept ``test.db`` in the cwd, +# so ``sqlite_sequence`` accumulated across runs and freshly minted task_ids +# collided with leftover ``upload//...`` files, intermittently failing +# uploads with a 400 "file already exists". The directory must be chosen at +# import time because the engine below is created at module import, before any +# pytest ``tmp_path``/``monkeypatch`` fixture could run. It is removed at the end +# of the session by the ``_hermetic_storage`` fixture. +_TEST_DATA_DIR = Path(tempfile.mkdtemp(prefix="labelu-test-")) + from labelu.internal.common.config import settings + +# Redirect media storage into the isolated directory and drop the memoized +# storage backend so it resolves paths against the new MEDIA_ROOT. +settings.STORAGE_BACKEND = "local" +settings.MEDIA_ROOT = _TEST_DATA_DIR / "media" +os.makedirs(settings.MEDIA_ROOT, exist_ok=True) + +from labelu.internal.common.storage import get_storage_backend + +get_storage_backend.cache_clear() + +from labelu.main import app from labelu.internal.common.db import Base from labelu.internal.common.db import begin_transaction from labelu.internal.common.db import get_db @@ -19,7 +46,7 @@ TEST_USER_PASSWORD = "test@123" -SQLALCHEMY_DATABASE_URL = "sqlite:///./test.db" +SQLALCHEMY_DATABASE_URL = f"sqlite:///{_TEST_DATA_DIR / 'test.db'}" engine = create_engine( SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False, "timeout": 30}, @@ -28,6 +55,23 @@ Base.metadata.create_all(bind=engine) +@pytest.fixture(autouse=True, scope="session") +def _hermetic_storage() -> Generator: + """Guarantee the isolated media root is active and clean it up afterwards. + + The redirect itself happens at import time (above); this fixture re-clears + the storage-backend cache in case anything memoized it during collection and + removes the throwaway directory (test DB + media) once the session ends. + """ + get_storage_backend.cache_clear() + try: + yield + finally: + engine.dispose() + get_storage_backend.cache_clear() + shutil.rmtree(_TEST_DATA_DIR, ignore_errors=True) + + def override_get_db(): db = None try: @@ -46,6 +90,12 @@ def db() -> Generator: db = None try: db = TestingSessionLocal() + # Seed the test user at session scope so it exists before the + # module-scoped ``testuser_token_headers`` fixture logs in. The + # per-test ``run_around_tests`` cleanup preserves the ``user`` table, so + # this single seed lasts the whole session. (Previously the user + # survived only because ``test.db`` persisted in the cwd across runs.) + init_db() yield db finally: if db is not None: From c2e76ba58a3faa06841cd1bd6ef576285e89d65e Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 13:34:38 +0800 Subject: [PATCH 233/236] fix(security): make private S3 endpoints configurable for on-prem deployments The SSRF guard rejected any endpoint resolving to a non-public address, which broke the common on-prem deployment backed by an internal MinIO/ S3-compatible store on a LAN or the same host. Adds ALLOW_PRIVATE_S3_ENDPOINT (default true): private (RFC1918/ULA) and loopback endpoints are permitted by default so internal deployments work out of the box, and can be locked down to public-only with a strict mode (set false). The cloud metadata endpoint (link-local 169.254.0.0/16, fe80::/10), multicast, reserved and unspecified addresses are ALWAYS rejected regardless of the flag, so the highest-impact SSRF (IAM credential theft) stays closed even in the permissive default. Co-Authored-By: Claude Fable 5 --- .../application/command/datasource.py | 29 +++++++--- labelu/internal/common/config.py | 8 +++ .../application/command/test_datasource.py | 58 +++++++++++++++---- 3 files changed, 78 insertions(+), 17 deletions(-) diff --git a/labelu/internal/application/command/datasource.py b/labelu/internal/application/command/datasource.py index 0776a081..fd1a27ce 100644 --- a/labelu/internal/application/command/datasource.py +++ b/labelu/internal/application/command/datasource.py @@ -5,13 +5,20 @@ from pydantic import BaseModel, Field, field_validator, model_validator +from labelu.internal.common.config import settings + def validate_s3_endpoint(endpoint: Union[str, None]) -> Union[str, None]: """Reject S3 endpoints that could be abused for SSRF. - Only http(s) URLs are allowed, and the host must not resolve to a - loopback/private/link-local/reserved address (e.g. the cloud metadata - endpoint 169.254.169.254 or internal services). + Only http(s) URLs are allowed. Link-local (cloud metadata + 169.254.0.0/16, fe80::/10), multicast, reserved and unspecified + addresses are always rejected — they are never valid S3 endpoints. + + Private (RFC1918 / ULA) and loopback addresses are allowed when + ``settings.ALLOW_PRIVATE_S3_ENDPOINT`` is true (the default, for on-prem + deployments backed by an internal MinIO/S3-compatible store) and + rejected otherwise (strict mode). """ if endpoint is None or endpoint == "": return endpoint @@ -28,17 +35,25 @@ def validate_s3_endpoint(endpoint: Union[str, None]) -> Union[str, None]: except socket.gaierror: raise ValueError("endpoint host cannot be resolved") + allow_private = settings.ALLOW_PRIVATE_S3_ENDPOINT for info in infos: addr = ipaddress.ip_address(info[4][0]) + # Never a legitimate S3 endpoint; always rejected regardless of config. + # Note: the metadata IP (169.254.x) is also flagged is_private on newer + # Python, so link-local is checked here and must win over the private + # carve-out below. Loopback is exempt from the reserved check because + # ::1 is also flagged is_reserved. if ( - addr.is_private - or addr.is_loopback - or addr.is_link_local - or addr.is_reserved + addr.is_link_local or addr.is_multicast or addr.is_unspecified + or (addr.is_reserved and not addr.is_loopback) ): raise ValueError("endpoint host resolves to a disallowed address") + # Internal networks (RFC1918 / ULA / loopback): allowed only when + # explicitly permitted via ALLOW_PRIVATE_S3_ENDPOINT. + if (addr.is_private or addr.is_loopback) and not allow_private: + raise ValueError("endpoint host resolves to a private address") return endpoint diff --git a/labelu/internal/common/config.py b/labelu/internal/common/config.py index 553b5d5a..83094783 100644 --- a/labelu/internal/common/config.py +++ b/labelu/internal/common/config.py @@ -39,6 +39,14 @@ class Settings(BaseSettings): S3_PATH_STYLE: bool = False S3_USE_SSL: bool = True + # Allow user-configured data-source S3 endpoints to point at private + # networks (RFC1918 / ULA / loopback). Enabled by default for on-prem + # deployments that use an internal MinIO/S3-compatible store. Even when + # enabled, link-local (cloud metadata 169.254.0.0/16), multicast, + # reserved and unspecified addresses are always rejected. Set to false to + # restrict data-source endpoints to public hosts only (strict SSRF mode). + ALLOW_PRIVATE_S3_ENDPOINT: bool = True + AI_AUTO_LABEL_ENABLED: bool = False AI_PROVIDER: str = "local_http" AI_MODEL_ENDPOINT: str = "" diff --git a/labelu/tests/internal/application/command/test_datasource.py b/labelu/tests/internal/application/command/test_datasource.py index 5ce9116d..a2d0fdf7 100644 --- a/labelu/tests/internal/application/command/test_datasource.py +++ b/labelu/tests/internal/application/command/test_datasource.py @@ -1,6 +1,7 @@ import pytest from pydantic import ValidationError +from labelu.internal.common.config import settings from labelu.internal.application.command.datasource import ( CreateDataSourceCommand, UpdateDataSourceCommand, @@ -18,25 +19,20 @@ def _create(endpoint): class TestDataSourceEndpointSSRF: - """Regression tests: the S3 endpoint must not point at internal targets.""" + """Regression tests for the data-source S3 endpoint SSRF guard.""" @pytest.mark.parametrize( "endpoint", [ - "http://169.254.169.254", # cloud metadata (IMDS) + "http://169.254.169.254", # cloud metadata (IMDS) - always blocked "http://169.254.169.254/latest/", # IMDS with path - "http://127.0.0.1:9000", # loopback - "http://localhost:9000", # loopback by name - "http://10.1.2.3", # private range - "http://192.168.0.1", # private range - "http://172.16.5.5", # private range - "http://[::1]:9000", # IPv6 loopback + "http://0.0.0.0", # unspecified - always blocked "file:///etc/passwd", # non-http scheme "ftp://example.com", # non-http scheme - "http://0.0.0.0", # unspecified + "s3.example.com", # missing scheme ], ) - def test_rejects_internal_or_bad_scheme_endpoint(self, endpoint): + def test_always_rejects_metadata_and_bad_scheme(self, endpoint): with pytest.raises(ValidationError): _create(endpoint) with pytest.raises(ValidationError): @@ -53,3 +49,45 @@ def test_rejects_internal_or_bad_scheme_endpoint(self, endpoint): def test_allows_public_endpoint(self, endpoint): cmd = _create(endpoint) assert cmd.endpoint == endpoint + + @pytest.mark.parametrize( + "endpoint", + [ + "http://10.1.2.3:9000", # private RFC1918 + "http://192.168.0.10:9000", # private RFC1918 + "http://172.16.5.5:9000", # private RFC1918 + "http://127.0.0.1:9000", # loopback (same-host MinIO) + "http://[::1]:9000", # IPv6 loopback + "http://localhost:9000", # loopback by name (same-host MinIO) + ], + ) + def test_allows_private_endpoint_by_default(self, endpoint): + # ALLOW_PRIVATE_S3_ENDPOINT defaults to True for on-prem deployments + assert settings.ALLOW_PRIVATE_S3_ENDPOINT is True + cmd = _create(endpoint) + assert cmd.endpoint == endpoint + + @pytest.mark.parametrize( + "endpoint", + [ + "http://10.1.2.3:9000", + "http://192.168.0.10:9000", + "http://127.0.0.1:9000", + "http://localhost:9000", + ], + ) + def test_strict_mode_rejects_private(self, endpoint, monkeypatch): + monkeypatch.setattr(settings, "ALLOW_PRIVATE_S3_ENDPOINT", False) + with pytest.raises(ValidationError): + _create(endpoint) + + def test_strict_mode_still_allows_public(self, monkeypatch): + monkeypatch.setattr(settings, "ALLOW_PRIVATE_S3_ENDPOINT", False) + cmd = _create("https://8.8.8.8:9000") + assert cmd.endpoint == "https://8.8.8.8:9000" + + def test_metadata_blocked_even_when_private_allowed(self): + # the flag must not re-open the cloud metadata endpoint + assert settings.ALLOW_PRIVATE_S3_ENDPOINT is True + with pytest.raises(ValidationError): + _create("http://169.254.169.254") From aa018c1fffb8824d7c3584935ab83fab35817189 Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 14:01:46 +0800 Subject: [PATCH 234/236] fix(security): block RFC6598 shared address space in S3 endpoint (Alibaba/Tencent metadata SSRF) The SSRF guard's always-block set only covered link-local (169.254.0.0/16), so the Alibaba/Tencent cloud metadata service at 100.100.100.200 was reachable: 100.64.0.0/10 (RFC6598 CGNAT) is not flagged private/link-local/reserved by the stdlib, so it passed in BOTH the default and strict modes, allowing an authenticated user to steal instance RAM credentials via list_objects/import_from_s3. 100.64.0.0/10 is now always rejected regardless of ALLOW_PRIVATE_S3_ENDPOINT. Co-Authored-By: Claude Fable 5 --- .../internal/application/command/datasource.py | 18 +++++++++++++++++- .../application/command/test_datasource.py | 4 +++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/labelu/internal/application/command/datasource.py b/labelu/internal/application/command/datasource.py index fd1a27ce..6a1a927f 100644 --- a/labelu/internal/application/command/datasource.py +++ b/labelu/internal/application/command/datasource.py @@ -8,11 +8,22 @@ from labelu.internal.common.config import settings +# Networks that are never a valid S3 endpoint and must be blocked regardless of +# ALLOW_PRIVATE_S3_ENDPOINT. 100.64.0.0/10 is RFC6598 shared address space +# (Carrier-Grade NAT) and is NOT flagged private/link-local/reserved by the +# stdlib, yet it hosts the Alibaba/Tencent cloud metadata service +# (100.100.100.200) — so it must be rejected explicitly. +_ALWAYS_BLOCK_NETWORKS = ( + ipaddress.ip_network("100.64.0.0/10"), +) + + def validate_s3_endpoint(endpoint: Union[str, None]) -> Union[str, None]: """Reject S3 endpoints that could be abused for SSRF. Only http(s) URLs are allowed. Link-local (cloud metadata - 169.254.0.0/16, fe80::/10), multicast, reserved and unspecified + 169.254.0.0/16, fe80::/10), multicast, reserved, unspecified and RFC6598 + shared-address-space (100.64.0.0/10, incl. Alibaba/Tencent metadata) addresses are always rejected — they are never valid S3 endpoints. Private (RFC1918 / ULA) and loopback addresses are allowed when @@ -43,11 +54,16 @@ def validate_s3_endpoint(endpoint: Union[str, None]) -> Union[str, None]: # Python, so link-local is checked here and must win over the private # carve-out below. Loopback is exempt from the reserved check because # ::1 is also flagged is_reserved. + in_blocked_net = any( + addr.version == net.version and addr in net + for net in _ALWAYS_BLOCK_NETWORKS + ) if ( addr.is_link_local or addr.is_multicast or addr.is_unspecified or (addr.is_reserved and not addr.is_loopback) + or in_blocked_net ): raise ValueError("endpoint host resolves to a disallowed address") # Internal networks (RFC1918 / ULA / loopback): allowed only when diff --git a/labelu/tests/internal/application/command/test_datasource.py b/labelu/tests/internal/application/command/test_datasource.py index a2d0fdf7..58c22236 100644 --- a/labelu/tests/internal/application/command/test_datasource.py +++ b/labelu/tests/internal/application/command/test_datasource.py @@ -24,8 +24,10 @@ class TestDataSourceEndpointSSRF: @pytest.mark.parametrize( "endpoint", [ - "http://169.254.169.254", # cloud metadata (IMDS) - always blocked + "http://169.254.169.254", # AWS/GCP/Azure metadata (IMDS) - link-local "http://169.254.169.254/latest/", # IMDS with path + "http://100.100.100.200/latest/", # Alibaba/Tencent cloud metadata (100.64.0.0/10) + "http://100.64.0.1:9000", # RFC6598 shared address space "http://0.0.0.0", # unspecified - always blocked "file:///etc/passwd", # non-http scheme "ftp://example.com", # non-http scheme From a07af39566a74b2cbb7118d9f1d469d420a44c36 Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 14:03:38 +0800 Subject: [PATCH 235/236] fix(security): enforce task access on batch auto-label (IDOR) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_batch_job loaded the task and checked only existence and media type, with no ownership check — unlike the single-sample auto_label.create which already verifies owner/collaborator. Any authenticated user could start a batch auto-label job on any task_id, overwriting another user's AI pre-annotations (when AI_AUTO_LABEL_ENABLED is on). Adds assert_task_access(task, current_user) after the task lookup. Co-Authored-By: Claude Fable 5 --- .../application/service/auto_label.py | 3 +++ .../internal/adapter/routers/test_sample.py | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/labelu/internal/application/service/auto_label.py b/labelu/internal/application/service/auto_label.py index 51539d58..265858f5 100644 --- a/labelu/internal/application/service/auto_label.py +++ b/labelu/internal/application/service/auto_label.py @@ -9,6 +9,7 @@ from loguru import logger from sqlalchemy.orm import Session +from labelu.internal.application.service.access import assert_task_access from labelu.internal.adapter.persistence import crud_pre_annotation, crud_sample, crud_task from labelu.internal.adapter.persistence import crud_auto_label_job from labelu.internal.application.command.auto_label import AutoLabelCommand, BatchAutoLabelCommand @@ -361,6 +362,8 @@ async def create_batch_job( status_code=status.HTTP_404_NOT_FOUND, ) + assert_task_access(task, current_user) + if task.media_type != MediaType.IMAGE.value: raise LabelUException( code=ErrorCode.CODE_56001_AUTO_LABEL_UNSUPPORTED_MEDIA, diff --git a/labelu/tests/internal/adapter/routers/test_sample.py b/labelu/tests/internal/adapter/routers/test_sample.py index a3417f32..88d6e2d9 100644 --- a/labelu/tests/internal/adapter/routers/test_sample.py +++ b/labelu/tests/internal/adapter/routers/test_sample.py @@ -353,6 +353,28 @@ def test_sample_list_by_sort_error( # check assert r.status_code == 422 + def test_batch_auto_label_forbidden_for_non_owner( + self, client: TestClient, testuser_token_headers: dict, db: Session, monkeypatch + ) -> None: + # enable the AI feature so the request reaches the access check + monkeypatch.setattr(settings, "AI_AUTO_LABEL_ENABLED", True) + monkeypatch.setattr(settings, "AI_MODEL_ENDPOINT", "http://model.local") + # task owned by another user + with begin_transaction(db): + task = crud_task.create( + db=db, + task=Task( + name="other", description="d", tips="t", media_type="IMAGE", + created_by=999, updated_by=999, + ), + ) + r = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/auto_label_job", + headers=testuser_token_headers, + json={"filter_by_labels": True}, + ) + assert r.status_code == 403 + def test_sample_access_forbidden_for_non_member( self, client: TestClient, testuser_token_headers: dict, db: Session ) -> None: From 60a5adf1069003d173280ece8348ae65bea7ab92 Mon Sep 17 00:00:00 2001 From: shlab Date: Mon, 13 Jul 2026 14:07:19 +0800 Subject: [PATCH 236/236] test(security): verify a collaborator can still annotate a shared task Regression guard for the IDOR access checks: adds a second user as a task collaborator and asserts they can list, read and patch (annotate) samples in the shared task, confirming assert_task_access grants owner-or-collaborator access and does not lock collaborators out. Co-Authored-By: Claude Fable 5 --- .../internal/adapter/routers/test_sample.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/labelu/tests/internal/adapter/routers/test_sample.py b/labelu/tests/internal/adapter/routers/test_sample.py index 88d6e2d9..843479f3 100644 --- a/labelu/tests/internal/adapter/routers/test_sample.py +++ b/labelu/tests/internal/adapter/routers/test_sample.py @@ -375,6 +375,69 @@ def test_batch_auto_label_forbidden_for_non_owner( ) assert r.status_code == 403 + def test_collaborator_can_annotate_shared_task( + self, client: TestClient, testuser_token_headers: dict, db: Session + ) -> None: + # owner = the default test user + owner = crud_user.get_user_by_username(db=db, username="test@example.com") + + # a second user, added as collaborator, with a real login token + collab_username = "collab_annotator@example.com" + client.post( + f"{settings.API_V1_STR}/users/signup", + json={"username": collab_username, "password": "collab@123"}, + ) + login = client.post( + f"{settings.API_V1_STR}/users/login", + json={"username": collab_username, "password": "collab@123"}, + ) + collab_headers = {"Authorization": login.json()["data"]["token"]} + collab_user = crud_user.get_user_by_username(db=db, username=collab_username) + + with begin_transaction(db): + task = crud_task.create( + db=db, + task=Task( + name="shared", description="d", tips="t", + created_by=owner.id, updated_by=owner.id, + ), + ) + with begin_transaction(db): + samples = crud_sample.batch( + db=db, + samples=[TaskSample( + task_id=task.id, file_id=1, + created_by=owner.id, updated_by=owner.id, data="{}", + )], + ) + sid = samples[0].id + + # owner adds the collaborator + r = client.post( + f"{settings.API_V1_STR}/tasks/{task.id}/collaborators", + headers=testuser_token_headers, + json={"user_id": collab_user.id}, + ) + assert r.status_code == 201 + + # the collaborator can list, read and annotate samples in the shared task + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/samples", + headers=collab_headers, params={"page": 0, "size": 10}, + ) + assert r.status_code == 200 + r = client.get( + f"{settings.API_V1_STR}/tasks/{task.id}/samples/{sid}", + headers=collab_headers, + ) + assert r.status_code == 200 + r = client.patch( + f"{settings.API_V1_STR}/tasks/{task.id}/samples/{sid}", + headers=collab_headers, + json={"data": {"k": "v"}, "annotated_count": 1}, + ) + assert r.status_code == 200 + def test_sample_access_forbidden_for_non_member( self, client: TestClient, testuser_token_headers: dict, db: Session ) -> None: