From b8691759573c1c00f195c2ddf37c63f12036bae9 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Thu, 13 Jun 2024 12:07:38 +0200 Subject: [PATCH 01/13] fix typos --- iop4lib/config.example.yaml | 6 +++--- tests/test_caha_cafos.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/iop4lib/config.example.yaml b/iop4lib/config.example.yaml index 5ad196fb..f87909a3 100644 --- a/iop4lib/config.example.yaml +++ b/iop4lib/config.example.yaml @@ -8,11 +8,11 @@ datadir: ~/.iop4data/ # Path to iop4data data folder. set_rawdata_readonly: False # True / False (sets raw fits file to readonly when downloading them or creating a RawFit object). db_path: ~/.iop4data/iop4.db # Path to iop4 sqlite database file. -astrometry_cache_path: ~/.astrometry_cache/ # Path to store the astromery index files. +astrometry_cache_path: ~/.astrometry_cache/ # Path to store the astrometry index files. nthreads: 4 # Number of threads / processes to use (e.g. 4). astrometry_timeout: 20 # Timeout in minutes for astrometry solving. astrometry_allsky_allow: false # Whether to allow all sky searches in the astrometry.net solver. If there are many all-sky images and nthreads is too high, it might cause very high memory consumption. -astrometry_allsky_septhreshold: 25 # Threshold of detected poinint mismatch to trigger an all-sky search. +astrometry_allsky_septhreshold: 25 # Threshold of detected pointing mismatch to trigger an all-sky search. ################ ### GRAPHICS ### @@ -54,7 +54,7 @@ osn_t150_password: null # osn_source_list_path: null # -# List of (case insentitive) regex patterns. +# List of (case insensitive) regex patterns. # Files matching any of these patterns will be downloaded from OSN. # For example, (^BLLac.*\.fits?$) will match file names starting with BLLac # and ending in .fit, .fits, or .fts. You can include your initials here, diff --git a/tests/test_caha_cafos.py b/tests/test_caha_cafos.py index eef43305..76a5f670 100644 --- a/tests/test_caha_cafos.py +++ b/tests/test_caha_cafos.py @@ -25,9 +25,9 @@ def test_build_single_proc(load_test_catalog): """ Test the whole building process of reduced fits in a single process This test is not really necessary since single-process reduction is already tested - in OSN-T090, therefore it is skipped by default. However CI actions at the momemnt + in OSN-T090, therefore it is skipped by default. However CI actions at the moment will fail without it, because if the astrometry index files are accessed by multiple - processes at the same time before they are catched httpdirfs will fail. To run it + processes at the same time before they are cached, httpdirfs will fail. To run it locally set the environment variable CI=true. """ @@ -61,7 +61,7 @@ def test_build_multi_proc_photopol(load_test_catalog): """ Test the whole building process of reduced fits through multiprocessing Also tests here relative photometry and polarimetry results and their - quality (value + uncertainties) (to avoud losing time reducing them + quality (value + uncertainties) (to avoid losing time reducing them in another test function). """ From 591b2ecf098bc365bf2bcf28b393ccd934a037f2 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Thu, 13 Jun 2024 12:08:03 +0200 Subject: [PATCH 02/13] add dependanbot for keeping CI up to date --- .github/.dependabot.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/.dependabot.yml diff --git a/.github/.dependabot.yml b/.github/.dependabot.yml new file mode 100644 index 00000000..74cb56ce --- /dev/null +++ b/.github/.dependabot.yml @@ -0,0 +1,7 @@ +# Maintain dependencies for GitHub Actions +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" \ No newline at end of file From 917285935e2924b4e5938fdf7ee41faab59b8ed6 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Thu, 13 Jun 2024 12:08:39 +0200 Subject: [PATCH 03/13] update versions in CI --- .github/workflows/docs.yml | 4 ++-- .github/workflows/release-drafter.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2285e278..6890a58b 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -37,9 +37,9 @@ jobs: run: git fetch --tags --force origin - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.10.9" + python-version: "3.11" - name: Update pip run: | diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index 201879c1..00aad6cc 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -15,6 +15,6 @@ jobs: runs-on: ubuntu-latest steps: # Drafts your next Release notes as Pull Requests are merged into "main" - - uses: release-drafter/release-drafter@v5 + - uses: release-drafter/release-drafter@v6 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 8a2545dc48bf5fe29b012eb40054fec6a1c5cbc5 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Thu, 13 Jun 2024 12:09:04 +0200 Subject: [PATCH 04/13] update action versions in CI --- .github/workflows/deploy_pypi.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_pypi.yml b/.github/workflows/deploy_pypi.yml index e720a95f..2d75905d 100644 --- a/.github/workflows/deploy_pypi.yml +++ b/.github/workflows/deploy_pypi.yml @@ -14,9 +14,9 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.10.9 + python-version: 3.11 - name: Install dependencies run: | From 1e4cad0a5b30dd6fec410515abb2d6371d367c6b Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Thu, 13 Jun 2024 12:09:18 +0200 Subject: [PATCH 05/13] update CI versions and upload coverage report --- .github/workflows/ci.yml | 59 ++++++++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c0388f7..d9bf4a08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,23 +31,23 @@ jobs: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.10.9" + python-version: "3.11" tests: runs-on: [self-hosted, x64, Linux, ubuntu, generic] strategy: max-parallel: 1 matrix: - python-version: ["3.10.9"] + python-version: ["3.11"] steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -83,9 +83,52 @@ jobs: ls -lh $HOME/.astrometry_cache/5200/index-5200-00.fits ls -lh $HOME/iop4testdata.$TESTDATA_MD5SUM.tar.gz - - name: Run tests (with -o log_cli=true -o log_cli_level=DEBUG to debug CI actions) - run: pytest -o log_cli=true -o log_cli_level=DEBUG -vxs tests + - name: Run tests with coverage report (with -o log_cli=true -o log_cli_level=DEBUG to debug CI actions) + run: pytest -o log_cli=true -o log_cli_level=DEBUG -vxs --cov=iop4lib tests + + - name: Upload coverage data + uses: actions/upload-artifact@v4 + with: + name: coverage-data-${{ matrix.python-version }} + path: .coverage.* + if-no-files-found: ignore + +coverage: + name: Combine & check coverage + if: always() + needs: tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - uses: actions/download-artifact@v4 + with: + pattern: coverage-data-* + merge-multiple: true + + - name: Combine coverage & fail if it's <100% + run: + python -Im pip install --upgrade coverage[toml]| + + python -Im coverage combine + python -Im coverage html --skip-covered --skip-empty + + # Report and write to summary. + python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY + + # Report again and fail if under 100%. + python -Im coverage report --fail-under=100 + + - name: Upload HTML report if check failed + uses: actions/upload-artifact@v4 + with: + name: html-report + path: htmlcov + if: ${{ failure() }} docs: runs-on: [self-hosted, x64, Linux, ubuntu, generic] @@ -98,9 +141,9 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.10.9" + python-version: "3.11" - name: Update pip run: | From 54169173e34fdced5798e6dcc2650eaa7512d0bc Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Thu, 13 Jun 2024 12:26:15 +0200 Subject: [PATCH 06/13] add ref for coverage action and fix syntax yml --- .github/workflows/ci.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9bf4a08..e23f8cd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,7 @@ jobs: path: .coverage.* if-no-files-found: ignore +# From Hynek Schlawack (https://hynek.me/articles/ditch-codecov-python/) coverage: name: Combine & check coverage if: always() @@ -111,19 +112,14 @@ coverage: merge-multiple: true - name: Combine coverage & fail if it's <100% - run: - python -Im pip install --upgrade coverage[toml]| - + run: | + python -Im pip install --upgrade coverage[toml] python -Im coverage combine python -Im coverage html --skip-covered --skip-empty - - # Report and write to summary. python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY - - # Report again and fail if under 100%. python -Im coverage report --fail-under=100 - - name: Upload HTML report if check failed + - name: Upload HTML report if check failed uses: actions/upload-artifact@v4 with: name: html-report From 9cd0f168927d8e166c6c6591ab577ff6d1829b71 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Thu, 13 Jun 2024 13:12:10 +0200 Subject: [PATCH 07/13] try ti fix the worng syntax --- .github/workflows/ci.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e23f8cd0..3eeb5e71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,9 +94,8 @@ jobs: if-no-files-found: ignore # From Hynek Schlawack (https://hynek.me/articles/ditch-codecov-python/) -coverage: + coverage: name: Combine & check coverage - if: always() needs: tests runs-on: ubuntu-latest @@ -105,27 +104,31 @@ coverage: - uses: actions/setup-python@v5 with: python-version: "3.11" - + - run: python -Im pip install --upgrade coverage[toml] - uses: actions/download-artifact@v4 with: pattern: coverage-data-* merge-multiple: true - - name: Combine coverage & fail if it's <100% + - name: Combine coverage & fail if it's <100%. run: | - python -Im pip install --upgrade coverage[toml] python -Im coverage combine python -Im coverage html --skip-covered --skip-empty + + # Report and write to summary. python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY + + # Report again and fail if under 100%. python -Im coverage report --fail-under=100 - - name: Upload HTML report if check failed + - name: Upload HTML report if check failed. uses: actions/upload-artifact@v4 with: name: html-report path: htmlcov if: ${{ failure() }} + docs: runs-on: [self-hosted, x64, Linux, ubuntu, generic] # Don't run if this is a tag push, already done during docs deployment (docs.yml) From 3c448f81313676e429b12eac81076b2e145c6aa3 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Thu, 13 Jun 2024 15:16:48 +0200 Subject: [PATCH 08/13] check if failure related to not having 100% cov --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3eeb5e71..2c04b774 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,14 +103,14 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - run: python -Im pip install --upgrade coverage[toml] - uses: actions/download-artifact@v4 with: pattern: coverage-data-* merge-multiple: true - - name: Combine coverage & fail if it's <100%. + - name: Combine coverage & fail if it's <60%. run: | python -Im coverage combine python -Im coverage html --skip-covered --skip-empty @@ -118,8 +118,8 @@ jobs: # Report and write to summary. python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY - # Report again and fail if under 100%. - python -Im coverage report --fail-under=100 + # Report again and fail if under 60%. + python -Im coverage report --fail-under=60 - name: Upload HTML report if check failed. uses: actions/upload-artifact@v4 From a215de6be09da12a221dbedf102f783eae14f8d0 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 18 Jun 2024 10:32:23 +0200 Subject: [PATCH 09/13] Do not combine coverage reports and do not fail with coverage below value --- .github/workflows/ci.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c04b774..c9871c53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,7 +95,7 @@ jobs: # From Hynek Schlawack (https://hynek.me/articles/ditch-codecov-python/) coverage: - name: Combine & check coverage + name: Check coverage needs: tests runs-on: ubuntu-latest @@ -107,12 +107,11 @@ jobs: - run: python -Im pip install --upgrade coverage[toml] - uses: actions/download-artifact@v4 with: - pattern: coverage-data-* + pattern: .coverage.* merge-multiple: true - - name: Combine coverage & fail if it's <60%. + - name: Coverage report in html run: | - python -Im coverage combine python -Im coverage html --skip-covered --skip-empty # Report and write to summary. From a6d90a56893662fb9f4818fa3ac1233db5cd0e5d Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 18 Jun 2024 10:53:33 +0200 Subject: [PATCH 10/13] set coverage file correctly when downloading it --- .github/workflows/ci.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c9871c53..500dd306 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,7 +91,7 @@ jobs: with: name: coverage-data-${{ matrix.python-version }} path: .coverage.* - if-no-files-found: ignore + # From Hynek Schlawack (https://hynek.me/articles/ditch-codecov-python/) coverage: @@ -107,8 +107,7 @@ jobs: - run: python -Im pip install --upgrade coverage[toml] - uses: actions/download-artifact@v4 with: - pattern: .coverage.* - merge-multiple: true + pattern: coverage-data-* - name: Coverage report in html run: | @@ -117,9 +116,6 @@ jobs: # Report and write to summary. python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY - # Report again and fail if under 60%. - python -Im coverage report --fail-under=60 - - name: Upload HTML report if check failed. uses: actions/upload-artifact@v4 with: From 1d1c2383a0dc4fd6fe6af1b42d6ab5a0243f6eaa Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 18 Jun 2024 11:41:10 +0200 Subject: [PATCH 11/13] integrate coverage report in tests --- .github/workflows/ci.yml | 32 +++++--------------------------- 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 500dd306..442574ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,8 +60,10 @@ jobs: pip install build python -m build - - name: Install the package - run: pip install .[test] + - name: Install the package and coverage library + run: | + pip install .[test] + python -Im pip install --upgrade coverage[toml] - name: Install the package in editable mode with all additional dependencies run: pip install --editable .[all] @@ -86,33 +88,9 @@ jobs: - name: Run tests with coverage report (with -o log_cli=true -o log_cli_level=DEBUG to debug CI actions) run: pytest -o log_cli=true -o log_cli_level=DEBUG -vxs --cov=iop4lib tests - - name: Upload coverage data - uses: actions/upload-artifact@v4 - with: - name: coverage-data-${{ matrix.python-version }} - path: .coverage.* - - -# From Hynek Schlawack (https://hynek.me/articles/ditch-codecov-python/) - coverage: - name: Check coverage - needs: tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - run: python -Im pip install --upgrade coverage[toml] - - uses: actions/download-artifact@v4 - with: - pattern: coverage-data-* - - - name: Coverage report in html + - name: Create coverage report in html and markdown run: | python -Im coverage html --skip-covered --skip-empty - # Report and write to summary. python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY From 1a4a266d596a6bea742022a7fbc9b83d109ffb00 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 25 Jun 2024 14:16:39 +0200 Subject: [PATCH 12/13] add coverage comment action --- .github/workflows/ci.yml | 24 +++++++++++++++++++---- .github/workflows/coverage.yml | 36 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 442574ea..a87ad87c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,14 @@ jobs: tests: runs-on: [self-hosted, x64, Linux, ubuntu, generic] + permissions: + # Gives the action the necessary permissions for publishing new + # comments in pull requests. + pull-requests: write + # Gives the action the necessary permissions for pushing data to the + # python-coverage-comment-action branch, and for editing existing + # comments (to avoid publishing multiple comments in the same PR) + contents: write strategy: max-parallel: 1 matrix: @@ -94,12 +102,20 @@ jobs: # Report and write to summary. python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY - - name: Upload HTML report if check failed. + - name: Coverage comment + id: coverage_comment + uses: py-cov-action/python-coverage-comment-action@v3 + with: + GITHUB_TOKEN: ${{ github.token }} + + - name: Store Pull Request comment to be posted uses: actions/upload-artifact@v4 + if: steps.coverage_comment.outputs.COMMENT_FILE_WRITTEN == 'true' with: - name: html-report - path: htmlcov - if: ${{ failure() }} + # If you use a different name, update COMMENT_ARTIFACT_NAME accordingly + name: python-coverage-comment-action + # If you use a different name, update COMMENT_FILENAME accordingly + path: python-coverage-comment-action.txt docs: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000..26a6ff93 --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,36 @@ +# .github/workflows/coverage.yml +name: Post coverage comment + +on: + workflow_run: + workflows: ["CI"] + types: + - completed + +jobs: + test: + name: Run tests & display coverage + runs-on: ubuntu-latest + if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' + permissions: + # Gives the action the necessary permissions for publishing new + # comments in pull requests. + pull-requests: write + # Gives the action the necessary permissions for editing existing + # comments (to avoid publishing multiple comments in the same PR) + contents: write + # Gives the action the necessary permissions for looking up the + # workflow that launched this workflow, and download the related + # artifact that contains the comment to be published + actions: read + steps: + # DO NOT run actions/checkout here, for security reasons + # For details, refer to https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ + - name: Post comment + uses: py-cov-action/python-coverage-comment-action@v3 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR_RUN_ID: ${{ github.event.workflow_run.id }} + # Update those if you changed the default values: + # COMMENT_ARTIFACT_NAME: python-coverage-comment-action + # COMMENT_FILENAME: python-coverage-comment-action.txt \ No newline at end of file From ec8c80f50041501910e582c7a666d2aaaa5f7d78 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 25 Jun 2024 15:27:12 +0200 Subject: [PATCH 13/13] rm old md conversion and bump comment action version --- .github/workflows/ci.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a87ad87c..d8a11de4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,16 +95,10 @@ jobs: - name: Run tests with coverage report (with -o log_cli=true -o log_cli_level=DEBUG to debug CI actions) run: pytest -o log_cli=true -o log_cli_level=DEBUG -vxs --cov=iop4lib tests - - - name: Create coverage report in html and markdown - run: | - python -Im coverage html --skip-covered --skip-empty - # Report and write to summary. - python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY - name: Coverage comment id: coverage_comment - uses: py-cov-action/python-coverage-comment-action@v3 + uses: py-cov-action/python-coverage-comment-action@v3.24 with: GITHUB_TOKEN: ${{ github.token }}