Myloasm stefanfixes

.github/workflows/deploy-docs.yaml

@@ -1,35 +1,50 @@
-name: Deploy documentation site
-
-on:
-  push:
-    branches:
-      - main
-    paths:
-      - 'docs/**'
-      - 'doctave.yml'
-  workflow_dispatch:
-
-jobs:
-  build:
-    name: Deploys
-    runs-on: ubuntu-latest
-    steps:
-    - name: 'Checkout GitHub Action'
-      uses: actions/checkout@main
-
-    - name: 'Setup homebrew'
-      uses: Homebrew/actions/setup-homebrew@master
-
-    - name: 'Build and deploy'
-      shell: bash
-      run: |
-        brew install doctave/doctave/doctave
-        doctave build --release --allow-failed-checks
-
-    - name: GitHub Pages
-      if: github.ref == 'refs/heads/main'
-      uses: crazy-max/ghaction-github-pages@v3.1.0
-      with:
-        build_dir: site/
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+name: Deploy documentation site
+
+on:
+  push:
+    branches:
+      - main
+      - update_doc
+    paths:
+      - 'docs/**'
+      - 'doctave.yaml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    name: Build docs
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+
+    - name: Setup homebrew
+      uses: Homebrew/actions/setup-homebrew@master
+
+    - name: Build with doctave
+      run: |
+        brew install doctave/doctave/doctave
+        doctave build --release --allow-failed-checks
+        touch site/.nojekyll
+
+    - name: Upload Pages artifact
+      uses: actions/upload-pages-artifact@v3
+      with:
+        path: site/
+
+  deploy:
+    name: Deploy to GitHub Pages
+    runs-on: ubuntu-latest
+    needs: build
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+    - name: Deploy to GitHub Pages
+      id: deployment
+      uses: actions/deploy-pages@v4

.github/workflows/python-publish.yml

@@ -20,7 +20,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Set env
-      run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      run: echo "RELEASE_VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_ENV
     - name: Set up Python 3.10
       uses: actions/setup-python@v3
       with:
@@ -34,21 +34,9 @@ jobs:
     - name: Build a binary wheel and a source tarball
       run: >-
         python setup.py sdist
-    - name: Move wheel to dist
-      run: mv dist/aviary-genome-*.tar.gz dist/aviary_genome-${{ env.RELEASE_VERSION }}.tar.gz
     - name: Publish distribution 📦 to PyPI
       if: startsWith(github.ref, 'refs/tags')
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         password: ${{ secrets.PYPI_API_TOKEN }}
-
-    - name: Create a Release
-      if: github.actor != 'dependabot[bot]'
-      uses: "marvinpinto/action-automatic-releases@latest"
-      with:
-        repo_token: "${{ secrets.GITHUB_TOKEN }}"
-        prerelease: false
-        automatic_release_tag: "${{ env.RELEASE_VERSION }}"
-        title: "${{ env.RELEASE_VERSION }}"
-        files: |
-          dist/aviary_genome-${{ env.RELEASE_VERSION }}.tar.gz
+        skip_existing: true

.gitignore

@@ -23,6 +23,5 @@ logs
 db
 mqsub.out
 test/qsub-test-log.txt
-aviary_tests_samples/*
-
-
+test_semibin_multi/*
+test_semibin_multi.sh

CHANGELOG.md

@@ -6,6 +6,13 @@ Forked from [wwood/aviary](https://github.com/wwood/aviary) at v0.12.0 (`myloasm
 
 ---
 
+### Changed
+
+- **SingleM updated to v0.21.3** — minimum version bumped from 0.20.3 to 0.21.3; `singlem-appraise` environment unpinned from 0.19.0 now that the v0.20 performance regression for `--genome-fasta-files` input is fixed in v0.21
+- **GTDB-Tk updated to v2.7.2** — minimum version bumped from 2.6.1 to 2.7.2. v2.7+ removes the `--skip_ani_screen` flag; replaced by `--place_species`
+- **Database updated to GTDB R232** — SingleM metapackage updated to `S6.5.0.GTDB_r232` and GTDB-Tk database updated to `release232`; download URL in `aviary configure --download` updated accordingly
+- **Benchmark added to `singlem_appraise` rule** — runtime now recorded in `benchmarks/singlem_appraise.benchmark.txt`
+
 ### Added
 
 #### Web Interface (`aviary/web/`) — experimental
@@ -59,9 +66,10 @@ Then open `http://localhost:8090` in your browser. Includes a pipeline monitor f
 - Removed unnecessary `--no-assign-taxonomy` flag from SingleM commands
 - Handle `"none"` input for read lists in `ReadContainer` initialisation
 - Added scratch directory to gtdbtk rule for better temporary file and memory handling
+- Removed `--skip_ani_screen` flag in gtdbtk rule (removed in GTDB-Tk v2.7.0)
 - Added sleep delays between GPU test submissions to prevent resource contention
 - Increased memory allocation for GPU and expensive tests in mqsub commands
-- Fixed isolate functionality, with updated medaka and dnaapler 
+- Fixed isolate functionality, with medaka updated to `>=2.2.1` (previously restricted to `<2.1.0` due to [nanoporetech/medaka#566](https://github.com/nanoporetech/medaka/issues/566), fixed in 2.2.x) and dnaapler updated to `>=1.0.0`
 
 ---
 

CITATION.cff

@@ -20,6 +20,31 @@ authors:
     given-names: Ben J.
     orcid: https://orcid.org/0000-0003-0670-7480
 title: "Aviary: Hybrid assembly and genome recovery from metagenomes with Aviary"
-version: 0.9.0
-doi: 10.5281/zenodo.10806928
+version: 0.13.0
+doi: 10.5281/zenodo.10158086
 date-released: 2024-03-12
+preferred-citation:
+  type: article
+  authors:
+    - family-names: Newell
+      given-names: Rhys J. P.
+      orcid: https://orcid.org/0000-0002-1300-6116
+    - family-names: Aroney
+      given-names: Samuel T. N.
+      orcid: https://orcid.org/0000-0001-9806-5846
+    - family-names: Zaugg
+      given-names: Julian
+      orcid: https://orcid.org/0000-0002-4919-1448
+    - family-names: Sternes
+      given-names: Peter
+      orcid: https://orcid.org/0000-0002-4456-150X
+    - family-names: Tyson
+      given-names: Gene W.
+      orcid: https://orcid.org/0000-0001-8559-9427
+    - family-names: Woodcroft
+      given-names: Ben J.
+      orcid: https://orcid.org/0000-0003-0670-7480
+  title: "Aviary: Hybrid assembly and genome recovery from metagenomes"
+  doi: 10.5281/zenodo.10158086
+  journal: "Zenodo"
+

README.md

@@ -117,8 +117,8 @@ lrwxrwxrwx - woodcrob 23 Apr 07:56 2.1.3 -> /mnt/hpccs01/work/microbiome/db/eggn
 lrwxrwxrwx - woodcrob 23 Apr 07:55 2015_01_16_v2 -> /work/microbiome/db/checkm/2015_01_16_v2
 lrwxrwxrwx - woodcrob 23 Apr 07:54 CheckM2_database -> /work/microbiome/db/CheckM2_database/uniref100.KO.1.dmnd
 lrwxrwxrwx - woodcrob 23 Apr 07:57 2024-3-28-GTDB214.1+humanT2T -> /work/microbiome/db/metabuli/2024-3-28-GTDB214.1+humanT2T/
-lrwxrwxrwx - woodcrob 23 Apr 07:56 release226 -> /work/microbiome/db/gtdb/gtdb_release226/auxillary_files/gtdbtk_package/full_package/release226
-lrwxrwxrwx - woodcrob 23 Apr 07:55 S5.4.0.GTDB_r226.metapackage_20250331.smpkg.zb -> /work/microbiome/db/singlem/S5.4.0.GTDB_r226.metapackage_20250331.smpkg.zb
+lrwxrwxrwx - woodcrob 23 Apr 07:56 release232 -> /work/microbiome/db/gtdb/gtdb_release232/auxillary_files/gtdbtk_package/full_package/release232
+lrwxrwxrwx - woodcrob 23 Apr 07:55 S6.5.0.GTDB_r232.metapackage_20260319.smpkg.zb -> /work/microbiome/db/singlem/S6.5.0.GTDB_r232.metapackage_20260319.smpkg.zb
 ```
 
 To check the expected database symlink names, see `admin/set_env_vars.sh` in the
@@ -198,7 +198,7 @@ use the `aviary configure` module to update the environment variables used by av
 These environment variables can also be configured manually, just set the following variables in your `.bashrc` file:
 
 ```bash
-export GTDBTK_DATA_PATH=/path/to/gtdb/gtdb_release226/db/ # https://gtdb.ecogenomic.org/downloads
+export GTDBTK_DATA_PATH=/path/to/gtdb/gtdb_release232/db/ # https://gtdb.ecogenomic.org/downloads
 export EGGNOG_DATA_DIR=/path/to/eggnog-mapper/2.1.8/ # https://github.com/eggnogdb/eggnog-mapper/wiki/eggNOG-mapper-v2.1.5-to-v2.1.8#setup
 export SINGLEM_METAPACKAGE_PATH=/path/to/singlem_metapackage.smpkg/
 export CHECKM2DB=/path/to/checkm2db/uniref100.KO.1.dmnd

admin/build_dep_defs_from_pixi.py

@@ -1,91 +1,15 @@
 #!/usr/bin/env python3
-
-import yaml
-import toml
-import subprocess
-import json
 import os
+from bird_tool_utils.pixi_deps import build_dep_defs
 
 base_dir = os.path.join(os.path.dirname(__file__), "..")
 
-# Load explicitly listed dependencies from pixi.toml
-with open(os.path.join(base_dir, "aviary", "pixi.lock"), "rb") as f:
-    lock_data = yaml.safe_load(f)
-
-# Load explicitly listed dependencies from pixi.toml
-with open(os.path.join(base_dir, "aviary", "pixi.toml"), "r") as f:
-    pixi_data = toml.load(f)
-
-# Gather conda packages and versions from "pixi list --json"
-result = subprocess.run(
-    ["pixi", "list", "--json"],
-    capture_output=True, text=True, check=True
+build_dep_defs(
+    pixi_toml_path=os.path.join(base_dir, "aviary", "pixi.toml"),
+    pixi_lock_path=os.path.join(base_dir, "aviary", "pixi.lock"),
+    environment_yml_path=os.path.join(base_dir, "admin", "environment.yml"),
+    requirements_txt_path=os.path.join(base_dir, "admin", "requirements.txt"),
+    conda_to_pip_name={"bird_tool_utils_python": "bird_tool_utils"},
+    project_name="aviary-genome",  # conda name differs from workspace name "aviary"
+    environment="dev",             # query the dev env for version pins
 )
-# Parse the JSON output
-pixi_list = json.loads(result.stdout)
-# convert to name -> version
-# exclude editable packages because that is this project
-pixi_list = {pkg["name"].lower(): pkg["version"] for pkg in pixi_list if 'is_editable' not in pkg or pkg['is_editable'] is not True}
-print("Found", len(pixi_list), "pixi packages from pixi_list e.g.", list(pixi_list.keys())[0])
-
-# Write requirements.txt via 'pip list --format json'
-result = subprocess.run(
-    ["pixi", "run", "pip", "list", "--format", "json"],
-    capture_output=True, text=True, check=True
-)
-# Parse the JSON output
-pip_list = json.loads(result.stdout)
-
-# Explicitly declared packages in pixi.toml
-declared_conda = set()
-to_parse = pixi_data['feature']['main']['dependencies']
-for pkg, val in to_parse.items():
-    pkg_lower = pkg.lower()
-    declared_conda.add(pkg_lower)
-print("Found", len(declared_conda), "declared conda packages from pixi.toml e.g.", list(declared_conda)[0])
-
-# Prepare environment.yml data
-conda_deps = [
-    f"{pkg}={pixi_list[pkg]}"
-    for pkg in declared_conda if pkg in pixi_list
-]
-conda_deps = sorted(conda_deps)
-env_data = {
-    "channels": ["conda-forge", "bioconda"],
-    "dependencies": conda_deps,
-}
-
-# Write environment.yml
-with open(os.path.join(base_dir, "..", "environment.yml"), "w") as f:
-    f.write("# This file was autogenerated by build_dep_defs_from_pixi.py - do not change manually.\n")
-    yaml.dump(env_data, f, default_flow_style=False)
-
-print("environment.yml written, with ", len(conda_deps), "dependencies.")
-
-# Find the name of the current project to exclude from requirements.txt
-# Set manually because the name of the project on conda is different to on pypi.
-project_name = "aviary-genome" #pixi_data.get("workspace", {}).get("name", "").lower()
-if not project_name:
-    raise ValueError("Project name not found in pixi.toml.")
-
-print("Excluding project name from requirements.txt:", project_name)
-
-# Write requirements.txt
-pip_count = 0
-with open(os.path.join(base_dir, "admin", "requirements.txt"), "w") as f:
-    f.write("# This file was autogenerated by build_dep_defs_from_pixi.py - do not change manually.\n")
-    for package in pip_list:
-        # only include if the package is specified in pixi.toml
-        if package["name"].lower() in declared_conda:
-            # Skip the project itself
-            if package["name"].lower() == project_name:
-                continue
-            pip_count += 1
-            # Sometimes the version is set to 0.0.0 - unsure why - for these ones just set '*'
-            if package["version"] == "0.0.0":
-                line = f"{package['name']}\n"
-            else:
-                # If the version is not 0.0.0, write it as well
-                line = f"{package['name']}=={package['version']}\n"
-            f.write(line)
-print("requirements.txt written, with ", pip_count, "dependencies.")

admin/environment.yml

@@ -0,0 +1,20 @@
+# This file was autogenerated by build_dep_defs_from_pixi.py - do not change manually.
+channels:
+- conda-forge
+- bioconda
+dependencies:
+- biopython=1.86
+- bird_tool_utils_python=0.6.0
+- extern=0.4.1
+- numpy=2.4.1
+- pandas=3.0.0
+- parallel=20251122
+- pigz=2.8
+- pip=25.3
+- pixi=0.63.2
+- polars=1.38.0
+- pyopenssl=25.3.0
+- python=3.13.11
+- ruamel.yaml=0.19.1
+- setuptools=80.10.1
+- snakemake=9.15.0

admin/requirements.txt

@@ -1,5 +1,6 @@
 # This file was autogenerated by build_dep_defs_from_pixi.py - do not change manually.
 biopython==1.86
+bird_tool_utils==0.6.0
 extern==0.4.1
 numpy==2.4.1
 pandas==3.0.0
@@ -8,4 +9,4 @@ polars==1.38.0
 pyOpenSSL==25.3.0
 ruamel.yaml==0.19.1
 setuptools==80.10.1
-snakemake==9.15.0
+snakemake

admin/set_env_vars.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 
-SINGLEM_METAPACKAGE_PATH_RELATIVE="$PIXI_PROJECT_ROOT/db/S5.4.0.GTDB_r226.metapackage_20250331.smpkg.zb"
+SINGLEM_METAPACKAGE_PATH_RELATIVE="$PIXI_PROJECT_ROOT/db/S6.5.0.GTDB_r232.metapackage_20260319.smpkg.zb"
 CHECKM_DATA_PATH_RELATIVE="$PIXI_PROJECT_ROOT/db/2015_01_16_v2"
-GTDBTK_DATA_PATH_RELATIVE="$PIXI_PROJECT_ROOT/db/release226"
+GTDBTK_DATA_PATH_RELATIVE="$PIXI_PROJECT_ROOT/db/release232"
 CHECKM2DB_RELATIVE="$PIXI_PROJECT_ROOT/db/CheckM2_database"
 EGGNOG_DATA_DIR_RELATIVE="$PIXI_PROJECT_ROOT/db/2.1.3"
 METABULI_DB_PATH_RELATIVE="$PIXI_PROJECT_ROOT/db/2024-3-28-GTDB214.1+humanT2T"

aviary/aviary.py

@@ -695,7 +695,7 @@ def main():
     binning_options.add_argument(
         '--skip-binners', '--skip_binners', '--skip_binner', '--skip-binner',
         help='Optional list of binning algorithms to skip. Can be any combination of: \n'
-             'rosella, semibin, metabat1, metabat2, metabat, vamb, quickbin \n'
+             'rosella, semibin, metabat1, metabat2, metabat, vamb \n'
              'N.B. specifying "metabat" will skip both MetaBAT1 and MetaBAT2. \n',
         dest='skip_binners',
         nargs='*',

aviary/modules/annotation/annotation.smk

@@ -89,7 +89,7 @@ rule download_gtdb:
         'mkdir -p {params.gtdbtk_folder}; '
         # Configuration
         # NOTE: Updating the database version here might require an update to the GTDB-Tk version in pixi.toml
-        'DB_URL="https://data.gtdb.ecogenomic.org/releases/release226/226.0/auxillary_files/gtdbtk_package/full_package/gtdbtk_r226_data.tar.gz"; '
+        'DB_URL="https://data.gtdb.ecogenomic.org/releases/release232/232.0/auxillary_files/gtdbtk_package/full_package/gtdbtk_r232_data.tar.gz"; '
         'TARGET_TAR_NAME="gtdbtk_data.tar.gz"; '
 
         # Script variables (no need to configure)
@@ -229,7 +229,7 @@ rule gtdbtk:
         "rm -rf data/gtdbtk data/gtdbtk_scratch && "
         "mkdir -p data/gtdbtk_scratch && "
         f'{pixi_run} -e gtdbtk '
-        "gtdbtk classify_wf --skip_ani_screen --cpus {threads} --pplacer_cpus {params.pplacer_threads} --extension {params.extension} "
+        "gtdbtk classify_wf --cpus {threads} --pplacer_cpus {params.pplacer_threads} --extension {params.extension} "
         "--genome_dir {input.mag_folder} --out_dir data/gtdbtk --scratch_dir data/gtdbtk_scratch "
         "> {resources.log_path} 2>&1 "
 

aviary/modules/binning/binning.smk

@@ -1306,6 +1306,8 @@ rule singlem_appraise:
         singlem = "data/singlem_out/singlem_appraisal.tsv"
     params:
         package_path = config['singlem_metapackage'],
+    benchmark:
+        "benchmarks/singlem_appraise.benchmark.txt"
     threads: min(config["max_threads"], 4) # Should not generally take a long time
     resources:
         # 16G since failed with 8G when running (with SingleM v0.20.3)