Sylph integration: graded absence weight, full-reconciliation mode, and strain-level brainstorm

.gitattributes

@@ -1,16 +1,2 @@
-db/4.07.ribosomal_protein_L2_rplB.gpkg.spkg/4.07.ribosomal_protein_L2_rplB/singlem_package_creatorDMgneQ.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.08.ribosomal_protein_L3_rplC.gpkg.spkg/4.08.ribosomal_protein_L3_rplC/singlem_package_creatorzY212C.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.09.ribosomal_protein_L5_rplE.gpkg.spkg/4.09.ribosomal_protein_L5_rplE/singlem_package_creatorIO_9V7.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.10.ribosomal_protein_L6_rplF.gpkg.spkg/4.10.ribosomal_protein_L6_rplF/singlem_package_creatoraYgaFe.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.11.ribosomal_protein_L10.gpkg.spkg/4.11.ribosomal_protein_L10/singlem_package_creatorbKHG6Z.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.12.ribosomal_protein_L11_rplK.gpkg.spkg/4.12.ribosomal_protein_L11_rplK/singlem_package_creatornLOQNK.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.13.ribosomal_protein_L14b_L23e_rplN.gpkg.spkg/4.13.ribosomal_protein_L14b_L23e_rplN/singlem_package_creatorvcVSZr.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.14.ribosomal_protein_L16_L10E_rplP.gpkg.spkg/4.14.ribosomal_protein_L16_L10E_rplP/singlem_package_creatorFBCD7u.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.15.ribosomal_protein_S2_rpsB.gpkg.spkg/4.15.ribosomal_protein_S2_rpsB/singlem_package_creatorK2almT.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.16.ribosomal_protein_S5.gpkg.spkg/4.16.ribosomal_protein_S5/singlem_package_creatorVrfWE1.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.17.ribosomal_protein_S7.gpkg.spkg/4.17.ribosomal_protein_S7/singlem_package_creatorm0oqmF.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.18.ribosomal_protein_S10_rpsJ.gpkg.spkg/4.18.ribosomal_protein_S10_rpsJ/singlem_package_creatorrSEuCX.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.19.ribosomal_protein_S12_S23.gpkg.spkg/4.19.ribosomal_protein_S12_S23/singlem_package_creatorTAU52H.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.20.ribosomal_protein_S15P_S13e.gpkg.spkg/4.20.ribosomal_protein_S15P_S13e/singlem_package_creatord8MMo3.dmnd filter=lfs diff=lfs merge=lfs -text
-db/4.21.ribosomal_protein_S19_rpsS.gpkg.spkg/4.21.ribosomal_protein_S19_rpsS/singlem_package_creatorEa_akI.dmnd filter=lfs diff=lfs merge=lfs -text
-*.dmnd filter=lfs diff=lfs merge=lfs -text
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true

.github/workflows/deploy-docs.yml

@@ -0,0 +1,29 @@
+name: Build manual pages and deploy documentation
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'docs/**'
+      - 'doctave.yml'
+      - '.github/workflows/deploy-docs.yml'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: 'Checkout cargo and install doctave'
+        uses: actions-rs/toolchain@v1.0.6
+        with:
+          toolchain: stable
+      - run: cargo install --git https://github.com/Doctave/doctave --tag 0.4.2
+      - name: 'Build doctave site'
+        run: doctave build --release
+      - name: 'GitHub Pages'
+        uses: crazy-max/ghaction-github-pages@v3.0.0
+        with:
+          build_dir: site/
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/publish-to-pypi.yml

@@ -0,0 +1,26 @@
+name: Publish to PyPI
+on:
+  push:
+    # Pattern matched against refs/tags
+    tags:        
+      - '*'           # Push events to every tag not containing /
+jobs:
+  Publish-to-PyPI:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      # admin/requirements.txt is tracked only as a stub; the real pinned
+      # contents are generated here from pixi.toml/pixi.lock so setuptools
+      # picks them up when building the wheel.
+      - uses: prefix-dev/setup-pixi@v0.8.1
+        with:
+          cache: true
+      - run: pixi run admin/build_dep_defs_from_pixi.py
+      - uses: casperdcl/deploy-pypi@v2
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          # pip: wheel -w dist/ --no-deps .
+          build: true
+          # only upload if a tag is pushed (otherwise just build & check)
+          upload: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') }}

.github/workflows/test-singlem.yml

@@ -0,0 +1,64 @@
+name: Current Build
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  pixi_test_dependencies_locked:
+    name: Pixi test_dependencies_locked ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        os: ["ubuntu-latest"]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Pixi
+        run: |
+          curl -fsSL https://pixi.sh/install.sh | bash
+          echo "$HOME/.pixi/bin" >> $GITHUB_PATH
+
+      - name: Show environment info
+        run: |
+          pixi run -e dev --frozen pixi info
+          pixi run -e dev --frozen pixi list
+
+      - name: Run tests with Pixi
+        run: |
+          pixi run -e dev --frozen pytest test -v
+
+  # Run after removing lock file so dependences are unlocked
+  pixi_test_dependencies_optimistic:
+    name: Pixi test_dependencies_optimistic ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    strategy:
+      matrix:
+        os: ["ubuntu-latest"]
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Pixi
+        run: |
+          curl -fsSL https://pixi.sh/install.sh | bash
+          echo "$HOME/.pixi/bin" >> $GITHUB_PATH
+
+      - name: Remove lock file
+        run: |
+          rm -fv pixi.lock admin/requirements.txt admin/environment.yml
+          touch admin/requirements.txt
+
+      - name: Create dev environment
+        run: |
+          pixi run -e dev pixi info
+
+      - name: Show environment info
+        run: |
+          pixi run -e dev pixi list
+
+      - name: Run tests with Pixi
+        run: |
+          pixi run -e dev pytest test -v

.gitignore

@@ -0,0 +1,23 @@
+guix*
+real_data
+db
+__pycache__/
+.snakemake/
+singlem.egg-info
+docker/singlem.yml
+docker/Dockerfile
+site
+extras/update_metapackage/env_master
+untracked
+build
+extras/new_package_creation/singlem_new_metapackage_working_directory/
+
+# pixi environments
+.pixi
+*.egg-info
+
+# Generated by admin/build_dep_defs_from_pixi.py. environment.yml is purely
+# a convenience artifact, so it's not tracked. admin/requirements.txt is
+# tracked as a stub (pyproject.toml needs it to exist) and overwritten by
+# the script locally and in CI.
+admin/environment.yml

AGENTS.md

@@ -0,0 +1,2 @@
+1. When making changes, run tests using `pixi run -e dev pytest test`. This command can take 500 seconds to complete.
+2. Do not modify markdown files in `docs/tools/` or `docs/advanced/` directories - these are auto-generated.

CHANGELOG.md

@@ -0,0 +1,177 @@
+## v0.21.3
+
+* Fix PyPI dependency list by generating `admin/requirements.txt` in CI before the wheel build; previous releases shipped without dependencies because the generated file was never committed to release tags.
+
+## v0.21.2
+
+* Fix PyPI dependency list structurally, and include bird_tool_utils.
+
+## v0.21.1
+
+* Fix PyPI dependency list
+
+## v0.21.0
+
+* Default metapackage updated to GTDB R232 (S6.5.0-R232)
+* `pipe`: Substantially improved logging, including progress bars for prefilter, hmmsearch and DIAMOND assignment steps. Thanks @thepatientwait.
+* `pipe`: Parallelised DIAMOND taxonomic assignment for faster runs. Thanks @thepatientwait.
+* `pipe`: Add optional `--context-window` if the entire query sequence is too large for output size.
+* `pipe`: Add zstd input support in places, including with `--read-chunk-size`/`--read-chunk-num` and `prokaryotic_fraction`
+* `pipe`: Restore v0.19.0 orfm+mux pipeline for genome input, restoring performance lost in v0.20
+* `pipe`: Use DIAMOND `--frameshift` when assigning taxonomy
+* `pipe`: `--sra-files`: Numerous fixes and refinements to chunked SRA extraction, including detection of `kingfisher` failures and stripping of `.sra` suffix from sample names
+* `pipe`/`renew`: Bugfix for missing sequences with Lyrebird
+* `supplement`: Add `--output-matched-faa` to output matched protein sequences, with marker name included in sequence IDs
+* `renew`: Accept zipped gzip archive OTU tables as input
+* `lyrebird`: Add `extras/lyrebird_metapackage_creation` snakemake workflow for building Lyrebird metapackages from scratch
+* `data`: Report DOI when database download or acquisition fails
+* Switch internal data handling from pandas to polars
+* Updated citation for `prokaryotic_fraction`
+* docs: Improved GlobDB documentation (#267)
+* Add `AGENTS.md` and `SKILL.md` to assist agentic coding tools
+* Assorted bug fixes, test improvements, and documentation updates
+
+Thanks @thepatientwait, @rzhao-2, @AroneyS, @EisenRa and others for contributions and testing, and thanks to @MCeciC (#269) and @magicprotoss (#267) for raising issues.
+
+## v0.20.3
+
+Minor bugfix release. Rare sequences tripped a bug in `pipe` mode.
+
+Thanks to @asuq for reporting #265.
+
+## v0.20.0 / 0.20.1 / 0.20.2
+
+Major new function - Long-read input support (Nanopore >= R10.4.1 or PacBio HiFi recommended), thanks to @thepatientwait.
+
+* Lyrebird database updated to v0.3.1, improving exclusion of off-target (non-phage) sequences
+* `microbial_fraction` subcommand renamed to `prokaryotic_fraction` (old name retained as synonym)
+* More flexible options for specifying genome input in `pipe` mode
+* `appriase` mode: Add `--stream-inputs`
+* [GlobDB R226 metapackage](https://fileshare.lisc.univie.ac.at/globdb/globdb_r226/taxonomic_profiling/globdb_r226_SingleM_metapackage.tar.gz) released
+
+Thanks to @AroneyS, @rzhao-2, @EisenRa, @thepatientwait, @dspeth, @Anna-MarieSeelen, @luigallucci, @ilnamkang and others for contributions and testing.
+
+## v0.19.0
+Major new function - profiling of Caudoviricetes (aka "Caudovirales") phage communities (Lyrebird), thanks to @rzhao-2.
+
+![Lyrebird](https://raw.githubusercontent.com/wwood/singlem/refs/tags/v0.19.0/docs/_include/lyrebird_resized.png?raw=1)
+
+Other changes:
+* Update default metapackage to GTDB R226
+* admin: Use pixi instead of conda
+* Use of diamond v2.1.10 specifically, to avoid segfault issues with diamond v2.1.11
+* Clarify non-standard metapackage usage (#220)
+* doc: Improve summarise --cluster (#210)
+
+Thanks @rzhao-2 @AroneyS @ilnamkang Phil Hugenholtz @pchaumeil @zackhenny @thepatientwait
+
+## v0.18.3
+A small patch release
+
+* `summarise`: Fix a regression
+* docs: Minor fixes
+
+## v0.18.1
+A small patch release.
+
+* Updates to singlem `supplement` and other modes for polars >1.0
+* Pin dependencies to help future proof singlem
+
+## v0.18.0
+Combined changelog for v0.17.0 and 0.18.0
+
+* Use of GTDB R220 reference metapackage by default
+* `pipe`/`condense`: Improve algorithm by delaying some filtering steps, leading to more accurate taxonomic profiles
+* `pipe`: update to [smafa](https://github.com/wwood/smafa) v0.8.0 for substantial speed improvement
+* `microbial_fraction`: Remove `%` from column data and add average genome size estimation
+* `supplement`: Change command line options in backwards incompatible way, clarifying their meaning
+* `summarise`: Add `--output-taxonomic-profile-with-extras` output to add relative abundance etc. to taxonomic profiles
+* `summarise`: Add `--output-species-by-site-relative-abundance-prefix` to create taxon-level specific relative abundances from taxonomic profiles
+* `summarise`: Add `--output-taxonomic-level-coverage` to show how much coverage and number of taxa assigned to each level
+* `pipe`: Faster processing when many genome fasta files are input
+* `seqs`: Prioritise high-info HMM positions.
+* dist: Fix singularity container
+* assorted bug and documentation fixes
+
+Thanks @AroneyS @EisenRa @jakobnissen @rzhao-2 @rrohwer @shaze @ellyyuyang @VadimDu @adityabandla @luispedro, and anonymous reviewers, among others.
+
+The `microbial_fraction` mode now has its own citation - https://www.biorxiv.org/content/10.1101/2024.05.16.594470v1
+
+## v0.16.0
+This version tweaks the method which assign taxonomy to OTUs (increasing the species-level threshold) and the method which summarises the OTUs to create a final taxonomic profile (very low abundance lineages are given lower taxonomic resolution, rather than ignored completely). This improves the rate over "overclassification" i.e. when novel species are classified wrongly to the species level, and improves the `read_fraction` (now called `microbial_fraction`) estimates in complex / shallowly sequenced metagenomes.
+
+We suggest recomputing community profiles using `renew` or `pipe` modes.
+
+* pipe/renew: Change default species-level assignment from 3bp or closer, to 2bp or closer.
+* pipe/renew/condense: Assign sub-min-taxon-coverage higher.
+* read_fraction mode renamed to microbial_fraction
+
+Thanks to Yu Yang, Caitlin Singleton, @MadsAlbertsen @EisenRa @BigDataBiology
+
+## v0.15.1
+Mostly minor bugfixes
+
+* pipe: extract: Apply --evalue to hmmsearch thresholding.
+* Fix for appraise --plot
+* pipe: Dedup hmmsearch results during diamond package assignment.
+* pipe/renew/condense: Prevent no_assign_taxonomy and taxonomic profile output.
+
+Thanks @kalonji08 @AroneyS @harmonydouwes
+
+## v0.15.0
+* Genomes that encode proteins with translation table 4 are now supported. This
+  works by assuming all genomes have translation table 4, since regular sequence
+  similarity search excludes inappropriately translated sequences from genomes
+  which use table 11 (the standard bacterial table). Thanks to Dr. Andy Leu for
+  useful test cases. NOTE: The `renew` mode is not sufficient for detecting
+  these lineages, `pipe` must be run again from scratch.
+* new_package_creation (beta): A snakemake pipeline included in the `extras`
+  directory used to create new SingleM metapackages from scratch. In
+  development. Thanks for @harmonydouwes @tvtv195 @JemmaSun for testing.
+* Version S3.2.1 of the default metapackage released, which includes updated
+  genome sizes for GTDB genomes (for use with `read_fraction`), now corrected
+  based on CheckM v2 estimates of completeness and contamination. Thanks to
+  @EisenRa for collaboration.
+* `seqs`: Output the best window position to STDOUT.
+* Other assorted bug fixes and documentation updates.
+
+## v0.14.0
+This release is a huge step forward for the SingleM software, comprising >750 git commits and several years work (particularly from @AroneyS and @EisenRa and @rzhao-2) since v0.13.2. 
+
+There are so many changes that generating a CHANGELOG would take too long.
+
+This release is equivalent to 1.0.0beta8, and is intended as a pre-release for version 1.0.0, but using a standard version number allows for a more streamlined release process.
+
+## v0.10.0 to v0.13.2
+(undocumented)
+
+ ## v0.9.0
+* Appraise can now generate 'appraisal plots'
+* Use of smafa / SQLite rather than BLAST+ / VSEARCH for 'query' and clustering. SingleM
+  databases (.sdb) will need to be regenerated.
+* SingleM databases can now be queried via taxonomy or sample name
+* Overhaul of command line help messages
+* Appraise can now appraise assemblies as well as genomes
+* Various bug fixes and enhancements
+
+## v0.8.2
+* Fix for installation through PyPI.
+
+## v0.8.1
+* Fix bug in singlem query where some results were omitted.
+* Detect when max_target_seqs has been reached in singlem query.
+
+## v0.8.0
+* Overhauled makedb/query. Database creation is now faster and querying more accurate, especially for OTUs with gaps. Old databases should be re-generated.
+* summarise: Added BIOM and wide format outputs - props to Steve Robbins and Louis Monaghan for the suggestions.
+* appraise: Default to genus-level similarity cutoff.
+
+## v0.7.1
+* Now installable via pip / PyPI.
+
+## v0.7.0
+* Speed improvements for singlem query
+* Memory improvements for singlem makedb
+* db: Use and require the new diamond version / database format
+* summarise: Only output to a single html for --krona where possible
+

CITATION.cff

@@ -0,0 +1,27 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - family-names: Woodcroft
+    given-names: Ben J.
+    orcid: https://orcid.org/0000-0003-0670-7480
+  - family-names: Aroney
+    given-names: Samuel T. N.
+    orcid: https://orcid.org/0000-0001-9806-5846
+  - family-names: Zhao
+    given-names: Rossen
+    orcid: http://orcid.org/0000-0002-7003-2791
+  - family-names: Cunningham
+    given-names: Mitchell
+  - family-names: Mitchell
+    given-names: Joshua	A. M.
+    orcid: http://orcid.org/0009-0006-9066-6683
+  - family-names: Blackall
+    given-names: Linda
+    orcid: http://orcid.org/0000-0002-8848-7698
+  - family-names: Tyson
+    given-names: Gene W.
+    orcid: https://orcid.org/0000-0001-8559-9427
+title: "SingleM and Sandpiper: Robust microbial taxonomic profiles from metagenomic data"
+version: 0.16.0
+doi: 10.1101/2024.01.30.578060
+date-released: 2024-01-31