From 629e68e6848d89fa09e68e2f6ed7b71a6c5ea80e Mon Sep 17 00:00:00 2001 From: Thomas Shin Date: Fri, 23 Oct 2026 11:37:46 -0400 Subject: [PATCH 1/2] Add initial CLI test and exports --- README.md | 96 +++++- docs/api/blast.rst | 18 +- docs/api/index.rst | 4 +- docs/api/pipeline.rst | 11 +- docs/user_guide.rst | 114 ++++++- pyproject.toml | 3 + src/lverage/blast.py | 172 ++++++++-- src/lverage/cli.py | 545 +++++++++++++++++++++++++++++++ src/lverage/evidence.py | 29 ++ src/lverage/ortholog_searcher.py | 3 + src/lverage/pipeline.py | 136 +++++++- tests/v2/test_cli.py | 435 ++++++++++++++++++++++++ tests/v2/test_local_blast.py | 163 +++++++++ tests/v2/test_pipeline.py | 95 ++++++ 14 files changed, 1769 insertions(+), 55 deletions(-) create mode 100644 src/lverage/cli.py create mode 100644 src/lverage/evidence.py create mode 100644 tests/v2/test_cli.py diff --git a/README.md b/README.md index ac703c7..6d0734d 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,10 @@ cd Lverage ``` ### Python -Lverage requires Python3.8 at minimum to run. The tool was developed using Python3.10.12. To obtain python and the required libraries, we advise using Conda. +The legacy workflow requires Python 3.8 at minimum and was developed with +Python 3.10.12. The V2 package requires Python 3.11 or newer; see the V2 CLI +section below. To obtain Python and the required libraries, we advise using +Conda. #### Downloading Conda To install Conda, follow the instructions on their [website user guide](https://conda.io/projects/conda/en/latest/user-guide/install/index.html "https://conda.io/projects/conda/en/latest/user-guide/install/index.html"). We suggest downloading Miniconda. @@ -95,7 +98,96 @@ the [API documentation](docs/api/index.rst) for how to pass them to the V2 pipeline. -## How to Use +## V2 Command-Line Interface + +The V2 CLI is a new interface to the modular V2 package. It does not reuse or +extend the legacy CLI below. From the repository root, install the package in +your Python environment with `python -m pip install .` to get the `lverage` +command, or run it directly from a source checkout with +`PYTHONPATH=src python -m lverage.cli`. The CLI's `--help` works without +BLAST+, PfamScan, databases, or network access. + +### Prerequisites + +- Python 3.11+ and the package dependencies from this repository. +- Local BLAST+ `blastp` and `blastdbcmd` executables. +- A local protein BLAST database suitable for complete sequence retrieval. +- A local PfamScan installation and database. +- Network access when running motif inference: JASPAR2024 is queried online. + +The local BLAST database setup above describes how to build a usable protein +database. Build and configure PfamScan separately according to its +distribution's instructions. + +### Input mapping and example + +Create a JSON file mapping bracketed scientific species names in BLAST hit +descriptions to positive NCBI taxonomy IDs, for example +`config/species-taxids.json`: + +```json +{ + "Homo sapiens": 9606, + "Mus musculus": 10090, + "Drosophila melanogaster": 7227 +} +``` + +Put one nucleotide FASTA file per gene/TF under `inputs/`. Supported suffixes +are `.fa`, `.fasta`, and `.fna`, matched case-insensitively. Multiple records +in one file are passed as independent fragments for that filename's gene; +they are never assembled, and headers are not used to verify that records +belong to the same gene. See the [V2 FASTA and CLI guide](docs/user_guide.rst) +for details. + +```bash +lverage --input-dir ./inputs --blast-db ./db/proteins \ + --species-map ./config/species-taxids.json --pfam-db ./db/pfam \ + --output-dir ./results +``` + +The required options are `--input-dir`, `--blast-db`, `--species-map`, +`--pfam-db`, and `--output-dir`. The output path must not already exist; the +CLI creates it and writes `results.tsv`, `orfs.fasta`, and `metadata.json`. +Optional executable overrides are +`--blastp` (default `blastp`), `--blastdbcmd` (default `blastdbcmd`), and +`--pfamscan` (default `pfam_scan.pl`). `--identity-threshold` defaults to +`0.7`. Repeat `--valid-pfam ACCESSION` to limit accepted domains; when omitted, +any detected domain is accepted. JASPAR defaults to 10 hits and E-value +threshold `1e-6`; the existing ORF and BLAST component defaults are retained +(75-nt minimum ORF; BLAST E-value `1e-6`, top 20 hits, and standard description +filters). + +The TSV contains explicit stage/status/error/outcome-detail fields, one motif +per motif row plus typed evidence rows for inputs, ORFs, query domains, +orthologs, ortholog domains, comparisons, stage statuses, and gene errors. +Earlier-stage rows are not motif or +ortholog hits; stage-status rows distinguish biological no-hits, below- +threshold completion, failure, and not-reached stages. Missing fields are blank. Rows retain BLAST accession, +description and native hit ID when returned, E-value, identity and query +coverage ratios and percentages, Pfam names/IDs, and each computed domain-pair +identity whether or not it meets the threshold. Candidate ORFs are uniquely +stored once in the FASTA by full-sequence SHA-256 ID; the TSV retains per-gene +associations and selected/not-selected/not-reached state. The existing +longest-first ORF-selection algorithm remains unchanged; downstream stages +run only on the selected ORF. `metadata.json` records settings, input +checksums, timestamps, tool versions when established (otherwise `unknown`), +errors, and completion counts. Bundle files are flushed as evidence arrives; +this is not checkpoint/resume or arbitrary-crash recovery. + +Use `--stop-after alignment` for pre-JASPAR validation. This intentional stop +does not contact JASPAR and exits 0. Exit 0 also means full completion, +including biological no-hits; exit 2 means preflight validation failure; exit +1 means one or more execution failures. Per-gene errors are recorded and +processing continues to other genes, preserving completed partial evidence. +Diagnostics go to stderr. The output directory is not removed after execution +starts, including when a downstream operation fails. + +## Legacy Command-Line Workflow + +This older workflow is separate from the V2 package and CLI. The options and +examples in this section refer only to the legacy implementation. + We warn against moving any file within the directory anywhere else as this will create errors. If you wish to access from other places, we suggest appending the directory to your PATH environment variable, creating an alias, or creating a shortcut. To call Lverage, ensure that the requirements above are all met. Within the directory is a file called *lverage.py*. All calls should be made with this file. diff --git a/docs/api/blast.rst b/docs/api/blast.rst index 398b8c8..c33e437 100644 --- a/docs/api/blast.rst +++ b/docs/api/blast.rst @@ -2,15 +2,13 @@ BLAST searchers =============== BLAST+ is an external prerequisite and is not installed as a Python package. -On the SCC, ``blast+/2.12.0`` and the ``human_mouse_nr`` database are useful -development examples; applications must still pass their own executable and -database locations. +Configure the executable and database locations for the environment where +Lverage is run. Local databases must be built with ``makeblastdb -parse_seqids`` because specific complete-sequence retrieval through ``blastdbcmd`` requires a parsed -sequence-ID index. The existing SCC ``human_mouse_nr`` database was built -without that index and must be rebuilt before it can support an end-to-end -``LocalBlastSearcher`` run. +sequence-ID index. Rebuild databases that lack this index before using them +with ``LocalBlastSearcher``. ``LocalBlastSearcher`` returns :class:`lverage.ortholog_searcher.OrthologRecord` objects. @@ -19,6 +17,14 @@ excluded, unresolved, or unretrievable hits are skipped and logged; executable, database, subprocess, timeout, and whole-result parsing failures are raised to the caller. +The existing ``get_orthologs(sequence)`` call remains unchanged. The shared +pipeline may additionally pass an evidence collector; in that mode each +retrieved ortholog and each failed/skipped sequence-retrieval attempt is +reported as it completes. This lets the CLI retain earlier hits if a later +retrieval fails and distinguishes a BLAST result with no alignments, alignments +filtered from retrieval, and retained hits whose sequences could not be +retrieved. The optional reporting mode does not rerun BLAST. + ``LocalBlastSearcher`` validates a database prefix with ``blastdbcmd -info`` and retrieves complete subject proteins from that database. diff --git a/docs/api/index.rst b/docs/api/index.rst index 6c48958..f5d2e03 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -2,8 +2,8 @@ API reference ============= This reference documents the current version 2 core interfaces. Package-level -exports and a command-line interface are intentionally deferred, so import -classes from the modules shown below. +exports are intentionally deferred, so import classes from the modules shown +below. The V2 command-line interface is documented in the user guide. .. toctree:: :maxdepth: 1 diff --git a/docs/api/pipeline.rst b/docs/api/pipeline.rst index 90b7632..01b79fe 100644 --- a/docs/api/pipeline.rst +++ b/docs/api/pipeline.rst @@ -7,9 +7,14 @@ local configuration and do not contact remote services. Each run deduplicates ORFs, selects the first longest ORF with an accepted domain, compares matching base PFAM accessions, and searches motif databases -for domain pairs meeting the identity threshold. Intermediate evidence remains -local to the run; the instance retains its copied configuration and the latest -:class:`lverage.pipeline.LverageCode`. +for domain pairs meeting the identity threshold. Intermediate evidence is not +retained by default. Callers may pass an optional ``evidence_collector`` to +``Lverage.run()`` to receive completed stage events as they occur; this supports +incremental reporting but is not checkpoint or crash recovery. The instance +retains its copied configuration and latest :class:`lverage.pipeline.LverageCode`. + +``stop_after="alignment"`` deliberately returns before motif-database searches +while preserving full-pipeline behavior when the option is omitted. Valid biological no-hit outcomes return an empty list and set a deterministic code. Adapter and service failures remain exceptions. diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 77beada..903ee15 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -5,8 +5,103 @@ Use nucleotide FASTA input to provide the gene or transcription-factor (TF) sequence from which Lverage infers a protein and its DNA-binding domain. Prepare one FASTA file per gene/TF and name the file after the gene, for example ``geneX.fa``. Keep records for different genes in separate files. The -command-line workflow requires this layout; for the Python API it is a -convention, since the caller reads the file. +V2 command-line interface derives the gene identifier from each filename +stem; for the Python API, the caller reads files and supplies any identifier. + +V2 command-line interface +------------------------- + +The V2 CLI uses the current modular pipeline and is distinct from the legacy +command-line workflow described at the end of this guide. It can be invoked as +an installed ``lverage`` command or, from a source checkout, as +``PYTHONPATH=src python -m lverage.cli``. ``--help`` only displays arguments; +it does not construct adapters, inspect external tools, or access a network. + +Prerequisites are the Python package dependencies, local BLAST+ executables +(``blastp`` and ``blastdbcmd``), a usable local protein database, and a local +PfamScan installation/database. JASPAR2024 is queried through its online +service, so each actual run also requires network access. The local BLAST +database must support complete protein retrieval with ``blastdbcmd``. See the +README's V2 local BLAST database instructions for building one. + +Create a JSON species-name-to-NCBI-tax-ID map. Species names must match the +bracketed names in BLAST descriptions (case-insensitive), and tax IDs must be +positive integers: + +.. code-block:: json + + { + "Homo sapiens": 9606, + "Mus musculus": 10090 + } + +Example using portable placeholder paths: + +.. code-block:: console + + lverage --input-dir ./inputs --blast-db ./db/proteins \ + --species-map ./config/species-taxids.json --pfam-db ./db/pfam \ + --output-dir ./results + +Required options are ``--input-dir``, ``--blast-db``, ``--species-map``, +``--pfam-db``, and ``--output-dir``. The output directory must be new. It +contains ``results.tsv``, ``orfs.fasta``, and ``metadata.json``. Optional +executable overrides are ``--blastp`` (``blastp``), +``--blastdbcmd`` (``blastdbcmd``), and ``--pfamscan`` (``pfam_scan.pl``). +``--identity-threshold`` defaults to ``0.7``. Repeat ``--valid-pfam`` to +restrict accepted domains; if it is omitted, any detected domain is accepted. +JASPAR options default to 10 hits and an inference E-value threshold of +``1e-6``. Other component options retain their V2 defaults: ORF minimum length +75 nucleotides; BLAST E-value ``1e-6``, at most 20 hits, and the standard +description filters. + +The CLI discovers ``.fa``, ``.fasta``, and ``.fna`` files nonrecursively, +case-insensitively, in filename order. The filename stem labels results and +must be unique ignoring case. Every FASTA record in a file is passed as an +independent sequence fragment to the same gene run. The CLI does not assemble +fragments, and it cannot determine from record headers whether they really +belong to the same gene. Empty files/records and unsupported DNA symbols are +rejected before external-tool adapters are constructed. + +The TSV has explicit ``Row Type``, ``Stage``, ``Status``, ``Error``, and +``Outcome Detail`` fields. ``Error`` is reserved for execution failures; +biological reasons and not-reached explanations use ``Outcome Detail``. +Row types include ``input``, ``orf``, ``query_domain``, ``ortholog``, +``ortholog_retrieval``, ``ortholog_domain``, ``domain_comparison``, ``motif``, +and ``gene_error``. +``stage_status`` rows record biological no-hits, below-threshold completion, +intentional stops, and stages not reached. +``ortholog_retrieval`` rows retain failed retrieval attempts and skipped hits, +including available native hit IDs/accessions, the attempted identifier, and a +diagnostic reason. A BLAST no-hit status is distinct from retrieval failure. +Earlier-stage rows are not ortholog or motif hits; missing values are blank. +Each motif has one row and identifies its input gene, query ORF ID, ortholog, +and matching query/ortholog domains. Multiple hits, domain occurrences, and +motifs remain distinct. ORF rows retain candidates even when no accepted +domain exists, and an input row records a gene with no ORF. ``orfs.fasta`` +stores each unique discovered candidate protein once under a deterministic +full-sequence SHA-256 identifier referenced by the TSV; gene associations and +selection status remain in the TSV. Candidate export does not change the +longest-first selection algorithm or run downstream analysis on unselected +candidates. BLAST fields include accession, native hit ID when returned, full +description, E-value, identity, and query coverage. Ratios and percentages are +separately labeled. Pfam IDs/names and every computed domain-pair identity are +kept, including values below the configured threshold. + +``metadata.json`` records run timestamps, settings, input paths/checksums, +sequence-ID conventions, known tool versions (unknown versions are reported as +``unknown``), and completion/failure summaries. Results and metadata are +flushed as pipeline evidence completes. This is incremental evidence retention, +not checkpoint/resume or recovery from arbitrary crashes. After execution +begins, the bundle is preserved on failure. Execution failures are caught per +gene and processing continues to other genes, retaining completed evidence if +a later stage fails. + +Use ``--stop-after alignment`` for pre-JASPAR validation. This intentional stop +does not call JASPAR and is not a failure. Exit status ``0`` means completed or +intentionally stopped, including biological no-hits; ``2`` means preflight +validation failure; ``1`` means one or more execution failures despite +preserved partial results. Diagnostics go to standard error. FASTA format ------------ @@ -34,9 +129,9 @@ same gene/TF, for example several scaffolds or contigs: >geneX_fragment_2 GCTAGCATCGATCGTA -The record headers identify fragments of the same gene in that one file. Do -not place unrelated genes or TFs together in a multi-record input file: use a -separate FASTA file for each one. +Treat all records in the file as intended fragments of the same gene. The +headers themselves are not validated as evidence that fragments belong to +that gene. Keep unrelated genes or TFs in separate files. The V2 ORF adapter accepts nonempty, case-insensitive DNA strings using the IUPAC symbols ``A C G T R Y S W K M B D H V N``. It does not accept RNA ``U``, @@ -91,13 +186,14 @@ BLASTP. Do not use those protein records as the gene DNA input to motif inference. The README's V2 local BLAST database section describes the protein FASTA used to build that database. -Legacy command-line workflow versus V2 API ------------------------------------------- +Legacy command-line workflow and V2 Python API +---------------------------------------------- The legacy command-line workflow is separate from the V2 Python API. Its ``-f/--fasta`` option takes a directory containing one file per gene/TF; a file may contain one record or multiple records for fragments of that same gene. The legacy implementation reads all FASTA records in each file. -For V2, configure the Python pipeline and call ``Lverage.run()`` with a DNA -string or list of DNA strings as shown above. +The V2 CLI is documented above. For the V2 Python API, configure the pipeline +and call ``Lverage.run()`` with a DNA string or list of DNA strings as shown +above. diff --git a/pyproject.toml b/pyproject.toml index 10ca740..cddcd66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,9 @@ dependencies = [ "requests>=2.32", ] +[project.scripts] +lverage = "lverage.cli:main" + [project.optional-dependencies] docs = [ "numpydoc>=1.10", diff --git a/src/lverage/blast.py b/src/lverage/blast.py index 58632e6..4d7588f 100644 --- a/src/lverage/blast.py +++ b/src/lverage/blast.py @@ -53,13 +53,15 @@ def _copy_excluded_terms(excluded_terms : Sequence[str] | None) -> list[str]: return list(terms) -def _parse_blast_hits(blast_xml, query_length, species_lookup, excluded_terms): +def _parse_blast_hits(blast_xml, query_length, species_lookup, excluded_terms, + evidence_collector=None): blast_records = list(NCBIXML.parse(StringIO(blast_xml))) if len(blast_records) != 1: raise ValueError("BLAST output did not contain exactly one query result") + raw_alignments = blast_records[0].alignments hits = [] - for alignment in blast_records[0].alignments: + for alignment in raw_alignments: try: description = alignment.hit_def if any(term.casefold() in description.casefold() for term in excluded_terms): @@ -79,7 +81,7 @@ def _parse_blast_hits(blast_xml, query_length, species_lookup, excluded_terms): hits.append({ "accession": alignment.accession, "description": description, - "hit_id": alignment.hit_id, + "hit_id": getattr(alignment, "hit_id", None), "species_name": species[0], "species_tax_id": species[1], "evalue": best_hsp.expect, @@ -90,6 +92,13 @@ def _parse_blast_hits(blast_xml, query_length, species_lookup, excluded_terms): }) except (AttributeError, TypeError, ValueError, ZeroDivisionError): LOGGER.warning("Skipping malformed BLAST hit", exc_info=True) + if evidence_collector is not None: + evidence_collector.emit( + "blast_search", + status=("no-hits" if not raw_alignments else "hits-found" if hits else "no-retained-hits"), + hit_count=len(raw_alignments), retained_hit_count=len(hits), + retrieved_count=0, skipped_count=0, error="", + ) return hits @@ -103,9 +112,23 @@ def _make_ortholog(hit, sequence): evalue=hit["evalue"], identity=hit["identity"], query_coverage=hit["query_coverage"], + hit_id=hit.get("hit_id"), ) +def _safe_retrieval_reason(error): + """Return a diagnostic without echoing command arguments or credentials.""" + detail = getattr(error, "stderr", None) + if isinstance(detail, bytes): + detail = detail.decode("utf-8", errors="replace") + if not isinstance(detail, str) or not detail.strip(): + detail = f"{type(error).__name__}: sequence retrieval failed" + detail = re.sub(r"https?://[^\s]+", "", detail) + detail = re.sub(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", "", detail) + detail = re.sub(r"(?i)(token|password|secret|api[_-]?key)=([^\s&]+)", r"\1=", detail) + return detail.strip() + + class LocalBlastSearcher(OrthologSearcherTemplate): """ Search a local BLAST protein database for orthologs. @@ -128,6 +151,8 @@ class LocalBlastSearcher(OrthologSearcherTemplate): Case-insensitive description terms to exclude; an empty sequence disables filtering """ + supports_evidence_collector = True + def __init__(self, database_path, species_map, @@ -162,7 +187,7 @@ def __init__(self, text=True, ) - def get_orthologs(self, sequence : str) -> list[OrthologRecord]: + def get_orthologs(self, sequence : str, *, evidence_collector=None) -> list[OrthologRecord]: """ Search the configured local database for complete ortholog proteins. @@ -180,6 +205,9 @@ def get_orthologs(self, sequence : str) -> list[OrthologRecord]: if not isinstance(sequence, str) or not sequence.strip(): raise ValueError("sequence must be a nonempty string") + if evidence_collector is not None: + evidence_collector.set_stage("blast") + result = subprocess.run( [ self.blastp_path, @@ -200,29 +228,125 @@ def get_orthologs(self, sequence : str) -> list[OrthologRecord]: len(sequence), self.__species_lookup, self.excluded_terms, + evidence_collector=evidence_collector, ) - for hit in hits: - complete_sequence = "" - entries = [hit["hit_id"]] - if hit["accession"] and hit["accession"] != hit["hit_id"]: - entries.append(hit["accession"]) - for entry in entries: - if not entry: + + skipped_count = 0 + retrieval_exception = None + record_exception = None + evidence_exception = None + + def emit_evidence(event, **payload): + nonlocal evidence_exception + if evidence_collector is None: + return + try: + evidence_collector.emit(event, **payload) + except Exception as error: + evidence_exception = error + raise + + try: + for hit in hits: + complete_sequence = "" + entries = [hit["hit_id"]] + if hit["accession"] and hit["accession"] != hit["hit_id"]: + entries.append(hit["accession"]) + retrieval_errors = [] + for entry in entries: + if not entry: + continue + try: + complete_sequence = self.__get_complete_sequence(entry) + except subprocess.CalledProcessError as error: + LOGGER.warning("Failed to retrieve BLAST hit: %s", entry) + reason = _safe_retrieval_reason(error) + retrieval_errors.append(f"{entry}: {reason}") + emit_evidence( + "ortholog_retrieval", status="attempt-failed", + hit_id=hit.get("hit_id"), accession=hit.get("accession"), + attempted_identifier=entry, error=reason, + ) + continue + except Exception as error: + retrieval_exception = error + try: + emit_evidence( + "ortholog_retrieval", status="failed", + hit_id=hit.get("hit_id"), accession=hit.get("accession"), + attempted_identifier=entry, + error=_safe_retrieval_reason(error), + ) + except Exception: + LOGGER.exception("Failed to emit BLAST retrieval diagnostic") + raise + if complete_sequence: + break + retrieval_errors.append(f"{entry}: empty sequence returned") + emit_evidence( + "ortholog_retrieval", status="attempt-failed", + hit_id=hit.get("hit_id"), accession=hit.get("accession"), + attempted_identifier=entry, + error="empty sequence returned", + ) + if not complete_sequence: + LOGGER.warning( + "Skipping BLAST hit whose complete protein could not be retrieved: %s", + hit.get("hit_id") or hit.get("accession"), + ) + skipped_count += 1 + reason = "; ".join(retrieval_errors) or "no usable hit identifier was available" + emit_evidence( + "ortholog_retrieval", status="skipped", + hit_id=hit.get("hit_id"), accession=hit.get("accession"), + attempted_identifier="", error=reason, + ) continue try: - complete_sequence = self.__get_complete_sequence(entry) - except subprocess.CalledProcessError: - LOGGER.warning("Failed to retrieve BLAST hit: %s", entry) - continue - if complete_sequence: - break - if not complete_sequence: - LOGGER.warning( - "Skipping BLAST hit whose complete protein could not be retrieved: %s", - hit["hit_id"], - ) - continue - orthologs.append(_make_ortholog(hit, complete_sequence)) + ortholog = _make_ortholog(hit, complete_sequence) + except Exception as error: + record_exception = error + raise + orthologs.append(ortholog) + emit_evidence("ortholog", ortholog=ortholog, status="retained", error="") + except Exception: + if evidence_collector is not None: + try: + if retrieval_exception is not None: + failure_status = "retrieval-failed" + failure_detail = _safe_retrieval_reason(retrieval_exception) + elif record_exception is not None: + failure_status = "record-construction-failed" + failure_detail = _safe_retrieval_reason(record_exception) + elif evidence_exception is not None: + failure_status = "evidence-write-failed" + failure_detail = "evidence callback failed while reporting BLAST results" + else: + failure_status = "processing-failed" + failure_detail = "unexpected BLAST hit-processing failure" + evidence_collector.emit( + "blast_search_summary", + status=failure_status, + hit_count=len(hits), retrieved_count=len(orthologs), + skipped_count=skipped_count, + error=failure_detail, + ) + except Exception: + LOGGER.exception("Failed to emit BLAST failure summary") + raise + + if evidence_collector is not None and hits: + retrieved_count = len(orthologs) + search_status = ( + "retrieval-failed" if retrieved_count == 0 + else "partial-retrieval-failure" if skipped_count + else "completed" + ) + evidence_collector.emit( + "blast_search_summary", status=search_status, + hit_count=len(hits), retrieved_count=retrieved_count, + skipped_count=skipped_count, error="", + ) return orthologs def __get_complete_sequence(self, entry): diff --git a/src/lverage/cli.py b/src/lverage/cli.py new file mode 100644 index 0000000..116c834 --- /dev/null +++ b/src/lverage/cli.py @@ -0,0 +1,545 @@ +"""Command-line interface for the V2 evidence-preserving pipeline.""" + +import argparse +import csv +from datetime import datetime, timezone +import hashlib +import importlib.metadata +import json +import math +import os +from pathlib import Path +import re +import shutil +import sys + + +DNA_ALPHABET = frozenset("ACGTRYSWKMBDHVN") +FASTA_SUFFIXES = frozenset({".fa", ".fasta", ".fna"}) +RESULT_HEADERS = [ + "Row Type", "Input Gene", "Query ORF ID", "Query ORF Selected", + "Stage", "Status", "Error", "Outcome Detail", "Attempted Hit Identifier", + "Ortholog Hit ID", "Ortholog Accession", + "Ortholog Description", "Ortholog Species Name", "Ortholog Species Tax ID", + "BLAST E-value", "Ortholog Identity Ratio", "Ortholog Identity Percent", + "Query Coverage Ratio", "Query Coverage Percent", + "Query Domain Name", "Query Domain ID", "Query Domain Start 0-based", + "Query Domain End 0-based-exclusive", "Ortholog Domain Name", "Ortholog Domain ID", + "Ortholog Domain Start 0-based", "Ortholog Domain End 0-based-exclusive", + "Domain Pair Identity Ratio", "Domain Pair Identity Percent", "Threshold Ratio", + "Threshold Passed", "Motif Database", "Matrix ID", "Motif Name", "Motif Class", + "Motif Inference E-value", "Motif URL", "PFM JSON", +] + + +class InputValidationError(Exception): + """A user-provided input or setting failed preflight validation.""" + + +def _positive_float(value): + try: + parsed = float(value) + except ValueError as error: + raise argparse.ArgumentTypeError("must be a number") from error + if not math.isfinite(parsed) or parsed <= 0: + raise argparse.ArgumentTypeError("must be greater than zero") + return parsed + + +def _identity_ratio(value): + try: + parsed = float(value) + except ValueError as error: + raise argparse.ArgumentTypeError("must be a number") from error + if not math.isfinite(parsed) or not 0 <= parsed <= 1: + raise argparse.ArgumentTypeError("must be between zero and one") + return parsed + + +def _positive_int(value): + try: + parsed = int(value) + except ValueError as error: + raise argparse.ArgumentTypeError("must be an integer") from error + if parsed <= 0: + raise argparse.ArgumentTypeError("must be greater than zero") + return parsed + + +def build_parser(): + """Build argument parser without loading any runtime adapters.""" + parser = argparse.ArgumentParser( + prog="lverage", + description=( + "Run V2 on one nucleotide FASTA file per gene. --output-dir creates " + "a results.tsv, orfs.fasta, and metadata.json evidence bundle." + ), + epilog=( + "Rows are explicitly typed: input, ORF, query_domain, ortholog, " + "ortholog_retrieval, ortholog_domain, domain_comparison, motif, stage_status, or gene_error. " + "Earlier-stage rows are not ortholog/motif hits. --stop-after alignment " + "is an intentional pre-JASPAR stop. Exit 0 means completed or intentionally " + "stopped (including biological no-hits), 2 is preflight failure, and 1 " + "means one or more execution failures with partial evidence preserved." + ), + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument("--input-dir", required=True, metavar="DIR", help="directory of per-gene nucleotide FASTA files") + parser.add_argument("--blast-db", required=True, metavar="PREFIX", help="local protein BLAST database prefix") + parser.add_argument("--species-map", required=True, metavar="JSON", help="JSON mapping species names to NCBI tax IDs") + parser.add_argument("--pfam-db", required=True, metavar="DIR", help="local PfamScan database directory") + parser.add_argument("--output-dir", required=True, metavar="DIR", help="new directory for results.tsv, orfs.fasta, and metadata.json") + parser.add_argument("--blastp", default="blastp", metavar="EXE", help="blastp executable name or path") + parser.add_argument("--blastdbcmd", default="blastdbcmd", metavar="EXE", help="blastdbcmd executable name or path") + parser.add_argument("--pfamscan", default="pfam_scan.pl", metavar="EXE", help="pfam_scan.pl executable name or path") + parser.add_argument("--identity-threshold", type=_identity_ratio, default=0.7, metavar="RATIO", help="minimum domain identity ratio") + parser.add_argument("--valid-pfam", action="append", default=None, metavar="ACCESSION", help="accepted Pfam accession; repeatable") + parser.add_argument("--jaspar-hits", type=_positive_int, default=10, metavar="N", help="maximum JASPAR motif hits") + parser.add_argument("--jaspar-evalue", type=_positive_float, default=1e-6, metavar="VALUE", help="maximum JASPAR inference E-value") + parser.add_argument("--stop-after", choices=("alignment",), help="intentionally stop after domain alignment, without contacting JASPAR") + return parser + + +def _read_species_map(path): + def reject_duplicate_keys(pairs): + mapping = {} + for key, value in pairs: + if key in mapping: + raise ValueError(f"duplicate JSON key: {key}") + mapping[key] = value + return mapping + + try: + with path.open(encoding="utf-8") as handle: + mapping = json.load(handle, object_pairs_hook=reject_duplicate_keys) + except (OSError, UnicodeError, ValueError) as error: + raise InputValidationError(f"cannot read species mapping {path}: {error}") from error + if not isinstance(mapping, dict) or not mapping: + raise InputValidationError("species mapping must be a nonempty JSON object") + normalized = set() + for name, tax_id in mapping.items(): + if not isinstance(name, str) or not name.strip() or name != name.strip(): + raise InputValidationError("species mapping keys must be nonempty, trimmed names") + if name.casefold() in normalized: + raise InputValidationError(f"duplicate species name ignoring case: {name}") + normalized.add(name.casefold()) + if isinstance(tax_id, bool) or not isinstance(tax_id, int) or tax_id <= 0: + raise InputValidationError(f"species mapping tax ID for {name!r} must be a positive integer") + return mapping + + +def _read_fasta(path): + from Bio import SeqIO + try: + with path.open(encoding="utf-8") as handle: + records = list(SeqIO.parse(handle, "fasta")) + except Exception as error: + raise InputValidationError(f"cannot parse FASTA {path}: {error}") from error + if not records: + raise InputValidationError(f"FASTA file contains no records: {path}") + fragments = [] + for index, record in enumerate(records, 1): + sequence = str(record.seq) + if not sequence: + raise InputValidationError(f"empty FASTA record {index} in {path}") + invalid = sorted(set(sequence.upper()) - DNA_ALPHABET) + if invalid: + raise InputValidationError(f"invalid DNA symbols in FASTA record {index} of {path}: {''.join(invalid)}") + fragments.append(sequence) + return fragments + + +def _sha256(path): + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _validate_inputs(args): + input_dir = Path(args.input_dir) + if not input_dir.is_dir(): + raise InputValidationError(f"input directory does not exist: {input_dir}") + fasta_files = sorted( + (item for item in input_dir.iterdir() if item.is_file() and item.suffix.casefold() in FASTA_SUFFIXES), + key=lambda item: (item.name.casefold(), item.name), + ) + if not fasta_files: + raise InputValidationError(f"no .fa, .fasta, or .fna files found in {input_dir}") + identifiers = set() + inputs = [] + for fasta_path in fasta_files: + gene = fasta_path.stem + if gene.casefold() in identifiers: + raise InputValidationError(f"duplicate gene identifier from filename stem: {gene}") + identifiers.add(gene.casefold()) + inputs.append({"gene": gene, "path": str(fasta_path), "fragments": _read_fasta(fasta_path), "sha256": _sha256(fasta_path)}) + species_path = Path(args.species_map) + species_map = _read_species_map(species_path) + pfam_dir = Path(args.pfam_db) + if not pfam_dir.is_dir(): + raise InputValidationError(f"PfamScan database directory does not exist: {pfam_dir}") + if not args.blast_db.strip(): + raise InputValidationError("--blast-db must be a nonempty database prefix") + if not args.blastp.strip() or not args.blastdbcmd.strip() or not args.pfamscan.strip(): + raise InputValidationError("executable paths must not be empty") + if args.valid_pfam is not None and any(not value.strip() for value in args.valid_pfam): + raise InputValidationError("--valid-pfam values must not be empty") + output_dir = Path(args.output_dir) + if os.path.lexists(output_dir): + raise InputValidationError(f"output directory already exists: {output_dir}") + if not output_dir.parent.is_dir(): + raise InputValidationError(f"output parent directory does not exist: {output_dir.parent}") + return inputs, species_map, species_path + + +def _build_pipeline(args, species_map): + from .blast import LocalBlastSearcher + from .jaspar import Jaspar2024MotifDB + from .orf_searcher import OrffinderOrfSearcher + from .pfam import LocalPfamScanner + from .pipeline import Lverage + ortholog_searcher = LocalBlastSearcher( + database_path=args.blast_db, species_map=species_map, + blastp_path=args.blastp, blastdbcmd_path=args.blastdbcmd, + ) + return Lverage( + motif_database_list=[Jaspar2024MotifDB(n_hits=args.jaspar_hits, escore_threshold=args.jaspar_evalue)], + orf_searcher=OrffinderOrfSearcher(), + domain_scanner=LocalPfamScanner(database_path=args.pfam_db, pfamscan_path=args.pfamscan), + ortholog_searcher=ortholog_searcher, + valid_pfam_list=args.valid_pfam or [], dbd_identity_thresh=args.identity_threshold, + ) + + +def _timestamp(): + return datetime.now(timezone.utc).isoformat() + + +def _redact_error(error): + text = re.sub(r"https?://[^\s]+", "", str(error)) + text = re.sub(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}", "", text) + text = re.sub(r"(?i)(token|password|secret|api[_-]?key)=([^\s&]+)", r"\1=", text) + text = re.sub(r"(?i)bearer\s+[A-Za-z0-9._~+/-]+=*", "Bearer ", text) + return text + + +def _orf_id(sequence): + return "orf_" + hashlib.sha256(sequence.encode("ascii")).hexdigest() + + +def _ratio_and_percent(value): + return value, value * 100 if value is not None else None + + +def _event_row(event, gene, sequence_ids, selection_status, selection_details): + row = {header: "" for header in RESULT_HEADERS} + row.update({"Input Gene": gene, "Stage": event.get("stage", ""), "Status": event.get("status", ""), "Error": event.get("error", ""), "Outcome Detail": event.get("detail", "")}) + kind = event["event"] + if kind == "gene_start": + row["Row Type"] = "input" + elif kind == "orf_candidate": + sequence = event["orf_sequence"] + row["Row Type"] = "orf" + row["Query ORF ID"] = sequence_ids[sequence] + row["Query ORF Selected"] = "true" if selection_status.get((gene, sequence)) == "selected" else "false" + if kind == "orf_candidate": + row["Stage"] = "orf" + row["Status"] = selection_status.get((gene, sequence), "not-reached") + row["Outcome Detail"] = selection_details.get((gene, sequence), "") + elif kind == "query_domain": + domain = event["domain"] + row["Row Type"] = "query_domain" + row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] + row["Query ORF Selected"] = "true" if selection_status.get((gene, event["orf_sequence"])) == "selected" else "false" + row.update({"Query Domain Name": domain.name, "Query Domain ID": domain.accession, "Query Domain Start 0-based": domain.start, "Query Domain End 0-based-exclusive": domain.end}) + elif kind == "ortholog": + ortholog = event["ortholog"] + row["Row Type"] = "ortholog" + row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] + row.update(_ortholog_fields(ortholog)) + elif kind == "ortholog_retrieval": + row["Row Type"] = "ortholog_retrieval" + row["Stage"] = "blast" + if event.get("orf_sequence"): + row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] + row["Attempted Hit Identifier"] = event.get("attempted_identifier", "") + row["Ortholog Hit ID"] = event.get("hit_id") or "" + row["Ortholog Accession"] = event.get("accession") or "" + elif kind in {"blast_search", "blast_search_summary"}: + row["Row Type"] = "stage_status" + row["Stage"] = "blast" + if event.get("orf_sequence"): + row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] + details = [] + for key in ("hit_count", "retained_hit_count", "retrieved_count", "skipped_count"): + if key in event: + details.append(f"{key}={event[key]}") + row["Outcome Detail"] = "; ".join(details) + elif kind == "ortholog_domain": + domain = event["domain"] + row["Row Type"] = "ortholog_domain" + row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] + row.update(_ortholog_fields(event["ortholog"])) + row.update({"Ortholog Domain Name": domain.name, "Ortholog Domain ID": domain.accession, "Ortholog Domain Start 0-based": domain.start, "Ortholog Domain End 0-based-exclusive": domain.end}) + elif kind == "domain_comparison": + qdomain, odomain = event["query_domain"], event["ortholog_domain"] + row["Row Type"] = "domain_comparison" + row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] + row.update(_ortholog_fields(event["ortholog"])) + row.update({"Query Domain Name": qdomain.name, "Query Domain ID": qdomain.accession, "Query Domain Start 0-based": qdomain.start, "Query Domain End 0-based-exclusive": qdomain.end, "Ortholog Domain Name": odomain.name, "Ortholog Domain ID": odomain.accession, "Ortholog Domain Start 0-based": odomain.start, "Ortholog Domain End 0-based-exclusive": odomain.end, "Domain Pair Identity Ratio": event["domain_identity"], "Domain Pair Identity Percent": event["domain_identity"] * 100, "Threshold Ratio": event.get("threshold"), "Threshold Passed": "true" if event["status"] == "accepted" else "false"}) + elif kind == "motif": + row["Row Type"] = "motif" + row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] + row.update(_ortholog_fields(event["ortholog"])) + qdomain, odomain = event["query_domain"], event["ortholog_domain"] + row.update({"Query Domain Name": qdomain.name, "Query Domain ID": qdomain.accession, "Query Domain Start 0-based": qdomain.start, "Query Domain End 0-based-exclusive": qdomain.end, "Ortholog Domain Name": odomain.name, "Ortholog Domain ID": odomain.accession, "Ortholog Domain Start 0-based": odomain.start, "Ortholog Domain End 0-based-exclusive": odomain.end, "Domain Pair Identity Ratio": event["domain_identity"], "Domain Pair Identity Percent": event["domain_identity"] * 100, "Threshold Ratio": event.get("threshold"), "Threshold Passed": "true", "Motif Database": event["motif_database_name"]}) + motif = event["motif_record"] + motif_values = dict(zip(motif.get_headers(), motif.get_values())) + mapping = {"Matrix ID": "Matrix ID", "Motif Name": "Motif Name", "Motif Class": "Motif Class", "Motif Inference E-value": "Inference E-value", "Motif URL": "Motif URL"} + for target, source in mapping.items(): + row[target] = motif_values.get(source, "") + pfm = motif_values.get("PFM", "") + row["PFM JSON"] = json.dumps(pfm, ensure_ascii=False, separators=(",", ":")) if pfm != "" else "" + elif kind == "gene_outcome": + row["Row Type"] = "stage_status" + elif kind == "gene_failure": + row["Row Type"] = "gene_error" + elif kind == "stage_status": + row["Row Type"] = "stage_status" + else: + return None + return row + + +def _ortholog_fields(ortholog): + identity, identity_percent = _ratio_and_percent(ortholog.identity) + coverage, coverage_percent = _ratio_and_percent(ortholog.query_coverage) + return { + "Ortholog Hit ID": ortholog.hit_id or "", "Ortholog Accession": ortholog.accession, + "Ortholog Description": ortholog.description, "Ortholog Species Name": ortholog.species_name, + "Ortholog Species Tax ID": ortholog.species_tax_id, "BLAST E-value": ortholog.evalue, + "Ortholog Identity Ratio": identity, "Ortholog Identity Percent": identity_percent, + "Query Coverage Ratio": coverage, "Query Coverage Percent": coverage_percent, + } + + +class _Bundle: + def __init__(self, destination, metadata): + self.path = destination + self.metadata = metadata + self.events = [] + self.sequence_by_id = {} + self.sequence_ids = {} + self._write_results() + self._write_orfs() + self.flush_metadata() + + def add_event(self, gene, event): + self.events.append((gene, event)) + sequence = event.get("orf_sequence") + if sequence: + identifier = _orf_id(sequence) + self.sequence_ids[sequence] = identifier + self.sequence_by_id[identifier] = sequence + self.metadata["current_gene"] = gene + self.metadata["current_stage"] = event.get("stage", "unknown") + self.metadata["updated_at"] = _timestamp() + self._write_results() + self._write_orfs() + self.flush_metadata() + + def _write_results(self): + selection = {} + selection_details = {} + for gene, event in self.events: + if event["event"] == "orf_selection": + selection[(gene, event["orf_sequence"])] = event["status"] + selection_details[(gene, event["orf_sequence"])] = event.get("detail", "") + selected_by_gene = { + gene: any(status == "selected" for (event_gene, _), status in selection.items() if event_gene == gene) + for gene, _ in self.events + } + for gene, event in self.events: + if event["event"] == "orf_candidate" and (gene, event["orf_sequence"]) not in selection and selected_by_gene.get(gene): + selection[(gene, event["orf_sequence"])] = "not-selected" + path = self.path / "results.tsv" + with path.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=RESULT_HEADERS, delimiter="\t", lineterminator="\n", extrasaction="ignore") + writer.writeheader() + for gene, event in self.events: + row = _event_row(event, gene, self.sequence_ids, selection, selection_details) + if row is not None: + writer.writerow(row) + handle.flush() + os.fsync(handle.fileno()) + + def _write_orfs(self): + with (self.path / "orfs.fasta").open("w", encoding="utf-8", newline="\n") as handle: + for identifier in sorted(self.sequence_by_id): + sequence = self.sequence_by_id[identifier] + handle.write(f">{identifier}\n") + for start in range(0, len(sequence), 80): + handle.write(sequence[start:start + 80] + "\n") + handle.flush() + os.fsync(handle.fileno()) + + def flush_metadata(self): + temporary = self.path / ".metadata.json.tmp" + with temporary.open("w", encoding="utf-8") as handle: + json.dump(self.metadata, handle, indent=2, ensure_ascii=False) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, self.path / "metadata.json") + + +def _metadata(args, inputs, species_path): + try: + package_version = importlib.metadata.version("lverage") + except importlib.metadata.PackageNotFoundError: + package_version = "unknown" + return { + "schema_version": 1, + "started_at": _timestamp(), "updated_at": _timestamp(), "finished_at": None, + "completion": "in-progress", "current_gene": None, "current_stage": "preflight-complete", + "summary": {"genes_total": len(inputs), "genes_completed": 0, "genes_failed": 0, "genes_intentionally_stopped": 0}, + "errors": [], + "tools": { + "lverage": {"version": package_version}, + "blastp": {"configured_executable": args.blastp, "resolved_executable": shutil.which(args.blastp), "version": "unknown"}, + "blastdbcmd": {"configured_executable": args.blastdbcmd, "resolved_executable": shutil.which(args.blastdbcmd), "version": "unknown"}, + "pfamscan": {"configured_executable": args.pfamscan, "resolved_executable": shutil.which(args.pfamscan), "version": "unknown"}, + "orffinder": {"version": _distribution_version("orffinder")}, + "biopython": {"version": _distribution_version("biopython")}, + "jaspar": {"release": "JASPAR2024", "service_version": "unknown"}, + }, + "settings": { + "identity_threshold_ratio": args.identity_threshold, + "valid_pfam": args.valid_pfam or [], "jaspar_hits": args.jaspar_hits, + "jaspar_evalue": args.jaspar_evalue, "stop_after": args.stop_after or "full-pipeline", + "orf_minimum_length_nt": 75, "blast_evalue": 1e-6, "blast_top_n": 20, + }, + "references": { + "input_files": [{"gene": item["gene"], "path": item["path"], "sha256": item["sha256"]} for item in inputs], + "species_map": {"path": str(species_path), "sha256": _sha256(species_path)}, + "blast_database_prefix": args.blast_db, "pfam_database_directory": args.pfam_db, + }, + "sequence_id_convention": "orf_; identical sequences share one FASTA record; table rows retain per-gene associations", + "limitations": ["Evidence is flushed as pipeline events complete; this is not checkpoint/resume or arbitrary-crash recovery."], + } + + +def _distribution_version(name): + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return "unknown" + + +def _stage_completion_events(final_status, last_stage): + stages = ("orf", "query_pfam", "blast", "ortholog_pfam", "alignment", "motif") + last_index = stages.index(last_stage) if last_stage in stages else -1 + events = [] + if final_status == "STOPPED_AFTER_ALIGNMENT": + events.append({"event": "stage_status", "stage": "motif", "status": "not-reached", "detail": "intentional stop after alignment"}) + return events + for stage in stages[last_index + 1:]: + events.append({"event": "stage_status", "stage": stage, "status": "not-reached", "detail": "upstream stage did not produce required evidence"}) + return events + + +def main(argv=None): + parser = build_parser() + args = parser.parse_args(argv) + try: + inputs, species_map, species_path = _validate_inputs(args) + except (InputValidationError, OSError, ValueError) as error: + print(f"lverage: invalid input/configuration: {_redact_error(error)}", file=sys.stderr) + return 2 + + output_dir = Path(args.output_dir) + try: + output_dir.mkdir() + except OSError as error: + print(f"lverage: could not create output directory: {_redact_error(error)}", file=sys.stderr) + return 2 + + metadata = _metadata(args, inputs, species_path) + bundle = _Bundle(output_dir, metadata) + failures = 0 + try: + pipeline = _build_pipeline(args, species_map) + except Exception as error: + failures = len(inputs) + safe_error = _redact_error(error) + for item in inputs: + gene = item["gene"] + metadata["errors"].append({"gene": gene, "stage": "initialization", "error": safe_error}) + bundle.add_event(gene, {"event": "gene_failure", "stage": "initialization", "status": "failed", "error": safe_error}) + metadata["summary"]["genes_failed"] = failures + metadata["completion"] = "failed" + metadata["finished_at"] = _timestamp() + bundle.flush_metadata() + print(f"lverage: pipeline initialization failed: {safe_error}", file=sys.stderr) + return 1 + + from .evidence import EvidenceCollector + for item in inputs: + gene = item["gene"] + bundle.add_event(gene, {"event": "gene_start", "stage": "input", "status": "in-progress", "error": ""}) + collector = EvidenceCollector(callback=lambda event, gene=gene: bundle.add_event(gene, event)) + try: + pipeline.run(item["fragments"], evidence_collector=collector, stop_after=args.stop_after) + status = pipeline.lverage_code.name + for event in _stage_completion_events(status, collector.stage): + bundle.add_event(gene, event) + retrieval_failures = [ + event for event in collector.events + if event.get("event") == "ortholog_retrieval" + and event.get("status") == "skipped" + ] + if retrieval_failures: + failures += 1 + metadata["summary"]["genes_failed"] = failures + metadata["errors"].append({ + "gene": gene, "stage": "blast", + "error": f"{len(retrieval_failures)} BLAST hit(s) could not be retrieved", + }) + print( + f"lverage: {len(retrieval_failures)} BLAST hit(s) could not be retrieved for gene {gene}", + file=sys.stderr, + ) + if status == "STOPPED_AFTER_ALIGNMENT": + metadata["summary"]["genes_intentionally_stopped"] += 1 + elif not retrieval_failures: + metadata["summary"]["genes_completed"] += 1 + if not any(event["event"] == "gene_outcome" for event in collector.events): + bundle.add_event(gene, {"event": "gene_outcome", "stage": collector.stage, "status": status, "error": ""}) + except Exception as error: + failures += 1 + safe_error = _redact_error(error) + metadata["errors"].append({"gene": gene, "stage": collector.stage, "error": safe_error}) + metadata["summary"]["genes_failed"] = failures + bundle.add_event(gene, {"event": "gene_failure", "stage": collector.stage, "status": "failed", "error": safe_error}) + stages = ("orf", "query_pfam", "blast", "ortholog_pfam", "alignment", "motif") + last_index = stages.index(collector.stage) if collector.stage in stages else -1 + for stage in stages[last_index + 1:]: + bundle.add_event(gene, {"event": "stage_status", "stage": stage, "status": "not-reached", "detail": "execution failed in upstream stage"}) + print(f"lverage: execution failed for gene {gene} at {collector.stage}: {safe_error}", file=sys.stderr) + metadata["current_gene"] = gene + metadata["current_stage"] = "complete" + metadata["updated_at"] = _timestamp() + bundle.flush_metadata() + + metadata["finished_at"] = _timestamp() + metadata["completion"] = "failed" if failures else ("intentionally-stopped" if metadata["summary"]["genes_intentionally_stopped"] else "completed") + metadata["current_gene"] = None + metadata["current_stage"] = "finished" + bundle.flush_metadata() + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/lverage/evidence.py b/src/lverage/evidence.py new file mode 100644 index 0000000..815b9e5 --- /dev/null +++ b/src/lverage/evidence.py @@ -0,0 +1,29 @@ +"""Small event collector for incremental pipeline evidence.""" + + +class EvidenceCollector: + """Forward completed pipeline evidence to a consumer as it is produced. + + The collector is intentionally in-memory and per invocation. It is not a + checkpoint or crash-recovery mechanism. + """ + + def __init__(self, callback=None): + self.callback = callback + self.events = [] + self.stage = "preflight" + self.context = {} + + def set_stage(self, stage): + self.stage = stage + + def set_context(self, **context): + """Attach identifying context to subsequently reported events.""" + self.context.update(context) + + def emit(self, event, **payload): + item = {"event": event, "stage": self.stage, **self.context, **payload} + self.events.append(item) + if self.callback is not None: + self.callback(item) + return item diff --git a/src/lverage/ortholog_searcher.py b/src/lverage/ortholog_searcher.py index 41a2fc6..f41f6e7 100644 --- a/src/lverage/ortholog_searcher.py +++ b/src/lverage/ortholog_searcher.py @@ -27,6 +27,8 @@ class OrthologRecord: Exact identity ratio from zero to one query_coverage : float Query coverage ratio from zero to one + hit_id : str, optional + Native identifier returned by the search backend, when available """ accession : str @@ -37,6 +39,7 @@ class OrthologRecord: evalue : float identity : float query_coverage : float + hit_id : str | None = None class OrthologSearcherTemplate(ABC): diff --git a/src/lverage/pipeline.py b/src/lverage/pipeline.py index a868912..abaedb3 100644 --- a/src/lverage/pipeline.py +++ b/src/lverage/pipeline.py @@ -45,6 +45,7 @@ class LverageCode(Enum): NO_ORTHOLOGS = 3 NO_VALID_DBD = 4 NO_MOTIF = 5 + STOPPED_AFTER_ALIGNMENT = 6 class Lverage: @@ -75,6 +76,7 @@ class Lverage: LverageCode.NO_ORTHOLOGS: "No orthologs were found", LverageCode.NO_VALID_DBD: "No valid DNA-binding domain was found in the orthologs", LverageCode.NO_MOTIF: "No motif was found in the orthologs", + LverageCode.STOPPED_AFTER_ALIGNMENT: "Intentionally stopped after alignment", } def __init__(self, @@ -127,18 +129,28 @@ def validate_arguments(self) -> None: def _select_query_orf(self, tf_sequences): return self._select_orf_from_candidates(self._get_sorted_orfs(tf_sequences)) - def _get_sorted_orfs(self, tf_sequences): + def _get_sorted_orfs(self, tf_sequences, evidence_collector=None): orf_list = [] + seen_orfs = set() for sequence in tf_sequences: - orf_list.extend(self.orf_searcher.get_orfs(sequence)) + for orf in self.orf_searcher.get_orfs(sequence): + if orf in seen_orfs: + continue + seen_orfs.add(orf) + orf_list.append(orf) + if evidence_collector is not None: + evidence_collector.emit( + "orf_candidate", orf_sequence=orf, + orf_index=len(orf_list), status="candidate", error="", + ) - orf_list = list(dict.fromkeys(orf_list)) orf_list.sort(key=len, reverse=True) return orf_list - def _select_orf_from_candidates(self, orf_list): - for orf in orf_list: + def _select_orf_from_candidates(self, orf_list, evidence_collector=None): + for orf_index, orf in enumerate(orf_list, start=1): domains = self.domain_scanner.get_domains(orf) + detected_domains = list(domains) if self.valid_pfam_list: domains = [ domain for domain in domains @@ -148,8 +160,26 @@ def _select_orf_from_candidates(self, orf_list): for valid_pfam in self.valid_pfam_list ) ] + if evidence_collector is not None: + for domain in detected_domains: + accepted = domain in domains + evidence_collector.emit( + "query_domain", orf_sequence=orf, orf_index=orf_index, + domain=domain, status="accepted" if accepted else "not-selected", + error="", + ) if domains: + if evidence_collector is not None: + evidence_collector.emit( + "orf_selection", orf_sequence=orf, orf_index=orf_index, + status="selected", error="", + ) return orf, domains + if evidence_collector is not None: + evidence_collector.emit( + "orf_selection", orf_sequence=orf, orf_index=orf_index, + status="biological-no-hit", detail="no accepted query domain", + ) return None, [] @staticmethod @@ -218,7 +248,8 @@ def _get_domain_slice(sequence, domain): raise ValueError("domain bounds must define a nonempty slice within the sequence") return sequence[start:end] - def run(self, tf_sequence : str | list[str]) -> list[LverageRecord]: + def run(self, tf_sequence : str | list[str], *, evidence_collector=None, + stop_after=None) -> list[LverageRecord]: """ Run motif inference for one transcription factor. @@ -234,27 +265,78 @@ def run(self, tf_sequence : str | list[str]) -> list[LverageRecord]: """ self.lverage_code = LverageCode.NOT_SET + if stop_after not in (None, "alignment"): + raise ValueError("stop_after must be None or 'alignment'") tf_sequences = self._validate_run_input(tf_sequence) - orf_list = self._get_sorted_orfs(tf_sequences) + if evidence_collector is not None: + evidence_collector.set_stage("orf") + orf_list = self._get_sorted_orfs(tf_sequences, evidence_collector) if not orf_list: self.lverage_code = LverageCode.NO_ORF + if evidence_collector is not None: + evidence_collector.emit("gene_outcome", status="biological-no-hit", detail="no ORF") return [] - query_orf, query_domains = self._select_orf_from_candidates(orf_list) + if evidence_collector is not None: + evidence_collector.set_stage("query_pfam") + query_orf, query_domains = self._select_orf_from_candidates(orf_list, evidence_collector) if query_orf is None: self.lverage_code = LverageCode.NO_VALID_DOMAIN + if evidence_collector is not None: + evidence_collector.emit("gene_outcome", status="biological-no-hit", detail="no accepted query domain") return [] - orthologs = self.ortholog_searcher.get_orthologs(query_orf) + if evidence_collector is not None: + evidence_collector.set_stage("blast") + if hasattr(evidence_collector, "set_context"): + evidence_collector.set_context(orf_sequence=query_orf) + supports_evidence = getattr( + self.ortholog_searcher, "supports_evidence_collector", False + ) + if evidence_collector is not None and supports_evidence: + orthologs = self.ortholog_searcher.get_orthologs( + query_orf, evidence_collector=evidence_collector + ) + else: + orthologs = self.ortholog_searcher.get_orthologs(query_orf) + if evidence_collector is not None and not supports_evidence: + for ortholog_index, ortholog in enumerate(orthologs, start=1): + evidence_collector.emit( + "ortholog", orf_sequence=query_orf, ortholog=ortholog, + ortholog_index=ortholog_index, status="retained", error="", + ) if not orthologs: self.lverage_code = LverageCode.NO_ORTHOLOGS + if evidence_collector is not None: + retrieval_failed = any( + event.get("event") == "blast_search_summary" + and event.get("status") == "retrieval-failed" + for event in getattr(evidence_collector, "events", []) + ) + evidence_collector.emit( + "gene_outcome", + status="failed" if retrieval_failed else "biological-no-hit", + detail=("all BLAST hits failed sequence retrieval" if retrieval_failed + else "no retained ortholog"), + ) return [] records = [] has_valid_domain_pair = False for ortholog in orthologs: + if evidence_collector is not None: + evidence_collector.set_stage("ortholog_pfam") ortholog_domains = self.domain_scanner.get_domains(ortholog.sequence) + if evidence_collector is not None: + for domain_index, domain in enumerate(ortholog_domains, start=1): + evidence_collector.emit( + "ortholog_domain", orf_sequence=query_orf, ortholog=ortholog, + domain=domain, domain_index=domain_index, + status="retained", error="", + ) + if evidence_collector is not None: + evidence_collector.set_stage("alignment") for query_domain in query_domains: for ortholog_domain in ortholog_domains: if self._base_accession(query_domain.accession) != self._base_accession(ortholog_domain.accession): @@ -265,6 +347,17 @@ def run(self, tf_sequence : str | list[str]) -> list[LverageRecord]: ortholog.sequence, ortholog_domain, ) + passes_threshold = domain_identity >= self.dbd_identity_thresh + if evidence_collector is not None: + evidence_collector.set_stage("alignment") + evidence_collector.emit( + "domain_comparison", orf_sequence=query_orf, ortholog=ortholog, + query_domain=query_domain, ortholog_domain=ortholog_domain, + domain_identity=domain_identity, + threshold=self.dbd_identity_thresh, + status="accepted" if passes_threshold else "below-threshold", + error="", + ) if domain_identity < self.dbd_identity_thresh: continue @@ -276,6 +369,10 @@ def run(self, tf_sequence : str | list[str]) -> list[LverageRecord]: ortholog_domain=ortholog_domain, ortholog_species_tax_id=ortholog.species_tax_id, ) + if stop_after == "alignment": + continue + if evidence_collector is not None: + evidence_collector.set_stage("motif") for motif_database in self.motif_database_list: if not motif_database.check_species_validity(ortholog.species_tax_id): LOGGER.info( @@ -293,6 +390,22 @@ def run(self, tf_sequence : str | list[str]) -> list[LverageRecord]: motif_database_name=motif_database.name, motif_record=motif_record, )) + if evidence_collector is not None: + evidence_collector.emit( + "motif", orf_sequence=query_orf, ortholog=ortholog, + query_domain=query_domain, ortholog_domain=ortholog_domain, + domain_identity=domain_identity, + threshold=self.dbd_identity_thresh, + motif_database_name=motif_database.name, + motif_record=motif_record, status="retained", error="", + ) + + if stop_after == "alignment": + self.lverage_code = LverageCode.STOPPED_AFTER_ALIGNMENT + if evidence_collector is not None: + evidence_collector.set_stage("alignment") + evidence_collector.emit("gene_outcome", status="intentional-stop", error="") + return [] if records: self.lverage_code = LverageCode.SUCCESS @@ -300,6 +413,11 @@ def run(self, tf_sequence : str | list[str]) -> list[LverageRecord]: self.lverage_code = LverageCode.NO_MOTIF else: self.lverage_code = LverageCode.NO_VALID_DBD + if evidence_collector is not None and self.lverage_code in (LverageCode.NO_MOTIF, LverageCode.NO_VALID_DBD): + evidence_collector.emit( + "gene_outcome", status="biological-no-hit", + detail=self.lverage_code.name, + ) return records @staticmethod diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py new file mode 100644 index 0000000..311f253 --- /dev/null +++ b/tests/v2/test_cli.py @@ -0,0 +1,435 @@ +import csv +import hashlib +import io +import json +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from lverage.cli import RESULT_HEADERS, _build_pipeline, _validate_inputs, build_parser, main +from lverage.domain_scanner import DomainRecord, DomainScannerTemplate +from lverage.jaspar import JasparRecord +from lverage.motif_database import MotifDBTemplate +from lverage.orf_searcher import OrfSearcherTemplate +from lverage.ortholog_searcher import OrthologRecord, OrthologSearcherTemplate +from lverage.pipeline import Lverage + + +class StubOrfSearcher(OrfSearcherTemplate): + + def __init__(self, results): + self.results = results + + def get_orfs(self, sequence): + return list(self.results.get(sequence, [])) + + +class StubDomainScanner(DomainScannerTemplate): + + def __init__(self, results=None, fail_on=None): + self.results = results or {} + self.fail_on = fail_on + self.scanned = [] + + def get_domains(self, sequence): + self.scanned.append(sequence) + if sequence == self.fail_on: + raise RuntimeError("simulated Pfam failure") + return list(self.results.get(sequence, [])) + + +class StubOrthologSearcher(OrthologSearcherTemplate): + + def __init__(self, orthologs=None, fail_on=None): + self.orthologs = orthologs or [] + self.fail_on = fail_on + + def get_orthologs(self, sequence): + if sequence == self.fail_on: + raise RuntimeError("simulated ortholog search failure") + return list(self.orthologs) + + +class IncrementalPartialSearcher(StubOrthologSearcher): + supports_evidence_collector = True + + def get_orthologs(self, sequence, *, evidence_collector=None): + evidence_collector.emit( + "blast_search", status="hits-found", hit_count=4, + retained_hit_count=4, retrieved_count=0, skipped_count=0, + ) + for ortholog in self.orthologs: + evidence_collector.emit("ortholog", ortholog=ortholog, status="retained", error="") + evidence_collector.emit( + "ortholog_retrieval", status="skipped", hit_id="native|hit4|", + accession="NP_4", attempted_identifier="", error="simulated retrieval failure", + ) + evidence_collector.emit( + "blast_search_summary", status="partial-retrieval-failure", + hit_count=4, retrieved_count=3, skipped_count=1, + ) + return list(self.orthologs) + + +class StubMotifDatabase(MotifDBTemplate): + + name = "StubJASPAR" + + def __init__(self, records=None, fail_on_call=None): + self.records = records or [] + self.fail_on_call = fail_on_call + self.calls = 0 + + def check_species_validity(self, species_tax_id): + return True + + def search(self, request): + self.calls += 1 + if self.calls == self.fail_on_call: + raise RuntimeError("simulated JASPAR failure") + return list(self.records) + + +def make_motif(matrix_id, name): + return JasparRecord( + matrix_id, name, {"A": [1, 2], "C": [3, 4]}, + f"https://example.org/matrix/{matrix_id}/", "Homeobox", 1e-8, + ) + + +def make_ortholog(accession="NP_1", sequence="AAAA", description=None): + return OrthologRecord( + accession, description or "Homeobox protein [Homo sapiens]", + "Homo sapiens", 9606, sequence, 1e-20, 0.8, 0.9, + hit_id=f"native|{accession}|", + ) + + +class LverageCliTests(unittest.TestCase): + + def setUp(self): + self.temp_dir = tempfile.TemporaryDirectory() + self.root = Path(self.temp_dir.name) + self.input_dir = self.root / "inputs" + self.input_dir.mkdir() + self.pfam_dir = self.root / "pfam" + self.pfam_dir.mkdir() + self.species_map = self.root / "species.json" + self.species_map.write_text('{"Homo sapiens": 9606}', encoding="utf-8") + + def tearDown(self): + self.temp_dir.cleanup() + + def write_fasta(self, name, sequence="ACGT"): + path = self.input_dir / name + path.write_text(f">{Path(name).stem}\n{sequence}\n", encoding="utf-8") + return path + + def arguments(self, output_name="bundle", *extra): + return [ + "--input-dir", str(self.input_dir), + "--blast-db", str(self.root / "blast"), + "--species-map", str(self.species_map), + "--pfam-db", str(self.pfam_dir), + "--output-dir", str(self.root / output_name), + *extra, + ] + + def make_pipeline(self, *, orfs=None, domains=None, orthologs=None, database=None, + scanner_fail_on=None, ortholog_fail_on=None): + return Lverage( + motif_database_list=[database or StubMotifDatabase()], + orf_searcher=StubOrfSearcher(orfs or {}), + domain_scanner=StubDomainScanner(domains, fail_on=scanner_fail_on), + ortholog_searcher=StubOrthologSearcher(orthologs, fail_on=ortholog_fail_on), + ) + + def run_cli(self, pipeline, *arguments): + stderr = io.StringIO() + with patch("lverage.cli._build_pipeline", return_value=pipeline): + with redirect_stdout(io.StringIO()), redirect_stderr(stderr): + code = main(list(arguments)) + return code, stderr.getvalue() + + def read_rows(self, output_dir): + with (output_dir / "results.tsv").open(encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle, delimiter="\t")) + + def test_bundle_contains_all_three_files_and_unique_orfs_with_gene_associations(self): + self.write_fasta("alpha.fa") + self.write_fasta("beta.fa", "TGCA") + pipeline = self.make_pipeline( + orfs={"ACGT": ["AAAA", "CCCC"], "TGCA": ["AAAA"]}, + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, + orthologs=[], + ) + destination = self.root / "bundle" + + code, _ = self.run_cli(pipeline, *self.arguments()) + + self.assertEqual(code, 0) + self.assertEqual({path.name for path in destination.iterdir()}, { + "results.tsv", "orfs.fasta", "metadata.json", + }) + fasta = (destination / "orfs.fasta").read_text(encoding="utf-8") + self.assertEqual(fasta.count(">"), 2) + shared_id = "orf_" + hashlib.sha256(b"AAAA").hexdigest() + self.assertIn(f">{shared_id}\nAAAA\n", fasta) + rows = self.read_rows(destination) + associations = { + row["Input Gene"] for row in rows + if row["Row Type"] == "orf" and row["Query ORF ID"] == shared_id + } + self.assertEqual(associations, {"alpha", "beta"}) + self.assertEqual(len(RESULT_HEADERS), len(next(csv.reader( + io.StringIO((destination / "results.tsv").read_text(encoding="utf-8")), delimiter="\t" + )))) + metadata = json.loads((destination / "metadata.json").read_text(encoding="utf-8")) + self.assertEqual(metadata["summary"]["genes_completed"], 2) + self.assertEqual(metadata["completion"], "completed") + + def test_no_orfs_and_no_blast_hits_are_biological_no_hits(self): + self.write_fasta("empty_orfs.fa") + no_orf_pipeline = self.make_pipeline(orfs={"ACGT": []}) + no_orf_dir = self.root / "no-orfs" + no_orf_code, _ = self.run_cli(no_orf_pipeline, *self.arguments("no-orfs")) + no_orf_rows = self.read_rows(no_orf_dir) + self.assertEqual(no_orf_code, 0) + self.assertTrue(any(row["Row Type"] == "input" for row in no_orf_rows)) + self.assertTrue(any(row["Status"] == "biological-no-hit" for row in no_orf_rows)) + self.assertTrue(any(row["Stage"] == "blast" and row["Status"] == "not-reached" for row in no_orf_rows)) + + self.write_fasta("no_hits.fa") + no_hit_pipeline = self.make_pipeline( + orfs={"ACGT": ["AAAA"]}, + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, + orthologs=[], + ) + no_hit_dir = self.root / "no-hits" + no_hit_code, _ = self.run_cli(no_hit_pipeline, *self.arguments("no-hits")) + no_hit_rows = self.read_rows(no_hit_dir) + self.assertEqual(no_hit_code, 0) + self.assertTrue(any(row["Stage"] == "blast" and row["Status"] == "biological-no-hit" for row in no_hit_rows)) + + def test_below_threshold_comparison_is_exported_with_distinct_status(self): + self.write_fasta("below.fa") + pipeline = self.make_pipeline( + orfs={"ACGT": ["AAAA"]}, + domains={ + "AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)], + "TTTT": [DomainRecord("Homeobox", "PF00046.2", 0, 4)], + }, + orthologs=[make_ortholog("NP_low", "TTTT")], + ) + destination = self.root / "below" + + code, _ = self.run_cli(pipeline, *self.arguments("below")) + + rows = self.read_rows(destination) + comparisons = [row for row in rows if row["Row Type"] == "domain_comparison"] + self.assertEqual(code, 0) + self.assertEqual(len(comparisons), 1) + self.assertEqual(comparisons[0]["Status"], "below-threshold") + self.assertEqual(comparisons[0]["Threshold Passed"], "false") + self.assertEqual(comparisons[0]["Domain Pair Identity Ratio"], "0.0") + self.assertTrue(any(row["Stage"] == "motif" and row["Status"] == "not-reached" for row in rows)) + + def test_hits_one_to_three_are_exported_once_when_hit_four_is_skipped(self): + self.write_fasta("blast_partial.fa") + retained = [make_ortholog(f"NP_{index}", "AAAA") for index in range(1, 4)] + pipeline = self.make_pipeline( + orfs={"ACGT": ["AAAA"]}, + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, + ) + pipeline.ortholog_searcher = IncrementalPartialSearcher(retained) + destination = self.root / "blast-partial" + + code, _ = self.run_cli(pipeline, *self.arguments("blast-partial")) + + rows = self.read_rows(destination) + ortholog_rows = [row for row in rows if row["Row Type"] == "ortholog"] + skipped_rows = [row for row in rows if row["Row Type"] == "ortholog_retrieval" and row["Status"] == "skipped"] + summaries = [row for row in rows if row["Row Type"] == "stage_status" and row["Stage"] == "blast"] + self.assertEqual(code, 1) + self.assertEqual([row["Ortholog Accession"] for row in ortholog_rows], ["NP_1", "NP_2", "NP_3"]) + self.assertEqual(len(ortholog_rows), 3) + self.assertEqual(len(skipped_rows), 1) + self.assertEqual(skipped_rows[0]["Ortholog Hit ID"], "native|hit4|") + self.assertTrue(any("partial-retrieval-failure" == row["Status"] for row in summaries)) + metadata = json.loads((destination / "metadata.json").read_text(encoding="utf-8")) + self.assertEqual(metadata["summary"]["genes_failed"], 1) + + def test_multiple_motifs_each_have_one_correctly_supported_row(self): + self.write_fasta("motifs.fa") + database = StubMotifDatabase([ + make_motif("MA0001.1", "motif one"), + make_motif("MA0002.1", "motif two"), + ]) + pipeline = self.make_pipeline( + orfs={"ACGT": ["AAAA"]}, + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, + orthologs=[make_ortholog("NP_1", "AAAA")], database=database, + ) + destination = self.root / "motifs" + + code, _ = self.run_cli(pipeline, *self.arguments("motifs")) + + motifs = [row for row in self.read_rows(destination) if row["Row Type"] == "motif"] + self.assertEqual(code, 0) + self.assertEqual(len(motifs), 2) + self.assertEqual({row["Matrix ID"] for row in motifs}, {"MA0001.1", "MA0002.1"}) + self.assertTrue(all(row["Input Gene"] == "motifs" for row in motifs)) + self.assertTrue(all(row["Ortholog Accession"] == "NP_1" for row in motifs)) + self.assertTrue(all(row["Query Domain ID"] == "PF00046.1" for row in motifs)) + self.assertTrue(all(row["Ortholog Domain ID"] == "PF00046.1" for row in motifs)) + self.assertEqual(json.loads(motifs[0]["PFM JSON"]), {"A": [1, 2], "C": [3, 4]}) + + def test_later_ortholog_failure_preserves_completed_first_ortholog_evidence(self): + self.write_fasta("later_ortholog.fa") + first = make_ortholog("NP_first", "AAAA") + second = make_ortholog("NP_second", "CCCC") + pipeline = self.make_pipeline( + orfs={"ACGT": ["AAAA"]}, + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, + orthologs=[first, second], scanner_fail_on="CCCC", + ) + destination = self.root / "ortholog-failure" + + code, _ = self.run_cli(pipeline, *self.arguments("ortholog-failure")) + + rows = self.read_rows(destination) + self.assertEqual(code, 1) + self.assertTrue(any(row["Row Type"] == "ortholog_domain" and row["Ortholog Accession"] == "NP_first" for row in rows)) + self.assertTrue(any(row["Row Type"] == "gene_error" and row["Stage"] == "ortholog_pfam" for row in rows)) + + def test_jaspar_failure_preserves_prior_motif_and_alignment_evidence(self): + self.write_fasta("jaspar_failure.fa") + sequence = "AAAAAAAA" + database = StubMotifDatabase([make_motif("MA0001.1", "first")], fail_on_call=2) + pipeline = self.make_pipeline( + orfs={"ACGT": [sequence]}, + domains={sequence: [ + DomainRecord("First", "PF00046.1", 0, 4), + DomainRecord("Second", "PF00046.2", 4, 8), + ]}, + orthologs=[make_ortholog("NP_1", sequence)], database=database, + ) + destination = self.root / "jaspar-failure" + + code, _ = self.run_cli(pipeline, *self.arguments("jaspar-failure")) + + rows = self.read_rows(destination) + self.assertEqual(code, 1) + self.assertEqual(len([row for row in rows if row["Row Type"] == "domain_comparison"]), 2) + self.assertEqual(len([row for row in rows if row["Row Type"] == "motif"]), 1) + self.assertTrue(any(row["Row Type"] == "gene_error" and row["Stage"] == "motif" for row in rows)) + + def test_cli_continues_after_gene_execution_failure(self): + self.write_fasta("first.fa") + self.write_fasta("second.fa", "TGCA") + first_orf = "AAAA" + second_orf = "CCCC" + pipeline = self.make_pipeline( + orfs={"ACGT": [first_orf], "TGCA": [second_orf]}, + domains={first_orf: [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, + orthologs=[], scanner_fail_on=first_orf, + ) + destination = self.root / "continue" + + code, stderr = self.run_cli(pipeline, *self.arguments("continue")) + + rows = self.read_rows(destination) + self.assertEqual(code, 1) + self.assertIn("gene first", stderr) + self.assertTrue(any(row["Input Gene"] == "first" and row["Row Type"] == "gene_error" for row in rows)) + self.assertTrue(any(row["Input Gene"] == "second" and row["Row Type"] == "input" for row in rows)) + self.assertEqual(json.loads((destination / "metadata.json").read_text())["summary"]["genes_failed"], 1) + self.assertEqual(json.loads((destination / "metadata.json").read_text())["summary"]["genes_completed"], 1) + + def test_stop_after_alignment_never_calls_jaspar(self): + self.write_fasta("alignment.fa") + database = StubMotifDatabase([make_motif("MA0001.1", "unused")]) + pipeline = self.make_pipeline( + orfs={"ACGT": ["AAAA"]}, + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, + orthologs=[make_ortholog()], database=database, + ) + destination = self.root / "alignment" + + code, _ = self.run_cli(pipeline, *self.arguments("alignment", "--stop-after", "alignment")) + + rows = self.read_rows(destination) + metadata = json.loads((destination / "metadata.json").read_text(encoding="utf-8")) + self.assertEqual(code, 0) + self.assertEqual(database.calls, 0) + self.assertEqual(metadata["completion"], "intentionally-stopped") + self.assertTrue(any(row["Stage"] == "alignment" and row["Row Type"] == "domain_comparison" for row in rows)) + self.assertTrue(any(row["Stage"] == "motif" and row["Status"] == "not-reached" for row in rows)) + + def test_invalid_inputs_and_existing_output_exit_two(self): + self.write_fasta("valid.fa") + existing = self.root / "already-there" + existing.mkdir() + stderr = io.StringIO() + with patch("lverage.cli._build_pipeline") as build: + with redirect_stderr(stderr): + code = main(self.arguments("already-there")) + self.assertEqual(code, 2) + self.assertIn("already exists", stderr.getvalue()) + build.assert_not_called() + + self.input_dir.joinpath("valid.fa").unlink() + with redirect_stderr(io.StringIO()): + self.assertEqual(main(self.arguments("missing-input")), 2) + + def test_parser_requires_output_dir_and_supports_stop_after(self): + args = self.arguments("parsed", "--stop-after", "alignment") + parsed = build_parser().parse_args(args) + self.assertEqual(parsed.output_dir, str(self.root / "parsed")) + self.assertEqual(parsed.stop_after, "alignment") + with self.assertRaises(SystemExit): + build_parser().parse_args([ + "--input-dir", str(self.input_dir), "--blast-db", "db", + "--species-map", str(self.species_map), "--pfam-db", str(self.pfam_dir), + ]) + + def test_validate_input_discovery_preserves_order_and_fragment_lists(self): + self.write_fasta("zeta.FNA") + self.write_fasta("Alpha.fa", "TGCA") + parsed = build_parser().parse_args(self.arguments("validate")) + inputs, species_map, species_path = _validate_inputs(parsed) + self.assertEqual([item["gene"] for item in inputs], ["Alpha", "zeta"]) + self.assertEqual(inputs[0]["fragments"], ["TGCA"]) + self.assertEqual(species_map, {"Homo sapiens": 9606}) + self.assertEqual(species_path, self.species_map) + + def test_build_pipeline_passes_existing_options(self): + parsed = build_parser().parse_args(self.arguments( + "configured", "--blastp", "/tools/blastp", "--blastdbcmd", "/tools/blastdbcmd", + "--pfamscan", "/tools/pfamscan", "--identity-threshold", "0.83", + "--valid-pfam", "PF00046", "--jaspar-hits", "7", "--jaspar-evalue", "2e-5", + )) + with patch("lverage.blast.LocalBlastSearcher") as blast, \ + patch("lverage.pfam.LocalPfamScanner") as pfam, \ + patch("lverage.orf_searcher.OrffinderOrfSearcher") as orf, \ + patch("lverage.jaspar.Jaspar2024MotifDB") as jaspar, \ + patch("lverage.pipeline.Lverage") as lverage: + _build_pipeline(parsed, {"Homo sapiens": 9606}) + blast.assert_called_once_with( + database_path=str(self.root / "blast"), species_map={"Homo sapiens": 9606}, + blastp_path="/tools/blastp", blastdbcmd_path="/tools/blastdbcmd", + ) + pfam.assert_called_once_with(database_path=str(self.pfam_dir), pfamscan_path="/tools/pfamscan") + jaspar.assert_called_once_with(n_hits=7, escore_threshold=2e-5) + lverage.assert_called_once_with( + motif_database_list=[jaspar.return_value], orf_searcher=orf.return_value, + domain_scanner=pfam.return_value, ortholog_searcher=blast.return_value, + valid_pfam_list=["PF00046"], dbd_identity_thresh=0.83, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/v2/test_local_blast.py b/tests/v2/test_local_blast.py index 789358f..9a2dbb4 100644 --- a/tests/v2/test_local_blast.py +++ b/tests/v2/test_local_blast.py @@ -5,6 +5,7 @@ from unittest.mock import patch from lverage.blast import DEFAULT_EXCLUDED_TERMS, LocalBlastSearcher, _parse_blast_hits +from lverage.evidence import EvidenceCollector FIXTURE_DIR = Path(__file__).parent / "fixtures" @@ -162,6 +163,168 @@ def test_sequence_retrieval_failure_for_both_identifiers_skips_hit(self): self.assertEqual(records, []) + def test_incremental_hits_remain_when_later_hit_retrieval_raises(self): + searcher = self.make_searcher() + alignments = [] + for index in range(1, 5): + alignment = make_alignment() + alignment.accession = f"NP_{index:06d}.1" + alignment.hit_id = f"ref|NP_{index:06d}.1|" + alignments.append(alignment) + original_error = RuntimeError("unexpected retrieval failure") + self.mock_run.side_effect = [ + SimpleNamespace(stdout=""), + SimpleNamespace(stdout="SEQUENCE1"), + SimpleNamespace(stdout="SEQUENCE2"), + SimpleNamespace(stdout="SEQUENCE3"), + original_error, + ] + evidence = EvidenceCollector() + + with patch("lverage.blast.NCBIXML.parse", return_value=iter([ + SimpleNamespace(alignments=alignments) + ])): + try: + searcher.get_orthologs("QUERY", evidence_collector=evidence) + except RuntimeError as error: + self.assertIs(error, original_error) + traceback_names = [] + traceback = error.__traceback__ + while traceback is not None: + traceback_names.append(traceback.tb_frame.f_code.co_name) + traceback = traceback.tb_next + self.assertIn("_execute_mock_call", traceback_names) + self.assertIn("__get_complete_sequence", traceback_names) + else: + self.fail("expected the injected BLAST retrieval exception") + + retained = [event for event in evidence.events if event["event"] == "ortholog"] + failures = [event for event in evidence.events if event["event"] == "ortholog_retrieval" and event["status"] == "failed"] + summaries = [event for event in evidence.events if event["event"] == "blast_search_summary"] + self.assertEqual([event["ortholog"].accession for event in retained], [ + "NP_000001.1", "NP_000002.1", "NP_000003.1", + ]) + self.assertEqual(len(failures), 1) + self.assertEqual(failures[0]["hit_id"], "ref|NP_000004.1|") + self.assertEqual(len(summaries), 1) + self.assertEqual(summaries[0]["status"], "retrieval-failed") + self.assertEqual(summaries[0]["retrieved_count"], 3) + + def test_fourth_hit_skip_keeps_first_three_and_reports_partial_retrieval(self): + searcher = self.make_searcher() + alignments = [] + for index in range(1, 5): + alignment = make_alignment() + alignment.accession = f"NP_{index:06d}.1" + alignment.hit_id = f"ref|NP_{index:06d}.1|" + alignments.append(alignment) + self.mock_run.side_effect = [ + SimpleNamespace(stdout=""), + SimpleNamespace(stdout="SEQUENCE1"), + SimpleNamespace(stdout="SEQUENCE2"), + SimpleNamespace(stdout="SEQUENCE3"), + subprocess.CalledProcessError(1, "blastdbcmd", stderr="missing hit 4"), + subprocess.CalledProcessError(1, "blastdbcmd", stderr="missing accession 4"), + ] + evidence = EvidenceCollector() + + with patch("lverage.blast.NCBIXML.parse", return_value=iter([ + SimpleNamespace(alignments=alignments) + ])): + records = searcher.get_orthologs("QUERY", evidence_collector=evidence) + + self.assertEqual([record.accession for record in records], [ + "NP_000001.1", "NP_000002.1", "NP_000003.1", + ]) + ortholog_events = [event for event in evidence.events if event["event"] == "ortholog"] + skipped = [event for event in evidence.events if event["event"] == "ortholog_retrieval" and event["status"] == "skipped"] + summaries = [event for event in evidence.events if event["event"] == "blast_search_summary"] + self.assertEqual(len(ortholog_events), 3) + self.assertEqual(len(skipped), 1) + self.assertEqual(skipped[0]["hit_id"], "ref|NP_000004.1|") + self.assertEqual(len(summaries), 1) + self.assertEqual(summaries[0]["status"], "partial-retrieval-failure") + + def test_evidence_callback_failure_is_not_a_retrieval_failure(self): + searcher = self.make_searcher() + alignment = make_alignment() + self.mock_run.side_effect = [ + SimpleNamespace(stdout=""), + SimpleNamespace(stdout="SEQUENCE"), + ] + + def fail_ortholog_row(event): + if event["event"] == "ortholog": + raise OSError("bundle write failed") + + evidence = EvidenceCollector(callback=fail_ortholog_row) + with patch("lverage.blast.NCBIXML.parse", return_value=iter([ + SimpleNamespace(alignments=[alignment]) + ])): + with self.assertRaisesRegex(OSError, "bundle write failed"): + searcher.get_orthologs("QUERY", evidence_collector=evidence) + + self.assertFalse(any( + event["event"] == "ortholog_retrieval" + and event["status"] in {"failed", "skipped"} + for event in evidence.events + )) + summaries = [event for event in evidence.events if event["event"] == "blast_search_summary"] + self.assertEqual(len(summaries), 1) + self.assertEqual(summaries[0]["status"], "evidence-write-failed") + + def test_ortholog_record_construction_failure_has_distinct_summary(self): + searcher = self.make_searcher() + original_error = ValueError("invalid ortholog record") + self.mock_run.side_effect = [ + SimpleNamespace(stdout=""), + SimpleNamespace(stdout="SEQUENCE"), + ] + evidence = EvidenceCollector() + + with patch("lverage.blast.NCBIXML.parse", return_value=iter([ + SimpleNamespace(alignments=[make_alignment()]) + ])), patch("lverage.blast._make_ortholog", side_effect=original_error): + with self.assertRaises(ValueError) as raised: + searcher.get_orthologs("QUERY", evidence_collector=evidence) + + self.assertIs(raised.exception, original_error) + summaries = [event for event in evidence.events if event["event"] == "blast_search_summary"] + self.assertEqual(len(summaries), 1) + self.assertEqual(summaries[0]["status"], "record-construction-failed") + self.assertFalse(any( + event["event"] == "blast_search_summary" + and event["status"] == "evidence-write-failed" + for event in evidence.events + )) + + def test_no_alignments_emit_explicit_no_hits_with_reporting(self): + searcher = self.make_searcher() + self.mock_run.return_value = SimpleNamespace(stdout="") + evidence = EvidenceCollector() + + with patch("lverage.blast.NCBIXML.parse", return_value=iter([ + SimpleNamespace(alignments=[]) + ])): + self.assertEqual(searcher.get_orthologs("QUERY", evidence_collector=evidence), []) + + searches = [event for event in evidence.events if event["event"] == "blast_search"] + self.assertEqual(len(searches), 1) + self.assertEqual(searches[0]["status"], "no-hits") + + def test_callers_without_evidence_reporting_keep_existing_signature_behavior(self): + searcher = self.make_searcher() + self.mock_run.side_effect = [ + SimpleNamespace(stdout=""), + SimpleNamespace(stdout="SEQUENCE"), + ] + with patch("lverage.blast.NCBIXML.parse", return_value=iter([ + SimpleNamespace(alignments=[make_alignment()]) + ])): + records = searcher.get_orthologs("QUERY") + self.assertEqual(len(records), 1) + self.assertEqual(records[0].hit_id, "ref|NP_000001.1|") + def test_species_resolution_is_case_insensitive(self): searcher = self.make_searcher() alignment = make_alignment("Homeobox protein [hOmO SaPiEnS]") diff --git a/tests/v2/test_pipeline.py b/tests/v2/test_pipeline.py index a4b30c1..a3a117d 100644 --- a/tests/v2/test_pipeline.py +++ b/tests/v2/test_pipeline.py @@ -1,6 +1,7 @@ import unittest from lverage.domain_scanner import DomainRecord, DomainScannerTemplate +from lverage.evidence import EvidenceCollector from lverage.motif_database import MotifDBRecordTemplate, MotifDBTemplate from lverage.orf_searcher import OrfSearcherTemplate from lverage.ortholog_searcher import OrthologRecord, OrthologSearcherTemplate @@ -228,6 +229,100 @@ def test_adapter_errors_propagate(self): pipeline.run("QUERY") self.assertEqual(pipeline.lverage_code, LverageCode.NOT_SET) + def test_no_orfs_and_below_threshold_emit_distinct_completed_evidence(self): + no_orfs = self.make_pipeline(orfs=[]) + no_orf_evidence = EvidenceCollector() + self.assertEqual(no_orfs.run("QUERY", evidence_collector=no_orf_evidence), []) + self.assertIn( + ("gene_outcome", "biological-no-hit"), + [(event["event"], event["status"]) for event in no_orf_evidence.events], + ) + + below_threshold = self.make_pipeline( + orthologs=[make_ortholog("TTTT")], threshold=0.75, + ) + below_threshold.domain_scanner.domains["TTTT"] = [ + DomainRecord("Homeobox", "PF00046.2", 0, 4), + ] + evidence = EvidenceCollector() + self.assertEqual(below_threshold.run("QUERY", evidence_collector=evidence), []) + comparisons = [event for event in evidence.events if event["event"] == "domain_comparison"] + self.assertEqual(len(comparisons), 1) + self.assertEqual(comparisons[0]["status"], "below-threshold") + self.assertEqual(comparisons[0]["domain_identity"], 0.0) + self.assertTrue(any( + event["event"] == "gene_outcome" and event["status"] == "biological-no-hit" + for event in evidence.events + )) + + def test_later_ortholog_domain_failure_keeps_earlier_events(self): + first = make_ortholog("AAAA") + second = make_ortholog("TTTT") + second.accession = "NP_000002.1" + + class FailingScanner(StubDomainScanner): + def get_domains(self, sequence): + if sequence == "TTTT": + raise RuntimeError("later ortholog Pfam failed") + return super().get_domains(sequence) + + pipeline = self.make_pipeline(orthologs=[first, second]) + pipeline.domain_scanner = FailingScanner({ + "AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)], + }) + evidence = EvidenceCollector() + with self.assertRaisesRegex(RuntimeError, "later ortholog Pfam failed"): + pipeline.run("QUERY", evidence_collector=evidence) + retained = [event for event in evidence.events if event["event"] == "ortholog"] + domains = [event for event in evidence.events if event["event"] == "ortholog_domain"] + comparisons = [event for event in evidence.events if event["event"] == "domain_comparison"] + self.assertEqual(len(retained), 2) + self.assertEqual(len(domains), 1) + self.assertEqual(len(comparisons), 1) + + def test_later_motif_failure_keeps_earlier_motif_event(self): + first_record = ExampleMotifRecord("MA0001.1") + + class FailsOnSecondSearch(StubMotifDB): + def search(self, request): + self.requests.append(request) + if len(self.requests) == 2: + raise RuntimeError("later JASPAR request failed") + return [first_record] + + database = FailsOnSecondSearch(motifs=[]) + sequence = "AAAAAAAA" + pipeline = self.make_pipeline( + orfs=[sequence], + domains={sequence: [ + DomainRecord("First", "PF00046.1", 0, 4), + DomainRecord("Second", "PF00046.2", 4, 8), + ]}, + orthologs=[make_ortholog(sequence)], + databases=[database], + ) + evidence = EvidenceCollector() + with self.assertRaisesRegex(RuntimeError, "later JASPAR request failed"): + pipeline.run("QUERY", evidence_collector=evidence) + motifs = [event for event in evidence.events if event["event"] == "motif"] + comparisons = [event for event in evidence.events if event["event"] == "domain_comparison"] + self.assertEqual(len(motifs), 1) + self.assertEqual(len(comparisons), 2) + + def test_stop_after_alignment_does_not_call_motif_database(self): + database = StubMotifDB(motifs=[ExampleMotifRecord("MA0001.1")]) + pipeline = self.make_pipeline(databases=[database]) + evidence = EvidenceCollector() + + self.assertEqual(pipeline.run("QUERY", evidence_collector=evidence, stop_after="alignment"), []) + + self.assertEqual(pipeline.lverage_code, LverageCode.STOPPED_AFTER_ALIGNMENT) + self.assertEqual(database.requests, []) + self.assertTrue(any( + event["event"] == "gene_outcome" and event["status"] == "intentional-stop" + for event in evidence.events + )) + def test_run_requires_nonempty_sequences(self): pipeline = self.make_pipeline() From fa60a3e9259a1f36262588e82be636be9d5a8f5a Mon Sep 17 00:00:00 2001 From: Thomas Shin Date: Wed, 28 Oct 2026 17:24:50 -0400 Subject: [PATCH 2/2] Complete CLI documentation and fix Sphinx parameter warnings --- README.md | 90 ++++------------------------------------- docs/user_guide.rst | 63 ++++++++++++++++++++--------- src/lverage/blast.py | 4 ++ src/lverage/pipeline.py | 21 +++++++++- 4 files changed, 76 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index 6d0734d..9c7e1b7 100644 --- a/README.md +++ b/README.md @@ -100,88 +100,14 @@ pipeline. ## V2 Command-Line Interface -The V2 CLI is a new interface to the modular V2 package. It does not reuse or -extend the legacy CLI below. From the repository root, install the package in -your Python environment with `python -m pip install .` to get the `lverage` -command, or run it directly from a source checkout with -`PYTHONPATH=src python -m lverage.cli`. The CLI's `--help` works without -BLAST+, PfamScan, databases, or network access. - -### Prerequisites - -- Python 3.11+ and the package dependencies from this repository. -- Local BLAST+ `blastp` and `blastdbcmd` executables. -- A local protein BLAST database suitable for complete sequence retrieval. -- A local PfamScan installation and database. -- Network access when running motif inference: JASPAR2024 is queried online. - -The local BLAST database setup above describes how to build a usable protein -database. Build and configure PfamScan separately according to its -distribution's instructions. - -### Input mapping and example - -Create a JSON file mapping bracketed scientific species names in BLAST hit -descriptions to positive NCBI taxonomy IDs, for example -`config/species-taxids.json`: - -```json -{ - "Homo sapiens": 9606, - "Mus musculus": 10090, - "Drosophila melanogaster": 7227 -} -``` - -Put one nucleotide FASTA file per gene/TF under `inputs/`. Supported suffixes -are `.fa`, `.fasta`, and `.fna`, matched case-insensitively. Multiple records -in one file are passed as independent fragments for that filename's gene; -they are never assembled, and headers are not used to verify that records -belong to the same gene. See the [V2 FASTA and CLI guide](docs/user_guide.rst) -for details. - -```bash -lverage --input-dir ./inputs --blast-db ./db/proteins \ - --species-map ./config/species-taxids.json --pfam-db ./db/pfam \ - --output-dir ./results -``` - -The required options are `--input-dir`, `--blast-db`, `--species-map`, -`--pfam-db`, and `--output-dir`. The output path must not already exist; the -CLI creates it and writes `results.tsv`, `orfs.fasta`, and `metadata.json`. -Optional executable overrides are -`--blastp` (default `blastp`), `--blastdbcmd` (default `blastdbcmd`), and -`--pfamscan` (default `pfam_scan.pl`). `--identity-threshold` defaults to -`0.7`. Repeat `--valid-pfam ACCESSION` to limit accepted domains; when omitted, -any detected domain is accepted. JASPAR defaults to 10 hits and E-value -threshold `1e-6`; the existing ORF and BLAST component defaults are retained -(75-nt minimum ORF; BLAST E-value `1e-6`, top 20 hits, and standard description -filters). - -The TSV contains explicit stage/status/error/outcome-detail fields, one motif -per motif row plus typed evidence rows for inputs, ORFs, query domains, -orthologs, ortholog domains, comparisons, stage statuses, and gene errors. -Earlier-stage rows are not motif or -ortholog hits; stage-status rows distinguish biological no-hits, below- -threshold completion, failure, and not-reached stages. Missing fields are blank. Rows retain BLAST accession, -description and native hit ID when returned, E-value, identity and query -coverage ratios and percentages, Pfam names/IDs, and each computed domain-pair -identity whether or not it meets the threshold. Candidate ORFs are uniquely -stored once in the FASTA by full-sequence SHA-256 ID; the TSV retains per-gene -associations and selected/not-selected/not-reached state. The existing -longest-first ORF-selection algorithm remains unchanged; downstream stages -run only on the selected ORF. `metadata.json` records settings, input -checksums, timestamps, tool versions when established (otherwise `unknown`), -errors, and completion counts. Bundle files are flushed as evidence arrives; -this is not checkpoint/resume or arbitrary-crash recovery. - -Use `--stop-after alignment` for pre-JASPAR validation. This intentional stop -does not contact JASPAR and exits 0. Exit 0 also means full completion, -including biological no-hits; exit 2 means preflight validation failure; exit -1 means one or more execution failures. Per-gene errors are recorded and -processing continues to other genes, preserving completed partial evidence. -Diagnostics go to stderr. The output directory is not removed after execution -starts, including when a downstream operation fails. +The V2 CLI is separate from the legacy workflow below. Run the installed +`lverage` command or use `PYTHONPATH=src python -m lverage.cli` from a source +checkout. It requires Python 3.11+, local BLAST+ and PfamScan tools/data, and a +local protein database; full motif lookup uses the online JASPAR2024 service. +See the [V2 FASTA and CLI guide](docs/user_guide.rst) for input mapping, +portable examples (including local runs stopped after alignment), output +formats, selection behavior, and status codes. See the local BLAST database +instructions above for database setup. ## Legacy Command-Line Workflow diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 903ee15..8119b89 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -20,9 +20,11 @@ it does not construct adapters, inspect external tools, or access a network. Prerequisites are the Python package dependencies, local BLAST+ executables (``blastp`` and ``blastdbcmd``), a usable local protein database, and a local PfamScan installation/database. JASPAR2024 is queried through its online -service, so each actual run also requires network access. The local BLAST -database must support complete protein retrieval with ``blastdbcmd``. See the -README's V2 local BLAST database instructions for building one. +service, so a full run that reaches motif lookup requires network access. A +local run stopped after alignment does not contact JASPAR and can run without +network access. The local BLAST database must support complete protein +retrieval with ``blastdbcmd``. See the README's V2 local BLAST database +instructions for building one. Create a JSON species-name-to-NCBI-tax-ID map. Species names must match the bracketed names in BLAST descriptions (case-insensitive), and tax IDs must be @@ -43,13 +45,26 @@ Example using portable placeholder paths: --species-map ./config/species-taxids.json --pfam-db ./db/pfam \ --output-dir ./results +For a local pre-JASPAR run that stops after domain alignment: + +.. code-block:: console + + lverage --input-dir ./inputs --blast-db ./db/proteins \ + --species-map ./config/species-taxids.json --pfam-db ./db/pfam \ + --output-dir ./alignment-results --stop-after alignment + Required options are ``--input-dir``, ``--blast-db``, ``--species-map``, ``--pfam-db``, and ``--output-dir``. The output directory must be new. It -contains ``results.tsv``, ``orfs.fasta``, and ``metadata.json``. Optional -executable overrides are ``--blastp`` (``blastp``), +contains three files: ``results.tsv`` contains typed pipeline evidence, +``orfs.fasta`` stores each unique candidate protein sequence once, and +``metadata.json`` records run settings and summaries. Optional executable +overrides are ``--blastp`` (``blastp``), ``--blastdbcmd`` (``blastdbcmd``), and ``--pfamscan`` (``pfam_scan.pl``). ``--identity-threshold`` defaults to ``0.7``. Repeat ``--valid-pfam`` to -restrict accepted domains; if it is omitted, any detected domain is accepted. +restrict accepted domains; if it is omitted, every detected Pfam domain is +accepted. The pipeline selects at most one query ORF per gene: candidates from +the input fragments are deduplicated and sorted longest-first, then the first +candidate with an accepted domain is selected for downstream analysis. JASPAR options default to 10 hits and an inference E-value threshold of ``1e-6``. Other component options retain their V2 defaults: ORF minimum length 75 nucleotides; BLAST E-value ``1e-6``, at most 20 hits, and the standard @@ -74,19 +89,29 @@ intentional stops, and stages not reached. ``ortholog_retrieval`` rows retain failed retrieval attempts and skipped hits, including available native hit IDs/accessions, the attempted identifier, and a diagnostic reason. A BLAST no-hit status is distinct from retrieval failure. -Earlier-stage rows are not ortholog or motif hits; missing values are blank. -Each motif has one row and identifies its input gene, query ORF ID, ortholog, -and matching query/ortholog domains. Multiple hits, domain occurrences, and -motifs remain distinct. ORF rows retain candidates even when no accepted -domain exists, and an input row records a gene with no ORF. ``orfs.fasta`` -stores each unique discovered candidate protein once under a deterministic -full-sequence SHA-256 identifier referenced by the TSV; gene associations and -selection status remain in the TSV. Candidate export does not change the -longest-first selection algorithm or run downstream analysis on unselected -candidates. BLAST fields include accession, native hit ID when returned, full -description, E-value, identity, and query coverage. Ratios and percentages are -separately labeled. Pfam IDs/names and every computed domain-pair identity are -kept, including values below the configured threshold. +``results.tsv`` has multiple evidence row types; a row is not necessarily a +separate ORF or a final motif result. For example, an ORF may have candidate, +domain, selection, ortholog, comparison, and stage-status rows, while a motif +row is emitted only for a returned motif. Each motif has one row identifying +its input gene, query ORF, ortholog, and matching query/ortholog domains. +Multiple hits, domain occurrences, and motifs remain distinct. ORF rows retain +candidates even when no accepted domain exists, and an input row records a +gene with no ORF. ``orfs.fasta`` stores each unique discovered candidate +protein once under a deterministic full-sequence SHA-256 identifier +referenced by the TSV; gene associations and selection status remain in the +TSV. Candidate export does not run downstream analysis on unselected +candidates. + +BLAST identity is the identity ratio from the HSP with the lowest E-value and +is distinct from domain-pair identity, which is the exact-match ratio across +every column of a global amino-acid alignment of matching Pfam domains +(including gaps). +The TSV reports BLAST identity and query coverage as ratios and percentages, +and records every computed domain-pair identity, including values below the +threshold. Domain start and end coordinates in exported rows are zero-based, +end-exclusive amino-acid positions. The domain-alignment identity score is +exported as a ratio and percentage, but the aligned sequences themselves are +not currently exported. ``metadata.json`` records run timestamps, settings, input paths/checksums, sequence-ID conventions, known tool versions (unknown versions are reported as diff --git a/src/lverage/blast.py b/src/lverage/blast.py index 4d7588f..9cd0a80 100644 --- a/src/lverage/blast.py +++ b/src/lverage/blast.py @@ -195,6 +195,10 @@ def get_orthologs(self, sequence : str, *, evidence_collector=None) -> list[Orth ---------- sequence : str Protein sequence used as the BLAST query + evidence_collector : object, optional + Evidence collector, defaulting to ``None``. When provided, BLAST + search, retrieval attempts, and retained ortholog evidence are + reported incrementally as the search progresses. Returns ------- diff --git a/src/lverage/pipeline.py b/src/lverage/pipeline.py index abaedb3..a341d05 100644 --- a/src/lverage/pipeline.py +++ b/src/lverage/pipeline.py @@ -257,11 +257,30 @@ def run(self, tf_sequence : str | list[str], *, evidence_collector=None, ---------- tf_sequence : str or list of str One sequence or multiple sequence fragments for one factor + evidence_collector : object, optional + Evidence collector, defaulting to ``None``. When provided, + pipeline evidence is reported incrementally as stages complete. + stop_after : {None, 'alignment'}, optional + ``None`` (the default) runs the full pipeline, including motif + lookup. ``'alignment'`` intentionally stops after domain + alignment and before motif lookup. Returns ------- list - Motif results with flattened query, ortholog, domain, and database evidence + Motif result records. Returns an empty list when no motif records + are found, including when stopped after alignment. + + Raises + ------ + TypeError + The input is not a string or a list containing only strings + ValueError + The input is empty, contains an empty sequence, or ``stop_after`` + is neither ``None`` nor ``'alignment'`` + Exception + An exception raised by an injected pipeline component or evidence + collector; such exceptions are propagated to the caller """ self.lverage_code = LverageCode.NOT_SET