diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index fd68e87..edbcf77 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -31,7 +31,9 @@ jobs: run: python -m unittest discover -s tests/v2 -p 'test_*.py' -v - name: Run the retained legacy tests - run: python -m unittest discover -s tests -p 'test_jaspar_window.py' -v + # Run only the legacy file; V2 tests run separately above. Recursive + # discovery mixes the legacy dependency stubs with real V2 imports. + run: python tests/test_jaspar_window.py -v - name: Build the documentation run: sphinx-build -W --keep-going --nitpicky -b html docs docs/_build/html diff --git a/README.md b/README.md index 9c7e1b7..4c11ea7 100644 --- a/README.md +++ b/README.md @@ -100,14 +100,28 @@ pipeline. ## V2 Command-Line Interface +Install the V2 package from the repository root into your Python 3.11+ +environment with `python -m pip install -e .`. This installs the Python +dependencies declared in `pyproject.toml`, including NumPy and the pinned +`orffinder` adapter dependency. The Python 3.10 Conda configuration above +is for the legacy workflow; it is not a V2 environment specification. +BLAST+, HMMER, PfamScan, and their reference databases are separate +requirements and are not installed by this command. + The V2 CLI is separate from the legacy workflow below. Run the installed -`lverage` command or use `PYTHONPATH=src python -m lverage.cli` from a source -checkout. It requires Python 3.11+, local BLAST+ and PfamScan tools/data, and a -local protein database; full motif lookup uses the online JASPAR2024 service. -See the [V2 FASTA and CLI guide](docs/user_guide.rst) for input mapping, -portable examples (including local runs stopped after alignment), output -formats, selection behavior, and status codes. See the local BLAST database -instructions above for database setup. +`lverage` command or use `PYTHONPATH=src python -m lverage.cli` from the Lverage +source checkout. It requires Python 3.11+, local BLAST+ and PfamScan tools/data, +and a local protein database; full motif inference uses the JASPAR calculations +adapted into Lverage and local JASPAR 2026 files, with no online requests. +Users need the external reference data directory and ``JASPAR2026.sqlite``, +but not the upstream checkout or a separate interpreter. HMMER/BLASTP remain +external executables. An alignment-only stop does not require JASPAR resources. +The cutoff is applied to the inference BLAST E-value; equivalence to the former +REST E-value has not been established, and identical motif results are not +claimed. See the [V2 FASTA and CLI guide](docs/user_guide.rst) for provenance, +local-data requirements, portable examples, output fields, selection behavior, +and status codes. See the local BLAST database instructions above for database +setup. ## Legacy Command-Line Workflow diff --git a/conda_lverage.yml b/conda_lverage.yml index 64b9eb2..4828210 100644 --- a/conda_lverage.yml +++ b/conda_lverage.yml @@ -34,6 +34,7 @@ dependencies: - idna==3.7 - itsdangerous==2.2.0 - msgspec==0.18.6 + - numpy>=1.24 - pytz==2024.1 - six==1.16.0 - urllib3==2.2.1 diff --git a/docs/api/jaspar.rst b/docs/api/jaspar.rst index 0e755a2..5e3555a 100644 --- a/docs/api/jaspar.rst +++ b/docs/api/jaspar.rst @@ -1,20 +1,37 @@ JASPAR ====== -The adapter uses the current ``jaspar.elixir.no`` REST API. Network requests -use finite timeouts, species availability is loaded lazily, and valid no-hit -responses return an empty list. Service, protocol, and malformed-response -failures are raised to the caller. +The V2 adapter implements the JASPAR profile-inference calculations within +Lverage, adapted from source revision +``ac4024148f1f140c7370d7b13a70556b4d13f7a3`` of +``amathelier/JASPAR-inference-tool``. It uses normal package imports and local +JASPAR reference data; no upstream checkout, Git history, CLI entry point, or +separate inference interpreter is required. Runtime inference makes no online +requests. Exact-version motif matrices and annotations are resolved from a local +``JASPAR2026.sqlite`` database. The algorithm applies the upstream HMMER domain +thresholds and overlap selection, BLASTP/Rost filter, insertion removal, +positional identity/BLOSUM62 scoring, and Cis-BP HSim sequence-identity or +similarity-regression decisions. -Inference uses a window of at most 2,000 ortholog residues while retaining the -complete ortholog domain. Accepted hits are sorted by inference E-value, checked -against every species listed in their motif details, and limited only after -species and threshold filtering. +The adapter filters by the upstream BLAST E-value, exact matrix species +taxonomy IDs, and then the configured hit limit. +The configured E-value cutoff applies specifically to the local inference +tool's BLAST E-value. Its equivalence to the former REST endpoint's ``evalue`` +field has not been established; identical motif results between the services +are not claimed. Exact matrix versions returned by the fork are retained, +subject to species filtering, the E-value cutoff, and the hit limit. + +The input is the unchanged ortholog-protein window of at most 2,000 residues +centered on the complete supporting domain. The inference BLAST E-value and DBD +identity are retained independently from Lverage BLAST metrics and Lverage's +global-alignment identity. Exact versioned matrix identifiers are required; +missing identifiers or local annotations are errors rather than an implicit +version substitution. .. autoclass:: lverage.jaspar.JasparRecord :members: :show-inheritance: -.. autoclass:: lverage.jaspar.Jaspar2024MotifDB +.. autoclass:: lverage.jaspar.Jaspar2026MotifDB :members: :show-inheritance: diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 8119b89..5d7b6a8 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -19,12 +19,39 @@ it does not construct adapters, inspect external tools, or access a network. Prerequisites are the Python package dependencies, local BLAST+ executables (``blastp`` and ``blastdbcmd``), a usable local protein database, and a local -PfamScan installation/database. JASPAR2024 is queried through its online -service, so a full run that reaches motif lookup requires network access. A -local run stopped after alignment does not contact JASPAR and can run without -network access. The local BLAST database must support complete protein -retrieval with ``blastdbcmd``. See the README's V2 local BLAST database -instructions for building one. +PfamScan installation/database. Local motif inference uses the JASPAR profile +inference calculations adapted into Lverage from +``amathelier/JASPAR-inference-tool`` and local JASPAR 2026 reference data; it +makes no online requests. Users do not need that source checkout, its Git +history, an upstream entry point, or a separate Python environment. A run +stopped after alignment does not require or inspect JASPAR data. The local BLAST +database must support complete protein retrieval with ``blastdbcmd``. See the +README's V2 local BLAST database instructions for building one. + +Configure the external inference data directory and local +``JASPAR2026.sqlite`` using CLI options or the ``LVERAGE_JASPAR_DATA_DIR`` and +``LVERAGE_JASPAR_SQLITE_DB`` environment variables. The implementation records +upstream source revision ``ac4024148f1f140c7370d7b13a70556b4d13f7a3`` and +version ``2026.1.0`` as algorithm provenance; neither is a runtime checkout +requirement. Keep reference data, databases, and temporary files outside the +repository. Configure BLASTP through ``--blastp`` and HMMER through ``--hmmscan`` +and ``--hmmalign`` (or their executable names on ``PATH``); those external +programs are invoked directly with argument lists. The Python inference +implementation imports Biopython and NumPy from the normal Lverage environment. +The configured data directory is a standalone data tree, not the upstream +source checkout. If reusing its downloaded reference files, stage the data +outside the checkout; Lverage does not read the upstream source tree at runtime. + +The adapted routines correspond to the upstream sequence-level inference flow: +load the per-taxon Pfam/profile/UniProt and Cis-BP models; detect and align +query Pfam domains with HMMER; search the local taxon BLAST databases and apply +the upstream Rost identity curve; remove HMM alignment insertions; calculate +the upstream positional identity and BLOSUM62 features; and apply the Cis-BP +identity or similarity-regression ``HSim`` decision before returning associated +JASPAR profiles. The upstream source revision is provenance only. The external +data directory contains the precomputed models, domain mappings, and BLAST +databases generated for the JASPAR 2026 profile set; the separate SQLite file +supplies exact-version matrices and annotations. Create a JSON species-name-to-NCBI-tax-ID map. Species names must match the bracketed names in BLAST descriptions (case-insensitive), and tax IDs must be @@ -41,34 +68,59 @@ Example using portable placeholder paths: .. code-block:: console - lverage --input-dir ./inputs --blast-db ./db/proteins \ - --species-map ./config/species-taxids.json --pfam-db ./db/pfam \ - --output-dir ./results + lverage --input-dir ./inputs --blast-db /path/to/local-data/blast/proteins \ + --species-map ./config/species-taxids.json \ + --pfam-db /path/to/local-data/pfam --output-dir /path/outside/repository/results \ + --jaspar-data-dir /path/to/local-data/jaspar-inference/files \ + --jaspar-sqlite-db /path/to/local-data/jaspar/JASPAR2026.sqlite \ + --blastp /path/to/blastp --hmmscan /path/to/hmmscan \ + --hmmalign /path/to/hmmalign For a local pre-JASPAR run that stops after domain alignment: .. code-block:: console - lverage --input-dir ./inputs --blast-db ./db/proteins \ - --species-map ./config/species-taxids.json --pfam-db ./db/pfam \ - --output-dir ./alignment-results --stop-after alignment + lverage --input-dir ./inputs --blast-db /path/to/local-data/blast/proteins \ + --species-map ./config/species-taxids.json \ + --pfam-db /path/to/local-data/pfam \ + --output-dir /path/outside/repository/alignment-results \ + --stop-after alignment Required options are ``--input-dir``, ``--blast-db``, ``--species-map``, ``--pfam-db``, and ``--output-dir``. The output directory must be new. It -contains three files: ``results.tsv`` contains typed pipeline evidence, +contains three files: ``results.csv`` contains typed pipeline evidence, ``orfs.fasta`` stores each unique candidate protein sequence once, and ``metadata.json`` records run settings and summaries. Optional executable overrides are ``--blastp`` (``blastp``), ``--blastdbcmd`` (``blastdbcmd``), and ``--pfamscan`` (``pfam_scan.pl``). +The adapted motif inference also accepts ``--hmmscan`` and ``--hmmalign`` +overrides. ``--identity-threshold`` defaults to ``0.7``. Repeat ``--valid-pfam`` to restrict accepted domains; if it is omitted, every detected Pfam domain is accepted. The pipeline selects at most one query ORF per gene: candidates from the input fragments are deduplicated and sorted longest-first, then the first candidate with an accepted domain is selected for downstream analysis. -JASPAR options default to 10 hits and an inference E-value threshold of -``1e-6``. Other component options retain their V2 defaults: ORF minimum length +JASPAR options default to 10 hits and a maximum inference-tool BLAST E-value of +``1e-6``; this cutoff applies to the BLAST E-value reported by the local +inference tool. Equivalence between that field and the former REST service's +``evalue`` field has not been established, so equivalent cutoff behavior or +identical motif results must not be assumed. The adapter retains all exact +versioned matrix IDs returned by the inference before applying the species +filter, this cutoff, and the configured hit limit. Set data paths using +``--jaspar-data-dir`` and ``--jaspar-sqlite-db``, or the corresponding +``LVERAGE_JASPAR_DATA_DIR`` and ``LVERAGE_JASPAR_SQLITE_DB`` environment +variables. Configure ``blastp``, ``hmmscan`` and ``hmmalign`` through their +executable options. The data directory must contain the locally stored JASPAR +profile-inference files, Pfam HMMs and pressed indexes, Cis-BP models, and the +per-taxon JSON and BLAST-database files used by the adapted algorithm. The +SQLite database can be the local JASPAR 2026 database distributed with +pyJASPAR; it supplies exact-version PFMs, TF classes, and species taxonomy +IDs. Other component options retain their V2 defaults: ORF minimum length 75 nucleotides; BLAST E-value ``1e-6``, at most 20 hits, and the standard -description filters. +description filters. By default, descriptions containing ``hypothetical``, +``unnamed``, ``uncharacterized``, ``unknown``, or ``partial`` are excluded +case-insensitively. An ``isoform`` label alone does not exclude a protein: +isoform-labelled proteins are retained unless another exclusion applies. The CLI discovers ``.fa``, ``.fasta``, and ``.fna`` files nonrecursively, case-insensitively, in filename order. The filename stem labels results and @@ -78,41 +130,76 @@ fragments, and it cannot determine from record headers whether they really belong to the same gene. Empty files/records and unsupported DNA symbols are rejected before external-tool adapters are constructed. -The TSV has explicit ``Row Type``, ``Stage``, ``Status``, ``Error``, and -``Outcome Detail`` fields. ``Error`` is reserved for execution failures; -biological reasons and not-reached explanations use ``Outcome Detail``. -Row types include ``input``, ``orf``, ``query_domain``, ``ortholog``, -``ortholog_retrieval``, ``ortholog_domain``, ``domain_comparison``, ``motif``, -and ``gene_error``. -``stage_status`` rows record biological no-hits, below-threshold completion, -intentional stops, and stages not reached. -``ortholog_retrieval`` rows retain failed retrieval attempts and skipped hits, -including available native hit IDs/accessions, the attempted identifier, and a -diagnostic reason. A BLAST no-hit status is distinct from retrieval failure. -``results.tsv`` has multiple evidence row types; a row is not necessarily a -separate ORF or a final motif result. For example, an ORF may have candidate, -domain, selection, ortholog, comparison, and stage-status rows, while a motif -row is emitted only for a returned motif. Each motif has one row identifying -its input gene, query ORF, ortholog, and matching query/ortholog domains. -Multiple hits, domain occurrences, and motifs remain distinct. ORF rows retain -candidates even when no accepted domain exists, and an input row records a -gene with no ORF. ``orfs.fasta`` stores each unique discovered candidate -protein once under a deterministic full-sequence SHA-256 identifier -referenced by the TSV; gene associations and selection status remain in the -TSV. Candidate export does not run downstream analysis on unselected -candidates. +``results.csv`` is a comma-separated UTF-8 table ordered by gene, ORF candidate, +query Pfam, BLAST hit, hit Pfam, global alignment, and motif hit. Each row is a +self-contained evidence record: every applicable gene, ORF, query-Pfam, +BLAST-hit, hit-Pfam, and alignment field is repeated on each descendant result +row. Columns are named for their stage (for example, ``Query Pfam ID``, +``BLAST Accession``, and ``Global Alignment Identity Ratio``). Candidates are +listed in longest-first examination order. The ORF examination and selection +columns identify examined candidates, candidates with no accepted domain, and +the selected candidate. Unexamined candidates still show their order, ID, and +length; their examination, selection, and downstream cells remain blank. All +unique ORFs are represented. + +Within the selected ORF, each query-domain occurrence has its own occurrence +number and retains its Pfam ID, name, and zero-based, end-exclusive amino-acid +coordinates. BLAST hits are grouped beneath query Pfams and retain accession, +native hit ID, description, species, E-value, identity, and query coverage. +Occurrence and hit-group numbers, BLAST descriptive fields, and supporting +alignment fields are repeated on every associated motif row. When a hit is +evaluated against multiple query-Pfam occurrences, each row identifies the +specific query/hit-domain occurrence pair. Hit Pfam occurrences remain visible +even when no query Pfam matches them. Domain comparisons are associated by the +actual query-domain and hit-domain occurrences, not by protein name. Each +computed global domain alignment has an identity ratio, threshold, and pass/fail +decision. There is one row per motif hit per supporting domain comparison; +multiple motifs and multiple comparisons therefore remain distinct rows. + +Rows remain grouped contiguously for convenient reading, but their meaning does +not depend on carrying values forward from preceding rows. Do not forward-fill +blank cells: a field is blank only when the information is unavailable or its +stage was not reached. No-hit outcomes and execution errors are recorded in the +relevant stage status, outcome, and error columns. Sorting or filtering may +change presentation order, but each row retains its own gene, candidate, hit, +domain-occurrence, alignment, and motif associations. +BLAST no hits and failed sequence retrieval are distinct outcomes. An empty +Pfam result, a domain pair below threshold, no matching Pfam, and a motif +search with no returned motifs are also distinguishable. + +``orfs.fasta`` stores each unique discovered candidate protein once under a +deterministic full-sequence SHA-256 identifier referenced by the table; gene +associations and selection status are retained in the grouped rows. Candidate export does not run downstream analysis on +unselected candidates. BLAST identity is the identity ratio from the HSP with the lowest E-value and is distinct from domain-pair identity, which is the exact-match ratio across every column of a global amino-acid alignment of matching Pfam domains (including gaps). -The TSV reports BLAST identity and query coverage as ratios and percentages, +The CSV reports BLAST identity and query coverage as ratios and percentages, and records every computed domain-pair identity, including values below the threshold. Domain start and end coordinates in exported rows are zero-based, end-exclusive amino-acid positions. The domain-alignment identity score is exported as a ratio and percentage, but the aligned sequences themselves are not currently exported. +Motif rows separately record the upstream inference tool's BLAST E-value, +query/TF regions, TF name, and upstream DBD identity ratio. This upstream DBD +identity is not the Lverage global-alignment identity and is not substituted +for it. The local BLAST E-value is not established to be equivalent to the old +REST inference E-value; identical motif results between those services are not +claimed. The input sent to local inference is the same ortholog-protein window +previously sent to the REST endpoint: at most 2,000 amino acids centered on the +supporting ortholog domain, retaining the complete domain. It is not replaced +by the query protein or by a domain-only sequence. Matrix IDs must be +versioned, and each ID is resolved exactly in the configured local SQLite +database; there is no version fallback. Motif species are filtered against +exact local NCBI taxonomy IDs, not broad taxonomic groups. + +Metadata schema version 2 identifies this layout with +``results_format="grouped-csv"``. It replaces the former event-row TSV schema. Fields containing commas, +quotation marks, or line breaks are quoted using standard CSV rules. + ``metadata.json`` records run timestamps, settings, input paths/checksums, sequence-ID conventions, known tool versions (unknown versions are reported as ``unknown``), and completion/failure summaries. Results and metadata are @@ -123,7 +210,8 @@ gene and processing continues to other genes, retaining completed evidence if a later stage fails. Use ``--stop-after alignment`` for pre-JASPAR validation. This intentional stop -does not call JASPAR and is not a failure. Exit status ``0`` means completed or +does not call JASPAR and does not require the JASPAR options or data; it is not +a failure. Exit status ``0`` means completed or intentionally stopped, including biological no-hits; ``2`` means preflight validation failure; ``1`` means one or more execution failures despite preserved partial results. Diagnostics go to standard error. diff --git a/pyproject.toml b/pyproject.toml index cddcd66..2dfa302 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ requires-python = ">=3.11" license = {file = "LICENSE"} dependencies = [ "biopython>=1.85", + "numpy>=1.24", "orffinder==1.8", "requests>=2.32", ] @@ -28,3 +29,6 @@ docs = [ [tool.setuptools.packages.find] where = ["src"] include = ["lverage*"] + +[tool.setuptools.package-data] +lverage = ["jaspar_inference_license.txt"] diff --git a/requirements.txt b/requirements.txt index 29411bd..ef65ff1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ selenium biopython +numpy>=1.24 six ete3 \ No newline at end of file diff --git a/src/lverage/blast.py b/src/lverage/blast.py index 9cd0a80..0671cf7 100644 --- a/src/lverage/blast.py +++ b/src/lverage/blast.py @@ -19,7 +19,6 @@ "uncharacterized", "unknown", "partial", - "isoform", ) @@ -251,7 +250,18 @@ def emit_evidence(event, **payload): raise try: - for hit in hits: + for hit_index, hit in enumerate(hits, start=1): + retrieval_fields = { + "hit_group": hit_index, + "description": hit["description"], + "species_name": hit["species_name"], + "species_tax_id": hit["species_tax_id"], + "evalue": hit["evalue"], + "identity_ratio": hit["identity"], + "identity_percent": hit["identity"] * 100, + "query_coverage_ratio": hit["query_coverage"], + "query_coverage_percent": hit["query_coverage"] * 100, + } complete_sequence = "" entries = [hit["hit_id"]] if hit["accession"] and hit["accession"] != hit["hit_id"]: @@ -270,6 +280,7 @@ def emit_evidence(event, **payload): "ortholog_retrieval", status="attempt-failed", hit_id=hit.get("hit_id"), accession=hit.get("accession"), attempted_identifier=entry, error=reason, + **retrieval_fields, ) continue except Exception as error: @@ -280,6 +291,7 @@ def emit_evidence(event, **payload): hit_id=hit.get("hit_id"), accession=hit.get("accession"), attempted_identifier=entry, error=_safe_retrieval_reason(error), + **retrieval_fields, ) except Exception: LOGGER.exception("Failed to emit BLAST retrieval diagnostic") @@ -292,6 +304,7 @@ def emit_evidence(event, **payload): hit_id=hit.get("hit_id"), accession=hit.get("accession"), attempted_identifier=entry, error="empty sequence returned", + **retrieval_fields, ) if not complete_sequence: LOGGER.warning( @@ -304,6 +317,7 @@ def emit_evidence(event, **payload): "ortholog_retrieval", status="skipped", hit_id=hit.get("hit_id"), accession=hit.get("accession"), attempted_identifier="", error=reason, + **retrieval_fields, ) continue try: diff --git a/src/lverage/cli.py b/src/lverage/cli.py index 116c834..73e660e 100644 --- a/src/lverage/cli.py +++ b/src/lverage/cli.py @@ -17,18 +17,29 @@ DNA_ALPHABET = frozenset("ACGTRYSWKMBDHVN") FASTA_SUFFIXES = frozenset({".fa", ".fasta", ".fna"}) RESULT_HEADERS = [ - "Row Type", "Input Gene", "Query ORF ID", "Query ORF Selected", - "Stage", "Status", "Error", "Outcome Detail", "Attempted Hit Identifier", - "Ortholog Hit ID", "Ortholog Accession", - "Ortholog Description", "Ortholog Species Name", "Ortholog Species Tax ID", - "BLAST E-value", "Ortholog Identity Ratio", "Ortholog Identity Percent", - "Query Coverage Ratio", "Query Coverage Percent", - "Query Domain Name", "Query Domain ID", "Query Domain Start 0-based", - "Query Domain End 0-based-exclusive", "Ortholog Domain Name", "Ortholog Domain ID", - "Ortholog Domain Start 0-based", "Ortholog Domain End 0-based-exclusive", - "Domain Pair Identity Ratio", "Domain Pair Identity Percent", "Threshold Ratio", - "Threshold Passed", "Motif Database", "Matrix ID", "Motif Name", "Motif Class", - "Motif Inference E-value", "Motif URL", "PFM JSON", + "Gene", "Gene Status", "Gene Error", "Gene Outcome", + "ORF Candidate Order", "ORF ID", "ORF Length (aa)", + "ORF Examination Status", "ORF Selection Status", "ORF Outcome", + "Query Pfam Occurrence", "Query Pfam ID", "Query Pfam Name", + "Query Pfam Start (0-based aa)", "Query Pfam End (0-based-exclusive aa)", + "Query Pfam Status", "Query Pfam Error", "Query Pfam Outcome", + "BLAST Hit Group", "BLAST Hit ID", "BLAST Accession", "BLAST Description", + "BLAST Species Name", "BLAST Species Tax ID", "BLAST E-value", + "BLAST Identity Ratio", "BLAST Identity Percent", "Query Coverage Ratio", + "Query Coverage Percent", "BLAST Status", "BLAST Error", "BLAST Outcome", + "BLAST Retrieval Attempted Identifier", + "Hit Pfam Occurrence", "Hit Pfam ID", "Hit Pfam Name", + "Hit Pfam Start (0-based aa)", "Hit Pfam End (0-based-exclusive aa)", + "Hit Pfam Status", "Hit Pfam Error", "Hit Pfam Outcome", + "Global Alignment ID", "Global Alignment Identity Ratio", + "Global Alignment Identity Percent", "Global Alignment Threshold Ratio", + "Global Alignment Threshold Passed", "Global Alignment Status", + "Global Alignment Error", + "Motif Database", "Motif Matrix ID", "Motif Name", "Motif Class", + "JASPAR Tool TF Name", "JASPAR Tool BLAST E-value", + "JASPAR Tool Query Region", "JASPAR Tool TF Region", + "JASPAR Tool DBD Identity Ratio", "Motif URL", "Motif PFM JSON", + "Motif Status", "Motif Error", "Motif Outcome", ] @@ -36,6 +47,15 @@ class InputValidationError(Exception): """A user-provided input or setting failed preflight validation.""" +def _require_external_path(path, description): + repository_root = Path(__file__).resolve().parents[2] + resolved = Path(path).expanduser().resolve() + if resolved == repository_root or repository_root in resolved.parents: + raise InputValidationError( + f"{description} must be outside the Lverage repository: {resolved}" + ) + + def _positive_float(value): try: parsed = float(value) @@ -72,12 +92,13 @@ def build_parser(): prog="lverage", description=( "Run V2 on one nucleotide FASTA file per gene. --output-dir creates " - "a results.tsv, orfs.fasta, and metadata.json evidence bundle." + "a grouped results.csv, orfs.fasta, and metadata.json evidence bundle." ), epilog=( - "Rows are explicitly typed: input, ORF, query_domain, ortholog, " - "ortholog_retrieval, ortholog_domain, domain_comparison, motif, stage_status, or gene_error. " - "Earlier-stage rows are not ortholog/motif hits. --stop-after alignment " + "results.csv groups evidence by gene, ORF candidate, query Pfam, " + "BLAST hit, hit Pfam, global alignment, and motif. Every row " + "repeats its applicable supporting fields; blank cells mean " + "unavailable or not reached. --stop-after alignment " "is an intentional pre-JASPAR stop. Exit 0 means completed or intentionally " "stopped (including biological no-hits), 2 is preflight failure, and 1 " "means one or more execution failures with partial evidence preserved." @@ -88,14 +109,18 @@ def build_parser(): parser.add_argument("--blast-db", required=True, metavar="PREFIX", help="local protein BLAST database prefix") parser.add_argument("--species-map", required=True, metavar="JSON", help="JSON mapping species names to NCBI tax IDs") parser.add_argument("--pfam-db", required=True, metavar="DIR", help="local PfamScan database directory") - parser.add_argument("--output-dir", required=True, metavar="DIR", help="new directory for results.tsv, orfs.fasta, and metadata.json") + parser.add_argument("--output-dir", required=True, metavar="DIR", help="new directory for results.csv, orfs.fasta, and metadata.json") parser.add_argument("--blastp", default="blastp", metavar="EXE", help="blastp executable name or path") parser.add_argument("--blastdbcmd", default="blastdbcmd", metavar="EXE", help="blastdbcmd executable name or path") parser.add_argument("--pfamscan", default="pfam_scan.pl", metavar="EXE", help="pfam_scan.pl executable name or path") parser.add_argument("--identity-threshold", type=_identity_ratio, default=0.7, metavar="RATIO", help="minimum domain identity ratio") parser.add_argument("--valid-pfam", action="append", default=None, metavar="ACCESSION", help="accepted Pfam accession; repeatable") parser.add_argument("--jaspar-hits", type=_positive_int, default=10, metavar="N", help="maximum JASPAR motif hits") - parser.add_argument("--jaspar-evalue", type=_positive_float, default=1e-6, metavar="VALUE", help="maximum JASPAR inference E-value") + parser.add_argument("--jaspar-evalue", type=_positive_float, default=1e-6, metavar="VALUE", help="maximum JASPAR inference-tool BLAST E-value") + parser.add_argument("--jaspar-data-dir", default=os.environ.get("LVERAGE_JASPAR_DATA_DIR"), metavar="DIR", help="external local JASPAR inference data directory (data only; no source checkout required)") + parser.add_argument("--jaspar-sqlite-db", default=os.environ.get("LVERAGE_JASPAR_SQLITE_DB"), metavar="FILE", help="local JASPAR2026.sqlite matrix and annotation database") + parser.add_argument("--hmmscan", default="hmmscan", metavar="EXE", help="hmmscan executable name or path for local JASPAR inference") + parser.add_argument("--hmmalign", default="hmmalign", metavar="EXE", help="hmmalign executable name or path for local JASPAR inference") parser.add_argument("--stop-after", choices=("alignment",), help="intentionally stop after domain alignment, without contacting JASPAR") return parser @@ -180,9 +205,13 @@ def _validate_inputs(args): pfam_dir = Path(args.pfam_db) if not pfam_dir.is_dir(): raise InputValidationError(f"PfamScan database directory does not exist: {pfam_dir}") + _require_external_path(pfam_dir, "PfamScan database") if not args.blast_db.strip(): raise InputValidationError("--blast-db must be a nonempty database prefix") - if not args.blastp.strip() or not args.blastdbcmd.strip() or not args.pfamscan.strip(): + _require_external_path(args.blast_db, "protein BLAST database") + if any(not executable.strip() for executable in ( + args.blastp, args.blastdbcmd, args.pfamscan, + args.hmmscan, args.hmmalign)): raise InputValidationError("executable paths must not be empty") if args.valid_pfam is not None and any(not value.strip() for value in args.valid_pfam): raise InputValidationError("--valid-pfam values must not be empty") @@ -191,12 +220,13 @@ def _validate_inputs(args): raise InputValidationError(f"output directory already exists: {output_dir}") if not output_dir.parent.is_dir(): raise InputValidationError(f"output parent directory does not exist: {output_dir.parent}") + _require_external_path(output_dir, "output directory") return inputs, species_map, species_path def _build_pipeline(args, species_map): from .blast import LocalBlastSearcher - from .jaspar import Jaspar2024MotifDB + from .jaspar import Jaspar2026MotifDB from .orf_searcher import OrffinderOrfSearcher from .pfam import LocalPfamScanner from .pipeline import Lverage @@ -204,8 +234,17 @@ def _build_pipeline(args, species_map): database_path=args.blast_db, species_map=species_map, blastp_path=args.blastp, blastdbcmd_path=args.blastdbcmd, ) + motif_database = Jaspar2026MotifDB( + n_hits=args.jaspar_hits, + escore_threshold=args.jaspar_evalue, + data_dir=args.jaspar_data_dir, + sqlite_db=args.jaspar_sqlite_db, + blastp_path=args.blastp, + hmmscan_path=args.hmmscan, + hmmalign_path=args.hmmalign, + ) return Lverage( - motif_database_list=[Jaspar2024MotifDB(n_hits=args.jaspar_hits, escore_threshold=args.jaspar_evalue)], + motif_database_list=[motif_database], orf_searcher=OrffinderOrfSearcher(), domain_scanner=LocalPfamScanner(database_path=args.pfam_db, pfamscan_path=args.pfamscan), ortholog_searcher=ortholog_searcher, @@ -233,98 +272,310 @@ def _ratio_and_percent(value): return value, value * 100 if value is not None else None -def _event_row(event, gene, sequence_ids, selection_status, selection_details): - row = {header: "" for header in RESULT_HEADERS} - row.update({"Input Gene": gene, "Stage": event.get("stage", ""), "Status": event.get("status", ""), "Error": event.get("error", ""), "Outcome Detail": event.get("detail", "")}) - kind = event["event"] - if kind == "gene_start": - row["Row Type"] = "input" - elif kind == "orf_candidate": - sequence = event["orf_sequence"] - row["Row Type"] = "orf" - row["Query ORF ID"] = sequence_ids[sequence] - row["Query ORF Selected"] = "true" if selection_status.get((gene, sequence)) == "selected" else "false" - if kind == "orf_candidate": - row["Stage"] = "orf" - row["Status"] = selection_status.get((gene, sequence), "not-reached") - row["Outcome Detail"] = selection_details.get((gene, sequence), "") - elif kind == "query_domain": - domain = event["domain"] - row["Row Type"] = "query_domain" - row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] - row["Query ORF Selected"] = "true" if selection_status.get((gene, event["orf_sequence"])) == "selected" else "false" - row.update({"Query Domain Name": domain.name, "Query Domain ID": domain.accession, "Query Domain Start 0-based": domain.start, "Query Domain End 0-based-exclusive": domain.end}) - elif kind == "ortholog": - ortholog = event["ortholog"] - row["Row Type"] = "ortholog" - row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] - row.update(_ortholog_fields(ortholog)) - elif kind == "ortholog_retrieval": - row["Row Type"] = "ortholog_retrieval" - row["Stage"] = "blast" - if event.get("orf_sequence"): - row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] - row["Attempted Hit Identifier"] = event.get("attempted_identifier", "") - row["Ortholog Hit ID"] = event.get("hit_id") or "" - row["Ortholog Accession"] = event.get("accession") or "" - elif kind in {"blast_search", "blast_search_summary"}: - row["Row Type"] = "stage_status" - row["Stage"] = "blast" - if event.get("orf_sequence"): - row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] - details = [] - for key in ("hit_count", "retained_hit_count", "retrieved_count", "skipped_count"): - if key in event: - details.append(f"{key}={event[key]}") - row["Outcome Detail"] = "; ".join(details) - elif kind == "ortholog_domain": - domain = event["domain"] - row["Row Type"] = "ortholog_domain" - row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] - row.update(_ortholog_fields(event["ortholog"])) - row.update({"Ortholog Domain Name": domain.name, "Ortholog Domain ID": domain.accession, "Ortholog Domain Start 0-based": domain.start, "Ortholog Domain End 0-based-exclusive": domain.end}) - elif kind == "domain_comparison": - qdomain, odomain = event["query_domain"], event["ortholog_domain"] - row["Row Type"] = "domain_comparison" - row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] - row.update(_ortholog_fields(event["ortholog"])) - row.update({"Query Domain Name": qdomain.name, "Query Domain ID": qdomain.accession, "Query Domain Start 0-based": qdomain.start, "Query Domain End 0-based-exclusive": qdomain.end, "Ortholog Domain Name": odomain.name, "Ortholog Domain ID": odomain.accession, "Ortholog Domain Start 0-based": odomain.start, "Ortholog Domain End 0-based-exclusive": odomain.end, "Domain Pair Identity Ratio": event["domain_identity"], "Domain Pair Identity Percent": event["domain_identity"] * 100, "Threshold Ratio": event.get("threshold"), "Threshold Passed": "true" if event["status"] == "accepted" else "false"}) - elif kind == "motif": - row["Row Type"] = "motif" - row["Query ORF ID"] = sequence_ids[event["orf_sequence"]] - row.update(_ortholog_fields(event["ortholog"])) - qdomain, odomain = event["query_domain"], event["ortholog_domain"] - row.update({"Query Domain Name": qdomain.name, "Query Domain ID": qdomain.accession, "Query Domain Start 0-based": qdomain.start, "Query Domain End 0-based-exclusive": qdomain.end, "Ortholog Domain Name": odomain.name, "Ortholog Domain ID": odomain.accession, "Ortholog Domain Start 0-based": odomain.start, "Ortholog Domain End 0-based-exclusive": odomain.end, "Domain Pair Identity Ratio": event["domain_identity"], "Domain Pair Identity Percent": event["domain_identity"] * 100, "Threshold Ratio": event.get("threshold"), "Threshold Passed": "true", "Motif Database": event["motif_database_name"]}) - motif = event["motif_record"] - motif_values = dict(zip(motif.get_headers(), motif.get_values())) - mapping = {"Matrix ID": "Matrix ID", "Motif Name": "Motif Name", "Motif Class": "Motif Class", "Motif Inference E-value": "Inference E-value", "Motif URL": "Motif URL"} - for target, source in mapping.items(): - row[target] = motif_values.get(source, "") - pfm = motif_values.get("PFM", "") - row["PFM JSON"] = json.dumps(pfm, ensure_ascii=False, separators=(",", ":")) if pfm != "" else "" - elif kind == "gene_outcome": - row["Row Type"] = "stage_status" - elif kind == "gene_failure": - row["Row Type"] = "gene_error" - elif kind == "stage_status": - row["Row Type"] = "stage_status" - else: - return None - return row - - def _ortholog_fields(ortholog): identity, identity_percent = _ratio_and_percent(ortholog.identity) coverage, coverage_percent = _ratio_and_percent(ortholog.query_coverage) return { - "Ortholog Hit ID": ortholog.hit_id or "", "Ortholog Accession": ortholog.accession, - "Ortholog Description": ortholog.description, "Ortholog Species Name": ortholog.species_name, - "Ortholog Species Tax ID": ortholog.species_tax_id, "BLAST E-value": ortholog.evalue, - "Ortholog Identity Ratio": identity, "Ortholog Identity Percent": identity_percent, + "BLAST Hit ID": ortholog.hit_id or "", "BLAST Accession": ortholog.accession, + "BLAST Description": ortholog.description, "BLAST Species Name": ortholog.species_name, + "BLAST Species Tax ID": ortholog.species_tax_id, "BLAST E-value": ortholog.evalue, + "BLAST Identity Ratio": identity, "BLAST Identity Percent": identity_percent, "Query Coverage Ratio": coverage, "Query Coverage Percent": coverage_percent, } +def _grouped_result_rows(gene, events, sequence_ids): + """Render one gene as contiguous, self-contained result rows. + + Domain associations use the original objects retained by EvidenceCollector, + not names or accessions: two occurrences can have identical annotations. + Parent evidence is copied onto each descendant row; associations are built + from the event hierarchy, never inferred by filling blank CSV cells. + """ + def blank(): + return {header: "" for header in RESULT_HEADERS} + + def domain_fields(prefix, event, occurrence): + domain = event["domain"] + return { + f"{prefix} Occurrence": occurrence, + f"{prefix} ID": domain.accession, + f"{prefix} Name": domain.name, + f"{prefix} Start (0-based aa)": domain.start, + f"{prefix} End (0-based-exclusive aa)": domain.end, + f"{prefix} Status": event.get("status", ""), + } + + def prepend(fields, children): + # Repeat supporting fields on each result row without changing its + # event-specific association. + if not children: + children = [blank()] + for child in children: + child.update(fields) + return children + + def pair_key(event): + return tuple(id(event.get(key)) for key in + ("ortholog", "query_domain", "ortholog_domain")) + + def motif_fields(event): + values = dict(zip(event["motif_record"].get_headers(), + event["motif_record"].get_values())) + row = blank() + row.update({ + "Motif Database": event["motif_database_name"], + "Motif Matrix ID": values.get("Matrix ID", ""), + "Motif Name": values.get("Motif Name", ""), + "Motif Class": values.get("Motif Class", ""), + "JASPAR Tool TF Name": values.get("JASPAR Tool TF Name", ""), + "JASPAR Tool BLAST E-value": values.get("JASPAR Tool BLAST E-value", ""), + "JASPAR Tool Query Region": values.get("JASPAR Tool Query Region", ""), + "JASPAR Tool TF Region": values.get("JASPAR Tool TF Region", ""), + "JASPAR Tool DBD Identity Ratio": values.get("JASPAR Tool DBD Identity Ratio", ""), + "Motif URL": values.get("Motif URL", ""), + "Motif Status": event.get("status", "retained"), + }) + pfm = values.get("PFM", "") + if pfm != "": + row["Motif PFM JSON"] = json.dumps( + pfm, ensure_ascii=False, separators=(",", ":") + ) + return row + + def diagnostic(prefix, event): + row = blank() + row[f"{prefix} Status"] = event.get("status", "failed") + row[f"{prefix} Error"] = event.get("error", "") + if f"{prefix} Outcome" in row: + row[f"{prefix} Outcome"] = event.get("detail", "") + return row + + def blast_outcome(summary): + return "; ".join( + [summary.get("status", "")] + + [f"{name}={summary[name]}" for name in + ("hit_count", "retrieved_count", "skipped_count") if name in summary] + ) + + failures = [event for event in events if event["event"] == "gene_failure"] + retrieval_failures = [event for event in events + if event["event"] == "ortholog_retrieval" + and event.get("status") == "skipped"] + handled_failures = set() + outcomes = [event for event in events if event["event"] == "gene_outcome"] + final = outcomes[-1] if outcomes else {} + gene_fields = { + "Gene": gene, + "Gene Status": "failed" if failures or any( + event["event"] == "ortholog_retrieval" and event.get("status") == "skipped" + for event in events) else final.get("status", "in-progress"), + "Gene Error": ( + failures[-1].get("error", "") if failures else + "; ".join(event.get("error", "") for event in retrieval_failures) + ), + "Gene Outcome": ( + f"{failures[-1].get('stage', 'unknown')}: execution failed" if failures + else "blast: partial-retrieval-failure" if retrieval_failures + else f"{final.get('stage', 'pipeline')}: {final.get('detail') or final.get('status', '')}" + if final else "" + ), + } + candidates = {} + for event in events: + if event["event"] == "orf_candidate": + candidates.setdefault(event["orf_sequence"], event) + ordered_candidates = sorted(candidates, key=lambda sequence: -len(sequence)) + selections = {event["orf_sequence"]: event for event in events + if event["event"] == "orf_selection"} + comparisons = {} + for event in events: + if event["event"] == "domain_comparison": + comparisons.setdefault(pair_key(event), []).append(event) + alignment_ids = {id(event): f"A{index}" for index, event in enumerate( + (event for event in events if event["event"] == "domain_comparison"), 1)} + + def comparison_rows(comparison): + key = pair_key(comparison) + children = [] + # Preserve completion order, including a database failure after motifs. + for event in events: + if pair_key(event) != key: + continue + kind = event["event"] + if kind == "motif": + children.append(motif_fields(event)) + elif (kind == "stage_result" and event.get("stage_name") == "motif" + and event.get("motif_count") == 0): + row = diagnostic("Motif", event) + row["Motif Database"] = event.get("motif_database_name", "") + row["Motif Outcome"] = "species unavailable" if event.get("status") == "species-unavailable" else "motif_count=0" + children.append(row) + elif kind == "gene_failure" and event.get("stage") == "motif": + row = diagnostic("Motif", event) + row["Motif Database"] = event.get("motif_database_name", "") + children.append(row) + handled_failures.add(id(event)) + return prepend({ + "Global Alignment ID": alignment_ids[id(comparison)], + "Global Alignment Identity Ratio": comparison["domain_identity"], + "Global Alignment Identity Percent": comparison["domain_identity"] * 100, + "Global Alignment Threshold Ratio": comparison.get("threshold", ""), + "Global Alignment Threshold Passed": str(comparison.get("status") == "accepted").lower(), + "Global Alignment Status": comparison.get("status", ""), + }, children) + + rows = [] + for order, sequence in enumerate(ordered_candidates, 1): + selection = selections.get(sequence, {}) + selected = selection.get("status") == "selected" + queries = [event for event in events if event["event"] == "query_domain" + and event.get("orf_sequence") == sequence] + examined = bool(selection or queries) or any( + event["event"] == "stage_attempt" and event.get("stage_name") == "query_pfam" + and event.get("orf_sequence") == sequence for event in events) + query_failures = [event for event in failures if event.get("stage") == "query_pfam" + and event.get("orf_sequence") == sequence] + candidate_fields = { + "ORF Candidate Order": order, + "ORF ID": sequence_ids.get(sequence, _orf_id(sequence)), + "ORF Length (aa)": len(sequence), + } + if examined: + candidate_fields["ORF Examination Status"] = "examined" + candidate_fields["ORF Selection Status"] = ( + "failed" if any(event.get("orf_sequence") == sequence for event in query_failures) + else selection.get("status", "not-selected") + ) + if selection.get("detail"): + candidate_fields["ORF Outcome"] = selection["detail"] + children = [] + if query_failures: + candidate_fields["ORF Examination Status"] = "failed" + for event in query_failures: + children.append(diagnostic("Query Pfam", event)) + handled_failures.add(id(event)) + if not queries and selection.get("status") == "biological-no-hit": + row = blank() + row["Query Pfam Status"] = "no accepted domain" + row["Query Pfam Outcome"] = selection.get("detail", "no accepted query domain") + children.append(row) + + hit_events = [event for event in events if event["event"] == "ortholog" + and event.get("orf_sequence") == sequence] + retrievals = [event for event in events if event["event"] == "ortholog_retrieval" + and event.get("orf_sequence") == sequence] + blast_summaries = [event for event in events if event["event"] in + {"blast_search", "blast_search_summary"} + and event.get("orf_sequence") == sequence] + accepted = [event for event in queries if event.get("status") == "accepted"] + first_query = accepted[0] if accepted else None + + for q_index, query in enumerate(queries, 1): + qdomain = query["domain"] + query_children = [] + if selected and query.get("status") == "accepted": + for h_index, hit_event in enumerate(hit_events, 1): + ortholog = hit_event["ortholog"] + domains = [event for event in events if event["event"] == "ortholog_domain" + and event.get("ortholog") is ortholog] + hit_children = [] + for d_index, domain in enumerate(domains, 1): + hdomain = domain["domain"] + key = (id(ortholog), id(qdomain), id(hdomain)) + matching = comparisons.get(key, []) + alignment_failures = [event for event in failures + if event.get("stage") == "alignment" + and pair_key(event) == key] + matched_any_query = any( + other["domain"].accession.split(".", 1)[0] == hdomain.accession.split(".", 1)[0] + for other in accepted) + # Display unpaired hit domains once, under the first query. + # Matched but not yet evaluated pairs stay visible with blank results. + same_family = qdomain.accession.split(".", 1)[0] == hdomain.accession.split(".", 1)[0] + if not same_family and (matched_any_query or query is not first_query): + continue + domain_children = [] + for comparison in matching: + domain_children.extend(comparison_rows(comparison)) + for event in alignment_failures: + domain_children.append(diagnostic("Global Alignment", event)) + handled_failures.add(id(event)) + fields = domain_fields("Hit Pfam", domain, domain.get("domain_index", d_index)) + if not matched_any_query: + fields["Hit Pfam Outcome"] = "no matching query Pfam" + hit_children.extend(prepend(fields, domain_children)) + scans = [event for event in events if event["event"] == "stage_result" + and event.get("stage_name") == "ortholog_pfam" + and event.get("ortholog") is ortholog] + scan_failures = [event for event in failures + if event.get("stage") == "ortholog_pfam" + and event.get("ortholog") is ortholog] + if query is first_query: + for event in scan_failures: + hit_children.append(diagnostic("Hit Pfam", event)) + handled_failures.add(id(event)) + if not domains and scans: + row = diagnostic("Hit Pfam", scans[-1]) + row["Hit Pfam Outcome"] = f"domain_count={scans[-1].get('domain_count', 0)}" + hit_children.append(row) + # Each hit is visible even before it is scanned. On later + # query parents, only display hits with relevant child evidence. + if query is first_query or hit_children: + fields = _ortholog_fields(ortholog) + summary = blast_summaries[-1] if blast_summaries else {} + fields["BLAST Outcome"] = blast_outcome(summary) + fields.update({"BLAST Hit Group": h_index, + "BLAST Status": hit_event.get("status", "retained")}) + query_children.extend(prepend(fields, hit_children)) + if query is first_query: + # Retrieval attempt diagnostics stay beside their native hit. + for event in retrievals: + row = diagnostic("BLAST", event) + if blast_summaries: + row["BLAST Outcome"] = blast_outcome(blast_summaries[-1]) + row.update({"BLAST Hit ID": event.get("hit_id", ""), + "BLAST Accession": event.get("accession", ""), + "BLAST Hit Group": event.get("hit_group", ""), + "BLAST Description": event.get("description", ""), + "BLAST Species Name": event.get("species_name", ""), + "BLAST Species Tax ID": event.get("species_tax_id", ""), + "BLAST E-value": event.get("evalue", ""), + "BLAST Identity Ratio": event.get("identity_ratio", ""), + "BLAST Identity Percent": event.get("identity_percent", ""), + "Query Coverage Ratio": event.get("query_coverage_ratio", ""), + "Query Coverage Percent": event.get("query_coverage_percent", ""), + "BLAST Retrieval Attempted Identifier": event.get("attempted_identifier", "")}) + query_children.append(row) + blast_failures = [event for event in failures if event.get("stage") == "blast"] + for event in blast_failures: + query_children.append(diagnostic("BLAST", event)) + handled_failures.add(id(event)) + if not hit_events and not retrievals and not blast_failures: + if blast_summaries: + summary = blast_summaries[-1] + row = diagnostic("BLAST", summary) + row["BLAST Outcome"] = blast_outcome(summary) + elif final.get("stage") == "blast": + row = diagnostic("BLAST", final) + else: + row = None + if row: + query_children.append(row) + children.extend(prepend(domain_fields("Query Pfam", query, q_index), query_children)) + rows.extend(prepend(candidate_fields, children)) + + # A failure without sufficiently specific evidence must still be visible. + prefixes = {"query_pfam": "Query Pfam", "blast": "BLAST", + "ortholog_pfam": "Hit Pfam", "alignment": "Global Alignment", "motif": "Motif"} + for event in failures: + if id(event) not in handled_failures and event.get("stage") in prefixes: + row = diagnostic(prefixes[event["stage"]], event) + row["Gene"] = gene # Restart context: do not attach to the last ORF. + row["Gene Outcome"] = f"unassociated failure at {event['stage']}" + rows.append(row) + return prepend(gene_fields, rows) + class _Bundle: def __init__(self, destination, metadata): self.path = destination @@ -351,29 +602,26 @@ def add_event(self, gene, event): self.flush_metadata() def _write_results(self): - selection = {} - selection_details = {} - for gene, event in self.events: - if event["event"] == "orf_selection": - selection[(gene, event["orf_sequence"])] = event["status"] - selection_details[(gene, event["orf_sequence"])] = event.get("detail", "") - selected_by_gene = { - gene: any(status == "selected" for (event_gene, _), status in selection.items() if event_gene == gene) - for gene, _ in self.events - } + grouped_events = {} + gene_order = [] for gene, event in self.events: - if event["event"] == "orf_candidate" and (gene, event["orf_sequence"]) not in selection and selected_by_gene.get(gene): - selection[(gene, event["orf_sequence"])] = "not-selected" - path = self.path / "results.tsv" - with path.open("w", encoding="utf-8", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=RESULT_HEADERS, delimiter="\t", lineterminator="\n", extrasaction="ignore") + if gene not in grouped_events: + grouped_events[gene] = [] + gene_order.append(gene) + grouped_events[gene].append(event) + path = self.path / "results.csv" + temporary = self.path / ".results.csv.tmp" + with temporary.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=RESULT_HEADERS, delimiter=",", lineterminator="\n", extrasaction="ignore") writer.writeheader() - for gene, event in self.events: - row = _event_row(event, gene, self.sequence_ids, selection, selection_details) - if row is not None: + for gene in gene_order: + for row in _grouped_result_rows( + gene, grouped_events[gene], self.sequence_ids + ): writer.writerow(row) handle.flush() os.fsync(handle.fileno()) + os.replace(temporary, path) def _write_orfs(self): with (self.path / "orfs.fasta").open("w", encoding="utf-8", newline="\n") as handle: @@ -401,7 +649,8 @@ def _metadata(args, inputs, species_path): except importlib.metadata.PackageNotFoundError: package_version = "unknown" return { - "schema_version": 1, + "schema_version": 2, + "results_format": "grouped-csv", "started_at": _timestamp(), "updated_at": _timestamp(), "finished_at": None, "completion": "in-progress", "current_gene": None, "current_stage": "preflight-complete", "summary": {"genes_total": len(inputs), "genes_completed": 0, "genes_failed": 0, "genes_intentionally_stopped": 0}, @@ -413,13 +662,38 @@ def _metadata(args, inputs, species_path): "pfamscan": {"configured_executable": args.pfamscan, "resolved_executable": shutil.which(args.pfamscan), "version": "unknown"}, "orffinder": {"version": _distribution_version("orffinder")}, "biopython": {"version": _distribution_version("biopython")}, - "jaspar": {"release": "JASPAR2024", "service_version": "unknown"}, + "jaspar": { + "release": "JASPAR2026", + "release_verification_method": "JASPAR2026.sqlite schema and exact per-taxonomy CORE inventory comparison", + "inference_algorithm_provenance": { + "repository": "https://github.com/amathelier/JASPAR-inference-tool", + "source_revision": "ac4024148f1f140c7370d7b13a70556b4d13f7a3", + "source_version": "2026.1.0", + "runtime_checkout_required": False, + "runtime_python_required": False, + }, + "data": { + "release": "JASPAR2026", + "directory": args.jaspar_data_dir, + "annotations_and_matrices_database": args.jaspar_sqlite_db, + "database_sha256": None, + }, + "executables": { + "blastp": args.blastp, + "hmmscan": args.hmmscan, + "hmmalign": args.hmmalign, + }, + "online_requests": False, + "validation": "not-required" if args.stop_after == "alignment" else "pending", + }, }, "settings": { "identity_threshold_ratio": args.identity_threshold, "valid_pfam": args.valid_pfam or [], "jaspar_hits": args.jaspar_hits, - "jaspar_evalue": args.jaspar_evalue, "stop_after": args.stop_after or "full-pipeline", - "orf_minimum_length_nt": 75, "blast_evalue": 1e-6, "blast_top_n": 20, + "jaspar_inference_blast_evalue_threshold": args.jaspar_evalue, + "stop_after": args.stop_after or "full-pipeline", + "orf_minimum_length_nt": 75, "blast_evalue": 1e-6, "blast_top_n": 20, "blast_seg": "default", + "blast_description_excluded_terms": None, }, "references": { "input_files": [{"gene": item["gene"], "path": item["path"], "sha256": item["sha256"]} for item in inputs], @@ -471,6 +745,15 @@ def main(argv=None): failures = 0 try: pipeline = _build_pipeline(args, species_map) + jaspar_database = pipeline.motif_database_list[0] + metadata_factory = getattr(jaspar_database, "configuration_metadata", None) + metadata["tools"]["jaspar"] = ( + metadata_factory() if callable(metadata_factory) + else {"release": getattr(jaspar_database, "name", "unknown")} + ) + metadata["tools"]["jaspar"]["validation"] = ( + "not-required" if args.stop_after == "alignment" else "not-run" + ) except Exception as error: failures = len(inputs) safe_error = _redact_error(error) @@ -485,6 +768,11 @@ def main(argv=None): print(f"lverage: pipeline initialization failed: {safe_error}", file=sys.stderr) return 1 + excluded_terms = getattr(pipeline.ortholog_searcher, "excluded_terms", None) + metadata["settings"]["blast_description_excluded_terms"] = ( + list(excluded_terms) if excluded_terms is not None else None + ) + from .evidence import EvidenceCollector for item in inputs: gene = item["gene"] @@ -522,12 +810,40 @@ def main(argv=None): safe_error = _redact_error(error) metadata["errors"].append({"gene": gene, "stage": collector.stage, "error": safe_error}) metadata["summary"]["genes_failed"] = failures - bundle.add_event(gene, {"event": "gene_failure", "stage": collector.stage, "status": "failed", "error": safe_error}) + failure_event = { + "event": "gene_failure", "stage": collector.stage, + "status": "failed", "error": safe_error, + } + attempt = next((event for event in reversed(collector.events) + if event.get("event") == "stage_attempt" + and event.get("stage_name") == collector.stage), None) + if attempt is not None: + for key in ( + "orf_sequence", "ortholog", "query_domain", + "ortholog_domain", "motif_database_name", + ): + if key in attempt: + failure_event[key] = attempt[key] + bundle.add_event(gene, failure_event) stages = ("orf", "query_pfam", "blast", "ortholog_pfam", "alignment", "motif") last_index = stages.index(collector.stage) if collector.stage in stages else -1 for stage in stages[last_index + 1:]: bundle.add_event(gene, {"event": "stage_status", "stage": stage, "status": "not-reached", "detail": "execution failed in upstream stage"}) print(f"lverage: execution failed for gene {gene} at {collector.stage}: {safe_error}", file=sys.stderr) + metadata_factory = getattr(jaspar_database, "configuration_metadata", None) + if callable(metadata_factory): + metadata["tools"]["jaspar"] = metadata_factory() + motif_attempted = any( + event.get("event") == "stage_attempt" + and event.get("stage_name") == "motif" + for event in collector.events + ) + if args.stop_after == "alignment": + metadata["tools"]["jaspar"]["validation"] = "not-required" + elif getattr(jaspar_database, "_validated", False): + metadata["tools"]["jaspar"]["validation"] = "verified" + elif motif_attempted: + metadata["tools"]["jaspar"]["validation"] = "failed" metadata["current_gene"] = gene metadata["current_stage"] = "complete" metadata["updated_at"] = _timestamp() diff --git a/src/lverage/jaspar.py b/src/lverage/jaspar.py index d1d2835..1ad13ee 100644 --- a/src/lverage/jaspar.py +++ b/src/lverage/jaspar.py @@ -1,309 +1,492 @@ -""" -This file contains the Jaspar2024 Motif Database class which can be found at https://jaspar.elixir.no/. -Citation: - Rauluseviciute I, Riudavets-Puig R, Blanc-Mathieu R, Castro-Mondragon JA, Ferenc K, Kumar V, Lemma RB, - Lucas J, Chèneby J, Baranasic D, Khan A, Fornes O, Gundersen S, Johansen M, Hovig E, Lenhard B, - Sandelin A, Wasserman WW, Parcy F, Mathelier A JASPAR 2024: 20th anniversary of the open-access - database of transcription factor binding profiles Nucleic Acids Res. 2024 Jan 5;52(D1):D174-D182.; - doi: 10.1093/nar/gkad1059 - -Copyright (C) Bradham Lab - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - -Correspondence: - - Cynthia A. Bradham - cbradham@bu.edu - * - - Anthony B. Garza - abgarza@bu.edu - ** - - Stephanie P. Hao - sphao@bu.edu - ** - - Yeting Li - yetingli@bu.edu - ** - - Thomas Shin - thshin@bu.edu - ** - - \* Principle Investigator, ** Software Developers -""" - -#@#@#@#@#@#@#@#@#@#@#@#@#@#@#@#@#@ -# Imports -from dataclasses import dataclass +"""Local JASPAR 2026 motif database adapter for Lverage.""" +from contextlib import closing +from dataclasses import dataclass +import hashlib +import json +import math +import os +from pathlib import Path +import re +import sqlite3 +import tempfile + +from .jaspar_inference import JASPAR_TAXA, infer_sequence, load_jaspar_data +from .jaspar_models import load_cisbp_models from .motif_database import MotifDBRecordTemplate, MotifDBTemplate, MotifSearchRequest -import requests + + +# Algorithm provenance only. Runtime does not import from or inspect this checkout. +UPSTREAM_REPOSITORY = "https://github.com/amathelier/JASPAR-inference-tool" +UPSTREAM_REVISION = "ac4024148f1f140c7370d7b13a70556b4d13f7a3" +UPSTREAM_VERSION = "2026.1.0" +JASPAR_RELEASE = "JASPAR2026" +MATRIX_ID_PATTERN = re.compile(r"^(MA\d{4,5})\.(\d+)$") @dataclass class JasparRecord(MotifDBRecordTemplate): - """ - Motif record returned by the JASPAR adapter. + """One exact-version JASPAR profile and its inference evidence. Parameters ---------- matrix_id : str - JASPAR matrix identifier + Exact versioned profile identifier, for example ``MA0593.1``. motif_name : str - Name of the motif + Profile name from the local matrix database. pfm : dict - Position frequency matrix supplied by JASPAR + Position frequency matrix with A, C, G and T count rows. motif_url : str - URL for the motif page or logo + Convenience link to the hosted profile; not used for inference. motif_class : str - JASPAR motif class + Transcription factor class annotated for the profile. inference_evalue : float - E-value reported by sequence inference + BLAST E-value reported by local motif inference. + inference_tf_name : str, optional + Reference transcription factor name reported by inference. + inference_query_region : str, optional + BLAST query endpoints, one-based and inclusive, in the submitted + ortholog protein or its window. + inference_tf_region : str, optional + One-based inclusive BLAST endpoints in the reference protein. + inference_dbd_identity : float, optional + Inference-tool domain identity ratio, distinct from Lverage's + query-domain versus ortholog-domain alignment identity. """ - matrix_id : str - motif_name : str - pfm : dict - motif_url : str - motif_class : str - inference_evalue : float + matrix_id: str + motif_name: str + pfm: dict + motif_url: str + motif_class: str + inference_evalue: float + inference_tf_name: str = "" + inference_query_region: str = "" + inference_tf_region: str = "" + inference_dbd_identity: float = 0.0 headers = ( - "Matrix ID", - "Motif Name", - "PFM", - "Motif URL", - "Motif Class", - "Inference E-value", + "Matrix ID", "Motif Name", "PFM", "Motif URL", "Motif Class", + "JASPAR Tool TF Name", "JASPAR Tool BLAST E-value", + "JASPAR Tool Query Region", "JASPAR Tool TF Region", + "JASPAR Tool DBD Identity Ratio", ) def get_values(self) -> list: - """ - Return values in the JASPAR record schema. + """Return the record fields in export-header order. Returns ------- list - Serialized JASPAR values + Field values in the order returned by ``get_headers()``. """ - return [ - self.matrix_id, - self.motif_name, - self.pfm, - self.motif_url, - self.motif_class, - self.inference_evalue, + self.matrix_id, self.motif_name, self.pfm, self.motif_url, + self.motif_class, self.inference_tf_name, self.inference_evalue, + self.inference_query_region, self.inference_tf_region, + self.inference_dbd_identity, ] -class Jaspar2024MotifDB(MotifDBTemplate): - """ - Class for the JASPAR2024 Motif Database, which can be found at https://jaspar.elixir.no/ +class Jaspar2026MotifDB(MotifDBTemplate): + """Run the adapted upstream algorithm against local JASPAR 2026 data. + + Only ``data_dir`` (the fork-generated ``files`` tree), ``sqlite_db``, and + external BLASTP/HMMER executable locations are configurable. The upstream + repository revision below records algorithm provenance, not a runtime + checkout or interpreter requirement. Parameters ---------- - n_hits : int - Number of motif hits to return - escore_threshold : float - Maximum e-score threshold for motif hits - request_timeout_seconds : float - Maximum number of seconds allowed for each API request - - Attributes - ---------- - name: str - Name of the motif database - metainfo: dict - Any extra information regarding this motif database - jaspar_rest_url: str - URL for the JASPAR REST API - n_hits: int - Number of motif hits to return - escore_threshold: float - Minimum e-score threshold for motif hits - request_timeout_seconds: float - Maximum number of seconds allowed for each API request + n_hits : int, optional + Maximum retained predictions per request; default is 10. + escore_threshold : float, optional + Maximum inference BLAST E-value; default is ``1e-6``. Equivalence + to the former REST endpoint's E-value has not been established. + data_dir : str or pathlib.Path, optional + Local inference reference-data tree. Defaults to the + ``LVERAGE_JASPAR_DATA_DIR`` environment variable. + sqlite_db : str or pathlib.Path, optional + Local ``JASPAR2026.sqlite`` matrix and annotation database. + Defaults to ``LVERAGE_JASPAR_SQLITE_DB``. + blastp_path : str, optional + BLASTP executable name or path; default is ``blastp``. + hmmscan_path : str, optional + HMMER domain-scan executable name or path; default is ``hmmscan``. + hmmalign_path : str, optional + HMMER alignment executable name or path; default is ``hmmalign``. """ - name = "JASPAR2024" + name = "JASPAR2026-local" metainfo = { - "citation":"Rauluseviciute I, Riudavets-Puig R, Blanc-Mathieu R, Castro-Mondragon JA, Ferenc K, Kumar V, Lemma RB, Lucas J, Chèneby J, Baranasic D, Khan A, Fornes O, Gundersen S, Johansen M, Hovig E, Lenhard B, Sandelin A, Wasserman WW, Parcy F, Mathelier A JASPAR 2024: 20th anniversary of the open-access database of transcription factor binding profiles Nucleic Acids Res. 2024 Jan 5;52(D1):D174-D182.; doi: 10.1093/nar/gkad1059", - "version": "JASPAR2024", - "collection":"JASPAR CORE", - "url":"https://jaspar.elixir.no/" - } - - # URL for the JASPAR REST API - jaspar_rest_url = "https://jaspar.elixir.no/api/v1" - - # URL for retrieving species information - jaspar_rest_species_url = "https://jaspar.elixir.no/api/v1/species/" - - # Parameters for getting all species - species_params = {"page":1, - "page_size":1000, - "release":"2024" - } - - def __init__(self, - n_hits : int = 10, - escore_threshold : float = 10**-6, - request_timeout_seconds : float = 30): + "version": JASPAR_RELEASE, + "collection": "JASPAR CORE", + "inference_source": UPSTREAM_REPOSITORY, + "inference_source_revision": UPSTREAM_REVISION, + "inference_source_version": UPSTREAM_VERSION, + "online_requests": False, + } + + def __init__(self, n_hits=10, escore_threshold=1e-6, *, data_dir=None, + sqlite_db=None, blastp_path="blastp", hmmscan_path="hmmscan", + hmmalign_path="hmmalign"): + if isinstance(n_hits, bool) or not isinstance(n_hits, int) or n_hits <= 0: + raise ValueError("n_hits must be a positive integer") + if (isinstance(escore_threshold, bool) + or not isinstance(escore_threshold, (int, float)) + or not math.isfinite(escore_threshold) or escore_threshold <= 0): + raise ValueError("escore_threshold must be a positive finite number") + + self.n_hits = n_hits + self.escore_threshold = float(escore_threshold) + self.data_dir = self._configured_path(data_dir, "LVERAGE_JASPAR_DATA_DIR") + self.sqlite_db = self._configured_path(sqlite_db, "LVERAGE_JASPAR_SQLITE_DB") + self.blastp_path = blastp_path + self.hmmscan_path = hmmscan_path + self.hmmalign_path = hmmalign_path + self._validated = False + self._species_ids = None + self._inference_data = None + self._database_sha256_value = None + + @staticmethod + def _configured_path(value, environment_name): + configured = value if value is not None else os.environ.get(environment_name) + if configured == "": + raise ValueError(f"{environment_name} must not be empty") + return Path(configured).expanduser() if configured is not None else None + + def configuration_metadata(self): + """Return code provenance separately from the required data release. + + Returns + ------- + dict + Source revision, data locations and release, configured executable + paths, and database checksum once configuration is validated. """ - Initialize a JASPAR motif database adapter. + return { + "release": JASPAR_RELEASE, + "release_verification_method": ( + "JASPAR2026.sqlite schema and exact per-taxonomy CORE inventory comparison" + ), + "inference_algorithm_provenance": { + "repository": UPSTREAM_REPOSITORY, + "source_revision": UPSTREAM_REVISION, + "source_version": UPSTREAM_VERSION, + "runtime_checkout_required": False, + "runtime_python_required": False, + }, + "data": { + "release": JASPAR_RELEASE, + "directory": str(self.data_dir) if self.data_dir else None, + "annotations_and_matrices_database": ( + str(self.sqlite_db) if self.sqlite_db else None + ), + "database_sha256": self._database_sha256_value, + }, + "executables": { + "blastp": self.blastp_path, + "hmmscan": self.hmmscan_path, + "hmmalign": self.hmmalign_path, + }, + "online_requests": False, + } + + def _validate_configuration(self): + if self._validated: + return + missing = [ + name for name, value in ( + ("--jaspar-data-dir / LVERAGE_JASPAR_DATA_DIR", self.data_dir), + ("--jaspar-sqlite-db / LVERAGE_JASPAR_SQLITE_DB", self.sqlite_db), + ) if value is None + ] + if missing: + raise RuntimeError( + "local JASPAR inference is not configured; provide " + ", ".join(missing) + ) + for name, executable in ( + ("blastp", self.blastp_path), + ("hmmscan", self.hmmscan_path), + ("hmmalign", self.hmmalign_path), + ): + if not isinstance(executable, str) or not executable.strip(): + raise RuntimeError(f"configured {name} executable must not be empty") + + data_dir = self.data_dir.resolve() + sqlite_db = self.sqlite_db.resolve() + repository_root = Path(__file__).resolve().parents[2] + for label, path in (("JASPAR inference data", data_dir), + ("JASPAR SQLite database", sqlite_db)): + if path == repository_root or repository_root in path.parents: + raise RuntimeError(f"{label} must be outside the Lverage repository: {path}") + if not data_dir.is_dir(): + raise RuntimeError(f"JASPAR inference data directory does not exist: {data_dir}") + if sqlite_db.name != "JASPAR2026.sqlite" or not sqlite_db.is_file(): + raise RuntimeError( + "JASPAR annotation database must be a local JASPAR2026.sqlite file" + ) + + self._verify_inference_data(data_dir) + self._verify_annotation_database(sqlite_db, data_dir) + self.data_dir, self.sqlite_db = data_dir, sqlite_db + self._database_sha256_value = self._database_sha256() + self._validated = True + + @staticmethod + def _verify_inference_data(data_dir): + hmm_prefix = data_dir / "pfam" / "All.hmm" + required = [data_dir / "cisbp", hmm_prefix] + required.extend(Path(str(hmm_prefix) + suffix) + for suffix in (".h3f", ".h3i", ".h3m", ".h3p")) + for taxon in JASPAR_TAXA: + required.extend(data_dir / f"{taxon}.{suffix}.json" + for suffix in ("pfam", "profiles", "uniprot")) + fasta_prefix = data_dir / f"{taxon}.fa" + required.append(fasta_prefix) + required.extend(Path(str(fasta_prefix) + suffix) + for suffix in (".phr", ".pin", ".psq")) + missing = [str(path) for path in required if not path.exists()] + if not (data_dir / "cisbp").is_dir(): + missing.append(str(data_dir / "cisbp/*.json")) + elif not any((data_dir / "cisbp").glob("*.json")): + missing.append(str(data_dir / "cisbp/*.json")) + if missing: + raise RuntimeError( + "JASPAR inference data directory is incomplete; missing: " + + ", ".join(missing[:12]) + ) + + @staticmethod + def _readonly_connection(path): + return sqlite3.connect(f"{path.as_uri()}?mode=ro", uri=True) + + def _verify_annotation_database(self, path, data_dir): + try: + with closing(self._readonly_connection(path)) as connection: + tables = {row[0] for row in connection.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + )} + required = {"MATRIX", "MATRIX_DATA", "MATRIX_SPECIES", "MATRIX_ANNOTATION"} + if not required.issubset(tables): + raise RuntimeError( + "JASPAR 2026 SQLite database lacks required motif tables: " + + ", ".join(sorted(required - tables)) + ) + if connection.execute("SELECT 1 FROM MATRIX LIMIT 1").fetchone() is None: + raise RuntimeError("JASPAR 2026 SQLite database contains no matrices") + except sqlite3.Error as error: + raise RuntimeError(f"cannot read local JASPAR 2026 SQLite database: {error}") from error + + for taxon in JASPAR_TAXA: + profile_path = data_dir / f"{taxon}.profiles.json" + try: + with profile_path.open(encoding="utf-8") as handle: + profiles = json.load(handle) + except (OSError, UnicodeError, ValueError) as error: + raise RuntimeError(f"cannot read local profile annotations {profile_path}: {error}") from error + if not isinstance(profiles, dict) or not profiles: + raise RuntimeError(f"local JASPAR profile annotations are empty or malformed: {profile_path}") + with closing(self._readonly_connection(path)) as connection: + rows = connection.execute( + "SELECT m.BASE_ID, m.VERSION, m.NAME FROM MATRIX m " + "JOIN MATRIX_ANNOTATION a ON a.ID = m.ID " + "WHERE upper(m.COLLECTION) = 'CORE' " + "AND a.TAG = 'tax_group' AND lower(a.VAL) = ?", + (taxon,), + ) + database_profiles = { + f"{base_id}.{version}": name + for base_id, version, name in rows + } + if profiles != database_profiles: + raise RuntimeError( + f"{profile_path} does not match the {taxon} CORE profiles " + "in the configured JASPAR2026 database" + ) + + def _database_sha256(self): + digest = hashlib.sha256() + with self.sqlite_db.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + def _load_inference_data(self): + if self._inference_data is None: + self._inference_data = ( + load_cisbp_models(self.data_dir), load_jaspar_data(self.data_dir), + ) + return self._inference_data + + def check_species_validity(self, species_tax_id: int) -> bool: + """Check whether local motif annotations include a species. Parameters ---------- - n_hits : int, optional - Maximum number of accepted motif hits - escore_threshold : float, optional - Maximum inference E-value - request_timeout_seconds : float, optional - Maximum number of seconds allowed for each API request - """ - - if isinstance(n_hits, bool) or not isinstance(n_hits, int) or n_hits <= 0: - raise ValueError("n_hits must be a positive integer") - if not isinstance(escore_threshold, (int, float)) or escore_threshold <= 0: - raise ValueError("escore_threshold must be positive") - if not isinstance(request_timeout_seconds, (int, float)) or request_timeout_seconds <= 0: - raise ValueError("request_timeout_seconds must be positive") - - self.n_hits = n_hits - self.escore_threshold = escore_threshold - self.request_timeout_seconds = request_timeout_seconds - self.jaspar_species = None + species_tax_id : int + Positive NCBI taxonomy identifier to match exactly. Blank and + null database annotations are treated as missing. - def search(self, request : MotifSearchRequest) -> list[JasparRecord]: + Returns + ------- + bool + Whether the identifier appears in the local species annotations. """ - Search JASPAR for motif records. + if isinstance(species_tax_id, bool) or not isinstance(species_tax_id, int) or species_tax_id <= 0: + raise ValueError("species_tax_id must be a positive integer") + self._validate_configuration() + if self._species_ids is None: + try: + with closing(self._readonly_connection(self.sqlite_db)) as connection: + values = connection.execute("SELECT DISTINCT TAX_ID FROM MATRIX_SPECIES") + self._species_ids = { + int(tax_id) + for (tax_id,) in values + if tax_id is not None and str(tax_id).strip() + } + except (sqlite3.Error, TypeError, ValueError) as error: + raise RuntimeError(f"cannot read local JASPAR species annotations: {error}") from error + return species_tax_id in self._species_ids + + def search(self, request: MotifSearchRequest) -> list[JasparRecord]: + """Infer local motifs from the supporting ortholog protein window. Parameters ---------- request : MotifSearchRequest - Query and ortholog evidence used for motif inference + Query and ortholog sequences, supporting domains, and ortholog + species identifier. Inference uses the ortholog sequence or an + at-most-2,000-residue window retaining the complete domain. Returns ------- - list - Accepted JASPAR motif records ordered by inference E-value + list of JasparRecord + Exact-version predictions passing the inference E-value and + species filters, limited to ``n_hits``. Empty if none survive. """ - if not isinstance(request, MotifSearchRequest): raise TypeError("request must be a MotifSearchRequest") - if isinstance(request.ortholog_species_tax_id, bool): - raise ValueError("ortholog_species_tax_id must be a positive integer") - if not isinstance(request.ortholog_species_tax_id, int) or request.ortholog_species_tax_id <= 0: + if (isinstance(request.ortholog_species_tax_id, bool) + or not isinstance(request.ortholog_species_tax_id, int) + or request.ortholog_species_tax_id <= 0): raise ValueError("ortholog_species_tax_id must be a positive integer") sequence = self._window_ortholog_sequence( - request.ortholog_sequence, - request.ortholog_domain, - ) - response = requests.get( - f"{self.jaspar_rest_url}/infer/{sequence}/", - timeout=self.request_timeout_seconds, + request.ortholog_sequence, request.ortholog_domain, ) - response.raise_for_status() - payload = response.json() - results = self.__get_results(payload, "JASPAR inference") - - try: - results = sorted(results, key=lambda result: float(result["evalue"])) - except (KeyError, TypeError, ValueError) as error: - raise ValueError("JASPAR inference returned malformed results") from error - - records = [] - for result in results: - try: - inference_evalue = float(result["evalue"]) - motif_url = result["url"] - if not isinstance(motif_url, str) or not motif_url: - raise ValueError - except (KeyError, TypeError, ValueError) as error: - raise ValueError("JASPAR inference returned a malformed hit") from error - if inference_evalue > self.escore_threshold: - continue - - motif_response = requests.get( - motif_url, - timeout=self.request_timeout_seconds, + self._validate_configuration() + cisbp_models, jaspar_data = self._load_inference_data() + temporary_root = Path(tempfile.gettempdir()).resolve() + repository_root = Path(__file__).resolve().parents[2] + if temporary_root == repository_root or repository_root in temporary_root.parents: + for candidate in (Path("/tmp"), Path("/var/tmp")): + if candidate.is_dir() and candidate.resolve() != repository_root \ + and repository_root not in candidate.resolve().parents: + temporary_root = candidate.resolve() + break + else: + raise RuntimeError("no temporary directory is available outside the Lverage repository") + + with tempfile.TemporaryDirectory(prefix="lverage-jaspar-", dir=temporary_root) as work_dir: + predictions = infer_sequence( + sequence, + data_dir=self.data_dir, + cisbp_models=cisbp_models, + jaspar_data=jaspar_data, + work_dir=work_dir, + blastp_path=self.blastp_path, + hmmscan_path=self.hmmscan_path, + hmmalign_path=self.hmmalign_path, + query_id="lverage_query", ) - motif_response.raise_for_status() - motif = motif_response.json() - if not isinstance(motif, dict): - raise ValueError("JASPAR returned malformed motif details") - try: - motif_species = motif["species"] - species_ids = {int(species["tax_id"]) for species in motif_species} - except (KeyError, TypeError, ValueError) as error: - raise ValueError("JASPAR motif details contain malformed species") from error + accepted = [] + for prediction in predictions: + if prediction["evalue"] > self.escore_threshold: + continue + record, species_ids = self._resolve_matrix(prediction) if request.ortholog_species_tax_id not in species_ids: continue - - records.append(self.__create_record(motif, inference_evalue)) - if len(records) == self.n_hits: + accepted.append(record) + if len(accepted) >= self.n_hits: break - return records + return accepted - def __create_record(self, motif, inference_evalue): + def _resolve_matrix(self, inference): + matrix_id = inference["matrix_id"] + match = MATRIX_ID_PATTERN.fullmatch(matrix_id) + if match is None: + raise ValueError(f"JASPAR matrix ID must be exact and versioned: {matrix_id!r}") + try: + with closing(self._readonly_connection(self.sqlite_db)) as connection: + matrix = connection.execute( + "SELECT ID, NAME, COLLECTION FROM MATRIX " + "WHERE BASE_ID = ? AND VERSION = ?", + (match.group(1), int(match.group(2))), + ).fetchone() + if matrix is None or str(matrix[2]).upper() != "CORE": + raise ValueError( + f"exact matrix {matrix_id} is absent from local JASPAR 2026 CORE data" + ) + internal_id, motif_name, _ = matrix + species = connection.execute( + "SELECT TAX_ID FROM MATRIX_SPECIES WHERE ID = ?", (internal_id,), + ).fetchall() + annotations = connection.execute( + "SELECT VAL FROM MATRIX_ANNOTATION WHERE ID = ? AND TAG = 'class'", + (internal_id,), + ).fetchall() + pfm = {} + for base in "ACGT": + values = connection.execute( + "SELECT VAL FROM MATRIX_DATA WHERE ID = ? AND ROW = ? ORDER BY COL", + (internal_id, base), + ).fetchall() + if not values: + raise ValueError(f"local JASPAR matrix {matrix_id} lacks its {base} row") + pfm[base] = [float(value[0]) for value in values] + except sqlite3.Error as error: + raise RuntimeError( + f"cannot resolve exact JASPAR matrix {matrix_id} from local data: {error}" + ) from error + + lengths = {len(values) for values in pfm.values()} + if len(lengths) != 1 or not lengths.pop(): + raise ValueError(f"local JASPAR matrix {matrix_id} has inconsistent PFM rows") + if not annotations or not annotations[0][0]: + raise ValueError(f"local JASPAR matrix {matrix_id} has no TF class annotation") try: - matrix_id = motif["matrix_id"] - motif_name = motif["name"] - pfm = motif["pfm"] - motif_class = motif["class"] - if isinstance(motif_class, list): - motif_class = motif_class[0] - if not all(isinstance(value, str) and value for value in [matrix_id, motif_name, motif_class]): - raise ValueError - if not isinstance(pfm, dict): - raise ValueError - except (KeyError, IndexError, TypeError, ValueError) as error: - raise ValueError("JASPAR returned malformed motif details") from error + species_ids = { + int(tax_id) + for (tax_id,) in species + if tax_id is not None and str(tax_id).strip() + } + except (TypeError, ValueError) as error: + raise ValueError(f"local JASPAR matrix {matrix_id} has invalid species annotations") from error return JasparRecord( matrix_id=matrix_id, motif_name=motif_name, pfm=pfm, motif_url=f"https://jaspar.elixir.no/matrix/{matrix_id}/", - motif_class=motif_class, - inference_evalue=inference_evalue, - ) - - @staticmethod - def __get_results(payload, service_name): - if not isinstance(payload, dict) or not isinstance(payload.get("results"), list): - raise ValueError(f"{service_name} returned a malformed response") - return payload["results"] + motif_class=annotations[0][0], + inference_evalue=inference["evalue"], + inference_tf_name=inference["tf_name"], + inference_query_region=inference["query_region"], + inference_tf_region=inference["tf_region"], + inference_dbd_identity=inference["dbd_identity"], + ), species_ids @staticmethod def _window_ortholog_sequence(sequence, ortholog_domain): - """ - Limit an ortholog sequence while retaining its complete domain. - - Parameters - ---------- - sequence : str - Complete ortholog protein sequence - ortholog_domain : DomainRecord - Domain with zero-based, half-open bounds - - Returns - ------- - str - Sequence window of at most 2,000 residues - - Raises - ------ - ValueError - The sequence or domain bounds are invalid - """ - + """Return the same at-most-2,000-aa ortholog window used previously.""" try: start = ortholog_domain.start end = ortholog_domain.end except AttributeError as error: raise ValueError("ortholog domains must provide start and end bounds") from error - if not isinstance(sequence, str) or not sequence: raise ValueError("ortholog sequences must be nonempty strings") if isinstance(start, bool) or isinstance(end, bool): @@ -312,55 +495,12 @@ def _window_ortholog_sequence(sequence, ortholog_domain): raise ValueError("ortholog domain bounds must be integers") if start < 0 or end > len(sequence) or start >= end: raise ValueError("ortholog domain bounds must define a nonempty slice within the sequence") - window_length = 2000 if end - start > window_length: raise ValueError("ortholog domains must fit inside the JASPAR sequence window") if len(sequence) <= window_length: return sequence - domain_center = (start + end) // 2 window_start = domain_center - window_length // 2 window_start = max(0, min(window_start, len(sequence) - window_length)) return sequence[window_start:window_start + window_length] - - def check_species_validity(self, species_tax_id : int) -> bool: - """ - Check whether a species appears in JASPAR. - - Parameters - ---------- - species_tax_id : int - Taxonomic identifier of the species - - Returns - ------- - bool - If the species appears in the database - """ - - if self.jaspar_species is None: - species_ids = set() - species_url = self.jaspar_rest_species_url - species_params = self.species_params.copy() - while species_url is not None: - response = requests.get( - species_url, - params=species_params, - timeout=self.request_timeout_seconds, - ) - response.raise_for_status() - payload = response.json() - results = self.__get_results(payload, "JASPAR species") - try: - species_ids.update(int(species["tax_id"]) for species in results) - except (KeyError, TypeError, ValueError) as error: - raise ValueError("JASPAR species returned malformed results") from error - next_url = payload.get("next") - if next_url is not None and (not isinstance(next_url, str) or not next_url): - raise ValueError("JASPAR species returned malformed pagination") - species_url = next_url - species_params = None - self.jaspar_species = species_ids - - return species_tax_id in self.jaspar_species diff --git a/src/lverage/jaspar_inference.py b/src/lverage/jaspar_inference.py new file mode 100644 index 0000000..62c4b11 --- /dev/null +++ b/src/lverage/jaspar_inference.py @@ -0,0 +1,319 @@ +"""Local JASPAR profile inference adapted for Lverage's motif adapter. + +This module adapts the inference functions from ``infer_profile.py`` in +https://github.com/amathelier/JASPAR-inference-tool, source revision +ac4024148f1f140c7370d7b13a70556b4d13f7a3 (version 2026.1.0). The original +copyright notice and MIT license terms are preserved in +``jaspar_inference_license.txt``. Upstream attribution names Anthony Mathelier, +Oriol Fornes, Xi Zhang, and the JASPAR Consortium. + +The inference calculations and decisions below retain the upstream procedure: +HMMER Pfam domain detection/alignment; BLASTP against each configured taxon DB; +Rost-curve identity filtering; lower-case insertion removal; the best positional +identity/BLOSUM62 alignment; Cis-BP HSim scoring; and one matched DBD's JASPAR +profiles per accepted BLAST target. External programs are called directly with +argument vectors, not through a shell. +""" + +from pathlib import Path +import json +import math +import re +import subprocess + +from Bio import SearchIO, SeqIO +from Bio.Align import substitution_matrices +from Bio.Seq import Seq +from Bio.SeqRecord import SeqRecord +import numpy as np + +from .jaspar_models import load_cisbp_models, score_alignment_result + + +BLOSUM62 = substitution_matrices.load("BLOSUM62") +JASPAR_TAXA = ("fungi", "insects", "nematodes", "plants", "urochordates", "vertebrates") +BLAST_OUTFMT = "sseqid pident length qstart qend sstart send evalue bitscore ppos qlen slen" + + +def load_jaspar_data(data_dir, taxons=JASPAR_TAXA): + """Load the fork's per-taxon Pfam/profile/UniProt mappings.""" + data_dir = Path(data_dir) + jaspar = {} + pfams = {} + profiles = {} + uniprots = {} + + for taxon in taxons: + for suffix, destination in ( + ("pfam", pfams), ("profiles", profiles), ("uniprot", uniprots), + ): + path = data_dir / f"{taxon}.{suffix}.json" + with path.open(encoding="utf-8") as handle: + for key, values in json.load(handle).items(): + destination.setdefault(key, values) + + for uniprot, domains in pfams.items(): + entry = jaspar.setdefault(uniprot, {}) + entry.setdefault("pfam", domains) + entry.setdefault("profiles", []) + for profile in uniprots[uniprot][0]: + entry["profiles"].append([profile, profiles[profile]]) + return jaspar + + +def _checked_run(arguments, *, stage, stdout=None): + try: + return subprocess.run( + arguments, + check=True, + stdout=subprocess.PIPE if stdout is None else stdout, + stderr=subprocess.PIPE, + text=True, + ) + except FileNotFoundError as error: + raise RuntimeError(f"{stage} executable was not found: {arguments[0]}") from error + except subprocess.CalledProcessError as error: + diagnostic = (error.stderr or error.stdout or "").strip() + if not diagnostic: + diagnostic = "no diagnostic output" + raise RuntimeError( + f"{stage} failed with status {error.returncode}: {diagnostic[-4000:]}" + ) from error + + +def _write_fasta(record, path): + with Path(path).open("w", encoding="utf-8") as handle: + SeqIO.write(record, handle, "fasta") + + +def read_hmmscan_domains(path): + """Apply upstream model (1e-5) and conditional-domain (0.01) cutoffs.""" + domains = [] + for result in SearchIO.parse(str(path), "hmmscan3-domtab"): + for model in result.iterhits(): + if model.evalue > 1e-5: + continue + for domain in model.hsps: + if domain.evalue_cond > 0.01: + continue + domains.append((model.id, domain.query_start, domain.query_end, domain.evalue_cond)) + return domains + + +def non_overlapping_domains(domains): + """Retain the lowest conditional-E-value domains with upstream overlap rule.""" + retained = [] + for domain in sorted(domains, key=lambda item: item[-1]): + if any(domain[1] < previous[2] and domain[2] > previous[1] + for previous in retained): + continue + retained.append(domain) + return retained + + +def scan_and_align_domains(record, data_dir, work_dir, *, hmmscan_path, hmmalign_path): + """Run upstream Pfam scan, thresholding, overlap selection, and HMM alignment.""" + data_dir = Path(data_dir) + work_dir = Path(work_dir) + sequence_path = work_dir / "query.fasta" + _write_fasta(record, sequence_path) + domtbl_path = work_dir / "query.domtbl" + _checked_run( + [hmmscan_path, "--domtblout", str(domtbl_path), + str(data_dir / "pfam" / "All.hmm"), str(sequence_path)], + stage="hmmscan", + stdout=subprocess.DEVNULL, + ) + domains = non_overlapping_domains(read_hmmscan_domains(domtbl_path)) + alignments = [] + domain_path = work_dir / "domain.fasta" + for pfam_id, start, end, evalue in sorted(domains, key=lambda item: item[1]): + domain_record = SeqRecord( + record.seq[start:end], id=record.id, name=record.name, + description=record.description, + ) + _write_fasta(domain_record, domain_path) + process = _checked_run( + [hmmalign_path, "--outformat", "PSIBLAST", + str(data_dir / "pfam" / f"{pfam_id}.hmm"), str(domain_path)], + stage="hmmalign", + ) + aligned = read_psiblast_alignment(process.stdout) + alignments.append((pfam_id, aligned, start + 1, end, evalue)) + return alignments + + +def read_psiblast_alignment(text): + """Concatenate non-whitespace trailing alignment blocks as upstream does.""" + aligned = "" + for chunk in text.split("\n"): + match = re.search(r"\s+(\S+)$", chunk) + if match: + aligned += match.group(1) + return aligned + + +def _rost_identity_cutoff(alignment_length, n=5): + """Upstream Rost pairwise sequence-identity curve.""" + return n + 480 * pow( + alignment_length, + -0.32 * (1 + math.exp(float(-alignment_length) / 1000)), + ) + + +def _is_over_rost_curve(percent_identity, alignment_length, n=5): + return percent_identity >= _rost_identity_cutoff(alignment_length, n) + + +def blast_search(record, data_dir, query_fasta, *, blastp_path, taxons=JASPAR_TAXA, n=5): + """Run upstream BLASTP outfmt and Rost identity/coverage filtering.""" + data_dir = Path(data_dir) + blast_results = set() + for taxon in taxons: + process = _checked_run( + [blastp_path, "-db", str(data_dir / f"{taxon}.fa"), + "-outfmt", f"6 {BLAST_OUTFMT}", "-query", str(query_fasta)], + stage=f"blastp ({taxon})", + ) + for line in process.stdout.splitlines(): + fields = line.split("\t") + if len(fields) != 12: + continue + target_id = fields[0] + percent_identity = float(fields[1]) + alignment_length = int(fields[2]) + query_start_end = f"{fields[3]}-{fields[4]}" + target_start_end = f"{fields[5]}-{fields[6]}" + evalue = float(fields[7]) + bit_score = float(fields[8]) + percent_similarity = float(fields[9]) + query_coverage = (int(fields[4]) - int(fields[3]) + 1) * 100 / float(fields[10]) + target_coverage = (int(fields[6]) - int(fields[5]) + 1) * 100 / float(fields[11]) + joint_coverage = math.sqrt(query_coverage * target_coverage) + blast_results.add(( + record.id, target_id, query_start_end, target_start_end, + evalue, bit_score, percent_identity, alignment_length, + percent_similarity, joint_coverage, + )) + + ordered = sorted(blast_results, key=lambda item: item[-1], reverse=True) + return [result for result in ordered + if _is_over_rost_curve(result[6], result[7], n)] + + +def _target_pfam_alignments(blast_results, jaspar): + pfam_alignments = {} + for result in blast_results: + target_id = result[1] + pfam_alignments.setdefault(target_id, {}) + for alignment in jaspar[target_id]["pfam"]: + pfam_alignments[target_id].setdefault(alignment[0], []).append(alignment[1]) + return pfam_alignments + + +def _remove_insertions(alignment): + return alignment.translate(str.maketrans("", "", "abcdefghijklmnopqrstuvwxyz")) + + +def _score_residue(first, second, similarity): + if similarity == "identity": + return 1 if first == second else 0 + if similarity == "blosum62": + if first == second == "-": + return 1 + if first == "-" or second == "-": + return -4 + try: + return int(BLOSUM62[first][second]) + except (KeyError, IndexError): + return 0 + return 0 + + +def positional_alignment_score(first_alignments, second_alignments, similarity="identity"): + """Upstream __get_X: maximize summed positional score over contiguous DBD alignments.""" + if not first_alignments or not second_alignments: + raise ValueError("at least one aligned domain is required for similarity regression") + if len(first_alignments) < len(second_alignments): + first_alignments, second_alignments = second_alignments, first_alignments + + scores = [] + for offset in range(len(first_alignments) - len(second_alignments) + 1): + row = [0] * len(first_alignments[offset]) + for second_offset, second_alignment in enumerate(second_alignments): + first_alignment = first_alignments[offset + second_offset] + if len(first_alignment) != len(second_alignment): + raise ValueError("upstream HMM alignment rows have unequal lengths") + for position, (first, second) in enumerate(zip(first_alignment, second_alignment)): + row[position] += _score_residue(first, second, similarity) + scores.append(np.asarray(row) / len(second_alignments)) + scores.sort(key=lambda values: sum(values), reverse=True) + return scores[0] + + +def infer_sequence(sequence, *, data_dir, cisbp_models, jaspar_data, work_dir, + blastp_path, hmmscan_path, hmmalign_path, query_id="lverage_query", + n=5, taxons=JASPAR_TAXA): + """Return upstream inference rows for one protein sequence, without CLI/TSV wrappers.""" + record = SeqRecord(Seq(sequence), id=query_id, name=query_id, description=query_id) + query_alignments = scan_and_align_domains( + record, data_dir, work_dir, + hmmscan_path=hmmscan_path, hmmalign_path=hmmalign_path, + ) + if not query_alignments: + return [] + + query_by_domain = {} + for pfam_id, aligned, _start, _end, _evalue in query_alignments: + query_by_domain.setdefault(pfam_id, []).append(aligned) + + blast_results = blast_search( + record, data_dir, Path(work_dir) / "query.fasta", + blastp_path=blastp_path, taxons=taxons, n=n, + ) + target_by_protein = _target_pfam_alignments(blast_results, jaspar_data) + predictions = [] + for result in blast_results: + target_id = result[1] + # Upstream directly indexes this result: a BLAST target absent from + # reference data is an execution/data mismatch, not a biological no-hit. + target_domains = target_by_protein[target_id] + for pfam_id, query_domain_alignments in query_by_domain.items(): + if pfam_id not in target_domains: + continue + query_rows = [_remove_insertions(row) for row in query_domain_alignments] + target_rows = [_remove_insertions(row) for row in target_domains[pfam_id]] + percent_identity_by_position = positional_alignment_score( + list(query_rows), list(target_rows), "identity", + ) + average_blosum62_by_position = positional_alignment_score( + list(query_rows), list(target_rows), "blosum62", + ) + percent_identity = sum(percent_identity_by_position) / len(percent_identity_by_position) + alignment_features = { + "ByPos.PctID": percent_identity_by_position, + "ByPos.AvgB62": average_blosum62_by_position, + "PctID_L": percent_identity, + } + model = cisbp_models.get(pfam_id, cisbp_models[None]) + _score, classification = score_alignment_result(alignment_features, model) + if classification != "HSim": + continue + + for matrix_id, tf_name in jaspar_data[target_id]["profiles"]: + predictions.append({ + "query": result[0], + "tf_name": tf_name, + "matrix_id": matrix_id, + "evalue": result[4], + "query_region": result[2], + "tf_region": result[3], + "dbd_identity": round(percent_identity, 3), + }) + # Upstream stops after the first matching/accepted query DBD per target. + break + + predictions.sort(key=lambda row: ( + row["evalue"], row["tf_name"], float(row["matrix_id"][2:]), + )) + return predictions diff --git a/src/lverage/jaspar_inference_license.txt b/src/lverage/jaspar_inference_license.txt new file mode 100644 index 0000000..c79ca8b --- /dev/null +++ b/src/lverage/jaspar_inference_license.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Oriol Fornes and Xi Zhang + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/lverage/jaspar_models.py b/src/lverage/jaspar_models.py new file mode 100644 index 0000000..8fc67a4 --- /dev/null +++ b/src/lverage/jaspar_models.py @@ -0,0 +1,107 @@ +"""Cis-BP similarity-regression support adapted for Lverage. + +Adapted from ``__init__.py`` in the JASPAR profile inference tool at +https://github.com/amathelier/JASPAR-inference-tool, source revision +ac4024148f1f140c7370d7b13a70556b4d13f7a3 (2026.1.0). + +Copyright (c) 2019 Oriol Fornes and Xi Zhang. The original MIT terms are +preserved in ``jaspar_inference_license.txt``. Upstream attribution also names +Anthony Mathelier and the JASPAR Consortium. +""" + +import json +from pathlib import Path + +import numpy as np + + +PFAM_TO_CISBP = { + "AP2": "AP2", + "KilA-N": "APSES", + "ARID": "ARID/BRIGHT", + "AT_hook": "AT hook", + "HLH": "bHLH", + "bZIP_1": "bZIP", + "zf-C2H2": "C2H2 ZF", + "DM": "DM", + "zf-Dof": "Dof", + "E2F_TDP": "E2F", + "Ets": "Ets", + "Forkhead": "Forkhead", + "GATA": "GATA", + "GCM": "GCM", + "Homeobox": "Homeodomain", + "Pou": "Homeodomain,POU", + "HSF_DNA-bind": "HSF", + "MADF_DNA_bdg": "MADF", + "Myb_DNA-binding": "Myb/SANT", + "NAM": "NAC/NAM", + "zf-C4": "Nuclear receptor", + "HTH_psq": "Pipsqueak", + "RFX_DNA_binding": "RFX", + "SAND": "SAND", + "SBP": "SBP", + "HMG_box": "Sox", + "T-box": "T-box", + "TCP": "TCP", + "TCR": "TCR/CxC", + "WRKY": "WRKY", + "Zn_clus": "Zinc cluster", + None: "NO_THRESHOLD", +} +CISBP_TO_PFAM = {value: key for key, value in PFAM_TO_CISBP.items()} + + +def read_sr_model(path): + """Load a Cis-BP SR model with the same array/NaN conversions upstream uses.""" + with Path(path).open(encoding="utf-8") as handle: + model = json.load(handle) + if "SR.Weights" in model: + model["SR.Weights"] = np.asarray(model["SR.Weights"]) + model["SR.FeatureScales.mean"] = np.asarray(model["SR.FeatureScales.mean"]) + standard_deviation = np.asarray(model["SR.FeatureScales.sd"]) + standard_deviation[standard_deviation == 0] = np.nan + model["SR.FeatureScales.sd"] = standard_deviation + if np.isnan(model["Threshold.Dis"]): + model["Threshold.Dis"] = None + return model + + +def load_cisbp_models(data_dir): + """Load family-keyed Cis-BP models exactly as the upstream loader does.""" + models = {} + model_dir = Path(data_dir) / "cisbp" + for filename in model_dir.iterdir(): + if filename.suffix != ".json": + continue + model = read_sr_model(filename) + models.setdefault(CISBP_TO_PFAM[model["Family_Name"]], model) + return models + + +def score_alignment_result(result, model, apply_identical_rule=True): + """Apply the upstream HSim identity/SR thresholds without approximation.""" + if apply_identical_rule and result["PctID_L"] == 1: + return result["PctID_L"], "HSim" + + if model["Model.Class"] == "SequenceIdentity": + score = result[model["Model.Name"]] + if score >= model["Threshold.HSim"]: + return score, "HSim" + else: + score = result[model["Baseline"]["Name"]] + if score >= model["Baseline"]["Threshold.HSim"]: + return score, "HSim" + + feature_name = "ByPos." + model["SR.Features"].replace("_", ".") + values = np.asarray(result[feature_name]) + if len(values) == len(model["SR.FeatureScales.mean"]): + scaled = (values - model["SR.FeatureScales.mean"]) / model["SR.FeatureScales.sd"] + scaled[np.isnan(scaled)] = 0 + score = model["SR.Intercept"] + np.dot(model["SR.Weights"], scaled) + if model["SR.LogisticTransform"]: + score = 1 / (1 + np.exp(-score)) + if score >= model["Threshold.HSim"]: + return score, "HSim" + + return np.nan, np.nan diff --git a/src/lverage/pipeline.py b/src/lverage/pipeline.py index a341d05..ee18d0d 100644 --- a/src/lverage/pipeline.py +++ b/src/lverage/pipeline.py @@ -149,6 +149,10 @@ def _get_sorted_orfs(self, tf_sequences, evidence_collector=None): def _select_orf_from_candidates(self, orf_list, evidence_collector=None): for orf_index, orf in enumerate(orf_list, start=1): + if evidence_collector is not None: + evidence_collector.emit( + "stage_attempt", stage_name="query_pfam", orf_sequence=orf, + ) domains = self.domain_scanner.get_domains(orf) detected_domains = list(domains) if self.valid_pfam_list: @@ -313,6 +317,10 @@ def run(self, tf_sequence : str | list[str], *, evidence_collector=None, supports_evidence = getattr( self.ortholog_searcher, "supports_evidence_collector", False ) + if evidence_collector is not None: + evidence_collector.emit( + "stage_attempt", stage_name="blast", orf_sequence=query_orf, + ) if evidence_collector is not None and supports_evidence: orthologs = self.ortholog_searcher.get_orthologs( query_orf, evidence_collector=evidence_collector @@ -346,8 +354,18 @@ def run(self, tf_sequence : str | list[str], *, evidence_collector=None, for ortholog in orthologs: if evidence_collector is not None: evidence_collector.set_stage("ortholog_pfam") - ortholog_domains = self.domain_scanner.get_domains(ortholog.sequence) + evidence_collector.emit( + "stage_attempt", stage_name="ortholog_pfam", + orf_sequence=query_orf, ortholog=ortholog, + ) + ortholog_domains = list(self.domain_scanner.get_domains(ortholog.sequence)) if evidence_collector is not None: + evidence_collector.emit( + "stage_result", stage_name="ortholog_pfam", + orf_sequence=query_orf, ortholog=ortholog, + status="domains-found" if ortholog_domains else "no-domains", + domain_count=len(ortholog_domains), error="", + ) for domain_index, domain in enumerate(ortholog_domains, start=1): evidence_collector.emit( "ortholog_domain", orf_sequence=query_orf, ortholog=ortholog, @@ -360,6 +378,13 @@ def run(self, tf_sequence : str | list[str], *, evidence_collector=None, for ortholog_domain in ortholog_domains: if self._base_accession(query_domain.accession) != self._base_accession(ortholog_domain.accession): continue + if evidence_collector is not None: + evidence_collector.emit( + "stage_attempt", stage_name="alignment", + orf_sequence=query_orf, ortholog=ortholog, + query_domain=query_domain, + ortholog_domain=ortholog_domain, + ) domain_identity = self._calculate_domain_identity( query_orf, query_domain, @@ -390,17 +415,36 @@ def run(self, tf_sequence : str | list[str], *, evidence_collector=None, ) if stop_after == "alignment": continue - if evidence_collector is not None: - evidence_collector.set_stage("motif") for motif_database in self.motif_database_list: + if evidence_collector is not None: + evidence_collector.set_stage("motif") + evidence_collector.emit( + "stage_attempt", stage_name="motif", + orf_sequence=query_orf, ortholog=ortholog, + query_domain=query_domain, + ortholog_domain=ortholog_domain, + motif_database_name=motif_database.name, + ) if not motif_database.check_species_validity(ortholog.species_tax_id): LOGGER.info( "Skipping %s for unavailable species %s", motif_database.name, ortholog.species_tax_id, ) + if evidence_collector is not None: + evidence_collector.emit( + "stage_result", stage_name="motif", + orf_sequence=query_orf, ortholog=ortholog, + query_domain=query_domain, + ortholog_domain=ortholog_domain, + motif_database_name=motif_database.name, + status="species-unavailable", motif_count=0, + error="", + ) continue + motif_count = 0 for motif_record in motif_database.search(request): + motif_count += 1 records.append(LverageRecord( query_domain=query_domain, ortholog=ortholog, @@ -418,6 +462,16 @@ def run(self, tf_sequence : str | list[str], *, evidence_collector=None, motif_database_name=motif_database.name, motif_record=motif_record, status="retained", error="", ) + if evidence_collector is not None: + evidence_collector.emit( + "stage_result", stage_name="motif", + orf_sequence=query_orf, ortholog=ortholog, + query_domain=query_domain, + ortholog_domain=ortholog_domain, + motif_database_name=motif_database.name, + status="motifs-found" if motif_count else "no-hits", + motif_count=motif_count, error="", + ) if stop_after == "alignment": self.lverage_code = LverageCode.STOPPED_AFTER_ALIGNMENT diff --git a/tests/v2/test_cli.py b/tests/v2/test_cli.py index 311f253..279f5b1 100644 --- a/tests/v2/test_cli.py +++ b/tests/v2/test_cli.py @@ -66,6 +66,10 @@ def get_orthologs(self, sequence, *, evidence_collector=None): evidence_collector.emit( "ortholog_retrieval", status="skipped", hit_id="native|hit4|", accession="NP_4", attempted_identifier="", error="simulated retrieval failure", + hit_group=4, description="Skipped protein [Homo sapiens]", + species_name="Homo sapiens", species_tax_id=9606, evalue=1e-20, + identity_ratio=0.8, identity_percent=80.0, + query_coverage_ratio=0.9, query_coverage_percent=90.0, ) evidence_collector.emit( "blast_search_summary", status="partial-retrieval-failure", @@ -155,219 +159,281 @@ def run_cli(self, pipeline, *arguments): return code, stderr.getvalue() def read_rows(self, output_dir): - with (output_dir / "results.tsv").open(encoding="utf-8", newline="") as handle: - return list(csv.DictReader(handle, delimiter="\t")) + with (output_dir / "results.csv").open(encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle, delimiter=",")) def test_bundle_contains_all_three_files_and_unique_orfs_with_gene_associations(self): self.write_fasta("alpha.fa") self.write_fasta("beta.fa", "TGCA") pipeline = self.make_pipeline( orfs={"ACGT": ["AAAA", "CCCC"], "TGCA": ["AAAA"]}, - domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, - orthologs=[], - ) - destination = self.root / "bundle" - + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}) + pipeline.ortholog_searcher.excluded_terms = ["configured description term"] code, _ = self.run_cli(pipeline, *self.arguments()) - + destination = self.root / "bundle" self.assertEqual(code, 0) - self.assertEqual({path.name for path in destination.iterdir()}, { - "results.tsv", "orfs.fasta", "metadata.json", - }) - fasta = (destination / "orfs.fasta").read_text(encoding="utf-8") - self.assertEqual(fasta.count(">"), 2) + self.assertEqual({p.name for p in destination.iterdir()}, {"results.csv", "orfs.fasta", "metadata.json"}) + fasta = (destination / "orfs.fasta").read_text() shared_id = "orf_" + hashlib.sha256(b"AAAA").hexdigest() + self.assertEqual(fasta.count(">"), 2) self.assertIn(f">{shared_id}\nAAAA\n", fasta) - rows = self.read_rows(destination) - associations = { - row["Input Gene"] for row in rows - if row["Row Type"] == "orf" and row["Query ORF ID"] == shared_id - } - self.assertEqual(associations, {"alpha", "beta"}) - self.assertEqual(len(RESULT_HEADERS), len(next(csv.reader( - io.StringIO((destination / "results.tsv").read_text(encoding="utf-8")), delimiter="\t" - )))) - metadata = json.loads((destination / "metadata.json").read_text(encoding="utf-8")) + self.assertEqual({r["Gene"] for r in self.read_rows(destination) if r["ORF ID"] == shared_id}, {"alpha", "beta"}) + metadata = json.loads((destination / "metadata.json").read_text()) self.assertEqual(metadata["summary"]["genes_completed"], 2) - self.assertEqual(metadata["completion"], "completed") + self.assertEqual(metadata["schema_version"], 2) + self.assertEqual(metadata["results_format"], "grouped-csv") + self.assertEqual( + metadata["settings"]["blast_description_excluded_terms"], + ["configured description term"], + ) + with (destination / "results.csv").open() as f: + lines = list(csv.reader(f, delimiter=",")) + self.assertEqual(lines[0], RESULT_HEADERS) + self.assertTrue(all(len(line) == len(RESULT_HEADERS) for line in lines)) + + def test_candidate_order_selection_and_blank_unexamined_stages(self): + self.write_fasta("candidates.fa") + selected = "AAAAAA" + pipeline = self.make_pipeline( + orfs={"ACGT": ["CCC", selected, "MMMMMMMM", selected]}, + domains={selected: [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}) + code, _ = self.run_cli(pipeline, *self.arguments()) + rows = self.read_rows(self.root / "bundle") + self.assertEqual(code, 0) + self.assertEqual([r["ORF Length (aa)"] for r in rows], ["8", "6", "3"]) + self.assertEqual(pipeline.domain_scanner.scanned, ["MMMMMMMM", selected]) + self.assertEqual(rows[0]["Query Pfam Status"], "no accepted domain") + self.assertEqual(rows[1]["ORF Selection Status"], "selected") + self.assertEqual(rows[2]["ORF Examination Status"], "") + self.assertEqual(rows[2]["ORF Selection Status"], "") + self.assertTrue(all(not rows[2][h] for h in RESULT_HEADERS[10:])) + self.assertTrue(all(r["Gene"] == "candidates" for r in rows)) def test_no_orfs_and_no_blast_hits_are_biological_no_hits(self): - self.write_fasta("empty_orfs.fa") - no_orf_pipeline = self.make_pipeline(orfs={"ACGT": []}) - no_orf_dir = self.root / "no-orfs" - no_orf_code, _ = self.run_cli(no_orf_pipeline, *self.arguments("no-orfs")) - no_orf_rows = self.read_rows(no_orf_dir) - self.assertEqual(no_orf_code, 0) - self.assertTrue(any(row["Row Type"] == "input" for row in no_orf_rows)) - self.assertTrue(any(row["Status"] == "biological-no-hit" for row in no_orf_rows)) - self.assertTrue(any(row["Stage"] == "blast" and row["Status"] == "not-reached" for row in no_orf_rows)) - - self.write_fasta("no_hits.fa") - no_hit_pipeline = self.make_pipeline( - orfs={"ACGT": ["AAAA"]}, - domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, - orthologs=[], - ) - no_hit_dir = self.root / "no-hits" - no_hit_code, _ = self.run_cli(no_hit_pipeline, *self.arguments("no-hits")) - no_hit_rows = self.read_rows(no_hit_dir) - self.assertEqual(no_hit_code, 0) - self.assertTrue(any(row["Stage"] == "blast" and row["Status"] == "biological-no-hit" for row in no_hit_rows)) + self.write_fasta("empty.fa") + code, _ = self.run_cli(self.make_pipeline(), *self.arguments("no-orfs")) + rows = self.read_rows(self.root / "no-orfs") + self.assertEqual(code, 0) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["Gene Status"], "biological-no-hit") + self.assertEqual(rows[0]["ORF ID"], "") + pipeline = self.make_pipeline(orfs={"ACGT": ["AAAA"]}, + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}) + code, _ = self.run_cli(pipeline, *self.arguments("no-hits")) + rows = self.read_rows(self.root / "no-hits") + self.assertEqual(code, 0) + self.assertEqual(rows[0]["BLAST Status"], "biological-no-hit") + self.assertEqual(rows[0]["BLAST Accession"], "") def test_below_threshold_comparison_is_exported_with_distinct_status(self): self.write_fasta("below.fa") - pipeline = self.make_pipeline( - orfs={"ACGT": ["AAAA"]}, - domains={ - "AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)], - "TTTT": [DomainRecord("Homeobox", "PF00046.2", 0, 4)], - }, - orthologs=[make_ortholog("NP_low", "TTTT")], - ) - destination = self.root / "below" - - code, _ = self.run_cli(pipeline, *self.arguments("below")) - - rows = self.read_rows(destination) - comparisons = [row for row in rows if row["Row Type"] == "domain_comparison"] + pipeline = self.make_pipeline(orfs={"ACGT": ["AAAA"]}, + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)], + "TTTT": [DomainRecord("Homeobox", "PF00046.2", 0, 4)]}, + orthologs=[make_ortholog("NP_low", "TTTT")]) + code, _ = self.run_cli(pipeline, *self.arguments()) + rows = self.read_rows(self.root / "bundle") self.assertEqual(code, 0) - self.assertEqual(len(comparisons), 1) - self.assertEqual(comparisons[0]["Status"], "below-threshold") - self.assertEqual(comparisons[0]["Threshold Passed"], "false") - self.assertEqual(comparisons[0]["Domain Pair Identity Ratio"], "0.0") - self.assertTrue(any(row["Stage"] == "motif" and row["Status"] == "not-reached" for row in rows)) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["Global Alignment Status"], "below-threshold") + self.assertEqual(rows[0]["Global Alignment Threshold Passed"], "false") + self.assertEqual(rows[0]["Global Alignment Identity Ratio"], "0.0") + self.assertEqual(rows[0]["Motif Status"], "") def test_hits_one_to_three_are_exported_once_when_hit_four_is_skipped(self): - self.write_fasta("blast_partial.fa") - retained = [make_ortholog(f"NP_{index}", "AAAA") for index in range(1, 4)] - pipeline = self.make_pipeline( - orfs={"ACGT": ["AAAA"]}, - domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, - ) - pipeline.ortholog_searcher = IncrementalPartialSearcher(retained) - destination = self.root / "blast-partial" - - code, _ = self.run_cli(pipeline, *self.arguments("blast-partial")) - - rows = self.read_rows(destination) - ortholog_rows = [row for row in rows if row["Row Type"] == "ortholog"] - skipped_rows = [row for row in rows if row["Row Type"] == "ortholog_retrieval" and row["Status"] == "skipped"] - summaries = [row for row in rows if row["Row Type"] == "stage_status" and row["Stage"] == "blast"] + self.write_fasta("partial.fa") + pipeline = self.make_pipeline(orfs={"ACGT": ["AAAA"]}, + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}) + pipeline.ortholog_searcher = IncrementalPartialSearcher([make_ortholog(f"NP_{i}") for i in range(1, 4)]) + code, _ = self.run_cli(pipeline, *self.arguments()) + rows = self.read_rows(self.root / "bundle") self.assertEqual(code, 1) - self.assertEqual([row["Ortholog Accession"] for row in ortholog_rows], ["NP_1", "NP_2", "NP_3"]) - self.assertEqual(len(ortholog_rows), 3) - self.assertEqual(len(skipped_rows), 1) - self.assertEqual(skipped_rows[0]["Ortholog Hit ID"], "native|hit4|") - self.assertTrue(any("partial-retrieval-failure" == row["Status"] for row in summaries)) - metadata = json.loads((destination / "metadata.json").read_text(encoding="utf-8")) - self.assertEqual(metadata["summary"]["genes_failed"], 1) + self.assertEqual([r["BLAST Accession"] for r in rows], ["NP_1", "NP_2", "NP_3", "NP_4"]) + self.assertEqual(rows[-1]["BLAST Status"], "skipped") + self.assertEqual(rows[-1]["BLAST Hit ID"], "native|hit4|") + self.assertEqual(rows[-1]["BLAST Hit Group"], "4") + self.assertEqual(rows[-1]["BLAST Description"], "Skipped protein [Homo sapiens]") + self.assertEqual(rows[-1]["BLAST Identity Percent"], "80.0") + self.assertEqual(rows[-1]["Query Coverage Percent"], "90.0") + self.assertIn("partial-retrieval-failure", rows[0]["BLAST Outcome"]) + self.assertEqual(rows[0]["Gene Status"], "failed") def test_multiple_motifs_each_have_one_correctly_supported_row(self): self.write_fasta("motifs.fa") - database = StubMotifDatabase([ - make_motif("MA0001.1", "motif one"), - make_motif("MA0002.1", "motif two"), - ]) - pipeline = self.make_pipeline( - orfs={"ACGT": ["AAAA"]}, + database = StubMotifDatabase([make_motif("MA0001.1", "one"), make_motif("MA0002.1", "two")]) + pipeline = self.make_pipeline(orfs={"ACGT": ["AAAA"]}, domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, - orthologs=[make_ortholog("NP_1", "AAAA")], database=database, - ) - destination = self.root / "motifs" - - code, _ = self.run_cli(pipeline, *self.arguments("motifs")) - - motifs = [row for row in self.read_rows(destination) if row["Row Type"] == "motif"] + orthologs=[make_ortholog()], database=database) + code, _ = self.run_cli(pipeline, *self.arguments()) + raw = self.read_rows(self.root / "bundle") + self.assertEqual(code, 0) + self.assertEqual(len(raw), 2) # No extra ORF/hit/alignment-only rows. + self.assertEqual([r["Motif Matrix ID"] for r in raw], ["MA0001.1", "MA0002.1"]) + repeated_fields = [ + "Gene", "Gene Status", "ORF Candidate Order", "ORF ID", + "ORF Selection Status", "Query Pfam Occurrence", "Query Pfam ID", + "BLAST Hit Group", "BLAST Hit ID", "BLAST Accession", + "Hit Pfam Occurrence", "Hit Pfam ID", "Global Alignment ID", + "Global Alignment Identity Ratio", + ] + self.assertTrue(all(all(row[field] for field in repeated_fields) for row in raw)) + self.assertEqual({row["Motif Matrix ID"] for row in raw}, {"MA0001.1", "MA0002.1"}) + self.assertTrue(all(row["Gene"] == "motifs" and row["BLAST Accession"] == "NP_1" for row in raw)) + self.assertTrue(all(row["Query Pfam ID"] == "PF00046.1" and row["Hit Pfam ID"] == "PF00046.1" for row in raw)) + self.assertEqual(raw[0]["Global Alignment ID"], raw[1]["Global Alignment ID"]) + self.assertEqual(json.loads(raw[0]["Motif PFM JSON"]), {"A": [1, 2], "C": [3, 4]}) + + def test_unpaired_domain_is_visible_once_without_alignment(self): + self.write_fasta("domains.fa") + query = "AAAAAAAA" + hit = "AAAAAAAT" + pipeline = self.make_pipeline(orfs={"ACGT": [query]}, domains={ + query: [DomainRecord("Q1", "PF00046.1", 0, 4), DomainRecord("Q2", "PF00047.1", 4, 8)], + hit: [DomainRecord("H1", "PF00046.2", 0, 4), DomainRecord("H2", "PF00047.2", 4, 8), + DomainRecord("Other", "PF99999.1", 0, 3)]}, + orthologs=[make_ortholog(sequence=hit)]) + code, _ = self.run_cli(pipeline, *self.arguments("domains", "--stop-after", "alignment")) + rows = self.read_rows(self.root / "domains") + self.assertEqual(code, 0) + self.assertEqual(len(rows), 3) + unmatched = [r for r in rows if r["Hit Pfam ID"] == "PF99999.1"] + self.assertEqual(len(unmatched), 1) + self.assertEqual(unmatched[0]["Global Alignment ID"], "") + matched = [r for r in rows if r.get("Global Alignment ID")] + self.assertEqual([(r["Query Pfam ID"].split('.')[0], r["Hit Pfam ID"].split('.')[0]) for r in matched], + [("PF00046", "PF00046"), ("PF00047", "PF00047")]) + + def test_identical_domain_annotations_keep_occurrence_associations(self): + self.write_fasta("occurrences.fa") + sequence = "AAAAAAAA" + domain1 = DomainRecord("Same", "PF00046.1", 0, 4) + domain2 = DomainRecord("Same", "PF00046.1", 4, 8) + pipeline = self.make_pipeline(orfs={"ACGT": [sequence]}, + domains={sequence: [domain1, domain2]}, orthologs=[make_ortholog(sequence=sequence)], + database=StubMotifDatabase([make_motif("MA0001.1", "one")])) + code, _ = self.run_cli(pipeline, *self.arguments()) + rows = self.read_rows(self.root / "bundle") self.assertEqual(code, 0) - self.assertEqual(len(motifs), 2) - self.assertEqual({row["Matrix ID"] for row in motifs}, {"MA0001.1", "MA0002.1"}) - self.assertTrue(all(row["Input Gene"] == "motifs" for row in motifs)) - self.assertTrue(all(row["Ortholog Accession"] == "NP_1" for row in motifs)) - self.assertTrue(all(row["Query Domain ID"] == "PF00046.1" for row in motifs)) - self.assertTrue(all(row["Ortholog Domain ID"] == "PF00046.1" for row in motifs)) - self.assertEqual(json.loads(motifs[0]["PFM JSON"]), {"A": [1, 2], "C": [3, 4]}) + self.assertEqual(len(rows), 4) + self.assertEqual({(r["Query Pfam Start (0-based aa)"], r["Hit Pfam Start (0-based aa)"]) for r in rows}, + {("0", "0"), ("0", "4"), ("4", "0"), ("4", "4")}) + self.assertTrue(all(r["Motif Matrix ID"] == "MA0001.1" for r in rows)) def test_later_ortholog_failure_preserves_completed_first_ortholog_evidence(self): - self.write_fasta("later_ortholog.fa") - first = make_ortholog("NP_first", "AAAA") - second = make_ortholog("NP_second", "CCCC") - pipeline = self.make_pipeline( - orfs={"ACGT": ["AAAA"]}, + self.write_fasta("later.fa") + pipeline = self.make_pipeline(orfs={"ACGT": ["AAAA"]}, domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, - orthologs=[first, second], scanner_fail_on="CCCC", - ) - destination = self.root / "ortholog-failure" - - code, _ = self.run_cli(pipeline, *self.arguments("ortholog-failure")) - - rows = self.read_rows(destination) + orthologs=[make_ortholog("NP_first"), make_ortholog("NP_second", "CCCC")], scanner_fail_on="CCCC") + code, _ = self.run_cli(pipeline, *self.arguments()) + rows = self.read_rows(self.root / "bundle") self.assertEqual(code, 1) - self.assertTrue(any(row["Row Type"] == "ortholog_domain" and row["Ortholog Accession"] == "NP_first" for row in rows)) - self.assertTrue(any(row["Row Type"] == "gene_error" and row["Stage"] == "ortholog_pfam" for row in rows)) + self.assertTrue(any(r["BLAST Accession"] == "NP_first" and r.get("Global Alignment ID") for r in rows)) + self.assertTrue(any(r["BLAST Accession"] == "NP_second" and r["Hit Pfam Status"] == "failed" for r in rows)) def test_jaspar_failure_preserves_prior_motif_and_alignment_evidence(self): - self.write_fasta("jaspar_failure.fa") + self.write_fasta("jaspar.fa") sequence = "AAAAAAAA" database = StubMotifDatabase([make_motif("MA0001.1", "first")], fail_on_call=2) - pipeline = self.make_pipeline( - orfs={"ACGT": [sequence]}, - domains={sequence: [ - DomainRecord("First", "PF00046.1", 0, 4), - DomainRecord("Second", "PF00046.2", 4, 8), - ]}, - orthologs=[make_ortholog("NP_1", sequence)], database=database, - ) - destination = self.root / "jaspar-failure" - - code, _ = self.run_cli(pipeline, *self.arguments("jaspar-failure")) - - rows = self.read_rows(destination) + pipeline = self.make_pipeline(orfs={"ACGT": [sequence]}, domains={sequence: [ + DomainRecord("First", "PF00046.1", 0, 4), DomainRecord("Second", "PF00046.2", 4, 8)]}, + orthologs=[make_ortholog(sequence=sequence)], database=database) + code, _ = self.run_cli(pipeline, *self.arguments()) + rows = self.read_rows(self.root / "bundle") self.assertEqual(code, 1) - self.assertEqual(len([row for row in rows if row["Row Type"] == "domain_comparison"]), 2) - self.assertEqual(len([row for row in rows if row["Row Type"] == "motif"]), 1) - self.assertTrue(any(row["Row Type"] == "gene_error" and row["Stage"] == "motif" for row in rows)) + self.assertEqual(sum(bool(r["Global Alignment ID"]) for r in rows), 2) + self.assertEqual(sum(bool(r["Motif Matrix ID"]) for r in rows), 1) + failed = [r for r in rows if r["Motif Status"] == "failed"] + self.assertEqual(len(failed), 1) + self.assertIn("simulated JASPAR failure", failed[0]["Motif Error"]) def test_cli_continues_after_gene_execution_failure(self): self.write_fasta("first.fa") self.write_fasta("second.fa", "TGCA") - first_orf = "AAAA" - second_orf = "CCCC" - pipeline = self.make_pipeline( - orfs={"ACGT": [first_orf], "TGCA": [second_orf]}, - domains={first_orf: [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, - orthologs=[], scanner_fail_on=first_orf, - ) - destination = self.root / "continue" - - code, stderr = self.run_cli(pipeline, *self.arguments("continue")) - - rows = self.read_rows(destination) + pipeline = self.make_pipeline(orfs={"ACGT": ["AAAA"], "TGCA": ["CCCC"]}, scanner_fail_on="AAAA") + code, stderr = self.run_cli(pipeline, *self.arguments()) + rows = self.read_rows(self.root / "bundle") self.assertEqual(code, 1) self.assertIn("gene first", stderr) - self.assertTrue(any(row["Input Gene"] == "first" and row["Row Type"] == "gene_error" for row in rows)) - self.assertTrue(any(row["Input Gene"] == "second" and row["Row Type"] == "input" for row in rows)) - self.assertEqual(json.loads((destination / "metadata.json").read_text())["summary"]["genes_failed"], 1) - self.assertEqual(json.loads((destination / "metadata.json").read_text())["summary"]["genes_completed"], 1) + self.assertEqual([r["Gene"] for r in rows], ["first", "second"]) + self.assertEqual(rows[0]["Query Pfam Status"], "failed") + self.assertEqual(rows[1]["Query Pfam Status"], "no accepted domain") + metadata = json.loads((self.root / "bundle/metadata.json").read_text()) + self.assertEqual(metadata["summary"]["genes_failed"], 1) + self.assertEqual(metadata["summary"]["genes_completed"], 1) def test_stop_after_alignment_never_calls_jaspar(self): self.write_fasta("alignment.fa") database = StubMotifDatabase([make_motif("MA0001.1", "unused")]) - pipeline = self.make_pipeline( - orfs={"ACGT": ["AAAA"]}, + pipeline = self.make_pipeline(orfs={"ACGT": ["AAAA"]}, domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, - orthologs=[make_ortholog()], database=database, - ) - destination = self.root / "alignment" - + orthologs=[make_ortholog()], database=database) code, _ = self.run_cli(pipeline, *self.arguments("alignment", "--stop-after", "alignment")) - - rows = self.read_rows(destination) - metadata = json.loads((destination / "metadata.json").read_text(encoding="utf-8")) + rows = self.read_rows(self.root / "alignment") self.assertEqual(code, 0) self.assertEqual(database.calls, 0) - self.assertEqual(metadata["completion"], "intentionally-stopped") - self.assertTrue(any(row["Stage"] == "alignment" and row["Row Type"] == "domain_comparison" for row in rows)) - self.assertTrue(any(row["Stage"] == "motif" and row["Status"] == "not-reached" for row in rows)) + self.assertEqual(len(rows), 1) + self.assertTrue(rows[0]["Global Alignment ID"]) + self.assertTrue(all(not rows[0][h] for h in RESULT_HEADERS if h.startswith("Motif "))) + self.assertIn("intentional-stop", rows[0]["Gene Outcome"]) + self.assertEqual({r["Query Pfam Occurrence"] for r in rows}, {"1"}) + self.assertEqual({r["Hit Pfam Occurrence"] for r in rows}, {"1"}) + + def test_alignment_failure_does_not_erase_domain_evidence(self): + self.write_fasta("alignment_error.fa") + pipeline = self.make_pipeline(orfs={"ACGT": ["AAAA"]}, domains={ + "AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)], + "CCCC": [DomainRecord("Homeobox", "PF00046.2", 0, 4)]}, + orthologs=[make_ortholog(sequence="CCCC")]) + with patch.object(pipeline, "_calculate_domain_identity", side_effect=RuntimeError("alignment failed")): + code, _ = self.run_cli(pipeline, *self.arguments()) + rows = self.read_rows(self.root / "bundle") + self.assertEqual(code, 1) + self.assertEqual(rows[0]["Hit Pfam ID"], "PF00046.2") + self.assertEqual(rows[0]["Global Alignment Status"], "failed") + self.assertEqual(rows[0]["Global Alignment Identity Ratio"], "") + self.assertEqual(rows[0]["Motif Matrix ID"], "") + + def test_motif_failure_after_partial_generator_preserves_first_motif(self): + self.write_fasta("partial_motifs.fa") + database = StubMotifDatabase() + def partial(request): + yield make_motif("MA0001.1", "first") + raise RuntimeError("later motif failed") + database.search = partial + pipeline = self.make_pipeline(orfs={"ACGT": ["AAAA"]}, + domains={"AAAA": [DomainRecord("Homeobox", "PF00046.1", 0, 4)]}, + orthologs=[make_ortholog()], database=database) + code, _ = self.run_cli(pipeline, *self.arguments()) + rows = self.read_rows(self.root / "bundle") + self.assertEqual(code, 1) + self.assertEqual(len(rows), 2) + self.assertEqual(rows[0]["Motif Matrix ID"], "MA0001.1") + self.assertEqual(rows[1]["Motif Status"], "failed") + self.assertEqual(rows[1]["Global Alignment ID"], rows[0]["Global Alignment ID"]) + self.assertEqual(rows[1]["BLAST Accession"], rows[0]["BLAST Accession"]) + + def test_filtered_query_domain_is_not_lost_when_another_is_accepted(self): + self.write_fasta("filtered.fa") + pipeline = self.make_pipeline(orfs={"ACGT": ["AAAAAAAA"]}, domains={"AAAAAAAA": [ + DomainRecord("Other", "PF99999.1", 0, 4), DomainRecord("Homeobox", "PF00046.1", 4, 8)]}) + pipeline.valid_pfam_list = ["PF00046"] + code, _ = self.run_cli(pipeline, *self.arguments()) + rows = self.read_rows(self.root / "bundle") + self.assertEqual(code, 0) + self.assertEqual([r["Query Pfam ID"] for r in rows], ["PF99999.1", "PF00046.1"]) + self.assertEqual(rows[0]["Query Pfam Status"], "not-selected") + self.assertEqual(rows[0]["BLAST Accession"], "") + + def test_atomic_result_write_preserves_previous_file_on_formatter_error(self): + from lverage.cli import _Bundle + destination = self.root / "atomic" + destination.mkdir() + bundle = _Bundle(destination, {}) + before = (destination / "results.csv").read_bytes() + with patch("lverage.cli._grouped_result_rows", side_effect=RuntimeError("format failed")): + with self.assertRaises(RuntimeError): + bundle.add_event("gene", {"event": "gene_start", "stage": "input"}) + self.assertEqual((destination / "results.csv").read_bytes(), before) def test_invalid_inputs_and_existing_output_exit_two(self): self.write_fasta("valid.fa") @@ -409,13 +475,14 @@ def test_validate_input_discovery_preserves_order_and_fragment_lists(self): def test_build_pipeline_passes_existing_options(self): parsed = build_parser().parse_args(self.arguments( "configured", "--blastp", "/tools/blastp", "--blastdbcmd", "/tools/blastdbcmd", - "--pfamscan", "/tools/pfamscan", "--identity-threshold", "0.83", + "--pfamscan", "/tools/pfamscan", "--hmmscan", "/tools/hmmscan", + "--hmmalign", "/tools/hmmalign", "--identity-threshold", "0.83", "--valid-pfam", "PF00046", "--jaspar-hits", "7", "--jaspar-evalue", "2e-5", )) with patch("lverage.blast.LocalBlastSearcher") as blast, \ patch("lverage.pfam.LocalPfamScanner") as pfam, \ patch("lverage.orf_searcher.OrffinderOrfSearcher") as orf, \ - patch("lverage.jaspar.Jaspar2024MotifDB") as jaspar, \ + patch("lverage.jaspar.Jaspar2026MotifDB") as jaspar, \ patch("lverage.pipeline.Lverage") as lverage: _build_pipeline(parsed, {"Homo sapiens": 9606}) blast.assert_called_once_with( @@ -423,7 +490,11 @@ def test_build_pipeline_passes_existing_options(self): blastp_path="/tools/blastp", blastdbcmd_path="/tools/blastdbcmd", ) pfam.assert_called_once_with(database_path=str(self.pfam_dir), pfamscan_path="/tools/pfamscan") - jaspar.assert_called_once_with(n_hits=7, escore_threshold=2e-5) + jaspar.assert_called_once_with( + n_hits=7, escore_threshold=2e-5, data_dir=None, sqlite_db=None, + blastp_path="/tools/blastp", hmmscan_path="/tools/hmmscan", + hmmalign_path="/tools/hmmalign", + ) lverage.assert_called_once_with( motif_database_list=[jaspar.return_value], orf_searcher=orf.return_value, domain_scanner=pfam.return_value, ortholog_searcher=blast.return_value, diff --git a/tests/v2/test_jaspar.py b/tests/v2/test_jaspar.py index 7ef98c8..b657f4d 100644 --- a/tests/v2/test_jaspar.py +++ b/tests/v2/test_jaspar.py @@ -1,162 +1,153 @@ +import sqlite3 +import tempfile import unittest -from unittest.mock import Mock, call, patch - -import requests +from pathlib import Path +from unittest.mock import patch from lverage.domain_scanner import DomainRecord -from lverage.jaspar import Jaspar2024MotifDB +from lverage.jaspar import Jaspar2026MotifDB from lverage.motif_database import MotifSearchRequest -def make_response(payload): - response = Mock() - response.json.return_value = payload - return response - - -def make_request(sequence="ORTHOLOG", tax_id=9606): +def make_request(sequence="ORTHOLOG", tax_id=9606, domain=None): + domain = domain or DomainRecord("Homeobox", "PF00046.1", 0, len(sequence)) return MotifSearchRequest( - "QUERY", - DomainRecord("Homeobox", "PF00046.1", 0, 5), - sequence, - DomainRecord("Homeobox", "PF00046.2", 0, 5), - tax_id, + "QUERY", DomainRecord("Homeobox", "PF00046.1", 0, 5), + sequence, domain, tax_id, ) -def make_motif(matrix_id, species_ids, motif_class=None): - if motif_class is None: - motif_class = ["Homeobox"] - return { - "matrix_id": matrix_id, - "name": f"Motif {matrix_id}", - "pfm": {"A": [1], "C": [2], "G": [3], "T": [4]}, - "class": motif_class, - "species": [{"tax_id": species_id} for species_id in species_ids], - } - - -class Jaspar2024MotifDBTests(unittest.TestCase): - - def test_import_and_constructor_do_not_make_requests(self): - with patch("lverage.jaspar.requests.get") as mock_get: - Jaspar2024MotifDB() - - mock_get.assert_not_called() - - @patch("lverage.jaspar.requests.get") - def test_search_uses_windowed_ortholog_sequence_and_timeout(self, mock_get): - mock_get.return_value = make_response({"results": []}) - sequence = "A" * 5000 - request = MotifSearchRequest( - "QUERY", - DomainRecord("Homeobox", "PF00046.1", 0, 5), - sequence, - DomainRecord("Homeobox", "PF00046.2", 2400, 2450), - 9606, +class LocalJasparTests(unittest.TestCase): + + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.root = Path(self.temporary.name) + self.database_path = self.root / "JASPAR2026.sqlite" + with sqlite3.connect(self.database_path) as connection: + connection.executescript(""" + CREATE TABLE MATRIX (ID INTEGER, BASE_ID TEXT, VERSION INTEGER, COLLECTION TEXT, NAME TEXT); + CREATE TABLE MATRIX_SPECIES (ID INTEGER, TAX_ID TEXT); + CREATE TABLE MATRIX_ANNOTATION (ID INTEGER, TAG TEXT, VAL TEXT); + CREATE TABLE MATRIX_DATA (ID INTEGER, ROW TEXT, COL INTEGER, VAL REAL); + """) + for matrix_id, internal_id, name, species in ( + ("MA0001.1", 1, "First motif", [9606]), + ("MA0001.2", 4, "First motif newer", [9606]), + ("MA0002.3", 2, "Second motif", [10090, 9606]), + ("MA0003.1", 3, "Mouse-only motif", [10090]), + ): + base_id, version = matrix_id.split(".") + connection.execute( + "INSERT INTO MATRIX VALUES (?, ?, ?, 'CORE', ?)", + (internal_id, base_id, int(version), name), + ) + connection.executemany( + "INSERT INTO MATRIX_SPECIES VALUES (?, ?)", + [(internal_id, str(tax_id)) for tax_id in species], + ) + connection.execute( + "INSERT INTO MATRIX_ANNOTATION VALUES (?, 'class', 'Homeobox')", + (internal_id,), + ) + for row_index, base in enumerate("ACGT"): + connection.executemany( + "INSERT INTO MATRIX_DATA VALUES (?, ?, ?, ?)", + [(internal_id, base, column, row_index + column) + for column in range(2)], + ) + self.adapter = Jaspar2026MotifDB( + data_dir=self.root, + sqlite_db=self.database_path, + blastp_path="blastp-test", + hmmscan_path="hmmscan-test", + hmmalign_path="hmmalign-test", ) - database = Jaspar2024MotifDB(request_timeout_seconds=7) - expected_window = database._window_ortholog_sequence(sequence, request.ortholog_domain) - - self.assertEqual(database.search(request), []) - mock_get.assert_called_once_with( - f"https://jaspar.elixir.no/api/v1/infer/{expected_window}/", - timeout=7, - ) - mock_get.return_value.raise_for_status.assert_called_once_with() - - @patch("lverage.jaspar.requests.get") - def test_results_are_sorted_filtered_and_limited_after_acceptance(self, mock_get): - infer_response = make_response({"results": [ - {"evalue": 1e-4, "url": "https://jaspar.elixir.no/api/v1/matrix/HIGH/"}, - {"evalue": 1e-10, "url": "https://jaspar.elixir.no/api/v1/matrix/WRONG/"}, - {"evalue": 1e-9, "url": "https://jaspar.elixir.no/api/v1/matrix/FIRST/"}, - {"evalue": 1e-8, "url": "https://jaspar.elixir.no/api/v1/matrix/SECOND/"}, - {"evalue": 1e-7, "url": "https://jaspar.elixir.no/api/v1/matrix/THIRD/"}, - ]}) - mock_get.side_effect = [ - infer_response, - make_response(make_motif("WRONG", [10090])), - make_response(make_motif("FIRST", [9606])), - make_response(make_motif("SECOND", [10090, 9606])), + self.adapter._validated = True + + def tearDown(self): + self.temporary.cleanup() + + def test_local_species_filter_threshold_and_limit_preserve_metrics(self): + self.adapter.n_hits = 1 + self.adapter.escore_threshold = 1e-6 + rows = [ + {"matrix_id": "MA0003.1", "tf_name": "Mouse", "evalue": 1e-12, + "query_region": "1-30", "tf_region": "4-32", "dbd_identity": 0.99}, + {"matrix_id": "MA0002.3", "tf_name": "Second", "evalue": 1e-10, + "query_region": "2-30", "tf_region": "3-31", "dbd_identity": 0.81}, + {"matrix_id": "MA0001.1", "tf_name": "First", "evalue": 1e-5, + "query_region": "1-30", "tf_region": "1-30", "dbd_identity": 0.7}, ] - database = Jaspar2024MotifDB(n_hits=2, escore_threshold=1e-6) - - records = database.search(make_request()) - - self.assertEqual([record.matrix_id for record in records], ["FIRST", "SECOND"]) - self.assertEqual([record.inference_evalue for record in records], [1e-9, 1e-8]) - self.assertEqual(mock_get.call_count, 4) - - @patch("lverage.jaspar.requests.get") - def test_record_parses_current_motif_fields_and_url(self, mock_get): - mock_get.side_effect = [ - make_response({"results": [{"evalue": "1e-12", "url": "DETAIL"}]}), - make_response(make_motif("MA0001.1", [9606], "Homeobox")), + with patch("lverage.jaspar.load_cisbp_models", return_value={None: {}}), \ + patch("lverage.jaspar.load_jaspar_data", return_value={}), \ + patch("lverage.jaspar.infer_sequence", return_value=rows): + records = self.adapter.search(make_request()) + + self.assertEqual([record.matrix_id for record in records], ["MA0002.3"]) + self.assertEqual(records[0].inference_evalue, 1e-10) + self.assertEqual(records[0].inference_dbd_identity, 0.81) + self.assertEqual(records[0].motif_name, "Second motif") + self.assertEqual(records[0].pfm["A"], [0.0, 1.0]) + + def test_exact_matrix_version_is_required(self): + with self.assertRaisesRegex(ValueError, "MA0001.3"): + self.adapter._resolve_matrix({ + "matrix_id": "MA0001.3", "tf_name": "Wrong version", + "evalue": 1e-10, "query_region": "1-10", + "tf_region": "1-10", "dbd_identity": 0.8, + }) + + def test_search_retains_multiple_returned_versions_of_one_matrix(self): + rows = [ + {"matrix_id": "MA0001.1", "tf_name": "First", "evalue": 1e-10, + "query_region": "1-30", "tf_region": "1-30", "dbd_identity": 0.8}, + {"matrix_id": "MA0001.2", "tf_name": "First newer", "evalue": 2e-10, + "query_region": "1-30", "tf_region": "1-30", "dbd_identity": 0.85}, ] + with patch("lverage.jaspar.load_cisbp_models", return_value={None: {}}), \ + patch("lverage.jaspar.load_jaspar_data", return_value={}), \ + patch("lverage.jaspar.infer_sequence", return_value=rows): + records = self.adapter.search(make_request()) - record = Jaspar2024MotifDB().search(make_request())[0] - - self.assertEqual(record.matrix_id, "MA0001.1") - self.assertEqual(record.motif_name, "Motif MA0001.1") - self.assertEqual(record.motif_class, "Homeobox") - self.assertEqual(record.motif_url, "https://jaspar.elixir.no/matrix/MA0001.1/") - self.assertEqual(record.inference_evalue, 1e-12) - - @patch("lverage.jaspar.requests.get") - def test_species_pagination_is_loaded_once_and_cached(self, mock_get): - mock_get.side_effect = [ - make_response({ - "results": [{"tax_id": 9606}], - "next": "https://jaspar.elixir.no/api/v1/species/?page=2", - }), - make_response({"results": [{"tax_id": "10090"}], "next": None}), - ] - database = Jaspar2024MotifDB(request_timeout_seconds=11) - - self.assertTrue(database.check_species_validity(9606)) - self.assertTrue(database.check_species_validity(10090)) - self.assertFalse(database.check_species_validity(7227)) - self.assertEqual(mock_get.call_args_list, [ - call( - database.jaspar_rest_species_url, - params=database.species_params, - timeout=11, - ), - call( - "https://jaspar.elixir.no/api/v1/species/?page=2", - params=None, - timeout=11, - ), - ]) - - @patch("lverage.jaspar.requests.get") - def test_http_errors_propagate(self, mock_get): - response = make_response({"results": []}) - response.raise_for_status.side_effect = requests.HTTPError("service failure") - mock_get.return_value = response - - with self.assertRaises(requests.HTTPError): - Jaspar2024MotifDB().search(make_request()) - - @patch("lverage.jaspar.requests.get") - def test_request_timeouts_propagate(self, mock_get): - mock_get.side_effect = requests.Timeout("service timeout") - - with self.assertRaises(requests.Timeout): - Jaspar2024MotifDB().search(make_request()) - - @patch("lverage.jaspar.requests.get") - def test_malformed_responses_raise_value_error(self, mock_get): - mock_get.return_value = make_response({"unexpected": []}) - - with self.assertRaises(ValueError): - Jaspar2024MotifDB().search(make_request()) - - def test_search_validates_request_contract(self): - with self.assertRaises(TypeError): - Jaspar2024MotifDB().search(object()) + self.assertEqual( + [record.matrix_id for record in records], ["MA0001.1", "MA0001.2"] + ) + + def test_species_check_uses_exact_local_taxonomy_ids(self): + self.adapter._validated = False + with patch.object(self.adapter, "_validate_configuration"): + self.assertTrue(self.adapter.check_species_validity(9606)) + self.assertTrue(self.adapter.check_species_validity(10090)) + self.assertFalse(self.adapter.check_species_validity(7227)) + + def test_search_calls_in_package_inference_with_window_and_configured_executables(self): + sequence = "A" * 2000 + prediction = { + "matrix_id": "MA0001.1", "tf_name": "Example TF", "evalue": 1e-12, + "query_region": "1-40", "tf_region": "2-41", "dbd_identity": 0.875, + } + with patch("lverage.jaspar.load_cisbp_models", return_value={None: {}}), \ + patch("lverage.jaspar.load_jaspar_data", return_value={}), \ + patch("lverage.jaspar.infer_sequence", return_value=[prediction]) as infer: + records = self.adapter.search(make_request(sequence=sequence)) + + self.assertEqual([record.matrix_id for record in records], ["MA0001.1"]) + self.assertEqual(infer.call_args.args[0], sequence) + self.assertEqual(infer.call_args.kwargs["query_id"], "lverage_query") + self.assertEqual(infer.call_args.kwargs["blastp_path"], "blastp-test") + self.assertEqual(infer.call_args.kwargs["hmmscan_path"], "hmmscan-test") + self.assertEqual(infer.call_args.kwargs["hmmalign_path"], "hmmalign-test") + self.assertEqual(infer.call_args.kwargs["data_dir"], self.root) + self.assertFalse(Path(infer.call_args.kwargs["work_dir"]).exists()) + + def test_existing_online_behavior_window_is_unchanged(self): + sequence = "A" * 5000 + domain = DomainRecord("Homeobox", "PF00046.1", 2400, 2450) + self.assertEqual( + self.adapter._window_ortholog_sequence(sequence, domain), + sequence[1425:3425], + ) if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file diff --git a/tests/v2/test_jaspar_inference.py b/tests/v2/test_jaspar_inference.py new file mode 100644 index 0000000..4b0740a --- /dev/null +++ b/tests/v2/test_jaspar_inference.py @@ -0,0 +1,98 @@ +import unittest +import math +from types import SimpleNamespace +from unittest.mock import patch + +import numpy as np +from Bio.Seq import Seq +from Bio.SeqRecord import SeqRecord + +from lverage.jaspar_inference import ( + _rost_identity_cutoff, + blast_search, + infer_sequence, + non_overlapping_domains, + positional_alignment_score, +) + + +class JasparInferenceAlgorithmTests(unittest.TestCase): + + def test_upstream_non_overlap_rule_keeps_lowest_conditional_evalue(self): + domains = [ + ("Homeobox", 0, 20, 1e-4), + ("Forkhead", 10, 25, 1e-6), + ("bZIP", 30, 45, 1e-3), + ] + self.assertEqual( + non_overlapping_domains(domains), + [("Forkhead", 10, 25, 1e-6), ("bZIP", 30, 45, 1e-3)], + ) + + def test_positional_features_match_upstream_identity_and_blosum_rules(self): + aligned = ["A-C"] + np.testing.assert_array_equal( + positional_alignment_score(aligned, aligned, "identity"), + np.array([1.0, 1.0, 1.0]), + ) + np.testing.assert_array_equal( + positional_alignment_score(aligned, aligned, "blosum62"), + np.array([4.0, 1.0, 9.0]), + ) + + def test_rost_cutoff_matches_source_formula(self): + length = 200 + expected = 5 + 480 * length ** (-0.32 * (1 + math.exp(-length / 1000))) + self.assertAlmostEqual(_rost_identity_cutoff(length), expected) + + def test_blast_search_uses_local_db_and_upstream_outfmt_as_argv(self): + record = SeqRecord(Seq("A" * 500), id="query") + row = "target\t100\t500\t1\t500\t1\t500\t1e-20\t1000\t100\t500\t500\n" + with patch( + "lverage.jaspar_inference._checked_run", + return_value=SimpleNamespace(stdout=row), + ) as run: + hits = blast_search( + record, "/local/jaspar/files", "/tmp/query.fasta", + blastp_path="/tools/blastp", taxons=("vertebrates",), + ) + command = run.call_args.args[0] + self.assertEqual(command[0], "/tools/blastp") + self.assertIn("/local/jaspar/files/vertebrates.fa", command) + self.assertEqual(command[command.index("-query") + 1], "/tmp/query.fasta") + self.assertEqual(command[command.index("-outfmt") + 1], + "6 sseqid pident length qstart qend sstart send evalue bitscore ppos qlen slen") + self.assertEqual(len(hits), 1) + + def test_sequence_inference_preserves_upstream_hsim_gate_and_versions(self): + query_domains = [("Homeobox", "AC-G", 1, 4, 1e-9)] + blast_hits = [("lverage_query", "target", "3-9", "2-8", 1e-12, 100.0, + 90.0, 7, 95.0, 90.0)] + data = { + "target": { + "pfam": [("Homeobox", "AC-G", 1, 4, 1e-9)], + "profiles": [["MA0001.1", "TF"], ["MA0001.2", "TF"]], + }, + } + models = {None: {"fallback": True}} + with patch("lverage.jaspar_inference.scan_and_align_domains", + return_value=query_domains), \ + patch("lverage.jaspar_inference.blast_search", + return_value=blast_hits), \ + patch("lverage.jaspar_inference.score_alignment_result", + return_value=(0.91, "HSim")) as score: + predictions = infer_sequence( + "ACGT", data_dir="/data", cisbp_models=models, + jaspar_data=data, work_dir="/tmp/work", blastp_path="blastp", + hmmscan_path="hmmscan", hmmalign_path="hmmalign", + ) + self.assertEqual([row["matrix_id"] for row in predictions], + ["MA0001.1", "MA0001.2"]) + self.assertEqual(predictions[0]["evalue"], 1e-12) + self.assertEqual(predictions[0]["dbd_identity"], 1.0) + self.assertEqual(predictions[0]["query"], "lverage_query") + self.assertEqual(score.call_args.args[1], models[None]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/v2/test_jaspar_models.py b/tests/v2/test_jaspar_models.py new file mode 100644 index 0000000..42f117e --- /dev/null +++ b/tests/v2/test_jaspar_models.py @@ -0,0 +1,52 @@ +import unittest + +import numpy as np + +from lverage.jaspar_models import score_alignment_result + + +class JasparSimilarityRegressionTests(unittest.TestCase): + + def test_upstream_identical_domain_rule_accepts_exact_identity(self): + model = { + "Model.Class": "SequenceIdentity", + "Model.Name": "PctID_L", + "Threshold.HSim": 0.9, + "Threshold.Dis": 0.5, + } + score, classification = score_alignment_result({"PctID_L": 1.0}, model) + self.assertEqual(score, 1.0) + self.assertEqual(classification, "HSim") + + def test_upstream_sequence_identity_threshold_is_inclusive(self): + model = { + "Model.Class": "SequenceIdentity", + "Model.Name": "PctID_L", + "Threshold.HSim": 0.8, + "Threshold.Dis": 0.5, + } + result = {"PctID_L": 0.8} + self.assertEqual(score_alignment_result(result, model), (0.8, "HSim")) + score, classification = score_alignment_result({"PctID_L": 0.79}, model) + self.assertTrue(np.isnan(score)) + self.assertTrue(np.isnan(classification)) + + def test_upstream_similarity_regression_baseline_then_scaled_score(self): + model = { + "Model.Class": "SimilarityRegression", + "Baseline": {"Name": "PctID_L", "Threshold.HSim": 1.1}, + "SR.Features": "PctID", + "SR.FeatureScales.mean": np.asarray([0.0, 0.0]), + "SR.FeatureScales.sd": np.asarray([1.0, 1.0]), + "SR.Weights": np.asarray([0.25, 0.25]), + "SR.Intercept": 0.0, + "SR.LogisticTransform": False, + "Threshold.HSim": 0.4, + "Threshold.Dis": None, + } + result = {"PctID_L": 0.5, "ByPos.PctID": np.asarray([1.0, 1.0])} + self.assertEqual(score_alignment_result(result, model), (0.5, "HSim")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/v2/test_jaspar_record.py b/tests/v2/test_jaspar_record.py index 2b9c00a..74de0ab 100644 --- a/tests/v2/test_jaspar_record.py +++ b/tests/v2/test_jaspar_record.py @@ -14,11 +14,17 @@ def test_record_serializes_in_header_order(self): "https://jaspar.elixir.no/matrix/MA0001.1/", "Homeobox", 1e-8, + "Example TF", + "1-80", + "2-81", + 0.875, ) self.assertEqual(record.get_headers(), [ "Matrix ID", "Motif Name", "PFM", "Motif URL", - "Motif Class", "Inference E-value", + "Motif Class", "JASPAR Tool TF Name", + "JASPAR Tool BLAST E-value", "JASPAR Tool Query Region", + "JASPAR Tool TF Region", "JASPAR Tool DBD Identity Ratio", ]) self.assertEqual(record.get_values(), [ "MA0001.1", @@ -26,7 +32,11 @@ def test_record_serializes_in_header_order(self): pfm, "https://jaspar.elixir.no/matrix/MA0001.1/", "Homeobox", + "Example TF", 1e-8, + "1-80", + "2-81", + 0.875, ]) def test_headers_are_immutable_and_returned_as_a_list_copy(self): diff --git a/tests/v2/test_jaspar_window.py b/tests/v2/test_jaspar_window.py index 71e8a57..f8d4da0 100644 --- a/tests/v2/test_jaspar_window.py +++ b/tests/v2/test_jaspar_window.py @@ -1,7 +1,7 @@ import unittest from lverage.domain_scanner import DomainRecord -from lverage.jaspar import Jaspar2024MotifDB +from lverage.jaspar import Jaspar2026MotifDB class JasparWindowTests(unittest.TestCase): @@ -10,13 +10,13 @@ def test_short_sequence_is_unchanged(self): sequence = "A" * 1500 domain = DomainRecord("Homeobox", "PF00046.1", 10, 20) - self.assertEqual(Jaspar2024MotifDB._window_ortholog_sequence(sequence, domain), sequence) + self.assertEqual(Jaspar2026MotifDB._window_ortholog_sequence(sequence, domain), sequence) def test_window_is_centered_on_domain(self): sequence = "A" * 5000 domain = DomainRecord("Homeobox", "PF00046.1", 2400, 2450) - window = Jaspar2024MotifDB._window_ortholog_sequence(sequence, domain) + window = Jaspar2026MotifDB._window_ortholog_sequence(sequence, domain) self.assertEqual(window, sequence[1425:3425]) self.assertEqual(len(window), 2000) @@ -26,7 +26,7 @@ def test_window_handles_n_terminal_domain(self): domain = DomainRecord("Homeobox", "PF00046.1", 50, 120) self.assertEqual( - Jaspar2024MotifDB._window_ortholog_sequence(sequence, domain), + Jaspar2026MotifDB._window_ortholog_sequence(sequence, domain), sequence[:2000], ) @@ -35,7 +35,7 @@ def test_window_handles_c_terminal_domain(self): domain = DomainRecord("Homeobox", "PF00046.1", 4900, 4980) self.assertEqual( - Jaspar2024MotifDB._window_ortholog_sequence(sequence, domain), + Jaspar2026MotifDB._window_ortholog_sequence(sequence, domain), sequence[-2000:], ) @@ -51,7 +51,7 @@ def test_invalid_bounds_raise_value_error(self): for domain in invalid_domains: with self.subTest(domain=domain): with self.assertRaises(ValueError): - Jaspar2024MotifDB._window_ortholog_sequence(sequence, domain) + Jaspar2026MotifDB._window_ortholog_sequence(sequence, domain) if __name__ == "__main__": diff --git a/tests/v2/test_local_blast.py b/tests/v2/test_local_blast.py index 9a2dbb4..f094da7 100644 --- a/tests/v2/test_local_blast.py +++ b/tests/v2/test_local_blast.py @@ -242,6 +242,11 @@ def test_fourth_hit_skip_keeps_first_three_and_reports_partial_retrieval(self): self.assertEqual(len(ortholog_events), 3) self.assertEqual(len(skipped), 1) self.assertEqual(skipped[0]["hit_id"], "ref|NP_000004.1|") + self.assertEqual(skipped[0]["hit_group"], 4) + self.assertEqual(skipped[0]["description"], "Homeobox protein [Homo sapiens]") + self.assertEqual(skipped[0]["species_tax_id"], 9606) + self.assertEqual(skipped[0]["identity_ratio"], 0.8) + self.assertEqual(skipped[0]["query_coverage_ratio"], 20.0) self.assertEqual(len(summaries), 1) self.assertEqual(summaries[0]["status"], "partial-retrieval-failure") @@ -335,6 +340,25 @@ def test_species_resolution_is_case_insensitive(self): self.assertEqual(records[0].species_name, "Homo sapiens") + def test_default_filters_retain_isoform_titled_hit_and_retrieve_sequence(self): + searcher = self.make_searcher() + description = "forkhead box protein P2 isoform I [Homo sapiens]" + blast_xml = (FIXTURE_DIR / "blastp.xml").read_text().replace( + "Homeobox protein [Homo sapiens]", + f"{description}", + ) + self.mock_run.side_effect = [ + SimpleNamespace(stdout=blast_xml), + SimpleNamespace(stdout="MPEPTIDE"), + ] + + records = searcher.get_orthologs("QUERY") + + self.assertEqual(searcher.excluded_terms, list(DEFAULT_EXCLUDED_TERMS)) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].description, description) + self.assertEqual(records[0].sequence, "MPEPTIDE") + def test_excluded_and_unresolved_hits_are_skipped(self): searcher = self.make_searcher() alignments = [