From 052a1ce9f988b691abf51151b8b57e1964d82841 Mon Sep 17 00:00:00 2001 From: Johnny Huynh <27847622+johnnyhuy@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:22:25 +1100 Subject: [PATCH] feat(api): preview the winning operator rule without delivery Use the shared RAG matcher to inspect approved answers without retrieval or Pi jobs. Cover matching policy, validation, failure handling and path parity with offline tests and a local curl transcript. Co-authored-by: OpenAI Codex --- .tmp/proof/rule-preview-api.txt | 137 +++++++++ .tmp/proof/rule-preview-backend-start.txt | 32 +++ .tmp/proof/rule-preview-backend-tests.txt | 312 +++++++++++++++++++++ .tmp/proof/rule-preview-capture.py | 42 +++ .tmp/proof/rule-preview-config.json | 33 +++ .tmp/proof/rule-preview-frontend-tests.txt | 69 +++++ .tmp/proof/rule-preview-install.txt | 182 ++++++++++++ .tmp/proof/rule-preview-runtime-tests.txt | 16 ++ .tmp/proof/rule-preview-sitecustomize.py | 21 ++ backend/app/main.py | 37 +++ backend/tests/test_rule_preview.py | 125 +++++++++ docs/operator-rules.md | 56 +++- mise.toml | 2 +- 13 files changed, 1058 insertions(+), 6 deletions(-) create mode 100644 .tmp/proof/rule-preview-api.txt create mode 100644 .tmp/proof/rule-preview-backend-start.txt create mode 100644 .tmp/proof/rule-preview-backend-tests.txt create mode 100644 .tmp/proof/rule-preview-capture.py create mode 100644 .tmp/proof/rule-preview-config.json create mode 100644 .tmp/proof/rule-preview-frontend-tests.txt create mode 100644 .tmp/proof/rule-preview-install.txt create mode 100644 .tmp/proof/rule-preview-runtime-tests.txt create mode 100644 .tmp/proof/rule-preview-sitecustomize.py create mode 100644 backend/tests/test_rule_preview.py diff --git a/.tmp/proof/rule-preview-api.txt b/.tmp/proof/rule-preview-api.txt new file mode 100644 index 0000000..b30737f --- /dev/null +++ b/.tmp/proof/rule-preview-api.txt @@ -0,0 +1,137 @@ +Offline operator-rule preview with rule-preview-config.json. +Slack API, Socket Mode, retrieval and Pi calls raise in the proof process. +No chunks or distances apply: this endpoint only inspects loaded rules. + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/health +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 20 +content-type: application/json +{ + "status": "healthy" +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": " How do I RESET VPN access?\n"}' +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 223 +content-type: application/json +{ + "question": "How do I RESET VPN access?", + "match": { + "rule_id": "vpn-reset", + "match_type": "contains", + "matched_keyword": "reset vpn", + "priority": 10, + "response": "Use the IT portal at https://it.example.com/vpn to reset VPN access." + } +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": "OFFICE HOURS"}' +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 194 +content-type: application/json +{ + "question": "OFFICE HOURS", + "match": { + "rule_id": "office-hours", + "match_type": "exact", + "matched_keyword": "office hours", + "priority": 0, + "response": "The support desk runs 9am to 5pm, Monday to Friday." + } +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": "Where is TICKET #4821?"}' +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 189 +content-type: application/json +{ + "question": "Where is TICKET #4821?", + "match": { + "rule_id": "ticket", + "match_type": "regex", + "matched_keyword": "ticket #?\\d+", + "priority": 0, + "response": "Look up the ticket in the support portal." + } +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": "How do I reset email?"}' +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 49 +content-type: application/json +{ + "question": "How do I reset email?", + "match": null +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": " \n\t "}' +HTTP/1.1 422 Unprocessable Content +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 154 +content-type: application/json +{ + "detail": [ + { + "type": "string_too_short", + "loc": [ + "body", + "question" + ], + "msg": "String should have at least 1 character", + "input": " \n\t ", + "ctx": { + "min_length": 1 + } + } + ] +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": 123}' +HTTP/1.1 422 Unprocessable Content +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 112 +content-type: application/json +{ + "detail": [ + { + "type": "string_type", + "loc": [ + "body", + "question" + ], + "msg": "Input should be a valid string", + "input": 123 + } + ] +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/ask -H 'Content-Type: application/json' -d '{"question": "How do I RESET VPN access?"}' +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 162 +content-type: application/json +{ + "answer": "Use the IT portal at https://it.example.com/vpn to reset VPN access.", + "sources": [ + { + "source": "operator_rule", + "rule_id": "vpn-reset" + } + ], + "confidence": "high" +} + +All response assertions passed. Preview and API answer select the same rule. diff --git a/.tmp/proof/rule-preview-backend-start.txt b/.tmp/proof/rule-preview-backend-start.txt new file mode 100644 index 0000000..c4fb312 --- /dev/null +++ b/.tmp/proof/rule-preview-backend-start.txt @@ -0,0 +1,32 @@ +[dev-backend] $ uv run uvicorn app.main:app --reload --host 127.0.0.1 --port 18… +INFO: Will watch for changes in these directories: ['/Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend'] +INFO: Uvicorn running on http://127.0.0.1:18059 (Press CTRL+C to quit) +INFO: Started reloader process [34598] using WatchFiles +INFO: Started server process [34657] +INFO: Waiting for application startup. +{"timestamp": "2026-11-16T14:21:28.106608+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "app.main", "message": "Starting Wingman backend...", "event_type": "log", "request_id": null, "trace_id": null} +{"timestamp": "2026-11-16T14:21:28.114661+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "app.main", "message": "Database initialized", "event_type": "log", "request_id": null, "trace_id": null} +INFO: Application startup complete. +INFO: 127.0.0.1:54831 - "GET /health HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.535859+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "b3b5cdccd1e942a58bf4af8cd7d77b9e", "trace_id": "b3b5cdccd1e942a58bf4af8cd7d77b9e", "request_method": "GET", "request_path": "/health", "route": "/health", "status_code": 200, "duration_ms": 0.53, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54832 - "POST /api/rules/preview HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.560690+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "d8893f36a4c34f3ab1fee3c749d809e6", "trace_id": "d8893f36a4c34f3ab1fee3c749d809e6", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 200, "duration_ms": 9.41, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54834 - "POST /api/rules/preview HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.577942+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "9c494068ece84b64aee19c973fbb441d", "trace_id": "9c494068ece84b64aee19c973fbb441d", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 200, "duration_ms": 0.72, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54835 - "POST /api/rules/preview HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.592262+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "2766ec1b0f744df4a43bfbc08b97b818", "trace_id": "2766ec1b0f744df4a43bfbc08b97b818", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 200, "duration_ms": 0.72, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54836 - "POST /api/rules/preview HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.606412+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "a9d41ea7e6b141cab377ab767b5dc139", "trace_id": "a9d41ea7e6b141cab377ab767b5dc139", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 200, "duration_ms": 0.61, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54837 - "POST /api/rules/preview HTTP/1.1" 422 Unprocessable Content +{"timestamp": "2026-11-16T14:21:48.620734+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "7baf10d4255c45ecafdcc5c8fd136bf0", "trace_id": "7baf10d4255c45ecafdcc5c8fd136bf0", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 422, "duration_ms": 0.33, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54838 - "POST /api/rules/preview HTTP/1.1" 422 Unprocessable Content +{"timestamp": "2026-11-16T14:21:48.634238+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "ad6871e0d2554cb795de6e699bca3094", "trace_id": "ad6871e0d2554cb795de6e699bca3094", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 422, "duration_ms": 0.29, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +{"timestamp": "2026-11-16T14:21:48.648333+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "app.rag", "message": "Operator rule vpn-reset answered a question", "event_type": "log", "request_id": "3be154cb62d04c1fa94a7a655b422087", "trace_id": "3be154cb62d04c1fa94a7a655b422087"} +INFO: 127.0.0.1:54839 - "POST /api/ask HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.648640+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "3be154cb62d04c1fa94a7a655b422087", "trace_id": "3be154cb62d04c1fa94a7a655b422087", "request_method": "POST", "request_path": "/api/ask", "route": "/api/ask", "status_code": 200, "duration_ms": 0.69, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: Shutting down +INFO: Waiting for application shutdown. +{"timestamp": "2026-11-16T14:21:54.451885+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "app.main", "message": "Shutting down Wingman backend...", "event_type": "log", "request_id": null, "trace_id": null} +INFO: Application shutdown complete. +INFO: Finished server process [34657] +INFO: Stopping reloader process [34598] diff --git a/.tmp/proof/rule-preview-backend-tests.txt b/.tmp/proof/rule-preview-backend-tests.txt new file mode 100644 index 0000000..5413bbe --- /dev/null +++ b/.tmp/proof/rule-preview-backend-tests.txt @@ -0,0 +1,312 @@ +[test-backend] $ uv run pytest tests/test_file_extraction_units.py tests/test_l… +============================= test session starts ============================== +platform darwin -- Python 3.13.13, pytest-9.0.2, pluggy-1.6.0 -- /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/.venv/bin/python +cachedir: .pytest_cache +rootdir: /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend +configfile: pytest.ini (WARNING: ignoring pytest config in pyproject.toml!) +plugins: langsmith-0.5.2, anyio-4.12.0, asyncio-1.3.0 +asyncio: mode=Mode.STRICT, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function +collecting ... collected 239 items + +tests/test_file_extraction_units.py::test_extract_plain_text PASSED [ 0%] +tests/test_file_extraction_units.py::test_extract_markdown PASSED [ 0%] +tests/test_file_extraction_units.py::test_extract_python_file PASSED [ 1%] +tests/test_file_extraction_units.py::test_extract_json PASSED [ 1%] +tests/test_file_extraction_units.py::test_extract_csv PASSED [ 2%] +tests/test_file_extraction_units.py::test_extract_html PASSED [ 2%] +tests/test_file_extraction_units.py::test_extract_html_removes_scripts PASSED [ 2%] +tests/test_file_extraction_units.py::test_extract_with_encoding_errors PASSED [ 3%] +tests/test_file_extraction_units.py::test_extract_xml PASSED [ 3%] +tests/test_file_extraction_units.py::test_extract_yaml PASSED [ 4%] +tests/test_file_extraction_units.py::test_javascript_file_extraction PASSED [ 4%] +tests/test_file_extraction_units.py::test_typescript_file_extraction PASSED [ 5%] +tests/test_file_extraction_units.py::test_shell_script_extraction PASSED [ 5%] +tests/test_file_extraction_units.py::TestConversationIds::test_thread_conversation_id_format PASSED [ 5%] +tests/test_file_extraction_units.py::TestConversationIds::test_dm_conversation_id_format PASSED [ 6%] +tests/test_file_extraction_units.py::TestConversationIds::test_mention_conversation_id_format PASSED [ 6%] +tests/test_logging_config.py::test_json_schema_and_redaction_variants PASSED [ 7%] +tests/test_logging_config.py::test_error_hash_not_stack PASSED [ 7%] +tests/test_logging_config.py::test_wide_event_one_per_request PASSED [ 7%] +tests/test_logging.py::test_json_stdout_preserves_controls_and_avoids_duplicate_handlers[wingman-backend] PASSED [ 8%] +tests/test_logging.py::test_json_stdout_preserves_controls_and_avoids_duplicate_handlers[wingman-bot] PASSED [ 8%] +tests/test_logging.py::test_text_format_and_configured_level PASSED [ 9%] +tests/test_logging.py::test_api_keeps_backend_service_when_importing_bot PASSED [ 9%] +tests/test_agent_runtime.py::test_transport_retry_keeps_request_id PASSED [ 10%] +tests/test_agent_runtime.py::test_rag_routes_retrieved_context_and_stable_slack_identity PASSED [ 10%] +tests/test_agent_runtime.py::test_failed_job_surfaces_error PASSED [ 10%] +tests/test_rules.py::TestMatching::test_contains_is_case_insensitive PASSED [ 11%] +tests/test_rules.py::TestMatching::test_contains_needs_the_keyword PASSED [ 11%] +tests/test_rules.py::TestMatching::test_any_keyword_can_match PASSED [ 12%] +tests/test_rules.py::TestMatching::test_exact_needs_the_whole_question PASSED [ 12%] +tests/test_rules.py::TestMatching::test_regex_match PASSED [ 12%] +tests/test_rules.py::TestMatching::test_blank_question_never_matches PASSED [ 13%] +tests/test_rules.py::TestMatching::test_no_rules_is_a_no_op PASSED [ 13%] +tests/test_rules.py::TestOrderingAndEnabled::test_higher_priority_wins PASSED [ 14%] +tests/test_rules.py::TestOrderingAndEnabled::test_config_order_breaks_priority_ties PASSED [ 14%] +tests/test_rules.py::TestOrderingAndEnabled::test_disabled_rule_never_matches PASSED [ 15%] +tests/test_rules.py::TestConfigLoading::test_parse_valid_rules PASSED [ 15%] +tests/test_rules.py::TestConfigLoading::test_parse_skips_invalid_entries PASSED [ 15%] +tests/test_rules.py::TestConfigLoading::test_parse_skips_bad_regex PASSED [ 16%] +tests/test_rules.py::TestConfigLoading::test_parse_rejects_non_list PASSED [ 16%] +tests/test_rules.py::TestConfigLoading::test_load_from_json_text PASSED [ 17%] +tests/test_rules.py::TestConfigLoading::test_load_invalid_json_is_empty PASSED [ 17%] +tests/test_rules.py::TestConfigLoading::test_load_from_file_and_file_wins PASSED [ 17%] +tests/test_rules.py::TestConfigLoading::test_load_missing_file_is_empty PASSED [ 18%] +tests/test_rules.py::TestConfigLoading::test_from_settings_reads_both_fields PASSED [ 18%] +tests/test_rules.py::TestRagShortCircuit::test_matched_question_skips_retrieval_and_generation PASSED [ 19%] +tests/test_rules.py::TestRagShortCircuit::test_unmatched_question_still_uses_generation PASSED [ 19%] +tests/test_rules.py::TestSlackShortCircuit::test_matched_slack_question_skips_retrieval_and_carries_pinned_answer PASSED [ 20%] +tests/test_rules.py::TestSlackShortCircuit::test_unmatched_slack_question_uses_retrieval_without_pinned_answer PASSED [ 20%] +tests/test_conversations.py::TestListConversations::test_lists_summaries_most_recent_first PASSED [ 20%] +tests/test_conversations.py::TestListConversations::test_summary_counts_and_preview PASSED [ 21%] +tests/test_conversations.py::TestListConversations::test_pagination PASSED [ 21%] +tests/test_conversations.py::TestListConversations::test_channel_filter PASSED [ 22%] +tests/test_conversations.py::TestListConversations::test_channel_filter_scopes_preview_for_shared_conversation[C1] PASSED [ 22%] +tests/test_conversations.py::TestListConversations::test_channel_filter_scopes_preview_for_shared_conversation[C2] PASSED [ 23%] +tests/test_conversations.py::TestListConversations::test_unfiltered_preview_includes_latest_turn_across_channels PASSED [ 23%] +tests/test_conversations.py::TestListConversations::test_scoped_preview_breaks_timestamp_ties_by_latest_id PASSED [ 23%] +tests/test_conversations.py::TestListConversations::test_invalid_pagination_rejected PASSED [ 24%] +tests/test_conversations.py::TestListConversations::test_empty_database PASSED [ 24%] +tests/test_conversations.py::TestConversationDetail::test_returns_turns_in_chronological_order PASSED [ 25%] +tests/test_conversations.py::TestConversationDetail::test_detail_pagination PASSED [ 25%] +tests/test_conversations.py::TestConversationDetail::test_missing_conversation_is_404 PASSED [ 25%] +tests/test_conversations.py::TestConversationDetail::test_detail_invalid_pagination_rejected PASSED [ 26%] +tests/test_documents.py::TestListDocuments::test_lists_documents_most_recent_first PASSED [ 26%] +tests/test_documents.py::TestListDocuments::test_preview_and_fields PASSED [ 27%] +tests/test_documents.py::TestListDocuments::test_pagination PASSED [ 27%] +tests/test_documents.py::TestListDocuments::test_source_filter PASSED [ 28%] +tests/test_documents.py::TestListDocuments::test_invalid_pagination_rejected PASSED [ 28%] +tests/test_documents.py::TestListDocuments::test_empty_database PASSED [ 28%] +tests/test_documents.py::TestSearchDocuments::test_matches_title_case_insensitively_and_trims_query[vpn] PASSED [ 29%] +tests/test_documents.py::TestSearchDocuments::test_matches_title_case_insensitively_and_trims_query[VpN] PASSED [ 29%] +tests/test_documents.py::TestSearchDocuments::test_matches_title_case_insensitively_and_trims_query[ \tVPN\n ] PASSED [ 30%] +tests/test_documents.py::TestSearchDocuments::test_matches_content_beyond_the_preview PASSED [ 30%] +tests/test_documents.py::TestSearchDocuments::test_filters_before_counting_and_paginating PASSED [ 30%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-%] PASSED [ 31%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-_] PASSED [ 31%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-/] PASSED [ 32%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-100%] PASSED [ 32%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-setup_notes] PASSED [ 33%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-a/b] PASSED [ 33%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-' OR 1=1 --] PASSED [ 33%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-%] PASSED [ 34%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-_] PASSED [ 34%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-/] PASSED [ 35%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-100%] PASSED [ 35%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-setup_notes] PASSED [ 35%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-a/b] PASSED [ 36%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-' OR 1=1 --] PASSED [ 36%] +tests/test_documents.py::TestSearchDocuments::test_blank_query_preserves_source_filter_and_existing_list[] PASSED [ 37%] +tests/test_documents.py::TestSearchDocuments::test_blank_query_preserves_source_filter_and_existing_list[ \t\n ] PASSED [ 37%] +tests/test_documents.py::TestSearchDocuments::test_blank_query_preserves_source_filter_and_existing_list[\u2003\xa0] PASSED [ 38%] +tests/test_documents.py::TestSearchDocuments::test_no_matches_return_an_empty_page[params0] PASSED [ 38%] +tests/test_documents.py::TestSearchDocuments::test_no_matches_return_an_empty_page[params1] PASSED [ 38%] +tests/test_documents.py::TestDocumentDetail::test_returns_full_content PASSED [ 39%] +tests/test_documents.py::TestDocumentDetail::test_missing_document_is_404 PASSED [ 39%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[-title] PASSED [ 40%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[-content] PASSED [ 40%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[-source] PASSED [ 41%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[ \n\t -title] PASSED [ 41%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[ \n\t -content] PASSED [ 41%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[ \n\t -source] PASSED [ 42%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[\u2003\xa0-title] PASSED [ 42%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[\u2003\xa0-content] PASSED [ 43%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[\u2003\xa0-source] PASSED [ 43%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[None-title] PASSED [ 43%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[None-content] PASSED [ 44%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[None-source] PASSED [ 44%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[123-title] PASSED [ 45%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[123-content] PASSED [ 45%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[123-source] PASSED [ 46%] +tests/test_documents.py::TestCreateDocument::test_rejects_missing_required_fields[title] PASSED [ 46%] +tests/test_documents.py::TestCreateDocument::test_rejects_missing_required_fields[content] PASSED [ 46%] +tests/test_documents.py::TestCreateDocument::test_creates_readable_document_and_indexes_preserved_content[None] PASSED [ 47%] +tests/test_documents.py::TestCreateDocument::test_creates_readable_document_and_indexes_preserved_content[ \tdocs\n ] PASSED [ 47%] +tests/test_retrieval.py::test_invalid_chunk_settings_are_rejected[config0] PASSED [ 48%] +tests/test_retrieval.py::test_invalid_chunk_settings_are_rejected[config1] PASSED [ 48%] +tests/test_retrieval.py::test_invalid_chunk_settings_are_rejected[config2] PASSED [ 48%] +tests/test_retrieval.py::test_invalid_chunk_settings_are_rejected[config3] PASSED [ 49%] +tests/test_retrieval.py::test_invalid_chunk_settings_are_rejected[config4] PASSED [ 49%] +tests/test_retrieval.py::test_valid_chunk_settings_are_accepted[1000-200] PASSED [ 50%] +tests/test_retrieval.py::test_valid_chunk_settings_are_accepted[1-0] PASSED [ 50%] +tests/test_retrieval.py::test_valid_chunk_settings_are_accepted[10-0] PASSED [ 51%] +tests/test_retrieval.py::test_valid_chunk_settings_are_accepted[10-9] PASSED [ 51%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[-5-2-expected0] PASSED [ 51%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[a-5-2-expected1] PASSED [ 52%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcd-5-2-expected2] PASSED [ 52%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcde-5-2-expected3] PASSED [ 53%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcdef-5-2-expected4] PASSED [ 53%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcdefgh-5-2-expected5] PASSED [ 53%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcdefghi-5-2-expected6] PASSED [ 54%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcdefghij-5-0-expected7] PASSED [ 54%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcdef-5-4-expected8] PASSED [ 55%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abc-1-0-expected9] PASSED [ 55%] +tests/test_retrieval.py::test_document_indexing_avoids_redundant_tail_chunks PASSED [ 56%] +tests/test_retrieval.py::test_repeated_thread_indexing_preserves_distinct_top_k PASSED [ 56%] +tests/test_retrieval.py::test_thread_reindex_updates_content_and_embedding PASSED [ 56%] +tests/test_retrieval.py::test_thread_chunk_ids_are_scoped_by_channel PASSED [ 57%] +tests/test_retrieval.py::test_thread_indexing_omits_absent_optional_metadata PASSED [ 57%] +tests/test_retrieval.py::test_thread_indexing_uses_last_duplicate_in_batch PASSED [ 58%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages0] PASSED [ 58%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages1] PASSED [ 58%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages2] PASSED [ 59%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages3] PASSED [ 59%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages4] PASSED [ 60%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages5] PASSED [ 60%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages6] PASSED [ 61%] +tests/test_retrieval.py::test_search_empty_collection_returns_no_matches[None-None] PASSED [ 61%] +tests/test_retrieval.py::test_search_empty_collection_returns_no_matches[None-CTARGET] PASSED [ 61%] +tests/test_retrieval.py::test_search_empty_collection_returns_no_matches[1.0-None] PASSED [ 62%] +tests/test_retrieval.py::test_search_empty_collection_returns_no_matches[1.0-CTARGET] PASSED [ 62%] +tests/test_retrieval.py::test_first_document_can_be_indexed_and_retrieved PASSED [ 63%] +tests/test_retrieval.py::test_missing_adapter_still_raises[add] PASSED [ 63%] +tests/test_retrieval.py::test_missing_adapter_still_raises[search] PASSED [ 64%] +tests/test_retrieval.py::test_channel_filter_runs_before_top_k PASSED [ 64%] +tests/test_retrieval.py::test_unscoped_search_preserves_global_ranking_and_explicit_k PASSED [ 64%] +tests/test_retrieval.py::test_distance_cutoff_is_inclusive_and_preserves_channel_scope PASSED [ 65%] +tests/test_retrieval.py::test_zero_distance_cutoff_keeps_only_exact_vector_matches PASSED [ 65%] +tests/test_retrieval.py::test_invalid_distance_config_is_rejected[-1] PASSED [ 66%] +tests/test_retrieval.py::test_invalid_distance_config_is_rejected[nan] PASSED [ 66%] +tests/test_retrieval.py::test_invalid_distance_config_is_rejected[inf] PASSED [ 66%] +tests/test_retrieval.py::test_invalid_distance_config_is_rejected[-inf] PASSED [ 67%] +tests/test_retrieval.py::test_invalid_distance_config_is_rejected[invalid] PASSED [ 67%] +tests/test_retrieval.py::test_distance_config_accepts_unset_or_nonnegative_finite_values[None] PASSED [ 68%] +tests/test_retrieval.py::test_distance_config_accepts_unset_or_nonnegative_finite_values[0] PASSED [ 68%] +tests/test_retrieval.py::test_distance_config_accepts_unset_or_nonnegative_finite_values[1.5] PASSED [ 69%] +tests/test_retrieval.py::test_empty_channel_does_not_fall_back_to_global_results PASSED [ 69%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[None-2-api] PASSED [ 69%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[None-2-slack] PASSED [ 70%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[1.0-1-api] PASSED [ 70%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[1.0-1-slack] PASSED [ 71%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[0.0-0-api] PASSED [ 71%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[0.0-0-slack] PASSED [ 71%] +tests/test_thread_indexing.py::test_saved_thread_includes_root_and_replies_in_timestamp_order PASSED [ 72%] +tests/test_thread_indexing.py::test_saved_root_only_thread_is_indexed_once[None] PASSED [ 72%] +tests/test_thread_indexing.py::test_saved_root_only_thread_is_indexed_once[1700000000.000001] PASSED [ 73%] +tests/test_thread_indexing.py::test_saved_replies_can_be_indexed_without_a_saved_root PASSED [ 73%] +tests/test_thread_indexing.py::test_missing_saved_thread_returns_404_without_indexing[CTARGET-1700000098.000001] PASSED [ 74%] +tests/test_thread_indexing.py::test_missing_saved_thread_returns_404_without_indexing[CEMPTY-1700000000.000001] PASSED [ 74%] +tests/test_thread_indexing.py::test_saved_thread_channel_filter_also_applies_to_root_lookup PASSED [ 74%] +tests/test_thread_indexing.py::test_saved_thread_indexing_failure_remains_500 PASSED [ 75%] +tests/test_thread_indexing.py::test_saved_thread_retrieval_includes_original_question PASSED [ 75%] +tests/test_retrieval_preview.py::test_preview_returns_ranked_chunks_metadata_distances_and_settings PASSED [ 76%] +tests/test_retrieval_preview.py::test_preview_applies_channel_scope_before_top_k PASSED [ 76%] +tests/test_retrieval_preview.py::test_preview_exposes_inclusive_distance_cutoff_with_transcript PASSED [ 76%] +tests/test_retrieval_preview.py::test_preview_returns_success_for_empty_results[collection] PASSED [ 77%] +tests/test_retrieval_preview.py::test_preview_returns_success_for_empty_results[channel] PASSED [ 77%] +tests/test_retrieval_preview.py::test_preview_returns_success_for_empty_results[cutoff] PASSED [ 78%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload0] PASSED [ 78%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload1] PASSED [ 79%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload2] PASSED [ 79%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload3] PASSED [ 79%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload4] PASSED [ 80%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload5] PASSED [ 80%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload6] PASSED [ 81%] +tests/test_retrieval_preview.py::test_preview_trims_question_and_channel PASSED [ 81%] +tests/test_retrieval_preview.py::test_preview_reports_unavailable_retrieval_without_exposing_error PASSED [ 82%] +tests/test_document_indexing.py::test_create_and_retry_keep_one_owned_chunk_and_preserve_saved_record PASSED [ 82%] +tests/test_document_indexing.py::test_retry_recovers_saved_document_after_embedding_failure_with_transcript PASSED [ 82%] +tests/test_document_indexing.py::test_retry_rechunks_and_removes_only_obsolete_owned_chunks PASSED [ 83%] +tests/test_document_indexing.py::test_retry_reads_latest_saved_content_title_and_source PASSED [ 83%] +tests/test_document_indexing.py::test_retry_failure_keeps_previous_chunks_and_saved_content[lookup] PASSED [ 84%] +tests/test_document_indexing.py::test_retry_failure_keeps_previous_chunks_and_saved_content[embed] PASSED [ 84%] +tests/test_document_indexing.py::test_retry_failure_keeps_previous_chunks_and_saved_content[upsert] PASSED [ 84%] +tests/test_document_indexing.py::test_cleanup_failure_is_reported_and_retry_finishes_without_duplicates PASSED [ 85%] +tests/test_document_indexing.py::test_missing_saved_document_does_not_initialise_vector_store PASSED [ 85%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[content- \n\t ] PASSED [ 86%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[content-None] PASSED [ 86%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[title-] PASSED [ 87%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[source-None] PASSED [ 87%] +tests/test_document_deletion.py::test_delete_removes_only_owned_chunks_with_retrieval_transcript PASSED [ 87%] +tests/test_document_deletion.py::test_missing_or_invalid_delete_does_not_initialise_vector_store[999-404] PASSED [ 88%] +tests/test_document_deletion.py::test_missing_or_invalid_delete_does_not_initialise_vector_store[-1-404] PASSED [ 88%] +tests/test_document_deletion.py::test_missing_or_invalid_delete_does_not_initialise_vector_store[invalid-422] PASSED [ 89%] +tests/test_document_deletion.py::test_missing_or_invalid_delete_does_not_initialise_vector_store[1.5-422] PASSED [ 89%] +tests/test_document_deletion.py::test_delete_unindexed_legacy_record_needs_no_valid_content_or_embeddings PASSED [ 89%] +tests/test_document_deletion.py::test_vector_delete_failure_retains_saved_record_and_allows_retry[initialise] PASSED [ 90%] +tests/test_document_deletion.py::test_vector_delete_failure_retains_saved_record_and_allows_retry[delete] PASSED [ 90%] +tests/test_document_deletion.py::test_vector_delete_failure_retains_saved_record_and_allows_retry[lost-acknowledgement] PASSED [ 91%] +tests/test_document_deletion.py::test_database_delete_failure_rolls_back_saved_record_and_can_retry[delete] PASSED [ 91%] +tests/test_document_deletion.py::test_database_delete_failure_rolls_back_saved_record_and_can_retry[commit] PASSED [ 92%] +tests/test_document_deletion.py::test_delete_still_requires_an_initialised_adapter PASSED [ 92%] +tests/test_rule_preview.py::test_preview_returns_winning_rule_and_exact_response[contains-keywords0-How do I RESET VPN access?-reset vpn] PASSED [ 92%] +tests/test_rule_preview.py::test_preview_returns_winning_rule_and_exact_response[exact-keywords1-OFFICE HOURS-office hours] PASSED [ 93%] +tests/test_rule_preview.py::test_preview_returns_winning_rule_and_exact_response[regex-keywords2-Where is TICKET #4821?-ticket #?\\d+] PASSED [ 93%] +tests/test_rule_preview.py::test_preview_uses_priority_config_order_and_disabled_rule_policy PASSED [ 94%] +tests/test_rule_preview.py::test_preview_returns_null_for_no_match_without_falling_back_to_generation[entries0] PASSED [ 94%] +tests/test_rule_preview.py::test_preview_returns_null_for_no_match_without_falling_back_to_generation[entries1] PASSED [ 94%] +tests/test_rule_preview.py::test_preview_returns_null_for_no_match_without_falling_back_to_generation[entries2] PASSED [ 95%] +tests/test_rule_preview.py::test_preview_trims_question_and_uses_current_shared_matcher PASSED [ 95%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload0] PASSED [ 96%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload1] PASSED [ 96%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload2] PASSED [ 97%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload3] PASSED [ 97%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload4] PASSED [ 97%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload5] PASSED [ 98%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload6] PASSED [ 98%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload7] PASSED [ 99%] +tests/test_rule_preview.py::test_preview_matches_api_answer_and_slack_job_without_submitting_a_job PASSED [ 99%] +tests/test_rule_preview.py::test_preview_reports_matcher_failure_without_exposing_details PASSED [100%] + +=============================== warnings summary =============================== +app/main.py:114 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/app/main.py:114: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.12/migration/ + class MessageResponse(BaseModel): + +app/main.py:141 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/app/main.py:141: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.12/migration/ + class ConversationTurn(BaseModel): + +app/main.py:178 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/app/main.py:178: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.12/migration/ + class DocumentDetail(BaseModel): + +app/main.py:191 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/app/main.py:191: DeprecationWarning: + on_event is deprecated, use lifespan event handlers instead. + + Read more about it in the + [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/). + + @app.on_event("startup") + +.venv/lib/python3.13/site-packages/fastapi/applications.py:4576 +.venv/lib/python3.13/site-packages/fastapi/applications.py:4576 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/.venv/lib/python3.13/site-packages/fastapi/applications.py:4576: DeprecationWarning: + on_event is deprecated, use lifespan event handlers instead. + + Read more about it in the + [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/). + + return self.router.on_event(event_type) + +app/main.py:204 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/app/main.py:204: DeprecationWarning: + on_event is deprecated, use lifespan event handlers instead. + + Read more about it in the + [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/). + + @app.on_event("shutdown") + +tests/test_documents.py: 4 warnings +tests/test_thread_indexing.py: 92 warnings +tests/test_document_indexing.py: 31 warnings +tests/test_document_deletion.py: 16 warnings + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/.venv/lib/python3.13/site-packages/sqlalchemy/sql/schema.py:3624: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + return util.wrap_callable(lambda ctx: fn(), fn) # type: ignore + +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[-5-2-expected0] + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/tests/test_retrieval.py:122: LangChainDeprecationWarning: The class `Chroma` was deprecated in LangChain 0.2.9 and will be removed in 1.0. An updated version of the class exists in the `langchain-chroma package and should be used instead. To use it run `pip install -U `langchain-chroma` and import as `from `langchain_chroma import Chroma``. + store.vector_store = Chroma( + +tests/test_retrieval.py: 384 warnings +tests/test_thread_indexing.py: 27 warnings +tests/test_retrieval_preview.py: 216 warnings +tests/test_document_indexing.py: 166 warnings +tests/test_document_deletion.py: 89 warnings + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/.venv/lib/python3.13/site-packages/chromadb/types.py:144: PydanticDeprecatedSince211: Accessing the 'model_fields' attribute on the instance is deprecated. Instead, you should access this attribute from the model class. Deprecated in Pydantic V2.11 to be removed in V3.0. + return self.model_fields # pydantic 2.x + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +====================== 239 passed, 1033 warnings in 6.31s ====================== diff --git a/.tmp/proof/rule-preview-capture.py b/.tmp/proof/rule-preview-capture.py new file mode 100644 index 0000000..3edcd10 --- /dev/null +++ b/.tmp/proof/rule-preview-capture.py @@ -0,0 +1,42 @@ +"""Capture curl requests against the offline backend started on port 18059.""" +import json +from pathlib import Path +import shlex +import subprocess + + +base = "http://127.0.0.1:18059" +transcript = [ + "Offline operator-rule preview with rule-preview-config.json.", + "Slack API, Socket Mode, retrieval and Pi calls raise in the proof process.", + "No chunks or distances apply: this endpoint only inspects loaded rules.", +] + + +def request(path, payload=None, status=200): + command = ["curl", "--silent", "--show-error", "--include", "--max-time", "10", f"{base}{path}"] + if payload is not None: + command += ["-H", "Content-Type: application/json", "-d", json.dumps(payload)] + result = subprocess.run(command, check=True, capture_output=True, text=True) + headers, body = result.stdout.split("\n\n", 1) + assert f" {status} " in headers.splitlines()[0], result.stdout + data = json.loads(body) + transcript.extend(["", "$ " + shlex.join(command), headers, json.dumps(data, indent=2)]) + return data + + +assert request("/health") == {"status": "healthy"} +winning = request("/api/rules/preview", {"question": " How do I RESET VPN access?\n"}) +assert winning["question"] == "How do I RESET VPN access?" +assert winning["match"]["rule_id"] == "vpn-reset" +assert winning["match"]["priority"] == 10 +assert request("/api/rules/preview", {"question": "OFFICE HOURS"})["match"]["match_type"] == "exact" +assert request("/api/rules/preview", {"question": "Where is TICKET #4821?"})["match"]["match_type"] == "regex" +assert request("/api/rules/preview", {"question": "How do I reset email?"})["match"] is None +request("/api/rules/preview", {"question": " \n\t "}, status=422) +request("/api/rules/preview", {"question": 123}, status=422) +answer = request("/api/ask", {"question": "How do I RESET VPN access?"}) +assert answer["answer"] == winning["match"]["response"] +assert answer["sources"] == [{"source": "operator_rule", "rule_id": winning["match"]["rule_id"]}] +transcript.extend(["", "All response assertions passed. Preview and API answer select the same rule."]) +Path(__file__).with_name("rule-preview-api.txt").write_text("\n".join(transcript) + "\n") diff --git a/.tmp/proof/rule-preview-config.json b/.tmp/proof/rule-preview-config.json new file mode 100644 index 0000000..4a7ff35 --- /dev/null +++ b/.tmp/proof/rule-preview-config.json @@ -0,0 +1,33 @@ +[ + { + "id": "vpn-low", + "keywords": ["reset vpn"], + "response": "A lower priority answer that should not win.", + "priority": 1 + }, + { + "id": "vpn-disabled", + "keywords": ["reset vpn"], + "response": "A disabled answer that should not win.", + "priority": 100, + "enabled": false + }, + { + "id": "vpn-reset", + "keywords": ["reset vpn", "vpn password"], + "response": "Use the IT portal at https://it.example.com/vpn to reset VPN access.", + "priority": 10 + }, + { + "id": "office-hours", + "keywords": ["office hours"], + "match_type": "exact", + "response": "The support desk runs 9am to 5pm, Monday to Friday." + }, + { + "id": "ticket", + "keywords": ["ticket #?\\d+"], + "match_type": "regex", + "response": "Look up the ticket in the support portal." + } +] diff --git a/.tmp/proof/rule-preview-frontend-tests.txt b/.tmp/proof/rule-preview-frontend-tests.txt new file mode 100644 index 0000000..30ff65f --- /dev/null +++ b/.tmp/proof/rule-preview-frontend-tests.txt @@ -0,0 +1,69 @@ +[test-frontend] $ bun run test +$ bun test --preload ./tests/setup.ts +bun test v1.3.5 (1e86cebd) + +tests/documents.test.tsx: +(pass) documents load on opening the tab, search only on submit, and retain the applied query while paging [176.30ms] +(pass) document client encodes literal search text, keeps source filters, and omits blank queries [0.17ms] +(pass) documents show loading, retry the submitted query after a list error, and distinguish empty searches [15.00ms] +(pass) documents render saved content as text and retry detail failures without losing results [22.97ms] +(pass) documents ignore stale successful and failed list responses after a newer search [9.05ms] +(pass) documents ignore late details after another selection, a page change, and clearing a search [35.51ms] +(pass) documents abort requests on leaving the tab and ignore late responses when reopened [10.11ms] + +tests/document-deletion.test.tsx: +(pass) deletion requires confirmation of the selected document and cancellation sends no mutation [18.01ms] +(pass) confirmed deletion handles HTTP %s by refreshing page one of the applied search [40.65ms] +(pass) confirmed deletion handles HTTP %s by refreshing page one of the applied search [37.14ms] +(pass) deletion reports an uncertain outcome after 503 and retries the same ID [22.40ms] +(pass) deletion reports an uncertain outcome after 403 and retries the same ID [24.72ms] +(pass) deletion reports an uncertain outcome after network and retries the same ID [25.24ms] +(pass) a deletion failure can be cancelled and re-indexing blocks deletion until it finishes [23.87ms] +(pass) a failed list refresh after deletion offers a list retry without repeating DELETE [19.81ms] +(pass) deletion ignores late success after a selection change [21.66ms] +(pass) deletion ignores late failure after a selection change [22.85ms] +(pass) deletion ignores late success after a search change [25.52ms] +(pass) deletion ignores late failure after a search change [31.68ms] +(pass) deletion ignores late success after a page change [25.23ms] +(pass) deletion ignores late failure after a page change [21.51ms] +(pass) deletion ignores late success after a tab change [23.88ms] +(pass) deletion ignores late failure after a tab change [23.96ms] + +tests/document-indexing.test.tsx: +(pass) re-indexing requires an explicit click, posts the saved ID once while pending, and keeps the content visible [15.98ms] +(pass) re-indexing reports a safe failure for 404 and retries the same saved document [18.27ms] +(pass) re-indexing reports a safe failure for 422 and retries the same saved document [22.28ms] +(pass) re-indexing reports a safe failure for 503 and retries the same saved document [32.95ms] +(pass) re-indexing reports a safe failure for network and retries the same saved document [21.25ms] +(pass) re-indexing ignores a late success after a selection change [25.17ms] +(pass) re-indexing ignores a late failure after a selection change [24.07ms] +(pass) re-indexing ignores a late success after a search change [31.50ms] +(pass) re-indexing ignores a late failure after a search change [28.36ms] +(pass) re-indexing ignores a late success after a page change [27.22ms] +(pass) re-indexing ignores a late failure after a page change [26.08ms] +(pass) re-indexing ignores a late success after a tab change [30.83ms] +(pass) re-indexing ignores a late failure after a tab change [30.03ms] + +tests/retrieval.test.tsx: +(pass) retrieval waits for a non-blank question and submits trimmed optional scope [12.25ms] +(pass) retrieval omits blank scope, preserves rank and raw distances, and shows an unset cutoff [9.16ms] +(pass) retrieval disables inputs and duplicate submissions while loading [8.03ms] +(pass) retrieval shows an empty result and a zero distance cutoff [5.85ms] +(pass) retrieval replaces a failed request with a successful retry [21.22ms] +(pass) retrieval clears old results when the question or channel changes [24.66ms] +(pass) retrieval aborts on leaving the tab and ignores late responses after reopening [13.91ms] +(pass) retrieval renders chunk and metadata markup as text [14.73ms] + +tests/conversations.test.tsx: +(pass) loads only on opening the tab and paginates summaries and chronological turns independently [135.73ms] +(pass) shows loading, an empty page and a list error with a working retry [8.90ms] +(pass) retries missing conversation details without discarding the list [22.70ms] +(pass) ignores stale detail responses after selecting another conversation [20.42ms] +(pass) ignores stale errors after leaving and reopening the tab [7.24ms] +(pass) renders stored markup as text and supports missing metadata [8.04ms] +(pass) encodes conversation IDs in detail request paths [1.58ms] + + 51 pass + 0 fail + 370 expect() calls +Ran 51 tests across 5 files. [1.78s] diff --git a/.tmp/proof/rule-preview-install.txt b/.tmp/proof/rule-preview-install.txt new file mode 100644 index 0000000..db18bb1 --- /dev/null +++ b/.tmp/proof/rule-preview-install.txt @@ -0,0 +1,182 @@ +[install-backend] $ uv sync +[install-frontend] $ bun install +[install-runtime] $ bun install --frozen-lockfile --ignore-scripts +[install-frontend] bun install v1.3.5 (1e86cebd) +[install-runtime] bun install v1.3.5 (1e86cebd) +[install-backend] Using CPython 3.13.13 +[install-backend] Creating virtual environment at: .venv +[install-runtime] +[install-runtime] + @earendil-works/chord@1.3.0 +[install-runtime] + @earendil-works/pi-ai@1.3.0 +[install-runtime] + @earendil-works/pi-durable@1.3.0 +[install-runtime] +[install-runtime] 91 packages installed [1075.00ms] +[install-runtime] Finished in 1.14s +[install-backend] Resolved 143 packages in 1.47s +[install-backend] warning: `numpy==2.4.0` is yanked (reason: "Backward compatibility bug") +[install-backend] Building wingman @ file:///Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend +[install-frontend] +[install-frontend] + @tailwindcss/postcss@4.1.18 +[install-frontend] + @testing-library/react@16.3.3 +[install-frontend] + @types/bun@1.4.4 +[install-frontend] + eslint@9.39.2 +[install-frontend] + eslint-config-next@16.1.1 +[install-frontend] + happy-dom@21.0.4 +[install-frontend] + tailwindcss@4.1.18 +[install-frontend] + @types/node@25.0.3 +[install-frontend] + @types/react@19.2.7 +[install-frontend] + @types/react-dom@19.2.3 +[install-frontend] + axios@1.13.2 +[install-frontend] + next@16.1.1 +[install-frontend] + react@19.2.3 +[install-frontend] + react-dom@19.2.3 +[install-frontend] + swr@2.3.8 +[install-frontend] + typescript@5.9.3 +[install-frontend] +[install-frontend] 382 packages installed [1.72s] +[install-frontend] Finished in 1.78s +[install-backend] Built wingman @ file:///Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend +[install-backend] Prepared 1 package in 401ms +[install-backend] Installed 140 packages in 727ms +[install-backend] + aiohappyeyeballs==2.6.1 +[install-backend] + aiohttp==3.13.2 +[install-backend] + aiosignal==1.4.0 +[install-backend] + alembic==1.17.2 +[install-backend] + annotated-doc==0.0.4 +[install-backend] + annotated-types==0.7.0 +[install-backend] + anyio==4.12.0 +[install-backend] + asgiref==3.11.0 +[install-backend] + attrs==25.4.0 +[install-backend] + backoff==2.2.1 +[install-backend] + bcrypt==5.0.0 +[install-backend] + build==1.3.0 +[install-backend] + cachetools==6.2.4 +[install-backend] + certifi==2025.11.12 +[install-backend] + charset-normalizer==3.4.4 +[install-backend] + chroma-hnswlib==0.7.6 +[install-backend] + chromadb==0.6.3 +[install-backend] + click==8.3.1 +[install-backend] + coloredlogs==15.0.1 +[install-backend] + dataclasses-json==0.6.7 +[install-backend] + distro==1.9.0 +[install-backend] + durationpy==0.10 +[install-backend] + fastapi==0.128.0 +[install-backend] + filelock==3.20.1 +[install-backend] + flatbuffers==25.12.19 +[install-backend] + frozenlist==1.8.0 +[install-backend] + fsspec==2025.12.0 +[install-backend] + google-auth==2.45.0 +[install-backend] + googleapis-common-protos==1.72.0 +[install-backend] + grpcio==1.76.0 +[install-backend] + h11==0.16.0 +[install-backend] + hf-xet==1.2.0 +[install-backend] + httpcore==1.0.9 +[install-backend] + httptools==0.7.1 +[install-backend] + httpx==0.28.1 +[install-backend] + httpx-sse==0.4.3 +[install-backend] + huggingface-hub==0.36.0 +[install-backend] + humanfriendly==10.0 +[install-backend] + idna==3.11 +[install-backend] + importlib-metadata==8.7.1 +[install-backend] + importlib-resources==6.5.2 +[install-backend] + iniconfig==2.3.0 +[install-backend] + jiter==0.12.0 +[install-backend] + jsonpatch==1.33 +[install-backend] + jsonpointer==3.0.0 +[install-backend] + kubernetes==34.1.0 +[install-backend] + langchain==1.2.0 +[install-backend] + langchain-classic==1.0.1 +[install-backend] + langchain-community==0.4.1 +[install-backend] + langchain-core==1.2.5 +[install-backend] + langchain-openai==1.1.6 +[install-backend] + langchain-text-splitters==1.1.0 +[install-backend] + langgraph==1.0.5 +[install-backend] + langgraph-checkpoint==3.0.1 +[install-backend] + langgraph-prebuilt==1.0.5 +[install-backend] + langgraph-sdk==0.3.1 +[install-backend] + langsmith==0.5.2 +[install-backend] + lxml==6.0.2 +[install-backend] + mako==1.3.10 +[install-backend] + markdown-it-py==4.0.0 +[install-backend] + markupsafe==3.0.3 +[install-backend] + marshmallow==3.26.2 +[install-backend] + mdurl==0.1.2 +[install-backend] + mmh3==5.2.0 +[install-backend] + mpmath==1.3.0 +[install-backend] + multidict==6.7.0 +[install-backend] + mypy-extensions==1.1.0 +[install-backend] + numpy==2.4.0 +[install-backend] + oauthlib==3.3.1 +[install-backend] + onnxruntime==1.23.2 +[install-backend] + openai==2.14.0 +[install-backend] + opentelemetry-api==1.39.1 +[install-backend] + opentelemetry-exporter-otlp-proto-common==1.39.1 +[install-backend] + opentelemetry-exporter-otlp-proto-grpc==1.39.1 +[install-backend] + opentelemetry-instrumentation==0.60b1 +[install-backend] + opentelemetry-instrumentation-asgi==0.60b1 +[install-backend] + opentelemetry-instrumentation-fastapi==0.60b1 +[install-backend] + opentelemetry-proto==1.39.1 +[install-backend] + opentelemetry-sdk==1.39.1 +[install-backend] + opentelemetry-semantic-conventions==0.60b1 +[install-backend] + opentelemetry-util-http==0.60b1 +[install-backend] + orjson==3.11.5 +[install-backend] + ormsgpack==1.12.1 +[install-backend] + overrides==7.7.0 +[install-backend] + packaging==25.0 +[install-backend] + pluggy==1.6.0 +[install-backend] + posthog==7.4.2 +[install-backend] + propcache==0.4.1 +[install-backend] + protobuf==6.33.2 +[install-backend] + psycopg2-binary==2.9.11 +[install-backend] + pyasn1==0.6.1 +[install-backend] + pyasn1-modules==0.4.2 +[install-backend] + pydantic==2.12.5 +[install-backend] + pydantic-core==2.41.5 +[install-backend] + pydantic-settings==2.12.0 +[install-backend] + pygments==2.19.2 +[install-backend] + pypdf==6.5.0 +[install-backend] + pypika==0.48.9 +[install-backend] + pyproject-hooks==1.2.0 +[install-backend] + pytest==9.0.2 +[install-backend] + pytest-asyncio==1.3.0 +[install-backend] + python-dateutil==2.9.0.post0 +[install-backend] + python-docx==1.2.0 +[install-backend] + python-dotenv==1.2.1 +[install-backend] + pyyaml==6.0.3 +[install-backend] + regex==2025.11.3 +[install-backend] + requests==2.32.5 +[install-backend] + requests-oauthlib==2.0.0 +[install-backend] + requests-toolbelt==1.0.0 +[install-backend] + rich==14.2.0 +[install-backend] + rsa==4.9.1 +[install-backend] + shellingham==1.5.4 +[install-backend] + six==1.17.0 +[install-backend] + slack-bolt==1.27.0 +[install-backend] + slack-sdk==3.42.0 +[install-backend] + sniffio==1.3.1 +[install-backend] + sqlalchemy==2.0.45 +[install-backend] + starlette==0.50.0 +[install-backend] + sympy==1.14.0 +[install-backend] + tenacity==9.1.2 +[install-backend] + tiktoken==0.12.0 +[install-backend] + tokenizers==0.22.1 +[install-backend] + tqdm==4.67.1 +[install-backend] + typer==0.21.0 +[install-backend] + typing-extensions==4.15.0 +[install-backend] + typing-inspect==0.9.0 +[install-backend] + typing-inspection==0.4.2 +[install-backend] + urllib3==2.3.0 +[install-backend] + uuid-utils==0.12.0 +[install-backend] + uvicorn==0.40.0 +[install-backend] + uvloop==0.22.1 +[install-backend] + watchfiles==1.1.1 +[install-backend] + websocket-client==1.9.0 +[install-backend] + websockets==15.0.1 +[install-backend] + wingman==1.1.0 (from file:///Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend) +[install-backend] + wrapt==1.17.3 +[install-backend] + xxhash==3.6.0 +[install-backend] + yarl==1.22.0 +[install-backend] + zipp==3.23.0 +[install-backend] + zstandard==0.25.0 +[install-backend] Finished in 2.89s +Finished in 2.90s diff --git a/.tmp/proof/rule-preview-runtime-tests.txt b/.tmp/proof/rule-preview-runtime-tests.txt new file mode 100644 index 0000000..474c51a --- /dev/null +++ b/.tmp/proof/rule-preview-runtime-tests.txt @@ -0,0 +1,16 @@ +[test-runtime] $ node --test +✔ real Pi SQLite storage preserves answers, deduplicates retries, and retains conversation history (36.818625ms) +✔ outbox resumes after restart without regenerating answer; retry admits only one job (4.123667ms) +✔ a process killed during generation resumes the admitted Pi submission (211.4125ms) +✔ pinned operator answer is delivered without calling the model (1.618084ms) +✔ model failure sends a persisted Slack failure notification (0.360125ms) +✔ runtime starts with openai catalog and protects job routes (645.452209ms) +✔ runtime starts with mimo catalog and protects job routes (466.165042ms) +ℹ tests 7 +ℹ suites 0 +ℹ pass 7 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms 1204.200291 diff --git a/.tmp/proof/rule-preview-sitecustomize.py b/.tmp/proof/rule-preview-sitecustomize.py new file mode 100644 index 0000000..14652fa --- /dev/null +++ b/.tmp/proof/rule-preview-sitecustomize.py @@ -0,0 +1,21 @@ +"""Offline proof hook: fixture Slack authentication and deny external work.""" +from slack_sdk import WebClient +from slack_bolt.adapter.socket_mode import SocketModeHandler + + +def blocked(*args, **kwargs): + raise AssertionError("Slack, retrieval and Pi calls are forbidden in this proof") + + +WebClient.auth_test = lambda *args, **kwargs: { + "ok": True, "bot_id": "BTEST", "user_id": "UTEST", "team_id": "TTEST", +} +WebClient.api_call = blocked +SocketModeHandler.start = blocked + +from app import agent_runtime +from app.vector_store import vector_store + +agent_runtime.generate = blocked +agent_runtime.submit = blocked +vector_store.similarity_search = blocked diff --git a/backend/app/main.py b/backend/app/main.py index 49c311f..890863e 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -61,6 +61,23 @@ class QuestionResponse(BaseModel): NonBlankString = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] +class OperatorRulePreviewRequest(BaseModel): + question: NonBlankString + + +class OperatorRuleMatch(BaseModel): + rule_id: str + match_type: str + matched_keyword: str + priority: int + response: str + + +class OperatorRulePreviewResponse(BaseModel): + question: str + match: OperatorRuleMatch | None + + class RetrievalPreviewRequest(BaseModel): question: NonBlankString channel_id: NonBlankString | None = None @@ -226,6 +243,26 @@ async def ask_question(request: QuestionRequest): raise HTTPException(status_code=500, detail=str(e)) +@app.post("/api/rules/preview", response_model=OperatorRulePreviewResponse) +def preview_operator_rule(request: OperatorRulePreviewRequest): + """Inspect the loaded rule winner without retrieval, generation or delivery.""" + try: + matched = rag_engine.match_operator_rule(request.question) + return OperatorRulePreviewResponse( + question=request.question, + match=OperatorRuleMatch( + rule_id=matched.rule.id, + match_type=matched.rule.match_type, + matched_keyword=matched.matched_keyword, + priority=matched.rule.priority, + response=matched.rule.response, + ) if matched is not None else None, + ) + except Exception: + logger.exception("Operator rule preview failed") + raise HTTPException(status_code=503, detail="Operator rule preview is unavailable") + + @app.post("/api/retrieval/preview", response_model=RetrievalPreviewResponse) def preview_retrieval(request: RetrievalPreviewRequest): """Inspect selected chunks and distances without generating an answer.""" diff --git a/backend/tests/test_rule_preview.py b/backend/tests/test_rule_preview.py new file mode 100644 index 0000000..aa348f8 --- /dev/null +++ b/backend/tests/test_rule_preview.py @@ -0,0 +1,125 @@ +"""Operator rule preview tests with fixture config and blocked external work.""" +import os +from unittest.mock import Mock, patch + +import pytest +from fastapi.testclient import TestClient + +os.environ.setdefault("SLACK_BOT_TOKEN", "xoxb-fixture-only") +os.environ.setdefault("SLACK_SIGNING_SECRET", "fixture-only") + +with patch("slack_sdk.WebClient.auth_test", return_value={ + "ok": True, "bot_id": "BTEST", "user_id": "UTEST", "team_id": "TTEST", +}): + import app.main as main + +from app import agent_runtime, database, rules + + +@pytest.fixture +def rule_api(monkeypatch): + blocked = Mock(side_effect=AssertionError("Preview must only match loaded rules")) + monkeypatch.setattr(main.vector_store, "similarity_search", blocked) + monkeypatch.setattr(agent_runtime, "generate", blocked) + monkeypatch.setattr(agent_runtime, "submit", blocked) + monkeypatch.setattr(database, "SessionLocal", blocked) + monkeypatch.setattr("slack_sdk.WebClient.api_call", blocked) + monkeypatch.setattr("slack_bolt.adapter.socket_mode.SocketModeHandler.start", blocked) + monkeypatch.setattr(rules, "rule_engine", rules.RuleEngine()) + monkeypatch.setattr(main, "init_db", lambda: None) + with TestClient(main.app, raise_server_exceptions=False) as client: + # Startup storage is unrelated; the endpoint itself needs no database. + yield client + blocked.assert_not_called() + + +def install_rules(monkeypatch, *entries): + monkeypatch.setattr(rules, "rule_engine", rules.RuleEngine(rules.parse_rules(list(entries)))) + + +def rule(rule_id="vpn", **options): + return {"id": rule_id, "keywords": ["reset vpn"], "response": "Use the IT portal.", **options} + + +def preview(client, question="How do I RESET VPN access?"): + return client.post("/api/rules/preview", json={"question": question}) + + +@pytest.mark.parametrize(("match_type", "keywords", "question", "winner"), [ + ("contains", ["vpn password", "reset vpn"], "How do I RESET VPN access?", "reset vpn"), + ("exact", ["office hours"], "OFFICE HOURS", "office hours"), + ("regex", [r"ticket #?\d+"], "Where is TICKET #4821?", r"ticket #?\d+"), +]) +def test_preview_returns_winning_rule_and_exact_response(rule_api, monkeypatch, match_type, keywords, question, winner): + response_text = " Approved wording.\nKeep the formatting. " + install_rules(monkeypatch, rule(match_type=match_type, keywords=keywords, response=response_text, priority=3)) + response = preview(rule_api, question) + assert response.status_code == 200 + assert response.json() == { + "question": question, + "match": { + "rule_id": "vpn", "match_type": match_type, "matched_keyword": winner, + "priority": 3, "response": response_text, + }, + } + + +def test_preview_uses_priority_config_order_and_disabled_rule_policy(rule_api, monkeypatch): + install_rules(monkeypatch, + rule("low", priority=1), rule("disabled", priority=100, enabled=False), + rule("first", priority=10), rule("second", priority=10), + ) + assert preview(rule_api).json()["match"]["rule_id"] == "first" + + +@pytest.mark.parametrize("entries", [[], [rule()], [rule(enabled=False)]]) +def test_preview_returns_null_for_no_match_without_falling_back_to_generation(rule_api, monkeypatch, entries): + install_rules(monkeypatch, *entries) + response = preview(rule_api, "How do I reset email?") + assert response.status_code == 200 + assert response.json() == {"question": "How do I reset email?", "match": None} + + +def test_preview_trims_question_and_uses_current_shared_matcher(rule_api, monkeypatch): + install_rules(monkeypatch, rule("first")) + question = "How do I reset VPN?" + response = preview(rule_api, f" \t{question}\n ") + assert response.json()["question"] == question + assert response.json()["match"]["rule_id"] == "first" + install_rules(monkeypatch, rule("replacement")) + assert preview(rule_api, question).json()["match"]["rule_id"] == "replacement" + + +@pytest.mark.parametrize("payload", [ + {}, {"question": ""}, {"question": " \n\t "}, {"question": None}, + {"question": 123}, {"question": True}, {"question": []}, {"question": {}}, +]) +def test_preview_rejects_invalid_input_before_matching(rule_api, monkeypatch, payload): + matcher = Mock(side_effect=AssertionError("Invalid input must not reach matching")) + monkeypatch.setattr(main.rag_engine, "match_operator_rule", matcher) + assert rule_api.post("/api/rules/preview", json=payload).status_code == 422 + matcher.assert_not_called() + + +def test_preview_matches_api_answer_and_slack_job_without_submitting_a_job(rule_api, monkeypatch): + install_rules(monkeypatch, rule("low"), rule("approved", priority=10)) + question = "How do I reset VPN?" + matched = preview(rule_api, question).json()["match"] + agent_runtime.submit.assert_not_called() + answer = main.rag_engine.generate_response(question) + assert answer["answer"] == matched["response"] + assert answer["sources"] == [{"source": "operator_rule", "rule_id": matched["rule_id"]}] + # Replace the blocked submit only for this separate parity check. Nothing is delivered. + submit = Mock(return_value={"id": "fixture-job", "status": "pending"}) + monkeypatch.setattr(agent_runtime, "submit", submit) + main.rag_engine.submit_slack(question, "CFIXTURE", "fixture-conversation", "fixture-request", "1.0") + payload = submit.call_args.args[0] + assert payload["operator_answer"] == matched["response"] + assert payload["sources"] == answer["sources"] + + +def test_preview_reports_matcher_failure_without_exposing_details(rule_api, monkeypatch): + monkeypatch.setattr(main.rag_engine, "match_operator_rule", Mock(side_effect=RuntimeError("private config details"))) + response = preview(rule_api) + assert response.status_code == 503 + assert response.json() == {"detail": "Operator rule preview is unavailable"} diff --git a/docs/operator-rules.md b/docs/operator-rules.md index df98f28..402100f 100644 --- a/docs/operator-rules.md +++ b/docs/operator-rules.md @@ -6,7 +6,9 @@ or a generated answer. Use them when the generic answer is wrong or stale and the correct wording is known. Rules are checked on the `/api/ask` path through `RAGEngine.generate_response` -in [backend/app/rag.py](../backend/app/rag.py). The matcher lives in +and on the Slack submit path through `RAGEngine.submit_slack` in +[backend/app/rag.py](../backend/app/rag.py). Slack's pinned answers use the Pi +outbox for delivery without model generation. The matcher lives in [backend/app/rules.py](../backend/app/rules.py). ## Configure @@ -64,9 +66,53 @@ match type, a non-integer priority or a bad regex is logged and skipped. Invalid JSON, a missing file or a non-list config loads as no rules. Bad config never stops the backend from starting. +## Preview the winning rule + +`POST /api/rules/preview` uses the backend's loaded matcher to show which rule +would answer a question. It does not retrieve chunks, generate an answer, +submit a Pi job, write to the database or send a Slack message. An unmatched +question returns a null match without falling back to generation. + +```sh +curl -sS http://localhost:8000/api/rules/preview \ + -H 'Content-Type: application/json' \ + -d '{"question":"How do I RESET VPN access?"}' +``` + +With the example rules above, the response is: + +```json +{ + "question": "How do I RESET VPN access?", + "match": { + "rule_id": "vpn-reset", + "match_type": "contains", + "matched_keyword": "reset vpn", + "priority": 10, + "response": "Use the IT portal at https://it.example.com/vpn to reset VPN access." + } +} +``` + +The winner follows the existing priority, config-order and enabled-rule policy. +The response preserves the approved answer's formatting. Surrounding question +whitespace is trimmed; missing, blank and non-string questions return HTTP 422 +before matching. No match returns HTTP 200 with `"match": null`. Matcher failures +return HTTP 503 with a generic detail rather than configuration internals. + +Rules are loaded once per process. Preview does not reload a file or validate +unsaved configuration. Restart both backend and bot after changing the rule +configuration; a backend preview cannot confirm what a separately running bot +has loaded. For Slack mentions, preview the question after removing the bot +mention, as the handler does before matching. + +Like the other management endpoints, preview has no built-in authentication. +Keep it behind trusted access controls: it returns the full approved answer +and the pattern that matched, which can contain internal information. It +does not evaluate source permissions or whether the answer is still correct. + ## Current limits -Rules apply to the API ask path in this slice. Slack delivery still goes -through the Pi Durable job flow, so wiring rules into the Slack handlers is -a follow-up. Combining a rule with fetched source data before summarising -is also still open under the parent issue. +Combining a rule with fetched source data before summarising remains open +under the parent issue. Preview inspects the current deterministic winner; +it does not list shadowed rules or change matching policy. diff --git a/mise.toml b/mise.toml index 48884ee..bb2f1be 100644 --- a/mise.toml +++ b/mise.toml @@ -51,7 +51,7 @@ run = "docker compose ps" [tasks.test-backend] description = "Run backend tests" dir = "backend" -run = "uv run pytest tests/test_file_extraction_units.py tests/test_logging_config.py tests/test_logging.py tests/test_agent_runtime.py tests/test_rules.py tests/test_conversations.py tests/test_documents.py tests/test_retrieval.py tests/test_thread_indexing.py tests/test_retrieval_preview.py tests/test_document_indexing.py tests/test_document_deletion.py -v" +run = "uv run pytest tests/test_file_extraction_units.py tests/test_logging_config.py tests/test_logging.py tests/test_agent_runtime.py tests/test_rules.py tests/test_conversations.py tests/test_documents.py tests/test_retrieval.py tests/test_thread_indexing.py tests/test_retrieval_preview.py tests/test_document_indexing.py tests/test_document_deletion.py tests/test_rule_preview.py -v" [tasks.migrate] description = "Run database migrations (upgrade to latest)"