diff --git a/.tmp/proof/rule-preview-api.txt b/.tmp/proof/rule-preview-api.txt new file mode 100644 index 0000000..b30737f --- /dev/null +++ b/.tmp/proof/rule-preview-api.txt @@ -0,0 +1,137 @@ +Offline operator-rule preview with rule-preview-config.json. +Slack API, Socket Mode, retrieval and Pi calls raise in the proof process. +No chunks or distances apply: this endpoint only inspects loaded rules. + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/health +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 20 +content-type: application/json +{ + "status": "healthy" +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": " How do I RESET VPN access?\n"}' +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 223 +content-type: application/json +{ + "question": "How do I RESET VPN access?", + "match": { + "rule_id": "vpn-reset", + "match_type": "contains", + "matched_keyword": "reset vpn", + "priority": 10, + "response": "Use the IT portal at https://it.example.com/vpn to reset VPN access." + } +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": "OFFICE HOURS"}' +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 194 +content-type: application/json +{ + "question": "OFFICE HOURS", + "match": { + "rule_id": "office-hours", + "match_type": "exact", + "matched_keyword": "office hours", + "priority": 0, + "response": "The support desk runs 9am to 5pm, Monday to Friday." + } +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": "Where is TICKET #4821?"}' +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 189 +content-type: application/json +{ + "question": "Where is TICKET #4821?", + "match": { + "rule_id": "ticket", + "match_type": "regex", + "matched_keyword": "ticket #?\\d+", + "priority": 0, + "response": "Look up the ticket in the support portal." + } +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": "How do I reset email?"}' +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 49 +content-type: application/json +{ + "question": "How do I reset email?", + "match": null +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": " \n\t "}' +HTTP/1.1 422 Unprocessable Content +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 154 +content-type: application/json +{ + "detail": [ + { + "type": "string_too_short", + "loc": [ + "body", + "question" + ], + "msg": "String should have at least 1 character", + "input": " \n\t ", + "ctx": { + "min_length": 1 + } + } + ] +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/rules/preview -H 'Content-Type: application/json' -d '{"question": 123}' +HTTP/1.1 422 Unprocessable Content +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 112 +content-type: application/json +{ + "detail": [ + { + "type": "string_type", + "loc": [ + "body", + "question" + ], + "msg": "Input should be a valid string", + "input": 123 + } + ] +} + +$ curl --silent --show-error --include --max-time 10 http://127.0.0.1:18059/api/ask -H 'Content-Type: application/json' -d '{"question": "How do I RESET VPN access?"}' +HTTP/1.1 200 OK +date: Mon, 16 Nov 2026 14:21:48 GMT +server: uvicorn +content-length: 162 +content-type: application/json +{ + "answer": "Use the IT portal at https://it.example.com/vpn to reset VPN access.", + "sources": [ + { + "source": "operator_rule", + "rule_id": "vpn-reset" + } + ], + "confidence": "high" +} + +All response assertions passed. Preview and API answer select the same rule. diff --git a/.tmp/proof/rule-preview-backend-start.txt b/.tmp/proof/rule-preview-backend-start.txt new file mode 100644 index 0000000..c4fb312 --- /dev/null +++ b/.tmp/proof/rule-preview-backend-start.txt @@ -0,0 +1,32 @@ +[dev-backend] $ uv run uvicorn app.main:app --reload --host 127.0.0.1 --port 18… +INFO: Will watch for changes in these directories: ['/Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend'] +INFO: Uvicorn running on http://127.0.0.1:18059 (Press CTRL+C to quit) +INFO: Started reloader process [34598] using WatchFiles +INFO: Started server process [34657] +INFO: Waiting for application startup. +{"timestamp": "2026-11-16T14:21:28.106608+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "app.main", "message": "Starting Wingman backend...", "event_type": "log", "request_id": null, "trace_id": null} +{"timestamp": "2026-11-16T14:21:28.114661+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "app.main", "message": "Database initialized", "event_type": "log", "request_id": null, "trace_id": null} +INFO: Application startup complete. +INFO: 127.0.0.1:54831 - "GET /health HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.535859+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "b3b5cdccd1e942a58bf4af8cd7d77b9e", "trace_id": "b3b5cdccd1e942a58bf4af8cd7d77b9e", "request_method": "GET", "request_path": "/health", "route": "/health", "status_code": 200, "duration_ms": 0.53, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54832 - "POST /api/rules/preview HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.560690+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "d8893f36a4c34f3ab1fee3c749d809e6", "trace_id": "d8893f36a4c34f3ab1fee3c749d809e6", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 200, "duration_ms": 9.41, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54834 - "POST /api/rules/preview HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.577942+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "9c494068ece84b64aee19c973fbb441d", "trace_id": "9c494068ece84b64aee19c973fbb441d", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 200, "duration_ms": 0.72, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54835 - "POST /api/rules/preview HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.592262+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "2766ec1b0f744df4a43bfbc08b97b818", "trace_id": "2766ec1b0f744df4a43bfbc08b97b818", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 200, "duration_ms": 0.72, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54836 - "POST /api/rules/preview HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.606412+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "a9d41ea7e6b141cab377ab767b5dc139", "trace_id": "a9d41ea7e6b141cab377ab767b5dc139", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 200, "duration_ms": 0.61, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54837 - "POST /api/rules/preview HTTP/1.1" 422 Unprocessable Content +{"timestamp": "2026-11-16T14:21:48.620734+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "7baf10d4255c45ecafdcc5c8fd136bf0", "trace_id": "7baf10d4255c45ecafdcc5c8fd136bf0", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 422, "duration_ms": 0.33, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: 127.0.0.1:54838 - "POST /api/rules/preview HTTP/1.1" 422 Unprocessable Content +{"timestamp": "2026-11-16T14:21:48.634238+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "ad6871e0d2554cb795de6e699bca3094", "trace_id": "ad6871e0d2554cb795de6e699bca3094", "request_method": "POST", "request_path": "/api/rules/preview", "route": "/api/rules/preview", "status_code": 422, "duration_ms": 0.29, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +{"timestamp": "2026-11-16T14:21:48.648333+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "app.rag", "message": "Operator rule vpn-reset answered a question", "event_type": "log", "request_id": "3be154cb62d04c1fa94a7a655b422087", "trace_id": "3be154cb62d04c1fa94a7a655b422087"} +INFO: 127.0.0.1:54839 - "POST /api/ask HTTP/1.1" 200 OK +{"timestamp": "2026-11-16T14:21:48.648640+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "wingman.wide_event", "message": "http_request", "event_type": "http_request", "request_id": "3be154cb62d04c1fa94a7a655b422087", "trace_id": "3be154cb62d04c1fa94a7a655b422087", "request_method": "POST", "request_path": "/api/ask", "route": "/api/ask", "status_code": 200, "duration_ms": 0.69, "error_class": null, "error_message": null, "sample_reason": "RANDOM"} +INFO: Shutting down +INFO: Waiting for application shutdown. +{"timestamp": "2026-11-16T14:21:54.451885+00:00", "level": "INFO", "service_name": "wingman-backend", "environment": "development", "service_version": "0.1.0", "logger": "app.main", "message": "Shutting down Wingman backend...", "event_type": "log", "request_id": null, "trace_id": null} +INFO: Application shutdown complete. +INFO: Finished server process [34657] +INFO: Stopping reloader process [34598] diff --git a/.tmp/proof/rule-preview-backend-tests.txt b/.tmp/proof/rule-preview-backend-tests.txt new file mode 100644 index 0000000..5413bbe --- /dev/null +++ b/.tmp/proof/rule-preview-backend-tests.txt @@ -0,0 +1,312 @@ +[test-backend] $ uv run pytest tests/test_file_extraction_units.py tests/test_l… +============================= test session starts ============================== +platform darwin -- Python 3.13.13, pytest-9.0.2, pluggy-1.6.0 -- /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/.venv/bin/python +cachedir: .pytest_cache +rootdir: /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend +configfile: pytest.ini (WARNING: ignoring pytest config in pyproject.toml!) +plugins: langsmith-0.5.2, anyio-4.12.0, asyncio-1.3.0 +asyncio: mode=Mode.STRICT, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function +collecting ... collected 239 items + +tests/test_file_extraction_units.py::test_extract_plain_text PASSED [ 0%] +tests/test_file_extraction_units.py::test_extract_markdown PASSED [ 0%] +tests/test_file_extraction_units.py::test_extract_python_file PASSED [ 1%] +tests/test_file_extraction_units.py::test_extract_json PASSED [ 1%] +tests/test_file_extraction_units.py::test_extract_csv PASSED [ 2%] +tests/test_file_extraction_units.py::test_extract_html PASSED [ 2%] +tests/test_file_extraction_units.py::test_extract_html_removes_scripts PASSED [ 2%] +tests/test_file_extraction_units.py::test_extract_with_encoding_errors PASSED [ 3%] +tests/test_file_extraction_units.py::test_extract_xml PASSED [ 3%] +tests/test_file_extraction_units.py::test_extract_yaml PASSED [ 4%] +tests/test_file_extraction_units.py::test_javascript_file_extraction PASSED [ 4%] +tests/test_file_extraction_units.py::test_typescript_file_extraction PASSED [ 5%] +tests/test_file_extraction_units.py::test_shell_script_extraction PASSED [ 5%] +tests/test_file_extraction_units.py::TestConversationIds::test_thread_conversation_id_format PASSED [ 5%] +tests/test_file_extraction_units.py::TestConversationIds::test_dm_conversation_id_format PASSED [ 6%] +tests/test_file_extraction_units.py::TestConversationIds::test_mention_conversation_id_format PASSED [ 6%] +tests/test_logging_config.py::test_json_schema_and_redaction_variants PASSED [ 7%] +tests/test_logging_config.py::test_error_hash_not_stack PASSED [ 7%] +tests/test_logging_config.py::test_wide_event_one_per_request PASSED [ 7%] +tests/test_logging.py::test_json_stdout_preserves_controls_and_avoids_duplicate_handlers[wingman-backend] PASSED [ 8%] +tests/test_logging.py::test_json_stdout_preserves_controls_and_avoids_duplicate_handlers[wingman-bot] PASSED [ 8%] +tests/test_logging.py::test_text_format_and_configured_level PASSED [ 9%] +tests/test_logging.py::test_api_keeps_backend_service_when_importing_bot PASSED [ 9%] +tests/test_agent_runtime.py::test_transport_retry_keeps_request_id PASSED [ 10%] +tests/test_agent_runtime.py::test_rag_routes_retrieved_context_and_stable_slack_identity PASSED [ 10%] +tests/test_agent_runtime.py::test_failed_job_surfaces_error PASSED [ 10%] +tests/test_rules.py::TestMatching::test_contains_is_case_insensitive PASSED [ 11%] +tests/test_rules.py::TestMatching::test_contains_needs_the_keyword PASSED [ 11%] +tests/test_rules.py::TestMatching::test_any_keyword_can_match PASSED [ 12%] +tests/test_rules.py::TestMatching::test_exact_needs_the_whole_question PASSED [ 12%] +tests/test_rules.py::TestMatching::test_regex_match PASSED [ 12%] +tests/test_rules.py::TestMatching::test_blank_question_never_matches PASSED [ 13%] +tests/test_rules.py::TestMatching::test_no_rules_is_a_no_op PASSED [ 13%] +tests/test_rules.py::TestOrderingAndEnabled::test_higher_priority_wins PASSED [ 14%] +tests/test_rules.py::TestOrderingAndEnabled::test_config_order_breaks_priority_ties PASSED [ 14%] +tests/test_rules.py::TestOrderingAndEnabled::test_disabled_rule_never_matches PASSED [ 15%] +tests/test_rules.py::TestConfigLoading::test_parse_valid_rules PASSED [ 15%] +tests/test_rules.py::TestConfigLoading::test_parse_skips_invalid_entries PASSED [ 15%] +tests/test_rules.py::TestConfigLoading::test_parse_skips_bad_regex PASSED [ 16%] +tests/test_rules.py::TestConfigLoading::test_parse_rejects_non_list PASSED [ 16%] +tests/test_rules.py::TestConfigLoading::test_load_from_json_text PASSED [ 17%] +tests/test_rules.py::TestConfigLoading::test_load_invalid_json_is_empty PASSED [ 17%] +tests/test_rules.py::TestConfigLoading::test_load_from_file_and_file_wins PASSED [ 17%] +tests/test_rules.py::TestConfigLoading::test_load_missing_file_is_empty PASSED [ 18%] +tests/test_rules.py::TestConfigLoading::test_from_settings_reads_both_fields PASSED [ 18%] +tests/test_rules.py::TestRagShortCircuit::test_matched_question_skips_retrieval_and_generation PASSED [ 19%] +tests/test_rules.py::TestRagShortCircuit::test_unmatched_question_still_uses_generation PASSED [ 19%] +tests/test_rules.py::TestSlackShortCircuit::test_matched_slack_question_skips_retrieval_and_carries_pinned_answer PASSED [ 20%] +tests/test_rules.py::TestSlackShortCircuit::test_unmatched_slack_question_uses_retrieval_without_pinned_answer PASSED [ 20%] +tests/test_conversations.py::TestListConversations::test_lists_summaries_most_recent_first PASSED [ 20%] +tests/test_conversations.py::TestListConversations::test_summary_counts_and_preview PASSED [ 21%] +tests/test_conversations.py::TestListConversations::test_pagination PASSED [ 21%] +tests/test_conversations.py::TestListConversations::test_channel_filter PASSED [ 22%] +tests/test_conversations.py::TestListConversations::test_channel_filter_scopes_preview_for_shared_conversation[C1] PASSED [ 22%] +tests/test_conversations.py::TestListConversations::test_channel_filter_scopes_preview_for_shared_conversation[C2] PASSED [ 23%] +tests/test_conversations.py::TestListConversations::test_unfiltered_preview_includes_latest_turn_across_channels PASSED [ 23%] +tests/test_conversations.py::TestListConversations::test_scoped_preview_breaks_timestamp_ties_by_latest_id PASSED [ 23%] +tests/test_conversations.py::TestListConversations::test_invalid_pagination_rejected PASSED [ 24%] +tests/test_conversations.py::TestListConversations::test_empty_database PASSED [ 24%] +tests/test_conversations.py::TestConversationDetail::test_returns_turns_in_chronological_order PASSED [ 25%] +tests/test_conversations.py::TestConversationDetail::test_detail_pagination PASSED [ 25%] +tests/test_conversations.py::TestConversationDetail::test_missing_conversation_is_404 PASSED [ 25%] +tests/test_conversations.py::TestConversationDetail::test_detail_invalid_pagination_rejected PASSED [ 26%] +tests/test_documents.py::TestListDocuments::test_lists_documents_most_recent_first PASSED [ 26%] +tests/test_documents.py::TestListDocuments::test_preview_and_fields PASSED [ 27%] +tests/test_documents.py::TestListDocuments::test_pagination PASSED [ 27%] +tests/test_documents.py::TestListDocuments::test_source_filter PASSED [ 28%] +tests/test_documents.py::TestListDocuments::test_invalid_pagination_rejected PASSED [ 28%] +tests/test_documents.py::TestListDocuments::test_empty_database PASSED [ 28%] +tests/test_documents.py::TestSearchDocuments::test_matches_title_case_insensitively_and_trims_query[vpn] PASSED [ 29%] +tests/test_documents.py::TestSearchDocuments::test_matches_title_case_insensitively_and_trims_query[VpN] PASSED [ 29%] +tests/test_documents.py::TestSearchDocuments::test_matches_title_case_insensitively_and_trims_query[ \tVPN\n ] PASSED [ 30%] +tests/test_documents.py::TestSearchDocuments::test_matches_content_beyond_the_preview PASSED [ 30%] +tests/test_documents.py::TestSearchDocuments::test_filters_before_counting_and_paginating PASSED [ 30%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-%] PASSED [ 31%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-_] PASSED [ 31%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-/] PASSED [ 32%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-100%] PASSED [ 32%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-setup_notes] PASSED [ 33%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-a/b] PASSED [ 33%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[title-' OR 1=1 --] PASSED [ 33%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-%] PASSED [ 34%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-_] PASSED [ 34%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-/] PASSED [ 35%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-100%] PASSED [ 35%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-setup_notes] PASSED [ 35%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-a/b] PASSED [ 36%] +tests/test_documents.py::TestSearchDocuments::test_treats_sql_wildcards_and_escape_characters_literally[content-' OR 1=1 --] PASSED [ 36%] +tests/test_documents.py::TestSearchDocuments::test_blank_query_preserves_source_filter_and_existing_list[] PASSED [ 37%] +tests/test_documents.py::TestSearchDocuments::test_blank_query_preserves_source_filter_and_existing_list[ \t\n ] PASSED [ 37%] +tests/test_documents.py::TestSearchDocuments::test_blank_query_preserves_source_filter_and_existing_list[\u2003\xa0] PASSED [ 38%] +tests/test_documents.py::TestSearchDocuments::test_no_matches_return_an_empty_page[params0] PASSED [ 38%] +tests/test_documents.py::TestSearchDocuments::test_no_matches_return_an_empty_page[params1] PASSED [ 38%] +tests/test_documents.py::TestDocumentDetail::test_returns_full_content PASSED [ 39%] +tests/test_documents.py::TestDocumentDetail::test_missing_document_is_404 PASSED [ 39%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[-title] PASSED [ 40%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[-content] PASSED [ 40%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[-source] PASSED [ 41%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[ \n\t -title] PASSED [ 41%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[ \n\t -content] PASSED [ 41%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[ \n\t -source] PASSED [ 42%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[\u2003\xa0-title] PASSED [ 42%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[\u2003\xa0-content] PASSED [ 43%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[\u2003\xa0-source] PASSED [ 43%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[None-title] PASSED [ 43%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[None-content] PASSED [ 44%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[None-source] PASSED [ 44%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[123-title] PASSED [ 45%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[123-content] PASSED [ 45%] +tests/test_documents.py::TestCreateDocument::test_rejects_invalid_fields_without_writes_or_indexing[123-source] PASSED [ 46%] +tests/test_documents.py::TestCreateDocument::test_rejects_missing_required_fields[title] PASSED [ 46%] +tests/test_documents.py::TestCreateDocument::test_rejects_missing_required_fields[content] PASSED [ 46%] +tests/test_documents.py::TestCreateDocument::test_creates_readable_document_and_indexes_preserved_content[None] PASSED [ 47%] +tests/test_documents.py::TestCreateDocument::test_creates_readable_document_and_indexes_preserved_content[ \tdocs\n ] PASSED [ 47%] +tests/test_retrieval.py::test_invalid_chunk_settings_are_rejected[config0] PASSED [ 48%] +tests/test_retrieval.py::test_invalid_chunk_settings_are_rejected[config1] PASSED [ 48%] +tests/test_retrieval.py::test_invalid_chunk_settings_are_rejected[config2] PASSED [ 48%] +tests/test_retrieval.py::test_invalid_chunk_settings_are_rejected[config3] PASSED [ 49%] +tests/test_retrieval.py::test_invalid_chunk_settings_are_rejected[config4] PASSED [ 49%] +tests/test_retrieval.py::test_valid_chunk_settings_are_accepted[1000-200] PASSED [ 50%] +tests/test_retrieval.py::test_valid_chunk_settings_are_accepted[1-0] PASSED [ 50%] +tests/test_retrieval.py::test_valid_chunk_settings_are_accepted[10-0] PASSED [ 51%] +tests/test_retrieval.py::test_valid_chunk_settings_are_accepted[10-9] PASSED [ 51%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[-5-2-expected0] PASSED [ 51%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[a-5-2-expected1] PASSED [ 52%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcd-5-2-expected2] PASSED [ 52%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcde-5-2-expected3] PASSED [ 53%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcdef-5-2-expected4] PASSED [ 53%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcdefgh-5-2-expected5] PASSED [ 53%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcdefghi-5-2-expected6] PASSED [ 54%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcdefghij-5-0-expected7] PASSED [ 54%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abcdef-5-4-expected8] PASSED [ 55%] +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[abc-1-0-expected9] PASSED [ 55%] +tests/test_retrieval.py::test_document_indexing_avoids_redundant_tail_chunks PASSED [ 56%] +tests/test_retrieval.py::test_repeated_thread_indexing_preserves_distinct_top_k PASSED [ 56%] +tests/test_retrieval.py::test_thread_reindex_updates_content_and_embedding PASSED [ 56%] +tests/test_retrieval.py::test_thread_chunk_ids_are_scoped_by_channel PASSED [ 57%] +tests/test_retrieval.py::test_thread_indexing_omits_absent_optional_metadata PASSED [ 57%] +tests/test_retrieval.py::test_thread_indexing_uses_last_duplicate_in_batch PASSED [ 58%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages0] PASSED [ 58%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages1] PASSED [ 58%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages2] PASSED [ 59%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages3] PASSED [ 59%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages4] PASSED [ 60%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages5] PASSED [ 60%] +tests/test_retrieval.py::test_thread_indexing_skips_unusable_messages[messages6] PASSED [ 61%] +tests/test_retrieval.py::test_search_empty_collection_returns_no_matches[None-None] PASSED [ 61%] +tests/test_retrieval.py::test_search_empty_collection_returns_no_matches[None-CTARGET] PASSED [ 61%] +tests/test_retrieval.py::test_search_empty_collection_returns_no_matches[1.0-None] PASSED [ 62%] +tests/test_retrieval.py::test_search_empty_collection_returns_no_matches[1.0-CTARGET] PASSED [ 62%] +tests/test_retrieval.py::test_first_document_can_be_indexed_and_retrieved PASSED [ 63%] +tests/test_retrieval.py::test_missing_adapter_still_raises[add] PASSED [ 63%] +tests/test_retrieval.py::test_missing_adapter_still_raises[search] PASSED [ 64%] +tests/test_retrieval.py::test_channel_filter_runs_before_top_k PASSED [ 64%] +tests/test_retrieval.py::test_unscoped_search_preserves_global_ranking_and_explicit_k PASSED [ 64%] +tests/test_retrieval.py::test_distance_cutoff_is_inclusive_and_preserves_channel_scope PASSED [ 65%] +tests/test_retrieval.py::test_zero_distance_cutoff_keeps_only_exact_vector_matches PASSED [ 65%] +tests/test_retrieval.py::test_invalid_distance_config_is_rejected[-1] PASSED [ 66%] +tests/test_retrieval.py::test_invalid_distance_config_is_rejected[nan] PASSED [ 66%] +tests/test_retrieval.py::test_invalid_distance_config_is_rejected[inf] PASSED [ 66%] +tests/test_retrieval.py::test_invalid_distance_config_is_rejected[-inf] PASSED [ 67%] +tests/test_retrieval.py::test_invalid_distance_config_is_rejected[invalid] PASSED [ 67%] +tests/test_retrieval.py::test_distance_config_accepts_unset_or_nonnegative_finite_values[None] PASSED [ 68%] +tests/test_retrieval.py::test_distance_config_accepts_unset_or_nonnegative_finite_values[0] PASSED [ 68%] +tests/test_retrieval.py::test_distance_config_accepts_unset_or_nonnegative_finite_values[1.5] PASSED [ 69%] +tests/test_retrieval.py::test_empty_channel_does_not_fall_back_to_global_results PASSED [ 69%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[None-2-api] PASSED [ 69%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[None-2-slack] PASSED [ 70%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[1.0-1-api] PASSED [ 70%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[1.0-1-slack] PASSED [ 71%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[0.0-0-api] PASSED [ 71%] +tests/test_retrieval.py::test_rag_payload_uses_scoped_matches[0.0-0-slack] PASSED [ 71%] +tests/test_thread_indexing.py::test_saved_thread_includes_root_and_replies_in_timestamp_order PASSED [ 72%] +tests/test_thread_indexing.py::test_saved_root_only_thread_is_indexed_once[None] PASSED [ 72%] +tests/test_thread_indexing.py::test_saved_root_only_thread_is_indexed_once[1700000000.000001] PASSED [ 73%] +tests/test_thread_indexing.py::test_saved_replies_can_be_indexed_without_a_saved_root PASSED [ 73%] +tests/test_thread_indexing.py::test_missing_saved_thread_returns_404_without_indexing[CTARGET-1700000098.000001] PASSED [ 74%] +tests/test_thread_indexing.py::test_missing_saved_thread_returns_404_without_indexing[CEMPTY-1700000000.000001] PASSED [ 74%] +tests/test_thread_indexing.py::test_saved_thread_channel_filter_also_applies_to_root_lookup PASSED [ 74%] +tests/test_thread_indexing.py::test_saved_thread_indexing_failure_remains_500 PASSED [ 75%] +tests/test_thread_indexing.py::test_saved_thread_retrieval_includes_original_question PASSED [ 75%] +tests/test_retrieval_preview.py::test_preview_returns_ranked_chunks_metadata_distances_and_settings PASSED [ 76%] +tests/test_retrieval_preview.py::test_preview_applies_channel_scope_before_top_k PASSED [ 76%] +tests/test_retrieval_preview.py::test_preview_exposes_inclusive_distance_cutoff_with_transcript PASSED [ 76%] +tests/test_retrieval_preview.py::test_preview_returns_success_for_empty_results[collection] PASSED [ 77%] +tests/test_retrieval_preview.py::test_preview_returns_success_for_empty_results[channel] PASSED [ 77%] +tests/test_retrieval_preview.py::test_preview_returns_success_for_empty_results[cutoff] PASSED [ 78%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload0] PASSED [ 78%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload1] PASSED [ 79%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload2] PASSED [ 79%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload3] PASSED [ 79%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload4] PASSED [ 80%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload5] PASSED [ 80%] +tests/test_retrieval_preview.py::test_preview_rejects_invalid_input_before_search[payload6] PASSED [ 81%] +tests/test_retrieval_preview.py::test_preview_trims_question_and_channel PASSED [ 81%] +tests/test_retrieval_preview.py::test_preview_reports_unavailable_retrieval_without_exposing_error PASSED [ 82%] +tests/test_document_indexing.py::test_create_and_retry_keep_one_owned_chunk_and_preserve_saved_record PASSED [ 82%] +tests/test_document_indexing.py::test_retry_recovers_saved_document_after_embedding_failure_with_transcript PASSED [ 82%] +tests/test_document_indexing.py::test_retry_rechunks_and_removes_only_obsolete_owned_chunks PASSED [ 83%] +tests/test_document_indexing.py::test_retry_reads_latest_saved_content_title_and_source PASSED [ 83%] +tests/test_document_indexing.py::test_retry_failure_keeps_previous_chunks_and_saved_content[lookup] PASSED [ 84%] +tests/test_document_indexing.py::test_retry_failure_keeps_previous_chunks_and_saved_content[embed] PASSED [ 84%] +tests/test_document_indexing.py::test_retry_failure_keeps_previous_chunks_and_saved_content[upsert] PASSED [ 84%] +tests/test_document_indexing.py::test_cleanup_failure_is_reported_and_retry_finishes_without_duplicates PASSED [ 85%] +tests/test_document_indexing.py::test_missing_saved_document_does_not_initialise_vector_store PASSED [ 85%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[content- \n\t ] PASSED [ 86%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[content-None] PASSED [ 86%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[title-] PASSED [ 87%] +tests/test_document_indexing.py::test_invalid_legacy_document_is_rejected_without_changing_chunks[source-None] PASSED [ 87%] +tests/test_document_deletion.py::test_delete_removes_only_owned_chunks_with_retrieval_transcript PASSED [ 87%] +tests/test_document_deletion.py::test_missing_or_invalid_delete_does_not_initialise_vector_store[999-404] PASSED [ 88%] +tests/test_document_deletion.py::test_missing_or_invalid_delete_does_not_initialise_vector_store[-1-404] PASSED [ 88%] +tests/test_document_deletion.py::test_missing_or_invalid_delete_does_not_initialise_vector_store[invalid-422] PASSED [ 89%] +tests/test_document_deletion.py::test_missing_or_invalid_delete_does_not_initialise_vector_store[1.5-422] PASSED [ 89%] +tests/test_document_deletion.py::test_delete_unindexed_legacy_record_needs_no_valid_content_or_embeddings PASSED [ 89%] +tests/test_document_deletion.py::test_vector_delete_failure_retains_saved_record_and_allows_retry[initialise] PASSED [ 90%] +tests/test_document_deletion.py::test_vector_delete_failure_retains_saved_record_and_allows_retry[delete] PASSED [ 90%] +tests/test_document_deletion.py::test_vector_delete_failure_retains_saved_record_and_allows_retry[lost-acknowledgement] PASSED [ 91%] +tests/test_document_deletion.py::test_database_delete_failure_rolls_back_saved_record_and_can_retry[delete] PASSED [ 91%] +tests/test_document_deletion.py::test_database_delete_failure_rolls_back_saved_record_and_can_retry[commit] PASSED [ 92%] +tests/test_document_deletion.py::test_delete_still_requires_an_initialised_adapter PASSED [ 92%] +tests/test_rule_preview.py::test_preview_returns_winning_rule_and_exact_response[contains-keywords0-How do I RESET VPN access?-reset vpn] PASSED [ 92%] +tests/test_rule_preview.py::test_preview_returns_winning_rule_and_exact_response[exact-keywords1-OFFICE HOURS-office hours] PASSED [ 93%] +tests/test_rule_preview.py::test_preview_returns_winning_rule_and_exact_response[regex-keywords2-Where is TICKET #4821?-ticket #?\\d+] PASSED [ 93%] +tests/test_rule_preview.py::test_preview_uses_priority_config_order_and_disabled_rule_policy PASSED [ 94%] +tests/test_rule_preview.py::test_preview_returns_null_for_no_match_without_falling_back_to_generation[entries0] PASSED [ 94%] +tests/test_rule_preview.py::test_preview_returns_null_for_no_match_without_falling_back_to_generation[entries1] PASSED [ 94%] +tests/test_rule_preview.py::test_preview_returns_null_for_no_match_without_falling_back_to_generation[entries2] PASSED [ 95%] +tests/test_rule_preview.py::test_preview_trims_question_and_uses_current_shared_matcher PASSED [ 95%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload0] PASSED [ 96%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload1] PASSED [ 96%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload2] PASSED [ 97%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload3] PASSED [ 97%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload4] PASSED [ 97%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload5] PASSED [ 98%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload6] PASSED [ 98%] +tests/test_rule_preview.py::test_preview_rejects_invalid_input_before_matching[payload7] PASSED [ 99%] +tests/test_rule_preview.py::test_preview_matches_api_answer_and_slack_job_without_submitting_a_job PASSED [ 99%] +tests/test_rule_preview.py::test_preview_reports_matcher_failure_without_exposing_details PASSED [100%] + +=============================== warnings summary =============================== +app/main.py:114 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/app/main.py:114: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.12/migration/ + class MessageResponse(BaseModel): + +app/main.py:141 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/app/main.py:141: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.12/migration/ + class ConversationTurn(BaseModel): + +app/main.py:178 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/app/main.py:178: PydanticDeprecatedSince20: Support for class-based `config` is deprecated, use ConfigDict instead. Deprecated in Pydantic V2.0 to be removed in V3.0. See Pydantic V2 Migration Guide at https://errors.pydantic.dev/2.12/migration/ + class DocumentDetail(BaseModel): + +app/main.py:191 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/app/main.py:191: DeprecationWarning: + on_event is deprecated, use lifespan event handlers instead. + + Read more about it in the + [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/). + + @app.on_event("startup") + +.venv/lib/python3.13/site-packages/fastapi/applications.py:4576 +.venv/lib/python3.13/site-packages/fastapi/applications.py:4576 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/.venv/lib/python3.13/site-packages/fastapi/applications.py:4576: DeprecationWarning: + on_event is deprecated, use lifespan event handlers instead. + + Read more about it in the + [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/). + + return self.router.on_event(event_type) + +app/main.py:204 + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/app/main.py:204: DeprecationWarning: + on_event is deprecated, use lifespan event handlers instead. + + Read more about it in the + [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/). + + @app.on_event("shutdown") + +tests/test_documents.py: 4 warnings +tests/test_thread_indexing.py: 92 warnings +tests/test_document_indexing.py: 31 warnings +tests/test_document_deletion.py: 16 warnings + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/.venv/lib/python3.13/site-packages/sqlalchemy/sql/schema.py:3624: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC). + return util.wrap_callable(lambda ctx: fn(), fn) # type: ignore + +tests/test_retrieval.py::test_document_chunks_stop_after_covering_content[-5-2-expected0] + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/tests/test_retrieval.py:122: LangChainDeprecationWarning: The class `Chroma` was deprecated in LangChain 0.2.9 and will be removed in 1.0. An updated version of the class exists in the `langchain-chroma package and should be used instead. To use it run `pip install -U `langchain-chroma` and import as `from `langchain_chroma import Chroma``. + store.vector_store = Chroma( + +tests/test_retrieval.py: 384 warnings +tests/test_thread_indexing.py: 27 warnings +tests/test_retrieval_preview.py: 216 warnings +tests/test_document_indexing.py: 166 warnings +tests/test_document_deletion.py: 89 warnings + /Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend/.venv/lib/python3.13/site-packages/chromadb/types.py:144: PydanticDeprecatedSince211: Accessing the 'model_fields' attribute on the instance is deprecated. Instead, you should access this attribute from the model class. Deprecated in Pydantic V2.11 to be removed in V3.0. + return self.model_fields # pydantic 2.x + +-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html +====================== 239 passed, 1033 warnings in 6.31s ====================== diff --git a/.tmp/proof/rule-preview-capture.py b/.tmp/proof/rule-preview-capture.py new file mode 100644 index 0000000..3edcd10 --- /dev/null +++ b/.tmp/proof/rule-preview-capture.py @@ -0,0 +1,42 @@ +"""Capture curl requests against the offline backend started on port 18059.""" +import json +from pathlib import Path +import shlex +import subprocess + + +base = "http://127.0.0.1:18059" +transcript = [ + "Offline operator-rule preview with rule-preview-config.json.", + "Slack API, Socket Mode, retrieval and Pi calls raise in the proof process.", + "No chunks or distances apply: this endpoint only inspects loaded rules.", +] + + +def request(path, payload=None, status=200): + command = ["curl", "--silent", "--show-error", "--include", "--max-time", "10", f"{base}{path}"] + if payload is not None: + command += ["-H", "Content-Type: application/json", "-d", json.dumps(payload)] + result = subprocess.run(command, check=True, capture_output=True, text=True) + headers, body = result.stdout.split("\n\n", 1) + assert f" {status} " in headers.splitlines()[0], result.stdout + data = json.loads(body) + transcript.extend(["", "$ " + shlex.join(command), headers, json.dumps(data, indent=2)]) + return data + + +assert request("/health") == {"status": "healthy"} +winning = request("/api/rules/preview", {"question": " How do I RESET VPN access?\n"}) +assert winning["question"] == "How do I RESET VPN access?" +assert winning["match"]["rule_id"] == "vpn-reset" +assert winning["match"]["priority"] == 10 +assert request("/api/rules/preview", {"question": "OFFICE HOURS"})["match"]["match_type"] == "exact" +assert request("/api/rules/preview", {"question": "Where is TICKET #4821?"})["match"]["match_type"] == "regex" +assert request("/api/rules/preview", {"question": "How do I reset email?"})["match"] is None +request("/api/rules/preview", {"question": " \n\t "}, status=422) +request("/api/rules/preview", {"question": 123}, status=422) +answer = request("/api/ask", {"question": "How do I RESET VPN access?"}) +assert answer["answer"] == winning["match"]["response"] +assert answer["sources"] == [{"source": "operator_rule", "rule_id": winning["match"]["rule_id"]}] +transcript.extend(["", "All response assertions passed. Preview and API answer select the same rule."]) +Path(__file__).with_name("rule-preview-api.txt").write_text("\n".join(transcript) + "\n") diff --git a/.tmp/proof/rule-preview-config.json b/.tmp/proof/rule-preview-config.json new file mode 100644 index 0000000..4a7ff35 --- /dev/null +++ b/.tmp/proof/rule-preview-config.json @@ -0,0 +1,33 @@ +[ + { + "id": "vpn-low", + "keywords": ["reset vpn"], + "response": "A lower priority answer that should not win.", + "priority": 1 + }, + { + "id": "vpn-disabled", + "keywords": ["reset vpn"], + "response": "A disabled answer that should not win.", + "priority": 100, + "enabled": false + }, + { + "id": "vpn-reset", + "keywords": ["reset vpn", "vpn password"], + "response": "Use the IT portal at https://it.example.com/vpn to reset VPN access.", + "priority": 10 + }, + { + "id": "office-hours", + "keywords": ["office hours"], + "match_type": "exact", + "response": "The support desk runs 9am to 5pm, Monday to Friday." + }, + { + "id": "ticket", + "keywords": ["ticket #?\\d+"], + "match_type": "regex", + "response": "Look up the ticket in the support portal." + } +] diff --git a/.tmp/proof/rule-preview-frontend-tests.txt b/.tmp/proof/rule-preview-frontend-tests.txt new file mode 100644 index 0000000..30ff65f --- /dev/null +++ b/.tmp/proof/rule-preview-frontend-tests.txt @@ -0,0 +1,69 @@ +[test-frontend] $ bun run test +$ bun test --preload ./tests/setup.ts +bun test v1.3.5 (1e86cebd) + +tests/documents.test.tsx: +(pass) documents load on opening the tab, search only on submit, and retain the applied query while paging [176.30ms] +(pass) document client encodes literal search text, keeps source filters, and omits blank queries [0.17ms] +(pass) documents show loading, retry the submitted query after a list error, and distinguish empty searches [15.00ms] +(pass) documents render saved content as text and retry detail failures without losing results [22.97ms] +(pass) documents ignore stale successful and failed list responses after a newer search [9.05ms] +(pass) documents ignore late details after another selection, a page change, and clearing a search [35.51ms] +(pass) documents abort requests on leaving the tab and ignore late responses when reopened [10.11ms] + +tests/document-deletion.test.tsx: +(pass) deletion requires confirmation of the selected document and cancellation sends no mutation [18.01ms] +(pass) confirmed deletion handles HTTP %s by refreshing page one of the applied search [40.65ms] +(pass) confirmed deletion handles HTTP %s by refreshing page one of the applied search [37.14ms] +(pass) deletion reports an uncertain outcome after 503 and retries the same ID [22.40ms] +(pass) deletion reports an uncertain outcome after 403 and retries the same ID [24.72ms] +(pass) deletion reports an uncertain outcome after network and retries the same ID [25.24ms] +(pass) a deletion failure can be cancelled and re-indexing blocks deletion until it finishes [23.87ms] +(pass) a failed list refresh after deletion offers a list retry without repeating DELETE [19.81ms] +(pass) deletion ignores late success after a selection change [21.66ms] +(pass) deletion ignores late failure after a selection change [22.85ms] +(pass) deletion ignores late success after a search change [25.52ms] +(pass) deletion ignores late failure after a search change [31.68ms] +(pass) deletion ignores late success after a page change [25.23ms] +(pass) deletion ignores late failure after a page change [21.51ms] +(pass) deletion ignores late success after a tab change [23.88ms] +(pass) deletion ignores late failure after a tab change [23.96ms] + +tests/document-indexing.test.tsx: +(pass) re-indexing requires an explicit click, posts the saved ID once while pending, and keeps the content visible [15.98ms] +(pass) re-indexing reports a safe failure for 404 and retries the same saved document [18.27ms] +(pass) re-indexing reports a safe failure for 422 and retries the same saved document [22.28ms] +(pass) re-indexing reports a safe failure for 503 and retries the same saved document [32.95ms] +(pass) re-indexing reports a safe failure for network and retries the same saved document [21.25ms] +(pass) re-indexing ignores a late success after a selection change [25.17ms] +(pass) re-indexing ignores a late failure after a selection change [24.07ms] +(pass) re-indexing ignores a late success after a search change [31.50ms] +(pass) re-indexing ignores a late failure after a search change [28.36ms] +(pass) re-indexing ignores a late success after a page change [27.22ms] +(pass) re-indexing ignores a late failure after a page change [26.08ms] +(pass) re-indexing ignores a late success after a tab change [30.83ms] +(pass) re-indexing ignores a late failure after a tab change [30.03ms] + +tests/retrieval.test.tsx: +(pass) retrieval waits for a non-blank question and submits trimmed optional scope [12.25ms] +(pass) retrieval omits blank scope, preserves rank and raw distances, and shows an unset cutoff [9.16ms] +(pass) retrieval disables inputs and duplicate submissions while loading [8.03ms] +(pass) retrieval shows an empty result and a zero distance cutoff [5.85ms] +(pass) retrieval replaces a failed request with a successful retry [21.22ms] +(pass) retrieval clears old results when the question or channel changes [24.66ms] +(pass) retrieval aborts on leaving the tab and ignores late responses after reopening [13.91ms] +(pass) retrieval renders chunk and metadata markup as text [14.73ms] + +tests/conversations.test.tsx: +(pass) loads only on opening the tab and paginates summaries and chronological turns independently [135.73ms] +(pass) shows loading, an empty page and a list error with a working retry [8.90ms] +(pass) retries missing conversation details without discarding the list [22.70ms] +(pass) ignores stale detail responses after selecting another conversation [20.42ms] +(pass) ignores stale errors after leaving and reopening the tab [7.24ms] +(pass) renders stored markup as text and supports missing metadata [8.04ms] +(pass) encodes conversation IDs in detail request paths [1.58ms] + + 51 pass + 0 fail + 370 expect() calls +Ran 51 tests across 5 files. [1.78s] diff --git a/.tmp/proof/rule-preview-install.txt b/.tmp/proof/rule-preview-install.txt new file mode 100644 index 0000000..db18bb1 --- /dev/null +++ b/.tmp/proof/rule-preview-install.txt @@ -0,0 +1,182 @@ +[install-backend] $ uv sync +[install-frontend] $ bun install +[install-runtime] $ bun install --frozen-lockfile --ignore-scripts +[install-frontend] bun install v1.3.5 (1e86cebd) +[install-runtime] bun install v1.3.5 (1e86cebd) +[install-backend] Using CPython 3.13.13 +[install-backend] Creating virtual environment at: .venv +[install-runtime] +[install-runtime] + @earendil-works/chord@1.3.0 +[install-runtime] + @earendil-works/pi-ai@1.3.0 +[install-runtime] + @earendil-works/pi-durable@1.3.0 +[install-runtime] +[install-runtime] 91 packages installed [1075.00ms] +[install-runtime] Finished in 1.14s +[install-backend] Resolved 143 packages in 1.47s +[install-backend] warning: `numpy==2.4.0` is yanked (reason: "Backward compatibility bug") +[install-backend] Building wingman @ file:///Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend +[install-frontend] +[install-frontend] + @tailwindcss/postcss@4.1.18 +[install-frontend] + @testing-library/react@16.3.3 +[install-frontend] + @types/bun@1.4.4 +[install-frontend] + eslint@9.39.2 +[install-frontend] + eslint-config-next@16.1.1 +[install-frontend] + happy-dom@21.0.4 +[install-frontend] + tailwindcss@4.1.18 +[install-frontend] + @types/node@25.0.3 +[install-frontend] + @types/react@19.2.7 +[install-frontend] + @types/react-dom@19.2.3 +[install-frontend] + axios@1.13.2 +[install-frontend] + next@16.1.1 +[install-frontend] + react@19.2.3 +[install-frontend] + react-dom@19.2.3 +[install-frontend] + swr@2.3.8 +[install-frontend] + typescript@5.9.3 +[install-frontend] +[install-frontend] 382 packages installed [1.72s] +[install-frontend] Finished in 1.78s +[install-backend] Built wingman @ file:///Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend +[install-backend] Prepared 1 package in 401ms +[install-backend] Installed 140 packages in 727ms +[install-backend] + aiohappyeyeballs==2.6.1 +[install-backend] + aiohttp==3.13.2 +[install-backend] + aiosignal==1.4.0 +[install-backend] + alembic==1.17.2 +[install-backend] + annotated-doc==0.0.4 +[install-backend] + annotated-types==0.7.0 +[install-backend] + anyio==4.12.0 +[install-backend] + asgiref==3.11.0 +[install-backend] + attrs==25.4.0 +[install-backend] + backoff==2.2.1 +[install-backend] + bcrypt==5.0.0 +[install-backend] + build==1.3.0 +[install-backend] + cachetools==6.2.4 +[install-backend] + certifi==2025.11.12 +[install-backend] + charset-normalizer==3.4.4 +[install-backend] + chroma-hnswlib==0.7.6 +[install-backend] + chromadb==0.6.3 +[install-backend] + click==8.3.1 +[install-backend] + coloredlogs==15.0.1 +[install-backend] + dataclasses-json==0.6.7 +[install-backend] + distro==1.9.0 +[install-backend] + durationpy==0.10 +[install-backend] + fastapi==0.128.0 +[install-backend] + filelock==3.20.1 +[install-backend] + flatbuffers==25.12.19 +[install-backend] + frozenlist==1.8.0 +[install-backend] + fsspec==2025.12.0 +[install-backend] + google-auth==2.45.0 +[install-backend] + googleapis-common-protos==1.72.0 +[install-backend] + grpcio==1.76.0 +[install-backend] + h11==0.16.0 +[install-backend] + hf-xet==1.2.0 +[install-backend] + httpcore==1.0.9 +[install-backend] + httptools==0.7.1 +[install-backend] + httpx==0.28.1 +[install-backend] + httpx-sse==0.4.3 +[install-backend] + huggingface-hub==0.36.0 +[install-backend] + humanfriendly==10.0 +[install-backend] + idna==3.11 +[install-backend] + importlib-metadata==8.7.1 +[install-backend] + importlib-resources==6.5.2 +[install-backend] + iniconfig==2.3.0 +[install-backend] + jiter==0.12.0 +[install-backend] + jsonpatch==1.33 +[install-backend] + jsonpointer==3.0.0 +[install-backend] + kubernetes==34.1.0 +[install-backend] + langchain==1.2.0 +[install-backend] + langchain-classic==1.0.1 +[install-backend] + langchain-community==0.4.1 +[install-backend] + langchain-core==1.2.5 +[install-backend] + langchain-openai==1.1.6 +[install-backend] + langchain-text-splitters==1.1.0 +[install-backend] + langgraph==1.0.5 +[install-backend] + langgraph-checkpoint==3.0.1 +[install-backend] + langgraph-prebuilt==1.0.5 +[install-backend] + langgraph-sdk==0.3.1 +[install-backend] + langsmith==0.5.2 +[install-backend] + lxml==6.0.2 +[install-backend] + mako==1.3.10 +[install-backend] + markdown-it-py==4.0.0 +[install-backend] + markupsafe==3.0.3 +[install-backend] + marshmallow==3.26.2 +[install-backend] + mdurl==0.1.2 +[install-backend] + mmh3==5.2.0 +[install-backend] + mpmath==1.3.0 +[install-backend] + multidict==6.7.0 +[install-backend] + mypy-extensions==1.1.0 +[install-backend] + numpy==2.4.0 +[install-backend] + oauthlib==3.3.1 +[install-backend] + onnxruntime==1.23.2 +[install-backend] + openai==2.14.0 +[install-backend] + opentelemetry-api==1.39.1 +[install-backend] + opentelemetry-exporter-otlp-proto-common==1.39.1 +[install-backend] + opentelemetry-exporter-otlp-proto-grpc==1.39.1 +[install-backend] + opentelemetry-instrumentation==0.60b1 +[install-backend] + opentelemetry-instrumentation-asgi==0.60b1 +[install-backend] + opentelemetry-instrumentation-fastapi==0.60b1 +[install-backend] + opentelemetry-proto==1.39.1 +[install-backend] + opentelemetry-sdk==1.39.1 +[install-backend] + opentelemetry-semantic-conventions==0.60b1 +[install-backend] + opentelemetry-util-http==0.60b1 +[install-backend] + orjson==3.11.5 +[install-backend] + ormsgpack==1.12.1 +[install-backend] + overrides==7.7.0 +[install-backend] + packaging==25.0 +[install-backend] + pluggy==1.6.0 +[install-backend] + posthog==7.4.2 +[install-backend] + propcache==0.4.1 +[install-backend] + protobuf==6.33.2 +[install-backend] + psycopg2-binary==2.9.11 +[install-backend] + pyasn1==0.6.1 +[install-backend] + pyasn1-modules==0.4.2 +[install-backend] + pydantic==2.12.5 +[install-backend] + pydantic-core==2.41.5 +[install-backend] + pydantic-settings==2.12.0 +[install-backend] + pygments==2.19.2 +[install-backend] + pypdf==6.5.0 +[install-backend] + pypika==0.48.9 +[install-backend] + pyproject-hooks==1.2.0 +[install-backend] + pytest==9.0.2 +[install-backend] + pytest-asyncio==1.3.0 +[install-backend] + python-dateutil==2.9.0.post0 +[install-backend] + python-docx==1.2.0 +[install-backend] + python-dotenv==1.2.1 +[install-backend] + pyyaml==6.0.3 +[install-backend] + regex==2025.11.3 +[install-backend] + requests==2.32.5 +[install-backend] + requests-oauthlib==2.0.0 +[install-backend] + requests-toolbelt==1.0.0 +[install-backend] + rich==14.2.0 +[install-backend] + rsa==4.9.1 +[install-backend] + shellingham==1.5.4 +[install-backend] + six==1.17.0 +[install-backend] + slack-bolt==1.27.0 +[install-backend] + slack-sdk==3.42.0 +[install-backend] + sniffio==1.3.1 +[install-backend] + sqlalchemy==2.0.45 +[install-backend] + starlette==0.50.0 +[install-backend] + sympy==1.14.0 +[install-backend] + tenacity==9.1.2 +[install-backend] + tiktoken==0.12.0 +[install-backend] + tokenizers==0.22.1 +[install-backend] + tqdm==4.67.1 +[install-backend] + typer==0.21.0 +[install-backend] + typing-extensions==4.15.0 +[install-backend] + typing-inspect==0.9.0 +[install-backend] + typing-inspection==0.4.2 +[install-backend] + urllib3==2.3.0 +[install-backend] + uuid-utils==0.12.0 +[install-backend] + uvicorn==0.40.0 +[install-backend] + uvloop==0.22.1 +[install-backend] + watchfiles==1.1.1 +[install-backend] + websocket-client==1.9.0 +[install-backend] + websockets==15.0.1 +[install-backend] + wingman==1.1.0 (from file:///Users/johnny/.paseo/worktrees/0x03bguv/tasty-hedgehog/backend) +[install-backend] + wrapt==1.17.3 +[install-backend] + xxhash==3.6.0 +[install-backend] + yarl==1.22.0 +[install-backend] + zipp==3.23.0 +[install-backend] + zstandard==0.25.0 +[install-backend] Finished in 2.89s +Finished in 2.90s diff --git a/.tmp/proof/rule-preview-runtime-tests.txt b/.tmp/proof/rule-preview-runtime-tests.txt new file mode 100644 index 0000000..474c51a --- /dev/null +++ b/.tmp/proof/rule-preview-runtime-tests.txt @@ -0,0 +1,16 @@ +[test-runtime] $ node --test +✔ real Pi SQLite storage preserves answers, deduplicates retries, and retains conversation history (36.818625ms) +✔ outbox resumes after restart without regenerating answer; retry admits only one job (4.123667ms) +✔ a process killed during generation resumes the admitted Pi submission (211.4125ms) +✔ pinned operator answer is delivered without calling the model (1.618084ms) +✔ model failure sends a persisted Slack failure notification (0.360125ms) +✔ runtime starts with openai catalog and protects job routes (645.452209ms) +✔ runtime starts with mimo catalog and protects job routes (466.165042ms) +ℹ tests 7 +ℹ suites 0 +ℹ pass 7 +ℹ fail 0 +ℹ cancelled 0 +ℹ skipped 0 +ℹ todo 0 +ℹ duration_ms 1204.200291 diff --git a/.tmp/proof/rule-preview-sitecustomize.py b/.tmp/proof/rule-preview-sitecustomize.py new file mode 100644 index 0000000..14652fa --- /dev/null +++ b/.tmp/proof/rule-preview-sitecustomize.py @@ -0,0 +1,21 @@ +"""Offline proof hook: fixture Slack authentication and deny external work.""" +from slack_sdk import WebClient +from slack_bolt.adapter.socket_mode import SocketModeHandler + + +def blocked(*args, **kwargs): + raise AssertionError("Slack, retrieval and Pi calls are forbidden in this proof") + + +WebClient.auth_test = lambda *args, **kwargs: { + "ok": True, "bot_id": "BTEST", "user_id": "UTEST", "team_id": "TTEST", +} +WebClient.api_call = blocked +SocketModeHandler.start = blocked + +from app import agent_runtime +from app.vector_store import vector_store + +agent_runtime.generate = blocked +agent_runtime.submit = blocked +vector_store.similarity_search = blocked diff --git a/backend/app/main.py b/backend/app/main.py index 49c311f..890863e 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -61,6 +61,23 @@ class QuestionResponse(BaseModel): NonBlankString = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] +class OperatorRulePreviewRequest(BaseModel): + question: NonBlankString + + +class OperatorRuleMatch(BaseModel): + rule_id: str + match_type: str + matched_keyword: str + priority: int + response: str + + +class OperatorRulePreviewResponse(BaseModel): + question: str + match: OperatorRuleMatch | None + + class RetrievalPreviewRequest(BaseModel): question: NonBlankString channel_id: NonBlankString | None = None @@ -226,6 +243,26 @@ async def ask_question(request: QuestionRequest): raise HTTPException(status_code=500, detail=str(e)) +@app.post("/api/rules/preview", response_model=OperatorRulePreviewResponse) +def preview_operator_rule(request: OperatorRulePreviewRequest): + """Inspect the loaded rule winner without retrieval, generation or delivery.""" + try: + matched = rag_engine.match_operator_rule(request.question) + return OperatorRulePreviewResponse( + question=request.question, + match=OperatorRuleMatch( + rule_id=matched.rule.id, + match_type=matched.rule.match_type, + matched_keyword=matched.matched_keyword, + priority=matched.rule.priority, + response=matched.rule.response, + ) if matched is not None else None, + ) + except Exception: + logger.exception("Operator rule preview failed") + raise HTTPException(status_code=503, detail="Operator rule preview is unavailable") + + @app.post("/api/retrieval/preview", response_model=RetrievalPreviewResponse) def preview_retrieval(request: RetrievalPreviewRequest): """Inspect selected chunks and distances without generating an answer.""" diff --git a/backend/tests/test_rule_preview.py b/backend/tests/test_rule_preview.py new file mode 100644 index 0000000..aa348f8 --- /dev/null +++ b/backend/tests/test_rule_preview.py @@ -0,0 +1,125 @@ +"""Operator rule preview tests with fixture config and blocked external work.""" +import os +from unittest.mock import Mock, patch + +import pytest +from fastapi.testclient import TestClient + +os.environ.setdefault("SLACK_BOT_TOKEN", "xoxb-fixture-only") +os.environ.setdefault("SLACK_SIGNING_SECRET", "fixture-only") + +with patch("slack_sdk.WebClient.auth_test", return_value={ + "ok": True, "bot_id": "BTEST", "user_id": "UTEST", "team_id": "TTEST", +}): + import app.main as main + +from app import agent_runtime, database, rules + + +@pytest.fixture +def rule_api(monkeypatch): + blocked = Mock(side_effect=AssertionError("Preview must only match loaded rules")) + monkeypatch.setattr(main.vector_store, "similarity_search", blocked) + monkeypatch.setattr(agent_runtime, "generate", blocked) + monkeypatch.setattr(agent_runtime, "submit", blocked) + monkeypatch.setattr(database, "SessionLocal", blocked) + monkeypatch.setattr("slack_sdk.WebClient.api_call", blocked) + monkeypatch.setattr("slack_bolt.adapter.socket_mode.SocketModeHandler.start", blocked) + monkeypatch.setattr(rules, "rule_engine", rules.RuleEngine()) + monkeypatch.setattr(main, "init_db", lambda: None) + with TestClient(main.app, raise_server_exceptions=False) as client: + # Startup storage is unrelated; the endpoint itself needs no database. + yield client + blocked.assert_not_called() + + +def install_rules(monkeypatch, *entries): + monkeypatch.setattr(rules, "rule_engine", rules.RuleEngine(rules.parse_rules(list(entries)))) + + +def rule(rule_id="vpn", **options): + return {"id": rule_id, "keywords": ["reset vpn"], "response": "Use the IT portal.", **options} + + +def preview(client, question="How do I RESET VPN access?"): + return client.post("/api/rules/preview", json={"question": question}) + + +@pytest.mark.parametrize(("match_type", "keywords", "question", "winner"), [ + ("contains", ["vpn password", "reset vpn"], "How do I RESET VPN access?", "reset vpn"), + ("exact", ["office hours"], "OFFICE HOURS", "office hours"), + ("regex", [r"ticket #?\d+"], "Where is TICKET #4821?", r"ticket #?\d+"), +]) +def test_preview_returns_winning_rule_and_exact_response(rule_api, monkeypatch, match_type, keywords, question, winner): + response_text = " Approved wording.\nKeep the formatting. " + install_rules(monkeypatch, rule(match_type=match_type, keywords=keywords, response=response_text, priority=3)) + response = preview(rule_api, question) + assert response.status_code == 200 + assert response.json() == { + "question": question, + "match": { + "rule_id": "vpn", "match_type": match_type, "matched_keyword": winner, + "priority": 3, "response": response_text, + }, + } + + +def test_preview_uses_priority_config_order_and_disabled_rule_policy(rule_api, monkeypatch): + install_rules(monkeypatch, + rule("low", priority=1), rule("disabled", priority=100, enabled=False), + rule("first", priority=10), rule("second", priority=10), + ) + assert preview(rule_api).json()["match"]["rule_id"] == "first" + + +@pytest.mark.parametrize("entries", [[], [rule()], [rule(enabled=False)]]) +def test_preview_returns_null_for_no_match_without_falling_back_to_generation(rule_api, monkeypatch, entries): + install_rules(monkeypatch, *entries) + response = preview(rule_api, "How do I reset email?") + assert response.status_code == 200 + assert response.json() == {"question": "How do I reset email?", "match": None} + + +def test_preview_trims_question_and_uses_current_shared_matcher(rule_api, monkeypatch): + install_rules(monkeypatch, rule("first")) + question = "How do I reset VPN?" + response = preview(rule_api, f" \t{question}\n ") + assert response.json()["question"] == question + assert response.json()["match"]["rule_id"] == "first" + install_rules(monkeypatch, rule("replacement")) + assert preview(rule_api, question).json()["match"]["rule_id"] == "replacement" + + +@pytest.mark.parametrize("payload", [ + {}, {"question": ""}, {"question": " \n\t "}, {"question": None}, + {"question": 123}, {"question": True}, {"question": []}, {"question": {}}, +]) +def test_preview_rejects_invalid_input_before_matching(rule_api, monkeypatch, payload): + matcher = Mock(side_effect=AssertionError("Invalid input must not reach matching")) + monkeypatch.setattr(main.rag_engine, "match_operator_rule", matcher) + assert rule_api.post("/api/rules/preview", json=payload).status_code == 422 + matcher.assert_not_called() + + +def test_preview_matches_api_answer_and_slack_job_without_submitting_a_job(rule_api, monkeypatch): + install_rules(monkeypatch, rule("low"), rule("approved", priority=10)) + question = "How do I reset VPN?" + matched = preview(rule_api, question).json()["match"] + agent_runtime.submit.assert_not_called() + answer = main.rag_engine.generate_response(question) + assert answer["answer"] == matched["response"] + assert answer["sources"] == [{"source": "operator_rule", "rule_id": matched["rule_id"]}] + # Replace the blocked submit only for this separate parity check. Nothing is delivered. + submit = Mock(return_value={"id": "fixture-job", "status": "pending"}) + monkeypatch.setattr(agent_runtime, "submit", submit) + main.rag_engine.submit_slack(question, "CFIXTURE", "fixture-conversation", "fixture-request", "1.0") + payload = submit.call_args.args[0] + assert payload["operator_answer"] == matched["response"] + assert payload["sources"] == answer["sources"] + + +def test_preview_reports_matcher_failure_without_exposing_details(rule_api, monkeypatch): + monkeypatch.setattr(main.rag_engine, "match_operator_rule", Mock(side_effect=RuntimeError("private config details"))) + response = preview(rule_api) + assert response.status_code == 503 + assert response.json() == {"detail": "Operator rule preview is unavailable"} diff --git a/docs/operator-rules.md b/docs/operator-rules.md index df98f28..402100f 100644 --- a/docs/operator-rules.md +++ b/docs/operator-rules.md @@ -6,7 +6,9 @@ or a generated answer. Use them when the generic answer is wrong or stale and the correct wording is known. Rules are checked on the `/api/ask` path through `RAGEngine.generate_response` -in [backend/app/rag.py](../backend/app/rag.py). The matcher lives in +and on the Slack submit path through `RAGEngine.submit_slack` in +[backend/app/rag.py](../backend/app/rag.py). Slack's pinned answers use the Pi +outbox for delivery without model generation. The matcher lives in [backend/app/rules.py](../backend/app/rules.py). ## Configure @@ -64,9 +66,53 @@ match type, a non-integer priority or a bad regex is logged and skipped. Invalid JSON, a missing file or a non-list config loads as no rules. Bad config never stops the backend from starting. +## Preview the winning rule + +`POST /api/rules/preview` uses the backend's loaded matcher to show which rule +would answer a question. It does not retrieve chunks, generate an answer, +submit a Pi job, write to the database or send a Slack message. An unmatched +question returns a null match without falling back to generation. + +```sh +curl -sS http://localhost:8000/api/rules/preview \ + -H 'Content-Type: application/json' \ + -d '{"question":"How do I RESET VPN access?"}' +``` + +With the example rules above, the response is: + +```json +{ + "question": "How do I RESET VPN access?", + "match": { + "rule_id": "vpn-reset", + "match_type": "contains", + "matched_keyword": "reset vpn", + "priority": 10, + "response": "Use the IT portal at https://it.example.com/vpn to reset VPN access." + } +} +``` + +The winner follows the existing priority, config-order and enabled-rule policy. +The response preserves the approved answer's formatting. Surrounding question +whitespace is trimmed; missing, blank and non-string questions return HTTP 422 +before matching. No match returns HTTP 200 with `"match": null`. Matcher failures +return HTTP 503 with a generic detail rather than configuration internals. + +Rules are loaded once per process. Preview does not reload a file or validate +unsaved configuration. Restart both backend and bot after changing the rule +configuration; a backend preview cannot confirm what a separately running bot +has loaded. For Slack mentions, preview the question after removing the bot +mention, as the handler does before matching. + +Like the other management endpoints, preview has no built-in authentication. +Keep it behind trusted access controls: it returns the full approved answer +and the pattern that matched, which can contain internal information. It +does not evaluate source permissions or whether the answer is still correct. + ## Current limits -Rules apply to the API ask path in this slice. Slack delivery still goes -through the Pi Durable job flow, so wiring rules into the Slack handlers is -a follow-up. Combining a rule with fetched source data before summarising -is also still open under the parent issue. +Combining a rule with fetched source data before summarising remains open +under the parent issue. Preview inspects the current deterministic winner; +it does not list shadowed rules or change matching policy. diff --git a/mise.toml b/mise.toml index 48884ee..bb2f1be 100644 --- a/mise.toml +++ b/mise.toml @@ -51,7 +51,7 @@ run = "docker compose ps" [tasks.test-backend] description = "Run backend tests" dir = "backend" -run = "uv run pytest tests/test_file_extraction_units.py tests/test_logging_config.py tests/test_logging.py tests/test_agent_runtime.py tests/test_rules.py tests/test_conversations.py tests/test_documents.py tests/test_retrieval.py tests/test_thread_indexing.py tests/test_retrieval_preview.py tests/test_document_indexing.py tests/test_document_deletion.py -v" +run = "uv run pytest tests/test_file_extraction_units.py tests/test_logging_config.py tests/test_logging.py tests/test_agent_runtime.py tests/test_rules.py tests/test_conversations.py tests/test_documents.py tests/test_retrieval.py tests/test_thread_indexing.py tests/test_retrieval_preview.py tests/test_document_indexing.py tests/test_document_deletion.py tests/test_rule_preview.py -v" [tasks.migrate] description = "Run database migrations (upgrade to latest)"