diff --git a/packages/gooddata-eval/AGENTS.md b/packages/gooddata-eval/AGENTS.md index b615bc592..183cb387c 100644 --- a/packages/gooddata-eval/AGENTS.md +++ b/packages/gooddata-eval/AGENTS.md @@ -27,7 +27,7 @@ experiment. The newest and most actively developed package in the repo. | `core/agentic/` | multi-turn agentic evaluation per capability, **plus** all Langfuse trace polling and linking (`_langfuse.py`, `_trace_linker.py`) | | `core/chat/` | SSE client for the agent's streaming chat endpoint | | `core/summary/` | HTTP client for the dedicated dashboard-summary endpoint — a single-shot chat backend, not reporting | -| `core/takeaway/` | HTTP client for the AI Publisher takeaway endpoint — a single-shot backend like `core/summary/`, keeping the whole response on `ChatResult.raw_response` — plus `evaluate_document_takeaway`, the K-run, gate and Langfuse entry point the Tavern shim calls | +| `core/takeaway/` | HTTP client for the AI Publisher takeaway endpoint — a single-shot backend like `core/summary/`, keeping the whole response on `ChatResult.raw_response` — plus `evaluate_document_takeaway`, the K-run, gate and Langfuse entry point the Tavern shim calls. Also used by the agentic document skill to write the takeaways the copilot leaves to the editor, before the narrative is graded | | `core/dataset/` | dataset format, loading, and `from_insights.py` — dataset generation from a workspace's real insights | | `core/evaluators/` | single-shot evaluators and their registry | | `core/langfuse/` | the whole Langfuse v4 client: `_env` (base URL + credentials), `otlp` (OTLP/JSON encoding), `experiment` (root-span construction, score targets), `item_scope` (an item's experiment scope sent to gen-ai as baggage, and which conversations joined), `observations` (trace reads), `client` (httpx calls), `sink` (single-shot results as experiments) | diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index 71efce2db..dfda520f4 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -277,7 +277,8 @@ Each item additionally carries a per-phase breakdown: ``` Every item reports `runs_ungraded` beside `runs_passed`: runs the agent answered but the LLM judge returned -nothing readable for. Agentic items also list them as `unscored_runs` / `judge_errors` in their `detail`, and a +nothing readable for. Agentic items also list them as `unscored_runs` / `judge_errors` in their `detail` (a document item whose takeaway-endpoint request failed +lists that failure under `request_errors` instead), and a `dashboard_summary` item carries `ungraded_criteria`. Such a run — or, for `dashboard_summary`, such a criterion — is excluded from pass@K and from the quality score rather than counted as a failure: scoring it 0 would be indistinguishable from the judge genuinely failing the answer, which is the confusion diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_editor_takeaways.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_editor_takeaways.py new file mode 100644 index 000000000..bbdd3c3cf --- /dev/null +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_editor_takeaways.py @@ -0,0 +1,274 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +"""The takeaway request the Publisher editor sends for an unwritten takeaway of a v2 document. + +Every visualization on the page, in layout order, each under the filters it is drawn with, a +visualization drawn twice the same way sent once; ``filterContext`` empty; the takeaway's and the +document's instructions. A visualization is drawn with the period on its own date dataset unless it +ignores the period, a document date filter on that dataset with both ends set in the period's place +(keeping its ``emptyValueHandling`` and, for a relative one, its ``boundedFilter``), +and every document attribute and text filter that narrows the data. An attribute filter excluding +nothing and a text filter with no value narrow nothing and are left out; measure value filters +and any other kind are left out too, since the renderer draws no chart differently for them. Widget +and page filters, and ``ignoreDocumentFilters``, are stored but not applied, so they are not applied +here either. + +Two parts of the editor's request are left out: ``maxCharacters``, which the editor derives from the +layout geometry and the theme, so the endpoint's default length applies; and resolving text variables +in the instructions, which are sent as written, as the editor does while it has no values for them. + +Mirrors gdc-ui at commit 7e0f3719ca2 (``documentTakeawayRequest`` and ``copilotTakeaways`` of the +publisher's v2 editor); check them again when the editor's request changes. + +Pure: no network. +""" + +from __future__ import annotations + +import json +from typing import Any + +from gooddata_eval.core.models import TakeawayInput, TakeawayInstructions, TakeawayVisualization + +# Publisher periods and dashboard date filters name granularities as GDC.time.*; AFM date filters +# take the platform enum. +_AFM_GRANULARITY = { + "GDC.time.year": "YEAR", + "GDC.time.quarter": "QUARTER", + "GDC.time.month": "MONTH", + "GDC.time.week_us": "WEEK", + "GDC.time.week": "WEEK", + "GDC.time.date": "DAY", + "GDC.time.hour": "HOUR", + "GDC.time.minute": "MINUTE", + "GDC.time.second": "SECOND", + "GDC.time.quarter_in_year": "QUARTER_OF_YEAR", + "GDC.time.month_in_year": "MONTH_OF_YEAR", + "GDC.time.week_in_year": "WEEK_OF_YEAR", + "GDC.time.day_in_year": "DAY_OF_YEAR", + "GDC.time.day_in_month": "DAY_OF_MONTH", + "GDC.time.day_in_week": "DAY_OF_WEEK", + "GDC.time.hour_in_day": "HOUR_OF_DAY", + "GDC.time.minute_in_hour": "MINUTE_OF_HOUR", + "GDC.time.minute_in_day": "MINUTE_OF_DAY", + "GDC.time.second_in_day": "SECOND_OF_DAY", + "GDC.time.second_in_minute": "SECOND_OF_MINUTE", + "GDC.time.fiscal_month": "FISCAL_MONTH", + "GDC.time.fiscal_quarter": "FISCAL_QUARTER", + "GDC.time.fiscal_year": "FISCAL_YEAR", +} + + +class _Unbuildable(Exception): + """The editor's request has a part this module cannot reproduce.""" + + +def _identifier(reference: Any) -> str | None: + """The id of a reference written as ``{"identifier": "..."}``, ``{"identifier": {"id": ...}}`` or a bare string.""" + if isinstance(reference, dict): + reference = reference.get("identifier") + if isinstance(reference, dict): + reference = reference.get("id") + return reference if isinstance(reference, str) else None + + +def _date_filter( + dataset: str, kind: Any, granularity: Any, start: Any, end: Any, options: dict[str, Any] | None = None +) -> dict: + """The AFM date filter; ``options`` are the document filter's ``emptyValueHandling`` and ``boundedFilter``.""" + dataset_ref = {"identifier": {"id": dataset, "type": "dataset"}} + options = options or {} + if kind == "absolute": + return { + "absoluteDateFilter": { + "dataset": dataset_ref, + "from": start, + "to": end, + **{key: value for key, value in options.items() if key == "emptyValueHandling"}, + } + } + if kind == "relative": + afm_granularity = _AFM_GRANULARITY.get(granularity) if isinstance(granularity, str) else None + if afm_granularity is None: + raise _Unbuildable(f"relative date granularity {granularity!r} has no AFM form") + return { + "relativeDateFilter": { + "dataset": dataset_ref, + "granularity": afm_granularity, + "from": start, + "to": end, + **options, + } + } + raise _Unbuildable(f"date filter type {kind!r} has no AFM form") + + +def _narrows_the_date(date_filter: dict) -> bool: + return date_filter.get("from") is not None and date_filter.get("to") is not None + + +def _date_filters(widget: dict, period: Any, filters: list[Any]) -> list[dict]: + dataset = _identifier(widget.get("dateDataSet")) + if dataset is None: + return [] + for item in filters: + date_filter = item.get("dateFilter") if isinstance(item, dict) else None + if ( + isinstance(date_filter, dict) + and _narrows_the_date(date_filter) + and _identifier(date_filter.get("dataSet")) == dataset + ): + return [ + _date_filter( + dataset, + date_filter.get("type"), + date_filter.get("granularity"), + date_filter.get("from"), + date_filter.get("to"), + { + key: date_filter[key] + for key in ("emptyValueHandling", "boundedFilter") + if date_filter.get(key) is not None + }, + ) + ] + if not isinstance(period, dict) or widget.get("ignoreDocumentPeriod") is True: + return [] + return [_date_filter(dataset, period.get("type"), period.get("granularity"), period.get("from"), period.get("to"))] + + +_MATCH_TYPES = {"contains": "CONTAINS", "startsWith": "STARTS_WITH", "endsWith": "ENDS_WITH"} + + +def _label_ref(body: dict) -> dict: + label = _identifier(body.get("displayForm")) + if label is None: + raise _Unbuildable(f"filter {body!r} names no label, so it has no AFM form") + return {"identifier": {"id": label, "type": "label"}} + + +def _element_filter(label_ref: dict, values: list[Any], negative: bool) -> dict | None: + """Selected elements as AFM; None for an exclusion of nothing, which keeps every row.""" + if negative: + return {"negativeAttributeFilter": {"label": label_ref, "notIn": {"values": values}}} if values else None + return {"positiveAttributeFilter": {"label": label_ref, "in": {"values": values}}} + + +def _attribute_filter(body: dict) -> dict | None: + elements = body.get("attributeElements") + elements = elements if isinstance(elements, dict) else {} + values = elements.get("uris") or elements.get("values") or [] + if not isinstance(values, list): + raise _Unbuildable(f"attribute filter {body!r} has no element list, so it has no AFM form") + negative = body.get("negativeSelection") is True + if not values and not negative: + raise _Unbuildable(f"attribute filter {body!r} selects no element, which has no AFM form") + return _element_filter(_label_ref(body), values, negative) + + +def _arbitrary_attribute_filter(body: dict) -> dict | None: + values = body.get("values") + if not isinstance(values, list): + raise _Unbuildable(f"text filter {body!r} has no value list, so it has no AFM form") + negative = body.get("negativeSelection") is True + # Tiger drops a text filter without values, so the editor's request carries none. + return _element_filter(_label_ref(body), values, negative) if values else None + + +def _match_attribute_filter(body: dict) -> dict: + operator = body.get("operator") + match_type = _MATCH_TYPES.get(operator) if isinstance(operator, str) else None + literal = body.get("literal") + if match_type is None or not isinstance(literal, str): + raise _Unbuildable(f"text filter {body!r} has no AFM form") + return { + "matchAttributeFilter": { + "label": _label_ref(body), + "literal": literal, + "matchType": match_type, + "caseSensitive": body.get("caseSensitive") is True, + "negate": body.get("negativeSelection") is True, + } + } + + +_ATTRIBUTE_KINDS = { + "attributeFilter": _attribute_filter, + "arbitraryAttributeFilter": _arbitrary_attribute_filter, + "matchAttributeFilter": _match_attribute_filter, +} + + +def _attribute_filters(filters: list[Any]) -> list[dict]: + """The AFM form of every document filter other than a date filter that narrows the data.""" + afm: list[dict] = [] + for item in filters: + if not isinstance(item, dict): + raise _Unbuildable(f"document filter {item!r} is not an object") + kind = next((kind for kind in _ATTRIBUTE_KINDS if kind in item), None) + if kind is None: + continue + body = item[kind] + if not isinstance(body, dict): + raise _Unbuildable(f"document filter {item!r} has no AFM form") + converted = _ATTRIBUTE_KINDS[kind](body) + if converted is not None: + afm.append(converted) + return afm + + +def _visualization_widgets(node: Any) -> list[dict]: + """Every visualization widget with a visualization under ``node``, in layout order.""" + if not isinstance(node, dict): + return [] + children = node.get("children") + if isinstance(children, list): + return [widget for child in children for widget in _visualization_widgets(child)] + if node.get("type") == "visualization" and _identifier(node.get("visualization")) is not None: + return [node] + return [] + + +def _instruction(text: Any) -> str | None: + written = text.strip() if isinstance(text, str) else "" + return written or None + + +def _generation_instructions(holder: Any) -> Any: + generation = holder.get("generation") if isinstance(holder, dict) else None + return generation.get("instructions") if isinstance(generation, dict) else None + + +def editor_takeaway_input(document: dict, page: dict, takeaway: dict) -> TakeawayInput | str: + """The request the editor sends to write ``takeaway`` on ``page``, or why it cannot be built. + + ``document`` is a v2 document in its declarative form, the one the editor reads, with its content + keys under ``content`` or at the top level. + """ + content = document.get("content") + content = content if isinstance(content, dict) else document + filters = content.get("filters") or [] + if not isinstance(filters, list): + return f"document filters {filters!r} are not a list" + try: + attribute_filters = _attribute_filters(filters) + placements: dict[str, TakeawayVisualization] = {} + for widget in _visualization_widgets(page.get("layout")): + placement = TakeawayVisualization( + visualization_id=_identifier(widget.get("visualization")) or "", + filters=[*_date_filters(widget, document.get("period"), filters), *attribute_filters], + ) + placements.setdefault(json.dumps(placement.model_dump(), sort_keys=True), placement) + except _Unbuildable as exc: + return str(exc) + if not placements: + return f"page {page.get('id')!r} has no visualization to summarize" + defaults = content.get("defaults") + takeaway_defaults = defaults.get("takeaway") if isinstance(defaults, dict) else None + return TakeawayInput( + visualizations=list(placements.values()), + filter_context=[], + instructions=TakeawayInstructions( + slot=_instruction(_generation_instructions(takeaway)), + document=_instruction(_generation_instructions(takeaway_defaults)), + ), + ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py index b0cf53062..b06182647 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/document_skill.py @@ -3,13 +3,18 @@ from __future__ import annotations +import copy import re import time from collections.abc import Iterator from dataclasses import dataclass, field, replace from typing import Any +import httpx +from gooddata_sdk.catalog.workspace.aac import aac_document_to_declarative + from gooddata_eval.core.agentic._conversation_context import classify_reply +from gooddata_eval.core.agentic._editor_takeaways import editor_takeaway_input from gooddata_eval.core.agentic._failed_runs import build_failed_runs from gooddata_eval.core.agentic._gate import ( DEFAULT_GATE, @@ -44,6 +49,7 @@ build_latency_breakdown, shift_and_index_events, ) +from gooddata_eval.core.takeaway.http_client import TakeawayClient, is_transient from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings _DEFAULT_K = 1 @@ -83,6 +89,9 @@ _V2 = "2" _V2_HEADING_WIDGETS = frozenset({"title", "heading"}) _V2_TAKEAWAY_WIDGET = "takeaway" +# The draft_document result field listing the pages whose takeaway the copilot left for the editor +# to write from the data. +_EDITOR_PAGES_KEY = "takeaway_editor_page_ids" _NARRATIVE_EVALUATION_STEPS = [ "The actual output is a document rendered as text: its title, period, and per page the heading, " @@ -541,20 +550,24 @@ class DocumentEvaluation: # Set when the judge returned something unreadable. The run then has no narrative verdict: # it is neither published as a 0 nor allowed to pass on the remaining checks. judge_error: str | None = None + # Set when the only takeaways left without text are those the takeaway endpoint failed transiently + # to write. The summaries and the narrative then have no verdict, as above. + fill_error: str | None = None failures: list[str] = field(default_factory=list) @property def strict_pass(self) -> bool: - return self.judge_error is None and all(self.strict_checks.values()) + return self.judge_error is None and self.fill_error is None and all(self.strict_checks.values()) @property def ungraded(self) -> bool: - """The judge returned nothing readable and the narrative was the only check still open. + """The judge or the takeaway endpoint left the narrative without a verdict, and it was the only check still open. A run that already failed another check is a failure whatever the judge would have said, - so a judge error there leaves it failed rather than ungraded. + so a judge or fill error there leaves it failed rather than ungraded. A fill error is only + set when nothing but the transient endpoint failure left a takeaway without text. """ - return self.judge_error is not None and all(self.strict_checks.values()) + return (self.judge_error is not None or self.fill_error is not None) and all(self.strict_checks.values()) @property def strict_checks(self) -> dict[str, bool]: @@ -573,7 +586,7 @@ def strict_checks(self) -> dict[str, bool]: checks["document_period_correct"] = self.period_correct if self.applies.charts: checks["document_charts_matched"] = self.charts_matched - if self.applies.narrative: + if self.applies.narrative and self.fill_error is None: checks["document_summaries_present"] = self.summaries_present if self.judge_error is None: checks["document_narrative_judged"] = self.narrative_judged @@ -618,6 +631,7 @@ def evaluate_document_response( *, skill_activated: bool, legacy_names: list[str] | None = None, + exempt_takeaways: frozenset[tuple[str, str]] = frozenset(), ) -> DocumentEvaluation: """Score one document response against its expectation. @@ -625,10 +639,15 @@ def evaluate_document_response( Any of them fails ``document_wire_names`` and leads the failures, because a gen-ai still on the Report names also fails every check that looks for the Document ones. + ``exempt_takeaways`` are ``(page id, takeaway id)`` pairs that may stay empty, from + ``_saved_empty_takeaways``. + Pure: no network and no conversation state, so the whole assertion surface is unit-testable without an agent. """ - evaluation = _score_response(tool_result, document_part, expected_output, skill_activated=skill_activated) + evaluation = _score_response( + tool_result, document_part, expected_output, skill_activated=skill_activated, exempt_takeaways=exempt_takeaways + ) if legacy_names: evaluation.wire_names_current = False evaluation.failures.insert(0, f"gen-ai sends the Report names: {', '.join(legacy_names)}") @@ -697,6 +716,7 @@ def _score_response( expected_output: dict, *, skill_activated: bool, + exempt_takeaways: frozenset[tuple[str, str]] = frozenset(), ) -> DocumentEvaluation: rework = _rework(expected_output) applies = _Applies( @@ -810,7 +830,7 @@ def _score_response( unwritten = [ f"page {page.get('id')!r} has a {_slot_name(slot)} with no written text" for page, slot in slots - if _summary_text(slot) is None + if _summary_text(slot) is None and (page.get("id"), slot.get("id")) not in exempt_takeaways ] if not slots: missing = "takeaway" if _is_v2(document) else "summary slot" @@ -839,6 +859,199 @@ def _score_response( ) +@dataclass +class _EditorFill: + """The document part with the takeaways the editor would write filled in, and how that went.""" + + document_part: dict + filled_page_ids: list[str] = field(default_factory=list) + failures: list[str] = field(default_factory=list) + # The pages whose takeaway the endpoint failed transiently to write, and the first such failure. + transient_page_ids: set[str] = field(default_factory=set) + error: str | None = None + # The editor starts every takeaway at once, so the reader waits for the slowest request, retries included. + waited_s: float = 0.0 + # The pages with a takeaway the editor writes, as the editor's rule decides; unknown when the draft + # did not convert. + generated_page_ids: set[str] = field(default_factory=set) + readable: bool = True + # The takeaways allowed to stay empty, see ``_saved_empty_takeaways``. + exempt: frozenset[tuple[str, str]] = frozenset() + + def ungraded_error(self) -> str | None: + """The transient failure, when it alone left takeaways of the filled part without text.""" + document = self.document_part.get(_PART_TYPE) + if self.error is None or self.failures or not isinstance(document, dict): + return None + unwritten = { + page.get("id") + for _number, page in _content_pages(document) + for slot in _summary_slots(page) + if _summary_text(slot) is None and (page.get("id"), slot.get("id")) not in self.exempt + } + return self.error if unwritten <= self.transient_page_ids else None + + +def _editor_page_ids(tool_result: dict | None, document_part: dict | None) -> list[str] | None: + """The pages the copilot says it left to the editor; None unless a v2 draft carries the field.""" + document, _ = _read_document(document_part) + listed = (tool_result or {}).get(_EDITOR_PAGES_KEY) + if document is None or not _is_v2(document) or not isinstance(listed, list): + return None + return [page_id for page_id in listed if isinstance(page_id, str)] + + +def _saved_definition(user_context: dict | None) -> dict | None: + """The open document the item sends as ``user_context.view.document.definition``, when it is a v2 document.""" + view = (user_context or {}).get("view") + document = view.get("document") if isinstance(view, dict) else None + definition = document.get("definition") if isinstance(document, dict) else None + return definition if isinstance(definition, dict) and _is_v2(definition) else None + + +def _saved_empty_takeaways(document_part: dict | None, user_context: dict | None) -> frozenset[tuple[str, str]]: + """The empty takeaways of the draft that the editor leaves empty, as ``(page id, takeaway id)``. + + The editor keeps an empty takeaway empty when the open document holds the same takeaway, empty, on + the same page and the editor's request for it (visualizations under their filters, period included, + and instructions) is identical. Every other empty takeaway is written. No open document, one that + is not a v2 document, or a request that cannot be built exempts nothing. + """ + saved_definition = _saved_definition(user_context) + document, _ = _read_document(document_part) + if saved_definition is None or document is None or not _is_v2(document): + return frozenset() + try: + saved = aac_document_to_declarative(saved_definition) + current = aac_document_to_declarative(document) + except ValueError: + return frozenset() + saved_pages = {page.get("id"): page for page in _pages(saved) if isinstance(page, dict)} + current_pages = {page.get("id"): page for page in _pages(current) if isinstance(page, dict)} + exempt: set[tuple[str, str]] = set() + for _number, page in _content_pages(document): + page_id = page.get("id") + saved_page = saved_pages.get(page_id) + if not isinstance(page_id, str) or saved_page is None: + continue + saved_slots = {slot.get("id"): slot for slot in _summary_slots(saved_page)} + current_slots = {slot.get("id"): slot for slot in _summary_slots(current_pages.get(page_id, {}))} + for slot in _summary_slots(page): + slot_id = slot.get("id") + if not isinstance(slot_id, str): + continue + saved_slot, current_slot = saved_slots.get(slot_id), current_slots.get(slot_id) + if not _is_takeaway(slot) or _summary_text(slot) is not None: + continue + if saved_slot is None or current_slot is None or _summary_text(saved_slot) is not None: + continue + saved_request = editor_takeaway_input(saved, saved_page, saved_slot) + current_request = editor_takeaway_input(current, current_pages[page_id], current_slot) + if not isinstance(saved_request, str) and saved_request == current_request: + exempt.add((page_id, slot_id)) + return frozenset(exempt) + + +def _unavailable_chart(exc: httpx.HTTPStatusError) -> str | None: + """The endpoint's ``detail`` when it refused a request because a chart does not exist or is not available to AI.""" + if exc.response.status_code != 404 or "not available to ai" not in exc.response.text.lower(): + return None + try: + detail = exc.response.json().get("detail") + except ValueError: + detail = None + return detail if isinstance(detail, str) and detail else exc.response.text + + +def _fill_editor_takeaways( + client: TakeawayClient, document_part: dict, exempt: frozenset[tuple[str, str]], agent_id: str | None +) -> _EditorFill: + """Write each unwritten takeaway the way the editor does, on a copy of the part. + + The editor writes every empty takeaway except the ones it keeps empty, see ``_saved_empty_takeaways``. + The part carries the document as code; each request is built from its declarative form, which is + what the editor reads, and the text is written into the copy of the part. A document that does not + convert is a failure of the run, and so is a request for a chart that does not exist or is not + available to AI. A takeaway that already has text is left as the copilot wrote it. A transient + endpoint failure leaves that takeaway unwritten, records its page, and the fill goes on. + + Raises: + httpx.HTTPError: the endpoint refused a request for a reason that is not transient (flag + off, unknown agent, invalid body): a setup problem, not a result. + """ + fill = _EditorFill(document_part=copy.deepcopy(document_part), exempt=exempt) + document = fill.document_part[_PART_TYPE] + try: + declarative = aac_document_to_declarative(document) + except ValueError as exc: + fill.failures.append(f"the eval cannot read the draft as the editor does: {exc}") + fill.readable = False + return fill + editor_pages = {page.get("id"): page for page in _pages(declarative) if isinstance(page, dict)} + for _number, page in _content_pages(document): + page_id = page.get("id") + if not isinstance(page_id, str): + continue + editor_page = editor_pages.get(page_id, {}) + # The conversion keeps each widget's id, so a takeaway finds its declarative form by id. + editor_slots = {editor_slot.get("id"): editor_slot for editor_slot in _summary_slots(editor_page)} + for slot in _summary_slots(page): + if not _is_takeaway(slot) or _summary_text(slot) is not None or (page_id, slot.get("id")) in exempt: + continue + fill.generated_page_ids.add(page_id) + editor_slot = editor_slots.get(slot.get("id")) + if editor_slot is None: + fill.failures.append( + f"page {page_id!r}: the eval cannot build the editor's takeaway request: " + f"takeaway {slot.get('id')!r} is missing from the converted document" + ) + continue + takeaway_input = editor_takeaway_input(declarative, editor_page, editor_slot) + if isinstance(takeaway_input, str): + fill.failures.append( + f"page {page_id!r}: the eval cannot build the editor's takeaway request: {takeaway_input}" + ) + continue + requested = time.monotonic() + try: + result = client.request(takeaway_input.model_copy(update={"agent_id": agent_id})) + except httpx.HTTPError as exc: + if isinstance(exc, httpx.HTTPStatusError) and (detail := _unavailable_chart(exc)) is not None: + fill.failures.append(f"page {page_id!r}: the takeaway endpoint refused the chart: {detail}") + continue + if not is_transient(exc): + raise + fill.transient_page_ids.add(page_id) + if fill.error is None: + fill.error = f"takeaway endpoint for page {page_id!r}: {type(exc).__name__}: {exc}" + continue + finally: + fill.waited_s = max(fill.waited_s, time.monotonic() - requested) + text = (result.text_response or "").strip() + if (result.raw_response or {}).get("noData") is True: + fill.failures.append(f"page {page_id!r}: the editor's takeaway request found no data") + elif not text: + fill.failures.append(f"page {page_id!r}: the takeaway endpoint wrote no text") + else: + slot["markdown"] = text + fill.filled_page_ids.append(page_id) + return fill + + +def _listed_pages_mismatch(listed: list[str], generated: set[str]) -> list[str]: + """Failures for ``takeaway_editor_page_ids`` where it differs from the pages the editor writes a takeaway on.""" + failures = [ + f"page {page_id!r} has a takeaway the editor writes, but draft_document does not list it in {_EDITOR_PAGES_KEY}" + for page_id in sorted(generated - set(listed)) + ] + failures.extend( + f"draft_document lists page {page_id!r} in {_EDITOR_PAGES_KEY}, but the editor writes no takeaway there" + for page_id in dict.fromkeys(listed) + if page_id not in generated + ) + return failures + + def _judge_narrative( evaluation: DocumentEvaluation, judge: LLMJudge, document_part: dict | None, question: str, narrative: str ) -> float: @@ -882,6 +1095,9 @@ class DocumentRunResult: timings: PhaseTimings = field(default_factory=PhaseTimings) # This run's own wall time, from just before its chat call to the end of its scoring. run_latency_s: float = 0.0 + # How many takeaways the editor's takeaway request wrote; None when no takeaway was requested: + # the copilot left none to the editor, or the fixture states no narrative. + takeaways_by_editor: int | None = None @property def diagnostics(self) -> dict[str, bool]: @@ -897,7 +1113,13 @@ def diagnostics(self) -> dict[str, bool]: @property def judge_error(self) -> str | None: - return self.evaluation.judge_error + """Why the run has no narrative verdict: the judge's error, or the takeaway endpoint's.""" + return self.evaluation.judge_error or self.evaluation.fill_error + + @property + def request_error(self) -> str | None: + """The takeaway endpoint's transient failure, when it alone left the run without a narrative verdict.""" + return self.evaluation.fill_error @property def ungraded(self) -> bool: @@ -924,9 +1146,24 @@ def scored_run_results(self) -> list[DocumentRunResult]: return [r for r in self.run_results if not r.ungraded] @property - def judge_errors(self) -> list[str]: + def unscored_errors(self) -> list[str]: + """Why each ungraded run has no narrative verdict, whichever source failed.""" return [r.judge_error for r in self.run_results if r.ungraded and r.judge_error is not None] + @property + def judge_errors(self) -> list[str]: + """The judge's errors on ungraded runs; the takeaway endpoint's are in ``request_errors``.""" + return [ + r.evaluation.judge_error + for r in self.run_results + if r.ungraded and r.request_error is None and r.evaluation.judge_error is not None + ] + + @property + def request_errors(self) -> list[str]: + """The takeaway endpoint's transient failures on ungraded runs.""" + return [r.request_error for r in self.run_results if r.ungraded and r.request_error is not None] + def _execute_single_document_run( client: ChatClient, @@ -935,11 +1172,23 @@ def _execute_single_document_run( expected_output: dict, max_iterations: int, judge: LLMJudge | None = None, + takeaways: TakeawayClient | None = None, + user_context: dict | None = None, + agent_id: str | None = None, ) -> DocumentRunResult: """Drive one conversation until the copilot drafts a document, then evaluate it. + When the fixture states a narrative and the draft carries ``takeaway_editor_page_ids``, + ``takeaways`` writes each empty takeaway as the editor does, as ``agent_id``, the agent of the + chat turn, and the document is graded with them. A takeaway the open document in + ``user_context`` held empty under the same request stays empty, see ``_saved_empty_takeaways``. + The field is checked against the pages the editor writes a takeaway on. + Nothing is cleaned up on the way out by design: gen-ai keeps the draft in conversation state and persists nothing until a user saves it. + + Raises: + httpx.HTTPError: the takeaway endpoint refused a request for a reason that is not transient. """ tool_result: dict | None = None document_part: dict | None = None @@ -1006,20 +1255,52 @@ def _execute_single_document_run( current_question = build_simulated_reply(expected_output, asked=asked and not replied) replied = replied or asked + graded_part = document_part + fill: _EditorFill | None = None + editor_page_ids = _editor_page_ids(tool_result, document_part) + # A draft that carries the field has the takeaway contract: the editor writes each empty takeaway, + # whatever the field lists. Any other draft is graded as before. + exempt = _saved_empty_takeaways(document_part, user_context) if editor_page_ids is not None else frozenset() + if ( + takeaways is not None + and document_part is not None + and editor_page_ids is not None + and _has_narrative(expected_output) + ): + fill_started = time.monotonic() + fill = _fill_editor_takeaways(takeaways, document_part, exempt, agent_id) + fill_elapsed = time.monotonic() - fill_started + # The takeaway endpoint is gen-ai too: the reader waits for the slowest request before the document is complete. + timings.agent_s += fill.waited_s + log_timer( + f"[timer] document_skill {conversation_id} editor takeaways complete after {fill_elapsed:.2f}s " + f"({fill.waited_s:.2f}s for the slowest); {len(fill.filled_page_ids)} written" + ) + graded_part = fill.document_part + evaluation = evaluate_document_response( tool_result, - document_part, + graded_part, expected_output, skill_activated=_skill_activated(all_tool_call_events, _BUILDER_SKILL), legacy_names=legacy_wire_names(all_parts, all_tool_call_events), + exempt_takeaways=exempt, ) if _asked_first_required(expected_output): evaluation.applies = replace(evaluation.applies, asked_first=True) evaluation.asked_first = asked_before_draft if tool_result is not None and not asked_before_draft: evaluation.failures.append("the copilot drafted without asking first") - if judge is not None and _has_narrative(expected_output): - timings.judge_s += _judge_narrative(evaluation, judge, document_part, question, expected_output["narrative"]) + if fill is not None: + evaluation.failures.extend(fill.failures) + if editor_page_ids is not None and fill.readable: + evaluation.failures.extend(_listed_pages_mismatch(editor_page_ids, fill.generated_page_ids)) + evaluation.fill_error = fill.ungraded_error() + if fill.error is not None and evaluation.fill_error is None: + # The run is graded on its other failures; the outage still explains its empty takeaway. + evaluation.failures.append(f"{fill.error}: the takeaway stays empty") + if judge is not None and _has_narrative(expected_output) and evaluation.fill_error is None: + timings.judge_s += _judge_narrative(evaluation, judge, graded_part, question, expected_output["narrative"]) return DocumentRunResult( conversation_id=conversation_id, @@ -1036,6 +1317,7 @@ def _execute_single_document_run( reasoning_step_events=all_reasoning_step_events, timings=timings, run_latency_s=time.monotonic() - run_started, + takeaways_by_editor=None if fill is None or not fill.generated_page_ids else len(fill.filled_page_ids), ) @@ -1062,10 +1344,16 @@ def run_agentic_document_skill( ``document_format`` is sent on every turn of every run, see ``ChatClient``; None leaves the format to the server's ``enablePublisherV2`` feature flag. + A takeaway the copilot left to the editor is written through the takeaway endpoint before + the narrative is graded, see ``_execute_single_document_run``. + Raises: ValueError: the fixture is unusable — see ``_validate_expectation``, ``_validate_user_context`` and ``_validate_rework_target``, or ``document_format`` is not a format gen-ai accepts. Raised before any chat request. + httpx.HTTPError: the takeaway endpoint refused a request for a reason that is not + transient and not the copilot's, such as a 404 while the takeaway feature flag is off + or for an agent that does not exist. """ _validate_expectation(expected_output) _validate_user_context(user_context) @@ -1083,12 +1371,24 @@ def run_agentic_document_skill( user_context=user_context, document_format=document_format, ) + # Opens no connection until the first request, which only a draft listing editor pages makes. + takeaways = TakeawayClient(host=host, token=token, workspace_id=workspace_id) try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: run_results.append( - _execute_single_document_run(client, conv_id_0, question, expected_output, max_iterations, judge) + _execute_single_document_run( + client, + conv_id_0, + question, + expected_output, + max_iterations, + judge, + takeaways, + user_context, + agent_id, + ) ) finally: if initial_conversation_id is None: # only delete conversations we created @@ -1098,14 +1398,25 @@ def run_agentic_document_skill( conv_id = client.create_conversation() try: run_results.append( - _execute_single_document_run(client, conv_id, question, expected_output, max_iterations, judge) + _execute_single_document_run( + client, + conv_id, + question, + expected_output, + max_iterations, + judge, + takeaways, + user_context, + agent_id, + ) ) finally: client.delete_conversation(conv_id) finally: client.close() + takeaways.close() - # An ungraded run is a fault of that judge request, not of the document: it is left out of + # An ungraded run is a fault of that judge or takeaway request, not of the document: it is left out of # pass@K, and it keeps pass^K from holding, since "every run passed" was never verified. scored = [r for r in run_results if not r.ungraded] return AgenticDocumentSummary( @@ -1159,7 +1470,11 @@ def evaluate_agentic_document_skill( request, so it means a fixture to fix rather than a result to read. Also raised, equally before any request, when the caller passes a ``document_format`` gen-ai does not accept. JudgeResponseError: the fixture states a narrative and the judge returned no readable - verdict for any run -- an item without a result, not K failures. + verdict for any run, or the takeaway endpoint failed transiently while writing the + takeaways the copilot left to the editor -- an item without a result, not K failures. + httpx.HTTPError: the takeaway endpoint refused a request for a reason that is not + transient and not the copilot's, such as a 404 while the takeaway feature flag is off + or for an agent that does not exist. """ document_format = normalize_document_format(document_format) if document_format is not None: @@ -1219,6 +1534,10 @@ def _write_scores(ctx: RunTraceContext) -> None: ctx.score( tid, name="document_summaries_from_data", value=run.summaries_from_data, data_type="NUMERIC" ) + if run.takeaways_by_editor is not None: + ctx.score( + tid, name="document_takeaways_by_editor", value=run.takeaways_by_editor, data_type="NUMERIC" + ) ctx.score(tid, name="turns", value=run.total_turns, data_type="NUMERIC") ctx.score(tid, name="steps", value=run.total_steps, data_type="NUMERIC") log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) @@ -1249,11 +1568,16 @@ def _write_scores(ctx: RunTraceContext) -> None: ) item_timings = sum_timings([r.timings for r in summary.run_results]) - unscored = summary.judge_errors + unscored = summary.unscored_errors + judge_errors = summary.judge_errors + request_errors = summary.request_errors if not summary.scored_run_results: + causes = [ + *([f"judge returned no readable verdict for {len(judge_errors)} run(s)"] if judge_errors else []), + *([f"request failed for {len(request_errors)} run(s)"] if request_errors else []), + ] exc_judge = JudgeResponseError( - f"judge returned no readable verdict for any of the {len(summary.run_results)} run(s): " - + " | ".join(unscored) + f"no run of {len(summary.run_results)} could be graded: {', '.join(causes)}: " + " | ".join(unscored) ) exc_judge.timings = item_timings exc_judge.run_latencies_s = [r.run_latency_s for r in summary.run_results] @@ -1272,6 +1596,7 @@ def _run_detail(run: DocumentRunResult) -> dict[str, Any]: **run.evaluation.strict_checks, **run.diagnostics, "summaries_from_data": run.summaries_from_data, + **({"takeaways_by_editor": run.takeaways_by_editor} if run.takeaways_by_editor is not None else {}), "turns": run.total_turns, **({"judge_reasoning": run.evaluation.judge_reasoning} if run.evaluation.applies.narrative else {}), "failures": run.evaluation.failures, @@ -1280,7 +1605,9 @@ def _run_detail(run: DocumentRunResult) -> dict[str, Any]: detail: dict[str, Any] = { **_run_detail(best), - **({"unscored_runs": len(unscored), "judge_errors": unscored} if unscored else {}), + **({"unscored_runs": len(unscored)} if unscored else {}), + **({"judge_errors": judge_errors} if judge_errors else {}), + **({"request_errors": request_errors} if request_errors else {}), } # Same predicate runs_passed is taken over, so an item's failed_runs and its counts # cannot disagree about which runs failed. diff --git a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py index 1ffa3e934..26d59ae4c 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/evaluate.py @@ -47,7 +47,7 @@ from gooddata_eval.core.evaluators.base import ItemEvaluation from gooddata_eval.core.evaluators.takeaway import DocumentTakeawayEvaluator from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ChatResult, DatasetItem -from gooddata_eval.core.takeaway.http_client import TakeawayClient +from gooddata_eval.core.takeaway.http_client import TakeawayClient, is_transient from gooddata_eval.core.timing import PhaseTimings, sum_timings _TEST_KIND = "document_takeaway" @@ -84,19 +84,12 @@ def _dataset_item(dataset_item: dict, dataset_name: str) -> DatasetItem: return item -def _is_transient(exc: httpx.HTTPError) -> bool: - if isinstance(exc, httpx.HTTPStatusError): - status = exc.response.status_code - return status == 429 or status >= 500 - return isinstance(exc, httpx.TransportError) and not isinstance(exc, httpx.UnsupportedProtocol) - - def _run_once(client: TakeawayClient, evaluator: DocumentTakeawayEvaluator, item: DatasetItem) -> _TakeawayRun: started = time.monotonic() try: chat_result = client.ask(item) except httpx.HTTPError as exc: - if not _is_transient(exc): + if not is_transient(exc): raise elapsed = time.monotonic() - started error = f"{type(exc).__name__}: {exc}" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py index d2ef8d9e5..d946aa5ab 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/takeaway/http_client.py @@ -40,6 +40,18 @@ def _is_retryable(exc: Exception) -> bool: return isinstance(exc, _RETRYABLE_TRANSPORT_ERRORS) +def is_transient(exc: httpx.HTTPError) -> bool: + """Whether a failure left after the client's retries says nothing about the request. + + 429, 5xx and transport failures (a read timeout included) are transient; any other 4xx + (flag off, no agent, invalid body) and an unsupported URL scheme are setup problems. + """ + if isinstance(exc, httpx.HTTPStatusError): + status = exc.response.status_code + return status == 429 or status >= 500 + return isinstance(exc, httpx.TransportError) and not isinstance(exc, httpx.UnsupportedProtocol) + + def _request_body(ti: TakeawayInput) -> dict[str, Any]: body: dict[str, Any] = {} if ti.agent_id is not None: @@ -77,14 +89,21 @@ def ask(self, item: DatasetItem) -> ChatResult: Raises: ValueError: The item carries no `takeaway_input`. - httpx.HTTPStatusError: The endpoint answered non-2xx; the message carries the - status code and the response body. Transient statuses and connection - failures are retried first, with the chat client's backoff and budget. + httpx.HTTPStatusError: As `request`. """ ti = item.takeaway_input if ti is None: raise ValueError(f"document_takeaway item '{item.id}' is missing required 'takeaway_input'.") + return self.request(ti) + def request(self, ti: TakeawayInput) -> ChatResult: + """Request a takeaway for one input and adapt it to a ChatResult. + + Raises: + httpx.HTTPStatusError: The endpoint answered non-2xx; the message carries the + status code and the response body. Transient statuses and connection + failures are retried first, with the chat client's backoff and budget. + """ body = _request_body(ti) def _post() -> httpx.Response: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/timing.py b/packages/gooddata-eval/src/gooddata_eval/core/timing.py index b2d203d7b..85622471d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/timing.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/timing.py @@ -41,7 +41,8 @@ class PhaseTimings: item, so an item's ``agent_s`` is the sum of every agent turn it took. """ - # The system under test: time inside ChatClient.send_message. + # The system under test: time inside ChatClient.send_message, and inside the takeaway + # endpoint when a run writes takeaways through it. agent_s: float = 0.0 # Our grading of the answer (LLMJudge). Post-hoc -- never blocks the agent. judge_s: float = 0.0 diff --git a/packages/gooddata-eval/tests/test_agentic_document_editor_takeaways.py b/packages/gooddata-eval/tests/test_agentic_document_editor_takeaways.py new file mode 100644 index 000000000..2ac8b9f0c --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_document_editor_takeaways.py @@ -0,0 +1,385 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +# SPDX-License-Identifier: LicenseRef-GoodData-Enterprise +from typing import Any + +import pytest +from gooddata_eval.core.agentic._editor_takeaways import editor_takeaway_input +from gooddata_eval.core.models import TakeawayInput + +_ABSOLUTE = {"type": "absolute", "granularity": "GDC.time.date", "from": "2026-01-01", "to": "2026-06-30"} +_RELATIVE = {"type": "relative", "granularity": "GDC.time.month", "from": -5, "to": 0} +_ORDER_DATE = {"identifier": {"id": "order_date", "type": "dataset"}} +_REGION = {"identifier": {"id": "region.name", "type": "label"}} +_TAKEAWAY = {"type": "takeaway", "id": "takeaway"} + + +def _chart(visualization: str, *, dataset: str | None = "order_date", **extra: Any) -> dict: + return { + "type": "visualization", + "id": f"w_{visualization}", + "visualization": {"identifier": visualization, "type": "insight"}, + **({"dateDataSet": {"identifier": dataset, "type": "dataset"}} if dataset else {}), + **extra, + } + + +def _page(*widgets: dict) -> dict: + return { + "id": "page2", + "kind": "content", + "layout": {"type": "column", "children": [{"type": "row", "children": [*widgets, _TAKEAWAY]}]}, + } + + +def _document(*, period: Any = _ABSOLUTE, filters: list[dict] | None = None, defaults: Any = None) -> dict: + content: dict[str, Any] = {"pages": []} + if filters is not None: + content["filters"] = filters + if defaults is not None: + content["defaults"] = defaults + return {"type": "document", "version": "2", "period": period, "content": content} + + +def _built(document: dict, page: dict, takeaway: dict = _TAKEAWAY) -> TakeawayInput: + built = editor_takeaway_input(document, page, takeaway) + assert isinstance(built, TakeawayInput), built + return built + + +def _filters_of(built: TakeawayInput) -> list[list[dict]]: + return [v.filters for v in built.visualizations if not isinstance(v, str)] + + +def _attribute_filter(values: list[str], *, negative: bool = False) -> dict: + return { + "attributeFilter": { + "displayForm": _REGION, + "negativeSelection": negative, + "attributeElements": {"values": values}, + } + } + + +def test_the_absolute_period_filters_each_chart_on_its_own_dataset() -> None: + # given + page = _page(_chart("revenue"), _chart("returns", dataset="return_date")) + + # when + built = _built(_document(), page) + + # then + assert [v.visualization_id for v in built.visualizations if not isinstance(v, str)] == ["revenue", "returns"] + assert _filters_of(built) == [ + [{"absoluteDateFilter": {"dataset": _ORDER_DATE, "from": "2026-01-01", "to": "2026-06-30"}}], + [ + { + "absoluteDateFilter": { + "dataset": {"identifier": {"id": "return_date", "type": "dataset"}}, + "from": "2026-01-01", + "to": "2026-06-30", + } + } + ], + ] + assert built.filter_context == [] + assert built.max_characters is None + assert built.agent_id is None + + +def test_a_relative_period_takes_the_afm_granularity() -> None: + # when + built = _built(_document(period=_RELATIVE), _page(_chart("revenue"))) + + # then + assert _filters_of(built) == [ + [{"relativeDateFilter": {"dataset": _ORDER_DATE, "granularity": "MONTH", "from": -5, "to": 0}}] + ] + + +def test_a_granularity_without_an_afm_form_cannot_be_built() -> None: + # when + built = editor_takeaway_input( + _document(period={**_RELATIVE, "granularity": "GDC.time.century"}), _page(_chart("revenue")), _TAKEAWAY + ) + + # then + assert built == "relative date granularity 'GDC.time.century' has no AFM form" + + +@pytest.mark.parametrize( + "chart", + [_chart("revenue", ignoreDocumentPeriod=True), _chart("revenue", dataset=None)], + ids=["ignores the period", "no date dataset"], +) +def test_a_chart_without_the_period_gets_no_date_filter(chart: dict) -> None: + # when + built = _built(_document(), _page(chart)) + + # then + assert _filters_of(built) == [[]] + + +def test_a_document_date_filter_on_the_charts_dataset_replaces_the_period() -> None: + # given + own = { + "dateFilter": {"type": "relative", "granularity": "GDC.time.year", "from": -1, "to": -1, "dataSet": _ORDER_DATE} + } + other = { + "dateFilter": {"type": "absolute", "from": "2020-01-01", "to": "2020-12-31", "dataSet": {"identifier": "x"}} + } + + # when + built = _built(_document(filters=[other, own]), _page(_chart("revenue", ignoreDocumentPeriod=True))) + + # then + assert _filters_of(built) == [ + [{"relativeDateFilter": {"dataset": _ORDER_DATE, "granularity": "YEAR", "from": -1, "to": -1}}] + ] + + +def test_a_document_date_filter_that_replaces_the_period_keeps_its_empty_value_handling_and_bounds() -> None: + # given + relative = { + "dateFilter": { + **_RELATIVE, + "dataSet": _ORDER_DATE, + "emptyValueHandling": "EXCLUDE", + "boundedFilter": {"granularity": "GDC.time.year", "from": -1, "to": 0}, + } + } + absolute = { + "dateFilter": { + "type": "absolute", + "from": "2026-02-01", + "to": "2026-02-28", + "dataSet": {"identifier": "return_date"}, + "emptyValueHandling": "INCLUDE", + "boundedFilter": {"granularity": "GDC.time.year", "from": -1, "to": 0}, + } + } + + # when + built = _built( + _document(filters=[relative, absolute]), _page(_chart("revenue"), _chart("returns", dataset="return_date")) + ) + + # then + assert _filters_of(built) == [ + [ + { + "relativeDateFilter": { + "dataset": _ORDER_DATE, + "granularity": "MONTH", + "from": -5, + "to": 0, + "emptyValueHandling": "EXCLUDE", + "boundedFilter": {"granularity": "GDC.time.year", "from": -1, "to": 0}, + } + } + ], + [ + { + "absoluteDateFilter": { + "dataset": {"identifier": {"id": "return_date", "type": "dataset"}}, + "from": "2026-02-01", + "to": "2026-02-28", + "emptyValueHandling": "INCLUDE", + } + } + ], + ] + + +def test_a_document_date_filter_with_an_open_end_leaves_the_period() -> None: + # given + all_time = {"dateFilter": {"type": "relative", "granularity": "GDC.time.year", "dataSet": _ORDER_DATE}} + + # when + built = _built(_document(filters=[all_time]), _page(_chart("revenue"))) + + # then + assert _filters_of(built) == [ + [{"absoluteDateFilter": {"dataset": _ORDER_DATE, "from": "2026-01-01", "to": "2026-06-30"}}] + ] + + +def test_every_attribute_filter_reaches_every_chart_and_one_that_narrows_nothing_is_dropped() -> None: + # given + filters = [ + _attribute_filter(["West"]), + _attribute_filter(["East"], negative=True), + _attribute_filter([], negative=True), + ] + + # when + built = _built(_document(filters=filters), _page(_chart("revenue", dataset=None))) + + # then + region = {"identifier": {"id": "region.name", "type": "label"}} + assert _filters_of(built) == [ + [ + {"positiveAttributeFilter": {"label": region, "in": {"values": ["West"]}}}, + {"negativeAttributeFilter": {"label": region, "notIn": {"values": ["East"]}}}, + ] + ] + + +def test_selected_elements_are_read_from_uris_and_then_from_values() -> None: + # given + by_uris = {"displayForm": _REGION, "negativeSelection": False, "attributeElements": {"uris": ["West"]}} + by_values = { + "displayForm": _REGION, + "negativeSelection": True, + "attributeElements": {"uris": [], "values": ["East"]}, + } + + # when + built = _built( + _document(filters=[{"attributeFilter": by_uris}, {"attributeFilter": by_values}]), + _page(_chart("revenue", dataset=None)), + ) + + # then + region = {"identifier": {"id": "region.name", "type": "label"}} + assert _filters_of(built) == [ + [ + {"positiveAttributeFilter": {"label": region, "in": {"values": ["West"]}}}, + {"negativeAttributeFilter": {"label": region, "notIn": {"values": ["East"]}}}, + ] + ] + + +def test_text_filters_on_whole_values_are_element_filters_and_one_without_values_is_dropped() -> None: + # given + filters = [ + {"arbitraryAttributeFilter": {"displayForm": _REGION, "values": ["West", None]}}, + {"arbitraryAttributeFilter": {"displayForm": _REGION, "values": ["East"], "negativeSelection": True}}, + {"arbitraryAttributeFilter": {"displayForm": _REGION, "values": [], "negativeSelection": True}}, + {"arbitraryAttributeFilter": {"displayForm": _REGION, "values": []}}, + ] + + # when + built = _built(_document(filters=filters), _page(_chart("revenue", dataset=None))) + + # then + region = {"identifier": {"id": "region.name", "type": "label"}} + assert _filters_of(built) == [ + [ + {"positiveAttributeFilter": {"label": region, "in": {"values": ["West", None]}}}, + {"negativeAttributeFilter": {"label": region, "notIn": {"values": ["East"]}}}, + ] + ] + + +def test_a_text_filter_on_a_fragment_is_a_match_filter() -> None: + # given + match = { + "displayForm": _REGION, + "operator": "startsWith", + "literal": "We", + "caseSensitive": True, + "negativeSelection": True, + } + + # when + built = _built(_document(filters=[{"matchAttributeFilter": match}]), _page(_chart("revenue", dataset=None))) + + # then + assert _filters_of(built) == [ + [ + { + "matchAttributeFilter": { + "label": {"identifier": {"id": "region.name", "type": "label"}}, + "literal": "We", + "matchType": "STARTS_WITH", + "caseSensitive": True, + "negate": True, + } + } + ] + ] + + +def test_a_filter_kind_that_draws_no_chart_differently_is_skipped() -> None: + # given + measure_value = {"measureValueFilter": {"measure": {"identifier": "revenue"}, "condition": {}}} + + # when + built = _built(_document(filters=[measure_value]), _page(_chart("revenue", dataset=None))) + + # then + assert _filters_of(built) == [[]] + + +@pytest.mark.parametrize( + "document_filter", + [ + {"attributeFilter": {"displayForm": _REGION, "negativeSelection": False, "attributeElements": {"uris": []}}}, + {"matchAttributeFilter": {"displayForm": _REGION, "operator": "like", "literal": "We"}}, + ], + ids=["empty selection", "unknown match operator"], +) +def test_a_document_filter_without_an_afm_form_cannot_be_built(document_filter: dict) -> None: + # when + built = editor_takeaway_input(_document(filters=[document_filter]), _page(_chart("revenue")), _TAKEAWAY) + + # then + assert isinstance(built, str) + assert "AFM form" in built + + +def test_the_takeaways_and_the_documents_instructions_are_sent_trimmed() -> None: + # given + takeaway = {**_TAKEAWAY, "generation": {"instructions": " Focus on the West. "}} + defaults = {"takeaway": {"generation": {"instructions": "Write for executives."}}} + + # when + built = _built(_document(defaults=defaults), _page(_chart("revenue")), takeaway) + + # then + assert built.instructions.slot == "Focus on the West." + assert built.instructions.document == "Write for executives." + + +def test_blank_instructions_are_left_out() -> None: + # given + takeaway = {**_TAKEAWAY, "generation": {"instructions": " "}} + defaults = {"takeaway": {"generation": {"instructions": ""}}} + + # when + built = _built(_document(defaults=defaults), _page(_chart("revenue")), takeaway) + + # then + assert built.instructions.slot is None + assert built.instructions.document is None + + +def test_a_chart_drawn_twice_the_same_way_is_sent_once() -> None: + # given + page = _page(_chart("revenue"), _chart("revenue"), _chart("revenue", ignoreDocumentPeriod=True)) + + # when + built = _built(_document(), page) + + # then + assert [len(f) for f in _filters_of(built)] == [1, 0] + + +def test_a_page_without_a_chart_cannot_be_built() -> None: + # when + built = editor_takeaway_input(_document(), _page(), _TAKEAWAY) + + # then + assert built == "page 'page2' has no visualization to summarize" + + +def test_content_keys_at_the_top_level_are_read() -> None: + # given + document = {**_document(), "filters": [_attribute_filter(["West"])]} + del document["content"] + + # when + built = _built(document, _page(_chart("revenue", dataset=None))) + + # then + assert len(_filters_of(built)[0]) == 1 diff --git a/packages/gooddata-eval/tests/test_agentic_document_skill.py b/packages/gooddata-eval/tests/test_agentic_document_skill.py index d0d58320c..364b06324 100644 --- a/packages/gooddata-eval/tests/test_agentic_document_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_document_skill.py @@ -4,8 +4,10 @@ import json from collections.abc import Iterator from contextlib import contextmanager +from types import SimpleNamespace from typing import Any +import httpx import pytest from gooddata_eval.cli.agentic_runner import _dispatch_agentic from gooddata_eval.core import agentic @@ -24,7 +26,7 @@ run_agentic_document_skill, ) from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError -from gooddata_eval.core.models import ChatResult, DatasetItem +from gooddata_eval.core.models import ChatResult, DatasetItem, TakeawayInput # Shapes follow what gen-ai writes for a drafted document (composed_report.aac.json): a cover # page, then content pages whose layout nests `column` and `row` entries down to the slots. @@ -1460,7 +1462,9 @@ def test_an_item_the_judge_could_never_grade_raises(monkeypatch: pytest.MonkeyPa turn = _narrative_turn(_cover_page(), _summary_page("page2", "Revenue grew 12%.", _REVENUE_TREND)) _install_client(monkeypatch, _ScriptedChatClient([turn])) judge = _FakeJudge(error=JudgeResponseError("empty body")) - with pytest.raises(JudgeResponseError, match="no readable verdict") as raised: + with pytest.raises( + JudgeResponseError, match=r"^no run of 1 could be graded: judge returned no readable verdict for 1 run\(s\): " + ) as raised: evaluate_agentic_document_skill("https://h", "tok", "ws", "q", {"narrative": _NARRATIVE}, judge=judge) assert len(raised.value.run_latencies_s) == 1 @@ -1757,6 +1761,605 @@ def test_a_document_whose_only_content_page_is_hidden_has_no_content_page() -> N assert evaluation.failures == ["the document has no content page"] +# ── takeaways the copilot leaves to the editor ────────────────────────────── + +_EDITOR_TEXT = "**Revenue grew 12%** in H1, led by the West." + + +class _StubTakeaways: + """Stands in for TakeawayClient: answers each request with the next scripted response.""" + + def __init__(self, *responses: dict | Exception) -> None: + self._responses = list(responses) + self.requests: list[TakeawayInput] = [] + self.closed = False + + def request(self, ti: TakeawayInput) -> ChatResult: + self.requests.append(ti) + response = self._responses.pop(0) if self._responses else {"takeaway": _EDITOR_TEXT, "noData": False} + if isinstance(response, Exception): + raise response + return ChatResult(text_response=response.get("takeaway") or "", raw_response=response) + + def close(self) -> None: + self.closed = True + + +def _install_takeaways(monkeypatch: pytest.MonkeyPatch, takeaways: _StubTakeaways) -> None: + monkeypatch.setattr(document_skill, "TakeawayClient", lambda **_kwargs: takeaways) + + +def _http_error(status: int) -> httpx.HTTPStatusError: + request = httpx.Request("POST", "https://h/api/v1/ai/workspaces/ws/takeaway") + return httpx.HTTPStatusError(f"{status}", request=request, response=httpx.Response(status, request=request)) + + +def _with_instructions(page: dict, instructions: str) -> dict: + takeaway = page["layout"]["children"][1]["children"][1] + takeaway["generation"] = {**takeaway["generation"], "instructions": instructions} + return page + + +# The copilot's part carries the document as code: a period of bare dates and a widget's date dataset in ``date``. +_AAC_PERIOD = {"from": "2026-01-01", "to": "2026-06-30"} + + +def _dated(page: dict, dataset: str) -> dict: + page["layout"]["children"][1]["children"][0]["children"][0]["date"] = dataset + return page + + +def _editor_pages() -> list[dict]: + """A cover, a page whose takeaway the copilot wrote, and two left to the editor, one dated and with instructions.""" + return [ + _v2_cover_page(), + _v2_content_page("page2", _REVENUE_TREND, takeaway="Revenue grew 12%.", bare_ids=True), + _v2_content_page("page3", _RETURNS_BY_CATEGORY, takeaway="", bare_ids=True), + _with_instructions( + _dated(_v2_content_page("page4", _REVENUE_TREND, takeaway="", bare_ids=True), "date"), "Focus on the West." + ), + ] + + +def _editor_turn( + pages: list[dict], + editor_page_ids: Any, + *, + version: str = "v2", + period: Any = _AAC_PERIOD, + filters: dict | None = None, +) -> ChatResult: + tool = _draft_result(page_count=len(pages)) + if editor_page_ids is not None: + tool[document_skill._EDITOR_PAGES_KEY] = editor_page_ids + part = _v2_document_part(pages, period=period) if version == "v2" else _document_part(pages) + if filters is not None: + part["document"]["content"]["filters"] = filters + return _chat_result(tool_calls=[_tool_call("draft_document", tool)], parts=[part], text="Done.") + + +def _editor_run( + pages: list[dict], + editor_page_ids: Any, + takeaways: _StubTakeaways | None, + judge: _FakeJudge | None = None, + user_context: dict | None = None, +) -> document_skill.DocumentRunResult: + return _execute_single_document_run( + _ScriptedChatClient([_editor_turn(pages, editor_page_ids)]), + "conv-1", + "q", + {"narrative": _NARRATIVE}, + 4, + judge or _FakeJudge(), + takeaways, + user_context, + ) + + +def test_each_takeaway_left_to_the_editor_is_written_the_editors_way_and_graded() -> None: + # given + takeaways = _StubTakeaways() + judge = _FakeJudge() + + # when + run = _editor_run(_editor_pages(), ["page3", "page4"], takeaways, judge) + + # then + assert [r.model_dump() for r in takeaways.requests] == [ + TakeawayInput(visualizations=[{"visualization_id": _RETURNS_BY_CATEGORY}]).model_dump(), + TakeawayInput( + visualizations=[ + { + "visualization_id": _REVENUE_TREND, + "filters": [ + { + "absoluteDateFilter": { + "dataset": {"identifier": {"id": "date", "type": "dataset"}}, + "from": "2026-01-01", + "to": "2026-06-30", + } + } + ], + } + ], + instructions={"slot": "Focus on the West."}, + ).model_dump(), + ] + assert run.evaluation.failures == [] + assert run.evaluation.strict_checks["document_summaries_present"] is True + assert run.evaluation.strict_checks["document_narrative_judged"] is True + assert judge.calls[0]["actual_output"].count(f"Summary: {_EDITOR_TEXT}") == 2 + assert run.takeaways_by_editor == 2 + + +def test_each_takeaway_on_a_page_sends_its_own_instructions() -> None: + # given + page = _with_instructions(_v2_content_page("page3", _RETURNS_BY_CATEGORY, takeaway="", bare_ids=True), "First.") + row = page["layout"]["children"][1]["children"] + row.append({**row[1], "id": "takeawayB", "generation": {**row[1]["generation"], "instructions": "Second."}}) + takeaways = _StubTakeaways() + + # when + run = _editor_run([_v2_cover_page(), page], ["page3"], takeaways) + + # then + assert [r.instructions.slot for r in takeaways.requests if r.instructions] == ["First.", "Second."] + assert run.takeaways_by_editor == 2 + + +def test_the_document_filters_as_the_editor_reads_them_reach_each_request() -> None: + # given + filters = { + "region": {"type": "attribute_filter", "using": "label/region", "state": {"include": ["West"]}}, + "q1": {"type": "date_filter", "granularity": "DAY", "date": "date", "from": "2026-01-01", "to": "2026-03-31"}, + } + takeaways = _StubTakeaways() + turn = _editor_turn(_editor_pages(), ["page3", "page4"], filters=filters) + + # when + run = _execute_single_document_run( + _ScriptedChatClient([turn]), "conv-1", "q", {"narrative": _NARRATIVE}, 4, _FakeJudge(), takeaways + ) + + # then + west = { + "positiveAttributeFilter": { + "label": {"identifier": {"id": "region", "type": "label"}}, + "in": {"values": ["West"]}, + } + } + q1 = { + "absoluteDateFilter": { + "dataset": {"identifier": {"id": "date", "type": "dataset"}}, + "from": "2026-01-01", + "to": "2026-03-31", + } + } + assert [[v.filters for v in r.visualizations if not isinstance(v, str)] for r in takeaways.requests] == [ + [[west]], + [[q1, west]], + ] + assert run.evaluation.failures == [] + assert run.takeaways_by_editor == 2 + + +def test_the_run_keeps_the_part_the_copilot_sent() -> None: + # when + run = _editor_run(_editor_pages(), ["page3", "page4"], _StubTakeaways()) + + # then + assert run.document_part is not None + assert ( + "markdown" not in run.document_part["document"]["content"]["pages"][2]["layout"]["children"][1]["children"][1] + ) + + +def test_a_takeaway_the_editor_writes_on_a_page_the_draft_does_not_list_fails() -> None: + # given + takeaways = _StubTakeaways() + + # when + run = _editor_run(_editor_pages(), ["page4"], takeaways) + + # then + assert len(takeaways.requests) == 2 + assert run.evaluation.failures == [ + "page 'page3' has a takeaway the editor writes, but draft_document does not list it in takeaway_editor_page_ids" + ] + + +def test_a_page_the_draft_lists_that_the_editor_writes_nothing_on_fails() -> None: + # given + takeaways = _StubTakeaways() + + # when + run = _editor_run(_editor_pages(), ["page2", "page3", "page4"], takeaways) + + # then + assert len(takeaways.requests) == 2 + assert run.evaluation.failures == [ + "draft_document lists page 'page2' in takeaway_editor_page_ids, but the editor writes no takeaway there" + ] + + +def _open_document(pages: list[dict], *, period: Any = _AAC_PERIOD) -> dict: + """The ``user_context`` of an item that has this document open in the editor.""" + return {"view": {"document": {"id": "sales_overview", "definition": _v2_document(pages, period=period)}}} + + +def test_an_empty_takeaway_the_open_document_holds_empty_under_the_same_request_stays_empty() -> None: + # given + takeaways = _StubTakeaways() + open_document = _open_document(_editor_pages()) + + # when + run = _editor_run(_editor_pages(), [], takeaways, user_context=open_document) + + # then + assert takeaways.requests == [] + assert run.evaluation.strict_checks["document_summaries_present"] is True + assert run.evaluation.failures == [] + + +def test_a_new_period_changes_only_the_request_of_a_takeaway_whose_chart_uses_it() -> None: + # given + takeaways = _StubTakeaways() + other_period = {"from": "2025-01-01", "to": "2025-06-30"} + open_document = _open_document(_editor_pages(), period=other_period) + + # when + run = _editor_run(_editor_pages(), ["page4"], takeaways, user_context=open_document) + + # then + assert [r.instructions.slot for r in takeaways.requests] == ["Focus on the West."] + assert run.evaluation.failures == [] + + +def test_an_empty_takeaway_is_written_when_the_open_document_wrote_it() -> None: + # given + written = _editor_pages() + written[2] = _v2_content_page("page3", _RETURNS_BY_CATEGORY, takeaway="Returns fell.", bare_ids=True) + takeaways = _StubTakeaways() + + # when + run = _editor_run(_editor_pages(), ["page3"], takeaways, user_context=_open_document(written)) + + # then + assert [r.visualizations[0].visualization_id for r in takeaways.requests] == [_RETURNS_BY_CATEGORY] + assert run.evaluation.failures == [] + + +def test_an_empty_takeaway_is_written_when_its_instructions_changed_since_the_open_document() -> None: + # given + changed = _editor_pages() + changed[2] = _with_instructions(changed[2], "Focus on the East.") + takeaways = _StubTakeaways() + + # when + run = _editor_run(changed, ["page3"], takeaways, user_context=_open_document(_editor_pages())) + + # then + assert [r.instructions.slot for r in takeaways.requests] == ["Focus on the East."] + assert run.evaluation.failures == [] + + +def test_a_listed_takeaway_the_copilot_wrote_is_graded_as_written() -> None: + # given + takeaways = _StubTakeaways() + + # when + run = _editor_run(_editor_pages(), ["page2", "page3", "page4"], takeaways) + + # then + assert len(takeaways.requests) == 2 + assert run.takeaways_by_editor == 2 + + +def test_a_draft_without_the_field_is_graded_as_before() -> None: + # given + takeaways = _StubTakeaways() + pages = _editor_pages() + + # when + run = _editor_run(pages, None, takeaways) + + # then + assert takeaways.requests == [] + assert run.takeaways_by_editor is None + without_client = _editor_run(pages, None, None) + assert run.evaluation == without_client.evaluation + assert run.evaluation.failures == [ + "page 'page3' has a takeaway with no written text", + "page 'page4' has a takeaway with no written text", + ] + + +def test_a_draft_without_the_field_gets_no_exemption_from_the_open_document() -> None: + # given + pages = _editor_pages() + + # when + run = _editor_run(pages, None, _StubTakeaways(), user_context=_open_document(pages)) + + # then + assert run.evaluation.strict_checks["document_summaries_present"] is False + assert run.evaluation.failures == [ + "page 'page3' has a takeaway with no written text", + "page 'page4' has a takeaway with no written text", + ] + + +def test_a_draft_that_lists_nothing_while_the_editor_writes_takeaways_fails() -> None: + # given + takeaways = _StubTakeaways() + + # when + run = _editor_run(_editor_pages(), [], takeaways) + + # then + assert len(takeaways.requests) == 2 + assert run.evaluation.failures == [ + "page 'page3' has a takeaway the editor writes, but draft_document does not list it in " + "takeaway_editor_page_ids", + "page 'page4' has a takeaway the editor writes, but draft_document does not list it in " + "takeaway_editor_page_ids", + ] + + +def test_each_takeaway_request_names_the_agent_of_the_chat_turn() -> None: + # given + takeaways = _StubTakeaways() + + # when + _execute_single_document_run( + _ScriptedChatClient([_editor_turn(_editor_pages(), ["page3", "page4"])]), + "conv-1", + "q", + {"narrative": _NARRATIVE}, + 4, + _FakeJudge(), + takeaways, + agent_id="custom-agent", + ) + + # then + assert [r.agent_id for r in takeaways.requests] == ["custom-agent", "custom-agent"] + + +def test_a_chart_the_endpoint_cannot_find_fails_that_run_with_its_detail() -> None: + # given + request = httpx.Request("POST", "https://h/api/v1/ai/workspaces/ws/takeaway") + detail = "Visualization 'returns_by_category' does not exist or is not available to AI." + refused = httpx.HTTPStatusError( + "404", request=request, response=httpx.Response(404, json={"detail": detail}, request=request) + ) + takeaways = _StubTakeaways(refused) + + # when + run = _editor_run(_editor_pages(), ["page3", "page4"], takeaways) + + # then + assert len(takeaways.requests) == 2 + assert run.evaluation.failures == [ + "page 'page3' has a takeaway with no written text", + f"page 'page3': the takeaway endpoint refused the chart: {detail}", + ] + assert run.takeaways_by_editor == 1 + + +def test_the_reader_waits_for_the_slowest_takeaway_not_for_their_sum(monkeypatch: pytest.MonkeyPatch) -> None: + # given + clock = iter([0.0, 3.0, 3.0, 4.0]) + monkeypatch.setattr(document_skill, "time", SimpleNamespace(monotonic=lambda: next(clock))) + part = _v2_document_part(_editor_pages(), period=_AAC_PERIOD) + + # when + fill = document_skill._fill_editor_takeaways(_StubTakeaways(), part, frozenset(), None) + + # then + assert fill.waited_s == 3.0 + + +def test_a_v1_draft_is_never_filled() -> None: + # given + takeaways = _StubTakeaways() + pages = [_cover_page(), _summary_page("page2", "", _REVENUE_TREND)] + + # when + run = _execute_single_document_run( + _ScriptedChatClient([_editor_turn(pages, ["page2"], version="v1")]), + "conv-1", + "q", + {"narrative": _NARRATIVE}, + 4, + _FakeJudge(), + takeaways, + ) + + # then + assert takeaways.requests == [] + assert run.evaluation.failures[0] == "page 'page2' has a summary slot with no written text" + + +@pytest.mark.parametrize( + ("response", "reason"), + [ + ({"takeaway": None, "noData": True}, "the editor's takeaway request found no data"), + ({"takeaway": " ", "noData": False}, "the takeaway endpoint wrote no text"), + ], + ids=["no data", "no text"], +) +def test_a_takeaway_the_endpoint_does_not_write_fails_with_why(response: dict, reason: str) -> None: + # when + run = _editor_run(_editor_pages(), ["page3", "page4"], _StubTakeaways(response)) + + # then + assert run.evaluation.strict_checks["document_summaries_present"] is False + assert run.evaluation.failures == ["page 'page3' has a takeaway with no written text", f"page 'page3': {reason}"] + assert run.takeaways_by_editor == 1 + + +def test_a_takeaway_whose_request_the_eval_cannot_build_fails_with_why() -> None: + # given + pages = _editor_pages() + pages[2] = _v2_content_page("page3", takeaway="") + + # when + run = _editor_run(pages, ["page3", "page4"], _StubTakeaways()) + + # then + assert run.evaluation.failures == [ + "page 'page3' has a takeaway with no written text", + "page 'page3': the eval cannot build the editor's takeaway request: page 'page3' has no visualization to summarize", + ] + + +def test_a_draft_the_editor_cannot_read_fails_with_why() -> None: + # given + takeaways = _StubTakeaways() + turn = _editor_turn( + _editor_pages(), ["page3"], period={"type": "relative", "granularity": "CENTURY", "from": -1, "to": -1} + ) + + # when + run = _execute_single_document_run( + _ScriptedChatClient([turn]), "conv-1", "q", {"narrative": _NARRATIVE}, 4, _FakeJudge(), takeaways + ) + + # then + assert takeaways.requests == [] + assert run.evaluation.failures[:2] == [ + "page 'page3' has a takeaway with no written text", + "page 'page4' has a takeaway with no written text", + ] + assert len(run.evaluation.failures) == 3 + assert run.evaluation.failures[2].startswith("the eval cannot read the draft as the editor does: ") + assert "CENTURY" in run.evaluation.failures[2] + + +def test_an_endpoint_that_refuses_the_request_stops_the_item(monkeypatch: pytest.MonkeyPatch) -> None: + # given + _install_client(monkeypatch, _ScriptedChatClient([_editor_turn(_editor_pages(), ["page3"])])) + takeaways = _StubTakeaways(_http_error(404)) + _install_takeaways(monkeypatch, takeaways) + + # when + with pytest.raises(httpx.HTTPStatusError, match="404"): + run_agentic_document_skill("https://h", "tok", "ws", "q", {"narrative": _NARRATIVE}, judge=_FakeJudge()) + + # then + assert takeaways.closed + + +def test_a_transient_endpoint_failure_leaves_the_run_ungraded(monkeypatch: pytest.MonkeyPatch) -> None: + # given + turns = [_editor_turn(_editor_pages(), ["page3", "page4"]) for _ in range(2)] + _install_client(monkeypatch, _ScriptedChatClient(turns)) + _install_takeaways(monkeypatch, _StubTakeaways(_http_error(503))) + judge = _FakeJudge() + + # when + summary = run_agentic_document_skill("https://h", "tok", "ws", "q", {"narrative": _NARRATIVE}, k=2, judge=judge) + + # then + first, second = summary.run_results + assert first.ungraded + assert first.judge_error is not None and first.judge_error.startswith("takeaway endpoint for page 'page3'") + assert "document_summaries_present" not in first.evaluation.strict_checks + assert not second.ungraded and second.evaluation.strict_pass + assert len(judge.calls) == 1, "a run with a takeaway still unwritten is not judged" + assert summary.request_errors == [first.judge_error] + assert summary.judge_errors == [] + assert summary.pass_at_k + assert not summary.pass_power_k + + +def test_an_item_whose_every_run_the_takeaway_endpoint_left_ungraded_raises_naming_the_takeaway_endpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # given + _install_client(monkeypatch, _ScriptedChatClient([_editor_turn(_editor_pages(), ["page3", "page4"])])) + _install_takeaways(monkeypatch, _StubTakeaways(_http_error(503))) + + # when + with pytest.raises(JudgeResponseError) as raised: + evaluate_agentic_document_skill("https://h", "tok", "ws", "q", {"narrative": _NARRATIVE}, judge=_FakeJudge()) + + # then + assert str(raised.value).startswith("no run of 1 could be graded: request failed for 1 run(s): ") + assert "takeaway endpoint for page 'page3'" in str(raised.value) + + +def test_a_transient_failure_after_a_listed_takeaway_already_failed_is_a_failure() -> None: + # when + run = _editor_run( + _editor_pages(), ["page3", "page4"], _StubTakeaways({"takeaway": None, "noData": True}, _http_error(503)) + ) + + # then + assert not run.ungraded + assert run.evaluation.fill_error is None + assert run.evaluation.strict_checks["document_summaries_present"] is False + assert "page 'page3': the editor's takeaway request found no data" in run.evaluation.failures + assert ( + "takeaway endpoint for page 'page4': HTTPStatusError: 503: the takeaway stays empty" in run.evaluation.failures + ) + + +def test_a_transient_failure_beside_a_takeaway_the_open_document_keeps_empty_leaves_the_run_ungraded() -> None: + # given + open_document = _open_document(_editor_pages()) + changed = _editor_pages() + changed[3] = _with_instructions(changed[3], "Focus on the East.") + + # when + run = _editor_run(changed, ["page4"], _StubTakeaways(_http_error(503)), user_context=open_document) + + # then + assert run.ungraded + assert run.evaluation.fill_error is not None + + +def test_a_transient_failure_does_not_stop_the_takeaways_after_it() -> None: + # given + takeaways = _StubTakeaways(httpx.ConnectError("refused")) + + # when + run = _editor_run(_editor_pages(), ["page3", "page4"], takeaways) + + # then + assert len(takeaways.requests) == 2 + assert run.ungraded + assert run.takeaways_by_editor == 1 + + +def test_the_takeaways_written_by_the_editor_reach_langfuse(monkeypatch: pytest.MonkeyPatch) -> None: + # given + _install_takeaways(monkeypatch, _StubTakeaways()) + + # when + ctx = _scored( + monkeypatch, {"narrative": _NARRATIVE}, [_editor_turn(_editor_pages(), ["page3", "page4"])], judge=_FakeJudge() + ) + + # then + assert ctx.scores["document_takeaways_by_editor"] == 2 + assert ctx.score_types["document_takeaways_by_editor"] == "NUMERIC" + assert ctx.scores["document_summaries_present"] == 1.0 + + +def test_no_takeaway_score_is_published_when_nothing_was_left_to_the_editor(monkeypatch: pytest.MonkeyPatch) -> None: + # given + _install_takeaways(monkeypatch, _StubTakeaways()) + + # when + ctx = _scored(monkeypatch, {"narrative": _NARRATIVE}, [_editor_turn(_v2_written_pages(), [])], judge=_FakeJudge()) + + # then + assert "document_takeaways_by_editor" not in ctx.scores + + def test_the_report_skill_module_still_exports_the_old_names() -> None: with pytest.warns(DeprecationWarning, match="gooddata_eval.core.agentic.document_skill"): report_skill = importlib.reload(importlib.import_module("gooddata_eval.core.agentic.report_skill")) diff --git a/packages/gooddata-eval/tests/test_takeaway_client.py b/packages/gooddata-eval/tests/test_takeaway_client.py index 5fe7b084d..515f57515 100644 --- a/packages/gooddata-eval/tests/test_takeaway_client.py +++ b/packages/gooddata-eval/tests/test_takeaway_client.py @@ -99,6 +99,21 @@ def test_placement_visualization_serialized_with_filters() -> None: } +def test_a_bare_input_posts_the_body_an_item_with_it_posts() -> None: + # given + takeaway_input = {"visualizations": [{"visualization_id": "v2"}], "instructions": {"document": "Be brief."}} + from_item: dict = {} + from_input: dict = {} + + # when + _client(_capturing(from_item)).ask(_item(takeaway_input)) + result = _client(_capturing(from_input)).request(TakeawayInput.model_validate(takeaway_input)) + + # then + assert from_input["body"] == from_item["body"] + assert result.text_response == _TAKEAWAY_RESPONSE["takeaway"] + + def test_instructions_omitted_when_empty() -> None: captured: dict = {} client = _client(_capturing(captured))