diff --git a/docs/reporting-source-adapters.md b/docs/reporting-source-adapters.md index 907a31552..16b32ecc6 100644 --- a/docs/reporting-source-adapters.md +++ b/docs/reporting-source-adapters.md @@ -5,6 +5,11 @@ sealed, replayable reporting publication. Return `InlineFetchResult` when metric availability differs within a constituent. Its `cell_availability` map is keyed by **request constituent ID, then metric name**. Constituent IDs are not necessarily media buy IDs; read them from `request.coverage.constituents`. +A package request uses `coverage.grain == "package_item"` and freezes its buy +selector in `coverage.media_buy_ids`. Each package constituent binds both +`media_buy_id` and `package_id`. An empty constituent list can represent a +complete package scan for the selected buys. For ordinary buy requests, +`request.media_buy_ids` derives the selected buys from the constituents. ```python from adcp.reporting.inline_source import InlineFetchResult, MetricEvidence @@ -117,11 +122,12 @@ Cell evidence cannot bypass that freshness gate. Rows can omit missing, unsupported, or delayed metrics; available measurements for other metrics are retained. A control total is a checksum over the staged rows -- a consumer recomputes it from the revision's rows -- so it covers every -staged row, including rows outside the requested coverage, which stay staged -with a warning. It is emitted when every staged row carries a valid finite -numeric value for the metric. Missing fields, nulls, booleans, and invalid -numbers prevent a total, and an omitted explicit-zero row value is not filled -in. Statuses the SDK derives on its own withdraw nothing: a result with no +staged row. Package requests reject rows outside the exact requested +`(media_buy_id, package_id)` pairs before staging; buy requests retain the +warning behavior for outside rows. A total is emitted when every staged row +carries a valid finite numeric value for the metric. Missing fields, nulls, +booleans, and invalid numbers prevent a total. An omitted explicit-zero row +value is not filled in. Statuses the SDK derives on its own withdraw nothing: a result with no `cell_availability` publishes exactly the totals it published before. ### The withdrawal invariant diff --git a/src/adcp/reporting/conformance.py b/src/adcp/reporting/conformance.py index c7b000ee4..f46823f43 100644 --- a/src/adcp/reporting/conformance.py +++ b/src/adcp/reporting/conformance.py @@ -34,10 +34,13 @@ import asyncio import hashlib +import json from collections.abc import Awaitable, Callable, Sequence from datetime import datetime, timezone from typing import Any, Literal, TypeVar +from pydantic import ValidationError + from adcp.reporting.source import ( AuthoritativeOfferingV1, ProvisionalSnapshotOfferingV1, @@ -152,9 +155,15 @@ async def _with_deadline( def _validate_request_against_capabilities( - capabilities: ReportingSourceCapabilitiesV1, request: ReportingSourceSliceRequestV1 + capabilities: ReportingSourceCapabilitiesV1, + request: ReportingSourceSliceRequestV1, ) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: """Is this frozen slice an exact atomic offering of this executor build?""" + if request.contract_version != capabilities.contract_version: + raise _fail( + "CAPABILITY_MISMATCH", + "the executor must explicitly declare the request contract version", + ) if capabilities.scope != "effective_account": raise _fail( "CAPABILITY_MISMATCH", @@ -200,6 +209,15 @@ def _validate_request_against_capabilities( _validate_window_bounds(offering, request) + if ( + request.coverage.grain == "package_item" + and "package_item" not in offering.constituent_kinds + ): + raise _fail( + "CAPABILITY_MISMATCH", + "a package slice requires package_item support in the selected offering", + ) + for constituent in request.coverage.constituents: if constituent.product_id not in offering.product_ids: raise _fail( @@ -294,6 +312,8 @@ def _validate_manifest_against_request( request: ReportingSourceSliceRequestV1, manifest: SourceBatchManifestV1, ) -> None: + if manifest.manifest_version != request.contract_version: + raise _fail("MANIFEST_MISMATCH", "the manifest version must match the request version") if request.coverage.expected == "full" and manifest.coverage.status != "full": raise _fail( "MANIFEST_MISMATCH", @@ -376,6 +396,22 @@ def _validate_manifest_against_request( request.coverage.denominator_fingerprint, ), ] + if request.coverage.grain == "package_item": + echoed.extend( + [ + ("coverage.grain", manifest.coverage.grain, request.coverage.grain), + ( + "coverage.media_buy_ids", + manifest.coverage.media_buy_ids, + request.coverage.media_buy_ids, + ), + ( + "coverage.constituents", + [item.constituent for item in manifest.coverage.constituents], + request.coverage.constituents, + ), + ] + ) for field, actual, expected in echoed: if actual != expected: raise _fail( @@ -470,12 +506,26 @@ async def validate_reporting_source_execution( manifest_bytes = result.manifest_bytes assert manifest_bytes is not None + if response.contract_version != request.contract_version: + raise _fail("MANIFEST_MISMATCH", "the execution response version must match the request") + if response.identity != request.identity: raise _fail( "IDENTITY_MISMATCH", "the response must echo the complete frozen request identity" ) - manifest = parse_verified_source_batch_manifest_v1(response.manifest, manifest_bytes) + try: + manifest = parse_verified_source_batch_manifest_v1(response.manifest, manifest_bytes) + except ValidationError as error: + payload = json.loads(manifest_bytes) + if ( + isinstance(payload, dict) + and payload.get("manifest_version") != request.contract_version + ): + raise _fail( + "MANIFEST_MISMATCH", "the manifest version must match the request" + ) from error + raise _validate_manifest_against_request(offering, request, manifest) cancel = cancel or asyncio.Event() diff --git a/src/adcp/reporting/fixtures.py b/src/adcp/reporting/fixtures.py index ec2d74b13..657f10805 100644 --- a/src/adcp/reporting/fixtures.py +++ b/src/adcp/reporting/fixtures.py @@ -387,6 +387,8 @@ def redacted_completed_result( coverage = SourceBatchCoverageV1( denominator_fingerprint=request.coverage.denominator_fingerprint, + grain=request.coverage.grain, + media_buy_ids=request.coverage.media_buy_ids, status="full" if available else "none", constituents=[ ReportingConstituentCoverageV1(constituent=item, status=status, **evidence) # type: ignore[arg-type] diff --git a/src/adcp/reporting/source.py b/src/adcp/reporting/source.py index 4661c3058..0c6b02594 100644 --- a/src/adcp/reporting/source.py +++ b/src/adcp/reporting/source.py @@ -138,6 +138,8 @@ "source_batch_coverage_fingerprint_v1", "source_batch_manifest_reference_v1", "source_batch_object_set_sha256_v1", + "validate_reporting_source_rows_v1", + "validate_reporting_source_staged_rows_v1", ] REPORTING_SOURCE_CONTRACT_VERSION_V1: Final[Literal["1.0"]] = "1.0" @@ -507,12 +509,13 @@ class ProductConstituentV1(_Frozen): class PackageItemConstituentV1(_Frozen): - """A package-item constituent: a product inside a named package.""" + """A product in one package of one media buy.""" constituent_kind: Literal["package_item"] = "package_item" constituent_id: ExternalId product_id: ExternalId package_id: ExternalId + media_buy_id: ExternalId class MediaBuyConstituentV1(_Frozen): @@ -530,13 +533,26 @@ class MediaBuyConstituentV1(_Frozen): ] -def coverage_denominator_fingerprint_v1(constituents: Sequence[Any]) -> str: +def _require_unique_package_pairs(constituents: Sequence[ReportingConstituent]) -> None: + pairs = [ + (item.media_buy_id, item.package_id) + for item in constituents + if isinstance(item, PackageItemConstituentV1) + ] + if len(set(pairs)) != len(pairs): + raise ValueError("a package pair may appear only once in the coverage denominator") + + +def coverage_denominator_fingerprint_v1( + constituents: Sequence[Any], + *, + grain: Literal["package_item"] | None = None, + media_buy_ids: Sequence[str] | None = None, +) -> str: """Fingerprint the *requested* constituent set, excluding publication status. - This is what makes "did I ask for the same thing?" answerable across a - request and its manifest without comparing two differently shaped - documents. Sorting by ``constituent_id`` means request order is not part - of identity; adding, removing, or re-binding a constituent changes it. + Package requests bind the grain and selected buys as well as constituents. + Sorting by ``constituent_id`` makes request order irrelevant. """ ordered = sorted( ( @@ -549,8 +565,22 @@ def coverage_denominator_fingerprint_v1(constituents: Sequence[Any]) -> str: ), key=lambda item: str(item["constituent_id"]).encode("utf-16-be", errors="surrogatepass"), ) + if grain is None and media_buy_ids is None: + return reporting_fingerprint_v1( + {"kind": "reporting_coverage_denominator_v1", "constituents": ordered} + ) + if grain != "package_item" or not media_buy_ids: + raise ValueError("package fingerprints require grain and nonempty media_buy_ids") return reporting_fingerprint_v1( - {"kind": "reporting_coverage_denominator_v1", "constituents": ordered} + { + "kind": "reporting_coverage_denominator_v1", + "grain": grain, + "media_buy_ids": sorted( + media_buy_ids, + key=lambda value: str(value).encode("utf-16-be", errors="surrogatepass"), + ), + "constituents": ordered, + } ) @@ -829,8 +859,14 @@ class ReportingSourceCoverageRequestV1(_Frozen): """ expected: Literal["full", "partial"] = "full" + grain: Literal["package_item"] | None = Field( + default=None, exclude_if=lambda value: value is None + ) + media_buy_ids: tuple[ExternalId, ...] | None = Field( + default=None, max_length=100_000, exclude_if=lambda value: value is None + ) constituents: list[ReportingConstituent] = Field( - min_length=1, max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 + max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 ) denominator_fingerprint: Fingerprint @@ -839,7 +875,10 @@ def _denominator_binds_the_constituents(self) -> ReportingSourceCoverageRequestV ids = [constituent.constituent_id for constituent in self.constituents] if len(set(ids)) != len(ids): raise ValueError("constituent_id must be unique within a coverage denominator") - expected = coverage_denominator_fingerprint_v1(self.constituents) + _validate_coverage_scope(self.grain, self.media_buy_ids, self.constituents) + expected = coverage_denominator_fingerprint_v1( + self.constituents, grain=self.grain, media_buy_ids=self.media_buy_ids + ) if expected != self.denominator_fingerprint: raise ValueError( "denominator_fingerprint must bind the complete constituent tuple set; " @@ -848,6 +887,29 @@ def _denominator_binds_the_constituents(self) -> ReportingSourceCoverageRequestV return self +def _validate_coverage_scope( + grain: Literal["package_item"] | None, + media_buy_ids: Sequence[str] | None, + constituents: Sequence[ReportingConstituent], +) -> None: + if grain is None: + if media_buy_ids is not None or not constituents: + raise ValueError("nonpackage coverage requires constituents and no buy selector") + if any(item.constituent_kind == "package_item" for item in constituents): + raise ValueError("package coverage requires grain and media_buy_ids") + return + if not media_buy_ids: + raise ValueError("package coverage needs media_buy_ids") + if len(set(media_buy_ids)) != len(media_buy_ids): + raise ValueError("media_buy_ids must be unique") + if any(item.constituent_kind != grain for item in constituents): + raise ValueError("coverage constituents must match the declared grain") + selected = {item.media_buy_id for item in constituents if hasattr(item, "media_buy_id")} + if not selected.issubset(media_buy_ids): + raise ValueError("package pairs must belong to the frozen media_buy_ids") + _require_unique_package_pairs(constituents) + + class ReportingSourceCheckpointV1(_Frozen): """Producer-owned cursor evidence an executor may read but never advance.""" @@ -921,7 +983,9 @@ def product_ids(self) -> tuple[str, ...]: @property def media_buy_ids(self) -> tuple[str, ...]: - """The unique media buys represented by the constituent denominator.""" + """The frozen buy selector, including buys with no package pairs.""" + if self.coverage.media_buy_ids is not None: + return self.coverage.media_buy_ids return tuple( dict.fromkeys( constituent.media_buy_id @@ -942,6 +1006,81 @@ def package_ids(self) -> tuple[str, ...]: ) +def validate_reporting_source_rows_v1( + request: ReportingSourceSliceRequestV1, + manifest: SourceBatchManifestV1, + rows: Sequence[Mapping[str, Any]], +) -> None: + """Reject package rows outside frozen coverage before revision staging.""" + if request.coverage.grain != "package_item": + return + if ( + manifest.coverage.grain != request.coverage.grain + or manifest.coverage.media_buy_ids != request.coverage.media_buy_ids + or manifest.coverage.denominator_fingerprint != request.coverage.denominator_fingerprint + or [item.constituent for item in manifest.coverage.constituents] + != request.coverage.constituents + ): + raise ValueError("manifest coverage differs from the requested coverage") + matched = _matched_reporting_source_rows_v1(request, rows) + statuses = {item.constituent_id: item.status for item in manifest.coverage.constituents} + for index, constituent in enumerate(matched): + if statuses.get(constituent.constituent_id) == "explicit_zero": + raise ValueError(f"row {index} contradicts explicit zero coverage") + + +def validate_reporting_source_staged_rows_v1( + request: ReportingSourceSliceRequestV1, rows: Sequence[Mapping[str, Any]] +) -> None: + """Fence normalized package rows against the frozen request before staging.""" + _matched_reporting_source_rows_v1(request, rows) + + +def _matched_reporting_source_rows_v1( + request: ReportingSourceSliceRequestV1, rows: Sequence[Mapping[str, Any]] +) -> list[PackageItemConstituentV1]: + if request.coverage.grain != "package_item": + return [] + packages = { + (item.media_buy_id, item.package_id): item + for item in request.coverage.constituents + if isinstance(item, PackageItemConstituentV1) + } + matched: list[PackageItemConstituentV1] = [] + seen_grains: set[tuple[str, str, bytes]] = set() + for index, row in enumerate(rows): + package_id = row.get("package_id") + media_buy_id = row.get("media_buy_id") + product_id = row.get("product_id") + constituent_id = row.get("constituent_id") + if any( + value is not None and not isinstance(value, str) + for value in (package_id, media_buy_id, product_id, constituent_id) + ): + raise ValueError(f"row {index} is outside the requested coverage") + constituent = ( + packages.get((media_buy_id, package_id)) + if isinstance(media_buy_id, str) and isinstance(package_id, str) + else None + ) + if ( + constituent is None + or (product_id is not None and product_id != constituent.product_id) + or (constituent_id is not None and constituent_id != constituent.constituent_id) + ): + raise ValueError(f"row {index} is outside the requested coverage") + grain = ( + constituent.media_buy_id, + constituent.package_id, + canonical_json_utf8_v1([row.get(name) for name in request.requested_dimensions]), + ) + if grain in seen_grains: + raise ValueError(f"row {index} is a duplicate row at the requested dimension grain") + seen_grains.add(grain) + matched.append(constituent) + return matched + + # -------------------------------------------------------------------------- # Manifest # -------------------------------------------------------------------------- @@ -1083,14 +1222,23 @@ class SourceBatchCoverageV1(_Frozen): """The slice's coverage roll-up plus its per-constituent evidence.""" denominator_fingerprint: Fingerprint + grain: Literal["package_item"] | None = Field( + default=None, exclude_if=lambda value: value is None + ) + media_buy_ids: tuple[ExternalId, ...] | None = Field( + default=None, max_length=100_000, exclude_if=lambda value: value is None + ) status: ReportingCoverageStatus - constituents: list[ReportingConstituentCoverageV1] = Field(min_length=1, max_length=100_000) + constituents: list[ReportingConstituentCoverageV1] = Field(max_length=100_000) @model_validator(mode="after") def _status_matches_the_evidence(self) -> SourceBatchCoverageV1: ids = [item.constituent_id for item in self.constituents] if len(set(ids)) != len(ids): raise ValueError("constituent_id must be unique within manifest coverage") + _validate_coverage_scope( + self.grain, self.media_buy_ids, [item.constituent for item in self.constituents] + ) covered = sum(1 for item in self.constituents if item.status in _AVAILABLE_STATUSES) partial = sum(1 for item in self.constituents if item.status == "partial") all_covered = covered == len(self.constituents) @@ -1104,7 +1252,9 @@ def _status_matches_the_evidence(self) -> SourceBatchCoverageV1: "partial coverage requires coverage evidence without every constituent covered" ) expected = coverage_denominator_fingerprint_v1( - [item.constituent for item in self.constituents] + [item.constituent for item in self.constituents], + grain=self.grain, + media_buy_ids=self.media_buy_ids, ) if expected != self.denominator_fingerprint: raise ValueError( @@ -1373,7 +1523,7 @@ class SourceBatchManifestV1(_Frozen): byte_count: int = Field(ge=0) control_totals: list[SourceControlTotalV1] = Field(default_factory=list, max_length=1_000) metric_availability: list[ReportingMetricAvailabilityV1] = Field( - min_length=1, max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 + max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 ) coverage: SourceBatchCoverageV1 explicit_zero: bool = False @@ -1519,6 +1669,8 @@ def _validate_finality(self) -> None: # -- cells ----------------------------------------------------------- def _validate_cells(self) -> None: + if self.coverage.grain is None and not self.metric_availability: + raise ValueError("metric availability must carry requested cells") cells = [(cell.constituent_id, cell.metric) for cell in self.metric_availability] if len(set(cells)) != len(cells): raise ValueError( @@ -1556,7 +1708,7 @@ def _validate_cells(self) -> None: # -- zero rows ------------------------------------------------------- def _validate_zero_rows(self) -> None: - """A zero-row period has exactly two honest shapes, and they differ. + """A zero-row period distinguishes an answer from unavailable data. ``explicit_zero``: the source answered for every requested cell and the answer was zero. That is data, and it satisfies an obligation. @@ -1564,7 +1716,22 @@ def _validate_zero_rows(self) -> None: Unavailable: the source did not answer. That is not data, and it must not be dressed up as a zero -- otherwise a dead feed looks like a quiet campaign forever. + + A package selector with no requested pairs is complete without metric + cells or an explicit-zero claim when its source watermark is current. """ + if self.coverage.grain == "package_item" and not self.coverage.constituents: + if self.coverage.status != "full": + raise ValueError("empty package coverage must be full") + if self.row_count != 0 or self.metric_availability or self.explicit_zero: + raise ValueError("complete empty package coverage requires zero rows and cells") + if self.event_time_range is not None or any( + not total.is_zero for total in self.control_totals + ): + raise ValueError("complete empty package coverage cannot declare events or totals") + if self.data_through != min(self.period.end, self.period.source_read_cutoff_at): + raise ValueError("complete empty package coverage needs the current source cutoff") + return if self.explicit_zero: if self.event_time_range is not None: raise ValueError("an explicit-zero batch must not declare an event-time range") @@ -1757,6 +1924,9 @@ def publication_content_fingerprint_v1(manifest: SourceBatchManifestV1 | Mapping }, "explicit_zero": payload.get("explicit_zero", False), } + if payload["coverage"].get("grain") == "package_item": + content["coverage"]["grain"] = payload["coverage"]["grain"] + content["coverage"]["media_buy_ids"] = payload["coverage"]["media_buy_ids"] # Retained manifests without an owner retain their original fingerprint. # Adding a null key would invalidate those immutable publications. if payload["identity"].get("consumer_id") is not None: @@ -1992,10 +2162,8 @@ def completed( manifest: SourceBatchManifestReferenceV1, manifest_bytes: bytes, ) -> ReportingSourceExecutorResult: - return cls( - response=completed_reporting_source_response_v1(request=request, manifest=manifest), - manifest_bytes=manifest_bytes, - ) + response = completed_reporting_source_response_v1(request=request, manifest=manifest) + return cls(response=response, manifest_bytes=manifest_bytes) @classmethod def failed(cls, error: ReportingSourceErrorV1) -> ReportingSourceExecutorResult: diff --git a/tests/test_reporting_source_package_contract.py b/tests/test_reporting_source_package_contract.py new file mode 100644 index 000000000..0fb08d0f6 --- /dev/null +++ b/tests/test_reporting_source_package_contract.py @@ -0,0 +1,315 @@ +"""One source contract covers buy and package denominators.""" + +from __future__ import annotations + +import hashlib + +import pytest +from pydantic import ValidationError + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.conformance import ( + ReportingSourceConformanceError, + validate_reporting_source_execution, +) +from adcp.reporting.fixtures import ( + redacted_capabilities, + redacted_completed_result, + redacted_snapshot_request, +) +from adcp.reporting.source import ( + PackageItemConstituentV1, + ReportingSourceCapabilitiesV1, + ReportingSourceCoverageRequestV1, + ReportingSourceSliceRequestV1, + SourceBatchManifestV1, + coverage_denominator_fingerprint_v1, + encode_source_batch_manifest_v1, + parse_verified_source_batch_manifest_v1, + publication_content_fingerprint_v1, + reporting_source_capabilities_sha256_v1, + source_batch_manifest_reference_v1, + validate_reporting_source_rows_v1, + validate_reporting_source_staged_rows_v1, +) + + +def _package_request(*, campaign_dimension: bool = False) -> ReportingSourceSliceRequestV1: + package = PackageItemConstituentV1( + constituent_id="package-constituent", + product_id="fixture-product", + media_buy_id="buy-1", + package_id="package-1", + ) + payload = redacted_snapshot_request().model_dump(mode="json") + payload["requested_dimensions"] = ( + ["package", "campaign_id"] if campaign_dimension else ["package"] + ) + payload["coverage"] = ReportingSourceCoverageRequestV1( + grain="package_item", + media_buy_ids=("buy-1",), + constituents=[package], + denominator_fingerprint=coverage_denominator_fingerprint_v1( + [package], grain="package_item", media_buy_ids=("buy-1",) + ), + ).model_dump(mode="json") + return ReportingSourceSliceRequestV1.model_validate(payload) + + +def _empty_package_request(buy_id: str = "buy-1") -> ReportingSourceSliceRequestV1: + payload = redacted_snapshot_request().model_dump(mode="json") + payload["coverage"] = { + "expected": "full", + "grain": "package_item", + "media_buy_ids": [buy_id], + "constituents": [], + "denominator_fingerprint": coverage_denominator_fingerprint_v1( + [], grain="package_item", media_buy_ids=(buy_id,) + ), + } + return ReportingSourceSliceRequestV1.model_validate(payload) + + +def test_ordinary_buy_v1_wire_and_hashes_remain_frozen() -> None: + request = redacted_snapshot_request() + request_json = request.model_dump_json() + canonical_request = canonical_json_utf8_v1(request.model_dump(mode="json", exclude_none=True)) + assert hashlib.sha256(canonical_request).hexdigest() == ( + "0b81860effeac0424f5d2efa9d8fc680ee86cb226749dbe17812bf94296c0f50" + ) + assert hashlib.sha256(request_json.encode()).hexdigest() == ( + "53b59efb8632ac2e314da2d4144ed2295ef1c7e9a49cb1fc3f4d55f9da724585" + ) + assert request.coverage.denominator_fingerprint == ( + "sha256:27bf44bb30e8ae26980327cf0913d74327dc2c344eaad02c194604672f16b74d" + ) + assert ReportingSourceSliceRequestV1.model_validate_json(request_json).model_dump_json() == ( + request_json + ) + assert "grain" not in request.coverage.model_dump() + assert "media_buy_ids" not in request.coverage.model_dump() + _, manifest, _ = redacted_completed_result(request) + assert manifest.content_fingerprint == ( + "sha256:178e0438af4767776bd58316b868c111a9f4df806f506c951ffae49e5410e45f" + ) + assert "grain" not in manifest.coverage.model_dump() + assert "media_buy_ids" not in manifest.coverage.model_dump() + manifest_bytes = encode_source_batch_manifest_v1(manifest) + assert ( + parse_verified_source_batch_manifest_v1( + source_batch_manifest_reference_v1("manifest-buy-v1", manifest_bytes), manifest_bytes + ) + == manifest + ) + + +def test_package_selector_and_pair_are_bound_on_the_v1_wire() -> None: + request = _package_request() + assert request.contract_version == "1.0" + assert request.media_buy_ids == ("buy-1",) + assert request.package_ids == ("package-1",) + assert request.coverage.grain == "package_item" + assert request.coverage.constituents[0].media_buy_id == "buy-1" + + +def test_complete_empty_package_scope_has_distinct_frozen_selectors() -> None: + first = _empty_package_request("buy-1") + second = _empty_package_request("buy-2") + assert first.media_buy_ids == ("buy-1",) + assert second.media_buy_ids == ("buy-2",) + assert first.coverage.denominator_fingerprint != second.coverage.denominator_fingerprint + + +async def test_complete_empty_package_scope_is_full_without_explicit_zero() -> None: + request = _empty_package_request() + result, manifest, reader = redacted_completed_result( + request, page_bodies=[""], row_counts=[0], explicit_zero=False + ) + assert manifest.manifest_version == "1.0" + assert manifest.coverage.grain == "package_item" + assert manifest.coverage.media_buy_ids == ("buy-1",) + assert manifest.coverage.status == "full" + assert not manifest.metric_availability + assert not manifest.explicit_zero + assert ( + await validate_reporting_source_execution( + capabilities=redacted_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + == manifest + ) + + +def test_complete_empty_package_scope_requires_current_watermark() -> None: + request = _empty_package_request() + _, manifest, _ = redacted_completed_result( + request, page_bodies=[""], row_counts=[0], explicit_zero=False + ) + payload = manifest.model_dump(mode="json") + payload["data_through"] = request.period.start.isoformat() + payload["content_fingerprint"] = publication_content_fingerprint_v1(payload) + with pytest.raises(ValidationError, match="current source cutoff"): + SourceBatchManifestV1.model_validate(payload) + + +def test_missing_wire_scope_fields_are_rejected() -> None: + for field in ("grain", "media_buy_ids"): + request_payload = _package_request().model_dump(mode="json") + request_payload["coverage"].pop(field) + with pytest.raises(ValidationError): + ReportingSourceSliceRequestV1.model_validate(request_payload) + _, manifest, _ = redacted_completed_result(_package_request()) + manifest_payload = manifest.model_dump(mode="json") + manifest_payload["coverage"].pop(field) + with pytest.raises(ValidationError): + SourceBatchManifestV1.model_validate(manifest_payload) + + +async def test_empty_package_scope_still_requires_package_capability() -> None: + request = _empty_package_request() + result, _, reader = redacted_completed_result( + request, page_bodies=[""], row_counts=[0], explicit_zero=False + ) + capabilities_payload = redacted_capabilities().model_dump(mode="json") + capabilities_payload["offerings"][0]["constituent_kinds"] = ["media_buy"] + capabilities_payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v1( + capabilities_payload + ) + capabilities = ReportingSourceCapabilitiesV1.model_validate(capabilities_payload) + with pytest.raises(ReportingSourceConformanceError, match="package_item support"): + await validate_reporting_source_execution( + capabilities=capabilities, + request=request, + result=result, + object_reader=reader, + ) + + +def test_package_denominator_rejects_duplicate_pairs_and_unselected_buy() -> None: + package = _package_request().coverage.constituents[0] + duplicate = package.model_copy(update={"constituent_id": "another-constituent"}) + for constituents, selector in (([package, duplicate], ("buy-1",)), ([package], ("buy-2",))): + with pytest.raises(ValidationError): + ReportingSourceCoverageRequestV1( + grain="package_item", + media_buy_ids=selector, + constituents=constituents, + denominator_fingerprint=coverage_denominator_fingerprint_v1( + constituents, grain="package_item", media_buy_ids=selector + ), + ) + + +def test_one_package_id_can_be_used_by_two_selected_buys() -> None: + first = _package_request().coverage.constituents[0] + second = first.model_copy( + update={"constituent_id": "second-constituent", "media_buy_id": "buy-2"} + ) + coverage = ReportingSourceCoverageRequestV1( + grain="package_item", + media_buy_ids=("buy-1", "buy-2"), + constituents=[first, second], + denominator_fingerprint=coverage_denominator_fingerprint_v1( + [first, second], grain="package_item", media_buy_ids=("buy-1", "buy-2") + ), + ) + assert [(item.media_buy_id, item.package_id) for item in coverage.constituents] == [ + ("buy-1", "package-1"), + ("buy-2", "package-1"), + ] + + +def test_manifest_must_echo_selector_and_pairs_exactly() -> None: + request = _package_request() + _, manifest, _ = redacted_completed_result(request) + payload = manifest.model_dump(mode="json") + payload["coverage"]["media_buy_ids"] = ["buy-1", "buy-2"] + payload["coverage"]["denominator_fingerprint"] = coverage_denominator_fingerprint_v1( + request.coverage.constituents, + grain="package_item", + media_buy_ids=("buy-1", "buy-2"), + ) + payload["content_fingerprint"] = publication_content_fingerprint_v1(payload) + widened = SourceBatchManifestV1.model_validate(payload) + with pytest.raises(ValueError, match="manifest coverage differs"): + validate_reporting_source_rows_v1(request, widened, []) + + +def test_explicit_zero_package_pair_rejects_a_row() -> None: + request = _package_request() + _, manifest, _ = redacted_completed_result(request, page_bodies=[""], row_counts=[0]) + with pytest.raises(ValueError, match="contradicts explicit zero"): + validate_reporting_source_rows_v1( + request, + manifest, + [{"media_buy_id": "buy-1", "package_id": "package-1", "impressions": 0}], + ) + + +def test_package_request_retains_the_1000_cell_limit() -> None: + first = _package_request().coverage.constituents[0] + for pair_count, allowed in ((500, True), (501, False)): + constituents = [ + first.model_copy( + update={"constituent_id": f"item-{index}", "package_id": f"package-{index}"} + ) + for index in range(pair_count) + ] + payload = _package_request().model_dump(mode="json") + payload["coverage"] = ReportingSourceCoverageRequestV1( + grain="package_item", + media_buy_ids=("buy-1",), + constituents=constituents, + denominator_fingerprint=coverage_denominator_fingerprint_v1( + constituents, grain="package_item", media_buy_ids=("buy-1",) + ), + ).model_dump(mode="json") + if allowed: + ReportingSourceSliceRequestV1.model_validate(payload) + else: + with pytest.raises(ValidationError, match="at most 1000 constituent-metric cells"): + ReportingSourceSliceRequestV1.model_validate(payload) + + +@pytest.mark.parametrize("stage", ["staged", "final"]) +def test_package_rows_reject_outside_pairs_and_duplicate_requested_grain(stage: str) -> None: + request = _package_request() + rows = [ + {"media_buy_id": "buy-1", "package_id": "package-1", "impressions": 4}, + {"media_buy_id": "buy-1", "package_id": "package-1", "impressions": 5}, + ] + manifest = redacted_completed_result(request)[1] if stage == "final" else None + + def validate(values: list[dict[str, str | int]]) -> None: + if manifest is None: + validate_reporting_source_staged_rows_v1(request, values) + else: + validate_reporting_source_rows_v1(request, manifest, values) + + with pytest.raises(ValueError, match="duplicate row"): + validate(rows) + for row in ( + {"media_buy_id": "buy-2", "package_id": "package-1", "impressions": 0}, + {"media_buy_id": "buy-1", "package_id": "package-2", "impressions": 0}, + {"media_buy_id": "buy-1", "impressions": 0}, + ): + with pytest.raises(ValueError, match="outside the requested coverage"): + validate([row]) + + +@pytest.mark.parametrize("stage", ["staged", "final"]) +def test_distinct_requested_campaign_values_remain_valid(stage: str) -> None: + request = _package_request(campaign_dimension=True) + rows = [ + {"media_buy_id": "buy-1", "package_id": "package-1", "campaign_id": "a"}, + {"media_buy_id": "buy-1", "package_id": "package-1", "campaign_id": "b"}, + ] + if stage == "staged": + validate_reporting_source_staged_rows_v1(request, rows) + else: + from adcp.reporting.fixtures import redacted_completed_result + + _, manifest, _ = redacted_completed_result(request) + validate_reporting_source_rows_v1(request, manifest, rows)