From 9e956c163dbacda6720ed60ff24c03f837a9f03f Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Thu, 19 Nov 2026 23:23:16 +0530 Subject: [PATCH 1/2] feat(reporting): bind package source coverage to media buys --- src/adcp/reporting/conformance.py | 80 ++- src/adcp/reporting/source.py | 425 +++++++++++- tests/test_reporting_source_contract_v2.py | 756 +++++++++++++++++++++ 3 files changed, 1249 insertions(+), 12 deletions(-) create mode 100644 tests/test_reporting_source_contract_v2.py diff --git a/src/adcp/reporting/conformance.py b/src/adcp/reporting/conformance.py index c7b000ee47..c56e571ea0 100644 --- a/src/adcp/reporting/conformance.py +++ b/src/adcp/reporting/conformance.py @@ -34,25 +34,33 @@ import asyncio import hashlib +import json from collections.abc import Awaitable, Callable, Sequence from datetime import datetime, timezone from typing import Any, Literal, TypeVar +from pydantic import ValidationError + from adcp.reporting.source import ( AuthoritativeOfferingV1, ProvisionalSnapshotOfferingV1, ReportingSourceCapabilitiesV1, + ReportingSourceCapabilitiesV2, ReportingSourceErrorCode, ReportingSourceErrorV1, ReportingSourceExecutor, ReportingSourceExecutorResult, ReportingSourceSliceRequestV1, + ReportingSourceSliceRequestV2, ReportingSourceStagedObjectReader, + SourceBatchManifestError, SourceBatchManifestV1, + SourceBatchManifestV2, _validate_metric_applicability, deterministic_source_publication_id_v1, iso_duration_milliseconds_v1, parse_verified_source_batch_manifest_v1, + parse_verified_source_batch_manifest_v2, source_batch_object_set_sha256_v1, ) @@ -152,9 +160,20 @@ async def _with_deadline( def _validate_request_against_capabilities( - capabilities: ReportingSourceCapabilitiesV1, request: ReportingSourceSliceRequestV1 + capabilities: ReportingSourceCapabilitiesV1 | ReportingSourceCapabilitiesV2, + request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, ) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: """Is this frozen slice an exact atomic offering of this executor build?""" + supported_versions = ( + capabilities.supported_contract_versions + if isinstance(capabilities, ReportingSourceCapabilitiesV2) + else (capabilities.contract_version,) + ) + if request.contract_version not in supported_versions: + raise _fail( + "CAPABILITY_MISMATCH", + "the executor must explicitly declare the request contract version", + ) if capabilities.scope != "effective_account": raise _fail( "CAPABILITY_MISMATCH", @@ -200,6 +219,15 @@ def _validate_request_against_capabilities( _validate_window_bounds(offering, request) + if ( + isinstance(request, ReportingSourceSliceRequestV2) + and "package_item" not in offering.constituent_kinds + ): + raise _fail( + "CAPABILITY_MISMATCH", + "a V2 package slice requires package_item support in the selected offering", + ) + for constituent in request.coverage.constituents: if constituent.product_id not in offering.product_ids: raise _fail( @@ -291,9 +319,11 @@ def _source_calendar_elapsed_ms(request: ReportingSourceSliceRequestV1) -> int: def _validate_manifest_against_request( offering: ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1, - request: ReportingSourceSliceRequestV1, - manifest: SourceBatchManifestV1, + request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, + manifest: SourceBatchManifestV1 | SourceBatchManifestV2, ) -> None: + if manifest.manifest_version != request.contract_version: + raise _fail("MANIFEST_MISMATCH", "the manifest version must match the request version") if request.coverage.expected == "full" and manifest.coverage.status != "full": raise _fail( "MANIFEST_MISMATCH", @@ -376,6 +406,22 @@ def _validate_manifest_against_request( request.coverage.denominator_fingerprint, ), ] + if isinstance(request, ReportingSourceSliceRequestV2): + assert isinstance(manifest, SourceBatchManifestV2) + echoed.extend( + [ + ( + "coverage.media_buy_ids", + manifest.coverage.media_buy_ids, + request.coverage.media_buy_ids, + ), + ( + "coverage.constituents", + [item.constituent for item in manifest.coverage.constituents], + request.coverage.constituents, + ), + ] + ) for field, actual, expected in echoed: if actual != expected: raise _fail( @@ -445,13 +491,13 @@ def _validate_manifest_against_request( async def validate_reporting_source_execution( *, - capabilities: ReportingSourceCapabilitiesV1, - request: ReportingSourceSliceRequestV1, + capabilities: ReportingSourceCapabilitiesV1 | ReportingSourceCapabilitiesV2, + request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, result: ReportingSourceExecutorResult, object_reader: ReportingSourceStagedObjectReader, cancel: asyncio.Event | None = None, clock: Callable[[], datetime] | None = None, -) -> SourceBatchManifestV1: +) -> SourceBatchManifestV1 | SourceBatchManifestV2: """Validate one execution end to end and return its verified manifest. Reads every staged object the manifest names and checks its bytes against @@ -470,12 +516,32 @@ async def validate_reporting_source_execution( manifest_bytes = result.manifest_bytes assert manifest_bytes is not None + if response.contract_version != request.contract_version: + raise _fail("MANIFEST_MISMATCH", "the execution response version must match the request") + if response.identity != request.identity: raise _fail( "IDENTITY_MISMATCH", "the response must echo the complete frozen request identity" ) - manifest = parse_verified_source_batch_manifest_v1(response.manifest, manifest_bytes) + manifest: SourceBatchManifestV1 | SourceBatchManifestV2 + if isinstance(request, ReportingSourceSliceRequestV2): + try: + manifest = parse_verified_source_batch_manifest_v2(response.manifest, manifest_bytes) + except SourceBatchManifestError as error: + if error.code == "SOURCE_MANIFEST_VERSION_MISMATCH": + raise _fail("MANIFEST_MISMATCH", "the manifest must declare version 2.0") from error + raise + else: + try: + manifest = parse_verified_source_batch_manifest_v1(response.manifest, manifest_bytes) + except ValidationError as error: + payload = json.loads(manifest_bytes) + if isinstance(payload, dict) and payload.get("manifest_version") == "2.0": + raise _fail( + "MANIFEST_MISMATCH", "the manifest version must match the request" + ) from error + raise _validate_manifest_against_request(offering, request, manifest) cancel = cancel or asyncio.Event() diff --git a/src/adcp/reporting/source.py b/src/adcp/reporting/source.py index 4661c3058b..90d6265ad3 100644 --- a/src/adcp/reporting/source.py +++ b/src/adcp/reporting/source.py @@ -88,18 +88,22 @@ __all__ = [ "REPORTING_EVIDENCE_REASON_MAX_LENGTH_V1", "REPORTING_SOURCE_CONTRACT_VERSION_V1", + "REPORTING_SOURCE_CONTRACT_VERSION_V2", "SOURCE_BATCH_MANIFEST_MAX_BYTES_V1", "SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1", "SOURCE_BATCH_MANIFEST_MAX_PROVIDER_CALLS_V1", "AuthoritativeOfferingV1", "MediaBuyConstituentV1", "PackageItemConstituentV1", + "PackageItemConstituentV2", "ProductConstituentV1", "ProvisionalSnapshotOfferingV1", "ReportingAdapterBuildIdentityV1", "ReportingAvailabilityStatus", "ReportingConstituent", + "ReportingConstituentV2", "ReportingConstituentCoverageV1", + "ReportingConstituentCoverageV2", "ReportingContractIdentityV1", "ReportingCoverageStatus", "ReportingFinalityEvidenceV1", @@ -111,10 +115,13 @@ "ReportingPublicationClass", "ReportingRevisionKind", "ReportingSourceCapabilitiesV1", + "ReportingSourceCapabilitiesV2", + "ReportingSourceCoverageRequestV2", "ReportingSourceError", "ReportingSourceErrorCode", "ReportingSourceErrorV1", "ReportingSourceExecutionResponseV1", + "ReportingSourceExecutionResponseV2", "ReportingSourceExecutor", "ReportingSourceExecutorResult", "ReportingSourceIdentityV1", @@ -122,25 +129,37 @@ "ReportingSourcePeriodV1", "ReportingSourceRetryClass", "ReportingSourceSliceRequestV1", + "ReportingSourceSliceRequestV2", "ReportingSourceStagedObjectReader", "ReportingSourceWindowingV1", "ReportingTriggerKind", "SourceBatchManifestReferenceV1", "SourceBatchManifestV1", + "SourceBatchManifestV2", + "SourceBatchCoverageV2", "SourceBatchObjectV1", "SourceControlTotalV1", "completed_reporting_source_response_v1", "coverage_denominator_fingerprint_v1", + "coverage_denominator_fingerprint_v2", "deterministic_source_publication_id_v1", "encode_source_batch_manifest_v1", + "encode_source_batch_manifest_v2", + "parse_verified_source_batch_manifest_v2", "publication_content_fingerprint_v1", + "publication_content_fingerprint_v2", "reporting_source_capabilities_sha256_v1", + "reporting_source_capabilities_sha256_v2", "source_batch_coverage_fingerprint_v1", + "source_batch_coverage_fingerprint_v2", "source_batch_manifest_reference_v1", "source_batch_object_set_sha256_v1", + "validate_reporting_source_rows_v2", + "validate_reporting_source_staged_rows_v2", ] REPORTING_SOURCE_CONTRACT_VERSION_V1: Final[Literal["1.0"]] = "1.0" +REPORTING_SOURCE_CONTRACT_VERSION_V2: Final[Literal["2.0"]] = "2.0" #: A retained manifest is evidence, not a payload. Rows belong in staged #: objects; the manifest that binds them stays small enough to keep forever. @@ -515,6 +534,16 @@ class PackageItemConstituentV1(_Frozen): package_id: ExternalId +class PackageItemConstituentV2(_Frozen): + """A package constituent scoped to exactly one media buy.""" + + constituent_kind: Literal["package_item"] = "package_item" + constituent_id: ExternalId + product_id: ExternalId + package_id: ExternalId + media_buy_id: ExternalId + + class MediaBuyConstituentV1(_Frozen): """A media-buy constituent: a product inside a named media buy.""" @@ -529,6 +558,21 @@ class MediaBuyConstituentV1(_Frozen): Field(discriminator="constituent_kind"), ] +ReportingConstituentV2 = Annotated[ + ProductConstituentV1 | PackageItemConstituentV2 | MediaBuyConstituentV1, + Field(discriminator="constituent_kind"), +] + + +def _require_unique_package_pairs(constituents: Sequence[ReportingConstituentV2]) -> None: + pairs = [ + (item.media_buy_id, item.package_id) + for item in constituents + if isinstance(item, PackageItemConstituentV2) + ] + if len(set(pairs)) != len(pairs): + raise ValueError("a package pair may appear only once in the coverage denominator") + def coverage_denominator_fingerprint_v1(constituents: Sequence[Any]) -> str: """Fingerprint the *requested* constituent set, excluding publication status. @@ -554,6 +598,35 @@ def coverage_denominator_fingerprint_v1(constituents: Sequence[Any]) -> str: ) +def coverage_denominator_fingerprint_v2( + constituents: Sequence[Any], *, media_buy_ids: Sequence[str] | None = None +) -> str: + """Fingerprint the frozen buy selector and its exact package pair set.""" + if media_buy_ids is None: + media_buy_ids = sorted({item.media_buy_id for item in constituents}) + ordered = sorted( + ( + { + key: value + for key, value in _constituent_identity(constituent).items() + if value is not None + } + for constituent in constituents + ), + key=lambda item: str(item["constituent_id"]).encode("utf-16-be", errors="surrogatepass"), + ) + return reporting_fingerprint_v1( + { + "kind": "reporting_coverage_denominator_v2", + "media_buy_ids": sorted( + media_buy_ids, + key=lambda value: str(value).encode("utf-16-be", errors="surrogatepass"), + ), + "constituents": ordered, + } + ) + + def _constituent_identity(constituent: Any) -> dict[str, Any]: """The identity half of a constituent, with mutable publication status dropped.""" if isinstance(constituent, BaseModel): @@ -800,6 +873,22 @@ def offering(self, offering_id: str) -> ProvisionalSnapshotOfferingV1 | Authorit raise KeyError(offering_id) +class ReportingSourceCapabilitiesV2(ReportingSourceCapabilitiesV1): + """An executor declaration with an explicit set of supported source versions.""" + + contract_version: Literal["2.0"] # type: ignore[assignment] + supported_contract_versions: tuple[Literal["1.0", "2.0"], ...] = Field( + min_length=1, max_length=2 + ) + + @model_validator(mode="after") + def _supported_versions_are_unique(self) -> ReportingSourceCapabilitiesV2: + versions = self.supported_contract_versions + if len(set(versions)) != len(versions) or "2.0" not in versions: + raise ValueError("supported_contract_versions must uniquely include version 2.0") + return self + + def reporting_source_capabilities_sha256_v1( capabilities: ReportingSourceCapabilitiesV1 | Mapping[str, Any], ) -> str: @@ -814,6 +903,13 @@ def reporting_source_capabilities_sha256_v1( return canonical_json_sha256_v1(payload) +def reporting_source_capabilities_sha256_v2( + capabilities: ReportingSourceCapabilitiesV2 | Mapping[str, Any], +) -> str: + """Bind the complete V2 capability declaration, including its version.""" + return reporting_source_capabilities_sha256_v1(capabilities) + + # -------------------------------------------------------------------------- # Slice request # -------------------------------------------------------------------------- @@ -848,6 +944,35 @@ def _denominator_binds_the_constituents(self) -> ReportingSourceCoverageRequestV return self +class ReportingSourceCoverageRequestV2(ReportingSourceCoverageRequestV1): + """The frozen buy selector and its requested package pairs.""" + + media_buy_ids: tuple[ExternalId, ...] = Field(min_length=1, max_length=100_000) + constituents: list[PackageItemConstituentV2] = Field( # type: ignore[assignment] + max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 + ) + + @model_validator(mode="after") + def _denominator_binds_the_constituents(self) -> ReportingSourceCoverageRequestV2: + ids = [constituent.constituent_id for constituent in self.constituents] + if len(set(ids)) != len(ids): + raise ValueError("constituent_id must be unique within a coverage denominator") + if len(set(self.media_buy_ids)) != len(self.media_buy_ids): + raise ValueError("media_buy_ids must be unique") + if any(item.media_buy_id not in self.media_buy_ids for item in self.constituents): + raise ValueError("package pairs must belong to the frozen media_buy_ids") + _require_unique_package_pairs(self.constituents) + expected = coverage_denominator_fingerprint_v2( + self.constituents, media_buy_ids=self.media_buy_ids + ) + if expected != self.denominator_fingerprint: + raise ValueError( + "denominator_fingerprint must bind the complete constituent tuple set; " + f"expected {expected}" + ) + return self + + class ReportingSourceCheckpointV1(_Frozen): """Producer-owned cursor evidence an executor may read but never advance.""" @@ -942,6 +1067,92 @@ def package_ids(self) -> tuple[str, ...]: ) +class ReportingSourceSliceRequestV2(ReportingSourceSliceRequestV1): + """A package-aware slice sent only to a V2-capable executor.""" + + contract_version: Literal["2.0"] # type: ignore[assignment] + coverage: ReportingSourceCoverageRequestV2 + + @property + def package_ids(self) -> tuple[str, ...]: + """The unique packages represented by the V2 denominator.""" + return tuple( + dict.fromkeys( + constituent.package_id + for constituent in self.coverage.constituents + if isinstance(constituent, PackageItemConstituentV2) + ) + ) + + @property + def media_buy_ids(self) -> tuple[str, ...]: + """The frozen media buys, including those with no package pairs.""" + return self.coverage.media_buy_ids + + +def validate_reporting_source_rows_v2( + request: ReportingSourceSliceRequestV2, + manifest: SourceBatchManifestV2, + rows: Sequence[Mapping[str, Any]], +) -> None: + """Reject normalized rows outside a frozen V2 coverage denominator before revision staging.""" + if manifest.coverage.denominator_fingerprint != request.coverage.denominator_fingerprint: + raise ValueError("manifest coverage differs from the requested coverage") + matched = _matched_reporting_source_rows_v2(request, rows) + statuses = {item.constituent_id: item.status for item in manifest.coverage.constituents} + for index, constituent in enumerate(matched): + if statuses.get(constituent.constituent_id) == "explicit_zero": + raise ValueError(f"row {index} contradicts explicit zero coverage") + + +def validate_reporting_source_staged_rows_v2( + request: ReportingSourceSliceRequestV2, rows: Sequence[Mapping[str, Any]] +) -> None: + """Fence normalized package rows against the frozen request before staging.""" + _matched_reporting_source_rows_v2(request, rows) + + +def _matched_reporting_source_rows_v2( + request: ReportingSourceSliceRequestV2, rows: Sequence[Mapping[str, Any]] +) -> list[PackageItemConstituentV2]: + packages = { + (item.media_buy_id, item.package_id): item for item in request.coverage.constituents + } + matched: list[PackageItemConstituentV2] = [] + seen_grains: set[tuple[str, str, bytes]] = set() + for index, row in enumerate(rows): + package_id = row.get("package_id") + media_buy_id = row.get("media_buy_id") + product_id = row.get("product_id") + constituent_id = row.get("constituent_id") + if any( + value is not None and not isinstance(value, str) + for value in (package_id, media_buy_id, product_id, constituent_id) + ): + raise ValueError(f"row {index} is outside the requested coverage") + constituent = ( + packages.get((media_buy_id, package_id)) + if isinstance(media_buy_id, str) and isinstance(package_id, str) + else None + ) + if ( + constituent is None + or (product_id is not None and product_id != constituent.product_id) + or (constituent_id is not None and constituent_id != constituent.constituent_id) + ): + raise ValueError(f"row {index} is outside the requested coverage") + grain = ( + constituent.media_buy_id, + constituent.package_id, + canonical_json_utf8_v1([row.get(name) for name in request.requested_dimensions]), + ) + if grain in seen_grains: + raise ValueError(f"row {index} is a duplicate row at the requested dimension grain") + seen_grains.add(grain) + matched.append(constituent) + return matched + + # -------------------------------------------------------------------------- # Manifest # -------------------------------------------------------------------------- @@ -1040,6 +1251,25 @@ def constituent_id(self) -> str: return self.constituent.constituent_id +class ReportingConstituentCoverageV2(_Frozen): + """Coverage evidence for a constituent with a complete package identity.""" + + constituent: PackageItemConstituentV2 + status: ReportingAvailabilityStatus + data_through: datetime | None = None + reason: EvidenceReason | None = None + + @model_validator(mode="after") + def _unavailable_coverage_is_explained(self) -> ReportingConstituentCoverageV2: + if self.status not in _AVAILABLE_STATUSES and not self.reason: + raise ValueError(f"constituent status {self.status!r} needs a stable reason") + return self + + @property + def constituent_id(self) -> str: + return self.constituent.constituent_id + + class ReportingMetricAvailabilityV1(_Frozen): """One ``(constituent, metric)`` cell's publication evidence. @@ -1114,6 +1344,49 @@ def _status_matches_the_evidence(self) -> SourceBatchCoverageV1: return self +class SourceBatchCoverageV2(SourceBatchCoverageV1): + """Coverage evidence for the frozen buy selector and package pairs.""" + + media_buy_ids: tuple[ExternalId, ...] = Field(min_length=1, max_length=100_000) + constituents: list[ReportingConstituentCoverageV2] = Field( # type: ignore[assignment] + max_length=100_000 + ) + + @model_validator(mode="after") + def _status_matches_the_evidence(self) -> SourceBatchCoverageV2: + ids = [item.constituent_id for item in self.constituents] + if len(set(ids)) != len(ids): + raise ValueError("constituent_id must be unique within manifest coverage") + if len(set(self.media_buy_ids)) != len(self.media_buy_ids): + raise ValueError("media_buy_ids must be unique") + if any( + item.constituent.media_buy_id not in self.media_buy_ids for item in self.constituents + ): + raise ValueError("package pairs must belong to the frozen media_buy_ids") + _require_unique_package_pairs([item.constituent for item in self.constituents]) + covered = sum(1 for item in self.constituents if item.status in _AVAILABLE_STATUSES) + partial = sum(1 for item in self.constituents if item.status == "partial") + all_covered = covered == len(self.constituents) + any_evidence = covered > 0 or partial > 0 + if self.status == "full" and not all_covered: + raise ValueError("full coverage requires every constituent present or explicit zero") + if self.status == "none" and any_evidence: + raise ValueError("none coverage must not contain a covered or partial constituent") + if self.status == "partial" and (not any_evidence or all_covered): + raise ValueError( + "partial coverage requires coverage evidence without every constituent covered" + ) + expected = coverage_denominator_fingerprint_v2( + [item.constituent for item in self.constituents], media_buy_ids=self.media_buy_ids + ) + if expected != self.denominator_fingerprint: + raise ValueError( + "denominator_fingerprint must bind the complete constituent tuple set; " + f"expected {expected}" + ) + return self + + class ReportingProviderPageV1(_Frozen): """One successful upstream result page. @@ -1649,6 +1922,50 @@ def _validate_strictness(self) -> None: ) +class SourceBatchManifestV2(SourceBatchManifestV1): + """A versioned publication that echoes the package and media-buy pairs.""" + + manifest_version: Literal["2.0"] # type: ignore[assignment] + coverage: SourceBatchCoverageV2 + metric_availability: list[ReportingMetricAvailabilityV1] = Field( + max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 + ) + + def _validate_zero_rows(self) -> None: + if not self.coverage.constituents and self.coverage.status == "full": + if self.row_count != 0 or self.metric_availability or self.explicit_zero: + raise ValueError("complete empty package coverage requires zero rows and cells") + if self.event_time_range is not None or any( + not total.is_zero for total in self.control_totals + ): + raise ValueError("complete empty package coverage cannot declare events or totals") + if self.data_through != min(self.period.end, self.period.source_read_cutoff_at): + raise ValueError("complete empty package coverage needs the current source cutoff") + return + super()._validate_zero_rows() + + @property + def contract_version(self) -> Literal["2.0"]: + """The negotiated source contract version.""" + return self.manifest_version + + @model_validator(mode="after") + def _manifest_is_internally_consistent(self) -> SourceBatchManifestV2: + self._validate_objects() + self._validate_time_evidence() + self._validate_finality() + self._validate_cells() + self._validate_zero_rows() + self._validate_event_range() + self._validate_strictness() + expected = publication_content_fingerprint_v2(self) + if expected != self.content_fingerprint: + raise ValueError( + f"content_fingerprint must bind the publication content; expected {expected}" + ) + return self + + # -------------------------------------------------------------------------- # Fingerprints and sealing # -------------------------------------------------------------------------- @@ -1698,6 +2015,30 @@ def source_batch_coverage_fingerprint_v1( ) +def source_batch_coverage_fingerprint_v2( + constituents: Sequence[ReportingConstituentCoverageV2], + *, + media_buy_ids: Sequence[str], +) -> str: + """Fingerprint package evidence and its frozen media-buy selector.""" + ordered = sorted( + (item.canonical_payload() for item in constituents), + key=lambda item: str(item["constituent"]["constituent_id"]).encode( + "utf-16-be", errors="surrogatepass" + ), + ) + return reporting_fingerprint_v1( + { + "kind": "reporting_coverage_evidence_v2", + "media_buy_ids": sorted( + media_buy_ids, + key=lambda value: str(value).encode("utf-16-be", errors="surrogatepass"), + ), + "constituents": ordered, + } + ) + + def publication_content_fingerprint_v1(manifest: SourceBatchManifestV1 | Mapping[str, Any]) -> str: """Fingerprint the source-neutral *content* of a publication. @@ -1764,6 +2105,16 @@ def publication_content_fingerprint_v1(manifest: SourceBatchManifestV1 | Mapping return reporting_fingerprint_v1(content) +def publication_content_fingerprint_v2(manifest: SourceBatchManifestV2 | Mapping[str, Any]) -> str: + """Bind V2 content and its version, including the pair-bound denominator.""" + return reporting_fingerprint_v1( + { + "kind": "reporting_source_content_v2", + "content_fingerprint": publication_content_fingerprint_v1(manifest), + } + ) + + def deterministic_source_publication_id_v1( *, source_execution_key: str, @@ -1809,6 +2160,19 @@ def encode_source_batch_manifest_v1(manifest: SourceBatchManifestV1) -> bytes: return encoded +def encode_source_batch_manifest_v2(manifest: SourceBatchManifestV2) -> bytes: + """Seal V2 manifest bytes with the retained-evidence size bound.""" + payload = manifest.model_dump(mode="json", exclude_none=True) + encoded = canonical_json_utf8_v1(payload) + if len(encoded) > SOURCE_BATCH_MANIFEST_MAX_BYTES_V1: + raise SourceBatchManifestError( + "SOURCE_MANIFEST_TOO_LARGE", + "the source manifest exceeds the 1 MiB retained-evidence limit; " + "large data belongs in staged objects", + ) + return encoded + + def source_batch_manifest_reference_v1( staged_commit_ref: str, manifest_bytes: bytes ) -> SourceBatchManifestReferenceV1: @@ -1874,6 +2238,45 @@ def parse_verified_source_batch_manifest_v1( return manifest +def parse_verified_source_batch_manifest_v2( + reference: SourceBatchManifestReferenceV1, raw_bytes: bytes +) -> SourceBatchManifestV2: + """Verify the reference, V2 schema, and exact canonical manifest bytes.""" + import hashlib + import json + + if ( + len(raw_bytes) > SOURCE_BATCH_MANIFEST_MAX_BYTES_V1 + or len(raw_bytes) != reference.byte_count + ): + raise SourceBatchManifestError( + "SOURCE_MANIFEST_TOO_LARGE", + "the staged manifest size does not match the bounded reference", + ) + if hashlib.sha256(raw_bytes).hexdigest() != reference.manifest_sha256: + raise SourceBatchManifestError( + "SOURCE_MANIFEST_CHECKSUM_MISMATCH", + "the staged manifest bytes do not match the declared checksum", + ) + try: + payload = json.loads(raw_bytes.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as error: + raise SourceBatchManifestError( + "SOURCE_MANIFEST_JSON_INVALID", "the staged manifest is not valid UTF-8 JSON" + ) from error + if not isinstance(payload, dict) or payload.get("manifest_version") != "2.0": + raise SourceBatchManifestError( + "SOURCE_MANIFEST_VERSION_MISMATCH", "the staged manifest must declare version 2.0" + ) + manifest = SourceBatchManifestV2.model_validate(payload) + if encode_source_batch_manifest_v2(manifest) != raw_bytes: + raise SourceBatchManifestError( + "SOURCE_MANIFEST_CANONICAL_ENCODING_MISMATCH", + "the staged manifest is not canonical_json_utf8_v1", + ) + return manifest + + # -------------------------------------------------------------------------- # Execution result # -------------------------------------------------------------------------- @@ -1888,6 +2291,12 @@ class ReportingSourceExecutionResponseV1(_Frozen): manifest: SourceBatchManifestReferenceV1 +class ReportingSourceExecutionResponseV2(ReportingSourceExecutionResponseV1): + """The V2 success arm with the request identity and manifest reference.""" + + contract_version: Literal["2.0"] # type: ignore[assignment] + + class ReportingSourceErrorV1(_Frozen): """The failure arm: a stable code, a retry classification, and a safe message. @@ -1965,7 +2374,7 @@ def _default_retry_class(code: ReportingSourceErrorCode) -> ReportingSourceRetry class ReportingSourceExecutorResult(_Frozen): """Exactly one of: a completed publication, or a typed failure.""" - response: ReportingSourceExecutionResponseV1 | None = None + response: ReportingSourceExecutionResponseV1 | ReportingSourceExecutionResponseV2 | None = None manifest_bytes: bytes | None = None error: ReportingSourceErrorV1 | None = None @@ -1988,14 +2397,20 @@ def ok(self) -> bool: def completed( cls, *, - request: ReportingSourceSliceRequestV1, + request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, manifest: SourceBatchManifestReferenceV1, manifest_bytes: bytes, ) -> ReportingSourceExecutorResult: - return cls( - response=completed_reporting_source_response_v1(request=request, manifest=manifest), - manifest_bytes=manifest_bytes, + response = ( + ReportingSourceExecutionResponseV2( + contract_version=REPORTING_SOURCE_CONTRACT_VERSION_V2, + identity=request.identity, + manifest=manifest, + ) + if isinstance(request, ReportingSourceSliceRequestV2) + else completed_reporting_source_response_v1(request=request, manifest=manifest) ) + return cls(response=response, manifest_bytes=manifest_bytes) @classmethod def failed(cls, error: ReportingSourceErrorV1) -> ReportingSourceExecutorResult: diff --git a/tests/test_reporting_source_contract_v2.py b/tests/test_reporting_source_contract_v2.py new file mode 100644 index 0000000000..2e3cc39108 --- /dev/null +++ b/tests/test_reporting_source_contract_v2.py @@ -0,0 +1,756 @@ +"""Versioned package source contracts keep each package bound to its media buy.""" + +from __future__ import annotations + +import hashlib + +import pytest +from pydantic import ValidationError + +from adcp.reporting.conformance import ( + ReportingSourceConformanceError, + validate_reporting_source_execution, +) +from adcp.reporting.fixtures import ( + redacted_capabilities, + redacted_completed_result, + redacted_snapshot_request, +) +from adcp.reporting.source import ( + PackageItemConstituentV1, + PackageItemConstituentV2, + ReportingConstituentCoverageV2, + ReportingSourceCapabilitiesV2, + ReportingSourceCoverageRequestV2, + ReportingSourceExecutionResponseV2, + ReportingSourceExecutorResult, + ReportingSourceSliceRequestV1, + ReportingSourceSliceRequestV2, + ReportingSourceStagedObjectReader, + SourceBatchCoverageV1, + SourceBatchCoverageV2, + SourceBatchManifestError, + SourceBatchManifestV2, + coverage_denominator_fingerprint_v1, + coverage_denominator_fingerprint_v2, + encode_source_batch_manifest_v2, + publication_content_fingerprint_v2, + reporting_source_capabilities_sha256_v1, + reporting_source_capabilities_sha256_v2, + source_batch_coverage_fingerprint_v2, + source_batch_manifest_reference_v1, + validate_reporting_source_rows_v2, + validate_reporting_source_staged_rows_v2, +) + + +def _package_request() -> ReportingSourceSliceRequestV2: + base = redacted_snapshot_request() + constituent = PackageItemConstituentV2( + constituent_id="package-constituent", + product_id="fixture-product", + media_buy_id="buy-1", + package_id="package-1", + ) + payload = base.model_dump(mode="json") + payload["contract_version"] = "2.0" + payload["coverage"] = ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1"], + constituents=[constituent], + denominator_fingerprint=coverage_denominator_fingerprint_v2([constituent]), + ).model_dump(mode="json") + return ReportingSourceSliceRequestV2.model_validate(payload) + + +def _v2_capabilities() -> ReportingSourceCapabilitiesV2: + payload = redacted_capabilities().model_dump(mode="json") + payload["contract_version"] = "2.0" + payload["supported_contract_versions"] = ["2.0"] + payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) + return ReportingSourceCapabilitiesV2.model_validate(payload) + + +def _v2_result( + request: ReportingSourceSliceRequestV2, +) -> tuple[ReportingSourceExecutorResult, SourceBatchManifestV2, ReportingSourceStagedObjectReader]: + base = redacted_snapshot_request() + _, manifest, reader = redacted_completed_result(base) + payload = manifest.model_dump(mode="json") + payload["manifest_version"] = "2.0" + payload["coverage"]["denominator_fingerprint"] = request.coverage.denominator_fingerprint + payload["coverage"]["media_buy_ids"] = list(request.coverage.media_buy_ids) + payload["coverage"]["constituents"][0]["constituent"] = request.coverage.constituents[ + 0 + ].model_dump(mode="json") + for cell in payload["metric_availability"]: + cell["constituent_id"] = request.coverage.constituents[0].constituent_id + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + versioned_manifest = SourceBatchManifestV2.model_validate(payload) + manifest_bytes = encode_source_batch_manifest_v2(versioned_manifest) + reference = source_batch_manifest_reference_v1("manifest-package-v2", manifest_bytes) + result = ReportingSourceExecutorResult.completed( + request=request, manifest=reference, manifest_bytes=manifest_bytes + ) + return result, versioned_manifest, reader + + +def _empty_package_request(media_buy_id: str) -> ReportingSourceSliceRequestV2: + payload = redacted_snapshot_request().model_dump(mode="json") + payload["contract_version"] = "2.0" + payload["coverage"] = { + "expected": "full", + "media_buy_ids": [media_buy_id], + "constituents": [], + "denominator_fingerprint": coverage_denominator_fingerprint_v2( + [], media_buy_ids=[media_buy_id] + ), + } + return ReportingSourceSliceRequestV2.model_validate(payload) + + +def test_v2_empty_pairs_keep_distinct_frozen_buy_selectors() -> None: + first = _empty_package_request("buy-1") + second = _empty_package_request("buy-2") + assert first.media_buy_ids == ("buy-1",) + assert second.media_buy_ids == ("buy-2",) + assert first.coverage.denominator_fingerprint != second.coverage.denominator_fingerprint + + +async def test_v2_complete_empty_report_admits_zero_rows_without_metric_cells() -> None: + request = _empty_package_request("buy-1") + base = redacted_snapshot_request() + _, manifest, reader = redacted_completed_result(base, page_bodies=[""], row_counts=[0]) + payload = manifest.model_dump(mode="json") + payload["manifest_version"] = "2.0" + payload["coverage"] = { + "media_buy_ids": ["buy-1"], + "denominator_fingerprint": request.coverage.denominator_fingerprint, + "status": "full", + "constituents": [], + } + payload["metric_availability"] = [] + payload["explicit_zero"] = False + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + complete = SourceBatchManifestV2.model_validate(payload) + manifest_bytes = encode_source_batch_manifest_v2(complete) + result = ReportingSourceExecutorResult.completed( + request=request, + manifest=source_batch_manifest_reference_v1("manifest-complete-empty", manifest_bytes), + manifest_bytes=manifest_bytes, + ) + assert ( + await validate_reporting_source_execution( + capabilities=_v2_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + == complete + ) + + +def test_v2_complete_empty_report_requires_current_source_watermark() -> None: + request = _empty_package_request("buy-1") + _, manifest, _ = redacted_completed_result( + redacted_snapshot_request(), page_bodies=[""], row_counts=[0] + ) + payload = manifest.model_dump(mode="json") + payload["manifest_version"] = "2.0" + payload["coverage"] = { + "media_buy_ids": ["buy-1"], + "denominator_fingerprint": request.coverage.denominator_fingerprint, + "status": "full", + "constituents": [], + } + payload["metric_availability"] = [] + payload["explicit_zero"] = False + payload["data_through"] = request.period.start.isoformat() + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + with pytest.raises(ValidationError, match="current source cutoff"): + SourceBatchManifestV2.model_validate(payload) + + +def test_v1_package_shape_and_request_bytes_remain_frozen() -> None: + request = redacted_snapshot_request() + assert request.contract_version == "1.0" + encoded = request.model_dump_json() + assert ReportingSourceSliceRequestV1.model_validate_json(encoded).model_dump_json() == encoded + assert hashlib.sha256(encoded.encode()).hexdigest() == ( + "53b59efb8632ac2e314da2d4144ed2295ef1c7e9a49cb1fc3f4d55f9da724585" + ) + with pytest.raises(ValidationError): + PackageItemConstituentV1( + constituent_id="item", + product_id="fixture-product", + package_id="package-1", + media_buy_id="buy-1", # type: ignore[call-arg] + ) + with pytest.raises(ValidationError): + ReportingSourceSliceRequestV1.model_validate( + {**request.model_dump(mode="json"), "contract_version": "2.0"} + ) + + +def test_v1_coverage_rejects_nested_v2_package_instance() -> None: + package = _package_request().coverage.constituents[0] + assert isinstance(package, PackageItemConstituentV2) + base = redacted_snapshot_request() + payload = base.coverage.model_dump(mode="json") + payload["constituents"] = [package] + payload["denominator_fingerprint"] = coverage_denominator_fingerprint_v1([package]) + with pytest.raises(ValidationError): + type(base.coverage).model_validate(payload) + + +def test_v1_manifest_coverage_rejects_completed_v2_constituent_evidence() -> None: + package = _package_request().coverage.constituents[0] + item = ReportingConstituentCoverageV2(constituent=package, status="present") + with pytest.raises(ValidationError): + SourceBatchCoverageV1( + denominator_fingerprint=coverage_denominator_fingerprint_v1([package]), + status="full", + constituents=[item], + ) + + +def test_v1_manifest_coverage_rejects_v2_batch_instance() -> None: + _, manifest, _ = _v2_result(_package_request()) + with pytest.raises(ValidationError): + SourceBatchCoverageV1.model_validate(manifest.coverage) + + +def test_v2_package_requires_both_parent_and_package() -> None: + package = _package_request().coverage.constituents[0] + for absent in ("media_buy_id", "package_id"): + payload = package.model_dump(mode="json") + payload.pop(absent) + with pytest.raises(ValidationError): + PackageItemConstituentV2.model_validate(payload) + + +def test_v2_denominator_fingerprint_binds_both_ids() -> None: + package = _package_request().coverage.constituents[0] + original = coverage_denominator_fingerprint_v2([package]) + assert original != coverage_denominator_fingerprint_v2( + [package.model_copy(update={"media_buy_id": "buy-2"})] + ) + assert original != coverage_denominator_fingerprint_v2( + [package.model_copy(update={"package_id": "package-2"})] + ) + assert _package_request().media_buy_ids == ("buy-1",) + assert _package_request().package_ids == ("package-1",) + + +def test_v2_selector_accepts_the_package_constituent_external_buy_id() -> None: + payload = _package_request().model_dump(mode="json") + package = ( + _package_request() + .coverage.constituents[0] + .model_copy(update={"media_buy_id": "buyer/buy-1"}) + ) + payload["coverage"] = { + "expected": "full", + "media_buy_ids": ["buyer/buy-1"], + "constituents": [package.model_dump(mode="json")], + "denominator_fingerprint": coverage_denominator_fingerprint_v2( + [package], media_buy_ids=["buyer/buy-1"] + ), + } + request = ReportingSourceSliceRequestV2.model_validate(payload) + _, manifest, _ = _v2_result(request) + assert request.media_buy_ids == manifest.coverage.media_buy_ids == ("buyer/buy-1",) + + +def test_v2_request_rejects_duplicate_pair_with_distinct_constituent_ids() -> None: + package = _package_request().coverage.constituents[0] + duplicate = package.model_copy(update={"constituent_id": "package-constituent-2"}) + with pytest.raises(ValidationError): + ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1"], + constituents=[package, duplicate], + denominator_fingerprint=coverage_denominator_fingerprint_v2([package, duplicate]), + ) + + +def test_v2_manifest_rejects_duplicate_pair_with_distinct_constituent_ids() -> None: + request = _package_request() + _, manifest, _ = _v2_result(request) + package = request.coverage.constituents[0] + duplicate = package.model_copy(update={"constituent_id": "package-constituent-2"}) + payload = manifest.model_dump(mode="json") + item = payload["coverage"]["constituents"][0].copy() + item["constituent"] = duplicate.model_dump(mode="json") + payload["coverage"]["constituents"].append(item) + payload["coverage"]["denominator_fingerprint"] = coverage_denominator_fingerprint_v2( + [package, duplicate] + ) + for cell in list(payload["metric_availability"]): + payload["metric_availability"].append({**cell, "constituent_id": duplicate.constituent_id}) + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + with pytest.raises(ValidationError): + SourceBatchManifestV2.model_validate(payload) + + +def test_v2_allows_one_package_id_under_two_buys() -> None: + package = _package_request().coverage.constituents[0] + shared_id = package.model_copy( + update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} + ) + denominator = coverage_denominator_fingerprint_v2([package, shared_id]) + request = ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1", "buy-2"], + constituents=[package, shared_id], + denominator_fingerprint=denominator, + ) + coverage = SourceBatchCoverageV2( + media_buy_ids=["buy-1", "buy-2"], + denominator_fingerprint=denominator, + status="full", + constituents=[ + ReportingConstituentCoverageV2(constituent=item, status="present") + for item in request.constituents + ], + ) + assert [(item.media_buy_id, item.package_id) for item in request.constituents] == [ + ("buy-1", "package-1"), + ("buy-2", "package-1"), + ] + assert coverage.status == "full" + + +def test_v2_rows_require_an_assigned_package_pair() -> None: + from adcp.reporting.source import validate_reporting_source_rows_v2 + + package = _package_request().coverage.constituents[0] + other_buy = package.model_copy( + update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} + ) + request = _package_request().model_copy( + update={ + "coverage": ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1", "buy-2"], + constituents=[package, other_buy], + denominator_fingerprint=coverage_denominator_fingerprint_v2([package, other_buy]), + ) + } + ) + _, base_manifest, _ = _v2_result(_package_request()) + manifest_payload = base_manifest.model_dump(mode="json") + manifest_payload["coverage"]["denominator_fingerprint"] = ( + request.coverage.denominator_fingerprint + ) + manifest_payload["coverage"]["media_buy_ids"] = list(request.coverage.media_buy_ids) + other_evidence = dict(manifest_payload["coverage"]["constituents"][0]) + other_evidence["constituent"] = other_buy.model_dump(mode="json") + manifest_payload["coverage"]["constituents"].append(other_evidence) + for cell in list(manifest_payload["metric_availability"]): + manifest_payload["metric_availability"].append( + {**cell, "constituent_id": other_buy.constituent_id} + ) + manifest_payload["content_fingerprint"] = publication_content_fingerprint_v2(manifest_payload) + manifest = SourceBatchManifestV2.model_validate(manifest_payload) + with pytest.raises(ValueError, match="manifest coverage differs"): + validate_reporting_source_rows_v2(_package_request(), manifest, []) + validate_reporting_source_rows_v2( + request, + manifest, + [ + {"media_buy_id": "buy-1", "package_id": "package-1", "impressions": 5}, + {"media_buy_id": "buy-2", "package_id": "package-1", "impressions": 0}, + ], + ) + for row in ( + {"media_buy_id": "buy-3", "package_id": "package-1"}, + {"package_id": "package-1"}, + {"media_buy_id": "buy-1", "package_id": "external-line-1"}, + { + "media_buy_id": "buy-1", + "package_id": "package-1", + "constituent_id": other_buy.constituent_id, + }, + {"line_id": "external-line-1"}, + ): + with pytest.raises(ValueError, match="outside the requested coverage"): + validate_reporting_source_rows_v2(request, manifest, [row]) + + +def test_v2_staged_rows_are_fenced_by_request_before_manifest_exists() -> None: + from adcp.reporting.source import validate_reporting_source_staged_rows_v2 + + request = _package_request() + validate_reporting_source_staged_rows_v2( + request, [{"media_buy_id": "buy-1", "package_id": "package-1"}] + ) + for rows in ( + [{"media_buy_id": "buy-1"}], + [{"media_buy_id": "buy-2", "package_id": "package-1"}], + [{"media_buy_id": "buy-1", "package_id": "package-2"}], + [{"media_buy_id": "buy-2", "package_id": "package-1", "impressions": 0}], + [{"media_buy_id": "buy-1", "package_id": "package-2", "impressions": 0}], + ): + with pytest.raises(ValueError, match="outside the requested coverage"): + validate_reporting_source_staged_rows_v2(request, rows) + + +@pytest.mark.parametrize("stage", ["staged", "final"]) +def test_v2_rows_reject_duplicate_package_and_requested_dimension_values(stage: str) -> None: + request = _package_request() + _, manifest, _ = _v2_result(request) + rows = [ + { + "media_buy_id": "buy-1", + "package_id": "package-1", + "campaign_id": "campaign-1", + "impressions": 5, + }, + { + "media_buy_id": "buy-1", + "package_id": "package-1", + "campaign_id": "campaign-1", + "impressions": 7, + }, + ] + with pytest.raises(ValueError, match="duplicate row"): + if stage == "staged": + validate_reporting_source_staged_rows_v2(request, rows) + else: + validate_reporting_source_rows_v2(request, manifest, rows) + + +@pytest.mark.parametrize("stage", ["staged", "final"]) +def test_v2_rows_preserve_distinct_requested_dimension_values(stage: str) -> None: + request = _package_request() + _, manifest, _ = _v2_result(request) + rows = [ + {"media_buy_id": "buy-1", "package_id": "package-1", "campaign_id": "campaign-1"}, + {"media_buy_id": "buy-1", "package_id": "package-1", "campaign_id": "campaign-2"}, + ] + if stage == "staged": + validate_reporting_source_staged_rows_v2(request, rows) + else: + validate_reporting_source_rows_v2(request, manifest, rows) + + +def test_v2_manifest_keeps_explicit_zero_on_one_package_pair() -> None: + package = _package_request().coverage.constituents[0] + other_buy = package.model_copy( + update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} + ) + _, manifest, _ = _v2_result(_package_request()) + payload = manifest.model_dump(mode="json") + payload["coverage"]["denominator_fingerprint"] = coverage_denominator_fingerprint_v2( + [package, other_buy] + ) + payload["coverage"]["media_buy_ids"] = ["buy-1", "buy-2"] + zero_evidence = dict(payload["coverage"]["constituents"][0]) + zero_evidence["constituent"] = other_buy.model_dump(mode="json") + zero_evidence["status"] = "explicit_zero" + payload["coverage"]["constituents"].append(zero_evidence) + for cell in list(payload["metric_availability"]): + payload["metric_availability"].append( + {**cell, "constituent_id": other_buy.constituent_id, "status": "explicit_zero"} + ) + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + versioned = SourceBatchManifestV2.model_validate(payload) + assert [item.status for item in versioned.coverage.constituents] == [ + "present", + "explicit_zero", + ] + from adcp.reporting.source import validate_reporting_source_rows_v2 + + request_payload = _package_request().model_dump(mode="json") + request_payload["coverage"] = ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1", "buy-2"], + constituents=[package, other_buy], + denominator_fingerprint=payload["coverage"]["denominator_fingerprint"], + ).model_dump(mode="json") + request = ReportingSourceSliceRequestV2.model_validate(request_payload) + with pytest.raises(ValueError, match="explicit zero"): + validate_reporting_source_rows_v2( + request, + versioned, + [{"media_buy_id": "buy-2", "package_id": "package-1"}], + ) + + +def test_v2_request_retains_metric_cell_limit() -> None: + package = _package_request().coverage.constituents[0] + other_buy = package.model_copy( + update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} + ) + payload = _package_request().model_dump(mode="json") + payload["coverage"] = ReportingSourceCoverageRequestV2( + media_buy_ids=["buy-1", "buy-2"], + constituents=[package, other_buy], + denominator_fingerprint=coverage_denominator_fingerprint_v2([package, other_buy]), + ).model_dump(mode="json") + payload["requested_metrics"] = [f"metric_{index}" for index in range(501)] + with pytest.raises(ValidationError, match="at most 1000 constituent-metric cells"): + ReportingSourceSliceRequestV2.model_validate(payload) + + +def test_v2_two_metric_package_limit_is_checked_at_request_validation() -> None: + first = _package_request().coverage.constituents[0] + + def request_with_packages(count: int) -> ReportingSourceSliceRequestV2: + packages = [ + first.model_copy( + update={"constituent_id": f"constituent-{index}", "package_id": f"package-{index}"} + ) + for index in range(count) + ] + payload = _package_request().model_dump(mode="json") + payload["coverage"] = { + "expected": "full", + "media_buy_ids": ["buy-1"], + "constituents": [item.model_dump(mode="json") for item in packages], + "denominator_fingerprint": coverage_denominator_fingerprint_v2( + packages, media_buy_ids=["buy-1"] + ), + } + return ReportingSourceSliceRequestV2.model_validate(payload) + + assert len(request_with_packages(500).coverage.constituents) == 500 + with pytest.raises(ValidationError, match="at most 1000 constituent-metric cells"): + request_with_packages(501) + + +def test_v2_capability_checksum_binds_declared_version() -> None: + payload = redacted_capabilities().model_dump(mode="json") + payload["contract_version"] = "2.0" + assert reporting_source_capabilities_sha256_v1(payload) != payload["capabilities_sha256"] + with pytest.raises(ValidationError): + ReportingSourceCapabilitiesV2.model_validate(payload) + + +def test_v2_contract_versions_are_explicit() -> None: + capabilities_payload = _v2_capabilities().model_dump(mode="json") + capabilities_payload.pop("contract_version") + with pytest.raises(ValidationError): + ReportingSourceCapabilitiesV2.model_validate(capabilities_payload) + capabilities_payload = _v2_capabilities().model_dump(mode="json") + capabilities_payload.pop("supported_contract_versions") + with pytest.raises(ValidationError): + ReportingSourceCapabilitiesV2.model_validate(capabilities_payload) + + request_payload = _package_request().model_dump(mode="json") + request_payload.pop("contract_version") + with pytest.raises(ValidationError): + ReportingSourceSliceRequestV2.model_validate(request_payload) + + result, manifest, _ = _v2_result(_package_request()) + manifest_payload = manifest.model_dump(mode="json") + manifest_payload.pop("manifest_version") + with pytest.raises(ValidationError): + SourceBatchManifestV2.model_validate(manifest_payload) + assert result.response is not None + response_payload = result.response.model_dump(mode="json") + response_payload.pop("contract_version") + with pytest.raises(ValidationError): + ReportingSourceExecutionResponseV2.model_validate(response_payload) + + +def test_supported_contract_versions_cannot_change_after_checksum_validation() -> None: + capabilities = _v2_capabilities() + checksum = capabilities.capabilities_sha256 + with pytest.raises(AttributeError): + capabilities.supported_contract_versions.append("1.0") + assert capabilities.capabilities_sha256 == checksum + assert capabilities.supported_contract_versions == ("2.0",) + + +@pytest.mark.parametrize("versions", [["1.0"], ["2.0", "2.0"]]) +def test_v2_capability_rejects_incoherent_supported_versions(versions: list[str]) -> None: + payload = _v2_capabilities().model_dump(mode="json") + payload["supported_contract_versions"] = versions + payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) + with pytest.raises(ValidationError): + ReportingSourceCapabilitiesV2.model_validate(payload) + + +def test_v2_coverage_fingerprint_binds_package_pair() -> None: + package = _package_request().coverage.constituents[0] + item = ReportingConstituentCoverageV2(constituent=package, status="present") + rebound = ReportingConstituentCoverageV2( + constituent=package.model_copy(update={"media_buy_id": "buy-2"}), status="present" + ) + assert source_batch_coverage_fingerprint_v2( + [item], media_buy_ids=["buy-1"] + ) != source_batch_coverage_fingerprint_v2([rebound], media_buy_ids=["buy-2"]) + + +def test_v2_coverage_fingerprint_binds_empty_buy_selector() -> None: + assert source_batch_coverage_fingerprint_v2( + [], media_buy_ids=["buy-1"] + ) != source_batch_coverage_fingerprint_v2([], media_buy_ids=["buy-2"]) + + +async def test_v2_execution_requires_explicit_capability_and_matching_manifest() -> None: + request = _package_request() + result, manifest, reader = _v2_result(request) + assert manifest.manifest_version == "2.0" + assert manifest.contract_version == "2.0" + with pytest.raises(ReportingSourceConformanceError) as error: + await validate_reporting_source_execution( + capabilities=redacted_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + assert error.value.code == "CAPABILITY_MISMATCH" + assert ( + await validate_reporting_source_execution( + capabilities=_v2_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + == manifest + ) + + +async def test_v2_empty_pair_request_requires_package_capability() -> None: + request = _empty_package_request("buy-1") + payload = _v2_capabilities().model_dump(mode="json") + for offering in payload["offerings"]: + offering["constituent_kinds"] = ["media_buy"] + payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) + capabilities = ReportingSourceCapabilitiesV2.model_validate(payload) + result, _, reader = _v2_result(_package_request()) + with pytest.raises(ReportingSourceConformanceError) as error: + await validate_reporting_source_execution( + capabilities=capabilities, + request=request, + result=result, + object_reader=reader, + ) + assert error.value.code == "CAPABILITY_MISMATCH" + + +async def test_one_executor_can_negotiate_v1_and_v2_requests() -> None: + payload = _v2_capabilities().model_dump(mode="json") + payload["supported_contract_versions"] = ["1.0", "2.0"] + payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) + capabilities = ReportingSourceCapabilitiesV2.model_validate(payload) + + v1_request = redacted_snapshot_request() + v1_result, v1_manifest, v1_reader = redacted_completed_result(v1_request) + assert ( + await validate_reporting_source_execution( + capabilities=capabilities, + request=v1_request, + result=v1_result, + object_reader=v1_reader, + ) + == v1_manifest + ) + + v2_request = _package_request() + v2_result, v2_manifest, v2_reader = _v2_result(v2_request) + assert ( + await validate_reporting_source_execution( + capabilities=capabilities, + request=v2_request, + result=v2_result, + object_reader=v2_reader, + ) + == v2_manifest + ) + + +async def test_v2_only_capability_rejects_v1_request() -> None: + request = redacted_snapshot_request() + result, _, reader = redacted_completed_result(request) + with pytest.raises(ReportingSourceConformanceError) as error: + await validate_reporting_source_execution( + capabilities=_v2_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + assert error.value.code == "CAPABILITY_MISMATCH" + + +async def test_mixed_request_manifest_versions_fail_admission() -> None: + request = _package_request() + _, v1_manifest, reader = redacted_completed_result(redacted_snapshot_request()) + payload = v1_manifest.model_dump(mode="json") + payload["coverage"]["denominator_fingerprint"] = request.coverage.denominator_fingerprint + payload["coverage"]["constituents"][0]["constituent"] = request.coverage.constituents[ + 0 + ].model_dump(mode="json") + for cell in payload["metric_availability"]: + cell["constituent_id"] = request.coverage.constituents[0].constituent_id + payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) + from adcp.reporting.canonical_json import canonical_json_utf8_v1 + + manifest_bytes = canonical_json_utf8_v1(payload) + result = ReportingSourceExecutorResult.completed( + request=request, + manifest=source_batch_manifest_reference_v1("manifest-mixed", manifest_bytes), + manifest_bytes=manifest_bytes, + ) + with pytest.raises(ReportingSourceConformanceError) as error: + await validate_reporting_source_execution( + capabilities=_v2_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + assert error.value.code == "MANIFEST_MISMATCH" + + +async def test_v1_request_rejects_v2_manifest() -> None: + request = redacted_snapshot_request() + versioned_request = _package_request() + versioned_result, _, reader = _v2_result(versioned_request) + assert versioned_result.response is not None + assert versioned_result.manifest_bytes is not None + result = ReportingSourceExecutorResult.completed( + request=request, + manifest=versioned_result.response.manifest, + manifest_bytes=versioned_result.manifest_bytes, + ) + with pytest.raises(ReportingSourceConformanceError) as error: + await validate_reporting_source_execution( + capabilities=redacted_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + assert error.value.code == "MANIFEST_MISMATCH" + + +def test_v2_manifest_rejects_package_without_parent_buy() -> None: + request = _package_request() + _, manifest, _ = _v2_result(request) + payload = manifest.model_dump(mode="json") + payload["coverage"]["constituents"][0]["constituent"].pop("media_buy_id") + with pytest.raises(ValidationError): + SourceBatchManifestV2.model_validate(payload) + + +def test_v2_execution_response_round_trips_without_downgrading() -> None: + result, _, _ = _v2_result(_package_request()) + round_trip = ReportingSourceExecutorResult.model_validate_json(result.model_dump_json()) + assert round_trip.response is not None + assert round_trip.response.contract_version == "2.0" + + +async def test_mixed_version_bytes_must_pass_reference_integrity_first() -> None: + result, _, reader = _v2_result(_package_request()) + assert result.response is not None + bad_reference = result.response.manifest.model_copy(update={"manifest_sha256": "0" * 64}) + mixed = ReportingSourceExecutorResult.completed( + request=redacted_snapshot_request(), + manifest=bad_reference, + manifest_bytes=result.manifest_bytes or b"", + ) + with pytest.raises(SourceBatchManifestError) as error: + await validate_reporting_source_execution( + capabilities=redacted_capabilities(), + request=redacted_snapshot_request(), + result=mixed, + object_reader=reader, + ) + assert error.value.code == "SOURCE_MANIFEST_CHECKSUM_MISMATCH" From 5e48d24f2bb645f3879ee509303503f57132083f Mon Sep 17 00:00:00 2001 From: rachitm022 Date: Fri, 20 Nov 2026 02:07:14 +0530 Subject: [PATCH 2/2] feat(reporting): support package coverage in V1 source contract --- docs/reporting-source-adapters.md | 16 +- src/adcp/reporting/conformance.py | 64 +- src/adcp/reporting/fixtures.py | 2 + src/adcp/reporting/source.py | 487 +++-------- tests/test_reporting_source_contract_v2.py | 756 ------------------ .../test_reporting_source_package_contract.py | 315 ++++++++ 6 files changed, 472 insertions(+), 1168 deletions(-) delete mode 100644 tests/test_reporting_source_contract_v2.py create mode 100644 tests/test_reporting_source_package_contract.py diff --git a/docs/reporting-source-adapters.md b/docs/reporting-source-adapters.md index 907a315525..16b32ecc68 100644 --- a/docs/reporting-source-adapters.md +++ b/docs/reporting-source-adapters.md @@ -5,6 +5,11 @@ sealed, replayable reporting publication. Return `InlineFetchResult` when metric availability differs within a constituent. Its `cell_availability` map is keyed by **request constituent ID, then metric name**. Constituent IDs are not necessarily media buy IDs; read them from `request.coverage.constituents`. +A package request uses `coverage.grain == "package_item"` and freezes its buy +selector in `coverage.media_buy_ids`. Each package constituent binds both +`media_buy_id` and `package_id`. An empty constituent list can represent a +complete package scan for the selected buys. For ordinary buy requests, +`request.media_buy_ids` derives the selected buys from the constituents. ```python from adcp.reporting.inline_source import InlineFetchResult, MetricEvidence @@ -117,11 +122,12 @@ Cell evidence cannot bypass that freshness gate. Rows can omit missing, unsupported, or delayed metrics; available measurements for other metrics are retained. A control total is a checksum over the staged rows -- a consumer recomputes it from the revision's rows -- so it covers every -staged row, including rows outside the requested coverage, which stay staged -with a warning. It is emitted when every staged row carries a valid finite -numeric value for the metric. Missing fields, nulls, booleans, and invalid -numbers prevent a total, and an omitted explicit-zero row value is not filled -in. Statuses the SDK derives on its own withdraw nothing: a result with no +staged row. Package requests reject rows outside the exact requested +`(media_buy_id, package_id)` pairs before staging; buy requests retain the +warning behavior for outside rows. A total is emitted when every staged row +carries a valid finite numeric value for the metric. Missing fields, nulls, +booleans, and invalid numbers prevent a total. An omitted explicit-zero row +value is not filled in. Statuses the SDK derives on its own withdraw nothing: a result with no `cell_availability` publishes exactly the totals it published before. ### The withdrawal invariant diff --git a/src/adcp/reporting/conformance.py b/src/adcp/reporting/conformance.py index c56e571ea0..f46823f435 100644 --- a/src/adcp/reporting/conformance.py +++ b/src/adcp/reporting/conformance.py @@ -45,22 +45,17 @@ AuthoritativeOfferingV1, ProvisionalSnapshotOfferingV1, ReportingSourceCapabilitiesV1, - ReportingSourceCapabilitiesV2, ReportingSourceErrorCode, ReportingSourceErrorV1, ReportingSourceExecutor, ReportingSourceExecutorResult, ReportingSourceSliceRequestV1, - ReportingSourceSliceRequestV2, ReportingSourceStagedObjectReader, - SourceBatchManifestError, SourceBatchManifestV1, - SourceBatchManifestV2, _validate_metric_applicability, deterministic_source_publication_id_v1, iso_duration_milliseconds_v1, parse_verified_source_batch_manifest_v1, - parse_verified_source_batch_manifest_v2, source_batch_object_set_sha256_v1, ) @@ -160,16 +155,11 @@ async def _with_deadline( def _validate_request_against_capabilities( - capabilities: ReportingSourceCapabilitiesV1 | ReportingSourceCapabilitiesV2, - request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, + capabilities: ReportingSourceCapabilitiesV1, + request: ReportingSourceSliceRequestV1, ) -> ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1: """Is this frozen slice an exact atomic offering of this executor build?""" - supported_versions = ( - capabilities.supported_contract_versions - if isinstance(capabilities, ReportingSourceCapabilitiesV2) - else (capabilities.contract_version,) - ) - if request.contract_version not in supported_versions: + if request.contract_version != capabilities.contract_version: raise _fail( "CAPABILITY_MISMATCH", "the executor must explicitly declare the request contract version", @@ -220,12 +210,12 @@ def _validate_request_against_capabilities( _validate_window_bounds(offering, request) if ( - isinstance(request, ReportingSourceSliceRequestV2) + request.coverage.grain == "package_item" and "package_item" not in offering.constituent_kinds ): raise _fail( "CAPABILITY_MISMATCH", - "a V2 package slice requires package_item support in the selected offering", + "a package slice requires package_item support in the selected offering", ) for constituent in request.coverage.constituents: @@ -319,8 +309,8 @@ def _source_calendar_elapsed_ms(request: ReportingSourceSliceRequestV1) -> int: def _validate_manifest_against_request( offering: ProvisionalSnapshotOfferingV1 | AuthoritativeOfferingV1, - request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, - manifest: SourceBatchManifestV1 | SourceBatchManifestV2, + request: ReportingSourceSliceRequestV1, + manifest: SourceBatchManifestV1, ) -> None: if manifest.manifest_version != request.contract_version: raise _fail("MANIFEST_MISMATCH", "the manifest version must match the request version") @@ -406,10 +396,10 @@ def _validate_manifest_against_request( request.coverage.denominator_fingerprint, ), ] - if isinstance(request, ReportingSourceSliceRequestV2): - assert isinstance(manifest, SourceBatchManifestV2) + if request.coverage.grain == "package_item": echoed.extend( [ + ("coverage.grain", manifest.coverage.grain, request.coverage.grain), ( "coverage.media_buy_ids", manifest.coverage.media_buy_ids, @@ -491,13 +481,13 @@ def _validate_manifest_against_request( async def validate_reporting_source_execution( *, - capabilities: ReportingSourceCapabilitiesV1 | ReportingSourceCapabilitiesV2, - request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, + capabilities: ReportingSourceCapabilitiesV1, + request: ReportingSourceSliceRequestV1, result: ReportingSourceExecutorResult, object_reader: ReportingSourceStagedObjectReader, cancel: asyncio.Event | None = None, clock: Callable[[], datetime] | None = None, -) -> SourceBatchManifestV1 | SourceBatchManifestV2: +) -> SourceBatchManifestV1: """Validate one execution end to end and return its verified manifest. Reads every staged object the manifest names and checks its bytes against @@ -524,24 +514,18 @@ async def validate_reporting_source_execution( "IDENTITY_MISMATCH", "the response must echo the complete frozen request identity" ) - manifest: SourceBatchManifestV1 | SourceBatchManifestV2 - if isinstance(request, ReportingSourceSliceRequestV2): - try: - manifest = parse_verified_source_batch_manifest_v2(response.manifest, manifest_bytes) - except SourceBatchManifestError as error: - if error.code == "SOURCE_MANIFEST_VERSION_MISMATCH": - raise _fail("MANIFEST_MISMATCH", "the manifest must declare version 2.0") from error - raise - else: - try: - manifest = parse_verified_source_batch_manifest_v1(response.manifest, manifest_bytes) - except ValidationError as error: - payload = json.loads(manifest_bytes) - if isinstance(payload, dict) and payload.get("manifest_version") == "2.0": - raise _fail( - "MANIFEST_MISMATCH", "the manifest version must match the request" - ) from error - raise + try: + manifest = parse_verified_source_batch_manifest_v1(response.manifest, manifest_bytes) + except ValidationError as error: + payload = json.loads(manifest_bytes) + if ( + isinstance(payload, dict) + and payload.get("manifest_version") != request.contract_version + ): + raise _fail( + "MANIFEST_MISMATCH", "the manifest version must match the request" + ) from error + raise _validate_manifest_against_request(offering, request, manifest) cancel = cancel or asyncio.Event() diff --git a/src/adcp/reporting/fixtures.py b/src/adcp/reporting/fixtures.py index ec2d74b138..657f10805e 100644 --- a/src/adcp/reporting/fixtures.py +++ b/src/adcp/reporting/fixtures.py @@ -387,6 +387,8 @@ def redacted_completed_result( coverage = SourceBatchCoverageV1( denominator_fingerprint=request.coverage.denominator_fingerprint, + grain=request.coverage.grain, + media_buy_ids=request.coverage.media_buy_ids, status="full" if available else "none", constituents=[ ReportingConstituentCoverageV1(constituent=item, status=status, **evidence) # type: ignore[arg-type] diff --git a/src/adcp/reporting/source.py b/src/adcp/reporting/source.py index 90d6265ad3..0c6b025940 100644 --- a/src/adcp/reporting/source.py +++ b/src/adcp/reporting/source.py @@ -88,22 +88,18 @@ __all__ = [ "REPORTING_EVIDENCE_REASON_MAX_LENGTH_V1", "REPORTING_SOURCE_CONTRACT_VERSION_V1", - "REPORTING_SOURCE_CONTRACT_VERSION_V2", "SOURCE_BATCH_MANIFEST_MAX_BYTES_V1", "SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1", "SOURCE_BATCH_MANIFEST_MAX_PROVIDER_CALLS_V1", "AuthoritativeOfferingV1", "MediaBuyConstituentV1", "PackageItemConstituentV1", - "PackageItemConstituentV2", "ProductConstituentV1", "ProvisionalSnapshotOfferingV1", "ReportingAdapterBuildIdentityV1", "ReportingAvailabilityStatus", "ReportingConstituent", - "ReportingConstituentV2", "ReportingConstituentCoverageV1", - "ReportingConstituentCoverageV2", "ReportingContractIdentityV1", "ReportingCoverageStatus", "ReportingFinalityEvidenceV1", @@ -115,13 +111,10 @@ "ReportingPublicationClass", "ReportingRevisionKind", "ReportingSourceCapabilitiesV1", - "ReportingSourceCapabilitiesV2", - "ReportingSourceCoverageRequestV2", "ReportingSourceError", "ReportingSourceErrorCode", "ReportingSourceErrorV1", "ReportingSourceExecutionResponseV1", - "ReportingSourceExecutionResponseV2", "ReportingSourceExecutor", "ReportingSourceExecutorResult", "ReportingSourceIdentityV1", @@ -129,37 +122,27 @@ "ReportingSourcePeriodV1", "ReportingSourceRetryClass", "ReportingSourceSliceRequestV1", - "ReportingSourceSliceRequestV2", "ReportingSourceStagedObjectReader", "ReportingSourceWindowingV1", "ReportingTriggerKind", "SourceBatchManifestReferenceV1", "SourceBatchManifestV1", - "SourceBatchManifestV2", - "SourceBatchCoverageV2", "SourceBatchObjectV1", "SourceControlTotalV1", "completed_reporting_source_response_v1", "coverage_denominator_fingerprint_v1", - "coverage_denominator_fingerprint_v2", "deterministic_source_publication_id_v1", "encode_source_batch_manifest_v1", - "encode_source_batch_manifest_v2", - "parse_verified_source_batch_manifest_v2", "publication_content_fingerprint_v1", - "publication_content_fingerprint_v2", "reporting_source_capabilities_sha256_v1", - "reporting_source_capabilities_sha256_v2", "source_batch_coverage_fingerprint_v1", - "source_batch_coverage_fingerprint_v2", "source_batch_manifest_reference_v1", "source_batch_object_set_sha256_v1", - "validate_reporting_source_rows_v2", - "validate_reporting_source_staged_rows_v2", + "validate_reporting_source_rows_v1", + "validate_reporting_source_staged_rows_v1", ] REPORTING_SOURCE_CONTRACT_VERSION_V1: Final[Literal["1.0"]] = "1.0" -REPORTING_SOURCE_CONTRACT_VERSION_V2: Final[Literal["2.0"]] = "2.0" #: A retained manifest is evidence, not a payload. Rows belong in staged #: objects; the manifest that binds them stays small enough to keep forever. @@ -526,16 +509,7 @@ class ProductConstituentV1(_Frozen): class PackageItemConstituentV1(_Frozen): - """A package-item constituent: a product inside a named package.""" - - constituent_kind: Literal["package_item"] = "package_item" - constituent_id: ExternalId - product_id: ExternalId - package_id: ExternalId - - -class PackageItemConstituentV2(_Frozen): - """A package constituent scoped to exactly one media buy.""" + """A product in one package of one media buy.""" constituent_kind: Literal["package_item"] = "package_item" constituent_id: ExternalId @@ -558,29 +532,27 @@ class MediaBuyConstituentV1(_Frozen): Field(discriminator="constituent_kind"), ] -ReportingConstituentV2 = Annotated[ - ProductConstituentV1 | PackageItemConstituentV2 | MediaBuyConstituentV1, - Field(discriminator="constituent_kind"), -] - -def _require_unique_package_pairs(constituents: Sequence[ReportingConstituentV2]) -> None: +def _require_unique_package_pairs(constituents: Sequence[ReportingConstituent]) -> None: pairs = [ (item.media_buy_id, item.package_id) for item in constituents - if isinstance(item, PackageItemConstituentV2) + if isinstance(item, PackageItemConstituentV1) ] if len(set(pairs)) != len(pairs): raise ValueError("a package pair may appear only once in the coverage denominator") -def coverage_denominator_fingerprint_v1(constituents: Sequence[Any]) -> str: +def coverage_denominator_fingerprint_v1( + constituents: Sequence[Any], + *, + grain: Literal["package_item"] | None = None, + media_buy_ids: Sequence[str] | None = None, +) -> str: """Fingerprint the *requested* constituent set, excluding publication status. - This is what makes "did I ask for the same thing?" answerable across a - request and its manifest without comparing two differently shaped - documents. Sorting by ``constituent_id`` means request order is not part - of identity; adding, removing, or re-binding a constituent changes it. + Package requests bind the grain and selected buys as well as constituents. + Sorting by ``constituent_id`` makes request order irrelevant. """ ordered = sorted( ( @@ -593,31 +565,16 @@ def coverage_denominator_fingerprint_v1(constituents: Sequence[Any]) -> str: ), key=lambda item: str(item["constituent_id"]).encode("utf-16-be", errors="surrogatepass"), ) - return reporting_fingerprint_v1( - {"kind": "reporting_coverage_denominator_v1", "constituents": ordered} - ) - - -def coverage_denominator_fingerprint_v2( - constituents: Sequence[Any], *, media_buy_ids: Sequence[str] | None = None -) -> str: - """Fingerprint the frozen buy selector and its exact package pair set.""" - if media_buy_ids is None: - media_buy_ids = sorted({item.media_buy_id for item in constituents}) - ordered = sorted( - ( - { - key: value - for key, value in _constituent_identity(constituent).items() - if value is not None - } - for constituent in constituents - ), - key=lambda item: str(item["constituent_id"]).encode("utf-16-be", errors="surrogatepass"), - ) + if grain is None and media_buy_ids is None: + return reporting_fingerprint_v1( + {"kind": "reporting_coverage_denominator_v1", "constituents": ordered} + ) + if grain != "package_item" or not media_buy_ids: + raise ValueError("package fingerprints require grain and nonempty media_buy_ids") return reporting_fingerprint_v1( { - "kind": "reporting_coverage_denominator_v2", + "kind": "reporting_coverage_denominator_v1", + "grain": grain, "media_buy_ids": sorted( media_buy_ids, key=lambda value: str(value).encode("utf-16-be", errors="surrogatepass"), @@ -873,22 +830,6 @@ def offering(self, offering_id: str) -> ProvisionalSnapshotOfferingV1 | Authorit raise KeyError(offering_id) -class ReportingSourceCapabilitiesV2(ReportingSourceCapabilitiesV1): - """An executor declaration with an explicit set of supported source versions.""" - - contract_version: Literal["2.0"] # type: ignore[assignment] - supported_contract_versions: tuple[Literal["1.0", "2.0"], ...] = Field( - min_length=1, max_length=2 - ) - - @model_validator(mode="after") - def _supported_versions_are_unique(self) -> ReportingSourceCapabilitiesV2: - versions = self.supported_contract_versions - if len(set(versions)) != len(versions) or "2.0" not in versions: - raise ValueError("supported_contract_versions must uniquely include version 2.0") - return self - - def reporting_source_capabilities_sha256_v1( capabilities: ReportingSourceCapabilitiesV1 | Mapping[str, Any], ) -> str: @@ -903,13 +844,6 @@ def reporting_source_capabilities_sha256_v1( return canonical_json_sha256_v1(payload) -def reporting_source_capabilities_sha256_v2( - capabilities: ReportingSourceCapabilitiesV2 | Mapping[str, Any], -) -> str: - """Bind the complete V2 capability declaration, including its version.""" - return reporting_source_capabilities_sha256_v1(capabilities) - - # -------------------------------------------------------------------------- # Slice request # -------------------------------------------------------------------------- @@ -925,8 +859,14 @@ class ReportingSourceCoverageRequestV1(_Frozen): """ expected: Literal["full", "partial"] = "full" + grain: Literal["package_item"] | None = Field( + default=None, exclude_if=lambda value: value is None + ) + media_buy_ids: tuple[ExternalId, ...] | None = Field( + default=None, max_length=100_000, exclude_if=lambda value: value is None + ) constituents: list[ReportingConstituent] = Field( - min_length=1, max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 + max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 ) denominator_fingerprint: Fingerprint @@ -935,7 +875,10 @@ def _denominator_binds_the_constituents(self) -> ReportingSourceCoverageRequestV ids = [constituent.constituent_id for constituent in self.constituents] if len(set(ids)) != len(ids): raise ValueError("constituent_id must be unique within a coverage denominator") - expected = coverage_denominator_fingerprint_v1(self.constituents) + _validate_coverage_scope(self.grain, self.media_buy_ids, self.constituents) + expected = coverage_denominator_fingerprint_v1( + self.constituents, grain=self.grain, media_buy_ids=self.media_buy_ids + ) if expected != self.denominator_fingerprint: raise ValueError( "denominator_fingerprint must bind the complete constituent tuple set; " @@ -944,33 +887,27 @@ def _denominator_binds_the_constituents(self) -> ReportingSourceCoverageRequestV return self -class ReportingSourceCoverageRequestV2(ReportingSourceCoverageRequestV1): - """The frozen buy selector and its requested package pairs.""" - - media_buy_ids: tuple[ExternalId, ...] = Field(min_length=1, max_length=100_000) - constituents: list[PackageItemConstituentV2] = Field( # type: ignore[assignment] - max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 - ) - - @model_validator(mode="after") - def _denominator_binds_the_constituents(self) -> ReportingSourceCoverageRequestV2: - ids = [constituent.constituent_id for constituent in self.constituents] - if len(set(ids)) != len(ids): - raise ValueError("constituent_id must be unique within a coverage denominator") - if len(set(self.media_buy_ids)) != len(self.media_buy_ids): - raise ValueError("media_buy_ids must be unique") - if any(item.media_buy_id not in self.media_buy_ids for item in self.constituents): - raise ValueError("package pairs must belong to the frozen media_buy_ids") - _require_unique_package_pairs(self.constituents) - expected = coverage_denominator_fingerprint_v2( - self.constituents, media_buy_ids=self.media_buy_ids - ) - if expected != self.denominator_fingerprint: - raise ValueError( - "denominator_fingerprint must bind the complete constituent tuple set; " - f"expected {expected}" - ) - return self +def _validate_coverage_scope( + grain: Literal["package_item"] | None, + media_buy_ids: Sequence[str] | None, + constituents: Sequence[ReportingConstituent], +) -> None: + if grain is None: + if media_buy_ids is not None or not constituents: + raise ValueError("nonpackage coverage requires constituents and no buy selector") + if any(item.constituent_kind == "package_item" for item in constituents): + raise ValueError("package coverage requires grain and media_buy_ids") + return + if not media_buy_ids: + raise ValueError("package coverage needs media_buy_ids") + if len(set(media_buy_ids)) != len(media_buy_ids): + raise ValueError("media_buy_ids must be unique") + if any(item.constituent_kind != grain for item in constituents): + raise ValueError("coverage constituents must match the declared grain") + selected = {item.media_buy_id for item in constituents if hasattr(item, "media_buy_id")} + if not selected.issubset(media_buy_ids): + raise ValueError("package pairs must belong to the frozen media_buy_ids") + _require_unique_package_pairs(constituents) class ReportingSourceCheckpointV1(_Frozen): @@ -1046,7 +983,9 @@ def product_ids(self) -> tuple[str, ...]: @property def media_buy_ids(self) -> tuple[str, ...]: - """The unique media buys represented by the constituent denominator.""" + """The frozen buy selector, including buys with no package pairs.""" + if self.coverage.media_buy_ids is not None: + return self.coverage.media_buy_ids return tuple( dict.fromkeys( constituent.media_buy_id @@ -1067,58 +1006,47 @@ def package_ids(self) -> tuple[str, ...]: ) -class ReportingSourceSliceRequestV2(ReportingSourceSliceRequestV1): - """A package-aware slice sent only to a V2-capable executor.""" - - contract_version: Literal["2.0"] # type: ignore[assignment] - coverage: ReportingSourceCoverageRequestV2 - - @property - def package_ids(self) -> tuple[str, ...]: - """The unique packages represented by the V2 denominator.""" - return tuple( - dict.fromkeys( - constituent.package_id - for constituent in self.coverage.constituents - if isinstance(constituent, PackageItemConstituentV2) - ) - ) - - @property - def media_buy_ids(self) -> tuple[str, ...]: - """The frozen media buys, including those with no package pairs.""" - return self.coverage.media_buy_ids - - -def validate_reporting_source_rows_v2( - request: ReportingSourceSliceRequestV2, - manifest: SourceBatchManifestV2, +def validate_reporting_source_rows_v1( + request: ReportingSourceSliceRequestV1, + manifest: SourceBatchManifestV1, rows: Sequence[Mapping[str, Any]], ) -> None: - """Reject normalized rows outside a frozen V2 coverage denominator before revision staging.""" - if manifest.coverage.denominator_fingerprint != request.coverage.denominator_fingerprint: + """Reject package rows outside frozen coverage before revision staging.""" + if request.coverage.grain != "package_item": + return + if ( + manifest.coverage.grain != request.coverage.grain + or manifest.coverage.media_buy_ids != request.coverage.media_buy_ids + or manifest.coverage.denominator_fingerprint != request.coverage.denominator_fingerprint + or [item.constituent for item in manifest.coverage.constituents] + != request.coverage.constituents + ): raise ValueError("manifest coverage differs from the requested coverage") - matched = _matched_reporting_source_rows_v2(request, rows) + matched = _matched_reporting_source_rows_v1(request, rows) statuses = {item.constituent_id: item.status for item in manifest.coverage.constituents} for index, constituent in enumerate(matched): if statuses.get(constituent.constituent_id) == "explicit_zero": raise ValueError(f"row {index} contradicts explicit zero coverage") -def validate_reporting_source_staged_rows_v2( - request: ReportingSourceSliceRequestV2, rows: Sequence[Mapping[str, Any]] +def validate_reporting_source_staged_rows_v1( + request: ReportingSourceSliceRequestV1, rows: Sequence[Mapping[str, Any]] ) -> None: """Fence normalized package rows against the frozen request before staging.""" - _matched_reporting_source_rows_v2(request, rows) + _matched_reporting_source_rows_v1(request, rows) -def _matched_reporting_source_rows_v2( - request: ReportingSourceSliceRequestV2, rows: Sequence[Mapping[str, Any]] -) -> list[PackageItemConstituentV2]: +def _matched_reporting_source_rows_v1( + request: ReportingSourceSliceRequestV1, rows: Sequence[Mapping[str, Any]] +) -> list[PackageItemConstituentV1]: + if request.coverage.grain != "package_item": + return [] packages = { - (item.media_buy_id, item.package_id): item for item in request.coverage.constituents + (item.media_buy_id, item.package_id): item + for item in request.coverage.constituents + if isinstance(item, PackageItemConstituentV1) } - matched: list[PackageItemConstituentV2] = [] + matched: list[PackageItemConstituentV1] = [] seen_grains: set[tuple[str, str, bytes]] = set() for index, row in enumerate(rows): package_id = row.get("package_id") @@ -1251,25 +1179,6 @@ def constituent_id(self) -> str: return self.constituent.constituent_id -class ReportingConstituentCoverageV2(_Frozen): - """Coverage evidence for a constituent with a complete package identity.""" - - constituent: PackageItemConstituentV2 - status: ReportingAvailabilityStatus - data_through: datetime | None = None - reason: EvidenceReason | None = None - - @model_validator(mode="after") - def _unavailable_coverage_is_explained(self) -> ReportingConstituentCoverageV2: - if self.status not in _AVAILABLE_STATUSES and not self.reason: - raise ValueError(f"constituent status {self.status!r} needs a stable reason") - return self - - @property - def constituent_id(self) -> str: - return self.constituent.constituent_id - - class ReportingMetricAvailabilityV1(_Frozen): """One ``(constituent, metric)`` cell's publication evidence. @@ -1313,57 +1222,23 @@ class SourceBatchCoverageV1(_Frozen): """The slice's coverage roll-up plus its per-constituent evidence.""" denominator_fingerprint: Fingerprint + grain: Literal["package_item"] | None = Field( + default=None, exclude_if=lambda value: value is None + ) + media_buy_ids: tuple[ExternalId, ...] | None = Field( + default=None, max_length=100_000, exclude_if=lambda value: value is None + ) status: ReportingCoverageStatus - constituents: list[ReportingConstituentCoverageV1] = Field(min_length=1, max_length=100_000) + constituents: list[ReportingConstituentCoverageV1] = Field(max_length=100_000) @model_validator(mode="after") def _status_matches_the_evidence(self) -> SourceBatchCoverageV1: ids = [item.constituent_id for item in self.constituents] if len(set(ids)) != len(ids): raise ValueError("constituent_id must be unique within manifest coverage") - covered = sum(1 for item in self.constituents if item.status in _AVAILABLE_STATUSES) - partial = sum(1 for item in self.constituents if item.status == "partial") - all_covered = covered == len(self.constituents) - any_evidence = covered > 0 or partial > 0 - if self.status == "full" and not all_covered: - raise ValueError("full coverage requires every constituent present or explicit zero") - if self.status == "none" and any_evidence: - raise ValueError("none coverage must not contain a covered or partial constituent") - if self.status == "partial" and (not any_evidence or all_covered): - raise ValueError( - "partial coverage requires coverage evidence without every constituent covered" - ) - expected = coverage_denominator_fingerprint_v1( - [item.constituent for item in self.constituents] + _validate_coverage_scope( + self.grain, self.media_buy_ids, [item.constituent for item in self.constituents] ) - if expected != self.denominator_fingerprint: - raise ValueError( - "denominator_fingerprint must bind the complete constituent tuple set; " - f"expected {expected}" - ) - return self - - -class SourceBatchCoverageV2(SourceBatchCoverageV1): - """Coverage evidence for the frozen buy selector and package pairs.""" - - media_buy_ids: tuple[ExternalId, ...] = Field(min_length=1, max_length=100_000) - constituents: list[ReportingConstituentCoverageV2] = Field( # type: ignore[assignment] - max_length=100_000 - ) - - @model_validator(mode="after") - def _status_matches_the_evidence(self) -> SourceBatchCoverageV2: - ids = [item.constituent_id for item in self.constituents] - if len(set(ids)) != len(ids): - raise ValueError("constituent_id must be unique within manifest coverage") - if len(set(self.media_buy_ids)) != len(self.media_buy_ids): - raise ValueError("media_buy_ids must be unique") - if any( - item.constituent.media_buy_id not in self.media_buy_ids for item in self.constituents - ): - raise ValueError("package pairs must belong to the frozen media_buy_ids") - _require_unique_package_pairs([item.constituent for item in self.constituents]) covered = sum(1 for item in self.constituents if item.status in _AVAILABLE_STATUSES) partial = sum(1 for item in self.constituents if item.status == "partial") all_covered = covered == len(self.constituents) @@ -1376,8 +1251,10 @@ def _status_matches_the_evidence(self) -> SourceBatchCoverageV2: raise ValueError( "partial coverage requires coverage evidence without every constituent covered" ) - expected = coverage_denominator_fingerprint_v2( - [item.constituent for item in self.constituents], media_buy_ids=self.media_buy_ids + expected = coverage_denominator_fingerprint_v1( + [item.constituent for item in self.constituents], + grain=self.grain, + media_buy_ids=self.media_buy_ids, ) if expected != self.denominator_fingerprint: raise ValueError( @@ -1646,7 +1523,7 @@ class SourceBatchManifestV1(_Frozen): byte_count: int = Field(ge=0) control_totals: list[SourceControlTotalV1] = Field(default_factory=list, max_length=1_000) metric_availability: list[ReportingMetricAvailabilityV1] = Field( - min_length=1, max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 + max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 ) coverage: SourceBatchCoverageV1 explicit_zero: bool = False @@ -1792,6 +1669,8 @@ def _validate_finality(self) -> None: # -- cells ----------------------------------------------------------- def _validate_cells(self) -> None: + if self.coverage.grain is None and not self.metric_availability: + raise ValueError("metric availability must carry requested cells") cells = [(cell.constituent_id, cell.metric) for cell in self.metric_availability] if len(set(cells)) != len(cells): raise ValueError( @@ -1829,7 +1708,7 @@ def _validate_cells(self) -> None: # -- zero rows ------------------------------------------------------- def _validate_zero_rows(self) -> None: - """A zero-row period has exactly two honest shapes, and they differ. + """A zero-row period distinguishes an answer from unavailable data. ``explicit_zero``: the source answered for every requested cell and the answer was zero. That is data, and it satisfies an obligation. @@ -1837,7 +1716,22 @@ def _validate_zero_rows(self) -> None: Unavailable: the source did not answer. That is not data, and it must not be dressed up as a zero -- otherwise a dead feed looks like a quiet campaign forever. + + A package selector with no requested pairs is complete without metric + cells or an explicit-zero claim when its source watermark is current. """ + if self.coverage.grain == "package_item" and not self.coverage.constituents: + if self.coverage.status != "full": + raise ValueError("empty package coverage must be full") + if self.row_count != 0 or self.metric_availability or self.explicit_zero: + raise ValueError("complete empty package coverage requires zero rows and cells") + if self.event_time_range is not None or any( + not total.is_zero for total in self.control_totals + ): + raise ValueError("complete empty package coverage cannot declare events or totals") + if self.data_through != min(self.period.end, self.period.source_read_cutoff_at): + raise ValueError("complete empty package coverage needs the current source cutoff") + return if self.explicit_zero: if self.event_time_range is not None: raise ValueError("an explicit-zero batch must not declare an event-time range") @@ -1922,50 +1816,6 @@ def _validate_strictness(self) -> None: ) -class SourceBatchManifestV2(SourceBatchManifestV1): - """A versioned publication that echoes the package and media-buy pairs.""" - - manifest_version: Literal["2.0"] # type: ignore[assignment] - coverage: SourceBatchCoverageV2 - metric_availability: list[ReportingMetricAvailabilityV1] = Field( - max_length=SOURCE_BATCH_MANIFEST_MAX_METRIC_AVAILABILITY_V1 - ) - - def _validate_zero_rows(self) -> None: - if not self.coverage.constituents and self.coverage.status == "full": - if self.row_count != 0 or self.metric_availability or self.explicit_zero: - raise ValueError("complete empty package coverage requires zero rows and cells") - if self.event_time_range is not None or any( - not total.is_zero for total in self.control_totals - ): - raise ValueError("complete empty package coverage cannot declare events or totals") - if self.data_through != min(self.period.end, self.period.source_read_cutoff_at): - raise ValueError("complete empty package coverage needs the current source cutoff") - return - super()._validate_zero_rows() - - @property - def contract_version(self) -> Literal["2.0"]: - """The negotiated source contract version.""" - return self.manifest_version - - @model_validator(mode="after") - def _manifest_is_internally_consistent(self) -> SourceBatchManifestV2: - self._validate_objects() - self._validate_time_evidence() - self._validate_finality() - self._validate_cells() - self._validate_zero_rows() - self._validate_event_range() - self._validate_strictness() - expected = publication_content_fingerprint_v2(self) - if expected != self.content_fingerprint: - raise ValueError( - f"content_fingerprint must bind the publication content; expected {expected}" - ) - return self - - # -------------------------------------------------------------------------- # Fingerprints and sealing # -------------------------------------------------------------------------- @@ -2015,30 +1865,6 @@ def source_batch_coverage_fingerprint_v1( ) -def source_batch_coverage_fingerprint_v2( - constituents: Sequence[ReportingConstituentCoverageV2], - *, - media_buy_ids: Sequence[str], -) -> str: - """Fingerprint package evidence and its frozen media-buy selector.""" - ordered = sorted( - (item.canonical_payload() for item in constituents), - key=lambda item: str(item["constituent"]["constituent_id"]).encode( - "utf-16-be", errors="surrogatepass" - ), - ) - return reporting_fingerprint_v1( - { - "kind": "reporting_coverage_evidence_v2", - "media_buy_ids": sorted( - media_buy_ids, - key=lambda value: str(value).encode("utf-16-be", errors="surrogatepass"), - ), - "constituents": ordered, - } - ) - - def publication_content_fingerprint_v1(manifest: SourceBatchManifestV1 | Mapping[str, Any]) -> str: """Fingerprint the source-neutral *content* of a publication. @@ -2098,6 +1924,9 @@ def publication_content_fingerprint_v1(manifest: SourceBatchManifestV1 | Mapping }, "explicit_zero": payload.get("explicit_zero", False), } + if payload["coverage"].get("grain") == "package_item": + content["coverage"]["grain"] = payload["coverage"]["grain"] + content["coverage"]["media_buy_ids"] = payload["coverage"]["media_buy_ids"] # Retained manifests without an owner retain their original fingerprint. # Adding a null key would invalidate those immutable publications. if payload["identity"].get("consumer_id") is not None: @@ -2105,16 +1934,6 @@ def publication_content_fingerprint_v1(manifest: SourceBatchManifestV1 | Mapping return reporting_fingerprint_v1(content) -def publication_content_fingerprint_v2(manifest: SourceBatchManifestV2 | Mapping[str, Any]) -> str: - """Bind V2 content and its version, including the pair-bound denominator.""" - return reporting_fingerprint_v1( - { - "kind": "reporting_source_content_v2", - "content_fingerprint": publication_content_fingerprint_v1(manifest), - } - ) - - def deterministic_source_publication_id_v1( *, source_execution_key: str, @@ -2160,19 +1979,6 @@ def encode_source_batch_manifest_v1(manifest: SourceBatchManifestV1) -> bytes: return encoded -def encode_source_batch_manifest_v2(manifest: SourceBatchManifestV2) -> bytes: - """Seal V2 manifest bytes with the retained-evidence size bound.""" - payload = manifest.model_dump(mode="json", exclude_none=True) - encoded = canonical_json_utf8_v1(payload) - if len(encoded) > SOURCE_BATCH_MANIFEST_MAX_BYTES_V1: - raise SourceBatchManifestError( - "SOURCE_MANIFEST_TOO_LARGE", - "the source manifest exceeds the 1 MiB retained-evidence limit; " - "large data belongs in staged objects", - ) - return encoded - - def source_batch_manifest_reference_v1( staged_commit_ref: str, manifest_bytes: bytes ) -> SourceBatchManifestReferenceV1: @@ -2238,45 +2044,6 @@ def parse_verified_source_batch_manifest_v1( return manifest -def parse_verified_source_batch_manifest_v2( - reference: SourceBatchManifestReferenceV1, raw_bytes: bytes -) -> SourceBatchManifestV2: - """Verify the reference, V2 schema, and exact canonical manifest bytes.""" - import hashlib - import json - - if ( - len(raw_bytes) > SOURCE_BATCH_MANIFEST_MAX_BYTES_V1 - or len(raw_bytes) != reference.byte_count - ): - raise SourceBatchManifestError( - "SOURCE_MANIFEST_TOO_LARGE", - "the staged manifest size does not match the bounded reference", - ) - if hashlib.sha256(raw_bytes).hexdigest() != reference.manifest_sha256: - raise SourceBatchManifestError( - "SOURCE_MANIFEST_CHECKSUM_MISMATCH", - "the staged manifest bytes do not match the declared checksum", - ) - try: - payload = json.loads(raw_bytes.decode("utf-8")) - except (UnicodeDecodeError, ValueError) as error: - raise SourceBatchManifestError( - "SOURCE_MANIFEST_JSON_INVALID", "the staged manifest is not valid UTF-8 JSON" - ) from error - if not isinstance(payload, dict) or payload.get("manifest_version") != "2.0": - raise SourceBatchManifestError( - "SOURCE_MANIFEST_VERSION_MISMATCH", "the staged manifest must declare version 2.0" - ) - manifest = SourceBatchManifestV2.model_validate(payload) - if encode_source_batch_manifest_v2(manifest) != raw_bytes: - raise SourceBatchManifestError( - "SOURCE_MANIFEST_CANONICAL_ENCODING_MISMATCH", - "the staged manifest is not canonical_json_utf8_v1", - ) - return manifest - - # -------------------------------------------------------------------------- # Execution result # -------------------------------------------------------------------------- @@ -2291,12 +2058,6 @@ class ReportingSourceExecutionResponseV1(_Frozen): manifest: SourceBatchManifestReferenceV1 -class ReportingSourceExecutionResponseV2(ReportingSourceExecutionResponseV1): - """The V2 success arm with the request identity and manifest reference.""" - - contract_version: Literal["2.0"] # type: ignore[assignment] - - class ReportingSourceErrorV1(_Frozen): """The failure arm: a stable code, a retry classification, and a safe message. @@ -2374,7 +2135,7 @@ def _default_retry_class(code: ReportingSourceErrorCode) -> ReportingSourceRetry class ReportingSourceExecutorResult(_Frozen): """Exactly one of: a completed publication, or a typed failure.""" - response: ReportingSourceExecutionResponseV1 | ReportingSourceExecutionResponseV2 | None = None + response: ReportingSourceExecutionResponseV1 | None = None manifest_bytes: bytes | None = None error: ReportingSourceErrorV1 | None = None @@ -2397,19 +2158,11 @@ def ok(self) -> bool: def completed( cls, *, - request: ReportingSourceSliceRequestV1 | ReportingSourceSliceRequestV2, + request: ReportingSourceSliceRequestV1, manifest: SourceBatchManifestReferenceV1, manifest_bytes: bytes, ) -> ReportingSourceExecutorResult: - response = ( - ReportingSourceExecutionResponseV2( - contract_version=REPORTING_SOURCE_CONTRACT_VERSION_V2, - identity=request.identity, - manifest=manifest, - ) - if isinstance(request, ReportingSourceSliceRequestV2) - else completed_reporting_source_response_v1(request=request, manifest=manifest) - ) + response = completed_reporting_source_response_v1(request=request, manifest=manifest) return cls(response=response, manifest_bytes=manifest_bytes) @classmethod diff --git a/tests/test_reporting_source_contract_v2.py b/tests/test_reporting_source_contract_v2.py deleted file mode 100644 index 2e3cc39108..0000000000 --- a/tests/test_reporting_source_contract_v2.py +++ /dev/null @@ -1,756 +0,0 @@ -"""Versioned package source contracts keep each package bound to its media buy.""" - -from __future__ import annotations - -import hashlib - -import pytest -from pydantic import ValidationError - -from adcp.reporting.conformance import ( - ReportingSourceConformanceError, - validate_reporting_source_execution, -) -from adcp.reporting.fixtures import ( - redacted_capabilities, - redacted_completed_result, - redacted_snapshot_request, -) -from adcp.reporting.source import ( - PackageItemConstituentV1, - PackageItemConstituentV2, - ReportingConstituentCoverageV2, - ReportingSourceCapabilitiesV2, - ReportingSourceCoverageRequestV2, - ReportingSourceExecutionResponseV2, - ReportingSourceExecutorResult, - ReportingSourceSliceRequestV1, - ReportingSourceSliceRequestV2, - ReportingSourceStagedObjectReader, - SourceBatchCoverageV1, - SourceBatchCoverageV2, - SourceBatchManifestError, - SourceBatchManifestV2, - coverage_denominator_fingerprint_v1, - coverage_denominator_fingerprint_v2, - encode_source_batch_manifest_v2, - publication_content_fingerprint_v2, - reporting_source_capabilities_sha256_v1, - reporting_source_capabilities_sha256_v2, - source_batch_coverage_fingerprint_v2, - source_batch_manifest_reference_v1, - validate_reporting_source_rows_v2, - validate_reporting_source_staged_rows_v2, -) - - -def _package_request() -> ReportingSourceSliceRequestV2: - base = redacted_snapshot_request() - constituent = PackageItemConstituentV2( - constituent_id="package-constituent", - product_id="fixture-product", - media_buy_id="buy-1", - package_id="package-1", - ) - payload = base.model_dump(mode="json") - payload["contract_version"] = "2.0" - payload["coverage"] = ReportingSourceCoverageRequestV2( - media_buy_ids=["buy-1"], - constituents=[constituent], - denominator_fingerprint=coverage_denominator_fingerprint_v2([constituent]), - ).model_dump(mode="json") - return ReportingSourceSliceRequestV2.model_validate(payload) - - -def _v2_capabilities() -> ReportingSourceCapabilitiesV2: - payload = redacted_capabilities().model_dump(mode="json") - payload["contract_version"] = "2.0" - payload["supported_contract_versions"] = ["2.0"] - payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) - return ReportingSourceCapabilitiesV2.model_validate(payload) - - -def _v2_result( - request: ReportingSourceSliceRequestV2, -) -> tuple[ReportingSourceExecutorResult, SourceBatchManifestV2, ReportingSourceStagedObjectReader]: - base = redacted_snapshot_request() - _, manifest, reader = redacted_completed_result(base) - payload = manifest.model_dump(mode="json") - payload["manifest_version"] = "2.0" - payload["coverage"]["denominator_fingerprint"] = request.coverage.denominator_fingerprint - payload["coverage"]["media_buy_ids"] = list(request.coverage.media_buy_ids) - payload["coverage"]["constituents"][0]["constituent"] = request.coverage.constituents[ - 0 - ].model_dump(mode="json") - for cell in payload["metric_availability"]: - cell["constituent_id"] = request.coverage.constituents[0].constituent_id - payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) - versioned_manifest = SourceBatchManifestV2.model_validate(payload) - manifest_bytes = encode_source_batch_manifest_v2(versioned_manifest) - reference = source_batch_manifest_reference_v1("manifest-package-v2", manifest_bytes) - result = ReportingSourceExecutorResult.completed( - request=request, manifest=reference, manifest_bytes=manifest_bytes - ) - return result, versioned_manifest, reader - - -def _empty_package_request(media_buy_id: str) -> ReportingSourceSliceRequestV2: - payload = redacted_snapshot_request().model_dump(mode="json") - payload["contract_version"] = "2.0" - payload["coverage"] = { - "expected": "full", - "media_buy_ids": [media_buy_id], - "constituents": [], - "denominator_fingerprint": coverage_denominator_fingerprint_v2( - [], media_buy_ids=[media_buy_id] - ), - } - return ReportingSourceSliceRequestV2.model_validate(payload) - - -def test_v2_empty_pairs_keep_distinct_frozen_buy_selectors() -> None: - first = _empty_package_request("buy-1") - second = _empty_package_request("buy-2") - assert first.media_buy_ids == ("buy-1",) - assert second.media_buy_ids == ("buy-2",) - assert first.coverage.denominator_fingerprint != second.coverage.denominator_fingerprint - - -async def test_v2_complete_empty_report_admits_zero_rows_without_metric_cells() -> None: - request = _empty_package_request("buy-1") - base = redacted_snapshot_request() - _, manifest, reader = redacted_completed_result(base, page_bodies=[""], row_counts=[0]) - payload = manifest.model_dump(mode="json") - payload["manifest_version"] = "2.0" - payload["coverage"] = { - "media_buy_ids": ["buy-1"], - "denominator_fingerprint": request.coverage.denominator_fingerprint, - "status": "full", - "constituents": [], - } - payload["metric_availability"] = [] - payload["explicit_zero"] = False - payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) - complete = SourceBatchManifestV2.model_validate(payload) - manifest_bytes = encode_source_batch_manifest_v2(complete) - result = ReportingSourceExecutorResult.completed( - request=request, - manifest=source_batch_manifest_reference_v1("manifest-complete-empty", manifest_bytes), - manifest_bytes=manifest_bytes, - ) - assert ( - await validate_reporting_source_execution( - capabilities=_v2_capabilities(), - request=request, - result=result, - object_reader=reader, - ) - == complete - ) - - -def test_v2_complete_empty_report_requires_current_source_watermark() -> None: - request = _empty_package_request("buy-1") - _, manifest, _ = redacted_completed_result( - redacted_snapshot_request(), page_bodies=[""], row_counts=[0] - ) - payload = manifest.model_dump(mode="json") - payload["manifest_version"] = "2.0" - payload["coverage"] = { - "media_buy_ids": ["buy-1"], - "denominator_fingerprint": request.coverage.denominator_fingerprint, - "status": "full", - "constituents": [], - } - payload["metric_availability"] = [] - payload["explicit_zero"] = False - payload["data_through"] = request.period.start.isoformat() - payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) - with pytest.raises(ValidationError, match="current source cutoff"): - SourceBatchManifestV2.model_validate(payload) - - -def test_v1_package_shape_and_request_bytes_remain_frozen() -> None: - request = redacted_snapshot_request() - assert request.contract_version == "1.0" - encoded = request.model_dump_json() - assert ReportingSourceSliceRequestV1.model_validate_json(encoded).model_dump_json() == encoded - assert hashlib.sha256(encoded.encode()).hexdigest() == ( - "53b59efb8632ac2e314da2d4144ed2295ef1c7e9a49cb1fc3f4d55f9da724585" - ) - with pytest.raises(ValidationError): - PackageItemConstituentV1( - constituent_id="item", - product_id="fixture-product", - package_id="package-1", - media_buy_id="buy-1", # type: ignore[call-arg] - ) - with pytest.raises(ValidationError): - ReportingSourceSliceRequestV1.model_validate( - {**request.model_dump(mode="json"), "contract_version": "2.0"} - ) - - -def test_v1_coverage_rejects_nested_v2_package_instance() -> None: - package = _package_request().coverage.constituents[0] - assert isinstance(package, PackageItemConstituentV2) - base = redacted_snapshot_request() - payload = base.coverage.model_dump(mode="json") - payload["constituents"] = [package] - payload["denominator_fingerprint"] = coverage_denominator_fingerprint_v1([package]) - with pytest.raises(ValidationError): - type(base.coverage).model_validate(payload) - - -def test_v1_manifest_coverage_rejects_completed_v2_constituent_evidence() -> None: - package = _package_request().coverage.constituents[0] - item = ReportingConstituentCoverageV2(constituent=package, status="present") - with pytest.raises(ValidationError): - SourceBatchCoverageV1( - denominator_fingerprint=coverage_denominator_fingerprint_v1([package]), - status="full", - constituents=[item], - ) - - -def test_v1_manifest_coverage_rejects_v2_batch_instance() -> None: - _, manifest, _ = _v2_result(_package_request()) - with pytest.raises(ValidationError): - SourceBatchCoverageV1.model_validate(manifest.coverage) - - -def test_v2_package_requires_both_parent_and_package() -> None: - package = _package_request().coverage.constituents[0] - for absent in ("media_buy_id", "package_id"): - payload = package.model_dump(mode="json") - payload.pop(absent) - with pytest.raises(ValidationError): - PackageItemConstituentV2.model_validate(payload) - - -def test_v2_denominator_fingerprint_binds_both_ids() -> None: - package = _package_request().coverage.constituents[0] - original = coverage_denominator_fingerprint_v2([package]) - assert original != coverage_denominator_fingerprint_v2( - [package.model_copy(update={"media_buy_id": "buy-2"})] - ) - assert original != coverage_denominator_fingerprint_v2( - [package.model_copy(update={"package_id": "package-2"})] - ) - assert _package_request().media_buy_ids == ("buy-1",) - assert _package_request().package_ids == ("package-1",) - - -def test_v2_selector_accepts_the_package_constituent_external_buy_id() -> None: - payload = _package_request().model_dump(mode="json") - package = ( - _package_request() - .coverage.constituents[0] - .model_copy(update={"media_buy_id": "buyer/buy-1"}) - ) - payload["coverage"] = { - "expected": "full", - "media_buy_ids": ["buyer/buy-1"], - "constituents": [package.model_dump(mode="json")], - "denominator_fingerprint": coverage_denominator_fingerprint_v2( - [package], media_buy_ids=["buyer/buy-1"] - ), - } - request = ReportingSourceSliceRequestV2.model_validate(payload) - _, manifest, _ = _v2_result(request) - assert request.media_buy_ids == manifest.coverage.media_buy_ids == ("buyer/buy-1",) - - -def test_v2_request_rejects_duplicate_pair_with_distinct_constituent_ids() -> None: - package = _package_request().coverage.constituents[0] - duplicate = package.model_copy(update={"constituent_id": "package-constituent-2"}) - with pytest.raises(ValidationError): - ReportingSourceCoverageRequestV2( - media_buy_ids=["buy-1"], - constituents=[package, duplicate], - denominator_fingerprint=coverage_denominator_fingerprint_v2([package, duplicate]), - ) - - -def test_v2_manifest_rejects_duplicate_pair_with_distinct_constituent_ids() -> None: - request = _package_request() - _, manifest, _ = _v2_result(request) - package = request.coverage.constituents[0] - duplicate = package.model_copy(update={"constituent_id": "package-constituent-2"}) - payload = manifest.model_dump(mode="json") - item = payload["coverage"]["constituents"][0].copy() - item["constituent"] = duplicate.model_dump(mode="json") - payload["coverage"]["constituents"].append(item) - payload["coverage"]["denominator_fingerprint"] = coverage_denominator_fingerprint_v2( - [package, duplicate] - ) - for cell in list(payload["metric_availability"]): - payload["metric_availability"].append({**cell, "constituent_id": duplicate.constituent_id}) - payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) - with pytest.raises(ValidationError): - SourceBatchManifestV2.model_validate(payload) - - -def test_v2_allows_one_package_id_under_two_buys() -> None: - package = _package_request().coverage.constituents[0] - shared_id = package.model_copy( - update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} - ) - denominator = coverage_denominator_fingerprint_v2([package, shared_id]) - request = ReportingSourceCoverageRequestV2( - media_buy_ids=["buy-1", "buy-2"], - constituents=[package, shared_id], - denominator_fingerprint=denominator, - ) - coverage = SourceBatchCoverageV2( - media_buy_ids=["buy-1", "buy-2"], - denominator_fingerprint=denominator, - status="full", - constituents=[ - ReportingConstituentCoverageV2(constituent=item, status="present") - for item in request.constituents - ], - ) - assert [(item.media_buy_id, item.package_id) for item in request.constituents] == [ - ("buy-1", "package-1"), - ("buy-2", "package-1"), - ] - assert coverage.status == "full" - - -def test_v2_rows_require_an_assigned_package_pair() -> None: - from adcp.reporting.source import validate_reporting_source_rows_v2 - - package = _package_request().coverage.constituents[0] - other_buy = package.model_copy( - update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} - ) - request = _package_request().model_copy( - update={ - "coverage": ReportingSourceCoverageRequestV2( - media_buy_ids=["buy-1", "buy-2"], - constituents=[package, other_buy], - denominator_fingerprint=coverage_denominator_fingerprint_v2([package, other_buy]), - ) - } - ) - _, base_manifest, _ = _v2_result(_package_request()) - manifest_payload = base_manifest.model_dump(mode="json") - manifest_payload["coverage"]["denominator_fingerprint"] = ( - request.coverage.denominator_fingerprint - ) - manifest_payload["coverage"]["media_buy_ids"] = list(request.coverage.media_buy_ids) - other_evidence = dict(manifest_payload["coverage"]["constituents"][0]) - other_evidence["constituent"] = other_buy.model_dump(mode="json") - manifest_payload["coverage"]["constituents"].append(other_evidence) - for cell in list(manifest_payload["metric_availability"]): - manifest_payload["metric_availability"].append( - {**cell, "constituent_id": other_buy.constituent_id} - ) - manifest_payload["content_fingerprint"] = publication_content_fingerprint_v2(manifest_payload) - manifest = SourceBatchManifestV2.model_validate(manifest_payload) - with pytest.raises(ValueError, match="manifest coverage differs"): - validate_reporting_source_rows_v2(_package_request(), manifest, []) - validate_reporting_source_rows_v2( - request, - manifest, - [ - {"media_buy_id": "buy-1", "package_id": "package-1", "impressions": 5}, - {"media_buy_id": "buy-2", "package_id": "package-1", "impressions": 0}, - ], - ) - for row in ( - {"media_buy_id": "buy-3", "package_id": "package-1"}, - {"package_id": "package-1"}, - {"media_buy_id": "buy-1", "package_id": "external-line-1"}, - { - "media_buy_id": "buy-1", - "package_id": "package-1", - "constituent_id": other_buy.constituent_id, - }, - {"line_id": "external-line-1"}, - ): - with pytest.raises(ValueError, match="outside the requested coverage"): - validate_reporting_source_rows_v2(request, manifest, [row]) - - -def test_v2_staged_rows_are_fenced_by_request_before_manifest_exists() -> None: - from adcp.reporting.source import validate_reporting_source_staged_rows_v2 - - request = _package_request() - validate_reporting_source_staged_rows_v2( - request, [{"media_buy_id": "buy-1", "package_id": "package-1"}] - ) - for rows in ( - [{"media_buy_id": "buy-1"}], - [{"media_buy_id": "buy-2", "package_id": "package-1"}], - [{"media_buy_id": "buy-1", "package_id": "package-2"}], - [{"media_buy_id": "buy-2", "package_id": "package-1", "impressions": 0}], - [{"media_buy_id": "buy-1", "package_id": "package-2", "impressions": 0}], - ): - with pytest.raises(ValueError, match="outside the requested coverage"): - validate_reporting_source_staged_rows_v2(request, rows) - - -@pytest.mark.parametrize("stage", ["staged", "final"]) -def test_v2_rows_reject_duplicate_package_and_requested_dimension_values(stage: str) -> None: - request = _package_request() - _, manifest, _ = _v2_result(request) - rows = [ - { - "media_buy_id": "buy-1", - "package_id": "package-1", - "campaign_id": "campaign-1", - "impressions": 5, - }, - { - "media_buy_id": "buy-1", - "package_id": "package-1", - "campaign_id": "campaign-1", - "impressions": 7, - }, - ] - with pytest.raises(ValueError, match="duplicate row"): - if stage == "staged": - validate_reporting_source_staged_rows_v2(request, rows) - else: - validate_reporting_source_rows_v2(request, manifest, rows) - - -@pytest.mark.parametrize("stage", ["staged", "final"]) -def test_v2_rows_preserve_distinct_requested_dimension_values(stage: str) -> None: - request = _package_request() - _, manifest, _ = _v2_result(request) - rows = [ - {"media_buy_id": "buy-1", "package_id": "package-1", "campaign_id": "campaign-1"}, - {"media_buy_id": "buy-1", "package_id": "package-1", "campaign_id": "campaign-2"}, - ] - if stage == "staged": - validate_reporting_source_staged_rows_v2(request, rows) - else: - validate_reporting_source_rows_v2(request, manifest, rows) - - -def test_v2_manifest_keeps_explicit_zero_on_one_package_pair() -> None: - package = _package_request().coverage.constituents[0] - other_buy = package.model_copy( - update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} - ) - _, manifest, _ = _v2_result(_package_request()) - payload = manifest.model_dump(mode="json") - payload["coverage"]["denominator_fingerprint"] = coverage_denominator_fingerprint_v2( - [package, other_buy] - ) - payload["coverage"]["media_buy_ids"] = ["buy-1", "buy-2"] - zero_evidence = dict(payload["coverage"]["constituents"][0]) - zero_evidence["constituent"] = other_buy.model_dump(mode="json") - zero_evidence["status"] = "explicit_zero" - payload["coverage"]["constituents"].append(zero_evidence) - for cell in list(payload["metric_availability"]): - payload["metric_availability"].append( - {**cell, "constituent_id": other_buy.constituent_id, "status": "explicit_zero"} - ) - payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) - versioned = SourceBatchManifestV2.model_validate(payload) - assert [item.status for item in versioned.coverage.constituents] == [ - "present", - "explicit_zero", - ] - from adcp.reporting.source import validate_reporting_source_rows_v2 - - request_payload = _package_request().model_dump(mode="json") - request_payload["coverage"] = ReportingSourceCoverageRequestV2( - media_buy_ids=["buy-1", "buy-2"], - constituents=[package, other_buy], - denominator_fingerprint=payload["coverage"]["denominator_fingerprint"], - ).model_dump(mode="json") - request = ReportingSourceSliceRequestV2.model_validate(request_payload) - with pytest.raises(ValueError, match="explicit zero"): - validate_reporting_source_rows_v2( - request, - versioned, - [{"media_buy_id": "buy-2", "package_id": "package-1"}], - ) - - -def test_v2_request_retains_metric_cell_limit() -> None: - package = _package_request().coverage.constituents[0] - other_buy = package.model_copy( - update={"constituent_id": "package-constituent-2", "media_buy_id": "buy-2"} - ) - payload = _package_request().model_dump(mode="json") - payload["coverage"] = ReportingSourceCoverageRequestV2( - media_buy_ids=["buy-1", "buy-2"], - constituents=[package, other_buy], - denominator_fingerprint=coverage_denominator_fingerprint_v2([package, other_buy]), - ).model_dump(mode="json") - payload["requested_metrics"] = [f"metric_{index}" for index in range(501)] - with pytest.raises(ValidationError, match="at most 1000 constituent-metric cells"): - ReportingSourceSliceRequestV2.model_validate(payload) - - -def test_v2_two_metric_package_limit_is_checked_at_request_validation() -> None: - first = _package_request().coverage.constituents[0] - - def request_with_packages(count: int) -> ReportingSourceSliceRequestV2: - packages = [ - first.model_copy( - update={"constituent_id": f"constituent-{index}", "package_id": f"package-{index}"} - ) - for index in range(count) - ] - payload = _package_request().model_dump(mode="json") - payload["coverage"] = { - "expected": "full", - "media_buy_ids": ["buy-1"], - "constituents": [item.model_dump(mode="json") for item in packages], - "denominator_fingerprint": coverage_denominator_fingerprint_v2( - packages, media_buy_ids=["buy-1"] - ), - } - return ReportingSourceSliceRequestV2.model_validate(payload) - - assert len(request_with_packages(500).coverage.constituents) == 500 - with pytest.raises(ValidationError, match="at most 1000 constituent-metric cells"): - request_with_packages(501) - - -def test_v2_capability_checksum_binds_declared_version() -> None: - payload = redacted_capabilities().model_dump(mode="json") - payload["contract_version"] = "2.0" - assert reporting_source_capabilities_sha256_v1(payload) != payload["capabilities_sha256"] - with pytest.raises(ValidationError): - ReportingSourceCapabilitiesV2.model_validate(payload) - - -def test_v2_contract_versions_are_explicit() -> None: - capabilities_payload = _v2_capabilities().model_dump(mode="json") - capabilities_payload.pop("contract_version") - with pytest.raises(ValidationError): - ReportingSourceCapabilitiesV2.model_validate(capabilities_payload) - capabilities_payload = _v2_capabilities().model_dump(mode="json") - capabilities_payload.pop("supported_contract_versions") - with pytest.raises(ValidationError): - ReportingSourceCapabilitiesV2.model_validate(capabilities_payload) - - request_payload = _package_request().model_dump(mode="json") - request_payload.pop("contract_version") - with pytest.raises(ValidationError): - ReportingSourceSliceRequestV2.model_validate(request_payload) - - result, manifest, _ = _v2_result(_package_request()) - manifest_payload = manifest.model_dump(mode="json") - manifest_payload.pop("manifest_version") - with pytest.raises(ValidationError): - SourceBatchManifestV2.model_validate(manifest_payload) - assert result.response is not None - response_payload = result.response.model_dump(mode="json") - response_payload.pop("contract_version") - with pytest.raises(ValidationError): - ReportingSourceExecutionResponseV2.model_validate(response_payload) - - -def test_supported_contract_versions_cannot_change_after_checksum_validation() -> None: - capabilities = _v2_capabilities() - checksum = capabilities.capabilities_sha256 - with pytest.raises(AttributeError): - capabilities.supported_contract_versions.append("1.0") - assert capabilities.capabilities_sha256 == checksum - assert capabilities.supported_contract_versions == ("2.0",) - - -@pytest.mark.parametrize("versions", [["1.0"], ["2.0", "2.0"]]) -def test_v2_capability_rejects_incoherent_supported_versions(versions: list[str]) -> None: - payload = _v2_capabilities().model_dump(mode="json") - payload["supported_contract_versions"] = versions - payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) - with pytest.raises(ValidationError): - ReportingSourceCapabilitiesV2.model_validate(payload) - - -def test_v2_coverage_fingerprint_binds_package_pair() -> None: - package = _package_request().coverage.constituents[0] - item = ReportingConstituentCoverageV2(constituent=package, status="present") - rebound = ReportingConstituentCoverageV2( - constituent=package.model_copy(update={"media_buy_id": "buy-2"}), status="present" - ) - assert source_batch_coverage_fingerprint_v2( - [item], media_buy_ids=["buy-1"] - ) != source_batch_coverage_fingerprint_v2([rebound], media_buy_ids=["buy-2"]) - - -def test_v2_coverage_fingerprint_binds_empty_buy_selector() -> None: - assert source_batch_coverage_fingerprint_v2( - [], media_buy_ids=["buy-1"] - ) != source_batch_coverage_fingerprint_v2([], media_buy_ids=["buy-2"]) - - -async def test_v2_execution_requires_explicit_capability_and_matching_manifest() -> None: - request = _package_request() - result, manifest, reader = _v2_result(request) - assert manifest.manifest_version == "2.0" - assert manifest.contract_version == "2.0" - with pytest.raises(ReportingSourceConformanceError) as error: - await validate_reporting_source_execution( - capabilities=redacted_capabilities(), - request=request, - result=result, - object_reader=reader, - ) - assert error.value.code == "CAPABILITY_MISMATCH" - assert ( - await validate_reporting_source_execution( - capabilities=_v2_capabilities(), - request=request, - result=result, - object_reader=reader, - ) - == manifest - ) - - -async def test_v2_empty_pair_request_requires_package_capability() -> None: - request = _empty_package_request("buy-1") - payload = _v2_capabilities().model_dump(mode="json") - for offering in payload["offerings"]: - offering["constituent_kinds"] = ["media_buy"] - payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) - capabilities = ReportingSourceCapabilitiesV2.model_validate(payload) - result, _, reader = _v2_result(_package_request()) - with pytest.raises(ReportingSourceConformanceError) as error: - await validate_reporting_source_execution( - capabilities=capabilities, - request=request, - result=result, - object_reader=reader, - ) - assert error.value.code == "CAPABILITY_MISMATCH" - - -async def test_one_executor_can_negotiate_v1_and_v2_requests() -> None: - payload = _v2_capabilities().model_dump(mode="json") - payload["supported_contract_versions"] = ["1.0", "2.0"] - payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v2(payload) - capabilities = ReportingSourceCapabilitiesV2.model_validate(payload) - - v1_request = redacted_snapshot_request() - v1_result, v1_manifest, v1_reader = redacted_completed_result(v1_request) - assert ( - await validate_reporting_source_execution( - capabilities=capabilities, - request=v1_request, - result=v1_result, - object_reader=v1_reader, - ) - == v1_manifest - ) - - v2_request = _package_request() - v2_result, v2_manifest, v2_reader = _v2_result(v2_request) - assert ( - await validate_reporting_source_execution( - capabilities=capabilities, - request=v2_request, - result=v2_result, - object_reader=v2_reader, - ) - == v2_manifest - ) - - -async def test_v2_only_capability_rejects_v1_request() -> None: - request = redacted_snapshot_request() - result, _, reader = redacted_completed_result(request) - with pytest.raises(ReportingSourceConformanceError) as error: - await validate_reporting_source_execution( - capabilities=_v2_capabilities(), - request=request, - result=result, - object_reader=reader, - ) - assert error.value.code == "CAPABILITY_MISMATCH" - - -async def test_mixed_request_manifest_versions_fail_admission() -> None: - request = _package_request() - _, v1_manifest, reader = redacted_completed_result(redacted_snapshot_request()) - payload = v1_manifest.model_dump(mode="json") - payload["coverage"]["denominator_fingerprint"] = request.coverage.denominator_fingerprint - payload["coverage"]["constituents"][0]["constituent"] = request.coverage.constituents[ - 0 - ].model_dump(mode="json") - for cell in payload["metric_availability"]: - cell["constituent_id"] = request.coverage.constituents[0].constituent_id - payload["content_fingerprint"] = publication_content_fingerprint_v2(payload) - from adcp.reporting.canonical_json import canonical_json_utf8_v1 - - manifest_bytes = canonical_json_utf8_v1(payload) - result = ReportingSourceExecutorResult.completed( - request=request, - manifest=source_batch_manifest_reference_v1("manifest-mixed", manifest_bytes), - manifest_bytes=manifest_bytes, - ) - with pytest.raises(ReportingSourceConformanceError) as error: - await validate_reporting_source_execution( - capabilities=_v2_capabilities(), - request=request, - result=result, - object_reader=reader, - ) - assert error.value.code == "MANIFEST_MISMATCH" - - -async def test_v1_request_rejects_v2_manifest() -> None: - request = redacted_snapshot_request() - versioned_request = _package_request() - versioned_result, _, reader = _v2_result(versioned_request) - assert versioned_result.response is not None - assert versioned_result.manifest_bytes is not None - result = ReportingSourceExecutorResult.completed( - request=request, - manifest=versioned_result.response.manifest, - manifest_bytes=versioned_result.manifest_bytes, - ) - with pytest.raises(ReportingSourceConformanceError) as error: - await validate_reporting_source_execution( - capabilities=redacted_capabilities(), - request=request, - result=result, - object_reader=reader, - ) - assert error.value.code == "MANIFEST_MISMATCH" - - -def test_v2_manifest_rejects_package_without_parent_buy() -> None: - request = _package_request() - _, manifest, _ = _v2_result(request) - payload = manifest.model_dump(mode="json") - payload["coverage"]["constituents"][0]["constituent"].pop("media_buy_id") - with pytest.raises(ValidationError): - SourceBatchManifestV2.model_validate(payload) - - -def test_v2_execution_response_round_trips_without_downgrading() -> None: - result, _, _ = _v2_result(_package_request()) - round_trip = ReportingSourceExecutorResult.model_validate_json(result.model_dump_json()) - assert round_trip.response is not None - assert round_trip.response.contract_version == "2.0" - - -async def test_mixed_version_bytes_must_pass_reference_integrity_first() -> None: - result, _, reader = _v2_result(_package_request()) - assert result.response is not None - bad_reference = result.response.manifest.model_copy(update={"manifest_sha256": "0" * 64}) - mixed = ReportingSourceExecutorResult.completed( - request=redacted_snapshot_request(), - manifest=bad_reference, - manifest_bytes=result.manifest_bytes or b"", - ) - with pytest.raises(SourceBatchManifestError) as error: - await validate_reporting_source_execution( - capabilities=redacted_capabilities(), - request=redacted_snapshot_request(), - result=mixed, - object_reader=reader, - ) - assert error.value.code == "SOURCE_MANIFEST_CHECKSUM_MISMATCH" diff --git a/tests/test_reporting_source_package_contract.py b/tests/test_reporting_source_package_contract.py new file mode 100644 index 0000000000..0fb08d0f69 --- /dev/null +++ b/tests/test_reporting_source_package_contract.py @@ -0,0 +1,315 @@ +"""One source contract covers buy and package denominators.""" + +from __future__ import annotations + +import hashlib + +import pytest +from pydantic import ValidationError + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.conformance import ( + ReportingSourceConformanceError, + validate_reporting_source_execution, +) +from adcp.reporting.fixtures import ( + redacted_capabilities, + redacted_completed_result, + redacted_snapshot_request, +) +from adcp.reporting.source import ( + PackageItemConstituentV1, + ReportingSourceCapabilitiesV1, + ReportingSourceCoverageRequestV1, + ReportingSourceSliceRequestV1, + SourceBatchManifestV1, + coverage_denominator_fingerprint_v1, + encode_source_batch_manifest_v1, + parse_verified_source_batch_manifest_v1, + publication_content_fingerprint_v1, + reporting_source_capabilities_sha256_v1, + source_batch_manifest_reference_v1, + validate_reporting_source_rows_v1, + validate_reporting_source_staged_rows_v1, +) + + +def _package_request(*, campaign_dimension: bool = False) -> ReportingSourceSliceRequestV1: + package = PackageItemConstituentV1( + constituent_id="package-constituent", + product_id="fixture-product", + media_buy_id="buy-1", + package_id="package-1", + ) + payload = redacted_snapshot_request().model_dump(mode="json") + payload["requested_dimensions"] = ( + ["package", "campaign_id"] if campaign_dimension else ["package"] + ) + payload["coverage"] = ReportingSourceCoverageRequestV1( + grain="package_item", + media_buy_ids=("buy-1",), + constituents=[package], + denominator_fingerprint=coverage_denominator_fingerprint_v1( + [package], grain="package_item", media_buy_ids=("buy-1",) + ), + ).model_dump(mode="json") + return ReportingSourceSliceRequestV1.model_validate(payload) + + +def _empty_package_request(buy_id: str = "buy-1") -> ReportingSourceSliceRequestV1: + payload = redacted_snapshot_request().model_dump(mode="json") + payload["coverage"] = { + "expected": "full", + "grain": "package_item", + "media_buy_ids": [buy_id], + "constituents": [], + "denominator_fingerprint": coverage_denominator_fingerprint_v1( + [], grain="package_item", media_buy_ids=(buy_id,) + ), + } + return ReportingSourceSliceRequestV1.model_validate(payload) + + +def test_ordinary_buy_v1_wire_and_hashes_remain_frozen() -> None: + request = redacted_snapshot_request() + request_json = request.model_dump_json() + canonical_request = canonical_json_utf8_v1(request.model_dump(mode="json", exclude_none=True)) + assert hashlib.sha256(canonical_request).hexdigest() == ( + "0b81860effeac0424f5d2efa9d8fc680ee86cb226749dbe17812bf94296c0f50" + ) + assert hashlib.sha256(request_json.encode()).hexdigest() == ( + "53b59efb8632ac2e314da2d4144ed2295ef1c7e9a49cb1fc3f4d55f9da724585" + ) + assert request.coverage.denominator_fingerprint == ( + "sha256:27bf44bb30e8ae26980327cf0913d74327dc2c344eaad02c194604672f16b74d" + ) + assert ReportingSourceSliceRequestV1.model_validate_json(request_json).model_dump_json() == ( + request_json + ) + assert "grain" not in request.coverage.model_dump() + assert "media_buy_ids" not in request.coverage.model_dump() + _, manifest, _ = redacted_completed_result(request) + assert manifest.content_fingerprint == ( + "sha256:178e0438af4767776bd58316b868c111a9f4df806f506c951ffae49e5410e45f" + ) + assert "grain" not in manifest.coverage.model_dump() + assert "media_buy_ids" not in manifest.coverage.model_dump() + manifest_bytes = encode_source_batch_manifest_v1(manifest) + assert ( + parse_verified_source_batch_manifest_v1( + source_batch_manifest_reference_v1("manifest-buy-v1", manifest_bytes), manifest_bytes + ) + == manifest + ) + + +def test_package_selector_and_pair_are_bound_on_the_v1_wire() -> None: + request = _package_request() + assert request.contract_version == "1.0" + assert request.media_buy_ids == ("buy-1",) + assert request.package_ids == ("package-1",) + assert request.coverage.grain == "package_item" + assert request.coverage.constituents[0].media_buy_id == "buy-1" + + +def test_complete_empty_package_scope_has_distinct_frozen_selectors() -> None: + first = _empty_package_request("buy-1") + second = _empty_package_request("buy-2") + assert first.media_buy_ids == ("buy-1",) + assert second.media_buy_ids == ("buy-2",) + assert first.coverage.denominator_fingerprint != second.coverage.denominator_fingerprint + + +async def test_complete_empty_package_scope_is_full_without_explicit_zero() -> None: + request = _empty_package_request() + result, manifest, reader = redacted_completed_result( + request, page_bodies=[""], row_counts=[0], explicit_zero=False + ) + assert manifest.manifest_version == "1.0" + assert manifest.coverage.grain == "package_item" + assert manifest.coverage.media_buy_ids == ("buy-1",) + assert manifest.coverage.status == "full" + assert not manifest.metric_availability + assert not manifest.explicit_zero + assert ( + await validate_reporting_source_execution( + capabilities=redacted_capabilities(), + request=request, + result=result, + object_reader=reader, + ) + == manifest + ) + + +def test_complete_empty_package_scope_requires_current_watermark() -> None: + request = _empty_package_request() + _, manifest, _ = redacted_completed_result( + request, page_bodies=[""], row_counts=[0], explicit_zero=False + ) + payload = manifest.model_dump(mode="json") + payload["data_through"] = request.period.start.isoformat() + payload["content_fingerprint"] = publication_content_fingerprint_v1(payload) + with pytest.raises(ValidationError, match="current source cutoff"): + SourceBatchManifestV1.model_validate(payload) + + +def test_missing_wire_scope_fields_are_rejected() -> None: + for field in ("grain", "media_buy_ids"): + request_payload = _package_request().model_dump(mode="json") + request_payload["coverage"].pop(field) + with pytest.raises(ValidationError): + ReportingSourceSliceRequestV1.model_validate(request_payload) + _, manifest, _ = redacted_completed_result(_package_request()) + manifest_payload = manifest.model_dump(mode="json") + manifest_payload["coverage"].pop(field) + with pytest.raises(ValidationError): + SourceBatchManifestV1.model_validate(manifest_payload) + + +async def test_empty_package_scope_still_requires_package_capability() -> None: + request = _empty_package_request() + result, _, reader = redacted_completed_result( + request, page_bodies=[""], row_counts=[0], explicit_zero=False + ) + capabilities_payload = redacted_capabilities().model_dump(mode="json") + capabilities_payload["offerings"][0]["constituent_kinds"] = ["media_buy"] + capabilities_payload["capabilities_sha256"] = reporting_source_capabilities_sha256_v1( + capabilities_payload + ) + capabilities = ReportingSourceCapabilitiesV1.model_validate(capabilities_payload) + with pytest.raises(ReportingSourceConformanceError, match="package_item support"): + await validate_reporting_source_execution( + capabilities=capabilities, + request=request, + result=result, + object_reader=reader, + ) + + +def test_package_denominator_rejects_duplicate_pairs_and_unselected_buy() -> None: + package = _package_request().coverage.constituents[0] + duplicate = package.model_copy(update={"constituent_id": "another-constituent"}) + for constituents, selector in (([package, duplicate], ("buy-1",)), ([package], ("buy-2",))): + with pytest.raises(ValidationError): + ReportingSourceCoverageRequestV1( + grain="package_item", + media_buy_ids=selector, + constituents=constituents, + denominator_fingerprint=coverage_denominator_fingerprint_v1( + constituents, grain="package_item", media_buy_ids=selector + ), + ) + + +def test_one_package_id_can_be_used_by_two_selected_buys() -> None: + first = _package_request().coverage.constituents[0] + second = first.model_copy( + update={"constituent_id": "second-constituent", "media_buy_id": "buy-2"} + ) + coverage = ReportingSourceCoverageRequestV1( + grain="package_item", + media_buy_ids=("buy-1", "buy-2"), + constituents=[first, second], + denominator_fingerprint=coverage_denominator_fingerprint_v1( + [first, second], grain="package_item", media_buy_ids=("buy-1", "buy-2") + ), + ) + assert [(item.media_buy_id, item.package_id) for item in coverage.constituents] == [ + ("buy-1", "package-1"), + ("buy-2", "package-1"), + ] + + +def test_manifest_must_echo_selector_and_pairs_exactly() -> None: + request = _package_request() + _, manifest, _ = redacted_completed_result(request) + payload = manifest.model_dump(mode="json") + payload["coverage"]["media_buy_ids"] = ["buy-1", "buy-2"] + payload["coverage"]["denominator_fingerprint"] = coverage_denominator_fingerprint_v1( + request.coverage.constituents, + grain="package_item", + media_buy_ids=("buy-1", "buy-2"), + ) + payload["content_fingerprint"] = publication_content_fingerprint_v1(payload) + widened = SourceBatchManifestV1.model_validate(payload) + with pytest.raises(ValueError, match="manifest coverage differs"): + validate_reporting_source_rows_v1(request, widened, []) + + +def test_explicit_zero_package_pair_rejects_a_row() -> None: + request = _package_request() + _, manifest, _ = redacted_completed_result(request, page_bodies=[""], row_counts=[0]) + with pytest.raises(ValueError, match="contradicts explicit zero"): + validate_reporting_source_rows_v1( + request, + manifest, + [{"media_buy_id": "buy-1", "package_id": "package-1", "impressions": 0}], + ) + + +def test_package_request_retains_the_1000_cell_limit() -> None: + first = _package_request().coverage.constituents[0] + for pair_count, allowed in ((500, True), (501, False)): + constituents = [ + first.model_copy( + update={"constituent_id": f"item-{index}", "package_id": f"package-{index}"} + ) + for index in range(pair_count) + ] + payload = _package_request().model_dump(mode="json") + payload["coverage"] = ReportingSourceCoverageRequestV1( + grain="package_item", + media_buy_ids=("buy-1",), + constituents=constituents, + denominator_fingerprint=coverage_denominator_fingerprint_v1( + constituents, grain="package_item", media_buy_ids=("buy-1",) + ), + ).model_dump(mode="json") + if allowed: + ReportingSourceSliceRequestV1.model_validate(payload) + else: + with pytest.raises(ValidationError, match="at most 1000 constituent-metric cells"): + ReportingSourceSliceRequestV1.model_validate(payload) + + +@pytest.mark.parametrize("stage", ["staged", "final"]) +def test_package_rows_reject_outside_pairs_and_duplicate_requested_grain(stage: str) -> None: + request = _package_request() + rows = [ + {"media_buy_id": "buy-1", "package_id": "package-1", "impressions": 4}, + {"media_buy_id": "buy-1", "package_id": "package-1", "impressions": 5}, + ] + manifest = redacted_completed_result(request)[1] if stage == "final" else None + + def validate(values: list[dict[str, str | int]]) -> None: + if manifest is None: + validate_reporting_source_staged_rows_v1(request, values) + else: + validate_reporting_source_rows_v1(request, manifest, values) + + with pytest.raises(ValueError, match="duplicate row"): + validate(rows) + for row in ( + {"media_buy_id": "buy-2", "package_id": "package-1", "impressions": 0}, + {"media_buy_id": "buy-1", "package_id": "package-2", "impressions": 0}, + {"media_buy_id": "buy-1", "impressions": 0}, + ): + with pytest.raises(ValueError, match="outside the requested coverage"): + validate([row]) + + +@pytest.mark.parametrize("stage", ["staged", "final"]) +def test_distinct_requested_campaign_values_remain_valid(stage: str) -> None: + request = _package_request(campaign_dimension=True) + rows = [ + {"media_buy_id": "buy-1", "package_id": "package-1", "campaign_id": "a"}, + {"media_buy_id": "buy-1", "package_id": "package-1", "campaign_id": "b"}, + ] + if stage == "staged": + validate_reporting_source_staged_rows_v1(request, rows) + else: + from adcp.reporting.fixtures import redacted_completed_result + + _, manifest, _ = redacted_completed_result(request) + validate_reporting_source_rows_v1(request, manifest, rows)