diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be1e649b1..8ac815209 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,6 +70,7 @@ jobs: run: | cargo fetch --locked cargo fetch --locked --manifest-path examples/app-demo/Cargo.toml + cargo fetch --locked --manifest-path examples/geo-kv-prototype/Cargo.toml cargo fetch --locked --manifest-path crates/edgezero-test-harness/fixtures/reusable-app/Cargo.toml cargo fetch --locked --manifest-path crates/edgezero-test-harness/Cargo.toml cargo fetch --locked --manifest-path crates/edgezero-test-harness/fixtures/request-fidelity/Cargo.toml @@ -215,6 +216,18 @@ jobs: - name: Native Geo real-peer hosting (loopback only) run: scripts/run_test_nonzero.sh geo_hosting_proves_real_peer_and_shared_outcome cargo test --offline --locked -p edgezero-adapter-axum --no-default-features --features geo-cloudfront,test-utils --test production_hosting geo_hosting_proves_real_peer_and_shared_outcome + - name: Portable KV Geo and managed listener (loopback only) + if: matrix.suite == 'workspace' + run: | + cargo fmt --manifest-path examples/geo-kv-prototype/Cargo.toml -- --check + cargo clippy --offline --locked --manifest-path examples/geo-kv-prototype/Cargo.toml --all-targets --all-features -- -D warnings + cargo test --offline --locked --manifest-path examples/geo-kv-prototype/Cargo.toml + cargo test --offline --locked --manifest-path examples/geo-kv-prototype/Cargo.toml --all-features + node --test scripts/run_geo_kv_integration.test.mjs + cargo test --offline --locked -p edgezero-core --features geo-kv-build --lib geo::kv + cargo check --offline --locked -p edgezero-core --no-default-features --features geo-kv --target wasm32-unknown-unknown + scripts/run_test_nonzero.sh kv_geo_listener_checks_reader_provenance_and_pinned_objects cargo test --offline --locked -p edgezero-adapter-axum --features geo-kv,geo-cloudfront,test-utils --test production_hosting kv_geo + - name: Native Geo public examples run: cargo test --offline --locked -p edgezero-adapter-axum --features geo-cloudfront --doc diff --git a/crates/edgezero-adapter-axum/Cargo.toml b/crates/edgezero-adapter-axum/Cargo.toml index fd8a2c15f..27626db40 100644 --- a/crates/edgezero-adapter-axum/Cargo.toml +++ b/crates/edgezero-adapter-axum/Cargo.toml @@ -39,6 +39,7 @@ aws-dynamodb-records = ["aws-dynamodb", "edgezero-store-aws/dynamodb-records"] celld-kv = ["axum", "edgezero-store-celld/kv"] celld-records = ["celld-kv", "edgezero-store-celld/records"] geo-cloudfront = ["axum"] +geo-kv = ["axum", "edgezero-core/geo-kv"] cli = [ "native-bindings", "dep:edgezero-adapter", @@ -121,7 +122,7 @@ required-features = ["celld-kv", "test-utils"] async-trait = { workspace = true } axum = { workspace = true, features = ["macros"] } edgezero-adapter-cloudflare = { path = "../edgezero-adapter-cloudflare", default-features = false } -edgezero-core = { path = "../edgezero-core", features = ["test-utils"] } +edgezero-core = { path = "../edgezero-core", features = ["test-utils", "geo-kv-build"] } flate2 = { workspace = true } http-body = { workspace = true } httparse = { workspace = true } diff --git a/crates/edgezero-adapter-axum/src/dev_server.rs b/crates/edgezero-adapter-axum/src/dev_server.rs index 426982332..26271a760 100644 --- a/crates/edgezero-adapter-axum/src/dev_server.rs +++ b/crates/edgezero-adapter-axum/src/dev_server.rs @@ -28,7 +28,8 @@ use crate::diagnostics::{NativeDiagnosticsHandle, NativeSession}; use crate::outbound_executor::{AxumOutboundExecutor, AxumOutboundExecutorHandle}; pub use crate::run_options::AxumRunOptions; use crate::run_options::{ - JsonBodyLimit, config_path, failure, native_limit_setting, native_settings, shutdown_grace, + JsonBodyLimit, config_path, failure, geo_setting, native_limit_setting, native_settings, + shutdown_grace, }; use crate::trusted_proxy::TrustedProxyPolicy; use anyhow::Context as _; @@ -51,6 +52,7 @@ use edgezero_core::config_store::{ConfigStoreError, ConfigStoreHandle, ConfigSto use edgezero_core::env_config::EnvConfig; use edgezero_core::key_value_store::KvHandle; use edgezero_core::logging::{BOOT_LOG_LEVEL, BOOT_LOG_TARGET, resolve_logging_level}; +use edgezero_core::manifest::BakedManifest; #[cfg(test)] use edgezero_core::router::RouterService; use edgezero_core::secret_store::SecretHandle; @@ -66,7 +68,7 @@ use tokio::time::{Instant as NativeInstant, sleep_until}; use crate::config_store::AxumConfigStore; use crate::config_store_limits::ConfigStoreLimits; use crate::connection::{ConnectionExit, serve_http1_with_shutdown}; -use crate::geo::NativeGeo; +use crate::geo::{self, NativeGeo}; use crate::ingress_config::AxumIngressConfig; use crate::key_value_store::PersistentKvStore; use crate::native_stores::{self, NativeSourcePlan, NativeStoreOverrides, NativeStoreSource}; @@ -1059,7 +1061,10 @@ where A: Hooks, F: FnOnce() -> anyhow::Result<()>, { - let (env, json_body_limit, native) = capture_dev_environment(vars_os())?; + let (env, json_body_limit, mut native) = capture_dev_environment(vars_os())?; + // The dev server ignores the manifest table: its deployment-only values are + // usually absent locally. Settings alone opt a dev run into geo. + let geo = geo::resolve(None, &env, &BakedManifest::Absent)?; if !A::owns_logging() { app_logger(&env) .init() @@ -1094,6 +1099,8 @@ where config: build_config_registry(metadata.config, &env, config_limits)?, secrets: build_secret_registry(metadata.secrets, &env), }; + let max_object_bytes = geo.max_object_bytes(None)?; + native.geo = geo.prepare(stores.kv(), max_object_bytes).await?; prepare_transport(app, stores, json_body_limit, native) }, ) @@ -1120,7 +1127,7 @@ where ["adapter", "trusted_proxy_cidrs"] => Some("TRUSTED_PROXY_CIDRS"), ["adapter", "forwarding_header_family"] => Some("FORWARDING_HEADER_FAMILY"), ["logging", "request_records"] => Some("LOGGING__REQUEST_RECORDS"), - _ => None, + _ => geo_setting(&path), } }) }); @@ -1319,12 +1326,12 @@ where &'startup PreparedStores, ) -> LocalStartupFuture<'startup>, { - let geo = options.native_geo()?; - let native = NativeHosting { + let geo = options.native_geo(&A::manifest())?; + let mut native = NativeHosting { diagnostics: options.diagnostics(), policy: options.trusted_proxy_policy(), request_records: options.request_records(), - geo, + geo: NativeGeo::default(), }; run_local( ListenerSource::Bind(options.addr()), @@ -1335,6 +1342,7 @@ where async move { let metadata = A::stores(); let sources = NativeSourcePlan::validate(metadata, &options, overrides)?; + let max_object_bytes = geo.max_object_bytes(Some(&sources))?; init_logging::(options.logging_level())?; let mut app = App::build::(crate::AXUM_PLATFORM) .map_err(|_error| anyhow::anyhow!("application configuration failed"))?; @@ -1345,6 +1353,7 @@ where app.record_store_requirements(), ) .await?; + native.geo = geo.prepare(stores.kv(), max_object_bytes).await?; let transport = AxumOutboundClient::try_transport() .map_err(|_error| failure("transport", "HTTP client", "initialization failed"))?; initialize(&mut app, &stores).await.map_err(|error| { diff --git a/crates/edgezero-adapter-axum/src/geo.rs b/crates/edgezero-adapter-axum/src/geo.rs index 4d3f86557..4731373bf 100644 --- a/crates/edgezero-adapter-axum/src/geo.rs +++ b/crates/edgezero-adapter-axum/src/geo.rs @@ -4,7 +4,9 @@ //! is not proof of an ALB credential rule or a particular `CloudFront` distribution. //! See the native Geo guide for the required network, header and cache controls. //! -//! Off is valid and is the default, with no ambient Geo selector: +//! Off is valid and is the default. Without an explicit selection the runner reads +//! `EDGEZERO__ADAPTER__GEO__*` settings, then the app's `[adapters.axum.geo]` manifest table. +//! An explicit selection overrides both: //! ``` //! use edgezero_adapter_axum::dev_server::AxumRunOptions; //! use edgezero_adapter_axum::geo::{NativeGeoCapability, NativeGeoSelection}; @@ -78,6 +80,9 @@ //! GeoOutcome::Selected { status, .. } => match status { //! GeoStatus::Unavailable(GeoUnavailableReason::UntrustedIngress) => "source not admitted", //! GeoStatus::Unavailable(GeoUnavailableReason::MissingMetadata) => "metadata missing", +//! GeoStatus::Unavailable(GeoUnavailableReason::MissingReaderIp) => "no checked reader", +//! GeoStatus::Unavailable(GeoUnavailableReason::NoMatch) => "no geographic coverage", +//! GeoStatus::Unavailable(GeoUnavailableReason::UnsupportedGeography) => "unknown country code", //! GeoStatus::Error(GeoError::InvalidAssertion) => "assertion rejected", //! GeoStatus::Error(GeoError::ProviderFailure) => "provider failed", //! GeoStatus::Located { location, .. } if location.subdivision().is_none() => "country only", @@ -92,7 +97,7 @@ reason = "public composition types precede private request-bound resolution and tests" )] #![cfg_attr( - feature = "geo-cloudfront", + any(feature = "geo-cloudfront", feature = "geo-kv"), expect( clippy::self_named_module_files, reason = "repo convention uses geo.rs with submodules under geo/, not mod.rs" @@ -106,14 +111,26 @@ use std::fmt; use std::net::SocketAddr; use std::sync::Arc; +use edgezero_core::env_config::EnvConfig; use edgezero_core::geo::{ GeoAssertion, GeoInput, GeoOutcome, GeoProvider, GeoProviderId, GeoResolution, GeoSource, GeoStatus, GeoTrustBasis, GeoUnavailableReason, }; use edgezero_core::http::{HeaderMap, HeaderName}; +use edgezero_core::manifest::{BakedManifest, ManifestAdapterGeo, ManifestGeoProvider}; +use edgezero_core::store_registry::KvRegistry; + +use crate::diagnostics::NativeIngressMetadata; +use crate::native_stores::NativeSourcePlan; use crate::peer_networks::PeerNetworks; +/// Native KV Geo selection and checked reader-IP policy. +#[cfg(feature = "geo-kv")] +pub mod kv; +#[cfg(feature = "geo-kv")] +use kv::{KvGeoReaderIp, KvGeoSelection}; + /// Bounded startup configuration categories. No policy inputs are retained. #[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] pub enum NativeGeoConfigError { @@ -125,6 +142,16 @@ pub enum NativeGeoConfigError { UnavailableCloudFront, #[error("native_geo: reserved supplied provider identity")] ReservedProviderId, + #[error("native_geo: invalid provider selection")] + InvalidSelection, + #[error("native_geo: KV implementation unavailable")] + UnavailableKv, + #[error("native_geo: selected KV binding is undeclared or unavailable")] + MissingKvBinding, + #[error("native_geo: KV preparation deadline")] + KvPreparationDeadline, + #[error("native_geo: pinned KV data could not be prepared")] + KvPreparationFailed, } /// An operator declaration of a credential-gated, isolated CloudFront/ALB path. @@ -201,6 +228,8 @@ impl fmt::Debug for ProtectedCloudFrontAlb { pub enum NativeGeoSelection { #[default] Off, + #[cfg(feature = "geo-kv")] + Kv(KvGeoSelection), CloudFront { policy: ProtectedCloudFrontAlb, }, @@ -216,6 +245,8 @@ impl fmt::Debug for NativeGeoSelection { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Off => f.write_str("Off"), + #[cfg(feature = "geo-kv")] + Self::Kv(selection) => fmt::Debug::fmt(selection, f), Self::CloudFront { policy } => f .debug_struct("CloudFront") .field("policy", policy) @@ -240,11 +271,254 @@ pub enum NativeGeoCapability { }, } +/// Resolves the startup selection: an explicit [`AxumRunOptions::with_geo`] wins, +/// then `EDGEZERO__ADAPTER__GEO__*` settings, then the baked `[adapters.axum.geo]` +/// table. With none of them geo stays off. +/// +/// [`AxumRunOptions::with_geo`]: crate::run_options::AxumRunOptions::with_geo +pub(crate) fn resolve( + explicit: Option<&NativeGeoSelection>, + env: &EnvConfig, + manifest: &BakedManifest, +) -> Result { + let configured = match explicit { + Some(selection) => selection.clone(), + None => configured_selection(env, manifest_geo(manifest))?, + }; + #[cfg(feature = "geo-kv")] + if let NativeGeoSelection::Kv(selection) = configured { + return Ok(NativeGeoPlan::Kv(selection)); + } + NativeGeo::validate(&configured).map(NativeGeoPlan::Ready) +} + +fn manifest_geo(baked: &BakedManifest) -> Option<&ManifestAdapterGeo> { + // An absent or malformed manifest selects nothing; app building rejects malformed. + let BakedManifest::Present(manifest) = baked else { + return None; + }; + let (_name, adapter) = manifest.adapter_entry("axum")?; + adapter.geo.as_ref() +} + +fn configured_selection( + env: &EnvConfig, + defaults: Option<&ManifestAdapterGeo>, +) -> Result { + let env_provider = env.get(&["adapter", "geo", "provider"]); + let env_networks = env.get(&["adapter", "geo", "alb_peer_cidrs"]); + let env_header = env.get(&["adapter", "geo", "origin_credential_header_name"]); + let provider = match env_provider { + Some("cloudfront") => Some(ManifestGeoProvider::CloudFront), + Some("kv") => Some(ManifestGeoProvider::Kv), + Some("off") => Some(ManifestGeoProvider::Off), + Some(_) => return Err(NativeGeoConfigError::InvalidSelection), + None => defaults.and_then(|geo| geo.provider), + }; + let kv_keys = [ + "store_id", + "pin", + "reader_ip", + "lookup_timeout_ms", + "max_in_flight", + ]; + let has_env_kv = kv_keys + .into_iter() + .any(|key| env.get(&["adapter", "geo", key]).is_some()); + if (provider != Some(ManifestGeoProvider::Kv) && has_env_kv) + || (provider == Some(ManifestGeoProvider::Kv) + && (env_networks.is_some() || env_header.is_some())) + { + return Err(NativeGeoConfigError::InvalidSelection); + } + // When settings switch providers, only their own fields may carry over. + let manifest = defaults.filter(|geo| geo.provider.is_none() || geo.provider == provider); + match provider { + Some(ManifestGeoProvider::Kv) => configured_kv(env, manifest), + Some(ManifestGeoProvider::CloudFront) => { + if manifest.is_some_and(ManifestAdapterGeo::has_kv_fields) { + return Err(NativeGeoConfigError::InvalidSelection); + } + let networks: Vec<&str> = match env_networks { + Some(raw) => raw.split(',').map(str::trim).collect(), + None => manifest + .and_then(|geo| geo.alb_peer_cidrs.as_deref()) + .unwrap_or_default() + .iter() + .map(String::as_str) + .collect(), + }; + let header = env_header + .or_else(|| manifest?.origin_credential_header_name.as_deref()) + .ok_or(NativeGeoConfigError::InvalidCredentialHeader)?; + let policy = ProtectedCloudFrontAlb::new(networks, header)?; + Ok(NativeGeoSelection::CloudFront { policy }) + } + // Policy without a selected provider is a misconfiguration, not a silent + // off. A manifest CloudFront default may still be overridden to off. + _ if env_networks.is_some() + || env_header.is_some() + || (provider.is_none() + && manifest.is_some_and(|geo| { + geo.alb_peer_cidrs.is_some() + || geo.origin_credential_header_name.is_some() + || geo.has_kv_fields() + })) => + { + Err(NativeGeoConfigError::InvalidSelection) + } + _ => Ok(NativeGeoSelection::Off), + } +} + +fn configured_kv( + env: &EnvConfig, + manifest: Option<&ManifestAdapterGeo>, +) -> Result { + #[cfg(not(feature = "geo-kv"))] + { + let _: (&EnvConfig, Option<&ManifestAdapterGeo>) = (env, manifest); + Err(NativeGeoConfigError::UnavailableKv) + } + #[cfg(feature = "geo-kv")] + { + use NativeGeoConfigError::InvalidSelection; + if manifest.is_some_and(|geo| { + geo.alb_peer_cidrs.is_some() || geo.origin_credential_header_name.is_some() + }) { + return Err(InvalidSelection); + } + let get = |key| env.get(&["adapter", "geo", key]); + let reader_ip = match get("reader_ip").or_else(|| manifest?.reader_ip.as_deref()) { + Some("direct-peer") => KvGeoReaderIp::DirectPeer, + Some("forwarded-required") => KvGeoReaderIp::ForwardedRequired, + _ => return Err(InvalidSelection), + }; + let timeout = match get("lookup_timeout_ms") { + Some(raw) => raw.parse().ok(), + None => manifest.and_then(|geo| geo.lookup_timeout_ms), + } + .ok_or(InvalidSelection)?; + let max_in_flight = match get("max_in_flight") { + Some(raw) => raw.parse().ok(), + None => manifest.and_then(|geo| geo.max_in_flight), + } + .ok_or(InvalidSelection)?; + let selection = KvGeoSelection::new( + get("store_id") + .or_else(|| manifest?.store_id.as_deref()) + .ok_or(InvalidSelection)?, + get("pin") + .or_else(|| manifest?.pin.as_deref()) + .ok_or(InvalidSelection)?, + reader_ip, + timeout, + max_in_flight, + )?; + Ok(NativeGeoSelection::Kv(selection)) + } +} + +/// A syntactically valid selection that still needs its serving binding. +pub(crate) enum NativeGeoPlan { + Ready(NativeGeo), + #[cfg(feature = "geo-kv")] + Kv(KvGeoSelection), +} + +impl NativeGeoPlan { + pub(crate) fn capability(&self) -> NativeGeoCapability { + match self { + Self::Ready(geo) => geo.capability(), + #[cfg(feature = "geo-kv")] + Self::Kv(selection) => NativeGeoCapability::Selected { + provider: GeoProviderId::kv(), + source: GeoSource::Kv, + trust_basis: selection.trust_basis(), + }, + } + } + + #[cfg_attr( + not(feature = "geo-kv"), + expect( + clippy::unnecessary_wraps, + clippy::unused_self, + reason = "share the runner's preparation path while compiled-out Geo needs no KV limit" + ) + )] + pub(crate) fn max_object_bytes( + &self, + sources: Option<&NativeSourcePlan>, + ) -> Result { + #[cfg(feature = "geo-kv")] + if let Self::Kv(selection) = self { + return match sources { + Some(plan) => { + use crate::native_stores::{NativeKvSource, NativeStoreSource}; + let provider_cap = match plan.kv.get(selection.store_id()) { + Some(NativeStoreSource::BuiltIn(NativeKvSource::DynamoDb(settings))) => { + settings.limits.max_value_bytes + } + Some(NativeStoreSource::BuiltIn(NativeKvSource::Celld { .. })) => { + usize::try_from(plan.policies.celld.max_value_bytes) + .map_err(|_error| NativeGeoConfigError::InvalidSelection)? + } + Some(NativeStoreSource::BuiltIn( + NativeKvSource::DynamoDbRecords(_) + | NativeKvSource::CelldRecords { .. }, + )) => return Err(NativeGeoConfigError::InvalidSelection), + Some( + NativeStoreSource::Local + | NativeStoreSource::Ready(_) + | NativeStoreSource::Prepare(_), + ) => kv::MAX_OBJECT_BYTES, + None => return Err(NativeGeoConfigError::MissingKvBinding), + }; + Ok(provider_cap.min(kv::MAX_OBJECT_BYTES)) + } + // Dev mode uses the declared local KV. It still validates the selected pin. + None => Ok(kv::MAX_OBJECT_BYTES), + }; + } + let _: Option<&NativeSourcePlan> = sources; + Ok(0) + } + + #[cfg_attr( + not(feature = "geo-kv"), + expect( + clippy::unused_async, + clippy::unused_async_trait_impl, + reason = "only the optional KV provider awaits preparation" + ) + )] + pub(crate) async fn prepare( + self, + stores: Option<&KvRegistry>, + max_object_bytes: usize, + ) -> Result { + match self { + Self::Ready(geo) => { + let _: (Option<&KvRegistry>, usize) = (stores, max_object_bytes); + Ok(geo) + } + #[cfg(feature = "geo-kv")] + Self::Kv(selection) => selection + .prepare(stores, max_object_bytes) + .await + .map(|prepared| NativeGeo::Kv(Arc::new(prepared))), + } + } +} + /// Validated runner-owned composition, never installed in public extensions. #[derive(Clone, Default)] pub(crate) enum NativeGeo { #[default] Off, + #[cfg(feature = "geo-kv")] + Kv(Arc), Selected { policy: ProtectedCloudFrontAlb, id: GeoProviderId, @@ -256,6 +530,8 @@ impl NativeGeo { pub(crate) fn validate(selection: &NativeGeoSelection) -> Result { match selection { NativeGeoSelection::Off => Ok(Self::Off), + #[cfg(feature = "geo-kv")] + NativeGeoSelection::Kv(_) => Err(NativeGeoConfigError::InvalidSelection), #[cfg(feature = "geo-cloudfront")] NativeGeoSelection::CloudFront { policy } => Ok(Self::Selected { policy: policy.clone(), @@ -271,7 +547,7 @@ impl NativeGeo { id, provider, } => { - if id == &GeoProviderId::cloudfront() { + if id == &GeoProviderId::cloudfront() || id == &GeoProviderId::kv() { return Err(NativeGeoConfigError::ReservedProviderId); } Ok(Self::Selected { @@ -286,6 +562,12 @@ impl NativeGeo { pub(crate) fn capability(&self) -> NativeGeoCapability { match self { Self::Off => NativeGeoCapability::Off, + #[cfg(feature = "geo-kv")] + Self::Kv(prepared) => NativeGeoCapability::Selected { + provider: GeoProviderId::kv(), + source: GeoSource::Kv, + trust_basis: prepared.trust_basis(), + }, Self::Selected { id, .. } => NativeGeoCapability::Selected { provider: id.clone(), source: GeoSource::CloudFront, @@ -300,6 +582,31 @@ impl NativeGeo { } } + #[cfg_attr( + not(feature = "geo-kv"), + expect( + clippy::unused_async, + clippy::unused_async_trait_impl, + reason = "only the optional KV provider awaits request I/O" + ) + )] + pub(crate) async fn resolve_request( + &self, + peer: Option, + headers: &HeaderMap, + checked: Option<&NativeIngressMetadata>, + ) -> GeoOutcome { + #[cfg(feature = "geo-kv")] + if let Self::Kv(provider) = self { + return GeoOutcome::Selected { + provider: GeoProviderId::kv(), + status: provider.resolve(peer, checked).await, + }; + } + let _: Option<&NativeIngressMetadata> = checked; + self.resolve(peer, headers) + } + pub(crate) fn resolve(&self, peer: Option, headers: &HeaderMap) -> GeoOutcome { let Self::Selected { policy, @@ -359,8 +666,12 @@ fn assertion(headers: &HeaderMap, name: &'static str) -> GeoAsse mod tests { use super::*; use crate::run_options::AxumRunOptions; + use edgezero_core::app::Hooks; + use edgezero_core::error::EdgeError; use edgezero_core::geo::{CountryCode, GeoError, GeoLocation}; use edgezero_core::http::HeaderValue; + use edgezero_core::manifest::Manifest; + use edgezero_core::router::RouterService; use std::iter::empty; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -368,6 +679,160 @@ mod tests { ProtectedCloudFrontAlb::new(["127.0.0.1/32"], "x-origin-verify-fixture").unwrap() } + fn resolved( + explicit: Option<&NativeGeoSelection>, + vars: &[(&str, &str)], + manifest: &BakedManifest, + ) -> Result { + let env = EnvConfig::from_vars(vars.iter().copied()); + resolve(explicit, &env, manifest).map(|geo| geo.capability()) + } + + fn baked(geo: &'static str) -> BakedManifest { + let json: &'static str = + Box::leak(format!(r#"{{"adapters":{{"axum":{{"geo":{geo}}}}}}}"#).into_boxed_str()); + let baked = Manifest::from_baked_json(json); + assert!(matches!(baked, BakedManifest::Present(_)), "{json}"); + baked + } + + #[test] + fn geo_settings_resolve_explicit_then_env_then_manifest() { + const CIDRS: &str = "EDGEZERO__ADAPTER__GEO__ALB_PEER_CIDRS"; + const HEADER: &str = "EDGEZERO__ADAPTER__GEO__ORIGIN_CREDENTIAL_HEADER_NAME"; + const PROVIDER: &str = "EDGEZERO__ADAPTER__GEO__PROVIDER"; + let none = BakedManifest::Absent; + let cloudfront = baked( + r#"{"provider":"cloudfront","alb_peer_cidrs":["10.0.0.0/24"],"origin_credential_header_name":"x-origin-verify"}"#, + ); + let header_only = + baked(r#"{"provider":"cloudfront","origin_credential_header_name":"x-origin-verify"}"#); + let off = Ok(NativeGeoCapability::Off); + + // Nothing selected stays off; an explicit selection beats settings and manifest. + assert_eq!(resolved(None, &[], &none), off); + assert_eq!( + resolved( + Some(&NativeGeoSelection::Off), + &[(PROVIDER, "cloudfront")], + &cloudfront + ), + off + ); + // Settings override the manifest provider and fill deployment-only values. + assert_eq!(resolved(None, &[(PROVIDER, "off")], &cloudfront), off); + assert_eq!( + resolved(None, &[], &header_only), + Err(NativeGeoConfigError::InvalidProtectedPolicy), + "manifest without networks needs the setting" + ); + assert_eq!( + resolved( + None, + &[(PROVIDER, "cloudfront"), (CIDRS, "10.0.0.0/24")], + &none + ), + Err(NativeGeoConfigError::InvalidCredentialHeader) + ); + assert_eq!( + resolved( + None, + &[ + (CIDRS, "10.0.0.0/24, 10.0.1.0/24"), + (HEADER, "x-forwarded-for") + ], + &header_only + ), + Err(NativeGeoConfigError::InvalidCredentialHeader), + "setting header replaces the manifest header and is validated" + ); + // Misconfiguration never falls back to off. + for (vars, manifest) in [ + (vec![(PROVIDER, "maxmind")], &none), + (vec![(CIDRS, "10.0.0.0/24")], &none), + ( + vec![(PROVIDER, "off"), (HEADER, "x-origin-verify")], + &cloudfront, + ), + ] { + assert_eq!( + resolved(None, &vars, manifest), + Err(NativeGeoConfigError::InvalidSelection), + "{vars:?}" + ); + } + assert_eq!( + resolved(None, &[], &baked(r#"{"alb_peer_cidrs":["10.0.0.0/24"]}"#)), + Err(NativeGeoConfigError::InvalidSelection) + ); + assert_eq!(resolved(None, &[], &baked(r#"{"provider":"off"}"#)), off); + + let selected = [ + resolved(None, &[], &cloudfront), + resolved(None, &[(CIDRS, "10.0.0.0/24, 10.0.1.0/24")], &header_only), + resolved( + None, + &[ + (PROVIDER, "cloudfront"), + (CIDRS, "10.0.0.0/24"), + (HEADER, "x-origin-verify"), + ], + &none, + ), + ]; + for result in selected { + #[cfg(feature = "geo-cloudfront")] + assert!( + matches!(&result, Ok(NativeGeoCapability::Selected { provider, .. }) if provider == &GeoProviderId::cloudfront()), + "{result:?}" + ); + #[cfg(not(feature = "geo-cloudfront"))] + assert_eq!(result, Err(NativeGeoConfigError::UnavailableCloudFront)); + } + } + + #[test] + fn geo_capability_for_applies_the_app_manifest() { + struct GeoApp; + #[expect( + clippy::missing_trait_methods, + reason = "fixture retains unused Hooks defaults" + )] + impl Hooks for GeoApp { + fn routes() -> Result { + Ok(RouterService::builder().build()) + } + fn manifest() -> BakedManifest { + Manifest::from_baked_json( + r#"{"adapters":{"axum":{"geo":{"provider":"cloudfront","alb_peer_cidrs":["10.0.0.0/24"],"origin_credential_header_name":"x-origin-verify"}}}}"#, + ) + } + } + let options = AxumRunOptions::new("127.0.0.1:8080".parse().unwrap()).unwrap(); + assert_eq!(options.geo_capability(), Ok(NativeGeoCapability::Off)); + #[cfg(feature = "geo-cloudfront")] + assert!(matches!( + options.geo_capability_for::(), + Ok(NativeGeoCapability::Selected { .. }) + )); + #[cfg(not(feature = "geo-cloudfront"))] + assert_eq!( + options.geo_capability_for::(), + Err(NativeGeoConfigError::UnavailableCloudFront) + ); + // Settings feed only the runner view; the explicit-only view never errors on them. + let partial = + AxumRunOptions::from_vars([("EDGEZERO__ADAPTER__GEO__ALB_PEER_CIDRS", "10.0.1.0/24")]) + .unwrap(); + assert_eq!(partial.geo_capability(), Ok(NativeGeoCapability::Off)); + let disabled = + AxumRunOptions::from_vars([("EDGEZERO__ADAPTER__GEO__PROVIDER", "off")]).unwrap(); + assert_eq!( + disabled.geo_capability_for::(), + Ok(NativeGeoCapability::Off) + ); + } + #[test] fn geo_protected_policy_reuses_network_bounds() { assert_eq!( diff --git a/crates/edgezero-adapter-axum/src/geo/kv.rs b/crates/edgezero-adapter-axum/src/geo/kv.rs new file mode 100644 index 000000000..0601a4b17 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/geo/kv.rs @@ -0,0 +1,192 @@ +//! Axum-owned preparation and execution budgets for a portable KV resolver. + +use std::fmt; +use std::net::{IpAddr, SocketAddr}; +use std::time::Duration; + +use edgezero_core::geo::kv::{IndexLimits, Receipt, Resolver}; +use edgezero_core::geo::{GeoError, GeoSource, GeoStatus, GeoTrustBasis, GeoUnavailableReason}; +use edgezero_core::ingress::CheckedHostSource; +use edgezero_core::store_registry::KvRegistry; +use tokio::sync::Semaphore; +use tokio::time::timeout; + +use super::NativeGeoConfigError; +use crate::diagnostics::NativeIngressMetadata; + +// The 16 KiB production-size prototype needed up to three nodes; the 64 KiB +// index needed two. The cache payload budget is the measured prototype budget. +const MAX_PATH_OBJECTS: usize = 3; +const CACHE_BYTES: usize = 4 * 1024 * 1024; +// 4 MiB holds 256 objects at the smaller measured object budget (16 KiB). +const CACHE_ENTRIES: usize = 256; +// Do not exceed the smaller qualified backend ceiling (ordinary DynamoDB). +pub(crate) const MAX_OBJECT_BYTES: usize = 384 * 1024; +// Use the native celld preparation window for the bounded manifest/root probe. +const STARTUP_TIMEOUT: Duration = Duration::from_secs(30); + +/// How this listener acquires a reader IP for KV geography. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum KvGeoReaderIp { + /// This listener receives reader connections directly. + DirectPeer, + /// Only the client approved by the trusted proxy policy is usable. + ForwardedRequired, +} + +impl KvGeoReaderIp { + pub(crate) const fn trust_basis(self) -> GeoTrustBasis { + match self { + Self::DirectPeer => GeoTrustBasis::CheckedDirectPeer, + Self::ForwardedRequired => GeoTrustBasis::CheckedForwardedClient, + } + } + + fn address( + self, + peer: Option, + checked: Option<&NativeIngressMetadata>, + ) -> Option { + match self { + Self::DirectPeer => peer.map(|address| address.ip().to_canonical()), + Self::ForwardedRequired => checked + .filter(|metadata| { + matches!( + metadata.client_source(), + CheckedHostSource::TrustedForwarded | CheckedHostSource::TrustedXForwarded + ) + }) + .and_then(NativeIngressMetadata::effective_client), + } + } +} + +/// Validated KV Geo selection. The serving credential comes from the named KV binding. +#[derive(Clone)] +pub struct KvGeoSelection { + store_id: String, + receipt: Receipt, + reader_ip: KvGeoReaderIp, + lookup_timeout: Duration, + max_in_flight: usize, +} + +impl fmt::Debug for KvGeoSelection { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("KvGeoSelection") + .field("reader_ip", &self.reader_ip) + .finish_non_exhaustive() + } +} + +impl KvGeoSelection { + /// Capture an explicit `@` pin and request budgets. + /// + /// # Errors + /// Rejects missing/invalid configuration and a malformed pin. + #[inline] + pub fn new( + store_id: &str, + pin: &str, + reader_ip: KvGeoReaderIp, + lookup_timeout_ms: u32, + max_in_flight: usize, + ) -> Result { + if store_id.is_empty() + || lookup_timeout_ms == 0 + || !(1..=Semaphore::MAX_PERMITS).contains(&max_in_flight) + { + return Err(NativeGeoConfigError::InvalidSelection); + } + let receipt = + Receipt::from_pin(pin).map_err(|_error| NativeGeoConfigError::InvalidSelection)?; + Ok(Self { + store_id: store_id.into(), + receipt, + reader_ip, + lookup_timeout: Duration::from_millis(lookup_timeout_ms.into()), + max_in_flight, + }) + } + + pub(super) const fn trust_basis(&self) -> GeoTrustBasis { + self.reader_ip.trust_basis() + } + + pub(super) fn store_id(&self) -> &str { + &self.store_id + } + + pub(super) async fn prepare( + self, + stores: Option<&KvRegistry>, + max_object_bytes: usize, + ) -> Result { + let handle = stores + .and_then(|registry| registry.named(&self.store_id)) + .ok_or(NativeGeoConfigError::MissingKvBinding)?; + let index = timeout(STARTUP_TIMEOUT, async { + let index = Resolver::open( + handle, + &self.receipt, + IndexLimits { + max_object_bytes, + max_path_objects: MAX_PATH_OBJECTS, + cache_bytes: CACHE_BYTES, + cache_entries: CACHE_ENTRIES, + }, + ) + .await?; + index.verify_roots().await?; + Ok::<_, anyhow::Error>(index) + }) + .await + .map_err(|_error| NativeGeoConfigError::KvPreparationDeadline)? + .map_err(|_error| NativeGeoConfigError::KvPreparationFailed)?; + log::info!(target: "edgezero::native", "event=geo_ready schema={} generation={}", index.manifest().schema, index.manifest().generation); + Ok(PreparedKvGeo { + index, + reader_ip: self.reader_ip, + lookup_timeout: self.lookup_timeout, + slots: Semaphore::new(self.max_in_flight), + }) + } +} + +pub(crate) struct PreparedKvGeo { + index: Resolver, + reader_ip: KvGeoReaderIp, + lookup_timeout: Duration, + slots: Semaphore, +} + +impl PreparedKvGeo { + pub(super) const fn trust_basis(&self) -> GeoTrustBasis { + self.reader_ip.trust_basis() + } + pub(super) async fn resolve( + &self, + peer: Option, + checked: Option<&NativeIngressMetadata>, + ) -> GeoStatus { + let Some(ip) = self.reader_ip.address(peer, checked) else { + return GeoStatus::Unavailable(GeoUnavailableReason::MissingReaderIp); + }; + let Ok(_slot) = self.slots.try_acquire() else { + return GeoStatus::Error(GeoError::ProviderFailure); + }; + match timeout(self.lookup_timeout, self.index.lookup(ip)).await { + Ok(Ok(Some(source))) => match source.geo_location() { + Some(location) => GeoStatus::Located { + location, + source: GeoSource::Kv, + trust_basis: self.reader_ip.trust_basis(), + }, + None => GeoStatus::Unavailable(GeoUnavailableReason::UnsupportedGeography), + }, + Ok(Ok(None)) => GeoStatus::Unavailable(GeoUnavailableReason::NoMatch), + _ => GeoStatus::Error(GeoError::ProviderFailure), + } + } +} diff --git a/crates/edgezero-adapter-axum/src/run_options.rs b/crates/edgezero-adapter-axum/src/run_options.rs index b2384b1de..6b30c605a 100644 --- a/crates/edgezero-adapter-axum/src/run_options.rs +++ b/crates/edgezero-adapter-axum/src/run_options.rs @@ -8,14 +8,17 @@ use std::str::FromStr as _; use std::time::{Duration, Instant}; use edgezero_core::addr; -use edgezero_core::app::StoresMetadata; +use edgezero_core::app::{Hooks, StoresMetadata}; use edgezero_core::env_config::EnvConfig; +use edgezero_core::manifest::BakedManifest; use edgezero_store_aws::settings::{AgentAccessToken, AwsPreparationLimits}; use edgezero_store_celld::settings::{CelldAccessToken, CelldPreparationLimits}; use log::LevelFilter; use crate::diagnostics::NativeDiagnosticsHandle; -use crate::geo::{NativeGeo, NativeGeoCapability, NativeGeoConfigError, NativeGeoSelection}; +use crate::geo::{ + self, NativeGeoCapability, NativeGeoConfigError, NativeGeoPlan, NativeGeoSelection, +}; use crate::native_bindings::{ NativeBindingsLimits, NativeConfigBinding, NativeKvBinding, NativePreparationPolicySources, NativeStoreBindings, NativeStoreBindingsInput, @@ -74,7 +77,8 @@ pub struct AxumRunOptions { data_dir: Option, diagnostics: NativeDiagnosticsHandle, env: EnvConfig, - geo: NativeGeoSelection, + /// `None` until [`Self::with_geo`]; the manifest and settings then decide. + geo: Option, grace: Duration, json_body_limit: JsonBodyLimit, level: LevelFilter, @@ -357,7 +361,7 @@ impl AxumRunOptions { data_dir: None, diagnostics: NativeDiagnosticsHandle::new(), env: EnvConfig::default(), - geo: NativeGeoSelection::Off, + geo: None, grace: Duration::from_secs(10), json_body_limit: JsonBodyLimit::DEFAULT, level: LevelFilter::Info, @@ -371,21 +375,46 @@ impl AxumRunOptions { #[must_use] #[inline] pub fn with_geo(mut self, selection: NativeGeoSelection) -> Self { - self.geo = selection; + self.geo = Some(selection); self } - /// Validate the finalized selection and capture its startup capability. + /// Validate an explicit [`Self::with_geo`] selection and capture its capability. + /// + /// Without one this reports `Off`; it does not read `EDGEZERO__ADAPTER__GEO__*` + /// settings or a manifest. Use [`Self::geo_capability_for`] for the selection the + /// runner will actually use. /// /// # Errors /// Rejects unavailable implementations and invalid protected-source policies. #[inline] pub fn geo_capability(&self) -> Result { - self.native_geo().map(|geo| geo.capability()) + geo::resolve( + self.geo.as_ref(), + &EnvConfig::default(), + &BakedManifest::Absent, + ) + .map(|geo| geo.capability()) } - pub(crate) fn native_geo(&self) -> Result { - NativeGeo::validate(&self.geo) + /// Capture the capability the runner will use for `A`: an explicit selection, + /// then `EDGEZERO__ADAPTER__GEO__*` settings, then its baked `[adapters.axum.geo]`. + /// + /// # Errors + /// Rejects unavailable implementations and invalid protected-source policies. + #[inline] + pub fn geo_capability_for(&self) -> Result + where + A: Hooks, + { + self.native_geo(&A::manifest()).map(|geo| geo.capability()) + } + + pub(crate) fn native_geo( + &self, + manifest: &BakedManifest, + ) -> Result { + geo::resolve(self.geo.as_ref(), &self.env, manifest) } pub(crate) fn request_records(&self) -> bool { @@ -747,10 +776,31 @@ fn fixed_setting(path: &[String]) -> Option<&'static str> { ["adapter", "json_body_limit_bytes"] => Some("JSON_BODY_LIMIT_BYTES"), ["logging", "level"] => Some("LOGGING__LEVEL"), ["store_bindings_file"] => Some("STORE_BINDINGS_FILE"), - _ => native_limit_setting(path), + _ => geo_setting(path).or_else(|| native_limit_setting(path)), } } +/// `EDGEZERO__ADAPTER__GEO__*` names, shared by the dev and production input checks. +pub(crate) fn geo_setting(path: &[String]) -> Option<&'static str> { + let segments: Vec<&str> = path.iter().map(String::as_str).collect(); + matches!( + segments.as_slice(), + [ + "adapter", + "geo", + "provider" + | "alb_peer_cidrs" + | "origin_credential_header_name" + | "store_id" + | "pin" + | "reader_ip" + | "lookup_timeout_ms" + | "max_in_flight" + ] + ) + .then_some("GEO") +} + pub(crate) fn native_limit_setting(path: &[String]) -> Option<&'static str> { let segments: Vec<&str> = path.iter().map(String::as_str).collect(); match segments.as_slice() { diff --git a/crates/edgezero-adapter-axum/src/service.rs b/crates/edgezero-adapter-axum/src/service.rs index c2e32737a..340c6da4b 100644 --- a/crates/edgezero-adapter-axum/src/service.rs +++ b/crates/edgezero-adapter-axum/src/service.rs @@ -493,7 +493,14 @@ impl AxumServiceState { .await; } }; - let geo = self.geo.resolve(remote_addr, &parts.headers); + let geo = self + .geo + .resolve_request( + remote_addr, + &parts.headers, + parts.extensions.get::(), + ) + .await; Ok(self .dispatch_prepared(prepared, parts, native_body, connection, guard, geo) .await) diff --git a/crates/edgezero-adapter-axum/tests/fixtures/geo_artifact.rs b/crates/edgezero-adapter-axum/tests/fixtures/geo_artifact.rs new file mode 100644 index 000000000..ac7b825e5 --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/fixtures/geo_artifact.rs @@ -0,0 +1,54 @@ +//! Files as exact-key KV fixtures for the managed listener test, not a backend. + +use std::fs; +use std::io::ErrorKind; +use std::path::PathBuf; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use edgezero_core::geo::kv::{digest, hex}; +use edgezero_core::key_value_store::{KvError, KvPage, KvStore}; + +pub static READS: AtomicUsize = AtomicUsize::new(0); + +pub struct Artifact(pub PathBuf); + +impl Artifact { + pub fn path(&self, key: &str) -> PathBuf { + self.0.join(hex(&digest(key.as_bytes()))) + } +} + +#[expect( + clippy::missing_trait_methods, + reason = "Geo fixture needs no record operations" +)] +#[async_trait(?Send)] +impl KvStore for Artifact { + async fn delete(&self, _: &str) -> Result<(), KvError> { + Err(KvError::Unavailable) + } + + async fn get_bytes(&self, key: &str) -> Result, KvError> { + READS.fetch_add(1, Ordering::Relaxed); + match fs::read(self.path(key)) { + Ok(bytes) => Ok(Some(bytes.into())), + Err(error) if error.kind() == ErrorKind::NotFound => Ok(None), + Err(_error) => Err(KvError::Unavailable), + } + } + + async fn list_keys_page(&self, _: &str, _: Option<&str>, _: usize) -> Result { + Err(KvError::Unavailable) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + fs::write(self.path(key), &value).map_err(|_error| KvError::Unavailable) + } + + async fn put_bytes_with_ttl(&self, _: &str, _: Bytes, _: Duration) -> Result<(), KvError> { + Err(KvError::Unavailable) + } +} diff --git a/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs b/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs index 8ae942637..8fb47c59d 100644 --- a/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs +++ b/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs @@ -3,6 +3,8 @@ #[path = "../../../edgezero-core/tests/fixtures/production_handlers.rs"] mod fixture; +#[cfg(feature = "geo-kv")] +mod geo_artifact; use std::cell::Cell; use std::env; @@ -38,6 +40,7 @@ use edgezero_core::ingress::{ AdmissionDecision, CheckedEffectiveHost, CheckedHostSource, IngressGrant, }; use edgezero_core::key_value_store::KvHandle; +use edgezero_core::manifest::{BakedManifest, Manifest}; use edgezero_core::middleware::{Middleware, Next}; use edgezero_core::params::PathParams; use edgezero_core::probe::{self, LifecyclePhase, LifecycleReader}; @@ -270,6 +273,9 @@ impl Hooks for FixtureApp { println!("HOOK_ENTERED"); if mode().starts_with("geo-") { app.set_ingress_admission_policy(|head| { + if head.target().path() == "/geo-abort" { + return AdmissionDecision::Abort; + } assert!( !head.headers().contains_key("x-origin-verify-fixture"), "origin credential removed before admission" @@ -295,6 +301,17 @@ impl Hooks for FixtureApp { Ok(()) } + /// `geo-manifest` bakes `CloudFront` geo with the credential header; the ALB + /// networks come from `EDGEZERO__ADAPTER__GEO__ALB_PEER_CIDRS`. + fn manifest() -> BakedManifest { + if mode().starts_with("geo-manifest") { + return Manifest::from_baked_json( + r#"{"adapters":{"axum":{"geo":{"provider":"cloudfront","origin_credential_header_name":"x-origin-verify-fixture"}}}}"#, + ); + } + BakedManifest::Absent + } + fn owns_logging() -> bool { true } @@ -343,6 +360,15 @@ impl Hooks for FixtureApp { } fn stores() -> StoresMetadata { + if mode() == "geo-kv" { + return StoresMetadata { + kv: Some(StoreMetadata { + ids: &["fixture_geo"], + default: "fixture_geo", + }), + ..StoresMetadata::default() + }; + } if mode().starts_with("geo-") { return StoresMetadata::default(); } @@ -925,6 +951,9 @@ fn geo_json(outcome: &GeoOutcome) -> Value { json!({"provider": provider.as_str(), "status": match reason { GeoUnavailableReason::MissingMetadata => "missing_metadata", GeoUnavailableReason::UntrustedIngress => "untrusted_ingress", + GeoUnavailableReason::MissingReaderIp => "missing_reader_ip", + GeoUnavailableReason::NoMatch => "no_match", + GeoUnavailableReason::UnsupportedGeography => "unsupported_geography", }}) } GeoStatus::Error(error) => { @@ -977,8 +1006,13 @@ async fn geo_observation(mut ctx: RequestContext) -> Result .headers() .get("cloudfront-viewer-country") .and_then(|value| value.to_str().ok()); + #[cfg(feature = "geo-kv")] + let kv_reads = geo_artifact::READS.load(Ordering::Relaxed); + #[cfg(not(feature = "geo-kv"))] + let kv_reads = 0_usize; json_response(&json!({ "geo": geo_json(ctx.geo()), + "kv_reads": kv_reads, "credential_absent": credential_absent, "immutable": ctx.geo() == &original, "mutated_country": mutated_country, @@ -992,6 +1026,10 @@ async fn geo_observation(mut ctx: RequestContext) -> Result reason = "test-only initializer asserts root-free startup, capability capture and known-valid forged state" )] fn run_geo_fixture(selected: &str) -> anyhow::Result<()> { + if selected == "geo-manifest-dev" { + // The dev runner ignores the manifest table; only settings select geo. + return run_app::(); + } let address: SocketAddr = env::var("FIXTURE_BIND")?.parse()?; let peer_network = if selected.ends_with("untrusted") { "192.0.2.1/32" @@ -1017,9 +1055,17 @@ fn run_geo_fixture(selected: &str) -> anyhow::Result<()> { } else { NativeGeoSelection::CloudFront { policy } }; - let options = AxumRunOptions::new(address)?.with_geo(selection); - // Capture from the same finalized options, without bypassing the runner's validation. - let capability = options.geo_capability(); + let (options, capability) = if selected == "geo-manifest" { + // No `with_geo`: the baked manifest and EDGEZERO__ settings select geo. + let options = AxumRunOptions::from_env()?; + let capability = options.geo_capability_for::(); + (options, capability) + } else { + let options = AxumRunOptions::new(address)?.with_geo(selection); + // Capture from the same finalized options, without bypassing the runner's validation. + let capability = options.geo_capability(); + (options, capability) + }; let is_off = selected == "geo-off"; let (sender, receiver) = oneshot::channel(); run_app_with_options_and_initializer::( @@ -1056,6 +1102,35 @@ fn run_geo_fixture(selected: &str) -> anyhow::Result<()> { ) } +#[cfg(feature = "geo-kv")] +fn run_kv_geo_fixture() -> anyhow::Result<()> { + use edgezero_adapter_axum::dev_server::run_app_with_native_stores; + use edgezero_adapter_axum::native_stores::NativeStoreOverrides; + let options = AxumRunOptions::from_env()?; + let mut overrides = NativeStoreOverrides::default(); + if let Some(root) = env::var_os("FIXTURE_GEO_ARTIFACT") { + overrides.insert_kv( + "fixture_geo", + KvHandle::new(Arc::new(geo_artifact::Artifact(root.into()))), + )?; + } + let (sender, receiver) = oneshot::channel(); + run_app_with_native_stores::( + options, + overrides, + async move { + let _closed = receiver.await; + }, + move |app, _stores| { + Box::pin(async move { + app.insert_state(Arc::new(StopState(Mutex::new(Some(sender))))); + println!("INITIALIZED"); + Ok(()) + }) + }, + ) +} + fn run_explicit_initializer_fixture() -> anyhow::Result<()> { let options = AxumRunOptions::new(env::var("FIXTURE_BIND")?.parse()?)? .with_config_dir(env::var("FIXTURE_CONFIG_ROOT")?)? @@ -1089,12 +1164,18 @@ fn run_explicit_initializer_fixture() -> anyhow::Result<()> { clippy::panic_in_result_fn, reason = "test-only executable asserts initializer ownership and call counts" )] +#[expect( + clippy::too_many_lines, + reason = "entrypoint dispatches independent fixture scenarios" +)] fn main() -> anyhow::Result<()> { let selected = mode(); if selected.starts_with("diagnostics") { return run_native_fixture(&selected); } match selected.as_str() { + #[cfg(feature = "geo-kv")] + "geo-kv" => return run_kv_geo_fixture(), geo_mode if geo_mode.starts_with("geo-") => return run_geo_fixture(geo_mode), "development" | "bare-dev" => return run_app::(), "framework" | "bare" | "hook-error" => return run_production_app::(), diff --git a/crates/edgezero-adapter-axum/tests/production_hosting.rs b/crates/edgezero-adapter-axum/tests/production_hosting.rs index c74a86c0f..114217d28 100644 --- a/crates/edgezero-adapter-axum/tests/production_hosting.rs +++ b/crates/edgezero-adapter-axum/tests/production_hosting.rs @@ -1,6 +1,10 @@ //! Real local processes, public entrypoints and ordinary main termination output. #![cfg(all(feature = "axum", feature = "test-utils", unix))] +#[cfg(all(test, feature = "geo-kv"))] +#[path = "fixtures/geo_artifact.rs"] +mod geo_artifact; + #[cfg(test)] #[path = "fixtures/native_proxy.rs"] mod native_proxy; @@ -2250,6 +2254,121 @@ mod tests { value } + #[cfg(feature = "geo-kv")] + fn kv_geo_command(cwd: &TempDir, addr: SocketAddr, pin: &str, mode: &str) -> Command { + let mut command = geo_command("geo-kv", cwd, addr); + command + .env("EDGEZERO__ADAPTER__HOST", addr.ip().to_string()) + .env("EDGEZERO__ADAPTER__PORT", addr.port().to_string()) + .env("EDGEZERO__ADAPTER__GEO__PROVIDER", "kv") + .env("EDGEZERO__ADAPTER__GEO__STORE_ID", "fixture_geo") + .env("EDGEZERO__ADAPTER__GEO__READER_IP", mode) + .env("EDGEZERO__ADAPTER__GEO__PIN", pin) + .env("EDGEZERO__ADAPTER__GEO__LOOKUP_TIMEOUT_MS", "200") + .env("EDGEZERO__ADAPTER__GEO__MAX_IN_FLIGHT", "4") + .env("EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS", "127.0.0.1/32") + .env("EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY", "forwarded"); + command + } + + #[test] + #[cfg(feature = "geo-kv")] + fn kv_geo_listener_checks_reader_provenance_and_pinned_objects() { + use super::geo_artifact::Artifact; + use edgezero_core::geo::kv::{Location, Range, Source, build, hex, node_key}; + use edgezero_core::key_value_store::KvHandle; + use futures::executor::block_on; + use std::net::Ipv6Addr; + + let artifact_dir = tempfile::tempdir().expect("artifact"); + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let artifact = Arc::new(Artifact(artifact_dir.path().to_owned())); + let handle = KvHandle::new(Arc::::clone(&artifact)); + let start = u128::from(u32::from(Ipv4Addr::new(192, 0, 2, 0))); + let v4: Vec<_> = (0..120) + .map(|i| Range { + start: start + i * 2, + end: start + i * 2, + location: if i == 1 { + Location::new("XK", "").expect("source country") + } else { + Location::new("US", "CA").expect("source country and subdivision") + }, + }) + .collect(); + let ip6: Ipv6Addr = "2001:db8::".parse().expect("IPv6"); + let v6 = [Range { + start: u128::from(ip6), + end: u128::from(ip6) + 1, + location: Location::new("GB", "").expect("country only"), + }]; + let (receipt, manifest, inventory) = block_on(build( + &handle, + Source { + edition: "fixture".into(), + files: vec![], + }, + &v4, + &v6, + 1024, + )) + .expect("fixture dataset"); + let pin = format!("{}@{}", manifest.generation, hex(&receipt.manifest_hash)); + let addr = address(); + let mut command = kv_geo_command(&cwd, addr, &pin, "forwarded-required"); + command.env("FIXTURE_GEO_ARTIFACT", artifact_dir.path()); + let mut server = Server::spawn(command, addr); + server.ready(); + + let missing = geo_response(&server, "CloudFront-Viewer-Country: US\r\n"); + assert_eq!(missing["geo"]["status"], "missing_reader_ip"); + assert_eq!(missing["kv_reads"], 3_u64); + // A trusted forwarded client needs no forwarded host or scheme. + let ipv4 = geo_response(&server, "Forwarded: for=192.0.2.0\r\n"); + assert_eq!(ipv4["geo"]["subdivision"], "CA"); + assert_eq!(ipv4["geo"]["trust_basis"], "CheckedForwardedClient"); + let mapped = geo_response(&server, "Forwarded: for=\"[::ffff:192.0.2.2]\"\r\n"); + assert_eq!(mapped["geo"]["status"], "unsupported_geography"); + let gap = geo_response(&server, "Forwarded: for=192.0.2.1\r\n"); + assert_eq!(gap["geo"]["status"], "no_match"); + let country_only = geo_response(&server, "Forwarded: for=\"[2001:db8::1]\"\r\n"); + assert_eq!(country_only["geo"]["country"], "GB"); + assert!(country_only["geo"]["subdivision"].is_null()); + let rejected = raw_request( + addr, + "GET /geo-abort HTTP/1.1\r\nHost: fixture\r\nForwarded: for=192.0.2.238\r\nConnection: close\r\n\r\n", + ); + assert!(rejected.is_empty()); + let untrusted = geo_response(&server, "Forwarded: for=unknown\r\n"); + assert_eq!(untrusted["geo"]["status"], "missing_reader_ip"); + assert_eq!(untrusted["kv_reads"], country_only["kv_reads"]); + + let v4_root = node_key(&manifest.generation, 4, &manifest.ipv4.expect("root").hash); + // Last IPv4 object before its root is the second leaf, not read by earlier requests. + let mut v4_entries = inventory + .iter() + .filter(|entry| entry.key.contains("/v4/")) + .rev(); + assert_eq!(v4_entries.next().expect("root").key, v4_root); + fs::remove_file(artifact.path(&v4_entries.next().expect("second leaf").key)) + .expect("remove leaf"); + let failed = geo_response(&server, "Forwarded: for=192.0.2.238\r\n"); + assert_eq!(failed["geo"]["status"], "provider_failure"); + finish_geo(&mut server, &cwd); + + fs::remove_file(artifact.path(&v4_root)).expect("remove root"); + let failed_addr = address(); + let mut failed_command = kv_geo_command(&cwd, failed_addr, &pin, "direct-peer"); + failed_command.env("FIXTURE_GEO_ARTIFACT", artifact_dir.path()); + let mut failed_startup = Server::spawn(failed_command, failed_addr); + assert!(!failed_startup.wait().success()); + let log = failed_startup.log(); + assert!(log.contains("pinned KV data could not be prepared")); + assert!(!log.contains("INITIALIZED")); + TcpStream::connect_timeout(&failed_addr, Duration::from_millis(50)) + .expect_err("invalid root never binds"); + } + fn finish_geo(server: &mut Server, cwd: &TempDir) { assert!( server @@ -2269,6 +2388,73 @@ mod tests { ); } + #[test] + #[cfg(feature = "geo-cloudfront")] + fn geo_hosting_binds_cloudfront_from_manifest_and_settings() { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let addr = address(); + let mut command = geo_command("geo-manifest", &cwd, addr); + command + .env("EDGEZERO__ADAPTER__HOST", addr.ip().to_string()) + .env("EDGEZERO__ADAPTER__PORT", addr.port().to_string()) + .env("EDGEZERO__ADAPTER__GEO__ALB_PEER_CIDRS", "127.0.0.1/32"); + let mut server = Server::spawn(command, addr); + server.ready(); + let value = geo_response( + &server, + "CloudFront-Viewer-Country: us\r\nCloudFront-Viewer-Country-Region: ca\r\nx-origin-verify-fixture: ORIGIN_CREDENTIAL_SENTINEL\r\n", + ); + assert_eq!(value["geo"]["status"], "located"); + assert_eq!(value["geo"]["provider"], "cloudfront"); + assert_eq!(value["geo"]["country"], "US"); + assert_eq!(value["geo"]["subdivision"], "CA"); + assert_eq!(value["geo"]["trust_basis"], "DeclaredProtectedAlb"); + finish_geo(&mut server, &cwd); + } + + #[test] + fn geo_dev_runner_ignores_manifest_and_reads_settings() { + let cwd = tempfile::tempdir().expect("unrelated cwd"); + let dev = |settings: &[(&str, &str)]| { + let addr = address(); + let mut command = geo_command("geo-manifest-dev", &cwd, addr); + command + .env("EDGEZERO__ADAPTER__HOST", addr.ip().to_string()) + .env("EDGEZERO__ADAPTER__PORT", addr.port().to_string()); + for (key, value) in settings { + command.env(key, value); + } + let mut server = Server::spawn(command, addr); + server.ready(); + server + }; + let request = "CloudFront-Viewer-Country: US\r\nCloudFront-Viewer-Country-Region: CA\r\n"; + + // The baked table lacks networks; the dev server still starts, with geo off. + let unselected = dev(&[]); + let value = geo_response(&unselected, request); + assert_eq!(value["geo"]["status"], "no_selection"); + drop(unselected); + + // Settings alone opt a dev run in. Without the built-in this fails startup instead. + #[cfg(feature = "geo-cloudfront")] + { + let selected = dev(&[ + ("EDGEZERO__ADAPTER__GEO__PROVIDER", "cloudfront"), + ("EDGEZERO__ADAPTER__GEO__ALB_PEER_CIDRS", "127.0.0.1/32"), + ( + "EDGEZERO__ADAPTER__GEO__ORIGIN_CREDENTIAL_HEADER_NAME", + "x-origin-verify-fixture", + ), + ]); + let located = geo_response(&selected, request); + assert_eq!(located["geo"]["status"], "located"); + assert_eq!(located["geo"]["provider"], "cloudfront"); + assert_eq!(located["geo"]["country"], "US"); + assert_eq!(located["geo"]["subdivision"], "CA"); + } + } + #[test] #[cfg(not(feature = "geo-cloudfront"))] fn geo_hosting_rejects_unavailable_cloudfront() { diff --git a/crates/edgezero-adapter-cloudflare/src/request.rs b/crates/edgezero-adapter-cloudflare/src/request.rs index 869225cd1..339ba55c4 100644 --- a/crates/edgezero-adapter-cloudflare/src/request.rs +++ b/crates/edgezero-adapter-cloudflare/src/request.rs @@ -11,12 +11,13 @@ use std::{io, task::Poll}; #[cfg(feature = "test-utils")] use bytes::Bytes; use edgezero_core::OutboundScopeOwner; -use edgezero_core::app::{App, StoreMetadata}; +use edgezero_core::app::{App, FrameworkErrorDetail, StoreMetadata}; use edgezero_core::body::Body; use edgezero_core::config_store::{ConfigStoreHandle, ConfigStoreOpenFailure}; use edgezero_core::context::RuntimeVariables; use edgezero_core::env_config::EnvConfig; use edgezero_core::error::EdgeError; +use edgezero_core::geo::{GeoOutcome, GeoSource}; use edgezero_core::http::{HeaderValue, Method as CoreMethod, Request, Uri, request_builder}; use edgezero_core::ingress::{ CapturedTarget, HeaderFidelity, InboundOrigin, IngressBeginOutcome, IngressDispatchOutcome, @@ -41,7 +42,8 @@ use futures_util::stream::poll_fn; use futures_util::stream::{LocalBoxStream, unfold}; use futures_util::{StreamExt as _, TryStreamExt as _}; use worker::{ - Context, Delay, Env, Error as WorkerError, Method, Request as CfRequest, Response as CfResponse, + Cf, Context, Delay, Env, Error as WorkerError, Method, Request as CfRequest, + Response as CfResponse, }; use crate::config_store::CloudflareConfigStore; @@ -506,6 +508,7 @@ where core_request, Stores::default(), request_start, + GeoOutcome::NoSelection, move || Ok(Some(source)), Ok, )) @@ -540,6 +543,7 @@ where core_request, Stores::default(), request_start, + GeoOutcome::NoSelection, move || make_source().map(Some), deliver, )) @@ -566,6 +570,7 @@ pub async fn dispatch_ingress_source_error_for_test( core_request, Stores::default(), request_start, + GeoOutcome::NoSelection, || -> Result>>, EdgeError> { Err(EdgeError::internal(anyhow::anyhow!( "cloudflare source construction failed" @@ -599,6 +604,7 @@ pub(crate) async fn dispatch_with_handles( CoreMethod::from_bytes(req.inner().method().as_bytes()).unwrap_or(CoreMethod::GET); let context = Arc::new(ctx); let runtime = CloudflareEgressRuntime::new(signal, Arc::clone(&context)); + let geo = request_geo(&req); let head_request = match into_core_request_head(&req, env, context, app.monotonic_clock()) { Ok(request) => request, Err(error) => { @@ -613,6 +619,7 @@ pub(crate) async fn dispatch_with_handles( head_request, stores, request_start, + geo, move || cloudflare_body_stream(req), move |egress| from_egress_response(egress, runtime), ) @@ -624,6 +631,7 @@ async fn dispatch_ingress_stream_and Result @@ -679,7 +687,17 @@ where *head_request.body_mut() = source.map_or_else(Body::empty, |stream| { cloudflare_deadline_body(stream, prepared.read_deadline(), prepared.monotonic_clock()) }); - dispatch_core_request(app, head_request, stores, prepared, deliver).await + dispatch_core_request(app, head_request, stores, prepared, geo, deliver).await +} + +/// Cloudflare's own geolocation from the request's `cf` metadata. +fn request_geo(req: &CfRequest) -> GeoOutcome { + let cf = req.cf(); + GeoOutcome::platform( + GeoSource::Cloudflare, + cf.and_then(Cf::country).as_deref(), + cf.and_then(Cf::region_code).as_deref(), + ) } /// Dispatch with per-id store registries built from baked metadata. @@ -827,6 +845,7 @@ async fn dispatch_core_request( mut core_request: Request, mut stores: Stores, prepared: PreparedIngress, + geo: GeoOutcome, deliver: Deliver, ) -> Result where @@ -861,7 +880,13 @@ where if let Some(registry) = secret_registry { core_request.extensions_mut().insert(registry); } - let response = Box::pin(app.dispatch_admitted(prepared, core_request)).await; + let response = Box::pin(app.dispatch_admitted_with_geo( + prepared, + core_request, + FrameworkErrorDetail::Default, + geo, + )) + .await; deliver(response).map_err(|err| edge_error_to_worker(&err)) } @@ -1010,6 +1035,7 @@ mod tests { use async_stream::stream; use edgezero_core::app::Hooks; use edgezero_core::context::{BodyKind, RequestContext}; + use edgezero_core::geo::GeoStatus; use edgezero_core::http::{Response, StatusCode, response_builder}; use edgezero_core::ingress::{AdmissionDecision, IngressGrant}; #[cfg(feature = "test-utils")] @@ -1887,6 +1913,49 @@ mod tests { assert!(RETAINED_UPLOAD.with(|upload_ref| upload_ref.borrow().upgrade().is_none())); } + /// Rebuilds a request whose JS object carries the given incoming `cf` metadata. + fn request_with_cf(fields: &[(&str, &str)]) -> CfRequest { + let inner = Clone::clone(native_request(Method::Get, "/", None).inner()); + let cf = Object::new(); + for (key, value) in fields { + Reflect::set(&cf, &JsValue::from_str(key), &JsValue::from_str(value)) + .expect("cf field"); + } + Reflect::set(&inner, &JsValue::from_str("cf"), &cf).expect("cf"); + CfRequest::from(inner) + } + + #[wasm_bindgen_test] + fn cf_metadata_becomes_cloudflare_geo() { + let platform = + |country, region| GeoOutcome::platform(GeoSource::Cloudflare, country, region); + let us_ca = request_with_cf(&[("country", "US"), ("regionCode", "CA")]); + assert_eq!(request_geo(&us_ca), platform(Some("US"), Some("CA"))); + assert!(matches!( + request_geo(&us_ca), + GeoOutcome::Selected { + status: GeoStatus::Located { .. }, + .. + } + )); + let country_only = request_with_cf(&[("country", "GB")]); + assert_eq!(request_geo(&country_only), platform(Some("GB"), None)); + // Cloudflare reports unknown and Tor traffic with placeholder countries. + let tor = request_with_cf(&[("country", "T1")]); + assert_eq!(request_geo(&tor), platform(Some("T1"), None)); + assert!(matches!( + request_geo(&tor), + GeoOutcome::Selected { + status: GeoStatus::Unavailable(_), + .. + } + )); + assert_eq!( + request_geo(&native_request(Method::Get, "/", None)), + platform(None, None) + ); + } + fn native_request(method: Method, path: &str, body: Option<&[u8]>) -> CfRequest { let mut init = worker::RequestInit::new(); init.with_method(method); @@ -1926,6 +1995,7 @@ mod tests { head, Stores::default(), app.monotonic_now(), + GeoOutcome::NoSelection, move || cloudflare_body_stream(req), Ok, ) diff --git a/crates/edgezero-adapter-cloudflare/tests/contract.rs b/crates/edgezero-adapter-cloudflare/tests/contract.rs index 48786e7a3..b41d76a3d 100644 --- a/crates/edgezero-adapter-cloudflare/tests/contract.rs +++ b/crates/edgezero-adapter-cloudflare/tests/contract.rs @@ -59,7 +59,7 @@ mod tests { }; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; - use edgezero_core::geo::{GeoError, GeoOutcome, GeoProviderId, GeoStatus}; + use edgezero_core::geo::{GeoError, GeoOutcome, GeoProviderId, GeoSource, GeoStatus}; use edgezero_core::http::{Method, Response, StatusCode, response_builder}; use edgezero_core::outbound::{ HttpClient, OutboundCollectedSlot, OutboundRequest, OutboundSlotResult, OutboundTime, @@ -216,9 +216,15 @@ mod tests { } } + /// Requests built in the test runtime carry no `cf` object, so Cloudflare geo is + /// selected but unavailable. + fn platform_geo() -> GeoOutcome { + GeoOutcome::platform(GeoSource::Cloudflare, None, None) + } + fn build_test_app() -> App { async fn capture_uri(ctx: RequestContext) -> Result { - assert_eq!(ctx.geo(), &GeoOutcome::NoSelection); + assert_eq!(ctx.geo(), &platform_geo()); assert_eq!(ctx.extensions().get::(), Some(&forged_geo())); let body = Body::text(ctx.uri().to_string()); let response = response_builder() @@ -408,6 +414,46 @@ mod tests { assert_eq!(bytes.as_slice(), b"echo"); } + #[wasm_bindgen_test] + async fn dispatch_exposes_cf_metadata_through_ctx_geo() { + async fn geo(ctx: RequestContext) -> Result { + let body = Body::text(format!("{:?}", ctx.geo())); + response_builder() + .status(StatusCode::OK) + .body(body) + .map_err(EdgeError::internal) + } + let app = App::new(RouterService::builder().get("/geo", geo).build()); + let inner = Clone::clone(cf_request(CfMethod::Get, "/geo", None).inner()); + let cf = Object::new(); + for (key, value) in [("country", "US"), ("regionCode", "CA")] { + Reflect::set(&cf, &JsValue::from_str(key), &JsValue::from_str(value)) + .expect("cf field"); + } + Reflect::set(&inner, &JsValue::from_str("cf"), &cf).expect("cf"); + let (env, ctx, _timers) = test_env_ctx(); + + let mut response = CloudflareService::new(&app) + .dispatch(CfRequest::from(inner), env, ctx) + .await + .expect("cf response"); + + assert_eq!(response.status_code(), StatusCode::OK.as_u16()); + let expected = GeoOutcome::platform(GeoSource::Cloudflare, Some("US"), Some("CA")); + assert!(matches!( + &expected, + GeoOutcome::Selected { + status: GeoStatus::Located { .. }, + .. + } + )); + assert_eq!( + response.text().await.expect("text"), + format!("{expected:?}"), + "cf metadata wins over the request's CloudFront headers" + ); + } + #[wasm_bindgen_test] async fn dispatch_runs_router_and_returns_response() { let app = build_test_app(); diff --git a/crates/edgezero-adapter-fastly/src/request.rs b/crates/edgezero-adapter-fastly/src/request.rs index 454728e3f..51a89daa0 100644 --- a/crates/edgezero-adapter-fastly/src/request.rs +++ b/crates/edgezero-adapter-fastly/src/request.rs @@ -3,15 +3,17 @@ use std::fmt::Display; #[cfg(all(feature = "test-utils", target_arch = "wasm32"))] use std::io::Cursor; use std::io::Read; +use std::net::IpAddr; use std::str::from_utf8; use std::sync::{Arc, Mutex, OnceLock, PoisonError}; use std::task::Poll; use bytes::Bytes; -use edgezero_core::app::{App, StoreMetadata, StoresMetadata}; +use edgezero_core::app::{App, FrameworkErrorDetail, StoreMetadata, StoresMetadata}; use edgezero_core::body::Body; use edgezero_core::config_store::{ConfigStoreHandle, ConfigStoreOpenFailure}; use edgezero_core::error::EdgeError; +use edgezero_core::geo::{GeoOutcome, GeoSource}; use edgezero_core::http::header::HOST; use edgezero_core::http::{ Extensions, HeaderValue, Method, Request, Response, Uri, request_builder, @@ -29,6 +31,7 @@ use edgezero_core::store_registry::{ BoundSecretStore, ConfigRegistry, ConfigStoreBinding, KvRegistry, SecretRegistry, StoreRegistry, }; use edgezero_core::time::{Deadline, MonotonicClock, MonotonicInstant}; +use fastly::geo::{Geo, geo_lookup}; use fastly::{Error as FastlyError, Request as FastlyRequest}; use futures::executor; use futures_util::stream; @@ -406,6 +409,7 @@ fn dispatch_core_request( mut core_request: Request, stores: Stores, prepared: PreparedIngress, + geo: GeoOutcome, ) -> DispatchOutcome { // Hard-cutoff: legacy bare handles are no longer // inserted into request extensions. `with_config_handle` @@ -440,7 +444,12 @@ fn dispatch_core_request( let entered = prepared.request_start(); let method = prepared.request_method().clone(); match executor::block_on(drive_callback( - Box::pin(app.dispatch_admitted(prepared, core_request)), + Box::pin(app.dispatch_admitted_with_geo( + prepared, + core_request, + FrameworkErrorDetail::Default, + geo, + )), clock, entered, )) { @@ -479,6 +488,7 @@ where Prepare: FnOnce(&mut FastlyRequest, &mut Extensions), { let ingress = capture_request_ingress(&req); + let geo = request_geo(&req); let mut scratch = apply_request_prepare(&mut req, prepare); scratch.remove::(); let synthesized_headers = scratch.remove::(); @@ -490,9 +500,10 @@ where ) })?; head_request.extensions_mut().extend(scratch); - let outcome = dispatch_ingress_reader(app, head_request, stores, request_start, move || { - Ok(req.try_take_body()) - }); + let outcome = + dispatch_ingress_reader(app, head_request, stores, request_start, geo, move || { + Ok(req.try_take_body()) + }); Ok(match (outcome, synthesized_headers) { (DispatchOutcome::Detached(envelope), Some(headers)) => { DispatchOutcome::Detached(envelope.with_synthesized_response_headers(headers)) @@ -509,6 +520,7 @@ fn dispatch_ingress_reader( mut head_request: Request, stores: Stores, request_start: MonotonicInstant, + geo: GeoOutcome, make_source: MakeSource, ) -> DispatchOutcome where @@ -554,7 +566,22 @@ where source, Some((prepared.read_deadline(), prepared.monotonic_clock())), ); - dispatch_core_request(app, head_request, stores, prepared) + dispatch_core_request(app, head_request, stores, prepared, geo) +} + +/// Fastly's own geolocation for the downstream client address. `Geo::region` is +/// already the subdivision part of the ISO 3166-2 code (`CA`, not `US-CA`). +fn request_geo(req: &FastlyRequest) -> GeoOutcome { + client_geo(req.get_client_ip_addr()) +} + +fn client_geo(client_ip: Option) -> GeoOutcome { + let geo = client_ip.and_then(geo_lookup); + GeoOutcome::platform( + GeoSource::Fastly, + geo.as_ref().map(Geo::country_code), + geo.as_ref().and_then(Geo::region), + ) } /// Dispatch with per-id store registries built from baked metadata and closed lifecycle hooks. @@ -1015,6 +1042,7 @@ where core_request, Stores::default(), request_start, + GeoOutcome::NoSelection, move || Ok(Some(source)), ) .into_envelope() @@ -1050,6 +1078,7 @@ where core_request, Stores::default(), request_start, + GeoOutcome::NoSelection, move || make_source().map(Some), ); match outcome { @@ -1083,6 +1112,7 @@ pub fn dispatch_ingress_source_error_for_test( core_request, Stores::default(), request_start, + GeoOutcome::NoSelection, || -> Result>>, EdgeError> { Err(EdgeError::internal(anyhow::anyhow!( "fastly source construction failed" @@ -1761,6 +1791,49 @@ mod synthesis_tests { fn resolve_secret_handle_builds_handle_when_required_true_matches_require_secrets() { let _handle = resolve_secret_handle(true); } + + #[cfg(target_arch = "wasm32")] + #[test] + fn client_geo_reads_fastly_country_and_region() { + use edgezero_core::geo::{ + GeoOutcome, GeoSource, GeoStatus, GeoUnavailableReason, SubdivisionCode, + }; + + // Viceroy answers loopback lookups with its default San Francisco record. + let loopback = client_geo(Some("127.0.0.1".parse().unwrap())); + assert_eq!( + loopback, + GeoOutcome::platform(GeoSource::Fastly, Some("US"), Some("CA")) + ); + let GeoOutcome::Selected { + provider, + status: GeoStatus::Located { location, .. }, + } = loopback + else { + panic!("loopback must be located: {loopback:?}"); + }; + assert_eq!(provider.as_str(), "fastly"); + assert_eq!(location.country().as_str(), "US"); + assert_eq!( + location.subdivision().map(SubdivisionCode::as_str), + Some("CA") + ); + + let unavailable = GeoOutcome::platform(GeoSource::Fastly, None, None); + assert!(matches!( + &unavailable, + GeoOutcome::Selected { + status: GeoStatus::Unavailable(GeoUnavailableReason::MissingMetadata), + .. + } + )); + assert_eq!(client_geo(None), unavailable, "no client address"); + assert_eq!( + client_geo(Some("192.0.2.1".parse().unwrap())), + unavailable, + "address without geolocation data" + ); + } } #[cfg(test)] diff --git a/crates/edgezero-adapter-fastly/tests/contract.rs b/crates/edgezero-adapter-fastly/tests/contract.rs index 77969805d..6184e4130 100644 --- a/crates/edgezero-adapter-fastly/tests/contract.rs +++ b/crates/edgezero-adapter-fastly/tests/contract.rs @@ -42,7 +42,7 @@ mod tests { }; use edgezero_core::context::{BodyKind, RequestContext}; use edgezero_core::error::EdgeError; - use edgezero_core::geo::{GeoError, GeoOutcome, GeoProviderId, GeoStatus}; + use edgezero_core::geo::{GeoError, GeoOutcome, GeoProviderId, GeoSource, GeoStatus}; use edgezero_core::http::{HeaderMap, Method, Response, StatusCode, response_builder}; use edgezero_core::ingress::{AdmissionDecision, IngressGrant}; use edgezero_core::outbound::{ @@ -148,9 +148,14 @@ mod tests { } } + /// Test requests have no downstream client address, so Fastly geo is selected but unavailable. + fn platform_geo() -> GeoOutcome { + GeoOutcome::platform(GeoSource::Fastly, None, None) + } + fn build_test_app() -> App { async fn capture_uri(ctx: RequestContext) -> Result { - assert_eq!(ctx.geo(), &GeoOutcome::NoSelection); + assert_eq!(ctx.geo(), &platform_geo()); assert_eq!(ctx.extensions().get::(), Some(&forged_geo())); let body = Body::text(ctx.uri().to_string()); let response = response_builder() @@ -604,7 +609,7 @@ mod tests { let request_events = Arc::clone(&handler_events); async move { request_events.lock().expect("events").push("handler"); - assert_eq!(ctx.geo(), &GeoOutcome::NoSelection); + assert_eq!(ctx.geo(), &platform_geo()); assert_eq!(ctx.extensions().get::(), Some(&forged_geo())); let value = ctx .extensions() diff --git a/crates/edgezero-core/Cargo.toml b/crates/edgezero-core/Cargo.toml index db650b349..d5fadb179 100644 --- a/crates/edgezero-core/Cargo.toml +++ b/crates/edgezero-core/Cargo.toml @@ -57,6 +57,8 @@ simd-adler32 = { workspace = true } web-time = { workspace = true } [features] +geo-kv = [] +geo-kv-build = ["geo-kv"] # Exposes `NoopKvStore` for use in downstream adapter and integration tests # that need a `KvHandle` without real storage. Add this feature to your crate's # `[dev-dependencies]` entry for `edgezero-core` to use it. diff --git a/crates/edgezero-core/src/geo.rs b/crates/edgezero-core/src/geo.rs index 6d6007c00..157a57c30 100644 --- a/crates/edgezero-core/src/geo.rs +++ b/crates/edgezero-core/src/geo.rs @@ -25,6 +25,10 @@ mod country_codes; +/// Versioned KV-backed geography index. +#[cfg(feature = "geo-kv")] +pub mod kv; + use std::fmt; /// A bounded validation category. Rejected source values are never retained. @@ -61,10 +65,18 @@ impl GeoProviderId { Self("cloudfront".to_owned()) } + /// Identity reserved for the built-in KV provider. + #[inline] + #[must_use] + pub fn kv() -> Self { + Self("kv".to_owned()) + } + /// Validate an application provider label matching `[a-z][a-z0-9._-]{0,63}`. /// /// # Errors - /// Returns a bounded category for invalid labels or the reserved `cloudfront` ID. + /// Returns a bounded category for invalid labels or the reserved `cloudfront`, `fastly`, + /// `cloudflare`, and `kv` IDs. #[inline] pub fn new(value: &str) -> Result { if value.is_empty() @@ -76,7 +88,7 @@ impl GeoProviderId { { return Err(GeoValueError::InvalidProviderId); } - if value == "cloudfront" { + if matches!(value, "cloudfront" | "fastly" | "cloudflare" | "kv") { return Err(GeoValueError::ReservedProviderId); } Ok(Self(value.to_owned())) @@ -308,21 +320,33 @@ pub trait GeoProvider { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GeoUnavailableReason { MissingMetadata, + MissingReaderIp, + NoMatch, + UnsupportedGeography, UntrustedIngress, } /// The source of a located assertion, not a distribution identity attestation. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum GeoSource { CloudFront, + Cloudflare, + Fastly, + Kv, } /// The declared infrastructure protection used for source admission. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum GeoTrustBasis { + CheckedDirectPeer, + CheckedForwardedClient, /// Operator-declared credential-gated, network-isolated ALB path. /// This does not prove the external listener or network configuration. DeclaredProtectedAlb, + /// Request metadata the hosting platform attaches itself; no client header is read. + PlatformMetadata, } /// Framework-owned status for a selected provider. @@ -350,6 +374,41 @@ pub enum GeoOutcome { }, } +impl GeoOutcome { + /// Outcome for geography an edge platform attaches to the request itself. + /// + /// `subdivision` is the ISO 3166-2 code without its country prefix. Platform + /// metadata is not client-supplied, so a value that fails validation, such as + /// Cloudflare's `XX` (unknown) or `T1` (Tor), means the platform has no usable + /// geography rather than a forged assertion: an absent or unusable country is + /// missing metadata, and an unusable subdivision leaves a country-only result. + #[inline] + #[must_use] + pub fn platform(source: GeoSource, country: Option<&str>, subdivision: Option<&str>) -> Self { + let provider = GeoProviderId( + match source { + GeoSource::CloudFront => "cloudfront", + GeoSource::Cloudflare => "cloudflare", + GeoSource::Fastly => "fastly", + GeoSource::Kv => return Self::NoSelection, + } + .to_owned(), + ); + let status = match country.and_then(|raw| CountryCode::new(raw).ok()) { + None => GeoStatus::Unavailable(GeoUnavailableReason::MissingMetadata), + Some(country_code) => GeoStatus::Located { + location: GeoLocation::new( + country_code, + subdivision.and_then(|raw| SubdivisionCode::new(raw).ok()), + ), + source, + trust_basis: GeoTrustBasis::PlatformMetadata, + }, + }; + Self::Selected { provider, status } + } +} + #[cfg(test)] mod tests { use super::*; @@ -461,11 +520,59 @@ mod tests { Err(GeoValueError::InvalidProviderId) ); } + for value in ["cloudfront", "fastly", "cloudflare", "kv"] { + assert_eq!( + GeoProviderId::new(value), + Err(GeoValueError::ReservedProviderId) + ); + } + assert_eq!(GeoProviderId::cloudfront().as_str(), "cloudfront"); + } + + #[test] + fn platform_outcome_treats_unusable_values_as_missing() { + let status = |outcome: GeoOutcome| match outcome { + GeoOutcome::Selected { provider, status } => { + assert_eq!(provider.as_str(), "fastly"); + status + } + GeoOutcome::NoSelection => panic!("platform geo is always selected"), + }; + let located = |country: &str, subdivision: Option<&str>| GeoStatus::Located { + location: GeoLocation::new( + CountryCode::new(country).unwrap(), + subdivision.map(|value| SubdivisionCode::new(value).unwrap()), + ), + source: GeoSource::Fastly, + trust_basis: GeoTrustBasis::PlatformMetadata, + }; + let platform = |country, subdivision| { + status(GeoOutcome::platform( + GeoSource::Fastly, + country, + subdivision, + )) + }; + assert_eq!(platform(Some("us"), Some("ca")), located("US", Some("CA"))); + assert_eq!(platform(Some("US"), None), located("US", None)); assert_eq!( - GeoProviderId::new("cloudfront"), - Err(GeoValueError::ReservedProviderId) + platform(None, None), + GeoStatus::Unavailable(GeoUnavailableReason::MissingMetadata) ); - assert_eq!(GeoProviderId::cloudfront().as_str(), "cloudfront"); + // Platform placeholders are unavailable geography, never a forged assertion. + for (country, subdivision) in [(Some("XX"), None), (Some("T1"), Some("CA"))] { + assert_eq!( + platform(country, subdivision), + GeoStatus::Unavailable(GeoUnavailableReason::MissingMetadata) + ); + } + assert_eq!(platform(Some("US"), Some("US-CA")), located("US", None)); + let GeoOutcome::Selected { provider, .. } = + GeoOutcome::platform(GeoSource::Cloudflare, None, None) + else { + panic!("selected") + }; + assert_eq!(provider.as_str(), "cloudflare"); } #[test] diff --git a/crates/edgezero-core/src/geo/kv.rs b/crates/edgezero-core/src/geo/kv.rs new file mode 100644 index 000000000..abb62d6cb --- /dev/null +++ b/crates/edgezero-core/src/geo/kv.rs @@ -0,0 +1,1129 @@ +//! Versioned, exact-key KV geography index. +// Keep the codec, writer, cache and reader together in wire-format order. +#![expect( + clippy::arbitrary_source_item_ordering, + reason = "wire-format and lifecycle grouping" +)] +// Header and record lengths are checked before byte addressing. The tree format uses +// network byte order and binary search/byte-budget division operates on bounded counts. +#![expect( + clippy::arithmetic_side_effects, + clippy::big_endian_bytes, + clippy::indexing_slicing, + clippy::integer_division, + clippy::integer_division_remainder_used, + clippy::missing_asserts_for_indexing, + reason = "fixed-width, validated wire-format data and bounded byte budgets" +)] + +use std::{ + collections::{HashMap, VecDeque}, + net::IpAddr, + sync::Mutex, +}; + +use super::{CountryCode, GeoLocation, SubdivisionCode}; +use crate::key_value_store::KvHandle; +use anyhow::{Context as _, Result, bail, ensure}; +use base16ct::lower::{decode as decode_hex, encode_string}; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use sha2::{Digest as _, Sha256}; +use std::str::from_utf8; + +const HEADER: usize = 12; +const MAGIC: &[u8; 4] = b"EZG1"; + +#[must_use] +#[inline] +pub fn digest(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} +#[must_use] +#[inline] +pub fn hex(bytes: &[u8]) -> String { + encode_string(bytes) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Location([u8; 6]); + +impl Location { + /// Normalize a source location without forcing it into core's country allowlist. + /// + /// # Errors + /// Rejects invalid source country or subdivision syntax. + #[inline] + pub fn new(raw_country: &str, raw_subdivision: &str) -> Result { + // Source-format DTO, not GeoLocation. Preserve XK and report the domain mismatch. + ensure!( + raw_country.len() == 2 && raw_country.bytes().all(|byte| byte.is_ascii_alphabetic()), + "invalid source country syntax" + ); + let country = raw_country.to_ascii_uppercase(); + let subdivision = if raw_subdivision.is_empty() { + None + } else { + Some(SubdivisionCode::new(raw_subdivision)?) + }; + let mut bytes = [0; 6]; + bytes[..2].copy_from_slice(country.as_bytes()); + if let Some(code) = subdivision { + let encoded = code.as_str().as_bytes(); + bytes[2] = u8::try_from(encoded.len())?; + bytes[3..3 + encoded.len()].copy_from_slice(encoded); + } + Ok(Self(bytes)) + } + + fn decode(bytes: &[u8]) -> Result { + let raw: [u8; 6] = bytes.try_into()?; + let len = usize::from(raw[2]); + ensure!( + len <= 3 + && raw[..2].iter().all(u8::is_ascii_uppercase) + && raw[3..3 + len] + .iter() + .all(|byte| byte.is_ascii_uppercase() || byte.is_ascii_digit()) + && raw[3 + len..].iter().all(|byte| *byte == 0), + "invalid location encoding" + ); + Ok(Self(raw)) + } + + /// Map a validated source location to core geography without inventing a country. + #[must_use] + #[inline] + pub fn geo_location(self) -> Option { + let country = CountryCode::new(from_utf8(&self.0[..2]).ok()?).ok()?; + let subdivision = if self.0[2] > 0 { + let text = from_utf8(&self.0[3..3 + usize::from(self.0[2])]).ok()?; + Some(SubdivisionCode::new(text).ok()?) + } else { + None + }; + Some(GeoLocation::new(country, subdivision)) + } + + #[must_use] + #[inline] + pub fn core_country_supported(self) -> bool { + self.geo_location().is_some() + } + + #[must_use] + #[inline] + pub fn json(self) -> serde_json::Value { + let country = from_utf8(&self.0[..2]).unwrap_or_default(); + let subdivision = if self.0[2] == 0 { + None + } else { + from_utf8(&self.0[3..3 + usize::from(self.0[2])]).ok() + }; + serde_json::json!({"country": country, "subdivision": subdivision, "core_country_supported": self.core_country_supported()}) + } +} + +#[cfg(feature = "geo-kv-build")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Range { + pub start: u128, + pub end: u128, + pub location: Location, +} + +/// Return the wire width in bytes for an IP address family. +/// +/// # Errors +/// Rejects unknown families. +fn width(family: u8) -> Result { + match family { + 4 => Ok(4), + 6 => Ok(16), + _ => bail!("invalid address family"), + } +} + +#[must_use] +#[inline] +pub fn address(input: IpAddr) -> (u8, u128) { + match input.to_canonical() { + IpAddr::V4(ip) => (4, u32::from(ip).into()), + IpAddr::V6(ip) => (6, u128::from(ip)), + } +} + +fn max_address(family: u8) -> u128 { + if family == 4 { + u32::MAX.into() + } else { + u128::MAX + } +} +fn read_address(bytes: &[u8]) -> u128 { + let mut raw = [0; 16]; + raw[16 - bytes.len()..].copy_from_slice(bytes); + u128::from_be_bytes(raw) +} +#[cfg(feature = "geo-kv-build")] +fn write_address(out: &mut Vec, value: u128, width: usize) { + out.extend_from_slice(&value.to_be_bytes()[16 - width..]); +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct NodeRef { + pub start: u128, + pub end: u128, + pub hash: [u8; 32], + pub level: u16, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Source { + pub edition: String, + pub files: Vec<(String, String)>, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Manifest { + pub schema: u32, + pub generation: String, + pub object_bytes: usize, + pub source: Source, + pub ipv4: Option, + pub ipv6: Option, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Receipt { + pub manifest_key: String, + pub manifest_hash: [u8; 32], +} + +#[cfg(feature = "geo-kv-build")] +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct InventoryEntry { + pub key: String, + pub sha256: String, + pub bytes: usize, +} + +#[must_use] +#[inline] +pub fn node_key(generation: &str, family: u8, hash: &[u8; 32]) -> String { + format!("geo/v1/{generation}/v{family}/{}", hex(hash)) +} + +#[cfg(feature = "geo-kv-build")] +fn header(family: u8, level: u16, count: usize) -> Result> { + let mut bytes = MAGIC.to_vec(); + bytes.extend_from_slice(&[family, u8::from(level > 0)]); + bytes.extend_from_slice(&level.to_be_bytes()); + bytes.extend_from_slice(&u32::try_from(count)?.to_be_bytes()); + Ok(bytes) +} + +#[cfg(feature = "geo-kv-build")] +async fn publish_node( + kv: &KvHandle, + generation: &str, + family: u8, + reference: &NodeRef, + bytes: Vec, + inventory: &mut Vec, +) -> Result<()> { + let key = node_key(generation, family, &reference.hash); + inventory.push(InventoryEntry { + key: key.clone(), + sha256: hex(&reference.hash), + bytes: bytes.len(), + }); + kv.put_bytes(&key, bytes.into()).await?; + Ok(()) +} + +#[cfg(feature = "geo-kv-build")] +async fn build_family( + kv: &KvHandle, + generation: &str, + family: u8, + ranges: &[Range], + object_bytes: usize, + inventory: &mut Vec, +) -> Result> { + let address_bytes = width(family)?; + let leaf_capacity = (object_bytes - HEADER) / (2 * address_bytes + 6); + let directory_capacity = (object_bytes - HEADER) / (2 * address_bytes + 32); + ensure!( + leaf_capacity > 0 && directory_capacity >= 2, + "object budget cannot fit a branching directory" + ); + let mut previous_end = None; + for range in ranges { + ensure!( + range.start <= range.end && range.end <= max_address(family), + "invalid range" + ); + ensure!( + previous_end.is_none_or(|end| range.start > end), + "overlapping or unordered ranges" + ); + Location::decode(&range.location.0)?; + previous_end = Some(range.end); + } + let mut children = Vec::new(); + for chunk in ranges.chunks(leaf_capacity) { + let mut bytes = header(family, 0, chunk.len())?; + for range in chunk { + write_address(&mut bytes, range.start, address_bytes); + write_address(&mut bytes, range.end, address_bytes); + bytes.extend_from_slice(&range.location.0); + } + let reference = NodeRef { + start: chunk[0].start, + end: chunk[chunk.len() - 1].end, + hash: digest(&bytes), + level: 0, + }; + publish_node(kv, generation, family, &reference, bytes, inventory).await?; + children.push(reference); + } + while children.len() > 1 { + let mut parents = Vec::new(); + for chunk in children.chunks(directory_capacity) { + let level = chunk[0] + .level + .checked_add(1) + .context("tree depth overflow")?; + let mut bytes = header(family, level, chunk.len())?; + for child in chunk { + write_address(&mut bytes, child.start, address_bytes); + write_address(&mut bytes, child.end, address_bytes); + bytes.extend_from_slice(&child.hash); + } + let reference = NodeRef { + start: chunk[0].start, + end: chunk[chunk.len() - 1].end, + hash: digest(&bytes), + level, + }; + publish_node(kv, generation, family, &reference, bytes, inventory).await?; + parents.push(reference); + } + children = parents; + } + Ok(children.pop()) +} + +/// Build a generation in an artifact store. Use the standalone publisher for remote writes. +/// +/// # Errors +/// Rejects invalid ranges, object budgets, encoding failures and storage errors. +#[cfg(feature = "geo-kv-build")] +#[inline] +pub async fn build( + kv: &KvHandle, + source: Source, + v4: &[Range], + v6: &[Range], + object_bytes: usize, +) -> Result<(Receipt, Manifest, Vec)> { + // The minimum follows the encoding: header plus two IPv6 directory records. + ensure!( + (MIN_OBJECT_BYTES..=KvHandle::MAX_VALUE_SIZE).contains(&object_bytes), + "invalid object byte budget" + ); + let generation = hex(&digest(&serde_json::to_vec(&( + 1_u32, + object_bytes, + &source, + ))?)); + let mut inventory = Vec::new(); + let ipv4 = build_family(kv, &generation, 4, v4, object_bytes, &mut inventory).await?; + let ipv6 = build_family(kv, &generation, 6, v6, object_bytes, &mut inventory).await?; + let manifest = Manifest { + schema: 1, + generation: generation.clone(), + object_bytes, + source, + ipv4, + ipv6, + }; + let bytes = serde_json::to_vec(&manifest)?; + ensure!( + bytes.len() <= object_bytes, + "manifest exceeds target value budget" + ); + let receipt = Receipt { + manifest_key: format!("geo/v1/{generation}/manifest"), + manifest_hash: digest(&bytes), + }; + inventory.push(InventoryEntry { + key: receipt.manifest_key.clone(), + sha256: hex(&receipt.manifest_hash), + bytes: bytes.len(), + }); + kv.put_bytes(&receipt.manifest_key, bytes.into()).await?; + Ok((receipt, manifest, inventory)) +} + +#[derive(Clone)] +struct Object { + bytes: Bytes, + width: usize, + record_bytes: usize, + count: usize, +} + +impl Object { + fn record(&self, index: usize) -> &[u8] { + &self.bytes[HEADER + index * self.record_bytes..HEADER + (index + 1) * self.record_bytes] + } + + fn bounds(&self, index: usize) -> (u128, u128) { + let record = self.record(index); + ( + read_address(&record[..self.width]), + read_address(&record[self.width..2 * self.width]), + ) + } + + fn decode(bytes: Bytes, family: u8, reference: &NodeRef, max_bytes: usize) -> Result { + ensure!( + bytes.len() <= max_bytes && bytes.len() >= HEADER, + "invalid object length" + ); + ensure!(digest(&bytes) == reference.hash, "object digest mismatch"); + ensure!( + &bytes[..4] == MAGIC && bytes[4] == family, + "invalid object header" + ); + let level = u16::from_be_bytes(bytes[6..8].try_into()?); + ensure!( + level == reference.level && bytes[5] == u8::from(level > 0), + "invalid object level" + ); + let address_bytes = width(family)?; + let count = usize::try_from(u32::from_be_bytes(bytes[8..12].try_into()?))?; + let record_bytes = 2 * address_bytes + if level == 0 { 6 } else { 32 }; + ensure!( + count > 0 + && count + .checked_mul(record_bytes) + .and_then(|n| n.checked_add(HEADER)) + == Some(bytes.len()), + "invalid object records" + ); + let object = Self { + bytes, + width: address_bytes, + record_bytes, + count, + }; + let mut previous_end = None; + for index in 0..count { + let (start, end) = object.bounds(index); + ensure!( + start <= end && previous_end.is_none_or(|prior| start > prior), + "invalid record ordering" + ); + if level == 0 { + Location::decode(&object.record(index)[2 * address_bytes..])?; + } + previous_end = Some(end); + } + ensure!( + object.bounds(0).0 == reference.start && object.bounds(count - 1).1 == reference.end, + "child envelope mismatch" + ); + Ok(object) + } + + fn child(&self, index: usize, level: u16) -> NodeRef { + let (start, end) = self.bounds(index); + let mut hash = [0; 32]; + hash.copy_from_slice(&self.record(index)[2 * self.width..]); + NodeRef { + start, + end, + hash, + level: level - 1, + } + } + + fn find(&self, ip: u128) -> Option { + let mut lo = 0; + let mut hi = self.count; + while lo < hi { + let mid = lo + (hi - lo) / 2; + if self.bounds(mid).0 <= ip { + lo = mid + 1; + } else { + hi = mid; + } + } + (lo > 0 && ip <= self.bounds(lo - 1).1).then(|| lo - 1) + } +} + +const MIN_OBJECT_BYTES: usize = HEADER + 2 * (32 + 32); + +/// Limits for one prepared generation. Time and concurrent request limits belong to the adapter. +#[derive(Clone, Copy, Debug)] +pub struct IndexLimits { + pub max_object_bytes: usize, + pub max_path_objects: usize, + pub cache_bytes: usize, + pub cache_entries: usize, +} + +#[derive(Clone, Copy, Default, Debug, Serialize)] +pub struct CacheStats { + pub hits: u64, + pub peak_payload_bytes: usize, + pub max_object_bytes: usize, +} + +#[derive(Default)] +struct Cache { + bytes: usize, + values: HashMap<[u8; 32], Object>, + order: VecDeque<[u8; 32]>, + stats: CacheStats, +} + +impl Cache { + fn get(&mut self, hash: &[u8; 32]) -> Option { + let object = self.values.get(hash)?.clone(); + self.order.retain(|cached_hash| cached_hash != hash); + self.order.push_back(*hash); + self.stats.hits = self.stats.hits.saturating_add(1); + Some(object) + } + + fn insert(&mut self, hash: [u8; 32], object: Object, limits: IndexLimits) { + let size = object.bytes.len(); + self.stats.max_object_bytes = self.stats.max_object_bytes.max(size); + if size > limits.cache_bytes || limits.cache_entries == 0 || self.values.contains_key(&hash) + { + return; + } + while self.bytes > limits.cache_bytes - size || self.values.len() >= limits.cache_entries { + let Some(evicted) = self.order.pop_front() else { + return; + }; + let Some(removed) = self.values.remove(&evicted) else { + return; + }; + self.bytes -= removed.bytes.len(); + } + self.bytes += size; + self.values.insert(hash, object); + self.order.push_back(hash); + self.stats.peak_payload_bytes = self.stats.peak_payload_bytes.max(self.bytes); + } +} + +impl Receipt { + /// Parse a single deployment pin, `@`. + /// + /// # Errors + /// Rejects noncanonical hex, incomplete pins, and malformed manifest identities. + #[inline] + pub fn from_pin(pin: &str) -> Result { + let (generation, digest) = pin.split_once('@').context("missing Geo pin separator")?; + ensure!(valid_generation(generation), "invalid Geo generation"); + let mut manifest_hash = [0; 32]; + ensure!( + decode_hex(digest, &mut manifest_hash)?.len() == 32, + "invalid Geo manifest digest length" + ); + Ok(Self { + manifest_key: manifest_key(generation), + manifest_hash, + }) + } +} + +fn valid_generation(generation: &str) -> bool { + generation.len() == 64 + && generation + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn manifest_key(generation: &str) -> String { + format!("geo/v1/{generation}/manifest") +} + +/// A pin-validated resolver. Its cache belongs exclusively to this binding and generation. +pub struct Resolver { + kv: KvHandle, + manifest: Manifest, + limits: IndexLimits, + cache: Mutex, +} + +impl Resolver { + /// Read and validate the pinned manifest, without preloading objects. + /// + /// # Errors + /// Returns an error for missing or incompatible index data or invalid limits. + #[inline] + pub async fn open(kv: KvHandle, receipt: &Receipt, limits: IndexLimits) -> Result { + ensure!( + (MIN_OBJECT_BYTES..=KvHandle::MAX_VALUE_SIZE).contains(&limits.max_object_bytes) + && limits.max_path_objects > 0, + "invalid Geo index limits" + ); + ensure!( + receipt + .manifest_key + .strip_prefix("geo/v1/") + .and_then(|tail| tail.strip_suffix("/manifest")) + .is_some_and(valid_generation), + "invalid manifest key" + ); + let bytes = kv + .get_bytes(&receipt.manifest_key) + .await? + .context("missing manifest")?; + ensure!( + bytes.len() <= limits.max_object_bytes, + "manifest exceeds value budget" + ); + ensure!( + digest(&bytes) == receipt.manifest_hash, + "manifest digest mismatch" + ); + let manifest: Manifest = serde_json::from_slice(&bytes)?; + ensure!( + manifest.schema == 1 && valid_generation(&manifest.generation), + "invalid manifest identity" + ); + ensure!( + receipt.manifest_key == manifest_key(&manifest.generation), + "manifest key mismatch" + ); + ensure!( + (MIN_OBJECT_BYTES..=limits.max_object_bytes).contains(&manifest.object_bytes) + && bytes.len() <= manifest.object_bytes, + "invalid manifest object budget" + ); + for (family, root) in [(4, &manifest.ipv4), (6, &manifest.ipv6)] { + if let Some(reference) = root { + ensure!( + reference.start <= reference.end + && reference.end <= max_address(family) + && usize::from(reference.level) < limits.max_path_objects, + "invalid family root" + ); + } + } + Ok(Self { + kv, + manifest, + limits, + cache: Mutex::new(Cache::default()), + }) + } + + #[must_use] + #[inline] + pub fn manifest(&self) -> &Manifest { + &self.manifest + } + + #[must_use] + #[inline] + pub fn cache_stats(&self) -> CacheStats { + self.cache + .lock() + .map_or_else(|_error| CacheStats::default(), |cache| cache.stats) + } + + async fn read_object(&self, family: u8, reference: &NodeRef) -> Result { + let key = node_key(&self.manifest.generation, family, &reference.hash); + let bytes = self + .kv + .get_bytes(&key) + .await? + .context("missing required index object")?; + Object::decode(bytes, family, reference, self.manifest.object_bytes) + } + + /// Verify both nonempty roots through the serving binding before readiness. + /// + /// # Errors + /// Fails when any expected root cannot be read or validated. + #[inline] + pub async fn verify_roots(&self) -> Result<()> { + for (family, root) in [(4, &self.manifest.ipv4), (6, &self.manifest.ipv6)] { + if let Some(reference) = root { + self.read_object(family, reference).await?; + } + } + Ok(()) + } + + /// Validate the complete tree with memory proportional to depth, without populating the cache. + /// + /// # Errors + /// Fails when any expected object cannot be read or validated. + #[cfg(feature = "geo-kv-build")] + #[inline] + pub async fn verify_tree(&self) -> Result { + let mut objects = 1_usize; // the manifest was verified by open + for (family, root) in [(4, &self.manifest.ipv4), (6, &self.manifest.ipv6)] { + let Some(root_ref) = root else { continue }; + let mut next = Some(root_ref.clone()); + let mut directories = Vec::new(); + loop { + if let Some(reference) = next.take() { + let object = self.read_object(family, &reference).await?; + objects = objects + .checked_add(1) + .context("Geo object count overflow")?; + if reference.level != 0 { + directories.push((object, reference.level, 0)); + } + } + while let Some((directory, level, slot)) = directories.last_mut() { + if *slot < directory.count { + next = Some(directory.child(*slot, *level)); + *slot += 1; + break; + } + directories.pop(); + } + if next.is_none() { + break; + } + } + } + Ok(objects) + } + + /// Resolve one normalized IP. A matched but unsupported country is still a source location. + /// + /// # Errors + /// Required missing or invalid objects are errors, not geographic gaps. + #[inline] + pub async fn lookup(&self, input: IpAddr) -> Result> { + let (family, ip) = address(input); + let root = if family == 4 { + &self.manifest.ipv4 + } else { + &self.manifest.ipv6 + }; + let Some(mut reference) = root.clone() else { + return Ok(None); + }; + loop { + if ip < reference.start || ip > reference.end { + return Ok(None); + } + let cached = self + .cache + .lock() + .map_err(|_error| anyhow::anyhow!("Geo cache unavailable"))? + .get(&reference.hash); + let object = if let Some(object) = cached { + // A digest identifies bytes, not the envelope stated by each parent. + ensure!( + object.bytes[4] == family + && object.bounds(0).0 == reference.start + && object.bounds(object.count - 1).1 == reference.end + && u16::from_be_bytes(object.bytes[6..8].try_into()?) == reference.level, + "cached child descriptor mismatch" + ); + object + } else { + let object = self.read_object(family, &reference).await?; + self.cache + .lock() + .map_err(|_error| anyhow::anyhow!("Geo cache unavailable"))? + .insert(reference.hash, object.clone(), self.limits); + object + }; + let Some(index) = object.find(ip) else { + return Ok(None); + }; + if reference.level == 0 { + return Ok(Some(Location::decode( + &object.record(index)[2 * object.width..], + )?)); + } + reference = object.child(index, reference.level); + } + } +} + +#[cfg(test)] +#[cfg(feature = "geo-kv-build")] +mod tests { + use super::*; + use crate::key_value_store::{KvError, KvPage, KvStore}; + use async_trait::async_trait; + use futures::executor::block_on; + use std::{ + sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, + }, + time::Duration, + }; + + #[derive(Default)] + struct MemoryStore(Mutex>, AtomicUsize); + #[expect( + clippy::missing_trait_methods, + reason = "fixture has no record operations" + )] + #[async_trait(?Send)] + impl KvStore for MemoryStore { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + self.1.fetch_add(1, Ordering::Relaxed); + Ok(self.0.lock().unwrap().get(key).cloned()) + } + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + self.0.lock().unwrap().insert(key.into(), value); + Ok(()) + } + async fn delete(&self, _: &str) -> Result<(), KvError> { + panic!("lookup must not delete") + } + async fn list_keys_page( + &self, + _: &str, + _: Option<&str>, + _: usize, + ) -> Result { + panic!("lookup must not list") + } + async fn put_bytes_with_ttl(&self, _: &str, _: Bytes, _: Duration) -> Result<(), KvError> { + panic!("no TTL") + } + } + fn source() -> Source { + Source { + edition: "synthetic".into(), + files: vec![], + } + } + fn limits(cache_bytes: usize) -> IndexLimits { + IndexLimits { + max_object_bytes: 1024, + max_path_objects: 8, + cache_bytes, + cache_entries: 1024, + } + } + fn ranges() -> Vec { + (0..10_000) + .map(|i| Range { + start: i * 4, + end: i * 4 + 1, + location: Location::new("US", if i % 2 == 0 { "CA" } else { "" }).unwrap(), + }) + .collect() + } + + #[test] + fn round_trip_multilevel_cache_and_determinism() { + block_on(async { + let raw = Arc::new(MemoryStore::default()); + let kv = KvHandle::new(Arc::::clone(&raw)); + let ranges = ranges(); + let (receipt, manifest, _) = + build(&kv, source(), &ranges, &ranges, 1024).await.unwrap(); + assert!(manifest.ipv6.as_ref().unwrap().level >= 3); + let before = raw.0.lock().unwrap().clone(); + let (again, _, _) = build(&kv, source(), &ranges, &ranges, 1024).await.unwrap(); + assert_eq!(receipt, again); + assert_eq!(before, *raw.0.lock().unwrap()); + for budget in [0, 1024, 1024 * 1024] { + let resolver = Resolver::open(kv.clone(), &receipt, limits(budget)) + .await + .unwrap(); + for family in [4_u8, 6] { + for point in 0..410_u128 { + let ip = if family == 4 { + IpAddr::from(u32::try_from(point).unwrap().to_be_bytes()) + } else { + IpAddr::from(point.to_be_bytes()) + }; + let expected = ranges + .iter() + .find(|range| range.start <= point && point <= range.end) + .map(|range| range.location); + assert_eq!(resolver.lookup(ip).await.unwrap(), expected); + } + } + assert!(resolver.cache_stats().peak_payload_bytes <= budget); + } + }); + } + + #[test] + fn repeated_lookup_uses_cache_without_backend_reads() { + block_on(async { + let raw = Arc::new(MemoryStore::default()); + let kv = KvHandle::new(Arc::::clone(&raw)); + let ranges = ranges(); + let (receipt, manifest, _) = + build(&kv, source(), &ranges, &ranges, 1024).await.unwrap(); + for budget in [0, 1024 * 1024] { + for (text, root) in [ + ("0.0.0.0", manifest.ipv4.as_ref().unwrap()), + ("::", manifest.ipv6.as_ref().unwrap()), + ] { + let before_open = raw.1.load(Ordering::Relaxed); + let resolver = Resolver::open(kv.clone(), &receipt, limits(budget)) + .await + .unwrap(); + assert_eq!(raw.1.load(Ordering::Relaxed), before_open + 1); + let path_objects = usize::from(root.level) + 1; + assert!(path_objects > 1); + let ip = text.parse().unwrap(); + assert_eq!(resolver.lookup(ip).await.unwrap(), Some(ranges[0].location)); + let after_cold = raw.1.load(Ordering::Relaxed); + assert_eq!(after_cold, before_open + 1 + path_objects); + assert_eq!(resolver.cache_stats().hits, 0); + assert_eq!(resolver.lookup(ip).await.unwrap(), Some(ranges[0].location)); + let stats = resolver.cache_stats(); + if budget == 0 { + assert_eq!(raw.1.load(Ordering::Relaxed), after_cold + path_objects); + assert_eq!(stats.hits, 0); + assert_eq!(stats.peak_payload_bytes, 0); + } else { + assert_eq!(raw.1.load(Ordering::Relaxed), after_cold); + assert_eq!(stats.hits, u64::try_from(path_objects).unwrap()); + assert!(stats.peak_payload_bytes > 0); + assert!(stats.peak_payload_bytes <= budget); + } + assert!(stats.max_object_bytes > 0); + assert!(stats.max_object_bytes <= 1024); + } + } + }); + } + + #[test] + fn cache_evicts_least_recently_used_objects_at_both_limits() { + block_on(async { + let raw = Arc::new(MemoryStore::default()); + let kv = KvHandle::new(Arc::::clone(&raw)); + let leaf_capacity = (1024 - HEADER) / (2 * 4 + 6); + let ranges: Vec<_> = ranges().into_iter().take(3 * leaf_capacity).collect(); + let (receipt, manifest, _) = build(&kv, source(), &ranges, &[], 1024).await.unwrap(); + let reference = manifest.ipv4.as_ref().unwrap(); + assert_eq!(reference.level, 1); + let root_bytes = + raw.0.lock().unwrap()[&node_key(&manifest.generation, 4, &reference.hash)].len(); + let leaf_bytes = HEADER + leaf_capacity * (2 * 4 + 6); + let payload_bytes = root_bytes + 2 * leaf_bytes; + for cache_limits in [ + IndexLimits { + cache_entries: 3, + ..limits(1024 * 1024) + }, + limits(payload_bytes), + ] { + let resolver = Resolver::open(kv.clone(), &receipt, cache_limits) + .await + .unwrap(); + let mut hits = 0; + for (slot, reads, new_hits) in [ + (0, 2, 0), // Load the root and leaf A. + (leaf_capacity, 1, 1), // Load B, retaining A. + (0, 0, 2), // Refresh A so B becomes least recently used. + (2 * leaf_capacity, 1, 1), // Load C, evicting B. + (0, 0, 2), // A must still be cached. + (leaf_capacity, 1, 1), // B must be fetched again. + ] { + let before = raw.1.load(Ordering::Relaxed); + let ip = IpAddr::from(u32::try_from(ranges[slot].start).unwrap().to_be_bytes()); + assert_eq!( + resolver.lookup(ip).await.unwrap(), + Some(ranges[slot].location) + ); + assert_eq!(raw.1.load(Ordering::Relaxed), before + reads); + hits += new_hits; + assert_eq!(resolver.cache_stats().hits, hits); + } + let stats = resolver.cache_stats(); + assert_eq!(stats.peak_payload_bytes, payload_bytes); + assert_eq!(stats.max_object_bytes, leaf_bytes); + } + }); + } + + #[test] + fn full_family_and_empty_family() { + block_on(async { + for family in [4, 6] { + let kv = KvHandle::new(Arc::new(MemoryStore::default())); + let ranges = [Range { + start: 0, + end: max_address(family), + location: Location::new("GB", "ENG").unwrap(), + }]; + let (receipt, _, _) = build( + &kv, + source(), + if family == 4 { &ranges } else { &[] }, + if family == 6 { &ranges } else { &[] }, + 1024, + ) + .await + .unwrap(); + let resolver = Resolver::open(kv, &receipt, limits(0)).await.unwrap(); + for point in [0, max_address(family)] { + let ip = if family == 4 { + IpAddr::from(u32::try_from(point).unwrap().to_be_bytes()) + } else { + IpAddr::from(point.to_be_bytes()) + }; + assert_eq!(resolver.lookup(ip).await.unwrap(), Some(ranges[0].location)); + } + let empty = if family == 4 { "::1" } else { "1.0.0.1" }; + assert_eq!(resolver.lookup(empty.parse().unwrap()).await.unwrap(), None); + } + }); + } + + #[test] + fn source_country_roundtrip_does_not_claim_domain_acceptance() { + block_on(async { + let kv = KvHandle::new(Arc::new(MemoryStore::default())); + let ranges = [Range { + start: 0, + end: 1, + location: Location::new("XK", "").unwrap(), + }]; + let (receipt, _, _) = build(&kv, source(), &ranges, &[], 1024).await.unwrap(); + let resolver = Resolver::open(kv, &receipt, limits(0)).await.unwrap(); + let location = resolver + .lookup("0.0.0.1".parse().unwrap()) + .await + .unwrap() + .unwrap(); + assert_eq!(location, ranges[0].location); + assert!(!location.core_country_supported()); + }); + } + + #[test] + fn missing_and_corrupt_objects_are_errors_not_misses() { + block_on(async { + let raw = Arc::new(MemoryStore::default()); + let kv = KvHandle::new(Arc::::clone(&raw)); + let (receipt, manifest, _) = build(&kv, source(), &ranges(), &[], 1024).await.unwrap(); + let key = node_key( + &manifest.generation, + 4, + &manifest.ipv4.as_ref().unwrap().hash, + ); + let old = raw.0.lock().unwrap().remove(&key).unwrap(); + let resolver = Resolver::open(kv.clone(), &receipt, limits(0)) + .await + .unwrap(); + assert!( + resolver + .lookup("0.0.0.0".parse().unwrap()) + .await + .unwrap_err() + .to_string() + .contains("missing required") + ); + let mut corrupt = old.to_vec(); + corrupt[0] ^= 1; + raw.0.lock().unwrap().insert(key, corrupt.into()); + assert!( + resolver + .lookup("0.0.0.0".parse().unwrap()) + .await + .unwrap_err() + .to_string() + .contains("digest mismatch") + ); + let mut bad_receipt = receipt.clone(); + bad_receipt.manifest_hash[0] ^= 1; + assert!(Resolver::open(kv, &bad_receipt, limits(0)).await.is_err()); + }); + } + + #[test] + fn malformed_records_with_matching_digest_are_rejected() { + let reference = |data: &[u8]| NodeRef { + start: 0, + end: 4, + hash: digest(data), + level: 0, + }; + let mut bytes = header(4, 0, 2).unwrap(); + for (start, end) in [(0, 2), (2, 4)] { + write_address(&mut bytes, start, 4); + write_address(&mut bytes, end, 4); + bytes.extend_from_slice(&Location::new("US", "CA").unwrap().0); + } + assert!(Object::decode(bytes.clone().into(), 4, &reference(&bytes), 140).is_err()); + let mut invalid = header(4, 0, 1).unwrap(); + write_address(&mut invalid, 0, 4); + write_address(&mut invalid, 4, 4); + invalid.extend_from_slice(&[b'U', b'S', 4, b'C', b'A', 0]); + assert!(Object::decode(invalid.clone().into(), 4, &reference(&invalid), 140).is_err()); + invalid.pop(); + assert!(Object::decode(invalid.clone().into(), 4, &reference(&invalid), 140).is_err()); + } + + #[test] + fn rejects_overlaps_with_a_compatible_object_budget() { + block_on(async { + for family in [4_u8, 6] { + let kv = KvHandle::new(Arc::new(MemoryStore::default())); + let valid: Vec<_> = ranges().into_iter().take(2).collect(); + let mut overlapping = valid.clone(); + overlapping[1].start = valid[0].end; + for input in [&valid, &overlapping] { + let result = build( + &kv, + source(), + if family == 4 { input } else { &[] }, + if family == 6 { input } else { &[] }, + 1024, + ) + .await; + if input == &valid { + result.unwrap(); + } else { + assert_eq!( + result.unwrap_err().to_string(), + "overlapping or unordered ranges" + ); + } + } + } + }); + } + + #[test] + fn rejects_too_small_object_budget() { + block_on(async { + let kv = KvHandle::new(Arc::new(MemoryStore::default())); + assert_eq!( + build(&kv, source(), &[], &[], 139) + .await + .unwrap_err() + .to_string(), + "invalid object byte budget" + ); + }); + } +} diff --git a/crates/edgezero-core/src/manifest.rs b/crates/edgezero-core/src/manifest.rs index cd1f49643..82c326b92 100644 --- a/crates/edgezero-core/src/manifest.rs +++ b/crates/edgezero-core/src/manifest.rs @@ -343,6 +343,7 @@ impl ManifestLoader { #[derive(Debug, Deserialize, Serialize, Validate)] #[serde(deny_unknown_fields)] #[validate(schema(function = "validate_manifest_adapter_keys_case_unique"))] +#[validate(schema(function = "validate_manifest_adapter_geo_placement"))] #[validate(schema( function = "validate_manifest_fastly_logging", skip_on_field_errors = false @@ -682,8 +683,13 @@ pub struct ManifestAdapter { #[serde(default)] #[validate(nested)] pub commands: ManifestAdapterCommands, - /// Catch-all for any sub-table other than the four canonical ones - /// (`adapter`, `build`, `commands`, `logging`). The pre-rewrite + /// `[adapters.axum.geo]`: native geo selection. Only Axum reads it; other + /// adapters use their platform's geo and reject the table. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[validate(nested)] + pub geo: Option, + /// Catch-all for any sub-table other than the canonical ones + /// (`adapter`, `build`, `commands`, `geo`, `logging`). The pre-rewrite /// `[adapters..stores.*]` tables land here and are rejected by /// `validate_manifest_adapter` with the migration-guide message. #[serde(flatten)] @@ -693,6 +699,64 @@ pub struct ManifestAdapter { pub logging: ManifestLoggingConfig, } +/// `[adapters.axum.geo]`. Each key is a default that the matching +/// `EDGEZERO__ADAPTER__GEO__*` variable overrides at startup, so deployment-only +/// values such as ALB networks can stay out of the compiled manifest. The +/// merged selection is validated before the server listens. +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize, Validate)] +#[serde(deny_unknown_fields)] +#[non_exhaustive] +#[validate(schema(function = "validate_manifest_adapter_geo"))] +pub struct ManifestAdapterGeo { + /// ALB peer networks admitted as the protected `CloudFront` source. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub alb_peer_cidrs: Option>, + /// Aggregate per-request KV lookup deadline in milliseconds. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub lookup_timeout_ms: Option, + /// Count of concurrent lookups allowed to start on this instance. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_in_flight: Option, + /// Header the ALB strips after checking the origin credential. Never the credential value. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub origin_credential_header_name: Option, + /// Atomic generation and manifest digest pin. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pin: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Reader provenance required by this listener's deployment. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reader_ip: Option, + /// Declared logical KV binding, not a backend resource name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub store_id: Option, +} + +impl ManifestAdapterGeo { + /// Whether any KV-specific settings were supplied. + #[inline] + #[must_use] + pub fn has_kv_fields(&self) -> bool { + self.store_id.is_some() + || self.pin.is_some() + || self.reader_ip.is_some() + || self.lookup_timeout_ms.is_some() + || self.max_in_flight.is_some() + } +} + +/// Geo provider an Axum deployment selects. +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "lowercase")] +#[non_exhaustive] +pub enum ManifestGeoProvider { + #[serde(rename = "cloudfront")] + CloudFront, + Kv, + Off, +} + #[derive(Debug, Default, Deserialize, Serialize, Validate)] #[non_exhaustive] #[validate(schema(function = "validate_manifest_adapter_definition"))] @@ -1449,6 +1513,42 @@ fn validate_manifest_adapter_definition( Ok(()) } +fn validate_manifest_adapter_geo(geo: &ManifestAdapterGeo) -> Result<(), ValidationError> { + let assertion = geo.alb_peer_cidrs.is_some() || geo.origin_credential_header_name.is_some(); + let kv = geo.has_kv_fields(); + if geo.provider == Some(ManifestGeoProvider::Off) && (assertion || kv) { + let mut error = ValidationError::new("geo_off_with_policy"); + error.message = Some( + "`[adapters.axum.geo]` with `provider = \"off\"` must not set source policy fields" + .into(), + ); + return Err(error); + } + if (assertion && (kv || geo.provider == Some(ManifestGeoProvider::Kv))) + || (kv && geo.provider == Some(ManifestGeoProvider::CloudFront)) + { + return Err(ValidationError::new("geo_mixed_provider_fields")); + } + Ok(()) +} + +fn validate_manifest_adapter_geo_placement(manifest: &Manifest) -> Result<(), ValidationError> { + for (name, adapter) in &manifest.adapters { + if adapter.geo.is_some() && !name.eq_ignore_ascii_case("axum") { + let mut error = ValidationError::new("geo_adapter_unsupported"); + error.message = Some( + format!( + "`[adapters.{name}.geo]` is not supported; only Axum selects geo, \ + other adapters use their platform's geo" + ) + .into(), + ); + return Err(error); + } + } + Ok(()) +} + /// Validates a single `[adapters.]` block. The portable manifest model /// has no per-adapter store / runtime tuning surface — all of that moved to /// `EDGEZERO__*` env vars. The pre-rewrite @@ -3537,6 +3637,66 @@ ids = ["default"] } } + #[test] + fn axum_geo_table_loads_partial_defaults() { + let loaded = ManifestLoader::try_load_from_str( + "[adapters.axum.geo]\nprovider = \"cloudfront\"\norigin_credential_header_name = \"x-origin-verify\"\n", + ) + .expect("axum geo table"); + let geo = loaded.manifest().adapters["axum"].geo.clone().expect("geo"); + assert_eq!(geo.provider, Some(ManifestGeoProvider::CloudFront)); + assert_eq!(geo.alb_peer_cidrs, None); + assert_eq!( + geo.origin_credential_header_name.as_deref(), + Some("x-origin-verify") + ); + // `app!` bakes the serialized manifest; the geo table must survive the round trip. + let json: &'static str = Box::leak( + serde_json::to_string(loaded.manifest()) + .expect("serialize") + .into_boxed_str(), + ); + let BakedManifest::Present(baked) = Manifest::from_baked_json(json) else { + panic!("baked manifest with geo must load: {json}"); + }; + assert_eq!(baked.adapters["axum"].geo.as_ref(), Some(&geo)); + assert!( + ManifestLoader::try_load_from_str("[adapters.axum]\n") + .expect("no geo") + .manifest() + .adapters["axum"] + .geo + .is_none() + ); + } + + #[test] + fn axum_geo_table_rejects_invalid_shapes() { + for (manifest, expected) in [ + ( + "[adapters.axum.geo]\nprovider = \"off\"\nalb_peer_cidrs = []\n", + "must not set source policy fields", + ), + ( + "[adapters.axum.geo]\nprovider = \"maxmind\"\n", + "unknown variant", + ), + ( + "[adapters.axum.geo]\nregion_header = \"x\"\n", + "unknown field", + ), + ( + "[adapters.fastly.geo]\nprovider = \"cloudfront\"\n", + "only Axum selects geo", + ), + ] { + let err = ManifestLoader::try_load_from_str(manifest) + .err() + .unwrap_or_else(|| panic!("must reject: {manifest}")); + assert!(err.to_string().contains(expected), "{manifest}: {err}"); + } + } + #[test] fn empty_manifest_has_no_config_store() { let mfest = ManifestLoader::load_from_str(""); diff --git a/docs/guide/adapters/axum.md b/docs/guide/adapters/axum.md index e5c95e48a..d6d08033b 100644 --- a/docs/guide/adapters/axum.md +++ b/docs/guide/adapters/axum.md @@ -290,7 +290,8 @@ request identity, counters, and deployment examples. ## Native Geo providers -Geo is off by default and selected only through `AxumRunOptions::with_geo`. The opt-in +Geo is off by default. Select it with `[adapters.axum.geo]` in `edgezero.toml` and +`EDGEZERO__ADAPTER__GEO__*` settings, or in code with `AxumRunOptions::with_geo`. The opt-in `geo-cloudfront` feature adds a bounded local CloudFront parser, not AWS SDKs or store setup. Both built-in and supplied providers require a separate `ProtectedCloudFrontAlb` policy using the actual socket peer. Ordinary trusted-proxy configuration does not admit geographic assertions. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 607c584f6..20569a7b6 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -561,6 +561,28 @@ implicitly applying manifest defaults or changing global state. All managed adapter loggers retain `edgezero::boot` errors and warnings at `Warn` even when ordinary runtime logging is `Off`. See [Logging Initialisation](/guide/adapters/overview#logging-initialisation). +### Geo (Axum only) + +`[adapters.axum.geo]` selects native geo so `RequestContext::geo()` works without application +code. Fastly and Cloudflare always report their platform's geo, and other adapters reject the +table. + +```toml +[adapters.axum.geo] +provider = "cloudfront" +origin_credential_header_name = "x-origin-verify-randomsuffix" +# alb_peer_cidrs usually comes from EDGEZERO__ADAPTER__GEO__ALB_PEER_CIDRS at deploy time +``` + +| Field | Runtime override | Description | +| ------------------------------- | ------------------------------------------------------- | ------------------------------------------- | +| `provider` | `EDGEZERO__ADAPTER__GEO__PROVIDER` | `cloudfront` or `off` | +| `alb_peer_cidrs` | `EDGEZERO__ADAPTER__GEO__ALB_PEER_CIDRS` (comma list) | ALB peer networks admitted as the source | +| `origin_credential_header_name` | `EDGEZERO__ADAPTER__GEO__ORIGIN_CREDENTIAL_HEADER_NAME` | Header the ALB strips; never the credential | + +Each runtime setting overrides its manifest key. See the [native Geo guide](./native-geo.md#select-from-the-manifest-and-settings) +for precedence and validation. + ## Full Example ```toml diff --git a/docs/guide/native-geo.md b/docs/guide/native-geo.md index 7a1b1f05e..529e2cf59 100644 --- a/docs/guide/native-geo.md +++ b/docs/guide/native-geo.md @@ -1,17 +1,131 @@ # Native Geo providers -Native Axum hosting can resolve CloudFront country and subdivision assertions once per admitted -request. Geo is off by default. There is no database, reader-IP lookup, remote call, fallback -provider, CLI option, environment selector, or manifest selector. +Native Axum hosting can resolve protected CloudFront assertions or look up a checked reader IP +in a pinned KV index. It resolves geography once per admitted request. Geo is off by default. +Select it from the manifest and `EDGEZERO__` settings, or in Rust. There is no fallback +provider. Geography is an estimate, not proof of residence, identity, consent, or permission. Applications own jurisdiction recognition and policy. A missing subdivision stays unknown, and a syntax-valid subdivision is not necessarily assigned or recognized. +## Select from the manifest and settings + +The runner reads `[adapters.axum.geo]` from the app's baked `edgezero.toml`, so an application +reads `RequestContext::geo()` without its own startup code: + +```toml +[adapters.axum.geo] +provider = "cloudfront" +origin_credential_header_name = "x-origin-verify-randomsuffix" +``` + +Each key is a default. The matching setting overrides it when the server starts, which keeps +deployment-only values such as ALB networks out of the compiled manifest: + +```sh +EDGEZERO__ADAPTER__GEO__ALB_PEER_CIDRS=10.42.1.0/24,10.42.2.0/24 +# EDGEZERO__ADAPTER__GEO__PROVIDER=off turns a baked selection off for one deployment +# EDGEZERO__ADAPTER__GEO__ORIGIN_CREDENTIAL_HEADER_NAME=x-origin-verify-other +``` + +Precedence, highest first: `AxumRunOptions::with_geo`, then the settings, then the manifest, +then off. Settings come from the options the runner receives: `from_env` and +`run_production_app` read them from the process, and options built with `new` carry none, but +the manifest still applies. The development `run_app` reads only the settings and ignores the +manifest table, because its deployment-only values are usually absent locally. +`geo_capability_for::()` reports the selection the runner will use. `geo_capability()` +reports only an explicit `with_geo` selection. + +The merged values pass the same validation as the Rust policy below, before the server listens. +Misconfiguration never falls back to off. An unknown provider, network or header settings with +no provider selected, settings-supplied policy fields with `provider = "off"`, and a CloudFront +selection missing networks or a header all fail startup. In the manifest, `provider = "off"` +with policy fields fails to load. + +## Serve a CSV-backed KV index + +1. Build and publish an artifact with the [standalone Geo tool](https://github.com/IABTechLab/edgezero/tree/main/examples/geo-kv-prototype#readme). + Use the publisher's credential for writes. Verify the entire generation and sampled CSV + results from the serving path before choosing a pin. The tool does not provision a store. +2. Compile with `geo-kv` plus the feature for your ordinary KV binding, such as `celld-kv` + or `aws-dynamodb`. Declare a dedicated logical KV ID in the app's `[stores.kv]`. + Bind it to a read-only namespace using the existing + [celld](native-celld-stores.md) or [DynamoDB](dynamodb-kv.md) store configuration. +3. Select `kv` and supply the pin and lookup budgets. Use `forwarded-required` behind a + trusted reverse proxy. Configure the trusted peer networks and forwarding family as + described in the [Axum ingress guide](adapters/axum.md). Use `direct-peer` only when the + listener receives reader connections directly. + +```toml +[adapters.axum.geo] +provider = "kv" +store_id = "geo_index" +reader_ip = "forwarded-required" +# Example budgets. Set them for your traffic and request latency requirements. +lookup_timeout_ms = 100 +max_in_flight = 32 +``` + +Set `EDGEZERO__ADAPTER__GEO__PIN` to the tool's `@` pin. +Override it as a single setting. The store ID, mode and budgets can likewise be overridden +by `EDGEZERO__ADAPTER__GEO__STORE_ID`, `READER_IP`, `LOOKUP_TIMEOUT_MS` and `MAX_IN_FLIGHT` +under the same prefix. + +The runner validates the manifest and both nonempty roots before readiness. It does not scan +the full dataset at startup. A lookup follows at most three index objects and uses a cache +limited to 4 MiB of validated payload and 256 entries. A missing checked reader requires no +reads. A missing or corrupt required object, timeout, or saturated lookup limit yields a +provider error. A verified gap yields `NoMatch`. A source code outside the supported country +set, such as `XK`, yields `UnsupportedGeography`. Neither is a storage error. + +With celld, grant the application only `read` on the Geo alias. With DynamoDB, the serving role +needs `DescribeTable` and `GetItem` on the chosen table; restrict `GetItem` to the Geo namespace +with `dynamodb:LeadingKeys` where applicable. The operator can publish and retire with the +same credential; no separate cleanup identity is needed. Geo never uses an EC record binding. + +Update or roll back by verifying another permitted generation and restarting with its pin. +There is no `active` pointer and no automatic fallback. Keep the CSV, artifacts and backups +within the [GeoLite license](https://www.maxmind.com/en/geolite/eula). Retire old objects after +stopping readers and within the license deadline. Apply attribution to your deployment. + +With the `geo-kv` feature, embedding applications can also select the provider with +`NativeGeoSelection::Kv(geo::kv::KvGeoSelection::new(...))`. The default dev runner can select through +settings and uses its declared local KV; use a managed production runner for remote bindings. + +### Reproduce the local backend proof + +From the EdgeZero checkout, build the operator tool and managed listener fixture. Use the +official celld 0.6.3 binary and a local Docker daemon with the DynamoDB Local image pinned in +`scripts/run_geo_kv_integration.mjs` already available. The script starts disposable loopback +services. It does not use hosted AWS or ambient AWS credentials. + +```sh +cargo build --locked --manifest-path examples/geo-kv-prototype/Cargo.toml \ + --target-dir "$PWD/examples/geo-kv-prototype/target" --all-features +cargo build --locked -p edgezero-adapter-axum \ + --target-dir "$PWD/target" --features geo-kv,celld-kv,aws-dynamodb,test-utils \ + --bin edgezero-hosting-fixture + +CELLD_BIN=/absolute/path/to/official/celld \ +GEO_KV_TOOL_BIN="$PWD/examples/geo-kv-prototype/target/debug/geo-kv-prototype" \ +GEO_KV_HOST_BIN="$PWD/target/debug/edgezero-hosting-fixture" \ +node scripts/run_geo_kv_integration.mjs +``` + +For the production-size comparison, set `GEO_KV_CSV_DIR` to an absolute path to a permitted +GeoLite City CSV extraction on Linux. The proof builds 16 KiB and 64 KiB artifacts, verifies +both backends against the CSV oracle, and reports process peak RSS, latency and backend reads. +The tool defaults to 16 KiB. In the local comparison, these objects fit the repeated-request +working set on both backends; 64 KiB used fewer reads on the first pass. The first pass starts +with an empty object cache; the second repeats its sampled addresses. +Response-body byte counts include backend serialization. Results describe this local workload +and build, not production SLOs, hosted IAM, or durability. + ## Select a provider in Rust The public selection API lives in `edgezero_adapter_axum::geo` with the `axum` feature. -Only the built-in implementation needs the default-off `geo-cloudfront` feature: +The built-in CloudFront implementation needs the default-off `geo-cloudfront` feature: ```toml [dependencies] @@ -19,13 +133,13 @@ edgezero-adapter-axum = { version = "", default-features ``` Use the same pinned EdgeZero release as your application. This feature enables `axum`, not -AWS stores or an AWS SDK. A Geo-only application needs no AWS credentials, config/data roots, -or declared stores. Existing declared stores keep their own startup requirements. +AWS stores or an AWS SDK. A CloudFront-only application needs no AWS credentials, config/data +roots, or declared stores. Existing declared stores keep their own startup requirements. ### Off -Every options constructor defaults to `Off`, including `from_env` and `from_vars`. Selection -is always programmatic, even when other hosting settings come from the environment. +Without a manifest table or settings, every options constructor resolves to `Off`, including +`from_env` and `from_vars`. An explicit `Off` here overrides both. ```rust use edgezero_adapter_axum::dev_server::AxumRunOptions; diff --git a/docs/superpowers/plans/2026-11-05-kv-geo-prototype-results.md b/docs/superpowers/plans/2026-11-05-kv-geo-prototype-results.md new file mode 100644 index 000000000..f607586c3 --- /dev/null +++ b/docs/superpowers/plans/2026-11-05-kv-geo-prototype-results.md @@ -0,0 +1,100 @@ +# KV Geo prototype results + +- Date: 2026-11-05. +- Status: historical `p1` file-backed experiment. See the [current design](../specs/2026-11-05-kv-geo-design.md) for production acceptance. +- Spec: [portable KV Geo](../specs/2026-11-05-kv-geo-design.md). +- Code and reproduction commands: [prototype README at the measured revision](https://github.com/IABTechLab/edgezero/blob/6e8bb0e1/examples/geo-kv-prototype/README.md). +- Measured revision: `6e8bb0e1`. The prototype source and manifest at `c52bdaf3` were unchanged, but its refreshed lockfile changes the fingerprint below to `e9c6bbc0c3e1323e195e1d1f44ebfa3832d24998d9a0b17c834bb3af37018088`. The timings and binary digest describe the measured revision only. +- Environment: Linux x86_64, Rust 1.95.0, optimized Cargo release build, local file-backed artifact KV. +- Tested source fingerprint: `70227e734eee4709c7072a45a50dcc00ea15bd1c947c82596025a309e9c2ddbf`. Compute with `sha256sum examples/geo-kv-prototype/src/*.rs examples/geo-kv-prototype/Cargo.toml examples/geo-kv-prototype/Cargo.lock | sha256sum`. +- Tested release binary SHA-256: `dd9af4b519c1213f2094b6805a41307fc637137eded40751491384dbfb5e6878`. + +## Dataset + +Used the user's `GeoLite2-City-CSV_20261103.zip`, kept outside Git. Archive SHA-256 is `94745697bcd25751e38c9386260eeb29527f851ec9e3732cfe58aa62a07b9ab8`. ZIP CRC checks passed; a publisher-provided checksum was not available for comparison. + +| Input | Count | +| -------------------------------- | --------- | +| English location rows | 75,449 | +| IPv4 source networks | 3,632,756 | +| IPv6 source networks | 2,091,942 | +| Merged IPv4 geographic intervals | 1,371,640 | +| Merged IPv6 geographic intervals | 605,625 | +| IPv4 networks without country | 768 | +| IPv6 networks without country | 724 | + +The import joins geolocation IDs only. It does not substitute registered-country IDs or include names, coordinates or postal codes in the index. Source networks without country produce gaps, not guessed locations. + +## Index shape + +All objects were written and read back through the actual EdgeZero `KvHandle`. The transport was the prototype's local artifact-file `KvStore`, not a cloud service. + +| Object budget | Encoded total | Objects, including manifest | Uncached path objects, IPv4/IPv6 | Import, build and read-back | +| ------------- | ------------- | --------------------------- | -------------------------------- | --------------------------- | +| 16 KiB | 40.423 MiB | 2595 | 3/3 | 1.482 s | +| 64 KiB | 40.302 MiB | 649 | 2/2 | 1.487 s | +| 256 KiB | 40.272 MiB | 165 | 2/2 | 1.490 s | +| 384 KiB | 40.269 MiB | 111 | 2/2 | 1.460 s | + +A resolver acquires the manifest with one additional read when opened. The path counts above exclude it and are maxima; root/directory gaps can return without a leaf read. + +The 64 KiB build contained 42,259,649 encoded bytes and 649 objects. Its largest object was 65,532 bytes. Both address families fit a root directory followed by a leaf. Increasing shards beyond 64 KiB reduced key count but did not remove another read from this dataset's path. + +Importer whole-process peak RSS was about 231.4 MiB. That process retains raw CIDRs for verification and builds merged ranges, so this is not serving memory. + +## Lookup checks and local timings + +For each size, checked 2,048 points per family in three passes: no object cache, an initially empty 4 MiB object cache, and a repeat using that same cache. All 49,152 comparisons across four sizes and three modes matched the independent source-CIDR oracle. These are repeated comparisons over the same 4,096-point workload, not 49,152 distinct random addresses or exhaustive dataset coverage. + +The sample mix includes family endpoints, stratified source-network endpoints and neighbors, and deterministic uniform random addresses. Each pass found 2,917 geographic locations and 1,179 no-matches. It is a correctness workload with correlated neighbors and many uncovered addresses, not a user-traffic distribution. + +| Object budget | Cache-disabled p95, microseconds | Repeated-cache p95, microseconds | Repeated-cache reads/query | Repeated-cache bytes/query | +| ------------- | -------------------------------- | -------------------------------- | -------------------------- | -------------------------- | +| 16 KiB | 63.5 | 46.5 | 0.240 | 3.83 KiB | +| 64 KiB | 158.5 | 142.9 | 0.228 | 14.51 KiB | +| 256 KiB | 524.9 | 514.6 | 0.184 | 46.51 KiB | +| 384 KiB | 1266.1 | 749.0 | 0.184 | 69.96 KiB | + +Timings include local file reads, hashing, validation and lookup. OS page cache was uncontrolled. They do not measure remote latency, cold storage, concurrency, provider billing or cache behavior across short-lived isolates. The mean read/byte rates are specific to this sample ordering and cache budget. + +A fresh 64 KiB `lookup` process resolved four sample inputs without loading source CSV or an inventory. It read one manifest, one directory and one leaf, retained 77,304 cache payload bytes, and peaked at 3,006,464 bytes of whole-process RSS. This proves a small lookup workload avoids the importer's full-data memory footprint. It does not establish maximum serving RSS under load. Cache limits count retained encoded payload, with allocator metadata and validation scratch memory in addition. + +## Country-code policy mismatch + +The real release contains `XK`, while EdgeZero's existing `CountryCode` rejects it. There are 393 IPv4 and 68 IPv6 source networks with that code. + +The initial strict import failed on this mismatch. To finish the format experiment without changing production country policy, the standalone prototype preserves syntactically valid source country codes in a separate DTO and reports whether core accepts them. A fresh lookup over an actual `XK` interval returned the preserved code with `core_country_supported: false`. No shared allowlist or application outcome changed. + +This is a decision needed before production implementation. Rejecting a complete release because it contains one unsupported code prevents using this dataset as supplied. Alternatives must explicitly define unsupported-code behavior rather than silently losing those intervals or calling them `NoMatch`. + +## Verification record + +| Check | Result | +| ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | +| `cargo test --offline --locked --manifest-path examples/geo-kv-prototype/Cargo.toml` | 10 prototype tests passed | +| `cargo clippy --offline --locked --manifest-path examples/geo-kv-prototype/Cargo.toml --all-targets -- -D warnings` | Passed | +| Prototype and workspace `cargo fmt -- --check` commands | Passed | +| `cargo test --offline --locked -p edgezero-core` | 753 unit tests and enabled doctests passed; 13 existing doctests ignored | +| `cargo build --offline --locked --release --manifest-path examples/geo-kv-prototype/Cargo.toml` | Passed | +| Real-data build at 16/64/256/384 KiB | Every exported object read back and digest-checked | +| Repeat real-data builds | Same receipts and immutable object bytes accepted for every size | +| CSV oracle with cache off/on/repeated | 49,152 comparisons, zero mismatches | +| Fresh-process lookup, including real `XK` interval | No CSV/inventory preload; unsupported-domain flag preserved | +| Pre-PR `cargo test --offline --locked --workspace --all-targets` | 2,115 passed, six existing tests ignored | +| Pre-PR `cargo clippy --offline --locked --workspace --all-targets --all-features -- -D warnings` | Passed | +| Pre-PR `cargo check --offline --locked --workspace --all-targets --features 'fastly cloudflare spin'` | Passed | +| Pre-PR `cargo check --offline --locked -p edgezero-adapter-spin --target wasm32-wasip2 --features spin` | Passed | + +Synthetic tests exercised multilevel traversal, deterministic output, IPv4/IPv6 zero and maximum addresses, empty families, gaps, country-only results, absent geolocation without registered-country fallback, quoted CSV fields, bad references, duplicate identities, bad family/CIDR input, missing objects, wrong digests and malformed records with matching digests. Missing expected data is an error, not a no-match. The original tests did not isolate overlap rejection from manifest-size rejection, or prove that cache hits and eviction occurred. + +Logs from this session are under `/tmp/geo-kv-prototype-*.log`. Dataset, artifacts and durable machine-readable measurements are under `~/.cache/edgezero/geo-kv-prototype/`; each `artifacts-k/` directory contains `receipt.json`, `inventory.json`, `build-report.json` and `verification-report.json`. These are local artifacts, not committed production data. + +## Next decisions + +64 KiB is a reasonable next remote-KV experiment: it preserved two-read lookup paths while transferring and validating less data than 256/384 KiB shards. The locally fastest 16 KiB objects needed a third uncached read, which may lose once network latency is included. No serving default or wire format is frozen by these results. + +Next work should decide country-code policy, exercise actual provider upload/read paths, and measure representative traffic and remote KV costs before changing request APIs or dispatch. City import and synthetic Country-shaped CSV were exercised; a full real Country release was not. + +Not verified: production async provider composition, request cancellation/deadlines, reader-IP admission on edge adapters, WASM runtime behavior, live KV latency/capacity, globally visible publication, crash-safe uploads, licensing review or deletion across deployed caches. No live credentials or cloud resources were used. The pre-PR workspace gates above passed without changing the prototype's measured source fingerprint; logs are under `/tmp/geo-kv-pr-*.log`. No new Fastly/Cloudflare WASM runtime execution or live-service acceptance was performed. + +At the measured revision, root workspace checks and the PR workflow did not build this standalone crate. The prototype commands above are local verification, not a claim of historical GitHub coverage. diff --git a/docs/superpowers/plans/2026-11-17-kv-geo-serving-results.md b/docs/superpowers/plans/2026-11-17-kv-geo-serving-results.md new file mode 100644 index 000000000..db0285437 --- /dev/null +++ b/docs/superpowers/plans/2026-11-17-kv-geo-serving-results.md @@ -0,0 +1,33 @@ +# Axum KV Geo local backend proof + +`scripts/run_geo_kv_integration.mjs` exercises publication, verification and managed Axum serving. See the [guide](../../guide/native-geo.md#reproduce-the-local-backend-proof) for the full command. The local run used Rust 1.98.1 debug binaries, official celld 0.6.3 and DynamoDB Local 3.3.1. The listener fixture was built at `1b7eeb89` with `geo-kv,celld-kv,aws-dynamodb,test-utils`; the later compiled-out lint change did not change the running fixture path. The operator tool was built from `7c0cd31e`. Both sample sizes were explicit command inputs, independent of the tool's default. Executables were copied to stable paths and their SHA-256 digests were checked again at the end of the run. + +| Executable | SHA-256 | +| ---------------- | ------------------------------------------------------------------ | +| Stock celld | `eb0a8158421b69d68dd5d28f2298a7d411dde498295833deeb1cc5ce53b7eb75` | +| Publisher | `f9eee2fe8ab97ae560eac6334f7ce9891dd0f52d7d4463895e11741f8aaedd3d` | +| Listener fixture | `fdd5252dacba39bee572920a2973138d77cf2f990016eb33589277a84ea04275` | + +## Correctness and lifecycle + +Both backends served the same synthetic artifacts. The runner checks interrupted writes, identical replay, immutable conflicts, read-only serving, verified IPv4 and IPv6 results, startup rejection, request errors and deadlines, simultaneous pins, restart, rollback and retirement. It verifies celld denying a write under a read-only token. DynamoDB Local confirms the serving code issues no writes, but does not enforce hosted IAM. + +Both backends then published matching 16 KiB and 64 KiB artifacts built once per size from the `GeoLite2-City` CSV release dated 2026-11-03. The proof checked every object through the serving binding and found no mismatches in 4,096 IPv4/IPv6 oracle comparisons per backend and size. The CSV and derived objects are not checked in. + +| Object budget | Objects | Encoded bytes | Inventory SHA-256 | +| ------------- | ------: | ------------: | ------------------------------------------------------------------ | +| 16 KiB | 2,595 | 42,386,355 | `f840b300927cc6f2075d5f8a20710de355059fee5d363d5cbc6cf6e053939c24` | +| 64 KiB | 649 | 42,259,651 | `7ec62707c1ba18204fe054222751f6d5330dfdd3428c6849b72a2c0660ca57c0` | + +## Measurements and default + +The HTTP workload contains 128 addresses sampled from the source blocks and sends four requests at a time. The first pass starts with an empty object cache. The second pass repeats those addresses in the same order. RSS is the managed host's process high-water mark. Response bytes include the backend's serialization. + +| Backend | Budget | Startup | Peak host RSS | First/repeat p95 | First/repeat KV reads per request | First/repeat response bytes per request | +| -------------- | -----: | ------: | ------------: | ---------------: | --------------------------------: | --------------------------------------: | +| celld | 16 KiB | 106 ms | 29.9 MiB | 48.0 / 1.1 ms | 1.25 / 0.00 | 19,151 / 0 | +| celld | 64 KiB | 106 ms | 31.7 MiB | 53.4 / 54.5 ms | 1.06 / 1.06 | 65,769 / 65,769 | +| DynamoDB Local | 16 KiB | 105 ms | 34.2 MiB | 29.4 / 1.9 ms | 1.27 / 0.00 | 26,130 / 0 | +| DynamoDB Local | 64 KiB | 106 ms | 37.6 MiB | 26.0 / 25.6 ms | 1.06 / 1.06 | 87,905 / 87,905 | + +Default to 16 KiB. It transfers less data and cached the entire repeated working set in this run. The 64 KiB build costs fewer writes to publish and needs at most two object reads, versus three for 16 KiB. An operator can choose it explicitly. These local results do not establish production latency, traffic capacity, hosted IAM or fleet durability. diff --git a/docs/superpowers/specs/2026-11-05-kv-geo-design.md b/docs/superpowers/specs/2026-11-05-kv-geo-design.md new file mode 100644 index 000000000..39324d301 --- /dev/null +++ b/docs/superpowers/specs/2026-11-05-kv-geo-design.md @@ -0,0 +1,209 @@ +# Portable KV-backed Geo + +- Status: design for [#454](https://github.com/IABTechLab/edgezero/pull/454). The prototype is measured; managed serving and backend proofs are not implemented or verified. +- Drafted: 2026-11-05. Revised: 2026-11-17. +- Dependencies: [native Geo providers (#439)](https://github.com/IABTechLab/edgezero/pull/439) and [Axum Geo manifest/settings selection (#502)](https://github.com/IABTechLab/edgezero/pull/502). Reconcile the shared settings and outcome changes before adding KV selection. +- Merge gate: reviewers must agree that the KV checks in [§10](#10-acceptance-and-delivery) replace the MMDB-specific requirements in [#438](https://github.com/IABTechLab/edgezero/issues/438) before claiming closure. + +## 1. Outcome and decisions + +Consumers can resolve an admitted reader IP to an estimated country and optional first-level subdivision without platform-provided geography, a runtime filesystem, or loading the entire dataset into memory. They supply a populated KV store through EdgeZero's existing store abstractions. + +Use GeoLite CSV as the initial import format. Transform it outside the serving runtime into an immutable, versioned IP index. Resolve requests asynchronously through `KvHandle`, with optional bounded in-memory caching. Geography remains an estimate; applications own jurisdiction recognition and policy. + +#454 ships opt-in managed Axum serving with a portable `KvHandle` reader. Stock celld with the EdgeZero store service and DynamoDB Local are the first end-to-end proof backends, not dependencies of Geo. Publish the same artifact to both. A compatible binding must support exact-key byte reads, fit the artifact's key and value limits, and make the pinned generation visible to readers before rollout. This design makes no performance claim over MMDB. + +Required invariants: + +- A deployment explicitly selects zero or one Geo provider. There is no automatic source selection or fallback. +- One immutable Geo outcome is resolved after ingress admission and before application middleware and handlers. +- The adapter owns reader-IP trust. The resolver receives a typed, admitted IP rather than parsing request headers. +- Request lookup uses exact-key reads, never key listing, full-dataset loading or per-prefix probing. +- Every request uses one pinned dataset generation, including cache hits and misses. +- Missing or corrupt required index data is a provider failure, not an unmatched address. +- Runtime Geo resolution requires no filesystem, local sidecar, MaxMind credentials or downloader. +- Portable code gains no native SDK, Tokio dependency, mandatory `Send` future or platform-specific timer. + +Preserve protected CloudFront and supplied providers. Fastly, Cloudflare, and Spin keep their existing Geo behavior; their managed KV Geo integration is separate work. Trusted Server wiring and deployment are also separate. + +## 2. Ownership and flow + +| Concern | Owner | +| ------------------------------------------------------------ | -------------------------------------- | +| Dataset acquisition, download integrity and licensing | Operator's update process | +| CSV validation, geographic projection and index construction | Offline importer | +| KV resource creation, credentials and backend bindings | Existing platform tooling and operator | +| Index lookup and decoding | Portable KV Geo implementation | +| Socket/platform client identity and forwarding trust | Adapter ingress | +| Per-request outcome and geographic value types | Portable Geo contract | +| Jurisdiction, consent, permissions and response caching | Application | + +```mermaid +flowchart TD + CSV[GeoLite City or Country CSV release] --> Import[Offline validation and index construction] + Import --> Publish[Publish immutable KV objects] + Publish --> Verify[Read back and verify generation] + Verify --> Select[Deploy with pinned manifest identity] + Request[Incoming request] --> Ingress[Adapter admission and checked reader IP] + Ingress --> Resolver[Portable async Geo resolution] + Select --> Resolver + Resolver --> Cache{Index object cached?} + Cache -->|Yes| Search[Search validated object] + Cache -->|No| Read[Read exact KV key and validate object] + Publish --> Read + Read --> Search + Search --> Next{Directory or leaf?} + Next -->|Directory| Cache + Next -->|Leaf or address gap| Outcome[Immutable request Geo outcome] + Outcome --> App[Application middleware and handlers] +``` + +## 3. Import contract + +Initially support GeoLite2 City and Country CSV releases. City is needed for subdivisions. For City, consume these files from the same archive: + +```text +GeoLite2-City-Blocks-IPv4.csv +GeoLite2-City-Blocks-IPv6.csv +GeoLite2-City-Locations-en.csv +``` + +Country uses the corresponding Country filenames and produces country-only locations. Read columns by header name so unrelated added columns do not change the projection. + +Join block `geoname_id` to location `geoname_id`. Keep `country_iso_code` and, for City, `subdivision_1_iso_code`. Do not substitute `registered_country_geoname_id` or `represented_country_geoname_id`; those identify different geographic facts. Names, coordinates, postal codes and network registration metadata are not needed in the serving index. + +Validate address-family membership, CIDRs, duplicate location identities, referenced locations and geographic code syntax. A populated reference with no matching location is an import error. An absent geolocation reference or location without country yields no geographic coverage, not a guessed country. A subdivision without country is invalid. Country-only locations remain partial; subdivision validation checks syntax, not worldwide assignment or application recognition. + +Preserve syntactically valid source codes that core does not recognize, including `XK`, in the source-format record. A matching interval with such a country yields unsupported geography. It is neither a gap nor a backend failure and does not change the `CountryCode` allowlist. + +Convert networks into sorted numeric intervals, separately for IPv4 and IPv6. Reject conflicting or overlapping source intervals rather than inventing precedence. Adjacent intervals with identical projected locations may be merged. Preserve gaps. Use overflow-safe address arithmetic, including the last address in each family. + +The importer emits deterministic objects, their keys and digests, and a publication inventory. Metadata records the dataset edition, source release identifier, source file digests, index schema version and importer version. Repeated import with the same source and format settings produces the same object bytes. + +The importer may use a filesystem and native tools. Those requirements belong to the publishing environment, not deployed applications. MaxMind authentication stays outside the serving runtime. + +## 4. KV index + +### Object layout + +Use a versioned tree of directory objects and leaf objects, separately addressable through exact KV keys. A small manifest identifies both family roots and the schema. Empty families are explicit. + +Conceptual keys: + +```text +geo/v1//manifest +geo/v1//v4/directory/ +geo/v1//v4/leaf/ +geo/v1//v6/directory/ +geo/v1//v6/leaf/ +``` + +A leaf contains sorted, disjoint inclusive intervals and their country and optional subdivision. A directory contains sorted, disjoint child envelopes and expected content digests. Derive each exact child key from the pinned generation, address family and digest rather than accepting arbitrary key strings from stored data. Envelopes include any internal gaps; the selected leaf confirms actual coverage. A directory gap proves no coverage without another read. + +Build leaves by encoded byte size, then build directories over contiguous groups until each family has one root. Additional directory levels allow the directory itself to exceed one KV value without requiring a whole-directory preload. Lookup chooses at most one child at each level by binary search. It never probes multiple prefixes or scans neighboring leaves. + +Use a versioned binary encoding with explicit record counts, address-family widths and network-byte-order addresses. Store locations with intervals to avoid a second read. Compare 16 KiB and 64 KiB objects on both proof backends before freezing the production schema and defaults. Record the resulting encoding and limits; the experimental `p1` encoding is not a stable contract. + +### Limits and validation + +All accesses go through `KvHandle`. Build to a key and value budget that fits the intended backends, accounting for storage overhead. The portable 25 MiB ceiling does not override the stock celld service's 1 MiB value ceiling or the DynamoDB binding's 384 KiB ceiling. Operators can configure lower limits. Celld caps bytes while collecting a response; DynamoDB checks its configured value cap after the AWS SDK decodes the item. Measure SDK allocations separately rather than treating the index limit as a wire limit. + +The reader validates schema, digest, encoded lengths, counts, ordering, interval bounds, child bounds and geographic values before using an object. Child keys must remain within the selected generation and address family, and children must fit the parent's declared envelope. Expected digests bind decoded descendants to the selected manifest. Hashes identify bytes, not publisher authenticity. + +Enforce limits for manifest and object bytes, record counts, tree depth, retained cache bytes and entries, and concurrent lookups. Include transport buffers, validation scratch and objects still held after eviction in memory measurements. Derive defaults from the backend limits and measured index shape. Reject an incompatible dataset rather than truncating it or skipping validation. + +## 5. Async resolution and request lifecycle + +Keep the public synchronous `GeoProvider` and assertion-only `GeoInput` unchanged. Axum's internal prepared provider selection distinguishes synchronous assertion providers from the async KV resolver. The portable reader takes a typed IP and owns its configured `KvHandle`; it does not get a request, headers or a service registry. No public async plugin trait, blocking KV bridge, native SDK or mandatory `Send` future is needed. + +Use the existing country and subdivision types and `RequestContext::geo()` as the consumer interface. Record reader-IP provenance without exposing the IP itself. Document new outcome variants and any exhaustive-match migration. + +Managed request sequence: + +1. Perform existing ingress validation, forwarding normalization and admission. Rejected or aborted ingress makes no Geo lookup. +2. Take the reader IP from checked ingress facts, including its provenance. Missing usable reader input produces `MissingReaderIp` without provider calls. +3. Await KV Geo once under an Axum-owned concurrency and time budget, separate from the body-read deadline. If lookup cannot start within that budget, return provider failure without entering an unbounded queue. +4. Attach one owned outcome before request preparers, application middleware and handlers. Response finalization uses that outcome without resolving again. + +A lookup follows at most one path, with up to one read per directory level plus one leaf read. Manifest acquisition is startup work. Concurrent cold requests may duplicate reads; request coalescing is not required. + +Geo adds no retry loop or detached task. Document backend retry and timeout behavior separately. The adapter owns deadline and cancellation mechanics. Dropping the request future stops Geo work owned by that request; it does not guarantee the backend aborted physical I/O. + +| Condition | Outcome | +| --------------------------------------------------------------- | ------------------------------ | +| Geo not selected | `NoSelection` | +| Selected, no admitted usable reader IP | `MissingReaderIp` | +| Validated directory or leaf proves address gap | `NoMatch` | +| Matched country, absent subdivision | Located, country only | +| Matched country and subdivision | Located, estimated subdivision | +| Matched interval contains an unsupported source country code | Unsupported geography | +| Expected object absent, invalid, unsupported or digest mismatch | Provider failure | +| KV error, saturation or Geo lookup deadline exhausted | Provider failure | + +Provider failure does not choose CloudFront, another dataset, or an older cache generation. Outcomes do not automatically choose HTTP status or permission policy. + +## 6. Configuration and adapter portability + +Extend #502's Axum selection path: `with_geo` overrides settings, then the Axum manifest, then Off. KV takes a dedicated logical store ID, a pin containing both generation and manifest digest, and a reader-IP mode. Override the pin as a unit. The selection contains no physical table, credentials or filesystem path. Reject undeclared bindings, incompatible limits and mixed provider fields. Do not silently select the application's default KV store. + +Resolve the named `KvHandle` once after store preparation. Under a bounded startup budget, fetch and validate the pinned manifest and both nonempty roots through the serving binding before readiness. If those reads fail, the selected provider prevents readiness. Full-tree verification belongs to rollout, not every startup. A missing or corrupt required descendant after startup remains a provider failure. + +Require one reader-IP mode: + +- **Direct peer** is for a listener that receives reader connections directly. Use the checked socket peer, not a forwarding or geographic header. +- **Forwarded required** uses the normalized effective client only when its client provenance is trusted forwarding. Checking that an IP exists is not enough: normalization can retain the socket peer on forwarding failure. A checked client can also appear in a `Partial` forwarding summary when other fields are absent, so use the client provenance rather than the summary alone. + +Preserve IPv4-mapped IPv6 normalization. A missing or untrusted reader input never falls back to the proxy's IP. Portable lookup does not imply trustworthy reader-IP acquisition on any other managed adapter. + +## 7. Cache behavior + +Caching is optional and bounded by bytes and entry count. Cache validated immutable objects by digest within one prepared binding and generation. Eviction only causes another KV read. With the same available generation, cache-disabled and warm lookups must produce the same geography. + +Use in-memory caching where the runtime permits it, without requiring cross-request or cross-instance persistence. Do not cache reader IPs or add an IP-to-location cache. A missing expected object is a publication/storage failure, not a negative geolocation cache entry. + +Retiring a dataset includes stopping or reconfiguring instances that retain its objects in memory. A version pin prevents accidental mixing; it does not establish freshness or permission to keep using old data. + +## 8. Publication and lifecycle + +Extend the [standalone tool](../../../examples/geo-kv-prototype/README.md) with offline `build`, plus `publish` and remote `verify` using the existing ordinary celld and DynamoDB preparation APIs. Only handle preparation knows the backend. Leave provisioning and credentials with the operator. The Geo serving identity is read-only; the operator may both publish and retire. No third identity, new authorization code, dedicated DynamoDB table or celld fork is required. + +1. **Build.** Produce the exact key/value inventory and a generation/digest receipt from one permitted CSV release. +2. **Publish.** Assign one publisher to each generation. Read each object first: skip identical bytes, write only if absent, and abort on different bytes. Verify the read-back of new writes. Publish the manifest last. An interrupted or uncertain write is reconciled by rerunning the same inventory. Ordinary `KvHandle` writes are unconditional; read-before-write is not a lock. +3. **Verify and select.** From the serving path, validate the full reachable tree through the serving binding and compare sampled lookups against the independent source oracle. One uploader's read-back or a manifest written last does not prove that other serving locations can read every object. Check visibility at the serving locations before rollout. Deploy the pin and restart canaries, then the fleet. Rollback re-verifies and restores a previous pin only while it is still permitted. There is no mutable `active` key or request-time fallback. +4. **Retire.** After readers stop referencing a generation, delete it by inventory within the applicable retention deadline. Include artifacts, caches and backups; don't use TTL as evidence of deletion. Keep licensed inputs and outputs out of Git and release images. Keep MaxMind credentials off serving hosts and document attribution and permitted use under the [GeoLite EULA](https://www.maxmind.com/en/geolite/eula). + +## 9. Diagnostics + +Expose bounded categories for configuration, initialization and lookup failures. Record schema and loaded dataset identity in startup/initialization diagnostics, not per-IP labels. Useful metrics include lookup latency, KV reads, cache hits, bytes decoded and outcome category. + +Do not retain reader IPs, raw forwarding headers, credentials or raw KV errors in Geo outcomes or default diagnostics. KV keys and values need the existing storage logging discipline; index keys encode dataset structure, not individual request addresses. Dependency logging is a separate operator concern. + +## 10. Acceptance and delivery + +Complete and record these checks before merging #454: + +1. **Deterministic CI.** Use synthetic City and Country fixtures. Prove deterministic bytes and compare IPv4/IPv6 boundary, neighbor, gap and sampled lookups against the independent CIDR oracle. Cover country-only, subdivisions and unsupported geography. Reject malformed imports and corrupt objects. Exercise cache disabled and warm, eviction and bounds. Include the standalone crate in CI and verify the portable reader's WASM dependency graph stays free of the importer and native SDKs. +2. **Managed Axum.** Prove selection precedence, Off and existing CloudFront/supplied behavior, direct and trusted forwarded input, mapped IPv4, and zero KV reads for rejected ingress or missing reader. Reject bad manifest/root data, unavailable bindings and denied reads before readiness. Exercise missing or corrupt descendants, deadlines, cancellation and concurrent limits. Assert the same immutable outcome reaches application dispatch and finalization without a second lookup. +3. **Two backend runs.** Publish the same artifact to stock celld with the EdgeZero service and DynamoDB Local. Verify through each serving binding and drive requests through a managed Axum listener. Cover interrupted and identical publication, conflicting bytes, incomplete visibility, simultaneous A/B pins, restart, rollback and inventory retirement. Prove stock celld denies writes under the serving grant. For DynamoDB, prove serving issues no writes and document a minimal IAM policy with `DescribeTable` and `GetItem`. Local DynamoDB does not enforce IAM; actual denial under a deployed role is hosted qualification. +4. **Licensed-size evidence.** Use a permitted production-size City release on both test backends. Compare 16 and 64 KiB objects, verify sampled source-oracle agreement, and report encoded size, object count, depth, startup time and RSS, cold and warm lookup latency, reads and bytes per request, and concurrent peak RSS alongside the enforced limits. Record exact commands, tested revision, and source and artifact digests. Synthetic Country coverage does not qualify production-size Country data. + +Managed AWS IAM, stock-celld fleet durability, production capacity and deployment latency targets require separate qualification. Neither local proof demonstrates them. Check the production owner or hot partition under the intended traffic: stock celld has one owning object per namespace, and DynamoDB uses the namespace as its partition key. + +## 11. Prototype findings + +The [standalone prototype](../../../examples/geo-kv-prototype/README.md) and [measurement record](../plans/2026-11-05-kv-geo-prototype-results.md) exercise the downloaded City CSV through `KvHandle` using a local artifact-store harness. A 64 KiB object budget produced about 40.3 MiB across 649 objects, with at most two object reads per lookup after manifest acquisition. This is not deployed-provider performance evidence or a frozen wire format. + +The measured CSV contains `XK`, which core rejects. The prototype preserved the source code and reported the mismatch. The serving contract in §3 keeps that distinction as unsupported geography. + +## 12. Scope boundaries and references + +Deferred: managed KV Geo outside Axum, hosted deployment qualification, Trusted Server selection and consent-path proof, automatic download/update scheduling, live version switching, remote geolocation APIs, richer location fields and MMDB support. The application continues to own jurisdiction, consent, permissions and response caching. + +- [GeoLite formats and account requirements](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data/). +- [CSV fields and join semantics](https://dev.maxmind.com/geoip/docs/databases/city-and-country/). +- [Account downloads](https://www.maxmind.com/en/accounts/current/geoip/downloads), choose City CSV for subdivisions. +- [Dummy City CSV sample](https://dev.maxmind.com/examples/GeoIP2-City-CSV_Example.zip). +- [KV interface and provider consistency](../../guide/kv.md). +- [Native DynamoDB limits and semantics](../../guide/dynamodb-kv.md). +- [Native celld store service and grants](../../guide/native-celld-stores.md). +- [Native startup lifecycle](2026-10-05-axum-production-startup-lifecycle-design.md). diff --git a/examples/geo-kv-prototype/Cargo.lock b/examples/geo-kv-prototype/Cargo.lock new file mode 100644 index 000000000..f304b74c0 --- /dev/null +++ b/examples/geo-kv-prototype/Cargo.lock @@ -0,0 +1,3128 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "arc-swap" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b5d51209f6de83d37a0b33d320acb8da88823e42a94c075140708f032a5" +dependencies = [ + "rustversion", +] + +[[package]] +name = "async-compression" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" +dependencies = [ + "compression-codecs", + "compression-core", + "futures-io", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "aws-config" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d7b388a9fc3a6db15a5ec778c38b354eff1364882c94d08e0252f7a47dcaa4" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 1.4.2", + "sha1", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bfbe32ac36b3c428696ce315464181bc4029d689fd58e3f2a4e58c252242f52" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "aws-runtime" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b8a9911551b4ea6ca13805ef52ed96f7d2bbb43cc3b4a14cb0776a71f33cfaa" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http 1.4.2", + "http-body", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-dynamodb" +version = "1.120.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f578638cafe182113da770e9798e7063d6de96cbbfe06f647d0026f197c9b97c" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "regex-lite", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.114.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12be2f9c8eef7f5fc919c96d538e629698469a02b4cb75408b26b1bd984ebe79" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.4.2", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.116.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7645db8724ea3b82fdccfb67e1b0f637c9d8ab0e7ef29d84884d8d8d73f805d" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 1.4.2", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.119.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e490aa904849b38e770922faac779dd245f5cf8be81f19065acf18d276e4ad" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 1.4.2", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2312577f088c9fbf4206dfdb884cf1de9407b43e1a923cbed5237775116fc24b" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 1.4.2", + "percent-encoding", + "sha2", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "639b4d8f8555f24a9be649811c3eb0b4d4616f4d61daf0c32e28873bc1ea9af1" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c89cc3f1f281d659a67a519a1b5c6d445b5ce09fa7e5aee40c2c2707e9509d" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2", + "http 1.4.2", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.63.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3385d469edbe8b60cc72002784652b5efca39178192aa9cc4b44c9875c6bdc18" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d1d71f6562be974caa85442ecd90194c40fdb5df045f182a6c2e872ce95056" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6e302ac1d88b99652489df31abdec6ac42a2ab2ac3982ad0ac49f64dfaf28ba" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http 1.4.2", + "http-body", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0730c16f91124c6a2abb4932c77e299288b3dd9f967ea2e9ec48cc6731e87a4" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http 1.4.2", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce237f890a2283c9b328955c3b0918d2d5bf66a1cb6ceccd36a0047246fbc6e6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f395d93304280b64b7632fea798d177e74897fe7f063416ce627cd6fa24829" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.2", +] + +[[package]] +name = "aws-smithy-types" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aff4bf014c512840b2bba80a20417b723cbd6a0a65c02e2d81f50b4268aedfc7" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 1.4.2", + "http-body", + "http-body-util", + "itoa", + "pin-project-lite", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b932c8d6dc127fc980eecd78f8694ae9b9551b69a93a7def2a199c1c0033daf" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209f3a6d82a6e9e5f94abbed94c7a26e1c052341002bf57a5fb5481f625896fc" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5962523e1b92ce1b5e793d9169b9943eece10d39f62550bc04bb605d75b94924" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "cc" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ef9e3e8154a45c7a95cff4c1299702106ad94062d7bb85fdc5d3a5afd91d45" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "ctutils" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f" +dependencies = [ + "cmov", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.6", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "edgezero-core" +version = "0.1.0" +dependencies = [ + "adler2", + "alloc-no-stdlib", + "alloc-stdlib", + "anyhow", + "async-compression", + "async-stream", + "async-trait", + "base16ct", + "brotli", + "brotli-decompressor", + "bytes", + "compression-codecs", + "compression-core", + "crc32fast", + "edgezero-macros", + "flate2", + "futures", + "futures-util", + "http 1.4.2", + "http-body", + "log", + "matchit", + "miniz_oxide", + "ryu", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha2", + "simd-adler32", + "thiserror", + "toml", + "tower-service", + "tracing", + "url", + "validator", + "web-time", +] + +[[package]] +name = "edgezero-macros" +version = "0.1.0" +dependencies = [ + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 3.0.3", + "toml", + "validator", +] + +[[package]] +name = "edgezero-store-aws" +version = "0.1.0" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-dynamodb", + "aws-smithy-runtime-api", + "aws-types", + "bytes", + "edgezero-core", + "percent-encoding", + "serde", + "serde_json", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "edgezero-store-celld" +version = "0.1.0" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "edgezero-core", + "futures-util", + "getrandom 0.4.2", + "reqwest", + "serde", + "serde_json", + "sha2", + "thiserror", + "tokio", + "url", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35d6e78f4cebb90e51d543db8fe41528851d6deeef6ef21ebd4bfdc1e784776" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "geo-kv-prototype" +version = "0.0.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "csv", + "edgezero-core", + "edgezero-store-aws", + "edgezero-store-celld", + "futures", + "ipnet", + "serde", + "serde_json", + "sha2", + "tempfile", + "tokio", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasip2", + "wasip3", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.2", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.4.2", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.2", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http 1.4.2", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http 1.4.2", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http 1.4.2", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +dependencies = [ + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.2", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "matchit" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8863b587001c1b9a8a4e36008cebc6b3612cb1226fe2de94858e06092687b608" + +[[package]] +name = "memchr" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1" + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bee6c73da26345c729282832b60b0363cf3dd9f4bfd81d8551b7a1c889a113" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.2", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6fa277c67e6ea590466f04c3fe82cb509f0864fe470643e5570de9514a9b11" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "102f35f7de7b06d8c61f1cff6592f5253e0dea431c74867aad5dfd70f44663fb" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a0760a887c14823fca5a1c970b01d47c328c696c0c64829d1df50e8f853fd5" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http 1.4.2", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.29.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c14b15dffa2f4bd712103e857140aa04a4f2ff6e955c77d7eb8e53ee02a3b64" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "validator" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d68c6633c483df6780cc5277a417c7c2d1bceee2649d06c8ab6b0fd2dd3c81" +dependencies = [ + "idna", + "regex", + "serde", + "serde_derive", + "serde_json", + "url", + "validator_derive", +] + +[[package]] +name = "validator_derive" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7df16e474ef958526d1205f6dda359fdfab79d9aa6d54bafcb92dcd07673dca" +dependencies = [ + "darling", + "once_cell", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen 0.57.1", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen 0.51.0", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.3", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "web-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "293198dcebfb821155b5092ab94b5a82ff180eddd7241df3bd6e66a4ffd1a585" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn 2.0.119", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.119", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "yoke" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ccff8a4805aba1eaa4cd4afcd49f7557dd2cf5fd79ffc37f8959466effc5aa" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/examples/geo-kv-prototype/Cargo.toml b/examples/geo-kv-prototype/Cargo.toml new file mode 100644 index 000000000..649d3acdc --- /dev/null +++ b/examples/geo-kv-prototype/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "geo-kv-prototype" +version = "0.0.0" +edition = "2024" +publish = false + +# Offline tooling stays outside the production workspace. +[workspace] + +[features] +default = [] +celld = ["dep:edgezero-store-celld", "dep:tokio"] +dynamodb = ["dep:edgezero-store-aws", "dep:tokio"] + +[dependencies] +anyhow = "1" +async-trait = "0.1" +bytes = "1" +csv = "=1.4.0" +edgezero-core = { path = "../../crates/edgezero-core", default-features = false, features = ["geo-kv-build"] } +edgezero-store-celld = { path = "../../crates/edgezero-store-celld", features = ["kv"], optional = true } +edgezero-store-aws = { path = "../../crates/edgezero-store-aws", features = ["dynamodb-kv"], optional = true } +tokio = { version = "1", features = ["rt", "time", "net"], optional = true } +futures = "0.3" +ipnet = { version = "2", features = ["serde"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.11" +tempfile = "3" diff --git a/examples/geo-kv-prototype/README.md b/examples/geo-kv-prototype/README.md new file mode 100644 index 000000000..6931a05dd --- /dev/null +++ b/examples/geo-kv-prototype/README.md @@ -0,0 +1,156 @@ +# KV Geo offline tools + +Standalone build, publication and verification for [portable KV Geo](../../docs/superpowers/specs/2026-11-05-kv-geo-design.md). CSV import and the independent CIDR oracle stay here. `src/index.rs` selectively re-exports the core `geo::kv` encoder and reader, so the tool and serving code use the same `geo/v1` production-candidate format. Old `geo/p1` artifacts are not compatible; rebuild them. + +This crate has its own workspace, lockfile and target directory. It does not provision resources or deploy an application. Native backends are optional dependencies, absent from default builds. The serving reader does not depend on this crate, CSV, the oracle, a filesystem or native SDKs. + +## Build and local checks + +From the repository root: + +```sh +MANIFEST=examples/geo-kv-prototype/Cargo.toml +TARGET=examples/geo-kv-prototype/target +cargo test --locked --manifest-path "$MANIFEST" --target-dir "$TARGET" +cargo build --locked --release --manifest-path "$MANIFEST" --target-dir "$TARGET" +BIN="$PWD/$TARGET/release/geo-kv-prototype" + +SOURCE="$HOME/.cache/edgezero/geo/source/GeoLite2-City-CSV_RELEASE" +OUT="$HOME/.cache/edgezero/geo/artifacts-64k" +"$BIN" build "$SOURCE" "$OUT" +"$BIN" verify "$SOURCE" "$OUT" 2048 4194304 +"$BIN" lookup "$OUT" 1.0.1.1 ::1 +``` + +Extract one permitted GeoLite City or Country release outside the repository. Supply the English Locations file and both Blocks files from that release. Import validates named columns, family membership, CIDRs, overlaps, referenced locations and geographic code syntax. It does not substitute registered or represented countries. City imports the first subdivision; Country produces country-only results. Empty references or locations without country leave gaps. + +`build` merges adjacent equal locations and exports byte-bounded objects through `KvHandle`. It reads the inventory back, validates the full reachable tree, and writes: + +- `receipt.json`, the exact manifest key and SHA-256 deployment identity. +- `inventory.json`, every exact key, byte count and SHA-256, including the manifest. +- `build-report.json`, source hashes, index shape, size, timings and whole-process peak RSS. + +The default object budget is 16 KiB. In the [local two-backend proof](../../docs/superpowers/plans/2026-11-17-kv-geo-serving-results.md), the smaller objects fit the repeated-request working set and avoided further KV reads. A 64 KiB budget reduces object count and maximum path reads. Use separate output directories when comparing sizes. Keep the source, receipt and inventory together. Rerunning an interrupted build with identical inputs reuses complete objects. If an existing object differs, build into a fresh directory and compare the receipt before publishing. + +`verify` checks inventory closure and the full local tree, then compares cache-disabled, cache-enabled first-pass and repeated-pass lookups with the independent original-CIDR oracle. Samples include family endpoints, stratified CIDR boundaries and neighbors, plus deterministic random points. IPv4-mapped IPv6 follows serving normalization. It writes `verification-report.json`. + +`lookup` opens only the receipt and the local reader, without CSV or inventory loading. Results follow argument order without echoing addresses. `null` means no coverage. Required missing or corrupt objects fail the command. `XK` remains a source location with `core_country_supported: false`, rather than becoming a gap or an invented core country. + +## Publish the same artifact to native backends + +Build the native tool once: + +```sh +cargo build --locked --release --manifest-path "$MANIFEST" \ + --target-dir "$TARGET" --features 'celld dynamodb' +``` + +A build with just `--features celld` or `--features dynamodb` works too. Native commands run on a current-thread Tokio runtime; the default offline tool uses the portable executor. + +```mermaid +flowchart TD + Build[CSV release builds one local artifact] --> Check[Validate receipt, inventory and full local tree] + Check --> Publish[Publisher binding reads each remote key] + Publish --> Existing{Remote bytes exist?} + Existing -->|Identical| Skip[Skip write] + Existing -->|Different| Abort[Abort, never overwrite] + Existing -->|Absent| Write[Write and read back exact bytes] + Skip --> Manifest[Process manifest last] + Write --> Manifest + Manifest --> Serving[Separate serving binding verifies full tree] + Serving --> Oracle[Compare sampled lookups with local source oracle] + Oracle --> Rollout[Operator selects pin after serving-location checks] +``` + +`publish ` validates all local bytes before preparing a backend. Inventory keys must belong to the pinned generation, match their hashes and exactly cover the reachable tree. Duplicate, omitted and extra objects fail validation. Each object is checked again when read for upload. Backend value limits are checked before publication. + +Remote publication reads before writing, skips identical bytes, aborts on different bytes, writes only absent objects, and reads new writes back. It processes the manifest last regardless of inventory ordering. After interruption or an uncertain write, rerun **the same command with the same artifact and inventory**. There is no internal retry loop, mutable `active` key, overwrite repair or fallback generation. Assign one publisher per generation. Ordinary KV writes are unconditional; read-before-write is not a lock against another publisher. + +`verify-remote ` opens the pinned manifest through the serving binding, verifies every reachable object and compares samples with the local CSV oracle. It issues only exact-key reads, never writes or listing. Source CSV is read by this offline verification process, not sent to the backend or loaded by serving applications. The source hashes must match the manifest. Success prints the pin, object count and checked sample count. + +### Ordinary celld + +Provision stock celld, install/configure the existing EdgeZero store service and create the alias and grants through existing operator tooling. See [native celld stores](../../docs/guide/native-celld-stores.md). No celld fork, special Geo service or namespace provisioning is built into this tool. + +Publisher settings, stored outside Git: + +```json +{ + "backend": "celld", + "settings": { + "endpoint": "https://celld.example.test:8443", + "store": "geo", + "access": { "mode": "authenticated", "token_env": "GEO_PUBLISH_TOKEN" } + }, + "limits": { "max_value_bytes": 1048576 } +} +``` + +Serving settings use the same endpoint and alias with `token_env: "GEO_SERVING_TOKEN"`. Inject token values through the environment, not JSON, arguments or shell history. The publisher needs read/write access. The serving grant must be read-only. The ordinary settings validator requires HTTPS for authenticated access. Local trusted-network experiments can use `"access": { "mode": "trusted-network" }` with an explicit loopback HTTP endpoint, but that does not prove grant enforcement. + +```sh +"$BIN" publish "$OUT" /secure/geo-celld-publisher.json +"$BIN" verify-remote "$OUT" /secure/geo-celld-serving.json "$SOURCE" 2048 +``` + +Preparation uses `CelldPreparation::prepare_kv` with optional `CelldPreparationLimits`. It does not probe permissions. Publication and serving verification exercise actual reads. Prove stock celld denies writes under the serving grant separately using existing service tooling; this verifier intentionally does not attempt a write. + +### DynamoDB and DynamoDB Local + +Provision an ordinary compatible table and namespace outside this tool. See [DynamoDB KV](../../docs/guide/dynamodb-kv.md) for table keys and value layout. Do not create a dedicated Geo table merely for this workflow. + +```json +{ + "backend": "dynamodb", + "settings": { + "region": "us-east-1", + "table": "edgezero-kv", + "namespace": "geo", + "limits": { "max_value_bytes": 393216 } + } +} +``` + +The tool calls `AwsPreparation::prepare_dynamodb`, including its ordinary `DescribeTable` validation, with ambient AWS credentials. `AWS_ENDPOINT_URL_DYNAMODB` selects DynamoDB Local through the SDK's normal configuration. For a disposable local instance only: + +```sh +AWS_ENDPOINT_URL_DYNAMODB=http://127.0.0.1:8000 \ +AWS_ACCESS_KEY_ID=local AWS_SECRET_ACCESS_KEY=local \ + "$BIN" publish "$OUT" /secure/geo-dynamodb-publisher.json +AWS_ENDPOINT_URL_DYNAMODB=http://127.0.0.1:8000 \ +AWS_ACCESS_KEY_ID=local AWS_SECRET_ACCESS_KEY=local \ + "$BIN" verify-remote "$OUT" /secure/geo-dynamodb-serving.json "$SOURCE" 2048 +``` + +Use a serving role allowing only `dynamodb:DescribeTable` and `dynamodb:GetItem` on the selected table ARN, with no write grants. A minimal identity policy is: + +```json +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["dynamodb:DescribeTable", "dynamodb:GetItem"], + "Resource": "arn:aws:dynamodb:us-east-1:123456789012:table/edgezero-kv" + }] +} +``` + +Check the role's other attached policies and resource policies too. The publishing operator also needs `PutItem`; retirement needs `DeleteItem`. DynamoDB Local does not enforce IAM, so a local read-only verifier proves it issues no writes, not that a deployed role denies them. Actual role denial remains hosted qualification. This tool adds no retries; ordinary backend operation timeouts and preparation limits still apply. + +## Rollout, retention and output safety + +Uploader read-back and a manifest written last do not prove propagation to all serving locations. Run full verification through the actual serving identity and path at the required locations before selecting the pin. The deployment pin is `@`, using the generation from `geo/v1//manifest`. Restart canaries and then the fleet through existing deployment tooling. Rollback requires re-verifying a still-permitted previous pin. There is no automatic runtime fallback. + +Retire only after readers stop referencing a generation. Use its exact inventory with existing operator deletion tooling, and include local artifacts, source files, retained caches and backups in the retention process. TTL is not deletion evidence. This tool does not delete remote data or prove retirement. + +Do not put licensed CSV, derived artifacts, reports or credentials into Git or release images. Follow the [GeoLite EULA](https://www.maxmind.com/en/geolite/eula), attribution and permitted-use obligations. Geography is an estimate; applications own jurisdiction recognition and policy. + +Successful commands print aggregate reports and dataset identity, never reader addresses or credentials. Errors print a fixed failure message and usage, not raw SDK, CSV, filesystem or parsing errors. Dependency logging is an operator concern. Explicit `lookup` results disclose geography in input order; use synthetic addresses when recording evidence. + +## Format and measurement limits + +Core owns the `EZG1` binary encoding and schema validation. Objects use sorted intervals, embedded six-byte source locations and digest-addressed children; the manifest binds both family roots and source hashes. No full index preload or key listing is required for lookup. Reader limits here are a 384 KiB object ceiling, 32 objects per path, and 4,096 cache entries. Cache payload bytes are configurable for local verification; remote verification disables caching. These are tool limits, not a claim about deployed capacity or final schema defaults. + +`store.rs` is a local file-backed `KvStore` for artifact IO, not a production async filesystem backend. Local timings include file reads, hashing, validation and search. OS page cache is uncontrolled. Cache-disabled does not mean a cold disk. Linux `VmHWM` includes importer/oracle memory in those commands, so it is not serving RSS. Fresh `lookup` avoids retaining CSV. + +Tests use synthetic City/Country data and in-memory fault injection. They cover deterministic exports, boundaries/gaps, unsupported codes, safe output, inventory tampering, manifest-last publication, identical replay, resuming after every uncertain write, conflict/read-back failures and missing/corrupt serving visibility without writes. Run optional-backend compilation and tests with `--all-features`; it does not contact services. Real stock-celld, DynamoDB Local, serving-grant denial, production-size 16/64 KiB measurements and hosted IAM qualification still require operator runs. Earlier [prototype measurements](../../docs/superpowers/plans/2026-11-05-kv-geo-prototype-results.md) describe the old `p1` experiment, not backend evidence for this implementation. diff --git a/examples/geo-kv-prototype/src/backend.rs b/examples/geo-kv-prototype/src/backend.rs new file mode 100644 index 000000000..9bf3cf434 --- /dev/null +++ b/examples/geo-kv-prototype/src/backend.rs @@ -0,0 +1,57 @@ +//! Existing native preparation APIs only. Provisioning belongs to the operator. +use anyhow::{Result, ensure}; +use edgezero_core::{key_value_store::KvHandle, time::MonotonicClock}; +use serde::Deserialize; +use std::{fs, path::Path}; + +#[derive(Deserialize)] +#[serde(tag = "backend", rename_all = "kebab-case", deny_unknown_fields)] +enum Settings { + #[cfg(feature = "celld")] + Celld { + settings: edgezero_store_celld::settings::CelldKvSettings, + #[serde(default)] + limits: edgezero_store_celld::settings::CelldPreparationLimits, + }, + #[cfg(feature = "dynamodb")] + Dynamodb { + settings: edgezero_store_aws::settings::DynamoDbKvSettings, + #[serde(default)] + limits: edgezero_store_aws::settings::AwsPreparationLimits, + }, +} + +pub async fn prepare(path: &Path, required_bytes: usize) -> Result { + let settings: Settings = serde_json::from_slice(&fs::read(path)?)?; + match settings { + #[cfg(feature = "celld")] + Settings::Celld { settings, limits } => { + use edgezero_store_celld::{CelldPreparation, settings::CelldAccessToken}; + settings.validate()?; + ensure!( + required_bytes as u64 <= limits.max_value_bytes, + "artifact exceeds backend budget" + ); + let token = settings + .access + .token_env() + .map(|name| Ok::<_, anyhow::Error>(CelldAccessToken::new(std::env::var(name)?)?)) + .transpose()?; + Ok(CelldPreparation::new(MonotonicClock::default(), limits)? + .prepare_kv(&settings, token.as_ref()) + .await?) + } + #[cfg(feature = "dynamodb")] + Settings::Dynamodb { settings, limits } => { + ensure!( + required_bytes <= settings.limits.max_value_bytes, + "artifact exceeds backend budget" + ); + Ok( + edgezero_store_aws::AwsPreparation::new(MonotonicClock::default(), limits)? + .prepare_dynamodb(&settings) + .await?, + ) + } + } +} diff --git a/examples/geo-kv-prototype/src/import.rs b/examples/geo-kv-prototype/src/import.rs new file mode 100644 index 000000000..f2dd88fa8 --- /dev/null +++ b/examples/geo-kv-prototype/src/import.rs @@ -0,0 +1,360 @@ +//! CSV import and independent, unmerged CIDR oracle for the format experiment. +use std::{collections::HashMap, fs::File, io::Read, net::IpAddr, path::Path}; + +use anyhow::{Context, Result, bail, ensure}; +use ipnet::IpNet; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::index::{Location, Range, Source, hex}; + +// CSV family membership is literal. Request lookup normalizes mapped IPv4 separately. +fn address(ip: IpAddr) -> (u8, u128) { + match ip { + IpAddr::V4(ip) => (4, u32::from(ip).into()), + IpAddr::V6(ip) => (6, u128::from(ip)), + } +} + +#[derive(Debug, Deserialize)] +struct LocationRow { + geoname_id: u32, + country_iso_code: String, + #[serde(default)] + subdivision_1_iso_code: String, +} + +#[derive(Debug, Deserialize)] +struct BlockRow { + network: IpNet, + geoname_id: Option, +} + +#[derive(Clone, Debug)] +pub struct SourceRange { + pub network: IpNet, + pub location: Option, +} + +impl SourceRange { + pub fn start(&self) -> u128 { + address(self.network.network()).1 + } + pub fn end(&self) -> u128 { + address(self.network.broadcast()).1 + } +} + +#[derive(Debug, Serialize)] +pub struct FamilyStats { + pub source_networks: usize, + pub without_country: usize, + pub country_only: usize, + pub merged_intervals: usize, + pub unsupported_core_country_networks: std::collections::BTreeMap, +} + +pub struct Imported { + pub source: Source, + pub locations: usize, + pub raw4: Vec, + pub raw6: Vec, +} + +impl Imported { + pub fn oracle(&self, ip: IpAddr) -> Option { + let ip = ip.to_canonical(); + oracle(if ip.is_ipv4() { &self.raw4 } else { &self.raw6 }, ip) + } +} + +fn locations(reader: impl Read) -> Result>> { + let mut out = HashMap::new(); + for row in csv::Reader::from_reader(reader).deserialize::() { + let row = row?; + let location = if row.country_iso_code.is_empty() { + ensure!( + row.subdivision_1_iso_code.is_empty(), + "subdivision without country for geoname {}", + row.geoname_id + ); + None + } else { + Some( + Location::new(&row.country_iso_code, &row.subdivision_1_iso_code).with_context( + || format!("invalid location codes for geoname {}", row.geoname_id), + )?, + ) + }; + ensure!( + out.insert(row.geoname_id, location).is_none(), + "duplicate geoname identity {}", + row.geoname_id + ); + } + Ok(out) +} + +fn blocks( + reader: impl Read, + locations: &HashMap>, + family: u8, +) -> Result> { + let mut out = Vec::new(); + for row in csv::Reader::from_reader(reader).deserialize::() { + let row = row?; + ensure!( + address(row.network.addr()).0 == family, + "wrong CSV address family" + ); + ensure!( + row.network.addr() == row.network.network(), + "CIDR has host bits" + ); + let location = row + .geoname_id + .map(|id| { + locations + .get(&id) + .copied() + .with_context(|| format!("missing referenced geoname {id}")) + }) + .transpose()? + .flatten(); + out.push(SourceRange { + network: row.network, + location, + }); + } + out.sort_unstable_by_key(SourceRange::start); + for pair in out.windows(2) { + ensure!( + pair[0].end() < pair[1].start(), + "overlapping source networks" + ); + } + Ok(out) +} + +struct HashedReader { + file: File, + hash: Sha256, +} + +impl HashedReader { + fn open(path: &Path) -> Result { + Ok(Self { + file: File::open(path).with_context(|| format!("open CSV {}", path.display()))?, + hash: Sha256::new(), + }) + } + fn finish(self) -> String { + hex(&self.hash.finalize()) + } +} + +impl Read for HashedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let n = self.file.read(buffer)?; + self.hash.update(&buffer[..n]); + Ok(n) + } +} + +pub fn load(directory: &Path) -> Result { + let city = directory + .join("GeoLite2-City-Locations-en.csv") + .try_exists()?; + let country = directory + .join("GeoLite2-Country-Locations-en.csv") + .try_exists()?; + let edition = match (city, country) { + (true, false) => "GeoLite2-City", + (false, true) => "GeoLite2-Country", + (false, false) => bail!("expected City or Country Locations-en CSV"), + (true, true) => bail!("City and Country Locations-en CSV files are both present"), + }; + let files: Vec = ["Locations-en.csv", "Blocks-IPv4.csv", "Blocks-IPv6.csv"] + .iter() + .map(|suffix| format!("{edition}-{suffix}")) + .collect(); + // Hash the same byte stream CSV consumes, rather than re-reading mutable files. + let mut reader = HashedReader::open(&directory.join(&files[0]))?; + let locations = locations(&mut reader)?; + let mut hashes = vec![reader.finish()]; + let mut reader = HashedReader::open(&directory.join(&files[1]))?; + let raw4 = blocks(&mut reader, &locations, 4)?; + hashes.push(reader.finish()); + let mut reader = HashedReader::open(&directory.join(&files[2]))?; + let raw6 = blocks(&mut reader, &locations, 6)?; + hashes.push(reader.finish()); + let files = files.into_iter().zip(hashes).collect(); + Ok(Imported { + source: Source { + edition: edition.into(), + files, + }, + locations: locations.len(), + raw4, + raw6, + }) +} + +pub fn project(raw: &[SourceRange]) -> (Vec, FamilyStats) { + let mut out: Vec = Vec::new(); + let mut stats = FamilyStats { + source_networks: raw.len(), + without_country: 0, + country_only: 0, + merged_intervals: 0, + unsupported_core_country_networks: std::collections::BTreeMap::new(), + }; + for row in raw { + let Some(location) = row.location else { + stats.without_country += 1; + continue; + }; + if !location.core_country_supported() { + let codes = location.json(); + let code = codes["country"].as_str().expect("validated source country"); + *stats + .unsupported_core_country_networks + .entry(code.into()) + .or_default() += 1; + } + if location.json()["subdivision"].is_null() { + stats.country_only += 1; + } + let start = row.start(); + let end = row.end(); + if let Some(previous) = out.last_mut() + && previous.location == location + && previous.end.checked_add(1) == Some(start) + { + previous.end = end; + continue; + } + out.push(Range { + start, + end, + location, + }); + } + stats.merged_intervals = out.len(); + (out, stats) +} + +// Uses the original CIDRs and ipnet.contains, not the encoded/merged interval search. +pub fn oracle(raw: &[SourceRange], ip: IpAddr) -> Option { + let value = address(ip).1; + let i = raw.partition_point(|r| r.start() <= value); + if i == 0 { + return None; + } + raw[i - 1] + .network + .contains(&ip) + .then_some(raw[i - 1].location) + .flatten() +} + +pub fn samples(raw: &[SourceRange], family: u8, count: usize) -> Vec { + let max = if family == 4 { + u32::MAX.into() + } else { + u128::MAX + }; + let mut values = vec![0, max]; + let boundary_rows = (count / 8).min(raw.len()); + for i in 0..boundary_rows { + let row = &raw[i * raw.len() / boundary_rows]; + values.extend([row.start(), row.end()]); + if let Some(before) = row.start().checked_sub(1) { + values.push(before); + } + if let Some(after) = row.end().checked_add(1).filter(|n| *n <= max) { + values.push(after); + } + } + let mut state = 0x6a09_e667_f3bc_c909_u64; + let mut random = || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + while values.len() < count { + let n = (u128::from(random()) << 64) | u128::from(random()); + values.push(n & max); + } + values + .into_iter() + .map(|n| { + if family == 4 { + IpAddr::from((n as u32).to_be_bytes()) + } else { + IpAddr::from(n.to_be_bytes()) + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + const LOCATIONS: &str = "geoname_id,country_iso_code,subdivision_1_iso_code,country_name\n1,US,CA,\"United, States\"\n2,GB,,United Kingdom\n3,,,\n"; + + #[test] + fn csv_join_country_only_and_no_registered_fallback() { + let loc = locations(LOCATIONS.as_bytes()).unwrap(); + let raw = blocks("network,geoname_id,registered_country_geoname_id\n1.0.0.0/24,,1\n1.0.1.0/24,1,2\n1.0.2.0/24,2,1\n1.0.3.0/24,3,1\n".as_bytes(), &loc, 4).unwrap(); + assert_eq!(raw[0].location, None); + assert_eq!(raw[1].location, Some(Location::new("US", "CA").unwrap())); + assert_eq!(raw[2].location, Some(Location::new("GB", "").unwrap())); + assert_eq!(raw[3].location, None); + assert_eq!(oracle(&raw, "1.0.1.255".parse().unwrap()), raw[1].location); + } + + #[test] + fn source_codes_are_preserved_without_changing_core_policy() { + let loc = locations("geoname_id,country_iso_code\n1,XK\n".as_bytes()).unwrap(); + let raw = blocks("network,geoname_id\n0.0.0.0/0,1\n".as_bytes(), &loc, 4).unwrap(); + let (ranges, stats) = project(&raw); + assert!(!ranges[0].location.core_country_supported()); + assert_eq!(stats.unsupported_core_country_networks.get("XK"), Some(&1)); + } + + #[test] + fn invalid_csv_data_is_rejected() { + assert!( + locations("geoname_id,country_iso_code,subdivision_1_iso_code\n1,,CA\n".as_bytes()) + .is_err() + ); + assert!(locations("geoname_id,country_iso_code\n1,US\n1,GB\n".as_bytes()).is_err()); + let loc = locations(LOCATIONS.as_bytes()).unwrap(); + for (csv, family) in [ + ("network,geoname_id\n1.0.0.0/24,999\n", 4), + ("network,geoname_id\n::/0,1\n", 4), + ("network,geoname_id\n1.0.0.1/24,1\n", 4), + ("network,geoname_id\n1.0.0.0/24,1\n1.0.0.0/25,2\n", 4), + ] { + assert!(blocks(csv.as_bytes(), &loc, family).is_err()); + } + } + + #[test] + fn projection_merges_only_adjacent_equal_locations_and_preserves_gaps() { + let loc = locations(LOCATIONS.as_bytes()).unwrap(); + let raw = blocks("network,geoname_id\n0.0.0.0/32,1\n0.0.0.1/32,1\n0.0.0.2/32,\n0.0.0.3/32,1\n255.255.255.255/32,2\n".as_bytes(), &loc, 4).unwrap(); + let (ranges, stats) = project(&raw); + assert_eq!(stats.without_country, 1); + assert_eq!(ranges.len(), 3); + assert_eq!((ranges[0].start, ranges[0].end), (0, 1)); + assert_eq!( + (ranges[2].start, ranges[2].end), + (u32::MAX.into(), u32::MAX.into()) + ); + let raw = blocks("network,geoname_id\n::/0,2\n".as_bytes(), &loc, 6).unwrap(); + assert_eq!(project(&raw).0[0].end, u128::MAX); + } +} diff --git a/examples/geo-kv-prototype/src/index.rs b/examples/geo-kv-prototype/src/index.rs new file mode 100644 index 000000000..a4f05f758 --- /dev/null +++ b/examples/geo-kv-prototype/src/index.rs @@ -0,0 +1,14 @@ +//! The standalone importer and tools share the production reader and encoder. +pub use edgezero_core::geo::kv::{ + IndexLimits, InventoryEntry, Location, Range, Receipt, Resolver, Source, build, digest, hex, +}; + +pub fn limits(cache_bytes: usize) -> IndexLimits { + IndexLimits { + max_object_bytes: 384 * 1024, + // Match managed Axum's startup acceptance limit. + max_path_objects: 3, + cache_bytes, + cache_entries: 4096, + } +} diff --git a/examples/geo-kv-prototype/src/main.rs b/examples/geo-kv-prototype/src/main.rs new file mode 100644 index 000000000..ff7b5b240 --- /dev/null +++ b/examples/geo-kv-prototype/src/main.rs @@ -0,0 +1,351 @@ +#[cfg(any(feature = "celld", feature = "dynamodb"))] +mod backend; +mod import; +mod index; +mod publication; +mod store; + +use std::{fs, net::IpAddr, path::Path, sync::Arc, time::Instant}; + +use anyhow::{Context, Result, bail, ensure}; +use edgezero_core::key_value_store::KvHandle; +#[cfg(not(any(feature = "celld", feature = "dynamodb")))] +use futures::executor::block_on; +use serde_json::json; + +use index::{Receipt, Resolver, digest, hex}; +use store::ArtifactStore; + +fn peak_rss_bytes() -> Option { + // Linux process high-water mark, including importer/oracle/backend/cache. + fs::read_to_string("/proc/self/status") + .ok()? + .lines() + .find_map(|line| { + line.strip_prefix("VmHWM:")? + .split_whitespace() + .next()? + .parse::() + .ok() + .map(|n| n * 1024) + }) +} + +fn write_json(path: &Path, value: &impl serde::Serialize) -> Result<()> { + fs::write(path, serde_json::to_vec_pretty(value)?)?; + Ok(()) +} + +fn read_receipt(out: &Path) -> Result { + Ok(serde_json::from_slice(&fs::read( + out.join("receipt.json"), + )?)?) +} + +async fn build(source: &Path, out: &Path, object_bytes: usize) -> Result<()> { + ensure!( + object_bytes <= index::limits(0).max_object_bytes, + "object budget exceeds tool reader limit" + ); + let started = Instant::now(); + let imported = import::load(source)?; + let import_seconds = started.elapsed().as_secs_f64(); + let (v4, stats4) = import::project(&imported.raw4); + let (v6, stats6) = import::project(&imported.raw6); + let projection_seconds = started.elapsed().as_secs_f64() - import_seconds; + fs::create_dir_all(out)?; + let store = Arc::new(ArtifactStore::new(out)); + let kv = KvHandle::new(store.clone()); + if out.join("receipt.json").try_exists()? { + let previous = read_receipt(out)?; + let selected = Resolver::open(kv.clone(), &previous, index::limits(0)).await?; + ensure!( + selected.manifest().source == imported.source + && selected.manifest().object_bytes == object_bytes, + "output already selects another generation; use a different output directory" + ); + } + let build_started = Instant::now(); + let (receipt, manifest, inventory) = + index::build(&kv, imported.source, &v4, &v6, object_bytes).await?; + let build_seconds = build_started.elapsed().as_secs_f64(); + let verify_started = Instant::now(); + for item in &inventory { + let bytes = kv + .get_bytes(&item.key) + .await? + .context("published object missing")?; + ensure!( + bytes.len() == item.bytes && hex(&digest(&bytes)) == item.sha256, + "published object read-back mismatch" + ); + } + let report = json!({ + "format": "geo/v1 production candidate", + "transport": "local artifact files through KvHandle; no remote KV measurements", + "source": manifest.source, + "manifest_key": receipt.manifest_key, + "manifest_sha256": hex(&receipt.manifest_hash), + "locations": imported.locations, + "ipv4": stats4, + "ipv6": stats6, + "object_budget_bytes": object_bytes, + "objects_including_manifest": inventory.len(), + "encoded_bytes_including_manifest": inventory.iter().map(|i| i.bytes).sum::(), + "largest_object_bytes": inventory.iter().map(|i| i.bytes).max(), + "ipv4_lookup_objects": manifest.ipv4.as_ref().map(|r| usize::from(r.level) + 1), + "ipv6_lookup_objects": manifest.ipv6.as_ref().map(|r| usize::from(r.level) + 1), + "import_seconds_including_source_hashing": import_seconds, + "projection_seconds": projection_seconds, + "build_and_write_seconds": build_seconds, + "readback_seconds": verify_started.elapsed().as_secs_f64(), + "total_seconds": started.elapsed().as_secs_f64(), + "process_peak_rss_bytes": peak_rss_bytes(), + "store": store.stats(), + }); + write_json(&out.join("receipt.json"), &receipt)?; + write_json(&out.join("inventory.json"), &inventory)?; + write_json(&out.join("build-report.json"), &report)?; + publication::Artifact::open(out).await?; + println!("{}", serde_json::to_string_pretty(&report)?); + Ok(()) +} + +async fn measure( + name: &str, + resolver: &Resolver, + store: &ArtifactStore, + queries: &[(IpAddr, Option)], +) -> Result { + store.reset_stats(); + let mut times = Vec::with_capacity(queries.len()); + let mut read_histogram = std::collections::BTreeMap::new(); + let mut located = 0; + let hits_before = resolver.cache_stats().hits; + for (ip, expected) in queries { + let before = store.stats().reads; + let started = Instant::now(); + let actual = resolver.lookup(*ip).await?; + times.push(started.elapsed().as_secs_f64() * 1_000_000.0); + ensure!(actual == *expected, "CSV oracle mismatch"); + located += usize::from(actual.is_some()); + *read_histogram + .entry(store.stats().reads - before) + .or_insert(0_usize) += 1; + } + times.sort_by(f64::total_cmp); + let stats = store.stats(); + Ok(json!({ + "pass": name, "queries": queries.len(), "located": located, + "no_match": queries.len() - located, "oracle_matches": queries.len(), + "mean_microseconds": times.iter().sum::() / times.len() as f64, + "p50_microseconds": times[times.len() / 2], + "p95_microseconds": times[(times.len() - 1) * 95 / 100], + "max_microseconds": times[times.len() - 1], + "reads_per_query_histogram": read_histogram, + "reads_per_query_mean": stats.reads as f64 / queries.len() as f64, + "bytes_per_query_mean": stats.read_bytes as f64 / queries.len() as f64, + "cache_hits": resolver.cache_stats().hits - hits_before, + "peak_cached_payload_bytes": resolver.cache_stats().peak_payload_bytes, + "max_decoded_object_bytes": resolver.cache_stats().max_object_bytes, + "store": stats, + })) +} + +async fn verify(source: &Path, out: &Path, count: usize, cache_bytes: usize) -> Result<()> { + ensure!( + count >= 2, + "at least two samples per family are needed for endpoints" + ); + let imported = import::load(source)?; + let mut queries = Vec::new(); + for (family, raw) in [(4, &imported.raw4), (6, &imported.raw6)] { + for ip in import::samples(raw, family, count) { + queries.push((ip, imported.oracle(ip))); + } + } + let receipt = read_receipt(out)?; + let store = Arc::new(ArtifactStore::new(out)); + let kv = KvHandle::new(store.clone()); + let artifact = publication::Artifact::open(out).await?; + let disabled = publication::verify(&artifact, kv.clone()).await?; + ensure!( + disabled.manifest().source == imported.source, + "oracle source differs from indexed source" + ); + let mut reports = vec![measure("cache_disabled", &disabled, &store, &queries).await?]; + let enabled = Resolver::open(kv, &receipt, index::limits(cache_bytes)).await?; + reports.push(measure("cache_enabled_first_pass", &enabled, &store, &queries).await?); + reports.push(measure("cache_enabled_repeat", &enabled, &store, &queries).await?); + let report = json!({ + "transport": "local file artifact KV; OS page cache uncontrolled; not network latency", + "manifest_sha256": hex(&receipt.manifest_hash), + "sample_method": "per family: zero/max, stratified source CIDR endpoints and neighbors, deterministic uniform random remainder", + "samples_per_family": count, + "manifest_reads_per_resolver_initialization": 1, + "cache_payload_budget_bytes": cache_bytes, + "passes": reports, + "process_peak_rss_bytes_including_csv_oracle": peak_rss_bytes(), + }); + write_json(&out.join("verification-report.json"), &report)?; + println!("{}", serde_json::to_string_pretty(&report)?); + Ok(()) +} + +async fn lookup(out: &Path, ips: &[String]) -> Result<()> { + ensure!(!ips.is_empty(), "provide at least one IP address"); + let receipt = read_receipt(out)?; + let store = Arc::new(ArtifactStore::new(out)); + let resolver = Resolver::open( + KvHandle::new(store.clone()), + &receipt, + index::limits(4 * 1024 * 1024), + ) + .await?; + let mut results = Vec::new(); + for text in ips { + let ip: IpAddr = text.parse()?; + results.push(resolver.lookup(ip).await?.map(index::Location::json)); + } + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "results_in_input_order": results, + "store": store.stats(), "cache": resolver.cache_stats(), + "process_peak_rss_bytes_without_csv_import": peak_rss_bytes(), + }))? + ); + Ok(()) +} + +async fn run(args: &[String]) -> Result<()> { + match args.first().map(String::as_str) { + Some("build") if (3..=4).contains(&args.len()) => { + build( + Path::new(&args[1]), + Path::new(&args[2]), + args.get(3) + .map(|s| s.parse()) + .transpose()? + .unwrap_or(16 * 1024), + ) + .await + } + Some("verify") if (3..=5).contains(&args.len()) => { + verify( + Path::new(&args[1]), + Path::new(&args[2]), + args.get(3).map(|s| s.parse()).transpose()?.unwrap_or(2048), + args.get(4) + .map(|s| s.parse()) + .transpose()? + .unwrap_or(4 * 1024 * 1024), + ) + .await + } + Some("lookup") if args.len() >= 3 => lookup(Path::new(&args[1]), &args[2..]).await, + Some("publish") if args.len() == 3 => remote_command(args, false).await, + Some("verify-remote") if args.len() == 5 => remote_command(args, true).await, + _ => bail!( + "usage: geo-kv-prototype build [object-bytes]\n geo-kv-prototype verify [samples-per-family] [cache-bytes]\n geo-kv-prototype lookup [ip ...]\n geo-kv-prototype publish \n geo-kv-prototype verify-remote " + ), + } +} + +#[cfg(any(feature = "celld", feature = "dynamodb"))] +async fn remote_command(args: &[String], verify: bool) -> Result<()> { + // All local bytes are checked before preparing a backend or making remote writes. + let artifact = publication::Artifact::open(Path::new(&args[1])).await?; + let required_bytes = artifact + .inventory + .iter() + .map(|item| item.bytes) + .max() + .unwrap_or(0); + if verify { + // Do offline work before opening a live binding. In particular, don't + // leave the connection from DynamoDB's preflight idle during CSV import. + let imported = import::load(Path::new(&args[3]))?; + let count: usize = args[4].parse()?; + ensure!(count >= 2, "at least two samples per family required"); + let kv = backend::prepare(Path::new(&args[2]), required_bytes).await?; + let resolver = publication::verify(&artifact, kv) + .await + .context("verify serving index tree")?; + ensure!( + resolver.manifest().source == imported.source, + "oracle source differs" + ); + let mut checked = 0; + for (family, raw) in [(4, &imported.raw4), (6, &imported.raw6)] { + for ip in import::samples(raw, family, count) { + ensure!( + resolver + .lookup(ip) + .await + .context("read serving oracle sample")? + == imported.oracle(ip), + "CSV oracle mismatch" + ); + checked += 1; + } + } + println!( + "{}", + json!({"status": "verified", "objects": artifact.inventory.len(), "oracle_samples": checked, + "manifest_key": artifact.receipt.manifest_key, "manifest_sha256": hex(&artifact.receipt.manifest_hash)}) + ); + } else { + let kv = backend::prepare(Path::new(&args[2]), required_bytes).await?; + println!( + "{}", + serde_json::to_string(&publication::publish(&artifact, &kv).await?)? + ); + } + Ok(()) +} + +#[cfg(not(any(feature = "celld", feature = "dynamodb")))] +async fn remote_command(_: &[String], _: bool) -> Result<()> { + bail!("remote commands need the celld or dynamodb feature") +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + #[cfg(any(feature = "celld", feature = "dynamodb"))] + let result = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(anyhow::Error::from) + .and_then(|runtime| runtime.block_on(run(&args))); + #[cfg(not(any(feature = "celld", feature = "dynamodb")))] + let result = block_on(run(&args)); + if let Err(error) = result { + // Print closed categories only; neither SDK errors nor source data are safe output. + let stage = error.to_string(); + if matches!( + stage.as_str(), + "check existing object" + | "write absent object" + | "read back written object" + | "immutable remote object conflict" + | "remote read-back mismatch" + | "remote read-back missing" + | "verify serving index tree" + | "read serving oracle sample" + | "CSV oracle mismatch" + | "output already selects another generation; use a different output directory" + | "expected City or Country Locations-en CSV" + | "City and Country Locations-en CSV files are both present" + ) { + eprintln!("geo publication failed: {stage}"); + } + eprintln!( + "geo command failed; check command syntax, settings, permissions and immutable artifact integrity" + ); + eprintln!( + "commands: build [object-bytes]; verify [samples] [cache-bytes]; lookup ...; publish ; verify-remote " + ); + std::process::exit(1); + } +} diff --git a/examples/geo-kv-prototype/src/publication.rs b/examples/geo-kv-prototype/src/publication.rs new file mode 100644 index 000000000..fa937ee4f --- /dev/null +++ b/examples/geo-kv-prototype/src/publication.rs @@ -0,0 +1,200 @@ +//! Immutable publication. One operator owns each generation; read-before-write is not a lock. +#[cfg(test)] +#[path = "publication_tests.rs"] +mod tests; +use std::{collections::HashMap, fs, path::Path, sync::Arc, time::Duration}; + +use anyhow::{Context, Result, ensure}; +use async_trait::async_trait; +use bytes::Bytes; +use edgezero_core::key_value_store::{KvError, KvHandle, KvPage, KvStore}; +#[cfg(any(test, feature = "celld", feature = "dynamodb"))] +use serde::Serialize; + +use crate::{ + index::{self, InventoryEntry, Receipt, Resolver, digest, hex}, + store::ArtifactStore, +}; + +pub struct Artifact { + pub receipt: Receipt, + pub inventory: Vec, + #[cfg(any(test, feature = "celld", feature = "dynamodb"))] + kv: KvHandle, +} + +// Verification is restricted to the inventory, so an omitted descendant cannot be +// hidden by a leftover file from an earlier build. Each read rechecks artifact bytes. +struct InventoryStore { + inner: ArtifactStore, + entries: HashMap, +} + +#[async_trait(?Send)] +impl KvStore for InventoryStore { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + let (size, hash) = self + .entries + .get(key) + .ok_or_else(|| KvError::Validation("object absent from inventory".into()))?; + let bytes = self.inner.get_bytes(key).await?; + if let Some(bytes) = &bytes + && (bytes.len() != *size || hex(&digest(bytes)) != *hash) + { + return Err(KvError::Validation("artifact digest mismatch".into())); + } + Ok(bytes) + } + async fn put_bytes(&self, _: &str, _: Bytes) -> Result<(), KvError> { + Err(read_only()) + } + async fn delete(&self, _: &str) -> Result<(), KvError> { + Err(read_only()) + } + async fn list_keys_page(&self, _: &str, _: Option<&str>, _: usize) -> Result { + Err(read_only()) + } + async fn put_bytes_with_ttl(&self, _: &str, _: Bytes, _: Duration) -> Result<(), KvError> { + Err(read_only()) + } +} +fn read_only() -> KvError { + KvError::Unsupported { + operation: "artifact mutation or listing".into(), + } +} + +impl Artifact { + pub async fn open(out: &Path) -> Result { + let receipt: Receipt = serde_json::from_slice(&fs::read(out.join("receipt.json"))?)?; + let inventory: Vec = + serde_json::from_slice(&fs::read(out.join("inventory.json"))?)?; + let prefix = receipt + .manifest_key + .strip_suffix("manifest") + .context("invalid manifest key")?; + let mut entries = HashMap::new(); + let mut manifests = 0; + for item in &inventory { + ensure!( + item.bytes > 0 && item.bytes <= index::limits(0).max_object_bytes, + "invalid inventory size" + ); + ensure!( + item.key.len() <= KvHandle::MAX_KEY_SIZE, + "invalid inventory key length" + ); + if item.key == receipt.manifest_key { + manifests += 1; + ensure!( + item.sha256 == hex(&receipt.manifest_hash), + "inventory manifest mismatch" + ); + } else { + let suffix = item + .key + .strip_prefix(prefix) + .context("inventory generation mismatch")?; + let hash = suffix + .strip_prefix("v4/") + .or_else(|| suffix.strip_prefix("v6/")) + .context("invalid inventory object key")?; + ensure!( + hash.len() == 64 + && hash + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + && hash == item.sha256, + "invalid inventory object digest" + ); + } + ensure!( + entries + .insert(item.key.clone(), (item.bytes, item.sha256.clone())) + .is_none(), + "duplicate inventory key" + ); + } + ensure!(manifests == 1, "inventory must contain one manifest"); + let kv = KvHandle::new(Arc::new(InventoryStore { + inner: ArtifactStore::new(out), + entries, + })); + let resolver = Resolver::open(kv.clone(), &receipt, index::limits(0)).await?; + ensure!( + resolver.verify_tree().await? == inventory.len(), + "inventory contains unreachable objects" + ); + Ok(Self { + receipt, + inventory, + #[cfg(any(test, feature = "celld", feature = "dynamodb"))] + kv, + }) + } +} + +#[cfg(any(test, feature = "celld", feature = "dynamodb"))] +#[derive(Debug, Default, Serialize)] +pub struct PublishReport { + pub objects: usize, + pub written: usize, + pub identical: usize, +} + +#[cfg(any(test, feature = "celld", feature = "dynamodb"))] +pub async fn publish(artifact: &Artifact, remote: &KvHandle) -> Result { + let mut report = PublishReport::default(); + // Inventory ordering is not trusted. Only the manifest is a publication marker. + for item in artifact + .inventory + .iter() + .filter(|i| i.key != artifact.receipt.manifest_key) + .chain( + artifact + .inventory + .iter() + .filter(|i| i.key == artifact.receipt.manifest_key), + ) + { + let bytes = artifact + .kv + .get_bytes(&item.key) + .await? + .context("missing artifact object")?; + match remote + .get_bytes(&item.key) + .await + .context("check existing object")? + { + Some(old) => { + ensure!(old == bytes, "immutable remote object conflict"); + report.identical += 1; + } + None => { + remote + .put_bytes(&item.key, bytes.clone()) + .await + .context("write absent object")?; + let read_back = remote + .get_bytes(&item.key) + .await + .context("read back written object")? + .context("remote read-back missing")?; + ensure!(read_back == bytes, "remote read-back mismatch"); + report.written += 1; + } + } + report.objects += 1; + } + Ok(report) +} + +pub async fn verify(artifact: &Artifact, serving: KvHandle) -> Result { + let resolver = Resolver::open(serving, &artifact.receipt, index::limits(0)).await?; + ensure!( + resolver.verify_tree().await? == artifact.inventory.len(), + "remote inventory mismatch" + ); + Ok(resolver) +} diff --git a/examples/geo-kv-prototype/src/publication_tests.rs b/examples/geo-kv-prototype/src/publication_tests.rs new file mode 100644 index 000000000..4d3315cd2 --- /dev/null +++ b/examples/geo-kv-prototype/src/publication_tests.rs @@ -0,0 +1,267 @@ +use super::*; +use crate::index::{Location, Range, Source}; +use futures::executor::block_on; +use std::sync::Mutex; + +#[derive(Default)] +struct State { + data: HashMap, + puts: Vec, + uncertain_after: Option, + deny_writes: bool, + corrupt_after_write: bool, +} +#[derive(Default)] +struct Remote(Mutex); +#[async_trait(?Send)] +impl KvStore for Remote { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + Ok(self.0.lock().unwrap().data.get(key).cloned()) + } + async fn put_bytes(&self, key: &str, bytes: Bytes) -> Result<(), KvError> { + let mut state = self.0.lock().unwrap(); + assert!( + !state.deny_writes, + "serving verification must issue no writes" + ); + state.puts.push(key.into()); + let corrupt = state.corrupt_after_write; + state.data.insert( + key.into(), + if corrupt { + Bytes::from_static(b"bad read-back") + } else { + bytes + }, + ); + if state.uncertain_after == Some(state.puts.len()) { + return Err(KvError::Validation( + "uncertain write, may have committed".into(), + )); + } + Ok(()) + } + async fn delete(&self, _: &str) -> Result<(), KvError> { + panic!("must not delete") + } + async fn list_keys_page(&self, _: &str, _: Option<&str>, _: usize) -> Result { + panic!("must not list") + } + async fn put_bytes_with_ttl(&self, _: &str, _: Bytes, _: Duration) -> Result<(), KvError> { + panic!("must not use TTL") + } +} + +async fn fixture() -> (tempfile::TempDir, Artifact) { + let dir = tempfile::tempdir().unwrap(); + let local = KvHandle::new(Arc::new(ArtifactStore::new(dir.path()))); + let ranges: Vec<_> = (0..300) + .map(|i| Range { + start: i * 4, + end: i * 4 + 1, + location: Location::new("US", "CA").unwrap(), + }) + .collect(); + let (receipt, _, mut inventory) = index::build( + &local, + Source { + edition: "synthetic".into(), + files: vec![], + }, + &ranges, + &ranges, + 1024, + ) + .await + .unwrap(); + // Deliberately put the manifest first, as publication must not trust input order. + inventory.reverse(); + fs::write( + dir.path().join("receipt.json"), + serde_json::to_vec(&receipt).unwrap(), + ) + .unwrap(); + fs::write( + dir.path().join("inventory.json"), + serde_json::to_vec(&inventory).unwrap(), + ) + .unwrap(); + let artifact = Artifact::open(dir.path()).await.unwrap(); + (dir, artifact) +} + +#[test] +fn manifest_last_identical_resume_and_serving_read_only() { + block_on(async { + let (_dir, artifact) = fixture().await; + let raw = Arc::new(Remote::default()); + let remote = KvHandle::new(raw.clone()); + raw.0.lock().unwrap().uncertain_after = Some(2); + assert!(publish(&artifact, &remote).await.is_err()); + assert!( + !raw.0 + .lock() + .unwrap() + .data + .contains_key(&artifact.receipt.manifest_key) + ); + raw.0.lock().unwrap().uncertain_after = None; + let report = publish(&artifact, &remote).await.unwrap(); + assert_eq!(report.identical, 2); + assert_eq!( + raw.0.lock().unwrap().puts.last(), + Some(&artifact.receipt.manifest_key) + ); + let before = raw.0.lock().unwrap().puts.len(); + let report = publish(&artifact, &remote).await.unwrap(); + assert_eq!(report.written, 0); + assert_eq!(report.identical, artifact.inventory.len()); + assert_eq!(before, raw.0.lock().unwrap().puts.len()); + raw.0.lock().unwrap().deny_writes = true; + let resolver = verify(&artifact, remote).await.unwrap(); + assert_eq!( + resolver.lookup("0.0.0.1".parse().unwrap()).await.unwrap(), + Some(Location::new("US", "CA").unwrap()) + ); + assert_eq!( + resolver.lookup("0.0.0.2".parse().unwrap()).await.unwrap(), + None + ); + }); +} + +#[test] +fn resume_after_every_write_including_manifest() { + block_on(async { + let (_dir, artifact) = fixture().await; + for stop in 1..=artifact.inventory.len() { + let raw = Arc::new(Remote::default()); + let remote = KvHandle::new(raw.clone()); + raw.0.lock().unwrap().uncertain_after = Some(stop); + assert!(publish(&artifact, &remote).await.is_err()); + raw.0.lock().unwrap().uncertain_after = None; + publish(&artifact, &remote).await.unwrap(); + verify(&artifact, remote).await.unwrap(); + assert_eq!(raw.0.lock().unwrap().puts.len(), artifact.inventory.len()); + } + }); +} + +#[test] +fn remote_conflict_and_bad_readback_do_not_publish_manifest() { + block_on(async { + let (_dir, artifact) = fixture().await; + for corrupt_readback in [false, true] { + let raw = Arc::new(Remote::default()); + let remote = KvHandle::new(raw.clone()); + { + let mut state = raw.0.lock().unwrap(); + state.corrupt_after_write = corrupt_readback; + if !corrupt_readback { + let key = &artifact + .inventory + .iter() + .find(|i| i.key != artifact.receipt.manifest_key) + .unwrap() + .key; + state + .data + .insert(key.clone(), Bytes::from_static(b"conflict")); + } + } + assert!(publish(&artifact, &remote).await.is_err()); + assert!( + !raw.0 + .lock() + .unwrap() + .data + .contains_key(&artifact.receipt.manifest_key) + ); + } + }); +} + +#[test] +fn serving_verification_detects_incomplete_visibility_and_corruption() { + block_on(async { + let (_dir, artifact) = fixture().await; + let raw = Arc::new(Remote::default()); + let remote = KvHandle::new(raw.clone()); + publish(&artifact, &remote).await.unwrap(); + raw.0.lock().unwrap().deny_writes = true; + let key = &artifact + .inventory + .iter() + .find(|i| i.key != artifact.receipt.manifest_key) + .unwrap() + .key; + let bytes = raw.0.lock().unwrap().data.remove(key).unwrap(); + assert!(verify(&artifact, remote.clone()).await.is_err()); + raw.0 + .lock() + .unwrap() + .data + .insert(key.clone(), Bytes::from_static(b"corrupt")); + assert!(verify(&artifact, remote.clone()).await.is_err()); + raw.0.lock().unwrap().data.insert(key.clone(), bytes); + verify(&artifact, remote).await.unwrap(); + }); +} + +#[test] +fn artifact_rejects_tamper_duplicates_omissions_extras_and_cross_generation() { + block_on(async { + for mode in [ + "tamper", + "duplicate", + "omit", + "extra", + "cross-generation", + "pin", + ] { + let (dir, artifact) = fixture().await; + let mut items: Vec = + serde_json::from_slice(&fs::read(dir.path().join("inventory.json")).unwrap()) + .unwrap(); + let item = items + .iter() + .find(|i| i.key != artifact.receipt.manifest_key) + .unwrap(); + match mode { + "tamper" => fs::write(dir.path().join(&item.key), b"bad").unwrap(), + "duplicate" => items.push(InventoryEntry { + key: item.key.clone(), + bytes: item.bytes, + sha256: item.sha256.clone(), + }), + "omit" => { + items.remove(1); + } + "extra" => items.push(InventoryEntry { + key: item.key.replace(&item.sha256, &"0".repeat(64)), + sha256: "0".repeat(64), + bytes: 1, + }), + "cross-generation" => { + items[1].key = items[1].key.replacen("geo/v1/", "geo/v1/wrong/", 1) + } + "pin" => { + let mut receipt = artifact.receipt; + receipt.manifest_hash[0] ^= 1; + fs::write( + dir.path().join("receipt.json"), + serde_json::to_vec(&receipt).unwrap(), + ) + .unwrap(); + } + _ => unreachable!(), + } + fs::write( + dir.path().join("inventory.json"), + serde_json::to_vec(&items).unwrap(), + ) + .unwrap(); + assert!(Artifact::open(dir.path()).await.is_err(), "accepted {mode}"); + } + }); +} diff --git a/examples/geo-kv-prototype/src/store.rs b/examples/geo-kv-prototype/src/store.rs new file mode 100644 index 000000000..7e7de4e45 --- /dev/null +++ b/examples/geo-kv-prototype/src/store.rs @@ -0,0 +1,151 @@ +//! Local artifact transport only. The resolver itself knows only KvHandle. +//! Synchronous filesystem calls here are NOT a production async KV backend. +use std::{ + fs, + io::{ErrorKind, Write as _}, + path::{Component, Path, PathBuf}, + sync::atomic::{AtomicU64, Ordering}, + time::Duration, +}; + +use async_trait::async_trait; +use bytes::Bytes; +use edgezero_core::key_value_store::{KvError, KvPage, KvStore}; +use serde::Serialize; + +#[derive(Default)] +pub struct ArtifactStore { + root: PathBuf, + reads: AtomicU64, + read_bytes: AtomicU64, + writes: AtomicU64, + write_bytes: AtomicU64, +} + +#[derive(Debug, Default, Clone, Serialize)] +pub struct StoreStats { + pub reads: u64, + pub read_bytes: u64, + pub writes: u64, + pub write_bytes: u64, +} + +impl ArtifactStore { + pub fn new(root: impl Into) -> Self { + Self { + root: root.into(), + ..Self::default() + } + } + + fn path(&self, key: &str) -> Result { + if !key.starts_with("geo/v1/") + || Path::new(key) + .components() + .any(|p| !matches!(p, Component::Normal(_))) + { + return Err(KvError::Validation("invalid artifact key".into())); + } + Ok(self.root.join(key)) + } + + pub fn stats(&self) -> StoreStats { + StoreStats { + reads: self.reads.load(Ordering::Relaxed), + read_bytes: self.read_bytes.load(Ordering::Relaxed), + writes: self.writes.load(Ordering::Relaxed), + write_bytes: self.write_bytes.load(Ordering::Relaxed), + } + } + + pub fn reset_stats(&self) { + self.reads.store(0, Ordering::Relaxed); + self.read_bytes.store(0, Ordering::Relaxed); + self.writes.store(0, Ordering::Relaxed); + self.write_bytes.store(0, Ordering::Relaxed); + } +} + +fn unsupported(operation: &str) -> KvError { + KvError::Unsupported { + operation: operation.into(), + } +} + +#[async_trait(?Send)] +impl KvStore for ArtifactStore { + async fn get_bytes(&self, key: &str) -> Result, KvError> { + self.reads.fetch_add(1, Ordering::Relaxed); + let value = match fs::read(self.path(key)?) { + Ok(value) => value, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(KvError::Internal(e.into())), + }; + self.read_bytes + .fetch_add(value.len() as u64, Ordering::Relaxed); + Ok(Some(value.into())) + } + + async fn put_bytes(&self, key: &str, value: Bytes) -> Result<(), KvError> { + let path = self.path(key)?; + let parent = path + .parent() + .ok_or_else(|| KvError::Validation("missing parent".into()))?; + fs::create_dir_all(parent).map_err(|e| KvError::Internal(e.into()))?; + // Finish writing before exposing the immutable object at its final key. + let mut temp = + tempfile::NamedTempFile::new_in(parent).map_err(|e| KvError::Internal(e.into()))?; + temp.write_all(&value) + .map_err(|e| KvError::Internal(e.into()))?; + match temp.persist_noclobber(&path) { + Ok(_) => {} + Err(e) if e.error.kind() == ErrorKind::AlreadyExists => { + let old = fs::read(path).map_err(|e| KvError::Internal(e.into()))?; + if old != value { + return Err(KvError::Validation("immutable artifact differs".into())); + } + } + Err(e) => return Err(KvError::Internal(e.error.into())), + } + self.writes.fetch_add(1, Ordering::Relaxed); + self.write_bytes + .fetch_add(value.len() as u64, Ordering::Relaxed); + Ok(()) + } + + async fn delete(&self, _: &str) -> Result<(), KvError> { + Err(unsupported("delete")) + } + async fn list_keys_page(&self, _: &str, _: Option<&str>, _: usize) -> Result { + Err(unsupported("list_keys_page")) + } + async fn put_bytes_with_ttl(&self, _: &str, _: Bytes, _: Duration) -> Result<(), KvError> { + Err(unsupported("put_bytes_with_ttl")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn artifact_keys_are_local_and_immutable() { + futures::executor::block_on(async { + let root = tempfile::tempdir().unwrap(); + let store = ArtifactStore::new(root.path()); + let key = "geo/v1/fixture/manifest"; + assert!(store.get_bytes(key).await.unwrap().is_none()); + assert!(store.get_bytes("geo/v1/../invalid").await.is_err()); + let original = Bytes::from_static(b"same"); + store.put_bytes(key, original.clone()).await.unwrap(); + store.put_bytes(key, original.clone()).await.unwrap(); + assert!( + store + .put_bytes(key, Bytes::from_static(b"different")) + .await + .is_err() + ); + assert_eq!(store.get_bytes(key).await.unwrap(), Some(original)); + }); + } +} diff --git a/examples/geo-kv-prototype/tests/cli.rs b/examples/geo-kv-prototype/tests/cli.rs new file mode 100644 index 000000000..2ba1e2efc --- /dev/null +++ b/examples/geo-kv-prototype/tests/cli.rs @@ -0,0 +1,143 @@ +use serde_json::{Value, json}; +use std::{ + fs, + path::Path, + process::{Command, Output}, +}; + +fn command(args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_geo-kv-prototype")) + .args(args) + .output() + .unwrap() +} +fn success(args: &[&str]) -> Value { + let output = command(args); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).unwrap() +} +fn text(path: &Path) -> &str { + path.to_str().unwrap() +} + +fn source(dir: &Path, city: bool) { + let edition = if city { "City" } else { "Country" }; + let locations = if city { + "geoname_id,country_iso_code,subdivision_1_iso_code,ignored\n1,US,CA,extra\n2,GB,,extra\n3,XK,,extra\n4,,,extra\n" + } else { + "geoname_id,country_iso_code,ignored\n1,US,extra\n2,GB,extra\n3,XK,extra\n4,,extra\n" + }; + fs::write( + dir.join(format!("GeoLite2-{edition}-Locations-en.csv")), + locations, + ) + .unwrap(); + fs::write(dir.join(format!("GeoLite2-{edition}-Blocks-IPv4.csv")), "network,geoname_id,registered_country_geoname_id\n0.0.0.0/32,1,2\n0.0.0.2/31,2,1\n1.0.0.0/24,,1\n2.0.0.0/24,4,1\n3.0.0.0/24,3,1\n255.255.255.255/32,2,1\n").unwrap(); + fs::write(dir.join(format!("GeoLite2-{edition}-Blocks-IPv6.csv")), "network,geoname_id\n::/128,1\n::2/127,2\n::ffff:0:0/96,3\n2001:db8::/32,3\nffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128,2\n").unwrap(); +} + +#[test] +fn city_and_country_cli_build_oracle_and_deterministic_inventory() { + for city in [true, false] { + let src = tempfile::tempdir().unwrap(); + let out = tempfile::tempdir().unwrap(); + let again = tempfile::tempdir().unwrap(); + source(src.path(), city); + success(&["build", text(src.path()), text(out.path()), "1024"]); + success(&["build", text(src.path()), text(again.path()), "1024"]); + let default_out = tempfile::tempdir().unwrap(); + let defaults = success(&["build", text(src.path()), text(default_out.path())]); + assert_eq!(defaults["object_budget_bytes"], 16 * 1024_u64); + // A completed build can also be repeated in-place. + success(&["build", text(src.path()), text(out.path()), "1024"]); + let rejected = command(&["build", text(src.path()), text(out.path()), "2048"]); + assert!(!rejected.status.success()); + assert!( + String::from_utf8_lossy(&rejected.stderr) + .contains("output already selects another generation") + ); + assert_eq!(fs::read_dir(out.path().join("geo/v1")).unwrap().count(), 1); + let build_report: Value = + serde_json::from_slice(&fs::read(out.path().join("build-report.json")).unwrap()) + .unwrap(); + for file in build_report["source"]["files"].as_array().unwrap() { + use sha2::{Digest as _, Sha256}; + let filename = file[0].as_str().unwrap(); + let csv = fs::read(src.path().join(filename)).unwrap(); + let actual: String = Sha256::digest(csv) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + assert_eq!(file[1], actual, "{filename}"); + } + for name in ["receipt.json", "inventory.json"] { + assert_eq!( + fs::read(out.path().join(name)).unwrap(), + fs::read(again.path().join(name)).unwrap() + ); + } + let inventory: Vec = + serde_json::from_slice(&fs::read(out.path().join("inventory.json")).unwrap()).unwrap(); + for item in inventory { + let key = item["key"].as_str().unwrap(); + assert!(key.starts_with("geo/v1/")); + assert_eq!( + fs::read(out.path().join(key)).unwrap(), + fs::read(again.path().join(key)).unwrap() + ); + } + let report = success(&["verify", text(src.path()), text(out.path()), "256", "1024"]); + assert_eq!(report["passes"][0]["oracle_matches"], 512); + let report = success(&[ + "lookup", + text(out.path()), + "0.0.0.0", + "0.0.0.1", + "255.255.255.255", + "::", + "::1", + "::ffff:0.0.0.0", + "2001:db8::1", + ]); + let results = &report["results_in_input_order"]; + assert_eq!(results[0]["country"], "US"); + assert_eq!( + results[0]["subdivision"], + if city { json!("CA") } else { Value::Null } + ); + assert!(results[1].is_null()); + assert_eq!(results[2]["country"], "GB"); + assert_eq!(results[3], results[0]); + assert!(results[4].is_null()); + assert_eq!(results[5], results[0]); + assert_eq!(results[6]["country"], "XK"); + assert_eq!(results[6]["core_country_supported"], false); + } +} + +#[test] +fn errors_do_not_echo_reader_addresses_or_raw_input() { + let source = tempfile::tempdir().unwrap(); + let out = tempfile::tempdir().unwrap(); + self::source(source.path(), true); + success(&["build", text(source.path()), text(out.path()), "1024"]); + let marker = "private-reader-input-not-an-ip"; + let output = command(&["lookup", text(out.path()), marker]); + assert!(!output.status.success()); + assert!(!String::from_utf8_lossy(&output.stderr).contains(marker)); + assert!(output.stdout.is_empty()); + let secret = "private-location-record-and-credential"; + fs::write( + source.path().join("GeoLite2-City-Locations-en.csv"), + format!("geoname_id,country_iso_code\n1,{secret}\n"), + ) + .unwrap(); + let output = command(&["build", text(source.path()), text(out.path()), "1024"]); + assert!(!output.status.success()); + assert!(!String::from_utf8_lossy(&output.stderr).contains(secret)); + assert!(output.stdout.is_empty()); +} diff --git a/scripts/native_aws_test_admin.mjs b/scripts/native_aws_test_admin.mjs index c63fb4d2c..1e6ae8abc 100644 --- a/scripts/native_aws_test_admin.mjs +++ b/scripts/native_aws_test_admin.mjs @@ -55,6 +55,9 @@ function client(endpoint, service) { }); } +// Reuse the same strictly loopback-only emulator client in other local proofs. +export { client as fixtureClient }; + async function seed(dynamo, secrets) { const keys = [ { AttributeName: "pk", KeyType: "HASH" }, diff --git a/scripts/run_geo_kv_integration.mjs b/scripts/run_geo_kv_integration.mjs new file mode 100644 index 000000000..800cbc274 --- /dev/null +++ b/scripts/run_geo_kv_integration.mjs @@ -0,0 +1,672 @@ +#!/usr/bin/env node +// Disposable local proof. Runs synthetic fixtures by default. Optionally add a +// permitted local City CSV with GEO_KV_CSV_DIR. No remote endpoints or downloads. +// GEO_KV_TOOL_BIN=/absolute/geo-kv-prototype GEO_KV_HOST_BIN=/absolute/edgezero-hosting-fixture \ +// CELLD_BIN=/absolute/official/celld node scripts/run_geo_kv_integration.mjs +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { constants, createReadStream } from "node:fs"; +import { access, cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import http from "node:http"; +import https from "node:https"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { isAbsolute, join } from "node:path"; +import { createInterface } from "node:readline"; +import { setTimeout as delay } from "node:timers/promises"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { start as startTls } from "./celld_store_tls_terminator.mjs"; +import { fixtureClient } from "./native_aws_test_admin.mjs"; +import { publishedEndpoint } from "./run_native_aws_integration.mjs"; + +const root = fileURLToPath(new URL("../", import.meta.url)); +export const DYNAMO_IMAGE = "amazon/dynamodb-local@sha256:ff89bd48ff32cd8d9be5fee8873b65b8854dc408f1afe881be6eb00247bc0dab"; +const table = "edgezero-geo-proof"; +const namespace = "geo-proof"; +const hash = (bytes) => createHash("sha256").update(bytes).digest("hex"); + +export function localEndpoint(endpoint, protocol = "http:") { + assert.match(endpoint, /^https?:\/\/127\.0\.0\.1:[1-9][0-9]*$/, "literal loopback endpoint required"); + const parsed = new URL(endpoint); + assert.equal(parsed.protocol, protocol); + assert.equal(parsed.hostname, "127.0.0.1", "proof endpoints must be literal loopback"); + assert(parsed.port && Number(parsed.port) > 0 && Number(parsed.port) <= 65535); + assert.equal(parsed.username + parsed.password + parsed.search + parsed.hash, ""); + assert.equal(parsed.pathname, "/"); + return parsed.origin; +} + +export function childEnvironment(source, work) { + // In particular do not inherit endpoints, profiles, proxies, Geo selection, + // bindings, FIXTURE_GEO_ARTIFACT, or real credentials from this shell. + return { + PATH: source.PATH ?? "/usr/bin:/bin", HOME: join(work, "home"), TMPDIR: work, + NO_COLOR: "1", AWS_ACCESS_KEY_ID: "DUMMYLOCAL", AWS_SECRET_ACCESS_KEY: "DUMMYLOCAL", + AWS_REGION: "us-east-1", AWS_EC2_METADATA_DISABLED: "true", + AWS_CONFIG_FILE: join(work, "absent-aws-config"), + AWS_SHARED_CREDENTIALS_FILE: join(work, "absent-aws-credentials"), + AWS_IGNORE_CONFIGURED_ENDPOINT_URLS: "false", RUST_LOG: "off", + }; +} + +export function serviceConfig(publisher, serving) { + const policy = { + version: 1, stores: [{ alias: "geo", namespace }], + access: { mode: "authenticated", tokens: [ + { id: "publisher", sha256: hash(publisher), grants: [{ store: "geo", operations: ["read", "write", "delete", "list"] }] }, + { id: "serving", sha256: hash(serving), grants: [{ store: "geo", operations: ["read"] }] }, + ] }, + }; + return { + name: "edgezero-store", main: "src/index.js", no_bundle: true, compatibility_date: "2026-10-01", + durable_objects: { bindings: [{ name: "EDGEZERO_STORE", class_name: "EdgeZeroStore" }] }, + migrations: [{ tag: "v1", new_sqlite_classes: ["EdgeZeroStore"] }], + vars: { EDGEZERO_STORE_POLICY: JSON.stringify(policy) }, + }; +} + +export function syntheticCsv(country = "US", subdivision = "CA") { + return { + "GeoLite2-City-Locations-en.csv": `geoname_id,country_iso_code,subdivision_1_iso_code\n1,${country},${subdivision}\n2,GB,\n3,XK,\n4,,\n`, + "GeoLite2-City-Blocks-IPv4.csv": "network,geoname_id,registered_country_geoname_id\n" + + Array.from({ length: 120 }, (_, i) => `192.0.2.${i * 2}/32,${i === 1 ? 3 : i === 2 ? 2 : 1},2\n`).join("") + + "198.51.100.0/24,,1\n203.0.113.0/24,4,1\n", + "GeoLite2-City-Blocks-IPv6.csv": "network,geoname_id\n2001:db8::/126,2\n", + }; +} + +async function within(promise, label, ms = 30_000) { + let timer; + try { + return await Promise.race([promise, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out`)), ms); + })]); + } finally { clearTimeout(timer); } +} + +class Process { + constructor(command, args, options, children) { + this.output = ""; + this.child = spawn(command, args, { ...options, detached: true, stdio: ["ignore", "pipe", "pipe"] }); + children.add(this); + for (const stream of [this.child.stdout, this.child.stderr]) { + stream.on("data", (chunk) => { this.output = (this.output + chunk).slice(-131072); }); + } + this.done = new Promise((resolve) => { + this.child.once("error", (error) => { this.result = { error }; resolve(this.result); }); + this.child.once("close", (code, signal) => { this.result = { code, signal }; resolve(this.result); }); + }); + } + kill(signal = "SIGKILL") { + if (!this.child.pid) return; + try { process.kill(-this.child.pid, signal); } catch (error) { if (error.code !== "ESRCH") throw error; } + } + async success(timeoutMs = 60_000) { + const result = await within(this.done, "child exit", timeoutMs); + assert.equal(result.code, 0, `child failed: ${JSON.stringify(result)}\n${this.output}`); + return this.output; + } +} + +async function waitFor(check, label, process) { + const deadline = Date.now() + 60_000; + let last; + while (Date.now() < deadline) { + if (process?.result) throw new Error(`${label} process exited: ${process.output}`); + try { if (await check()) return; } catch (error) { last = error; } + await delay(50); + } + throw new Error(`waiting for ${label} failed`, { cause: last }); +} + +async function listen(server) { + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + return `http://127.0.0.1:${server.address().port}`; +} + +async function port() { + const server = createServer(); + const endpoint = await listen(server); + await new Promise((resolve) => server.close(resolve)); + return new URL(endpoint).port; +} + +function request(url, { method = "GET", headers = {}, body, ca } = {}) { + const parsed = new URL(url); + localEndpoint(parsed.origin, parsed.protocol); + return new Promise((resolve, reject) => { + const client = parsed.protocol === "https:" ? https : http; + const req = client.request(url, { method, headers, ca }, (res) => { + const chunks = []; + res.on("data", (chunk) => chunks.push(chunk)); + res.on("error", reject); + res.on("end", () => resolve({ status: res.statusCode, bytes: Buffer.concat(chunks) })); + }); + const timer = setTimeout(() => req.destroy(new Error("local request timed out")), 3000); + req.on("error", reject); + req.on("close", () => clearTimeout(timer)); + req.end(body); + }); +} + +// No redirects or environment proxy discovery. The publisher fault waits until +// the backend applied a write, then withholds its acknowledgment until we kill +// the publisher. Serving traffic has its own audit and cannot be mistaken for +// direct fixture administration. +export async function auditProxy(target, classify) { + localEndpoint(target); + const state = { calls: [], fault: null, heldReads: null }; + const server = http.createServer((incoming, outgoing) => { + const call = { ...classify(incoming), responseBytes: 0 }; + state.calls.push(call); + if (state.heldReads && call.role === "serving" && !call.write && + state.heldReads.remaining > 0) { + incoming.resume(); + const gate = state.heldReads; + gate.remaining--; + if (gate.remaining === 0) gate.resolve(); + return; // Leave the request pending until its owner cancels or times out. + } + const upstream = http.request(`${target}${incoming.url}`, { + method: incoming.method, headers: incoming.headers, + }, (response) => { + call.status = response.statusCode; + response.on("data", (chunk) => { call.responseBytes += chunk.length; }); + if (state.fault && call.write) { + const fault = state.fault; + state.fault = null; + response.resume(); + response.on("end", () => { + fault.resolve({ call, status: response.statusCode }); + // Deliberately never send the successful reply. No retry here. + }); + } else { + outgoing.writeHead(response.statusCode, response.headers); + response.pipe(outgoing); + } + }); + upstream.on("error", () => { if (!outgoing.destroyed) outgoing.destroy(); }); + outgoing.on("close", () => upstream.destroy()); + incoming.pipe(upstream); + }); + state.endpoint = await listen(server); + state.server = server; + state.arm = () => new Promise((resolve) => { state.fault = { resolve }; }); + state.hold = (remaining) => new Promise((resolve) => { state.heldReads = { remaining, resolve }; }); + return state; +} + +function hostEnvironment(base, endpoint, bindings, pin) { + return { + ...base, FIXTURE_MODE: "geo-kv", FIXTURE_BIND: new URL(endpoint).host, + EDGEZERO__ADAPTER__HOST: "127.0.0.1", EDGEZERO__ADAPTER__PORT: new URL(endpoint).port, + EDGEZERO__STORE_BINDINGS_FILE: bindings, + EDGEZERO__ADAPTER__GEO__PROVIDER: "kv", EDGEZERO__ADAPTER__GEO__STORE_ID: "fixture_geo", + EDGEZERO__ADAPTER__GEO__READER_IP: "forwarded-required", EDGEZERO__ADAPTER__GEO__PIN: pin, + EDGEZERO__ADAPTER__GEO__LOOKUP_TIMEOUT_MS: "1000", EDGEZERO__ADAPTER__GEO__MAX_IN_FLIGHT: "4", + EDGEZERO__ADAPTER__TRUSTED_PROXY_CIDRS: "127.0.0.1/32", + EDGEZERO__ADAPTER__FORWARDING_HEADER_FAMILY: "forwarded", + EDGEZERO__ADAPTER__SHUTDOWN_GRACE_SECONDS: "1", + }; +} + +// Approximate process peak RSS, including runtime and SDK allocations, not +// individual index allocations. No reader addresses are written to the report. +async function peakRss(pid) { + const status = await readFile(`/proc/${pid}/status`, "utf8"); + const match = /^VmHWM:\s+(\d+) kB$/m.exec(status); + assert(match, "licensed-size proof requires Linux VmHWM"); + return Number(match[1]) * 1024; +} + +async function sampleReaderAddresses(source, buildReport) { + const workload = []; + for (const [family, stats] of [[4, buildReport.ipv4], [6, buildReport.ipv6]]) { + const stride = Math.max(1, Math.floor(stats.source_networks / 64)); + const input = createReadStream(join(source, `GeoLite2-City-Blocks-IPv${family}.csv`)); + const lines = createInterface({ input, crlfDelay: Infinity }); + let row = -1; + try { + for await (const line of lines) { + row++; + if (row === 0 || row % stride !== 0) continue; + const address = line.split(",", 1)[0].split("/", 1)[0]; + workload.push(family === 4 ? address : `"[${address}]"`); + if (row >= stride * 64) break; + } + } finally { + lines.close(); + input.destroy(); + } + } + assert(workload.length > 0, "City workload requires source networks"); + return workload; +} + +function latencyStats(samples) { + const sorted = [...samples].sort((a, b) => a - b); + const quantile = (value) => sorted[Math.ceil(value * sorted.length) - 1]; + return { p50_ms: quantile(0.5), p95_ms: quantile(0.95), p99_ms: quantile(0.99) }; +} + +export async function runIntegration({ env = process.env, log = console.log, backends = ["celld", "dynamodb"] } = {}) { + const licensedSource = env.GEO_KV_CSV_DIR; + if (licensedSource) assert(isAbsolute(licensedSource), "GEO_KV_CSV_DIR must be absolute"); + const binaries = { celld: env.CELLD_BIN, tool: env.GEO_KV_TOOL_BIN, host: env.GEO_KV_HOST_BIN }; + for (const [name, binary] of Object.entries(binaries)) { + assert(binary && isAbsolute(binary), `${name} requires an explicitly selected absolute executable`); + await access(binary, constants.X_OK); + } + const fingerprints = {}; + for (const [name, binary] of Object.entries(binaries)) fingerprints[name] = hash(await readFile(binary)); + const work = await mkdtemp(join(tmpdir(), "edgezero-geo-kv-")); + const children = new Set(); + const servers = []; + const container = `edgezero-geo-kv-${process.pid}-${randomBytes(4).toString("hex")}`; + const network = `${container}-network`; + let containerCreated = false; + let networkCreated = false; + let interrupted; + const onInterrupt = () => { + interrupted = true; + for (const child of children) child.kill(); + for (const server of servers) server.closeAllConnections?.(); + }; + process.once("SIGINT", onInterrupt); + process.once("SIGTERM", onInterrupt); + const base = childEnvironment(env, work); + const start = (command, args, extra = {}) => { + assert(!interrupted, "proof interrupted"); + return new Process(command, args, { cwd: work, env: base, ...extra }, children); + }; + const run = async (command, args, extra = {}, timeoutMs) => { + const child = start(command, args, extra); + try { return await child.success(timeoutMs); } catch (error) { child.kill(); throw error; } + }; + const jsonFile = async (path, value) => writeFile(path, JSON.stringify(value, null, 2)); + const results = []; + try { + await mkdir(base.HOME); + assert.equal((await run(binaries.celld, ["--version"])).trim(), "celld 0.6.3"); + assert(!(await run(binaries.celld, ["--help"])).includes("--store-api-config"), "modified celld is not stock proof"); + await run("docker", ["image", "inspect", DYNAMO_IMAGE]); + // Cached image only, telemetry disabled, exclusively loopback-published. + // A normal bridge permits host publication; this is not an egress sandbox. + // All clients below receive explicit loopback service endpoints. + await run("docker", ["network", "create", network]); + networkCreated = true; + await run("docker", ["run", "-d", "--pull=never", "--name", container, "--network", network, + "--publish", "127.0.0.1::8000", DYNAMO_IMAGE, + "-jar", "DynamoDBLocal.jar", "-inMemory", "-sharedDb", "-disableTelemetry"]); + containerCreated = true; + const dynamoEndpoint = publishedEndpoint(await run("docker", ["port", container, "8000/tcp"])); + const dynamo = fixtureClient(dynamoEndpoint, "dynamodb"); + await waitFor(async () => { await dynamo("ListTables"); return true; }, "DynamoDB Local"); + await dynamo("CreateTable", { + TableName: table, BillingMode: "PAY_PER_REQUEST", + KeySchema: [{ AttributeName: "pk", KeyType: "HASH" }, { AttributeName: "sk", KeyType: "RANGE" }], + AttributeDefinitions: [{ AttributeName: "pk", AttributeType: "S" }, { AttributeName: "sk", AttributeType: "S" }], + }); + + const publisher = randomBytes(32).toString("hex"); + const serving = randomBytes(32).toString("hex"); + const service = join(work, "service"); + await mkdir(service); + await cp(join(root, "services/celld-store/src"), join(service, "src"), { recursive: true }); + await jsonFile(join(service, "wrangler.json"), serviceConfig(publisher, serving)); + const cellPort = await port(); + const cell = start(binaries.celld, ["dev", ".", "--no-watch", "--logs", "--port", cellPort], { cwd: service }); + await waitFor(() => cell.output.includes(`celld listening on 127.0.0.1:${cellPort}`), "stock celld listener", cell); + const cellAudit = await auditProxy(`http://127.0.0.1:${cellPort}`, (req) => ({ + role: req.headers.authorization === `Bearer ${serving}` ? "serving" : "publisher", + operation: req.url.split("?")[0].split("/").at(-1), write: req.method !== "GET" && req.method !== "HEAD", + })); + servers.push(cellAudit.server); + const cert = join(work, "cert.pem"); + const key = join(work, "key.pem"); + await run("openssl", ["req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", + "-subj", "/CN=geo-kv-proof", "-addext", "subjectAltName=IP:127.0.0.1", + "-addext", "basicConstraints=critical,CA:FALSE", "-addext", "extendedKeyUsage=serverAuth", + "-keyout", key, "-out", cert]); + const tls = startTls({ cert, key, target: new URL(cellAudit.endpoint).host, listen: "127.0.0.1:0" }, () => {}); + servers.push(tls); + if (!tls.listening) await within(new Promise((resolve, reject) => { tls.once("listening", resolve); tls.once("error", reject); }), "TLS listener"); + const cellEndpoint = localEndpoint(`https://127.0.0.1:${tls.address().port}`, "https:"); + const ca = await readFile(cert); + const cellBase = { ...base, SSL_CERT_FILE: cert }; + + const dynamoPublish = await auditProxy(dynamoEndpoint, (req) => { + const operation = req.headers["x-amz-target"]?.split(".").at(-1); + return { role: "publisher", operation, write: operation === "PutItem" }; + }); + const dynamoServe = await auditProxy(dynamoEndpoint, (req) => { + const operation = req.headers["x-amz-target"]?.split(".").at(-1); + return { role: "serving", operation, write: !["DescribeTable", "GetItem"].includes(operation) }; + }); + servers.push(dynamoPublish.server, dynamoServe.server); + const artifacts = []; + for (const [name, country, subdivision] of [["a", "US", "CA"], ["b", "DE", "BE"]]) { + const source = join(work, `csv-${name}`); + const dir = join(work, `artifact-${name}`); + await mkdir(source); + for (const [file, bytes] of Object.entries(syntheticCsv(country, subdivision))) await writeFile(join(source, file), bytes); + await run(binaries.tool, ["build", source, dir, "1024"]); + const receipt = JSON.parse(await readFile(join(dir, "receipt.json"))); + const inventory = JSON.parse(await readFile(join(dir, "inventory.json"))); + const manifest = JSON.parse(await readFile(join(dir, receipt.manifest_key))); + artifacts.push({ dir, source, receipt, inventory, manifest, pin: `${manifest.generation}@${Buffer.from(receipt.manifest_hash).toString("hex")}` }); + } + + const licensed = []; + if (licensedSource) { + for (const bytes of [16 * 1024, 64 * 1024]) { + const dir = join(work, `licensed-${bytes}`); + await run(binaries.tool, ["build", licensedSource, dir, String(bytes)], {}, 600_000); + const receipt = JSON.parse(await readFile(join(dir, "receipt.json"))); + const manifest = JSON.parse(await readFile(join(dir, receipt.manifest_key))); + const inventory = JSON.parse(await readFile(join(dir, "inventory.json"))); + const report = JSON.parse(await readFile(join(dir, "build-report.json"))); + const workload = await sampleReaderAddresses(licensedSource, report); + licensed.push({ dir, source: licensedSource, receipt, manifest, inventory, report, workload, + pin: `${manifest.generation}@${Buffer.from(receipt.manifest_hash).toString("hex")}` }); + } + } + const licensedResults = []; + + for (const backend of backends) { + assert(["celld", "dynamodb"].includes(backend), "unknown proof backend"); + const audit = backend === "celld" ? cellAudit : dynamoPublish; + const publisherEnv = backend === "celld" ? { ...cellBase, GEO_PUBLISH_TOKEN: publisher } : { ...base, AWS_ENDPOINT_URL_DYNAMODB: dynamoPublish.endpoint }; + const servingEnv = backend === "celld" ? { ...cellBase, GEO_SERVE_TOKEN: serving } : { ...base, AWS_ENDPOINT_URL_DYNAMODB: dynamoServe.endpoint }; + const settings = (reader) => backend === "celld" + ? { backend, settings: { endpoint: cellEndpoint, store: "geo", access: { mode: "authenticated", token_env: reader ? "GEO_SERVE_TOKEN" : "GEO_PUBLISH_TOKEN" } } } + : { backend, settings: { region: "us-east-1", table, namespace } }; + const publisherFile = join(work, `${backend}-publisher.json`); + const servingFile = join(work, `${backend}-serving.json`); + const bindings = join(work, `${backend}-bindings.toml`); + await jsonFile(publisherFile, settings(false)); + await jsonFile(servingFile, settings(true)); + await writeFile(bindings, "version = 1\n[kv.fixture_geo]\n" + (backend === "celld" + ? `provider = "celld-kv"\nendpoint = "${cellEndpoint}"\nstore = "geo"\n[kv.fixture_geo.access]\nmode = "authenticated"\ntoken_env = "GEO_SERVE_TOKEN"\n` + : `provider = "aws-dynamodb"\nregion = "us-east-1"\ntable = "${table}"\nnamespace = "${namespace}"\n`)); + const tool = (args, reader = false) => run(binaries.tool, args, { env: reader ? servingEnv : publisherEnv }); + const publish = async (artifact) => JSON.parse(await tool(["publish", artifact.dir, publisherFile])); + const verify = async (artifact) => { + const observed = backend === "celld" ? cellAudit : dynamoServe; + const before = observed.calls.length; + const result = await tool(["verify-remote", artifact.dir, servingFile, artifact.source, "256"], true) + .catch((error) => { + const calls = observed.calls.slice(before); + throw new Error(`${backend} verify failed after ${calls.length} calls; last responses: ${JSON.stringify(calls.slice(-8))}`, { cause: error }); + }); + return JSON.parse(result); + }; + const expectFailure = async (args, reader = false) => { + const child = start(binaries.tool, args, { env: reader ? servingEnv : publisherEnv }); + const outcome = await within(child.done, "expected CLI rejection", 30_000); + assert.equal(outcome.code, 1, `expected failed command, got ${JSON.stringify(outcome)}\n${child.output}`); + }; + const cellRequest = (operation, objectKey, value, token = publisher) => request( + `${cellEndpoint}/v1/stores/geo/${operation}?key=${encodeURIComponent(objectKey)}`, + { ca, method: operation === "value" ? "GET" : "POST", body: value, + headers: { Authorization: `Bearer ${token}`, "x-edgezero-mutation-id": randomBytes(16).toString("base64url") } }, + ); + const keyOf = (objectKey) => ({ pk: { S: namespace }, sk: { S: objectKey } }); + const read = async (objectKey) => { + if (backend === "celld") { + const response = await cellRequest("value", objectKey); + assert([200, 404].includes(response.status), `celld read status ${response.status}`); + return response.status === 200 ? response.bytes : null; + } + const response = await dynamo("GetItem", { TableName: table, Key: keyOf(objectKey), ConsistentRead: true }); + return response.Item ? Buffer.from(response.Item.v.B, "base64") : null; + }; + const put = async (objectKey, value) => { + if (backend === "celld") assert.equal((await cellRequest("put", objectKey, value)).status, 204); + else await dynamo("PutItem", { TableName: table, Item: { ...keyOf(objectKey), v: { B: value.toString("base64") } } }); + }; + const remove = async (objectKey) => { + if (backend === "celld") assert.equal((await cellRequest("delete", objectKey)).status, 204); + else await dynamo("DeleteItem", { TableName: table, Key: keyOf(objectKey) }); + }; + const [a, b] = artifacts; + // Actual uncertain acknowledgment, not just a preseeded partial inventory. + const applied = audit.arm(); + const uploader = start(binaries.tool, ["publish", a.dir, publisherFile], { env: publisherEnv }); + try { assert.equal((await within(applied, "interrupted applied publication")).status, backend === "celld" ? 204 : 200); } + finally { uploader.kill(); await uploader.done; audit.fault = null; } + assert.equal(await read(a.receipt.manifest_key), null, "manifest must not be published by interrupted uploader"); + const resumed = await publish(a); + assert(resumed.identical >= 1 && resumed.written >= 1, "rerun must reconcile the applied write"); + assert.deepEqual(await publish(a), { objects: a.inventory.length, written: 0, identical: a.inventory.length }); + assert.equal((await verify(a)).oracle_samples, 512); + + const damaged = a.inventory.find((item) => item.key !== a.receipt.manifest_key); + const original = await readFile(join(a.dir, damaged.key)); + await put(damaged.key, Buffer.from("conflicting immutable bytes")); + await expectFailure(["publish", a.dir, publisherFile]); + assert.equal((await read(damaged.key)).toString(), "conflicting immutable bytes", "conflict must not be overwritten"); + await put(damaged.key, original); + await remove(damaged.key); + await expectFailure(["verify-remote", a.dir, servingFile, a.source, "256"], true); + await publish(a); + assert.equal((await verify(a)).status, "verified"); + await publish(b); + assert.equal((await verify(b)).status, "verified"); + + if (backend === "celld") { + assert.equal((await cellRequest("put", "must-not-write", Buffer.from("denied"), serving)).status, 403); + assert.equal((await cellRequest("delete", a.receipt.manifest_key, undefined, serving)).status, 403); + assert.equal(await read("must-not-write"), null); + } + const hosts = new Set(); + const launch = async (artifact) => { + const started = performance.now(); + const endpoint = `http://127.0.0.1:${await port()}`; + const cwd = join(work, `${backend}-host-${randomBytes(4).toString("hex")}`); + await mkdir(cwd); + const child = start(binaries.host, [], { cwd, env: hostEnvironment(servingEnv, endpoint, bindings, artifact.pin) }); + const host = { endpoint, child }; + hosts.add(host); + await waitFor(async () => (await request(`${endpoint}/ready`)).status === 200, `${backend} managed readiness`, child); + assert(child.output.includes("INITIALIZED")); + host.startupMs = performance.now() - started; + return host; + }; + const stop = async (host) => { + assert.equal((await request(`${host.endpoint}/stop`)).status, 200); + await host.child.success(); + hosts.delete(host); + }; + const geo = async (host, forwarded) => { + const response = await request(`${host.endpoint}/geo`, { + headers: { "cloudfront-viewer-country": "FR", ...(forwarded ? { Forwarded: `for=${forwarded}` } : {}) }, + }); + assert.equal(response.status, 200); + const value = JSON.parse(response.bytes); + assert.equal(value.immutable, true); + assert.equal(value.credential_absent, true); + assert.equal(value.geo.provider, "kv"); + return value.geo; + }; + const hostA = await launch(a); + const hostB = await launch(b); + const servingCalls = () => backend === "celld" ? cellAudit.calls.filter((call) => call.role === "serving") : dynamoServe.calls; + const budgetAudit = backend === "celld" ? cellAudit : dynamoServe; + const blocked = budgetAudit.hold(4); + const timedRequests = Array.from({ length: 4 }, () => geo(hostA, "192.0.2.238")); + // Use a fully concurrent fixture before exercising ordinary cold or cached paths. + const failures = Promise.all(timedRequests); + failures.catch(() => {}); // Also join in finally if any earlier assertion fails. + try { + await within(blocked, "four lookup slots entered", 3000); + const beforeSaturation = servingCalls().length; + assert.equal((await geo(hostA, "192.0.2.238")).status, "provider_failure"); + assert.equal(servingCalls().length, beforeSaturation, "a saturated request must not begin I/O"); + for (const failure of await failures) assert.equal(failure.status, "provider_failure"); + } finally { + budgetAudit.heldReads = null; + await Promise.allSettled(timedRequests); + } + // The subsequent cold-path requests also prove timed-out work released its slots. + const beforeMissing = servingCalls().length; + assert.equal((await geo(hostA)).status, "missing_reader_ip"); + assert.equal((await geo(hostA, "unknown")).status, "missing_reader_ip"); + assert.equal(servingCalls().length, beforeMissing, "missing/untrusted reader must issue zero KV reads"); + await assert.rejects(request(`${hostA.endpoint}/geo-abort`, { headers: { Forwarded: "for=192.0.2.238" } }), /socket hang up/); + assert.equal(servingCalls().length, beforeMissing, "aborted admission must issue zero KV reads"); + assert.equal((await geo(hostA, "192.0.2.1")).status, "no_match"); + assert.equal((await geo(hostA, "192.0.2.2")).status, "unsupported_geography"); + assert.equal((await geo(hostA, "198.51.100.1")).status, "no_match", "registered country must not substitute"); + const first = await geo(hostA, "192.0.2.0"); + assert.equal(first.country, "US"); + assert.equal(first.subdivision, "CA"); + assert.equal(first.trust_basis, "CheckedForwardedClient"); + assert.deepEqual(await geo(hostA, '"[::ffff:192.0.2.0]"'), first); + const ipv6 = await geo(hostA, '"[2001:db8::1]"'); + assert.equal(ipv6.country, "GB"); + assert.equal(ipv6.subdivision, null); + assert.equal((await geo(hostB, "192.0.2.0")).country, "DE"); + assert.deepEqual(await geo(hostA, "192.0.2.0"), first, "simultaneous B pin cannot change A cache"); + const rootKey = `geo/v1/${a.manifest.generation}/v4/${Buffer.from(a.manifest.ipv4.hash).toString("hex")}`; + const leaves = a.inventory.filter((item) => item.key.includes("/v4/") && item.key !== rootKey); + assert(leaves.length >= 2, "synthetic artifact must exercise uncached descendants"); + const coldLeaf = leaves.at(-1); + const leafBytes = await readFile(join(a.dir, coldLeaf.key)); + await remove(coldLeaf.key); + assert.equal((await geo(hostA, "192.0.2.238")).status, "provider_failure", "missing descendant is not a gap or fallback"); + await put(coldLeaf.key, Buffer.from("corrupt descendant")); + assert.equal((await geo(hostA, "192.0.2.238")).status, "provider_failure", "corrupt descendant is not a gap or fallback"); + await put(coldLeaf.key, leafBytes); + assert.equal((await geo(hostA, "192.0.2.238")).country, "US", "restored immutable descendant resolves"); + await stop(hostA); + const restarted = await launch(a); + assert.deepEqual(await geo(restarted, "192.0.2.0"), first, "restart retains explicit A pin"); + await stop(restarted); + await stop(hostB); + await verify(a); + const rollback = await launch(a); + assert.deepEqual(await geo(rollback, "192.0.2.0"), first, "reverified rollback to A"); + await stop(rollback); + + const rootBytes = await readFile(join(a.dir, rootKey)); + await remove(rootKey); + const failedEndpoint = `http://127.0.0.1:${await port()}`; + const failed = start(binaries.host, [], { env: hostEnvironment(servingEnv, failedEndpoint, bindings, a.pin) }); + assert.equal((await within(failed.done, "invalid root startup rejection")).code, 1); + assert(!failed.output.includes("INITIALIZED"), "invalid root must fail before initialization/readiness"); + await assert.rejects(request(`${failedEndpoint}/ready`), /ECONNREFUSED/); + await put(rootKey, rootBytes); + if (backend === "dynamodb") { + assert(servingCalls().some((call) => call.operation === "GetItem")); + assert(servingCalls().some((call) => call.operation === "DescribeTable")); + assert(servingCalls().every((call) => ["DescribeTable", "GetItem"].includes(call.operation)), "serving must issue no writes or scans"); + } else { + assert(servingCalls().filter((call) => call.write).length === 2, "only the two intentional denied serving mutations"); + } + assert.equal(hosts.size, 0); + for (const artifact of [b, a]) { + for (const item of artifact.inventory) await remove(item.key); + for (const item of artifact.inventory) assert.equal(await read(item.key), null, "inventory retirement readback"); + } + for (const artifact of licensed) { + const startCall = audit.calls.length; + const written = await run(binaries.tool, ["publish", artifact.dir, publisherFile], { env: publisherEnv }, 600_000) + .then(JSON.parse) + .catch((error) => { + const calls = audit.calls.slice(startCall); + throw new Error(`${backend} publish failed after ${calls.length} calls; last responses: ${JSON.stringify(calls.slice(-8))}`, { cause: error }); + }); + assert.equal(written.objects, artifact.inventory.length); + const verifyAudit = backend === "celld" ? cellAudit : dynamoServe; + const beforeVerify = verifyAudit.calls.length; + const checked = await run(binaries.tool, [ + "verify-remote", artifact.dir, servingFile, artifact.source, "2048", + ], { env: servingEnv }, 600_000) + .then(JSON.parse) + .catch((error) => { + const calls = verifyAudit.calls.slice(beforeVerify); + throw new Error(`${backend} verify failed after ${calls.length} calls; last responses: ${JSON.stringify(calls.slice(-8))}`, { cause: error }); + }); + assert.equal(checked.oracle_samples, 4096); + const host = await launch(artifact); + const startupRss = await peakRss(host.child.child.pid); + const passes = []; + // First pass starts with an empty serving cache. Repeating the same workload + // measures reuse; neither pass claims production traffic or backend disk state. + for (const pass of ["cold-cache-start", "repeated-pass"]) { + const before = servingCalls().length; + const durations = []; + for (let offset = 0; offset < artifact.workload.length; offset += 4) { + await Promise.all(artifact.workload.slice(offset, offset + 4).map(async (address) => { + const startTime = performance.now(); + const outcome = await geo(host, address); + assert(["located", "no_match", "unsupported_geography"].includes(outcome.status), "licensed-size lookup failed"); + durations.push(performance.now() - startTime); + })); + } + const reads = servingCalls().slice(before); + passes.push({ pass, requests: durations.length, ...latencyStats(durations), + reads_per_request: reads.length / durations.length, + backend_response_body_bytes_per_request: reads.reduce((total, call) => total + call.responseBytes, 0) / durations.length, + peak_process_rss_bytes: await peakRss(host.child.child.pid) }); + } + await stop(host); + for (const item of artifact.inventory) await remove(item.key); + // The run's loopback stores and temporary artifacts are also destroyed in finally. + licensedResults.push({ backend, pin: artifact.pin, object_bytes: artifact.report.object_budget_bytes, + objects: written.objects, encoded_bytes: artifact.report.encoded_bytes_including_manifest, + ipv4_lookup_objects: artifact.report.ipv4_lookup_objects, ipv6_lookup_objects: artifact.report.ipv6_lookup_objects, + source: artifact.manifest.source, inventory_sha256: hash(JSON.stringify(artifact.inventory)), + oracle_samples: checked.oracle_samples, startup_ms: host.startupMs, + startup_peak_process_rss_bytes: startupRss, request_concurrency: 4, passes }); + log(`${backend}: ${artifact.report.object_budget_bytes}-byte City objects published, verified and measured`); + } + results.push({ backend, pin: a.pin, objects: a.inventory.length, interrupted_resume: resumed, + oracle_samples_per_generation: 512, serving_operations: [...new Set(servingCalls().map((call) => call.operation))], + checks: ["interrupted-and-identical-publication", "conflict-preserved", "incomplete-tree-rejected", "read-only-serving", + "managed-axum-ipv4-ipv6-mapped-gap-unsupported", "missing-reader-and-admission-abort-zero-reads", + "missing-corrupt-descendant-provider-failure", "missing-root-prevents-readiness", "deadline-and-saturation-release-slots", + "simultaneous-pins", "restart", "rollback", "inventory-retirement"] }); + log(`${backend}: synthetic publication, remote verification, managed requests and lifecycle passed`); + } + for (const [name, binary] of Object.entries(binaries)) assert.equal(hash(await readFile(binary)), fingerprints[name], `${name} executable changed during proof`); + const report = { status: "passed", scope: licensedSource ? "loopback synthetic and licensed City local proof; no hosted qualification" + : "loopback synthetic local proof only; no licensed data or hosted IAM qualification", + binaries, sha256: fingerprints, dynamodb_image: DYNAMO_IMAGE, + generations: artifacts.map((artifact) => ({ pin: artifact.pin, objects: artifact.inventory.length, + encoded_bytes: artifact.inventory.reduce((total, item) => total + item.bytes, 0), + inventory_sha256: hash(JSON.stringify(artifact.inventory)), source: artifact.manifest.source })), + results, licensed_results: licensedResults }; + log(JSON.stringify(report, null, 2)); + return report; + } finally { + for (const child of children) child.kill(); + await Promise.allSettled([...children].map((child) => within(child.done, "cleanup child", 5000))); + for (const server of servers.reverse()) { + server.closeAllConnections?.(); + server.close(); + } + // Cleanup uses the same cleared environment. Never touches unowned resources. + const cleanupCommand = async (args) => { + const child = new Process("docker", args, { cwd: work, env: base }, children); + return child.success(); + }; + try { + if (containerCreated) await cleanupCommand(["rm", "-f", container]); + if (networkCreated) await cleanupCommand(["network", "rm", network]); + } finally { + await rm(work, { recursive: true, force: true }); + process.removeListener("SIGINT", onInterrupt); + process.removeListener("SIGTERM", onInterrupt); + } + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runIntegration().catch((error) => { console.error(error.stack); process.exitCode = 1; }); +} diff --git a/scripts/run_geo_kv_integration.test.mjs b/scripts/run_geo_kv_integration.test.mjs new file mode 100644 index 000000000..1acd41e1d --- /dev/null +++ b/scripts/run_geo_kv_integration.test.mjs @@ -0,0 +1,127 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import test from "node:test"; +import { auditProxy, childEnvironment, DYNAMO_IMAGE, localEndpoint, runIntegration, serviceConfig, syntheticCsv } from "./run_geo_kv_integration.mjs"; +import { fixtureClient } from "./native_aws_test_admin.mjs"; + +async function server(t, handler) { + const instance = http.createServer(handler); + await new Promise((resolve) => instance.listen(0, "127.0.0.1", resolve)); + t.after(() => { instance.closeAllConnections(); instance.close(); }); + return `http://127.0.0.1:${instance.address().port}`; +} + +function post(endpoint, operation, body = "{}") { + return new Promise((resolve, reject) => { + const req = http.request(endpoint, { + method: "POST", headers: { "x-amz-target": `DynamoDB_20120810.${operation}` }, + }, (response) => { + let text = ""; + response.on("data", (chunk) => { text += chunk; }); + response.on("end", () => resolve({ status: response.statusCode, text })); + response.on("error", reject); + }); + req.on("error", reject); + req.end(body); + }); +} + +const classify = (req) => ({ operation: req.headers["x-amz-target"].split(".").at(-1), write: req.headers["x-amz-target"].endsWith(".PutItem") }); + +test("local proof rejects external, ambiguous and credential-bearing endpoints", () => { + assert.equal(localEndpoint("http://127.0.0.1:1234"), "http://127.0.0.1:1234"); + assert.equal(localEndpoint("https://127.0.0.1:1234", "https:"), "https://127.0.0.1:1234"); + for (const endpoint of [ + "http://aws.example:8000", "http://localhost:8000", "http://127.1:8000", "http://2130706433:8000", + "http://127.0.0.1:0", "http://127.0.0.1:65536", "http://127.0.0.1:8000/", + "http://user:secret@127.0.0.1:8000", "https://127.0.0.1:8000", "http://127.0.0.1:8000?redirect=remote", + ]) assert.throws(() => localEndpoint(endpoint), undefined, endpoint); +}); + +test("child environment cannot inherit production credentials, endpoints, proxy or artifact overrides", () => { + const env = childEnvironment({ + PATH: "/bin", HOME: "/production-home", AWS_PROFILE: "production", AWS_ACCESS_KEY_ID: "real-key", + AWS_SECRET_ACCESS_KEY: "real-secret", AWS_ENDPOINT_URL_DYNAMODB: "https://aws.example", + HTTP_PROXY: "https://proxy.example", SSL_CERT_FILE: "/production-ca", + FIXTURE_GEO_ARTIFACT: "/other-artifact", EDGEZERO__STORE_BINDINGS_FILE: "/production-bindings", + EDGEZERO__ADAPTER__GEO__PROVIDER: "cloudfront", + }, "/tmp/local-proof"); + assert.equal(env.HOME, "/tmp/local-proof/home"); + assert.equal(env.AWS_ACCESS_KEY_ID, "DUMMYLOCAL"); + assert.equal(env.AWS_SECRET_ACCESS_KEY, "DUMMYLOCAL"); + assert.equal(env.AWS_EC2_METADATA_DISABLED, "true"); + for (const key of ["AWS_PROFILE", "AWS_ENDPOINT_URL_DYNAMODB", "HTTP_PROXY", "SSL_CERT_FILE", "FIXTURE_GEO_ARTIFACT", "EDGEZERO__STORE_BINDINGS_FILE", "EDGEZERO__ADAPTER__GEO__PROVIDER"]) assert.equal(env[key], undefined); + assert.match(DYNAMO_IMAGE, /^amazon\/dynamodb-local@sha256:[0-9a-f]{64}$/); +}); + +test("missing binaries fail prerequisites, never produce a passing skip or build", async () => { + await assert.rejects(runIntegration({ env: {} }), /explicitly selected absolute executable/); + await assert.rejects(runIntegration({ env: { CELLD_BIN: "relative-celld" } }), /explicitly selected absolute executable/); +}); + +test("celld policy separates publisher grants and read-only serving token without storing plaintext", () => { + const config = serviceConfig("publisher-token", "serving-token"); + const policy = JSON.parse(config.vars.EDGEZERO_STORE_POLICY); + assert.equal(policy.access.mode, "authenticated"); + assert.deepEqual(policy.access.tokens[0].grants[0].operations, ["read", "write", "delete", "list"]); + assert.deepEqual(policy.access.tokens[1].grants[0].operations, ["read"]); + assert(!JSON.stringify(config).includes("publisher-token")); + assert(!JSON.stringify(config).includes("serving-token")); +}); + +test("synthetic A and B preserve boundaries, unsupported country, country-only and missing-geoname cases", () => { + const a = syntheticCsv(); + const b = syntheticCsv("DE", "BE"); + assert.equal(Object.keys(a).length, 3); + assert.match(a["GeoLite2-City-Locations-en.csv"], /1,US,CA\n2,GB,\n3,XK,/); + assert.match(b["GeoLite2-City-Locations-en.csv"], /1,DE,BE/); + assert.equal(a["GeoLite2-City-Blocks-IPv4.csv"], b["GeoLite2-City-Blocks-IPv4.csv"]); + assert.match(a["GeoLite2-City-Blocks-IPv4.csv"], /192\.0\.2\.0\/32,1,2\n192\.0\.2\.2\/32,3,2/); + assert.match(a["GeoLite2-City-Blocks-IPv4.csv"], /192\.0\.2\.238\/32/); + assert.match(a["GeoLite2-City-Blocks-IPv4.csv"], /198\.51\.100\.0\/24,,1/); + assert.match(a["GeoLite2-City-Blocks-IPv6.csv"], /2001:db8::\/126,2/); +}); + +test("audit forwards each call exactly once and records operation without retaining reader IPs or credentials", async (t) => { + const requests = []; + const target = await server(t, async (req, res) => { + let body = ""; + for await (const chunk of req) body += chunk; + requests.push(body); + res.end('{"Item":null}'); + }); + const proxy = await auditProxy(target, classify); + t.after(() => { proxy.server.closeAllConnections(); proxy.server.close(); }); + const result = await post(proxy.endpoint, "GetItem", '{"key":"synthetic"}'); + assert.equal(result.status, 200); + assert.deepEqual(requests, ['{"key":"synthetic"}']); + assert.deepEqual(proxy.calls, [{ operation: "GetItem", write: false, responseBytes: 13, status: 200 }]); +}); + +test("uncertain-write fault observes backend completion but withholds acknowledgement and never retries", async (t) => { + let applied = 0; + const target = await server(t, async (req, res) => { + for await (const chunk of req) void chunk; + applied++; + res.end("{}"); + }); + const proxy = await auditProxy(target, classify); + t.after(() => { proxy.server.closeAllConnections(); proxy.server.close(); }); + const appliedPromise = proxy.arm(); + const req = http.request(proxy.endpoint, { method: "POST", headers: { "x-amz-target": "DynamoDB_20120810.PutItem" } }); + let acknowledged = false; + req.on("response", () => { acknowledged = true; }); + req.on("error", () => {}); + req.end("{}"); + const result = await appliedPromise; + assert.equal(result.status, 200); + assert.equal(applied, 1); + assert.equal(acknowledged, false); + req.destroy(); + assert.equal((await post(proxy.endpoint, "GetItem")).status, 200); + assert.equal(applied, 2, "one call per explicit request; fault does not replay"); +}); + +test("exported admin client still rejects external endpoint before any I/O", () => { + assert.throws(() => fixtureClient("https://dynamodb.us-east-1.amazonaws.com", "dynamodb"), /literal/); +});