diff --git a/.env.example b/.env.example index 1121ecd9b..c2ac88e3a 100644 --- a/.env.example +++ b/.env.example @@ -37,7 +37,7 @@ TRUSTED_SERVER__REQUEST_SIGNING__ENABLED=false # Prebid TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=false -# TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid-server.com/openrtb2/auction +# TRUSTED_SERVER__INTEGRATIONS__PREBID__SERVER_URL=https://prebid-server.example.com/openrtb2/auction # TRUSTED_SERVER__INTEGRATIONS__PREBID__TIMEOUT_MS=1000 # TRUSTED_SERVER__INTEGRATIONS__PREBID__BIDDERS=kargo,rubicon,appnexus # TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDES='{"bidder-name":{"param1":12345,"param2":"value"}}' @@ -45,6 +45,7 @@ TRUSTED_SERVER__INTEGRATIONS__PREBID__ENABLED=false # TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_ZONE_OVERRIDES='{"kargo":{"header":{"placementId":"_abc"}}}' # Preferred canonical env shape for future generic rules # TRUSTED_SERVER__INTEGRATIONS__PREBID__BID_PARAM_OVERRIDE_RULES='[{"when":{"bidder":"kargo","zone":"header"},"set":{"placementId":"_abc"}}]' +# TRUSTED_SERVER__INTEGRATIONS__PREBID__SUPPRESS_NURL_BIDDERS=exampleBidder,anotherBidder # TRUSTED_SERVER__INTEGRATIONS__PREBID__AUTO_CONFIGURE=false # TRUSTED_SERVER__INTEGRATIONS__PREBID__DEBUG=false # TRUSTED_SERVER__INTEGRATIONS__PREBID__TEST_MODE=false diff --git a/.gitignore b/.gitignore index dabeec55d..a616db591 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,9 @@ src/*.html /crates/trusted-server-integration-tests/browser/test-results/ /crates/trusted-server-integration-tests/browser/playwright-report/ /crates/trusted-server-integration-tests/browser/.browser-test-state.json + +# Defunct pre-rename crate dirs (renamed to crates/trusted-server-*); ignore the +# leftover local build artifacts (node_modules, target, dist) that remain on disk. +/crates/js/ +/crates/integration-tests/ + diff --git a/Cargo.lock b/Cargo.lock index c77167838..0427b972d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2028,6 +2028,12 @@ dependencies = [ "r-efi 6.0.0", ] +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + [[package]] name = "group" version = "0.13.0" @@ -5283,6 +5289,7 @@ dependencies = [ "flate2", "futures", "getrandom 0.2.17", + "glob", "hex", "hmac", "http", @@ -5299,6 +5306,7 @@ dependencies = [ "sha2 0.10.9", "subtle", "temp-env", + "tokio", "toml", "trusted-server-js", "trusted-server-openrtb", diff --git a/Cargo.toml b/Cargo.toml index 3d139333e..f410c011b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ fastly = "0.12" fern = "0.7.1" flate2 = "1.1" futures = "0.3" +glob = "0.3" getrandom = "0.2" hex = "0.4.3" hmac = "0.12.1" diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index f312b171e..2f4329574 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -19,7 +19,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - buffer_publisher_response, handle_publisher_request, handle_tsjs_dynamic, + AuctionDispatch, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -134,6 +135,34 @@ where .unwrap_or_else(|e| http_error(&e))) } +// --------------------------------------------------------------------------- +// EC context +// --------------------------------------------------------------------------- + +/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, +/// `/__ts/page-bids`, and the publisher fallback). +/// +/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction +/// Unknown, which fails the auction consent gate closed even for consented +/// users. Geo comes from the platform (a no-op on the local Axum dev server, so +/// jurisdiction stays Unknown there unless the request carries TCF consent). A +/// malformed consent string is logged and falls back to the default +/// (fail-closed) context rather than being silently swallowed. +fn build_ec_context(state: &AppState, services: &RuntimeServices, req: &Request) -> EcContext { + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + EcContext::read_from_request_with_geo(&state.settings, req, services, geo_info.as_ref()) + .unwrap_or_else(|e| { + log::warn!("EC context read failed: {e:?}"); + EcContext::default() + }) +} + // --------------------------------------------------------------------------- // Fallback dispatcher (tsjs / integration proxy / publisher) // --------------------------------------------------------------------------- @@ -171,9 +200,34 @@ async fn dispatch_fallback( }); } - handle_publisher_request(&state.settings, &state.registry, services, req) - .await - .and_then(|pr| buffer_publisher_response(pr, &method, &state.settings, &state.registry)) + // Run the server-side auction with the configured creative-opportunity + // slots; `handle_publisher_request` matches them against the request path. + let mut ec_context = build_ec_context(state, services, &req); + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + let publisher_response = handle_publisher_request( + &state.settings, + services, + None, + &mut ec_context, + auction, + req, + ) + .await?; + // Async finalize so the dispatched auction is collected and its bids are + // injected before `` (the sync buffer path would drop them). + buffer_publisher_response_async( + publisher_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + services, + ) + .await } fn fallback_handler( @@ -202,6 +256,7 @@ enum NamedRouteHandler { /// reach the publisher fallback (which would leak admin credentials). LegacyAdminDenied, Auction, + PageBids, FirstPartyProxy, FirstPartyClick, FirstPartySign, @@ -224,7 +279,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 11] { +fn named_routes() -> [NamedRoute; 12] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -270,6 +325,13 @@ fn named_routes() -> [NamedRoute; 11] { primary_methods: &[Method::POST], handler: NamedRouteHandler::Auction, }, + // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS + // preflight guard for this side-effecting endpoint. + NamedRoute { + path: "/__ts/page-bids", + primary_methods: &[Method::GET, Method::OPTIONS], + handler: NamedRouteHandler::PageBids, + }, NamedRoute { path: "/first-party/proxy", primary_methods: &[Method::GET], @@ -328,7 +390,10 @@ fn named_route_handler( } NamedRouteHandler::LegacyAdminDenied => Ok(legacy_admin_alias_denied()), NamedRouteHandler::Auction => { - let ec_context = EcContext::default(); + // Build the geo-aware EC context so the auction consent + // gate sees the caller's jurisdiction — `EcContext::default()` + // fails it closed for consented users. + let ec_context = build_ec_context(&state, &services, &req); handle_auction( &state.settings, &state.orchestrator, @@ -340,6 +405,30 @@ fn named_route_handler( ) .await } + NamedRouteHandler::PageBids => { + // SPA re-auction endpoint. `OPTIONS` is a CORS preflight + // for this side-effecting GET and is always denied so the + // GET handler's `X-TSJS-Page-Bids` gate stays trustworthy. + if req.method() == Method::OPTIONS { + Ok(page_bids_preflight_denied()) + } else { + let ec_context = build_ec_context(&state, &services, &req); + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + handle_page_bids( + &state.settings, + &services, + None, + auction, + &ec_context, + req, + ) + .await + } + } NamedRouteHandler::FirstPartyProxy => { handle_first_party_proxy(&state.settings, &services, req).await } diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs index 8ad362a97..45cbedc2c 100644 --- a/crates/trusted-server-adapter-axum/src/middleware.rs +++ b/crates/trusted-server-adapter-axum/src/middleware.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response}; +use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; @@ -88,26 +88,19 @@ impl Middleware for AuthMiddleware { /// /// Unlike the Fastly variant, geo is always unavailable so `X-Geo-Info-Available: false` /// is unconditionally emitted. Fastly-specific headers are omitted. -/// Operator-configured `settings.response_headers` are applied last and can override -/// any managed header. +/// Operator-configured `settings.response_headers` are applied last (with the +/// shared cookie cache-privacy hardening) and can override any managed header. pub(crate) fn apply_finalize_headers(settings: &Settings, response: &mut Response) { response.headers_mut().insert( HEADER_X_GEO_INFO_AVAILABLE, HeaderValue::from_static("false"), ); - for (key, value) in &settings.response_headers { - let header_name = HeaderName::from_bytes(key.as_bytes()); - let header_value = HeaderValue::from_str(value); - if let (Ok(header_name), Ok(header_value)) = (header_name, header_value) { - response.headers_mut().insert(header_name, header_value); - } else { - log::warn!( - "Skipping invalid configured response header value for {}", - key - ); - } - } + // Cookie-bearing responses stay private to shared caches and operator + // headers cannot re-enable caching for uncacheable per-user payloads. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 2765131dd..461b567d1 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -642,6 +642,7 @@ mod tests { port: None, certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), host_header_override: None, }; let name1 = backend.predict_name(&spec).expect("should return a name"); @@ -661,6 +662,7 @@ mod tests { port: None, certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), host_header_override: None, }; assert_eq!( diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index 0319ab1c9..353f55a2c 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -21,7 +21,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - PublisherResponse, buffer_publisher_response, handle_publisher_request, handle_tsjs_dynamic, + AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_page_bids, + handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -124,6 +125,29 @@ fn build_per_request_services(ctx: &RequestContext) -> RuntimeServices { build_runtime_services(ctx) } +/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, +/// `/__ts/page-bids`, and the publisher fallback). +/// +/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction +/// Unknown, which fails the auction consent gate closed even for consented +/// users. Geo comes from the Workers `cf` object when deployed. A malformed +/// consent string is logged and falls back to the default (fail-closed) context +/// rather than being silently swallowed. +fn build_ec_context(settings: &Settings, services: &RuntimeServices, req: &Request) -> EcContext { + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) + .unwrap_or_else(|e| { + log::warn!("EC context read failed: {e:?}"); + EcContext::default() + }) +} + // --------------------------------------------------------------------------- // Handler factory // --------------------------------------------------------------------------- @@ -161,16 +185,27 @@ where /// Collapse a [`PublisherResponse`] into a plain [`Response`]. /// -/// Delegates to the shared [`buffer_publisher_response`], which enforces +/// Delegates to the shared [`buffer_publisher_response_async`], which collects +/// the dispatched server-side auction and enforces /// `settings.publisher.max_buffered_body_bytes`, then removes any /// `Transfer-Encoding` header since the buffered body is no longer chunked. -fn resolve_publisher_response( +async fn resolve_publisher_response( publisher_response: PublisherResponse, method: &Method, settings: &Settings, registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, ) -> Result> { - let mut response = buffer_publisher_response(publisher_response, method, settings, registry)?; + let mut response = buffer_publisher_response_async( + publisher_response, + method, + settings, + registry, + orchestrator, + services, + ) + .await?; response.headers_mut().remove(header::TRANSFER_ENCODING); Ok(response) } @@ -340,11 +375,35 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - handle_publisher_request(&state.settings, &state.registry, &services, req) - .await - .and_then(|pr| { - resolve_publisher_response(pr, &method, &state.settings, &state.registry) - }) + let mut ec_context = build_ec_context(&state.settings, &services, &req); + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + match handle_publisher_request( + &state.settings, + &services, + None, + &mut ec_context, + auction, + req, + ) + .await + { + Ok(pr) => { + resolve_publisher_response( + pr, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &services, + ) + .await + } + Err(e) => Err(e), + } }; Ok(result.unwrap_or_else(|e| http_error(&e))) @@ -398,7 +457,10 @@ fn build_router(state: &Arc) -> RouterService { .post( "/auction", make_handler(Arc::clone(&state), |s, services, req| async move { - let ec_context = EcContext::default(); + // Build the geo-aware EC context so the auction consent gate + // sees the caller's jurisdiction — `EcContext::default()` + // fails it closed for consented users. + let ec_context = build_ec_context(&s.settings, &services, &req); handle_auction( &s.settings, &s.orchestrator, @@ -411,6 +473,28 @@ fn build_router(state: &Arc) -> RouterService { .await }), ) + // SPA re-auction endpoint. The OPTIONS preflight for this + // side-effecting GET is denied so the GET handler's `X-TSJS-Page-Bids` + // gate stays trustworthy. + .route( + "/__ts/page-bids", + Method::OPTIONS, + make_handler(Arc::clone(&state), |_s, _services, _req| async move { + Ok(page_bids_preflight_denied()) + }), + ) + .get( + "/__ts/page-bids", + make_handler(Arc::clone(&state), |s, services, req| async move { + let ec_context = build_ec_context(&s.settings, &services, &req); + let auction = AuctionDispatch { + orchestrator: &s.orchestrator, + slots: s.settings.creative_opportunity_slots(), + registry: None, + }; + handle_page_bids(&s.settings, &services, None, auction, &ec_context, req).await + }), + ) .get( "/first-party/proxy", make_handler(Arc::clone(&state), |s, services, req| async move { diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs index 3b60cae3f..5b605bcff 100644 --- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs +++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response}; +use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; @@ -96,8 +96,8 @@ impl Middleware for AuthMiddleware { /// /// `geo_available` controls `X-Geo-Info-Available`; pass `true` when /// `cf-ipcountry` was present and non-`XX` in the incoming request. -/// Operator-configured `settings.response_headers` are applied last and can -/// override any managed header. +/// Operator-configured `settings.response_headers` are applied last (with the +/// shared cookie cache-privacy hardening) and can override any managed header. pub(crate) fn apply_finalize_headers( settings: &Settings, geo_available: bool, @@ -108,18 +108,12 @@ pub(crate) fn apply_finalize_headers( HeaderValue::from_static(if geo_available { "true" } else { "false" }), ); - for (key, value) in &settings.response_headers { - let header_name = HeaderName::from_bytes(key.as_bytes()); - let header_value = HeaderValue::from_str(value); - if let (Ok(header_name), Ok(header_value)) = (header_name, header_value) { - response.headers_mut().insert(header_name, header_value); - } else { - log::warn!( - "Skipping invalid configured response header value for {}", - key - ); - } - } + // Cloudflare is a real shared cache: cookie-bearing responses must stay + // private and operator headers must not re-enable caching for uncacheable + // per-user payloads. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index ee0e94185..27fb18918 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -115,7 +115,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, AssetProxyCachePolicy, }; use trusted_server_core::publisher::{ - buffer_publisher_response, handle_publisher_request, handle_tsjs_dynamic, + buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, AuctionDispatch, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -603,6 +604,38 @@ async fn run_named_route( ) .await } + NamedRouteHandler::PageBids => { + // SPA re-auction endpoint. `OPTIONS` is a CORS preflight for this + // side-effecting GET and is always denied so the GET handler's + // `X-TSJS-Page-Bids` gate stays trustworthy. + if req.method() == Method::OPTIONS { + return Ok(page_bids_preflight_denied()); + } + // Like the auction, page-bids reads consent data, so the consent KV + // store must be available — fail closed with 503 when configured but + // unopenable, matching legacy. + let consent_services = runtime_services_for_consent_route(&state.settings, services)?; + let partner_registry = PartnerRegistry::from_config(&state.settings.ec.partners)?; + let registry_ref = if partner_registry.is_empty() { + None + } else { + Some(&partner_registry) + }; + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: registry_ref, + }; + handle_page_bids( + &state.settings, + &consent_services, + ec.kv_graph.as_ref(), + auction, + &ec.ec_context, + req, + ) + .await + } NamedRouteHandler::FirstPartyProxy => { handle_first_party_proxy(&state.settings, services, req).await } @@ -734,16 +767,47 @@ async fn dispatch_fallback( // be opened, matching legacy behavior. match runtime_services_for_consent_route(&state.settings, services) { Ok(publisher_services) => { - handle_publisher_request(&state.settings, &state.registry, &publisher_services, req) - .await - .and_then(|pub_response| { - buffer_publisher_response( - pub_response, - &method, + // Run the server-side auction with the configured creative- + // opportunity slots and collect the dispatched bids in the + // buffered finalize (`buffer_publisher_response_async`), matching + // the legacy streaming path. `handle_publisher_request` matches the + // slots against the request path. The partner registry plus the + // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with + // server-side EIDs, same as the legacy auction. + let slots = state.settings.creative_opportunity_slots(); + match PartnerRegistry::from_config(&state.settings.ec.partners) { + Ok(partner_registry) => { + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots, + registry: Some(&partner_registry), + }; + match handle_publisher_request( &state.settings, - &state.registry, + &publisher_services, + ec.kv_graph.as_ref(), + &mut ec.ec_context, + auction, + req, ) - }) + .await + { + Ok(pub_response) => { + buffer_publisher_response_async( + pub_response, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &publisher_services, + ) + .await + } + Err(e) => Err(e), + } + } + Err(e) => Err(e), + } } Err(e) => Err(e), } @@ -926,6 +990,7 @@ enum NamedRouteHandler { SetTester, ClearTester, Auction, + PageBids, FirstPartyProxy, FirstPartyClick, FirstPartySign, @@ -1010,6 +1075,13 @@ const NAMED_ROUTES: &[NamedRoute] = &[ primary_methods: &[Method::POST], handler: NamedRouteHandler::Auction, }, + // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS + // preflight guard for this side-effecting endpoint. + NamedRoute { + path: "/__ts/page-bids", + primary_methods: &[Method::GET, Method::OPTIONS], + handler: NamedRouteHandler::PageBids, + }, NamedRoute { path: "/first-party/proxy", primary_methods: &[Method::GET], diff --git a/crates/trusted-server-adapter-fastly/src/backend.rs b/crates/trusted-server-adapter-fastly/src/backend.rs index 7763eaf0e..4056c81da 100644 --- a/crates/trusted-server-adapter-fastly/src/backend.rs +++ b/crates/trusted-server-adapter-fastly/src/backend.rs @@ -49,6 +49,8 @@ fn sanitize_backend_name_component(value: &str) -> String { /// Default first-byte timeout for backends (15 seconds). pub(crate) const DEFAULT_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +/// Default timeout between response body bytes for backends (10 seconds). +pub(crate) const DEFAULT_BETWEEN_BYTES_TIMEOUT: Duration = Duration::from_secs(10); /// Configuration for creating a dynamic Fastly backend. /// @@ -60,6 +62,7 @@ pub struct BackendConfig<'a> { port: Option, certificate_check: bool, first_byte_timeout: Duration, + between_bytes_timeout: Duration, host_header_override: Option<&'a str>, } @@ -76,6 +79,7 @@ impl<'a> BackendConfig<'a> { port: None, certificate_check: true, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_BETWEEN_BYTES_TIMEOUT, host_header_override: None, } } @@ -106,6 +110,17 @@ impl<'a> BackendConfig<'a> { self } + /// Set the maximum time to wait between response body bytes. + /// + /// Defaults to 10 seconds. Auction backends should set this to the same + /// remaining budget as the first-byte timeout so slow-drip bodies cannot + /// hold the auction past its deadline. + #[must_use] + pub fn between_bytes_timeout(mut self, timeout: Duration) -> Self { + self.between_bytes_timeout = timeout; + self + } + /// Set the outbound Host header sent to the backend origin. #[must_use] pub fn host_header_override(mut self, host: Option<&'a str>) -> Self { @@ -159,13 +174,15 @@ impl<'a> BackendConfig<'a> { } else { "_nocert" }; - let timeout_ms = self.first_byte_timeout.as_millis(); + let first_byte_timeout_ms = self.first_byte_timeout.as_millis(); + let between_bytes_timeout_ms = self.between_bytes_timeout.as_millis(); let backend_name = format!( - "backend_{}{}{}_t{}", + "backend_{}{}{}_fb{}_bb{}", sanitize_backend_name_component(&name_base), host_override_suffix, cert_suffix, - timeout_ms + first_byte_timeout_ms, + between_bytes_timeout_ms ); Ok((backend_name, target_port)) @@ -187,9 +204,10 @@ impl<'a> BackendConfig<'a> { /// Ensure a dynamic backend exists for this configuration and return its name. /// /// The backend name is derived from the scheme, host, port, certificate - /// setting, and `first_byte_timeout` to avoid collisions. Different - /// timeout values produce different backend registrations so that a - /// tight deadline cannot be silently widened by an earlier registration. + /// setting, `first_byte_timeout`, and `between_bytes_timeout` to avoid + /// collisions. Different timeout values produce different backend + /// registrations so that a tight deadline cannot be silently widened by an + /// earlier registration. /// /// # Errors /// @@ -210,7 +228,7 @@ impl<'a> BackendConfig<'a> { .override_host(&host_header) .connect_timeout(Duration::from_secs(1)) .first_byte_timeout(self.first_byte_timeout) - .between_bytes_timeout(Duration::from_secs(10)); + .between_bytes_timeout(self.between_bytes_timeout); if self.scheme.eq_ignore_ascii_case("https") { builder = builder.enable_ssl().sni_hostname(self.host); if self.certificate_check { @@ -381,7 +399,7 @@ mod tests { let name = BackendConfig::new("https", "origin.example.com") .ensure() .expect("should create backend for valid HTTPS origin"); - assert_eq!(name, "backend_https_origin_example_com_443_t15000"); + assert_eq!(name, "backend_https_origin_example_com_443_fb15000_bb10000"); } #[test] @@ -390,7 +408,10 @@ mod tests { .certificate_check(false) .ensure() .expect("should create backend with cert check disabled"); - assert_eq!(name, "backend_https_origin_example_com_443_nocert_t15000"); + assert_eq!( + name, + "backend_https_origin_example_com_443_nocert_fb15000_bb10000" + ); } #[test] @@ -399,7 +420,7 @@ mod tests { .port(Some(8080)) .ensure() .expect("should create backend for HTTP origin with explicit port"); - assert_eq!(name, "backend_http_api_test-site_org_8080_t15000"); + assert_eq!(name, "backend_http_api_test-site_org_8080_fb15000_bb10000"); } #[test] @@ -407,7 +428,7 @@ mod tests { let name = BackendConfig::new("http", "example.org") .ensure() .expect("should create backend defaulting to port 80 for HTTP"); - assert_eq!(name, "backend_http_example_org_80_t15000"); + assert_eq!(name, "backend_http_example_org_80_fb15000_bb10000"); } #[test] @@ -464,11 +485,11 @@ mod tests { ); assert_eq!( name_a, - "backend_https_origin_example_com_443_oh_www_example_com_t15000" + "backend_https_origin_example_com_443_oh_www_example_com_fb15000_bb10000" ); assert_eq!( name_b, - "backend_https_origin_example_com_443_oh_m_example_com_t15000" + "backend_https_origin_example_com_443_oh_m_example_com_fb15000_bb10000" ); } @@ -523,12 +544,39 @@ mod tests { "backends with different timeouts should have different names" ); assert!( - name_a.ends_with("_t2000"), - "name should include timeout suffix" + name_a.ends_with("_fb2000_bb10000"), + "name should include first-byte and between-bytes timeout suffix" + ); + assert!( + name_b.ends_with("_fb500_bb10000"), + "name should include first-byte and between-bytes timeout suffix" + ); + } + + #[test] + fn different_between_bytes_timeouts_produce_different_names() { + use std::time::Duration; + + let (name_a, _) = BackendConfig::new("https", "origin.example.com") + .between_bytes_timeout(Duration::from_secs(2)) + .compute_name() + .expect("should compute name with 2000ms between-bytes timeout"); + let (name_b, _) = BackendConfig::new("https", "origin.example.com") + .between_bytes_timeout(Duration::from_millis(500)) + .compute_name() + .expect("should compute name with 500ms between-bytes timeout"); + + assert_ne!( + name_a, name_b, + "backends with different between-bytes timeouts should have different names" + ); + assert!( + name_a.ends_with("_fb15000_bb2000"), + "name should include first-byte and between-bytes timeout suffix" ); assert!( - name_b.ends_with("_t500"), - "name should include timeout suffix" + name_b.ends_with("_fb15000_bb500"), + "name should include first-byte and between-bytes timeout suffix" ); } } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 489c4982d..4b4d9daff 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -46,7 +46,7 @@ use trusted_server_core::proxy::{ AssetProxyCachePolicy, }; use trusted_server_core::publisher::{ - handle_publisher_request, handle_tsjs_dynamic, stream_publisher_body, + handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, stream_publisher_body_async, OwnedProcessResponseParams, PublisherResponse, }; use trusted_server_core::request_signing::{ @@ -94,7 +94,7 @@ enum HandlerOutcome { Streaming { response: HttpResponse, body: EdgeBody, - params: OwnedProcessResponseParams, + params: Box, }, AssetStreaming { response: HttpResponse, @@ -680,6 +680,11 @@ fn send_edgezero_response( effects.apply_to_response(&mut response); } + // Final cache guard: EC finalization and request-filter effects may have + // added a per-user Set-Cookie after `apply_finalize_headers` ran, so + // re-apply the privacy downgrade before send, mirroring legacy_main. + crate::middleware::enforce_set_cookie_cache_privacy(&mut response); + let (parts, body) = response.into_parts(); match body { EdgeBody::Stream(_) => { @@ -771,6 +776,7 @@ fn legacy_main(mut req: FastlyRequest) { &state.registry, &partner_registry, &runtime_services, + state.settings.creative_opportunity_slots(), http_req, device_signals, )) @@ -828,8 +834,14 @@ fn legacy_main(mut req: FastlyRequest) { &mut response, ); } + // Apply request-filter response effects (e.g. a DataDome allow + // Set-Cookie) before the final cache guard so any per-user cookie + // they add is covered. EC finalization above may also have added the + // identity Set-Cookie; the guard runs last so it observes both. request_filter_effects.apply_to_response(&mut response); - compat::to_fastly_response(response).send_to_client(); + let mut fastly_resp = compat::to_fastly_response(response); + enforce_set_cookie_cache_privacy(&mut fastly_resp); + fastly_resp.send_to_client(); if is_real_browser { if let Some(context) = build_pull_sync_context(&ec_context) { @@ -845,7 +857,7 @@ fn legacy_main(mut req: FastlyRequest) { HandlerOutcome::Streaming { mut response, body, - params, + mut params, } => { finalize_response(&state.settings, geo_info.as_ref(), &mut response); asset_cache_policy.apply_after_route_finalization(&mut response); @@ -860,17 +872,24 @@ fn legacy_main(mut req: FastlyRequest) { &mut response, ); } + // Apply request-filter response effects (e.g. a DataDome allow + // Set-Cookie) before the final cache guard so any per-user cookie + // they add is covered. EC finalization above may also have added the + // identity Set-Cookie; the guard runs last so it observes both. request_filter_effects.apply_to_response(&mut response); - let fastly_resp = compat::to_fastly_response_skeleton(response); + let mut fastly_resp = compat::to_fastly_response_skeleton(response); + enforce_set_cookie_cache_privacy(&mut fastly_resp); let mut streaming_body = fastly_resp.stream_to_client(); let mut stream_succeeded = false; - match stream_publisher_body( + match futures::executor::block_on(stream_publisher_body_async( body, &mut streaming_body, - ¶ms, + &mut params, &state.settings, &state.registry, - ) { + &state.orchestrator, + &runtime_services, + )) { Ok(()) => { if let Err(e) = streaming_body.finish() { log::error!("failed to finish streaming body: {e}"); @@ -900,8 +919,12 @@ fn legacy_main(mut req: FastlyRequest) { HandlerOutcome::AssetStreaming { mut response, body } => { finalize_response(&state.settings, geo_info.as_ref(), &mut response); asset_cache_policy.apply_after_route_finalization(&mut response); + // A request filter (e.g. DataDome allow) can append a per-user + // Set-Cookie via response effects even on an otherwise cacheable + // asset, so guard against shared caching after applying them. request_filter_effects.apply_to_response(&mut response); - let fastly_resp = compat::to_fastly_response_skeleton(response); + let mut fastly_resp = compat::to_fastly_response_skeleton(response); + enforce_set_cookie_cache_privacy(&mut fastly_resp); let mut streaming_body = fastly_resp.stream_to_client(); if let Err(e) = futures::executor::block_on(stream_asset_body(body, &mut streaming_body)) @@ -963,12 +986,18 @@ fn build_ja4_debug_response(req: &FastlyRequest) -> FastlyResponse { .with_body(body) } +// Combines the server-side ad-stack inputs (creative-opportunity `slots`) with +// the EdgeZero dual-path requirement that `device_signals` be derived from the +// `FastlyRequest` before conversion and passed in, pushing this central +// dispatch helper to eight arguments. +#[allow(clippy::too_many_arguments)] async fn route_request( settings: &Settings, orchestrator: &AuctionOrchestrator, integration_registry: &IntegrationRegistry, partner_registry: &PartnerRegistry, runtime_services: &RuntimeServices, + slots: &[trusted_server_core::creative_opportunities::CreativeOpportunitySlot], mut req: HttpRequest, device_signals: DeviceSignals, ) -> Result> { @@ -1142,6 +1171,12 @@ async fn route_request( let path = req.uri().path().to_string(); let method = req.method().clone(); + let registry_ref = if partner_registry.is_empty() { + None + } else { + Some(partner_registry) + }; + let mut asset_cache_policy = AssetProxyCachePolicy::OriginControlled; let mut should_finalize_ec = true; @@ -1202,28 +1237,55 @@ async fn route_request( (Method::GET, "/_ts/set-tester") => (handle_set_tester(settings), false), (Method::GET, "/_ts/clear-tester") => (handle_clear_tester(settings), false), - // Unified auction endpoint. - (Method::POST, "/auction") => { - let registry_ref = if partner_registry.is_empty() { - None - } else { - Some(partner_registry) - }; - ( - handle_auction( - settings, - orchestrator, - kv_graph.as_ref(), - registry_ref, - &ec_context, - runtime_services, - req, - ) - .await, - false, + // Unified auction endpoint (returns creative HTML inline) + (Method::POST, "/auction") => ( + handle_auction( + settings, + orchestrator, + kv_graph.as_ref(), + registry_ref, + &ec_context, + runtime_services, + req, ) + .await, + false, + ), + + // Reject CORS preflight for the side-effecting page-bids endpoint at the + // adapter. The GET handler's legacy fallback trusts `X-TSJS-Page-Bids` + // precisely because this endpoint never grants a preflight; letting + // OPTIONS fall through to the publisher origin (which may return + // permissive CORS) would defeat that, allowing a cross-site page to + // trigger real PBS/APS auctions from a visitor's browser. + (Method::OPTIONS, "/__ts/page-bids") => { + let mut response = HttpResponse::new(EdgeBody::from("Forbidden")); + *response.status_mut() = edgezero_core::http::StatusCode::FORBIDDEN; + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + (Ok(response), false) } + // SPA/CSR navigation endpoint — returns slots + bids JSON for the given path + (Method::GET, "/__ts/page-bids") => ( + handle_page_bids( + settings, + runtime_services, + kv_graph.as_ref(), + trusted_server_core::publisher::AuctionDispatch { + orchestrator, + slots, + registry: registry_ref, + }, + &ec_context, + req, + ) + .await, + false, + ), + // First-party proxy/click/sign/rebuild endpoints (Method::GET, "/first-party/proxy") => ( handle_first_party_proxy(settings, runtime_services, req).await, @@ -1315,8 +1377,14 @@ async fn route_request( match handle_publisher_request( settings, - integration_registry, runtime_services, + kv_graph.as_ref(), + &mut ec_context, + trusted_server_core::publisher::AuctionDispatch { + orchestrator, + slots, + registry: registry_ref, + }, req, ) .await @@ -1410,9 +1478,45 @@ fn run_pull_sync_after_send( /// version/staging, then operator-configured `settings.response_headers`. /// This means operators can intentionally override any managed header. fn finalize_response(settings: &Settings, geo_info: Option<&GeoInfo>, response: &mut HttpResponse) { + // Legacy and EdgeZero paths share one protected finalizer so the cache / + // Set-Cookie privacy hardening cannot drift between them. `HttpResponse` and + // the middleware's `Response` are the same `edgezero_core::http::Response`. apply_finalize_headers(settings, geo_info, response); } +/// Forces cookie-bearing Fastly responses to stay private to shared caches. +/// +/// [`finalize_response`] applies this same downgrade on the [`HttpResponse`], +/// but the EC identity cookie is written later by [`ec_finalize_response`] onto +/// the converted [`FastlyResponse`], so the earlier guard never sees it. +/// Re-apply it here so a first-visit navigation whose only per-user payload is +/// the EC `Set-Cookie` can never be served with `public`/surrogate cache headers +/// inherited from the origin or operator response headers — a shared cache must +/// not be able to store and replay one visitor's EC cookie to others. +/// +/// Idempotent: a response already marked `private`/`no-store` keeps its stricter +/// `Cache-Control`, but the surrogate cache headers are stripped regardless so a +/// `no-store` cookie response can never retain shared Fastly cacheability. +fn enforce_set_cookie_cache_privacy(response: &mut FastlyResponse) { + if response.get_header("set-cookie").is_none() { + return; + } + // Strip surrogate cache headers on every cookie-bearing response, even when + // keeping a stricter `no-store`/`private` directive — Surrogate-Control is + // independent of Cache-Control and would otherwise let a shared cache store + // and replay one visitor's Set-Cookie. + for name in crate::middleware::SURROGATE_CACHE_HEADERS { + response.remove_header(*name); + } + let already_uncacheable = response + .get_header_str("cache-control") + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + if !already_uncacheable { + response.set_header("cache-control", "private, max-age=0"); + } +} + /// Builds the local `404 Not Found` returned for legacy `/admin/keys/*` /// aliases. /// diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index ceb470b7d..238afa9eb 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -16,7 +16,7 @@ use async_trait::async_trait; use edgezero_adapter_fastly::context::FastlyRequestContext; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response, StatusCode}; +use edgezero_core::http::{HeaderValue, Response, StatusCode}; use edgezero_core::middleware::{Middleware, Next}; use edgezero_core::response::IntoResponse; use std::net::IpAddr; @@ -181,13 +181,19 @@ where /// Applies all standard Trusted Server response headers to the given response. /// /// Mirrors [`crate::finalize_response`] exactly, operating on [`Response`] from -/// `edgezero_core::http` instead of `HttpResponse`. +/// `edgezero_core::http` instead of `HttpResponse`. [`crate::finalize_response`] +/// delegates here so the legacy and `EdgeZero` paths share one protected +/// finalizer. /// /// Header write order (last write wins): /// 1. Geo headers (`x-geo-*`) — or `X-Geo-Info-Available: false` when absent /// 2. `X-TS-Version` from `FASTLY_SERVICE_VERSION` env var /// 3. `X-TS-ENV: staging` when `FASTLY_IS_STAGING == "1"` -/// 4. `settings.response_headers` — operator-configured overrides applied last +/// 4. Set-Cookie cache privacy — strip surrogate cache headers and downgrade +/// `Cache-Control` to `private, max-age=0` on cookie-bearing responses +/// 5. `settings.response_headers` — operator-configured overrides, except the +/// cache-controlling headers are skipped on uncacheable (`private`/`no-store`) +/// responses so operators cannot re-enable shared caching for per-user payloads pub(crate) fn apply_finalize_headers( settings: &Settings, geo_info: Option<&GeoInfo>, @@ -216,16 +222,31 @@ pub(crate) fn apply_finalize_headers( .insert(HEADER_X_TS_ENV, HeaderValue::from_static("staging")); } - for (key, value) in &settings.response_headers { - let header_name = HeaderName::from_bytes(key.as_bytes()) - .expect("should be a valid header name: response_headers validated in prepare_runtime"); - let header_value = HeaderValue::from_str(value).expect( - "should be a valid header value: response_headers validated in prepare_runtime", - ); - response.headers_mut().insert(header_name, header_value); - } + // Any response that sets a per-user cookie (notably the EC identity cookie) + // must never be shared-cached, and per-user responses (assembled HTML, + // page-bids, cookie-bearing navigations) must not have their uncacheable + // Cache-Control re-enabled by operator headers. This shared helper runs + // byte-identically on every adapter so the privacy guarantee can't drift. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } +/// Surrogate cache headers stripped from every cookie-bearing response. +/// +/// Re-exported from [`trusted_server_core::response_privacy`] so the legacy +/// [`crate::enforce_set_cookie_cache_privacy`] `FastlyResponse` variant and the +/// shared [`Response`] downgrade cannot drift apart. +pub(crate) use trusted_server_core::response_privacy::SURROGATE_CACHE_HEADERS; + +/// Forces cookie-bearing responses to stay private to shared caches. +/// +/// Re-exported from [`trusted_server_core::response_privacy`] so the `EdgeZero` +/// entry point (`main.rs`) can re-apply it after +/// [`ec_finalize_response`](trusted_server_core::ec::finalize::ec_finalize_response) +/// writes the EC identity `Set-Cookie`, using the single shared implementation. +pub(crate) use trusted_server_core::response_privacy::enforce_set_cookie_cache_privacy; + // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -241,7 +262,7 @@ mod tests { use edgezero_core::body::Body; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; - use edgezero_core::http::{request_builder, response_builder, Method, StatusCode}; + use edgezero_core::http::{request_builder, response_builder, HeaderName, Method, StatusCode}; use edgezero_core::middleware::Next; use edgezero_core::params::PathParams; use error_stack::Report; @@ -342,6 +363,149 @@ mod tests { ); } + fn response_with_headers(pairs: &[(&'static str, &'static str)]) -> Response { + let mut response = empty_response(); + for (key, value) in pairs { + response.headers_mut().insert( + HeaderName::from_static(key), + HeaderValue::from_static(value), + ); + } + response + } + + #[test] + fn apply_finalize_headers_downgrades_public_set_cookie_response() { + // A per-user cookie response that arrives shared-cacheable (origin-public + // plus a surrogate directive) must be downgraded so a shared cache cannot + // store and replay one visitor's Set-Cookie. + let settings = settings_with_response_headers(vec![]); + let mut response = response_with_headers(&[ + ("set-cookie", "ts-ec=abc; Path=/"), + ("cache-control", "public, max-age=600"), + ("surrogate-control", "max-age=600"), + ]); + + apply_finalize_headers(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "should downgrade a public cookie response to private" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip surrogate-control from a cookie-bearing response" + ); + } + + #[test] + fn apply_finalize_headers_blocks_operator_surrogate_on_private_response() { + // Operator response_headers must not re-enable shared caching for an + // uncacheable (private) per-user response — neither by replacing + // Cache-Control nor by reintroducing surrogate cache headers. + let settings = settings_with_response_headers(vec![ + ("cache-control", "public, max-age=3600"), + ("surrogate-control", "max-age=3600"), + ("x-operator", "kept"), + ]); + let mut response = response_with_headers(&[("cache-control", "private, max-age=0")]); + + apply_finalize_headers(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "operator cache-control must not weaken a private response" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "operator surrogate-control must not be applied to a private response" + ); + assert_eq!( + response + .headers() + .get("x-operator") + .and_then(|v| v.to_str().ok()), + Some("kept"), + "non-cache operator headers must still apply" + ); + } + + #[test] + fn enforce_set_cookie_cache_privacy_downgrades_late_cookie() { + // Mirrors the EdgeZero post-ec_finalize guard: a Set-Cookie added after + // finalize headers ran (origin-public response) must be downgraded. + let mut response = response_with_headers(&[ + ("set-cookie", "ts-ec=abc; Path=/"), + ("cache-control", "public, max-age=600"), + ("surrogate-control", "max-age=600"), + ]); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "should downgrade a late public cookie response to private" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip surrogate-control from the late cookie response" + ); + } + + #[test] + fn enforce_set_cookie_cache_privacy_keeps_stricter_no_store() { + // Idempotent: a stricter no-store directive is preserved, but surrogate + // headers still come off. + let mut response = response_with_headers(&[ + ("set-cookie", "ts-ec=abc; Path=/"), + ("cache-control", "no-store"), + ("surrogate-control", "max-age=600"), + ]); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("no-store"), + "should keep the stricter no-store directive" + ); + assert!( + response.headers().get("surrogate-control").is_none(), + "should strip surrogate-control even when keeping no-store" + ); + } + + #[test] + fn enforce_set_cookie_cache_privacy_ignores_cookieless_response() { + let mut response = response_with_headers(&[("cache-control", "public, max-age=600")]); + + enforce_set_cookie_cache_privacy(&mut response); + + assert_eq!( + response + .headers() + .get("cache-control") + .and_then(|v| v.to_str().ok()), + Some("public, max-age=600"), + "should leave a cookieless response untouched" + ); + } + // --------------------------------------------------------------------------- // FinalizeResponseMiddleware::handle tests // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 9b1a73422..935c957ea 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -159,6 +159,7 @@ fn backend_config_from_spec(spec: &PlatformBackendSpec) -> BackendConfig<'_> { .host_header_override(spec.host_header_override.as_deref()) .certificate_check(spec.certificate_check) .first_byte_timeout(spec.first_byte_timeout) + .between_bytes_timeout(spec.between_bytes_timeout) } impl PlatformBackend for FastlyPlatformBackend { @@ -676,6 +677,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let name = backend @@ -683,7 +685,7 @@ mod tests { .expect("should compute backend name for valid spec"); assert_eq!( - name, "backend_https_origin_example_com_443_t15000", + name, "backend_https_origin_example_com_443_fb15000_bb15000", "should match BackendConfig naming convention" ); } @@ -698,6 +700,7 @@ mod tests { host_header_override: Some("www.example.com".to_string()), certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let name = backend @@ -705,7 +708,7 @@ mod tests { .expect("should compute backend name for host header override"); assert_eq!( - name, "backend_https_origin_example_com_443_oh_www_example_com_t15000", + name, "backend_https_origin_example_com_443_oh_www_example_com_fb15000_bb15000", "should match BackendConfig naming convention with host header override" ); } @@ -720,6 +723,7 @@ mod tests { host_header_override: None, certificate_check: false, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let name = backend @@ -742,6 +746,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_secs(15), + between_bytes_timeout: Duration::from_secs(15), }; let result = backend.predict_name(&spec); @@ -759,6 +764,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_millis(2000), + between_bytes_timeout: Duration::from_millis(2000), }; let name = backend @@ -766,8 +772,8 @@ mod tests { .expect("should compute name with custom timeout"); assert!( - name.ends_with("_t2000"), - "should encode 2000ms timeout in name" + name.ends_with("_fb2000_bb2000"), + "should encode 2000ms first-byte and between-bytes timeouts in name" ); } diff --git a/crates/trusted-server-adapter-fastly/src/route_tests.rs b/crates/trusted-server-adapter-fastly/src/route_tests.rs index ce542a7c0..5c336dda8 100644 --- a/crates/trusted-server-adapter-fastly/src/route_tests.rs +++ b/crates/trusted-server-adapter-fastly/src/route_tests.rs @@ -35,6 +35,16 @@ use trusted_server_core::settings::{ use super::{route_request, HandlerOutcome}; +#[test] +fn streaming_publisher_path_uses_async_auction_collector() { + let router_source = include_str!("main.rs"); + + assert!( + router_source.contains("stream_publisher_body_async("), + "streaming publisher responses must collect dispatched auctions before injection" + ); +} + struct StubJwksConfigStore; impl PlatformConfigStore for StubJwksConfigStore { @@ -376,6 +386,23 @@ pub(crate) fn us_california_geo() -> GeoInfo { } } +/// Geo resolving to a non-regulated jurisdiction, so the server-side auction +/// consent gate (which fails closed for GDPR/unknown jurisdictions without TCF +/// Purpose 1) allows the auction to proceed. Used by `/auction` route tests +/// that exercise orchestration behavior rather than consent. +fn non_regulated_geo() -> GeoInfo { + GeoInfo { + city: "Example City".to_string(), + country: "AU".to_string(), + continent: "OC".to_string(), + latitude: -33.8, + longitude: 151.2, + metro_code: 0, + region: Some("NSW".to_string()), + asn: None, + } +} + fn valid_ec_id() -> String { format!("{}.Abc123", "a".repeat(64)) } @@ -615,8 +642,12 @@ fn route_result_to_fastly_response( &mut response, ); } + // Apply request-filter response effects (which may append a per-user + // Set-Cookie) before the final cache guard so the guard observes them. request_filter_effects.apply_to_response(&mut response); - compat::to_fastly_response(response) + let mut fastly_response = compat::to_fastly_response(response); + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + fastly_response } fn browser_device_signals() -> DeviceSignals { @@ -644,7 +675,16 @@ fn route_auction_with_stack( let req = Request::post("https://test.com/auction") .with_header(header::CONTENT_TYPE, "application/json") .with_body(body.into()); - let services = test_runtime_services(&req); + // Resolve to a non-regulated jurisdiction so the server-side auction consent + // gate allows the auction; these tests assert orchestration behavior, not + // consent gating (covered separately in endpoints.rs). + let services = test_runtime_services_with_secret_http_client_and_geo( + &req, + Arc::new(NoopBackend), + Arc::new(NoopSecretStore), + Arc::new(NoopHttpClient) as Arc, + Arc::new(FixedGeo(non_regulated_geo())), + ); let route_result = futures::executor::block_on(route_request( settings, @@ -652,6 +692,7 @@ fn route_auction_with_stack( integration_registry, &partner_registry, &services, + &[], compat::from_fastly_request(req), browser_device_signals(), )) @@ -676,6 +717,7 @@ fn route_buffered_response( integration_registry, &partner_registry, services, + &[], compat::from_fastly_request(req), browser_device_signals(), )) @@ -884,11 +926,15 @@ fn datadome_allow_applies_downstream_headers_and_protects_auction() { ("x-dd-b", "allowed"), ]), ); - let services = test_runtime_services_with_secret_and_http_client( + // Resolve to a non-regulated jurisdiction so the server-side auction consent + // gate allows the auction to run; this test asserts DataDome protection plus + // auction orchestration, not consent gating (covered in endpoints.rs). + let services = test_runtime_services_with_secret_http_client_and_geo( &req, Arc::new(FixedBackend), datadome_secret_store(), Arc::clone(&http_client) as Arc, + Arc::new(FixedGeo(non_regulated_geo())), ); let response = route_buffered_response( @@ -1102,6 +1148,7 @@ fn routes_use_request_local_consent() { &integration_registry, &partner_registry, &discovery_services, + &[], compat::from_fastly_request(discovery_fastly_req), DeviceSignals::derive("", None, None), )) @@ -1120,6 +1167,7 @@ fn routes_use_request_local_consent() { &integration_registry, &partner_registry, &admin_services, + &[], compat::from_fastly_request(admin_fastly_req), DeviceSignals::derive("", None, None), )) @@ -1166,6 +1214,7 @@ fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { &integration_registry, &partner_registry, &services, + &[], compat::from_fastly_request(alias_req), DeviceSignals::derive("", None, None), )) @@ -1499,6 +1548,7 @@ fn asset_routes_stream_asset_responses_directly() { &integration_registry, &partner_registry, &services, + &[], req, browser_device_signals(), )) @@ -1633,6 +1683,546 @@ fn asset_handler_error_stays_uncacheable_after_global_headers() { ); } +#[test] +fn finalize_response_preserves_origin_cache_headers_for_plain_html() { + // Reviewer P1.2: a zero-slot / non-matching navigation injects no per-user + // data and sets no cookie, so the publisher path leaves Cache-Control alone. + // finalize_response must not downgrade shared cacheability for it. + let settings = create_test_settings(); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "public, max-age=3600") + .header("surrogate-control", "max-age=86400") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("public, max-age=3600"), + "plain cookieless HTML should keep its shared cache directive" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|v| v.to_str().ok()), + Some("max-age=86400"), + "plain cookieless HTML should keep its surrogate cache directive" + ); +} + +#[test] +fn finalize_response_makes_cookie_bearing_responses_private() { + // A first-visit navigation that only sets the EC identity cookie must not be + // shared-cached, even though no ad data was injected. + let settings = create_test_settings(); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "public, max-age=3600") + .header("surrogate-control", "max-age=86400") + .header(header::SET_COOKIE, "ec=abc; Path=/; HttpOnly") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "a Set-Cookie response must be downgraded to private" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "a Set-Cookie response must not retain surrogate cacheability" + ); +} + +#[test] +fn ec_set_cookie_added_after_finalize_downgrades_origin_public_cache() { + // First-visit navigation: the origin response is shared-cacheable and carries + // no cookie, so the HttpResponse-stage finalizer keeps its cache headers. EC + // finalization then mints the identity Set-Cookie on the converted Fastly + // response, after that guard has already run. The post-EC privacy guard must + // downgrade caching so a shared cache cannot replay one visitor's EC cookie. + let settings = create_test_settings(); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "public, max-age=3600") + .header("surrogate-control", "max-age=86400") + .body(EdgeBody::empty()) + .expect("should build test response"); + + // No cookie at this stage, so the cookie net does not fire and the origin + // cache directive survives finalize_response — reproducing the gap. + super::finalize_response(&settings, None, &mut response); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("public, max-age=3600"), + "a cookieless response should keep its origin cache directive" + ); + + let mut fastly_response = compat::to_fastly_response(response); + // Stand in for ec_finalize_response minting the first-visit identity cookie: + // its EcContext constructors are #[cfg(test)] in trusted-server-core and are + // not reachable from this crate, but the only behavior under test here is the + // post-EC ordering — a Set-Cookie appearing after finalize_response ran. + fastly_response.set_header(header::SET_COOKIE, "ec=abc; Path=/; HttpOnly"); + + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("private, max-age=0"), + "an EC Set-Cookie added after finalize_response must downgrade caching" + ); + assert!( + fastly_response.get_header("surrogate-control").is_none(), + "EC Set-Cookie responses must not retain surrogate cacheability" + ); +} + +#[test] +fn enforce_set_cookie_cache_privacy_keeps_stricter_no_store() { + // A stricter directive minted alongside the cookie must not be weakened to + // the `private, max-age=0` downgrade. + let mut fastly_response = compat::to_fastly_response( + edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"), + ); + + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("no-store"), + "an already-uncacheable response should keep its stricter directive" + ); +} + +#[test] +fn enforce_set_cookie_cache_privacy_strips_surrogate_on_no_store() { + // A `no-store` cookie response keeps its stricter Cache-Control but must still + // lose the surrogate cache headers — they are independent of Cache-Control and + // would otherwise let a shared cache store and replay the visitor's cookie. + let mut fastly_response = compat::to_fastly_response( + edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .header("surrogate-control", "max-age=86400") + .header("fastly-surrogate-control", "max-age=86400") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"), + ); + + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("no-store"), + "should keep the stricter no-store directive" + ); + assert!( + fastly_response.get_header("surrogate-control").is_none(), + "no-store cookie responses must not retain Surrogate-Control" + ); + assert!( + fastly_response + .get_header("fastly-surrogate-control") + .is_none(), + "no-store cookie responses must not retain Fastly-Surrogate-Control" + ); +} + +#[test] +fn request_filter_set_cookie_after_guard_still_downgrades_cache() { + // A request filter (e.g. a DataDome allow) can append a per-user Set-Cookie via + // response effects. main applies those effects before the final cache guard, so + // an origin response still marked `public` with surrogate headers must be + // downgraded once the filter cookie is present. + use trusted_server_core::integrations::{HeaderMutation, RequestFilterEffects}; + + let mut edge_response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "public, max-age=3600") + .header("surrogate-control", "max-age=86400") + .body(EdgeBody::empty()) + .expect("should build test response"); + + let effects = RequestFilterEffects { + request_headers: vec![], + response_headers: vec![HeaderMutation::append( + "set-cookie", + "datadome=allow; Path=/; HttpOnly", + )], + }; + + // Mirror main's ordering: apply effects first, then the guard. + effects.apply_to_response(&mut edge_response); + let mut fastly_response = compat::to_fastly_response(edge_response); + super::enforce_set_cookie_cache_privacy(&mut fastly_response); + + assert_eq!( + fastly_response.get_header_str("cache-control"), + Some("private, max-age=0"), + "a filter-added Set-Cookie must downgrade a public origin response" + ); + assert!( + fastly_response.get_header("surrogate-control").is_none(), + "a filter-added Set-Cookie must strip surrogate cacheability" + ); +} + +#[test] +fn finalize_response_no_store_cookie_blocks_operator_surrogate_reenable() { + // Operator response_headers must not re-add surrogate caching to a Set-Cookie + // response carrying the stricter `no-store` directive — the operator guard must + // treat no-store as protected, not just `private`. + let mut settings = create_test_settings(); + settings + .response_headers + .insert("Surrogate-Control".to_string(), "max-age=86400".to_string()); + settings.response_headers.insert( + "Fastly-Surrogate-Control".to_string(), + "max-age=86400".to_string(), + ); + settings.response_headers.insert( + header::CACHE_CONTROL.as_str().to_string(), + "public, max-age=3600".to_string(), + ); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "no-store") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store"), + "operator Cache-Control must not weaken the stricter no-store directive" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "operator Surrogate-Control must not re-enable caching for a no-store cookie response" + ); + assert_eq!( + response + .headers() + .get("fastly-surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "operator Fastly-Surrogate-Control must not re-enable caching for a no-store cookie response" + ); +} + +#[test] +fn finalize_response_leaves_stricter_no_store_untouched() { + let settings = create_test_settings(); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "no-store") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("no-store"), + "an already-uncacheable response should keep its stricter directive" + ); +} + +#[test] +fn finalize_response_treats_mixed_case_no_store_as_uncacheable() { + // Cache-Control directives are case-insensitive: `No-Store` on a Set-Cookie + // response must be recognized as already-uncacheable and left untouched, not + // downgraded to the weaker `private, max-age=0`. + let settings = create_test_settings(); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CACHE_CONTROL, "No-Store") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("No-Store"), + "mixed-case No-Store must be treated as uncacheable and preserved" + ); +} + +#[test] +fn finalize_response_mixed_case_private_blocks_operator_surrogate_reenable() { + // A mixed-case `Private` directive must still mark the response private so + // operator response_headers cannot re-enable shared caching. + let mut settings = create_test_settings(); + settings + .response_headers + .insert("Surrogate-Control".to_string(), "max-age=86400".to_string()); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::CACHE_CONTROL, "Private, max-age=0") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "operator Surrogate-Control must not re-enable caching for a mixed-case Private response" + ); +} + +#[test] +fn finalize_response_cookie_net_blocks_operator_surrogate_reenable() { + // Operator response_headers must not re-add surrogate caching once the + // cookie net has marked the response private. + let mut settings = create_test_settings(); + settings + .response_headers + .insert("Surrogate-Control".to_string(), "max-age=86400".to_string()); + settings.response_headers.insert( + header::CACHE_CONTROL.as_str().to_string(), + "public, max-age=3600".to_string(), + ); + let mut response = edge_response_builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .header(header::SET_COOKIE, "ec=abc; Path=/") + .body(EdgeBody::empty()) + .expect("should build test response"); + + super::finalize_response(&settings, None, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "operator Cache-Control must not override the cookie privacy directive" + ); + assert_eq!( + response + .headers() + .get("surrogate-control") + .and_then(|v| v.to_str().ok()), + None, + "operator Surrogate-Control must not re-enable shared caching for a cookie response" + ); +} + +#[test] +fn page_bids_response_cannot_regain_surrogate_headers_from_settings() { + let base = base_route_settings_toml(); + let prebid = prebid_integration_toml(); + let config = format!( + r#"{base} + +{prebid} + + [auction] + enabled = true + providers = ["prebid"] + timeout_ms = 2000 + + [creative_opportunities] + gam_network_id = "1234" + "#, + ); + let mut settings = + Settings::from_toml(&config).expect("should parse page-bids route test settings"); + settings + .response_headers + .insert("Surrogate-Control".to_string(), "max-age=86400".to_string()); + settings.response_headers.insert( + "Fastly-Surrogate-Control".to_string(), + "max-age=86400".to_string(), + ); + settings.response_headers.insert( + header::CACHE_CONTROL.as_str().to_string(), + "public, max-age=3600".to_string(), + ); + let (orchestrator, integration_registry) = build_route_stack(&settings); + + let mut req = Request::get("https://test-publisher.com/__ts/page-bids?path=/2024/article/"); + req.set_header("sec-fetch-site", "same-origin"); + let services = test_runtime_services(&req); + + let resp = route_buffered_response( + &settings, + &orchestrator, + &integration_registry, + &services, + req, + "should route page-bids request", + ); + + assert_eq!( + resp.get_status(), + StatusCode::OK, + "should serve the page-bids response" + ); + assert_eq!( + resp.get_header_str(header::CACHE_CONTROL), + Some("private, no-store"), + "should keep the per-user cache directive despite operator Cache-Control" + ); + assert_eq!( + resp.get_header_str("surrogate-control"), + None, + "should not let operator headers re-enable shared surrogate caching" + ); + assert_eq!( + resp.get_header_str("fastly-surrogate-control"), + None, + "should not let operator headers re-enable Fastly surrogate caching" + ); +} + +#[test] +fn page_bids_cross_site_request_is_rejected_at_the_route() { + let base = base_route_settings_toml(); + let prebid = prebid_integration_toml(); + let config = format!( + r#"{base} + +{prebid} + + [auction] + enabled = true + providers = ["prebid"] + timeout_ms = 2000 + + [creative_opportunities] + gam_network_id = "1234" + "#, + ); + let settings = + Settings::from_toml(&config).expect("should parse page-bids route test settings"); + let (orchestrator, integration_registry) = build_route_stack(&settings); + + let mut req = Request::get("https://test-publisher.com/__ts/page-bids?path=/2024/article/"); + req.set_header("sec-fetch-site", "cross-site"); + let services = test_runtime_services(&req); + + let resp = route_buffered_response( + &settings, + &orchestrator, + &integration_registry, + &services, + req, + "should route cross-site page-bids request", + ); + + assert_eq!( + resp.get_status(), + StatusCode::FORBIDDEN, + "should reject cross-site page-bids requests" + ); +} + +#[test] +fn page_bids_options_preflight_is_rejected_at_the_route() { + // OPTIONS must not fall through to the publisher origin (which may return + // permissive CORS); the GET handler's legacy `X-TSJS-Page-Bids` fallback + // relies on this endpoint never granting a preflight. + let base = base_route_settings_toml(); + let prebid = prebid_integration_toml(); + let config = format!( + r#"{base} + +{prebid} + + [auction] + enabled = true + providers = ["prebid"] + timeout_ms = 2000 + + [creative_opportunities] + gam_network_id = "1234" + "#, + ); + let settings = + Settings::from_toml(&config).expect("should parse page-bids route test settings"); + let (orchestrator, integration_registry) = build_route_stack(&settings); + + let req = Request::new( + Method::OPTIONS, + "https://test-publisher.com/__ts/page-bids?path=/2024/article/", + ); + let services = test_runtime_services(&req); + + let resp = route_buffered_response( + &settings, + &orchestrator, + &integration_registry, + &services, + req, + "should route page-bids preflight request", + ); + + assert_eq!( + resp.get_status(), + StatusCode::FORBIDDEN, + "should reject the page-bids CORS preflight at the adapter" + ); + assert_eq!( + resp.get_header_str(header::CACHE_CONTROL), + Some("private, no-store"), + "preflight rejection must not be shared-cached" + ); +} + #[test] fn s3_asset_origin_error_stays_uncacheable_after_global_headers() { let mut settings = create_test_settings(); diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 0dddd7a1d..f6de6779b 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -14,12 +14,14 @@ use trusted_server_core::ec::EcContext; use trusted_server_core::error::{IntoHttpResponse as _, TrustedServerError}; use trusted_server_core::http_util::sanitize_forwarded_headers; use trusted_server_core::integrations::{IntegrationRegistry, ProxyDispatchInput}; +use trusted_server_core::platform::RuntimeServices; use trusted_server_core::proxy::{ handle_first_party_click, handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - PublisherResponse, buffer_publisher_response, handle_publisher_request, handle_tsjs_dynamic, + AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_page_bids, + handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -77,16 +79,27 @@ fn build_state_with_settings( /// Collapse a [`PublisherResponse`] into a plain [`Response`]. /// -/// Delegates to the shared [`buffer_publisher_response`], which enforces -/// `settings.publisher.max_buffered_body_bytes` so a large processable -/// origin response fails safely instead of exhausting the Wasm heap. -fn resolve_publisher_response( +/// Delegates to the shared [`buffer_publisher_response_async`], which collects +/// the dispatched server-side auction and enforces +/// `settings.publisher.max_buffered_body_bytes` so a large processable origin +/// response fails safely instead of exhausting the Wasm heap. +async fn resolve_publisher_response( publisher_response: PublisherResponse, method: &Method, settings: &Settings, registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, ) -> Result> { - buffer_publisher_response(publisher_response, method, settings, registry) + buffer_publisher_response_async( + publisher_response, + method, + settings, + registry, + orchestrator, + services, + ) + .await } // --------------------------------------------------------------------------- @@ -129,7 +142,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 11] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 12] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), @@ -138,6 +151,7 @@ fn named_fallback_paths() -> [(&'static str, &'static [Method]); 11] { ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), ("/auction", &[Method::POST]), + ("/__ts/page-bids", &[Method::GET, Method::OPTIONS]), ("/first-party/proxy", &[Method::GET]), ("/first-party/click", &[Method::GET]), ("/first-party/sign", &[Method::GET, Method::POST]), @@ -308,6 +322,30 @@ fn health_response() -> Response { resp } +/// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, +/// `/__ts/page-bids`, and the publisher fallback). +/// +/// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction +/// Unknown, which fails the auction consent gate closed even for consented +/// users. Spin's platform geo is a no-op, so jurisdiction stays Unknown unless +/// the request carries TCF consent. A malformed consent string is logged and +/// falls back to the default (fail-closed) context rather than being silently +/// swallowed. +fn build_ec_context(settings: &Settings, services: &RuntimeServices, req: &Request) -> EcContext { + let geo_info = services + .geo() + .lookup(services.client_info().client_ip) + .unwrap_or_else(|e| { + log::warn!("geo lookup failed: {e}"); + None + }); + EcContext::read_from_request_with_geo(settings, req, services, geo_info.as_ref()) + .unwrap_or_else(|e| { + log::warn!("EC context read failed: {e:?}"); + EcContext::default() + }) +} + // --------------------------------------------------------------------------- // Error helper // --------------------------------------------------------------------------- @@ -492,7 +530,10 @@ fn build_router(state: &Arc) -> RouterService { // OpenRTB metadata that auction signing derives from // `RequestInfo::from_request` uses the trusted runtime authority. let req = ctx.into_request(); - let ec_context = EcContext::default(); + // Build the geo-aware EC context so the auction consent gate sees + // the caller's jurisdiction — `EcContext::default()` fails it + // closed for consented users. + let ec_context = build_ec_context(&s.settings, &services, &req); Ok(handle_auction( &s.settings, &s.orchestrator, @@ -507,6 +548,33 @@ fn build_router(state: &Arc) -> RouterService { } }; + // GET /__ts/page-bids — SPA re-auction endpoint. + let s = Arc::clone(&state); + let page_bids_handler = move |ctx: RequestContext| { + let s = Arc::clone(&s); + async move { + let services = build_runtime_services(&ctx); + let req = ctx.into_request(); + let ec_context = build_ec_context(&s.settings, &services, &req); + let auction = AuctionDispatch { + orchestrator: &s.orchestrator, + slots: s.settings.creative_opportunity_slots(), + registry: None, + }; + Ok( + handle_page_bids(&s.settings, &services, None, auction, &ec_context, req) + .await + .unwrap_or_else(|e| http_error(&e)), + ) + } + }; + + // OPTIONS /__ts/page-bids — deny the CORS preflight for this + // side-effecting GET so the `X-TSJS-Page-Bids` gate stays trustworthy. + let page_bids_options_handler = |_ctx: RequestContext| async { + Ok::(page_bids_preflight_denied()) + }; + // GET /first-party/proxy let s = Arc::clone(&state); let fp_proxy_handler = move |ctx: RequestContext| { @@ -597,11 +665,35 @@ fn build_router(state: &Arc) -> RouterService { })) }) } else { - handle_publisher_request(&state.settings, &state.registry, &services, req) - .await - .and_then(|pr| { - resolve_publisher_response(pr, &method, &state.settings, &state.registry) - }) + let mut ec_context = build_ec_context(&state.settings, &services, &req); + let auction = AuctionDispatch { + orchestrator: &state.orchestrator, + slots: state.settings.creative_opportunity_slots(), + registry: None, + }; + match handle_publisher_request( + &state.settings, + &services, + None, + &mut ec_context, + auction, + req, + ) + .await + { + Ok(pr) => { + resolve_publisher_response( + pr, + &method, + &state.settings, + &state.registry, + &state.orchestrator, + &services, + ) + .await + } + Err(e) => Err(e), + } }; Ok(result.unwrap_or_else(|e| http_error(&e))) @@ -646,6 +738,12 @@ fn build_router(state: &Arc) -> RouterService { .post("/_ts/admin/keys/rotate", rotate_handler) .post("/_ts/admin/keys/deactivate", deactivate_handler) .post("/auction", auction_handler) + .get("/__ts/page-bids", page_bids_handler) + .route( + "/__ts/page-bids", + Method::OPTIONS, + page_bids_options_handler, + ) .get("/first-party/proxy", fp_proxy_handler) .get("/first-party/click", fp_click_handler) .get("/first-party/sign", fp_sign_handler) diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs index 62f83e1ea..1bcede1fc 100644 --- a/crates/trusted-server-adapter-spin/src/middleware.rs +++ b/crates/trusted-server-adapter-spin/src/middleware.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; -use edgezero_core::http::{HeaderName, HeaderValue, Response}; +use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; @@ -124,8 +124,8 @@ impl Middleware for NormalizeMiddleware { /// /// `geo_available` controls `X-Geo-Info-Available`. Spin passes `false` /// because it has no geo headers. Operator-configured -/// `settings.response_headers` are applied last and can override any managed -/// header. +/// `settings.response_headers` are applied last (with the shared cookie +/// cache-privacy hardening) and can override any managed header. pub(crate) fn apply_finalize_headers( settings: &Settings, geo_available: bool, @@ -136,18 +136,11 @@ pub(crate) fn apply_finalize_headers( HeaderValue::from_static(if geo_available { "true" } else { "false" }), ); - for (key, value) in &settings.response_headers { - let header_name = HeaderName::from_bytes(key.as_bytes()); - let header_value = HeaderValue::from_str(value); - if let (Ok(header_name), Ok(header_value)) = (header_name, header_value) { - response.headers_mut().insert(header_name, header_value); - } else { - log::warn!( - "Skipping invalid configured response header value for {}", - key - ); - } - } + // Cookie-bearing responses stay private to shared caches and operator + // headers cannot re-enable caching for uncacheable per-user payloads. + trusted_server_core::response_privacy::apply_response_headers_with_cache_privacy( + settings, response, + ); } // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index d593390bb..ba88f361f 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -25,6 +25,7 @@ edgezero-core = { workspace = true } error-stack = { workspace = true } flate2 = { workspace = true } futures = { workspace = true } +glob = { workspace = true } hex = { workspace = true } hmac = { workspace = true } http = { workspace = true } @@ -67,6 +68,7 @@ config = { workspace = true } criterion = { workspace = true } edgezero-core = { workspace = true, features = ["test-utils"] } temp-env = { workspace = true } +tokio = { workspace = true } [[bench]] name = "consent_decode" diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 016d5c51e..9e81d67d9 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -9,6 +9,8 @@ fn make_config() -> HtmlProcessorConfig { request_host: "proxy.bench.example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, } } diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 000f1c482..c642c9ec3 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -1,12 +1,15 @@ //! HTTP endpoint handlers for auction requests. +use std::collections::HashMap; + use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{header, Request, Response, StatusCode}; use serde_json::Value as JsonValue; use crate::auction::formats::AdRequest; -use crate::consent::gate_eids_by_consent; +use crate::auction::orchestrator::OrchestrationResult; +use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; use crate::constants::COOKIE_TS_EIDS; use crate::ec::eids::{resolve_partner_ids, to_eids}; use crate::ec::kv::KvIdentityGraph; @@ -34,11 +37,62 @@ const MAX_CLIENT_EID_SOURCE_BYTES: usize = 255; /// arbitrary WASM linear memory. const MAX_AUCTION_BODY_SIZE: usize = 256 * 1024; -/// Handle auction request from /auction endpoint. +/// Handle auction request from `POST /auction`. +/// +/// Accepts a JSON body matching [`AdRequest`][`super::formats::AdRequest`]. +/// The minimum valid request is: +/// +/// ```json +/// { +/// "adUnits": [{ +/// "code": "atf_sidebar_ad", +/// "mediaTypes": { "banner": { "sizes": [[300, 250]] } } +/// }] +/// } +/// ``` +/// +/// ## Bidder params: inline vs. stored-request +/// +/// Each ad unit's `bids` array is **optional**. When absent or empty the PBS +/// integration falls back to a stored-request keyed by the unit's `code` +/// field (`imp.ext.prebid.storedrequest = { id: "" }`). A PBS stored +/// request must therefore exist for every slot code that omits inline params. +/// +/// When `bids` is supplied, each entry's `bidder`/`params` pair is forwarded +/// directly as `imp.ext.prebid.bidder.`. +/// +/// ## Context passthrough (`config`) +/// +/// The optional `config` object is filtered through +/// [`auction.allowed_context_keys`][`crate::settings::AuctionConfig::allowed_context_keys`]. +/// Only keys listed there reach the auction providers (e.g. `"permutive_segments"`). +/// All other keys are silently dropped. Values must be either strings or arrays of +/// strings. +/// +/// ## Response +/// +/// Returns an `OpenRTB 2.x` response. Creative HTML is inlined in each bid's +/// `adm` field after sanitisation and first-party URL rewriting. Response +/// headers include `X-TS-EC` (the caller's Edge Cookie ID) and +/// `X-TS-EC-Fresh` (a freshly generated ID for cookie renewal). +/// +/// ## Scroll, refresh, and SPA navigation /// -/// This is the main entry point for running header bidding auctions. -/// It orchestrates bids from multiple providers (Prebid, APS, GAM, etc.) and returns -/// the winning bids in `OpenRTB` format with creative HTML inline in the `adm` field. +/// This endpoint is intended for **initial page render** and **programmatic +/// callers** (e.g. slim-Prebid, native apps, server-to-server integrations). +/// It is **not** the intended path for scroll or GPT refresh events. +/// +/// **SPA navigation** is handled by `GET /__ts/page-bids`: the client-side SPA +/// hook (`installSpaAuctionHook`) intercepts `pushState`/`replaceState`/`popstate` +/// events and calls that endpoint to fetch fresh slots and bids for each new +/// route, then invokes `window.tsjs.adInit()` with the updated data. +/// +/// **Scroll and GPT refresh** are owned by slim-Prebid in Phase 1: it runs +/// post-`window.load`, listens for GPT refresh events, and runs client-side +/// auctions independently of this endpoint. +/// +/// A slot-template-aware refresh API (`POST /auction/refresh`) is deferred to a +/// future phase and not designed here. /// /// # Errors /// @@ -112,14 +166,64 @@ pub async fn handle_auction( }; let consent_context = ec_context.consent().clone(); + // Server-side auction consent gate. The publisher-navigation and + // `/__ts/page-bids` paths fail closed for GDPR/unknown jurisdictions that + // lack effective TCF Purpose 1. `/auction` is the programmatic entry point + // for the same server-side auction, so it must gate identically: returning + // a no-bid response here prevents outbound PBS/APS calls and the forwarding + // of request-derived signals (UA/IP/geo, and cookies under some Prebid + // consent-forwarding modes) for traffic that must not run an auction. + if !consent_allows_server_side_auction(&consent_context) { + log::info!( + "/auction: server-side auction consent gate denied; returning no-bid response without contacting providers" + ); + // Build the request shape locally (no outbound calls, no geo lookup, no + // EID resolution) so the no-bid OpenRTB response echoes the request id. + let auction_request = convert_tsjs_to_auction_request( + &body, + settings, + services, + &http_req, + consent_context, + ec_id, + None, + )?; + let empty_result = OrchestrationResult { + provider_responses: Vec::new(), + mediator_response: None, + winning_bids: HashMap::new(), + total_time_ms: 0, + metadata: HashMap::new(), + }; + return convert_to_openrtb_response( + &empty_result, + settings, + &auction_request, + ec_context.ec_allowed(), + ); + } + // Parse client-provided EIDs from the current request body. When the // current request does not include them, fall back to the persisted // `ts-eids` cookie so later requests can still forward the browser's // full OpenRTB-style EID structure. - let client_eids = resolve_client_auction_eids( - body.eids.as_ref(), - extract_cookie_value(&http_req, COOKIE_TS_EIDS).as_deref(), - ); + // + // Gate this on the same identity-consent condition as the EC ID + // (`ec_id.is_some()`, which is already filtered by `ec_context.ec_allowed()`). + // Otherwise a US/GPC or US-Privacy opt-out context — where EC identity use is + // denied but a non-personalized auction may still run — could forward + // persistent client EIDs from the body/cookie, since `gate_eids_by_consent` + // only strips on TCF/GDPR signals. This matches the publisher and + // `/__ts/page-bids` paths, which also resolve client EIDs only when + // `ec_id.is_some()`. + let client_eids = if ec_id.is_some() { + resolve_client_auction_eids( + body.eids.as_ref(), + extract_cookie_value(&http_req, COOKIE_TS_EIDS).as_deref(), + ) + } else { + None + }; // Resolve partner EIDs from the KV identity graph when the user has // a valid EC and both KV and partner stores are available. @@ -188,7 +292,7 @@ pub async fn handle_auction( /// Returns `None` when any prerequisite is missing (no KV store, no partner /// store, no EC, consent denied). On KV or partner-resolution errors, logs a /// warning and returns empty EIDs so the auction can proceed in degraded mode. -fn resolve_auction_eids( +pub(crate) fn resolve_auction_eids( kv: Option<&KvIdentityGraph>, registry: Option<&PartnerRegistry>, ec_context: &EcContext, @@ -234,7 +338,7 @@ fn extract_cookie_value(req: &Request, name: &str) -> Option { None } -fn resolve_client_auction_eids( +pub(crate) fn resolve_client_auction_eids( raw: Option<&JsonValue>, cookie_value: Option<&str>, ) -> Option> { @@ -330,7 +434,7 @@ fn parse_client_auction_uid(raw: &JsonValue) -> Option { Some(Uid { id, atype, ext }) } -fn merge_auction_eids( +pub(crate) fn merge_auction_eids( client_eids: Option>, resolved_eids: Option>, ) -> Option> { @@ -393,12 +497,19 @@ fn merge_auction_eids( #[cfg(test)] mod tests { use super::*; + use crate::auction::config::AuctionConfig; + use crate::auction::provider::AuctionProvider; + use crate::auction::types::{AuctionRequest, AuctionResponse}; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::ConsentContext; use crate::openrtb::Uid; + use crate::platform::test_support::noop_services; + use crate::platform::{PlatformPendingRequest, PlatformResponse}; + use crate::test_support::tests::create_test_settings; use base64::engine::general_purpose::STANDARD as BASE64; use base64::Engine as _; use serde_json::json; + use std::sync::Arc; fn make_ec_context(jurisdiction: Jurisdiction, ec_value: Option<&str>) -> EcContext { EcContext::new_for_test( @@ -410,6 +521,234 @@ mod tests { ) } + /// Provider that fails the test if it is ever contacted. Used to prove the + /// `/auction` consent gate short-circuits before any outbound bid request. + struct PanicOnBidProvider; + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for PanicOnBidProvider { + fn provider_name(&self) -> &'static str { + "panic_provider" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + panic!("provider must not be contacted when the consent gate denies the auction"); + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("provider must not parse a response when the auction is gated off"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("panic-backend".to_string()) + } + } + + #[tokio::test] + async fn auction_endpoint_consent_gate_returns_no_bid_without_contacting_providers() { + // GDPR/unknown jurisdiction lacking effective TCF Purpose 1 must not run + // a server-side auction. The /auction endpoint must short-circuit to a + // no-bid response before dispatching to any provider — matching the + // publisher-navigation and /__ts/page-bids paths. + let settings = create_test_settings(); + let config = AuctionConfig { + enabled: true, + providers: vec!["panic_provider".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(PanicOnBidProvider)); + let services = noop_services(); + let ec_id = format!("{}.ABC123", "a".repeat(64)); + let ec_context = make_ec_context(Jurisdiction::Unknown, Some(&ec_id)); + + let body = json!({ + "adUnits": [ + { + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + let response = handle_auction( + &settings, + &orchestrator, + None, + None, + &ec_context, + &services, + req, + ) + .await + .expect("gated auction should still return a valid response"); + + assert_eq!( + response.status(), + StatusCode::OK, + "gated auction should return a 200 no-bid response" + ); + let body_bytes = response.into_body().into_bytes().unwrap_or_default(); + let parsed: JsonValue = + serde_json::from_slice(&body_bytes).expect("response body should be valid JSON"); + let seatbid_empty = match parsed.get("seatbid").and_then(JsonValue::as_array) { + Some(seatbid) => seatbid.is_empty(), + None => true, + }; + assert!( + seatbid_empty, + "gated auction must return no bids, got: {parsed}" + ); + } + + /// Provider that records whether the auction request it received carried + /// EIDs, then fails its launch so no real transport handle is needed. + struct EidCapturingProvider { + had_eids: Arc>>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for EidCapturingProvider { + fn provider_name(&self) -> &'static str { + "eid_capturing_provider" + } + + async fn request_bids( + &self, + request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + *self.had_eids.lock().expect("should lock captured eids") = + Some(request.user.eids.is_some()); + Err(Report::new(TrustedServerError::Auction { + message: "capture only".to_string(), + })) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("parse_response must not run when the launch fails"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("capture-backend".to_string()) + } + } + + #[tokio::test] + async fn auction_strips_client_eids_when_ec_identity_denied() { + // US-state opt-out via GPC: the server-side auction consent gate still + // allows a non-personalized auction, but EC identity use is denied + // (`ec_allowed()` is false) and `gate_eids_by_consent` does not strip + // because no TCF signal is present and GDPR does not apply. Client EIDs + // supplied in the request body/cookie must NOT be forwarded — the + // outgoing auction request must have `user.eids == None`. + let settings = create_test_settings(); + let config = AuctionConfig { + enabled: true, + providers: vec!["eid_capturing_provider".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + let had_eids = Arc::new(std::sync::Mutex::new(None)); + orchestrator.register_provider(Arc::new(EidCapturingProvider { + had_eids: Arc::clone(&had_eids), + })); + let services = noop_services(); + + // US-state jurisdiction with an explicit GPC opt-out: auction allowed, + // EC identity denied. + let ec_context = EcContext::new_for_test( + None, + ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gpc: true, + ..ConsentContext::default() + }, + ); + + // Persistent EIDs supplied in both the request body and the ts-eids cookie. + let cookie_payload = json!([ + { + "source": "sharedid.org", + "uids": [{ "id": "cookie_uid", "atype": 3 }] + } + ]); + let encoded_cookie = BASE64 + .encode(serde_json::to_vec(&cookie_payload).expect("should serialize cookie payload")); + let body = json!({ + "adUnits": [ + { + "code": "div-gpt-ad-1", + "mediaTypes": { "banner": { "sizes": [[300, 250]] } } + } + ], + "eids": [ + { + "source": "id5-sync.com", + "uids": [{ "id": "body_uid", "atype": 1 }] + } + ] + }); + let req = Request::builder() + .method("POST") + .uri("https://test-publisher.com/auction") + .header("cookie", format!("{COOKIE_TS_EIDS}={encoded_cookie}")) + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize body"), + )) + .expect("should build auction request"); + + // The capturing provider fails its launch, so the auction errors overall; + // the assertion is on the EIDs observed by the provider, not the result. + let _ = handle_auction( + &settings, + &orchestrator, + None, + None, + &ec_context, + &services, + req, + ) + .await; + + assert_eq!( + *had_eids.lock().expect("should lock captured eids"), + Some(false), + "outgoing auction request must carry no EIDs when EC identity is denied" + ); + } + #[test] fn resolve_auction_eids_returns_none_without_kv() { let registry = PartnerRegistry::empty(); diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index f6e6b43f4..691021d50 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -29,7 +29,11 @@ use super::types::{ PublisherInfo, SiteInfo, UserInfo, }; -/// Request body format for auction endpoints (tsjs/Prebid.js format). +/// Request body for `POST /auction` (tsjs / Prebid.js wire format). +/// +/// `adUnits` lists the placements to bid on. `config` carries optional +/// context values (e.g. audience segments) filtered through +/// [`auction.allowed_context_keys`][`crate::settings::AuctionConfig::allowed_context_keys`]. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AdRequest { @@ -38,6 +42,15 @@ pub struct AdRequest { pub eids: Option, } +/// A single ad placement in an [`AdRequest`]. +/// +/// `code` identifies the slot (e.g. `"atf_sidebar_ad"`) and becomes the +/// impression ID in the outgoing `OpenRTB` request. +/// +/// `bids` is optional. When absent or empty the PBS provider falls back to +/// a stored-request keyed by `code` (`imp.ext.prebid.storedrequest.id`). +/// When present, each entry's params are forwarded inline to PBS as +/// `imp.ext.prebid.bidder.`. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AdUnit { @@ -46,7 +59,11 @@ pub struct AdUnit { pub bids: Option>, } -/// Bidder configuration from the request. +/// Inline bidder params for one SSP within an [`AdUnit`]. +/// +/// `params` is passed verbatim to the corresponding PBS bidder adapter. +/// When the `bids` array is absent, the slot falls back to PBS stored +/// requests — see [`AdUnit`] for details. #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BidConfig { @@ -420,6 +437,10 @@ mod tests { height: 250, nurl: None, burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: HashMap::new(), } } @@ -1129,3 +1150,184 @@ mod tests { assert!(bid.get("h").is_none(), "should omit out-of-range height"); } } + +#[cfg(test)] +mod convert_tests { + use super::*; + use crate::consent::ConsentContext; + use crate::platform::test_support::noop_services; + use crate::test_support::tests::crate_test_settings_str; + use http::Method; + + fn make_settings() -> Settings { + Settings::from_toml(&crate_test_settings_str()).expect("should parse test settings") + } + + fn make_req() -> Request { + Request::builder() + .method(Method::POST) + .uri("https://test-publisher.com/auction") + .body(EdgeBody::empty()) + .expect("should build test request") + } + + fn call_convert(body: &AdRequest) -> AuctionRequest { + let settings = make_settings(); + let services = noop_services(); + let req = make_req(); + convert_tsjs_to_auction_request( + body, + &settings, + &services, + &req, + ConsentContext::default(), + Some("test-ec-id"), + None, + ) + .expect("should convert without error") + } + + #[test] + fn no_bids_produces_empty_bidders_map() { + // An ad unit with no `bids` array must produce an empty bidders map. + // An empty bidders map triggers the PBS stored-request fallback: + // the PBS provider sets imp.ext.prebid.storedrequest = { id: "" }. + let body = AdRequest { + ad_units: vec![AdUnit { + code: "atf_sidebar_ad".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![300, 250]], + }), + }), + bids: None, + }], + config: None, + eids: None, + }; + + let auction_request = call_convert(&body); + + assert_eq!(auction_request.slots.len(), 1, "should have one slot"); + let slot = &auction_request.slots[0]; + assert_eq!(slot.id, "atf_sidebar_ad", "slot id should match unit code"); + assert!( + slot.bidders.is_empty(), + "absent bids array should yield empty bidders map (PBS stored-request path)" + ); + } + + #[test] + fn inline_bids_populate_bidders_map() { + // When bids are supplied, each bidder+params pair should appear in the + // slot's bidders map so PBS receives inline params. + let body = AdRequest { + ad_units: vec![AdUnit { + code: "homepage_header_ad".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![970, 90]], + }), + }), + bids: Some(vec![BidConfig { + bidder: "kargo".to_string(), + params: serde_json::json!({ "placementId": "client_123" }), + }]), + }], + config: None, + eids: None, + }; + + let auction_request = call_convert(&body); + + let slot = &auction_request.slots[0]; + assert!( + slot.bidders.contains_key("kargo"), + "kargo bidder should be present in slot bidders map" + ); + assert_eq!( + slot.bidders["kargo"]["placementId"], "client_123", + "bidder params should be forwarded verbatim" + ); + } + + #[test] + fn config_allowed_key_passes_through() { + // Keys in auction.allowed_context_keys must reach the auction context. + // The test settings do not set allowed_context_keys so the default + // (empty) applies — verify a key is NOT present rather than IS. + // To test the allow-list, inject a key via a custom settings string. + let settings_str = format!( + "{}\n[auction]\nallowed_context_keys = [\"permutive_segments\"]\n", + crate_test_settings_str() + ); + let settings = Settings::from_toml(&settings_str).expect("should parse"); + let services = noop_services(); + let req = make_req(); + + let body = AdRequest { + ad_units: vec![], + config: Some(serde_json::json!({ + "permutive_segments": ["seg1", "seg2"], + "disallowed_key": "should be dropped", + })), + eids: None, + }; + + let auction_request = convert_tsjs_to_auction_request( + &body, + &settings, + &services, + &req, + ConsentContext::default(), + Some("test-ec-id"), + None, + ) + .expect("should convert"); + + assert!( + auction_request.context.contains_key("permutive_segments"), + "allowed key should be in auction context" + ); + assert!( + !auction_request.context.contains_key("disallowed_key"), + "unlisted key should be dropped" + ); + } + + #[test] + fn invalid_banner_size_returns_error() { + // Banner sizes must be [width, height] pairs; a 3-element size is invalid. + let body = AdRequest { + ad_units: vec![AdUnit { + code: "bad_slot".to_string(), + media_types: Some(MediaTypes { + banner: Some(BannerUnit { + sizes: vec![vec![300, 250, 99]], // invalid — 3 elements + }), + }), + bids: None, + }], + config: None, + eids: None, + }; + + let settings = make_settings(); + let services = noop_services(); + let req = make_req(); + let result = convert_tsjs_to_auction_request( + &body, + &settings, + &services, + &req, + ConsentContext::default(), + Some("test-ec-id"), + None, + ); + + assert!( + result.is_err(), + "3-element banner size should return an error" + ); + } +} diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index c89f30ff4..7137ebdb3 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -7,11 +7,43 @@ use std::time::Duration; use web_time::Instant; use crate::error::TrustedServerError; +use crate::platform::{PlatformPendingRequest, RuntimeServices}; use super::config::AuctionConfig; use super::provider::AuctionProvider; use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus}; +/// In-flight auction requests dispatched to SSP backends. +/// +/// Created by [`AuctionOrchestrator::dispatch_auction`] and consumed by +/// [`AuctionOrchestrator::collect_dispatched_auction`]. Carrying this handle +/// across `pending_origin.wait()` lets origin response and SSP HTTP requests +/// race in Fastly's native layer, enabling TTFB ≈ origin latency rather than +/// TTFB ≈ auction timeout. +pub struct DispatchedAuction { + pending_requests: Vec, + backend_to_provider: HashMap)>, + auction_start: Instant, + timeout_ms: u32, + floor_prices: HashMap, + /// Carried so the mediator call in collect can pass it as the auction request. + request: AuctionRequest, +} + +#[cfg(test)] +impl DispatchedAuction { + pub(crate) fn empty_for_test(request: AuctionRequest, timeout_ms: u32) -> Self { + Self { + pending_requests: Vec::new(), + backend_to_provider: HashMap::new(), + auction_start: Instant::now(), + timeout_ms, + floor_prices: HashMap::new(), + request, + } + } +} + const PROVIDER_ERROR_MESSAGE_CHARS: usize = 500; const ERROR_TYPE_PARSE_RESPONSE: &str = "parse_response"; @@ -212,7 +244,9 @@ impl AuctionOrchestrator { let mediator_context = AuctionContext { settings: context.settings, request: context.request, - timeout_ms: remaining_ms, + // Bound by both the remaining auction budget and the mediator's + // own configured timeout, matching the dispatched collect path. + timeout_ms: remaining_ms.min(mediator.timeout_ms()), provider_responses: Some(&provider_responses), services: context.services, }; @@ -235,8 +269,19 @@ impl AuctionOrchestrator { })?; let response_time_ms = start_time.elapsed().as_millis() as u64; + // Use the context-aware parse so mediators (e.g. adserver_mock) can + // restore nurl/burl/ad_id and PBS cache fields from the collected SSP + // responses. The dispatched collect path already does this; the + // synchronous mediation path used by POST /auction and + // /__ts/page-bids must match or mediated cache bids lose the metadata + // needed for creative rendering and win/billing beacons. let mediator_resp = mediator - .parse_response(platform_resp, response_time_ms) + .parse_response_with_context( + platform_resp, + response_time_ms, + request, + &mediator_context, + ) .await .change_context(TrustedServerError::Auction { message: format!("Mediator {} parse failed", mediator.provider_name()), @@ -366,9 +411,9 @@ impl AuctionOrchestrator { } // Give each provider only the remaining time from the auction - // deadline so that its backend first_byte_timeout doesn't extend - // past the overall budget. Also respect the provider's own - // configured timeout when it is tighter than the remaining budget. + // deadline so that backend transport timeouts do not extend past + // the overall budget. Also respect the provider's own configured + // timeout when it is tighter than the remaining budget. let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); let effective_timeout = remaining_ms.min(provider.timeout_ms()); @@ -465,10 +510,11 @@ impl AuctionOrchestrator { // Enforce the auction deadline: after each select() returns, check // elapsed time and drop remaining requests if the timeout is exceeded. // - // NOTE: `select()` blocks until at least one backend responds (or its - // transport timeout fires). Hard deadline enforcement therefore depends - // on every backend's `first_byte_timeout` being set to at most the - // remaining auction budget — which Phase 1 above guarantees. + // NOTE: `select()` blocks until at least one backend responds and, on + // some adapters, buffers the selected response body before returning. + // Hard deadline enforcement therefore depends on every backend's + // first-byte and between-bytes timeouts being set to at most the + // remaining auction budget, which Phase 1 above guarantees. let mut remaining = pending_requests; while !remaining.is_empty() { @@ -500,7 +546,19 @@ impl AuctionOrchestrator { { let response_time_ms = start_time.elapsed().as_millis() as u64; - match provider.parse_response(response, response_time_ms).await { + // Use the context-aware parse so a provider overriding + // `parse_response_with_context` behaves identically on the + // parallel (`/auction`, page-bids) and collect (publisher) + // paths. The default impl delegates to `parse_response`. + match provider + .parse_response_with_context( + response, + response_time_ms, + request, + context, + ) + .await + { Ok(auction_response) => { log::info!( "Provider '{}' returned {} bids (status: {:?}, time: {}ms)", @@ -633,28 +691,29 @@ impl AuctionOrchestrator { } let starting_count = winning_bids.len(); - winning_bids.retain(|slot_id, bid| match floor_prices.get(slot_id) { - Some(floor) => { - // Bids without price (e.g., APS) pass through - floor checked in mediation - match bid.price { - Some(price) if price >= *floor => true, - Some(_) => { - log::info!( - "Dropping winning bid below floor price for slot '{}'", - slot_id - ); - false - } - None => { - log::debug!( - "Passing bid with encoded price for slot '{}' - floor check deferred to mediation", - slot_id - ); - true - } - } + winning_bids.retain(|slot_id, bid| match (floor_prices.get(slot_id), bid.price) { + (Some(floor), Some(price)) if price >= *floor => true, + (Some(_), Some(_)) => { + log::info!( + "Dropping winning bid below floor price for slot '{}'", + slot_id + ); + false } - None => true, + (_, None) => { + // Any caller that needs to keep an undecoded (encoded-price) + // bid must decode it *before* invoking this function — both + // `select_winning_bids` and the mediator path already do. + // Letting `None`-price bids through here would cause + // `winning_bids.len()` to overcount what `build_bid_map` + // downstream is willing to emit, so they get dropped instead. + log::debug!( + "Dropping bid for slot '{}' - no decoded price (caller must decode before apply_floor_prices)", + slot_id + ); + false + } + (None, Some(_)) => true, }); if winning_bids.len() != starting_count { @@ -694,6 +753,437 @@ impl AuctionOrchestrator { }) } + /// Dispatch SSP bid requests without blocking WASM. + /// + /// Calls each enabled provider's [`AuctionProvider::request_bids`] (which + /// internally calls Fastly's `send_async`), then returns immediately with a + /// [`DispatchedAuction`] token. The Fastly host begins the SSP round-trips + /// while WASM continues to `pending_origin.wait()`. + /// + /// Returns `None` when no providers are configured or all providers are + /// disabled / over budget. The caller should fall back to the synchronous + /// `run_auction` path. + #[must_use] + pub async fn dispatch_auction( + &self, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Option { + let provider_names = self.config.provider_names(); + if provider_names.is_empty() { + return None; + } + + // Mirror run_providers_parallel: reject multi-provider fan-out before + // any request launches when the platform executes `send_async` eagerly + // (e.g. Cloudflare Workers, Spin). Sequential execution would accrue + // the sum of provider latencies before the origin fetch and then fail + // collection with empty bids. + if provider_names.len() > 1 && !context.services.http_client().supports_concurrent_fanout() + { + log::warn!( + "{} auction providers configured, but this platform's HTTP client \ + executes requests sequentially — skipping initial-page auction \ + dispatch; configure a single provider, or use an adapter with \ + concurrent fan-out support", + provider_names.len(), + ); + return None; + } + + let auction_start = Instant::now(); + let mut backend_to_provider: HashMap)> = + HashMap::new(); + let mut pending_requests: Vec = Vec::new(); + + for provider_name in provider_names { + let provider = match self.providers.get(provider_name) { + Some(p) => p, + None => { + // lgtm[rust/cleartext-logging] + // The provider name is a static config identifier (e.g. "prebid"), not a secret. + log::warn!("Provider '{}' not registered, skipping", provider_name); + continue; + } + }; + + if !provider.is_enabled() { + log::debug!( + "Provider '{}' is disabled, skipping", + provider.provider_name() + ); + continue; + } + + let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); + let effective_timeout = remaining_ms.min(provider.timeout_ms()); + + if effective_timeout == 0 { + log::warn!( + "Auction timeout ({}ms) exhausted before launching '{}' — skipping", + context.timeout_ms, + provider.provider_name() + ); + continue; + } + + let backend_name = match provider.backend_name(context.services, effective_timeout) { + Some(name) => name, + None => { + log::warn!( + "Provider '{}' has no backend_name, skipping", + provider.provider_name() + ); + continue; + } + }; + + let provider_context = AuctionContext { + settings: context.settings, + request: context.request, + timeout_ms: effective_timeout, + provider_responses: context.provider_responses, + services: context.services, + }; + + let start_time = Instant::now(); + match provider.request_bids(request, &provider_context).await { + Ok(pending) => { + log::info!( + "Dispatching bid request to '{}' (backend: {}, budget: {}ms)", + provider.provider_name(), + backend_name, + effective_timeout + ); + backend_to_provider.insert( + backend_name.clone(), + ( + provider.provider_name().to_string(), + start_time, + Arc::clone(provider), + ), + ); + pending_requests.push(pending.with_backend_name(backend_name)); + } + Err(e) => { + log::warn!( + "Provider '{}' failed to dispatch request: {:?}", + provider.provider_name(), + e + ); + } + } + } + + if pending_requests.is_empty() { + return None; + } + + log::info!( + "Dispatched {} SSP requests (timeout: {}ms); Fastly host will race them against origin", + pending_requests.len(), + context.timeout_ms + ); + + Some(DispatchedAuction { + pending_requests, + backend_to_provider, + auction_start, + timeout_ms: context.timeout_ms, + floor_prices: self.floor_prices_by_slot(request), + request: request.clone(), + }) + } + + /// Collect bid responses from a previously-dispatched auction. + /// + /// Runs the select-loop phase (equivalent to Phase 2 of + /// `run_providers_parallel`) and, if the orchestrator has a mediator + /// configured, forwards collected bids to it. The overall auction deadline + /// is enforced from `dispatched.auction_start`. + /// + /// On any error or partial failure the method returns the best available + /// result rather than propagating — the caller should still inject the + /// winning bids even if some providers timed out. + pub async fn collect_dispatched_auction( + &self, + dispatched: DispatchedAuction, + services: &RuntimeServices, + context: &AuctionContext<'_>, + ) -> OrchestrationResult { + let DispatchedAuction { + pending_requests, + mut backend_to_provider, + auction_start, + timeout_ms, + floor_prices, + request, + } = dispatched; + + log::info!( + "Collecting {} in-flight SSP responses (timeout: {}ms remaining: {}ms)", + pending_requests.len(), + timeout_ms, + remaining_budget_ms(auction_start, timeout_ms), + ); + + let mut responses: Vec = Vec::new(); + let mut remaining = pending_requests; + + while !remaining.is_empty() { + let select_result = match services + .http_client() + .select(remaining) + .await + .change_context(TrustedServerError::Auction { + message: "HTTP select failed".to_string(), + }) { + Ok(r) => r, + Err(e) => { + log::warn!("select() failed during auction collection: {:?}", e); + break; + } + }; + // Destructure so transport failures can be attributed to a provider + // via `failed_backend_name`, mirroring run_providers_parallel. + let crate::platform::PlatformSelectResult { + ready, + remaining: new_remaining, + failed_backend_name, + } = select_result; + remaining = new_remaining; + + match ready { + Ok(platform_response) => { + let backend_name = platform_response.backend_name.clone().unwrap_or_default(); + if let Some((provider_name, start_time, provider)) = + backend_to_provider.remove(&backend_name) + { + let response_time_ms = start_time.elapsed().as_millis() as u64; + // Mirror run_providers_parallel: use the context-aware + // parse so providers behave identically on both paths. + match provider + .parse_response_with_context( + platform_response, + response_time_ms, + &request, + context, + ) + .await + { + Ok(auction_response) => { + log::info!( + "Provider '{}' returned {} bids ({}ms)", + auction_response.provider, + auction_response.bids.len(), + auction_response.response_time_ms + ); + responses.push(auction_response); + } + Err(e) => { + log::warn!("Provider '{}' parse failed: {:?}", provider_name, e); + // Mirror the parallel path so a parse failure is + // attributed (error_type + message) in provider_details. + responses.push(provider_error_response( + &provider_name, + response_time_ms, + ERROR_TYPE_PARSE_RESPONSE, + &e, + )); + } + } + } else { + log::warn!( + "Received response from unknown backend '{}', ignoring", + backend_name + ); + } + } + Err(e) => { + // Mirror the parallel path: attribute the transport failure to + // the provider behind `failed_backend_name` so it appears in + // provider_details instead of vanishing. + if let Some(ref backend_name) = failed_backend_name { + if let Some((provider_name, start_time, _)) = + backend_to_provider.remove(backend_name) + { + let response_time_ms = start_time.elapsed().as_millis() as u64; + log::warn!("Provider '{}' request failed: {:?}", provider_name, e); + responses.push(provider_transport_failed_response( + &provider_name, + response_time_ms, + )); + } else { + log::warn!( + "A provider request failed (backend '{}' not tracked): {:?}", + backend_name, + e + ); + } + } else { + log::warn!( + "A provider request failed during collection (backend not identified): {:?}", + e + ); + } + } + } + + // Drain every dispatched request. Each backend was capped with + // first-byte and between-bytes timeouts at dispatch time, so by the + // collect phase the remaining handles may already be ready even if + // wall-clock time elapsed while the origin was slow. Dropping them + // here would discard SSP responses that already arrived. The + // mediator launch below still observes A_deadline via + // `remaining_budget_ms`. + } + + let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { + match self.providers.get(mediator_name.as_str()) { + Some(mediator) => { + // Cap the mediator at whichever is tighter: its own configured + // timeout or the remaining auction budget (A_deadline). The old + // comment here claimed origin drain could exhaust the budget before + // collection, but SSP backends are given first-byte and between-bytes + // timeouts equal to effective_timeout (capped at their provider + // timeout) at dispatch time, so they cannot run past A_deadline + // independently. Giving the mediator an uncapped timeout lets it run + // past A_deadline, violating the bounded hold invariant. + let remaining = remaining_budget_ms(auction_start, timeout_ms); + if remaining == 0 { + log::warn!( + "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", + mediator.provider_name(), + responses.len(), + ); + let winning = self.select_winning_bids(&responses, &floor_prices); + return OrchestrationResult { + provider_responses: responses, + mediator_response: None, + winning_bids: winning, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: HashMap::new(), + }; + } + let mediator_timeout = remaining.min(mediator.timeout_ms()); + let mediator_start = Instant::now(); + log::info!( + "Running mediator '{}' with {}ms budget (A_deadline remaining: {}ms, configured: {}ms)", + mediator.provider_name(), + mediator_timeout, + remaining, + mediator.timeout_ms(), + ); + // The mediator runs on the collect path. See the doc-comment on + // `AuctionContext::request`: the real client request was already + // consumed by `send_async` during dispatch, so we substitute a + // canonical placeholder URL. Any future mediator that needs real + // client headers must snapshot them at dispatch time onto + // `DispatchedAuction` rather than reading `context.request` here. + let placeholder = http::Request::builder() + .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) + .body(edgezero_core::body::Body::empty()) + .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); + let mediator_context = AuctionContext { + settings: context.settings, + request: &placeholder, + timeout_ms: mediator_timeout, + provider_responses: Some(&responses), + services: context.services, + }; + match mediator.request_bids(&request, &mediator_context).await { + Ok(pending) => { + let platform_resp = services.http_client().wait(pending).await; + match platform_resp.change_context(TrustedServerError::Auction { + message: format!( + "Mediator {} request failed", + mediator.provider_name() + ), + }) { + Ok(platform_resp) => { + let response_time_ms = + mediator_start.elapsed().as_millis() as u64; + // Mirror run_parallel_mediation: use the + // context-aware parse so the mediator sees + // the collected provider responses. + match mediator + .parse_response_with_context( + platform_resp, + response_time_ms, + &request, + &mediator_context, + ) + .await + { + Ok(mediator_resp) => { + let winning = mediator_resp + .bids + .iter() + .filter_map(|bid| { + if bid.price.is_none() { + log::warn!( + "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", + mediator.provider_name(), + bid.slot_id + ); + None + } else { + Some((bid.slot_id.clone(), bid.clone())) + } + }) + .collect(); + let winning = + self.apply_floor_prices(winning, &floor_prices); + (Some(mediator_resp), winning) + } + Err(e) => { + log::warn!( + "Mediator '{}' parse failed: {:?}", + mediator.provider_name(), + e + ); + let winning = + self.select_winning_bids(&responses, &floor_prices); + (None, winning) + } + } + } + Err(e) => { + log::warn!("Mediator request failed: {:?}", e); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } + Err(e) => { + log::warn!( + "Mediator '{}' failed to dispatch: {:?}", + mediator.provider_name(), + e + ); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } + None => { + // lgtm[rust/cleartext-logging] + // The mediator name is a static config identifier, not a secret. + log::warn!("Mediator '{}' not registered", mediator_name); + (None, self.select_winning_bids(&responses, &floor_prices)) + } + } + } else { + (None, self.select_winning_bids(&responses, &floor_prices)) + }; + + OrchestrationResult { + provider_responses: responses, + mediator_response, + winning_bids, + total_time_ms: auction_start.elapsed().as_millis() as u64, + metadata: HashMap::new(), + } + } + /// Check if orchestrator is enabled. #[must_use] pub fn is_enabled(&self) -> bool { @@ -823,6 +1313,160 @@ mod tests { } } + /// Mediator whose context-aware parse restores `nurl`/`ad_id` (mirroring + /// `adserver_mock`), while its context-free parse does not. Lets a test prove + /// the synchronous mediation path calls `parse_response_with_context`. + struct CacheRestoringMediator; + + fn mediated_bid(nurl: Option) -> Bid { + Bid { + slot_id: "header-banner".to_string(), + price: Some(2.5), + currency: "USD".to_string(), + creative: Some("
ad
".to_string()), + adomain: None, + bidder: "mediator".to_string(), + width: 728, + height: 90, + nurl: nurl.clone(), + burl: nurl, + ad_id: Some("creative-123".to_string()), + cache_id: Some("cache-abc".to_string()), + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + } + } + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for CacheRestoringMediator { + fn provider_name(&self) -> &'static str { + "mediator" + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let req = PlatformHttpRequest::new( + http::Request::builder() + .method("POST") + .uri("https://example.com/mediate") + .body(edgezero_core::body::Body::empty()) + .expect("should build mediator request"), + "mediator-backend", + ); + context + .services + .http_client() + .send_async(req) + .await + .change_context(TrustedServerError::Auction { + message: "mediator launch failed".to_string(), + }) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + response_time_ms: u64, + ) -> Result> { + // Context-free path: cannot restore SSP-only render/accounting fields. + Ok(AuctionResponse::success( + "mediator", + vec![mediated_bid(None)], + response_time_ms, + )) + } + + async fn parse_response_with_context( + &self, + _response: PlatformResponse, + response_time_ms: u64, + _request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + // Context-aware path: restores nurl/ad_id from the collected SSP bids. + Ok(AuctionResponse::success( + "mediator", + vec![mediated_bid(Some("https://nurl.example/win".to_string()))], + response_time_ms, + )) + } + + fn timeout_ms(&self) -> u32 { + 2000 + } + + fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + Some("mediator-backend".to_string()) + } + } + + #[tokio::test] + async fn mediated_bid_preserves_restored_fields_through_run_auction() { + // run_parallel_mediation must parse the mediator response via + // parse_response_with_context so cache/nurl fields restored from SSP + // responses survive the synchronous mediation path (POST /auction, + // /__ts/page-bids), matching the dispatched collect path. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); // bidder send_async + stub.push_response(200, b"{}".to_vec()); // mediator send_async + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let config = AuctionConfig { + enabled: true, + providers: vec!["bidder".to_string()], + mediator: Some("mediator".to_string()), + timeout_ms: 2000, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider { + name: "bidder", + backend: "bidder-backend", + })); + orchestrator.register_provider(Arc::new(CacheRestoringMediator)); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 2000, + provider_responses: None, + services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("mediated auction should complete"); + + let bid = result + .winning_bids + .get("header-banner") + .expect("mediator should produce a winning bid for the slot"); + assert_eq!( + bid.nurl.as_deref(), + Some("https://nurl.example/win"), + "synchronous mediation must restore nurl via parse_response_with_context" + ); + assert_eq!( + bid.ad_id.as_deref(), + Some("creative-123"), + "mediated bid must keep its restored ad_id" + ); + } + fn create_test_auction_request() -> AuctionRequest { AuctionRequest { id: "test-auction-123".to_string(), @@ -1025,6 +1669,10 @@ mod tests { height: 250, nurl: None, burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: HashMap::new(), }, ); @@ -1041,6 +1689,10 @@ mod tests { height: 250, nurl: None, burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: HashMap::new(), }, ); @@ -1338,9 +1990,78 @@ mod tests { } #[test] - fn test_apply_floor_prices_allows_none_prices_for_encoded_bids() { - // Test that bids with None prices (APS-style) pass through floor pricing - // This is correct behavior for parallel-only strategy where mediation happens later + fn dispatch_auction_skips_multi_provider_fanout_on_sequential_platform() { + futures::executor::block_on(async { + // Arrange: two configured providers on a platform whose HTTP + // client executes send_async eagerly (no concurrent fan-out). + // The initial-page dispatch path must apply the same guard as + // run_providers_parallel or the summed provider latency lands + // before the origin fetch. + let stub = Arc::new(StubHttpClient::new()); + stub.set_concurrent_fanout(false); + let stub_for_assertion = Arc::clone(&stub); + + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + // The leaked allocation is bounded to the test process lifetime. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let config = AuctionConfig { + enabled: true, + providers: vec!["provider-a".to_string(), "provider-b".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider { + name: "provider-a", + backend: "backend-a", + })); + orchestrator.register_provider(Arc::new(StubAuctionProvider { + name: "provider-b", + backend: "backend-b", + })); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 2000, + provider_responses: None, + services, + }; + + // Act + let dispatched = orchestrator.dispatch_auction(&request, &context).await; + + // Assert: no dispatch and no provider request launched. + assert!( + dispatched.is_none(), + "should skip initial-page dispatch on sequential platforms" + ); + assert!( + stub_for_assertion.recorded_backend_names().is_empty(), + "should not launch any provider request on a sequential platform" + ); + }); + } + + #[test] + fn test_apply_floor_prices_drops_bids_with_undecoded_price() { + // Bids that reach apply_floor_prices with `price=None` cannot have a + // floor compared against them — and they would not survive downstream + // (build_bid_map filters them) — so apply_floor_prices drops them so + // the count it reports matches what eventually ships to the client. + // Both production paths (select_winning_bids and the mediator filter) + // already decode/skip None prices before calling this function; this + // test pins the contract. let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); let mut floor_prices = HashMap::new(); floor_prices.insert("slot-1".to_string(), 1.00); @@ -1350,7 +2071,7 @@ mod tests { "slot-1".to_string(), Bid { slot_id: "slot-1".to_string(), - price: None, // APS bid with encoded price + price: None, currency: "USD".to_string(), creative: Some("
Ad
".to_string()), adomain: None, @@ -1359,6 +2080,10 @@ mod tests { height: 250, nurl: None, burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: { let mut m = HashMap::new(); m.insert( @@ -1370,25 +2095,96 @@ mod tests { }, ); - // Apply floor pricing - should pass through with None price let filtered = orchestrator.apply_floor_prices(winning_bids, &floor_prices); - assert_eq!( - filtered.len(), - 1, - "APS bid with None price should pass through floor check" + assert!( + filtered.is_empty(), + "bid with None price should be dropped by apply_floor_prices" ); assert!( - filtered.contains_key("slot-1"), - "Slot-1 should still be present" + !filtered.contains_key("slot-1"), + "slot-1 should not survive when its bid has no decoded price" ); + } + + #[test] + fn test_apply_floor_prices_drops_decoded_aps_bid_below_floor() { + // After mediation decodes an APS bid, apply_floor_prices must enforce the + // slot floor on the resulting price=Some(x) value. This test simulates the + // state of a bid after mediator decoding: price is Some, amznbid is gone. + let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let mut floor_prices = HashMap::new(); + floor_prices.insert("atf".to_string(), 0.50); + + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf".to_string(), + Bid { + slot_id: "atf".to_string(), + price: Some(0.30), // decoded APS price — below $0.50 floor + currency: "USD".to_string(), + creative: Some("
APS Ad
".to_string()), + adomain: None, + bidder: "aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + }, + ); + + let filtered = orchestrator.apply_floor_prices(winning_bids, &floor_prices); + assert!( - filtered - .get("slot-1") - .expect("slot-1 should be present") - .price - .is_none(), - "Price should still be None (not decoded yet)" + filtered.is_empty(), + "Decoded APS bid below slot floor should be dropped" + ); + } + + #[test] + fn test_apply_floor_prices_keeps_decoded_aps_bid_at_or_above_floor() { + let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let mut floor_prices = HashMap::new(); + floor_prices.insert("atf".to_string(), 0.50); + + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf".to_string(), + Bid { + slot_id: "atf".to_string(), + price: Some(0.75), // decoded APS price — above floor + currency: "USD".to_string(), + creative: Some("
APS Ad
".to_string()), + adomain: None, + bidder: "aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + }, + ); + + let filtered = orchestrator.apply_floor_prices(winning_bids, &floor_prices); + + assert_eq!( + filtered.len(), + 1, + "Decoded APS bid at or above floor should be kept" + ); + assert_eq!( + filtered.get("atf").expect("atf should be present").price, + Some(0.75), + "Price should be preserved" ); } } diff --git a/crates/trusted-server-core/src/auction/provider.rs b/crates/trusted-server-core/src/auction/provider.rs index f5d235aa0..86305bf1b 100644 --- a/crates/trusted-server-core/src/auction/provider.rs +++ b/crates/trusted-server-core/src/auction/provider.rs @@ -49,6 +49,28 @@ pub trait AuctionProvider: Send + Sync { response_time_ms: u64, ) -> Result>; + /// Parse the response with access to the original auction request and context. + /// + /// Providers that need request-local metadata while transforming responses + /// can override this method. `request` is the [`AuctionRequest`] the + /// orchestrator dispatched, so request-scoped data (e.g. slot ID mappings) + /// can be derived here instead of stored on the shared provider instance. + /// The default preserves the existing response-only provider contract. + /// + /// # Errors + /// + /// Returns an error if the response cannot be parsed into a valid [`AuctionResponse`]. + async fn parse_response_with_context( + &self, + response: PlatformResponse, + response_time_ms: u64, + request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let _ = (request, context); + self.parse_response(response, response_time_ms).await + } + /// Check if this provider supports a specific media type. fn supports_media_type(&self, media_type: &super::types::MediaType) -> bool { // By default, support banner ads diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 80314318e..14c7713f8 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -53,9 +53,15 @@ pub struct AdFormat { } /// Media type enumeration. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +/// +/// `Default` is `Banner` for programmatic construction only. Do **not** add +/// `#[serde(default)]` to any field of this type: it would coerce an +/// unknown/missing media type to `Banner` rather than failing, silently +/// mis-typing video/native slots. Deserialization must stay strict. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum MediaType { + #[default] Banner, Video, Native, @@ -83,11 +89,13 @@ pub struct UserInfo { /// cookies/headers, not from stored data. #[serde(skip)] pub consent: Option, - /// Consent-gated Extended User IDs resolved from the KV identity graph. + /// Extended User IDs parsed from the [`crate::constants::COOKIE_TS_EIDS`] cookie. /// - /// Populated by the auction handler from partner data when the user has - /// a valid EC and consent permits EID transmission. `None` when no EIDs - /// are available (no EC, consent denied, or KV read failure). + /// Raw (un-gated) values from the browser; consent gating via + /// [`crate::consent::gate_eids_by_consent`] is applied centrally in the + /// endpoint handlers (the auction and page-bids paths) before any EID + /// reaches a bid request — the provider layer just forwards already-gated + /// EIDs. #[serde(skip)] pub eids: Option>, } @@ -108,6 +116,29 @@ pub struct SiteInfo { } /// Context passed to auction providers. +/// +/// # The `request` field is path-dependent +/// +/// `request` carries the **real downstream client request** in the dispatch +/// path ([`AuctionOrchestrator::run_auction`][run] and +/// [`dispatch_auction`][dispatch]). Providers there can read client headers +/// (DNT, User-Agent, cookies, X-* customs) directly off it. +/// +/// In the **collect path** ([`collect_dispatched_auction`][collect]) the +/// mediator is invoked with a synthetic placeholder request +/// (`https://placeholder.invalid/`), because the real client request has +/// already been consumed by `send_async` during dispatch and the host pipeline +/// can't lend it across the `.await`. **Mediators must not depend on reading +/// client state from `context.request`** — the placeholder has none of the +/// real headers. If a future mediator needs that data, snapshot it into a new +/// field on this struct at dispatch time and stash it on the +/// [`DispatchedAuction`] token so collect can attach it to the mediator's +/// context. See +/// (P2-1) for the open follow-up. +/// +/// [run]: crate::auction::AuctionOrchestrator::run_auction +/// [dispatch]: crate::auction::AuctionOrchestrator::dispatch_auction +/// [collect]: crate::auction::AuctionOrchestrator::collect_dispatched_auction pub struct AuctionContext<'a> { pub settings: &'a Settings, pub request: &'a Request, @@ -119,6 +150,12 @@ pub struct AuctionContext<'a> { pub services: &'a RuntimeServices, } +/// URL used by the orchestrator when invoking a mediator from the collect +/// path. Providers can `debug_assert` against this value to catch a mediator +/// that has accidentally started depending on `context.request` carrying real +/// client headers. +pub const MEDIATOR_PLACEHOLDER_URL: &str = "https://placeholder.invalid/"; + /// Response from a single auction provider. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AuctionResponse { @@ -159,6 +196,24 @@ pub struct Bid { pub nurl: Option, /// Billing notification URL pub burl: Option, + /// Ad ID from the bidder + pub ad_id: Option, + /// Prebid Cache UUID for this bid. + /// + /// Populated from `ext.prebid.cache.bids.cacheId` in the PBS response. + /// Used as `hb_adid` targeting value in `window.tsjs.bids`. `None` for + /// non-PBS providers (e.g., APS) and PBS bids without Prebid Cache enabled. + pub cache_id: Option, + /// Prebid Cache host (e.g., `"openads.adsrvr.org"`). + /// + /// Populated from the host of `ext.prebid.cache.bids.url`. Used as + /// `hb_cache_host` targeting value. `None` when cache is absent. + pub cache_host: Option, + /// Prebid Cache path (e.g., `"/cache"`). + /// + /// Populated from the path of `ext.prebid.cache.bids.url`. Used as + /// `hb_cache_path` targeting value. `None` when cache is absent. + pub cache_path: Option, /// Provider-specific bid metadata /// For APS bids, contains encoded price in "amznbid" field pub metadata: HashMap, @@ -283,6 +338,10 @@ mod tests { height: 250, nurl: None, burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: HashMap::new(), } } @@ -394,4 +453,73 @@ mod tests { "should omit metadata field when empty" ); } + + #[test] + fn bid_with_cache_fields_round_trips_through_json() { + let bid = Bid { + slot_id: "atf".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "thetradedesk".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-id".to_string()), + cache_id: Some("cache-uuid".to_string()), + cache_host: Some("cache.example.com".to_string()), + cache_path: Some("/pbc/v1/cache".to_string()), + metadata: HashMap::new(), + }; + let json = serde_json::to_string(&bid).expect("should serialize Bid"); + let restored: Bid = serde_json::from_str(&json).expect("should deserialize Bid"); + assert_eq!( + restored.cache_id.as_deref(), + Some("cache-uuid"), + "should round-trip cache_id" + ); + assert_eq!( + restored.cache_host.as_deref(), + Some("cache.example.com"), + "should round-trip cache_host" + ); + assert_eq!( + restored.cache_path.as_deref(), + Some("/pbc/v1/cache"), + "should round-trip cache_path" + ); + } + + #[test] + fn media_type_defaults_to_banner() { + assert_eq!( + MediaType::default(), + MediaType::Banner, + "should default to Banner for serde field defaults" + ); + } + + #[test] + fn bid_has_ad_id_field() { + let bid = Bid { + slot_id: "s".to_string(), + price: Some(1.0), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("prebid-ad-id-abc".to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }; + assert_eq!(bid.ad_id.as_deref(), Some("prebid-ad-id-abc")); + } } diff --git a/crates/trusted-server-core/src/consent/mod.rs b/crates/trusted-server-core/src/consent/mod.rs index ebb5e032c..fb49d513c 100644 --- a/crates/trusted-server-core/src/consent/mod.rs +++ b/crates/trusted-server-core/src/consent/mod.rs @@ -330,6 +330,26 @@ fn effective_tcf(ctx: &ConsentContext) -> Option<&types::TcfConsent> { }) } +/// Returns whether a server-side auction may be dispatched for this request. +/// +/// Fails closed for GDPR-relevant traffic: when an EU TCF signal is present +/// (`gdpr_applies`) **or** the request's geo jurisdiction is GDPR or unknown, +/// the effective TCF consent (standalone TC string or GPP EU TCF section) +/// must grant Purpose 1 (storage/access). Only requests from a known +/// non-GDPR jurisdiction with no EU TCF signal are freely allowed. +#[must_use] +pub fn consent_allows_server_side_auction(ctx: &ConsentContext) -> bool { + let requires_tcf_purpose1 = ctx.gdpr_applies + || matches!( + ctx.jurisdiction, + jurisdiction::Jurisdiction::Gdpr | jurisdiction::Jurisdiction::Unknown + ); + if !requires_tcf_purpose1 { + return true; + } + effective_tcf(ctx).is_some_and(|tcf| tcf.has_purpose_consent(1)) +} + /// Returns whether TCF consent allows EID transmission. #[must_use] fn allows_eid_transmission(tcf: &types::TcfConsent) -> bool { @@ -683,8 +703,8 @@ mod tests { use super::{ allows_ec_creation, apply_expiration_check, apply_tcf_conflict_resolution, - build_consent_context, build_context_from_signals, has_explicit_ec_withdrawal, - ConsentPipelineInput, + build_consent_context, build_context_from_signals, consent_allows_server_side_auction, + has_explicit_ec_withdrawal, ConsentPipelineInput, }; use crate::consent::jurisdiction::Jurisdiction; use crate::consent::types::{ @@ -785,6 +805,95 @@ mod tests { } } + #[test] + fn auction_allowed_for_known_non_gdpr_jurisdiction_without_tcf_signal() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + ..ConsentContext::default() + }; + + assert!( + consent_allows_server_side_auction(&ctx), + "known non-GDPR jurisdiction with no EU TCF signal should allow auction" + ); + } + + #[test] + fn auction_fails_closed_for_gdpr_jurisdiction_without_consent() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + ..ConsentContext::default() + }; + + assert!( + !consent_allows_server_side_auction(&ctx), + "GDPR jurisdiction without a TCF signal should fail closed" + ); + } + + #[test] + fn auction_fails_closed_for_unknown_jurisdiction_without_consent() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Unknown, + ..ConsentContext::default() + }; + + assert!( + !consent_allows_server_side_auction(&ctx), + "unknown jurisdiction without a TCF signal should fail closed" + ); + } + + #[test] + fn auction_fails_closed_when_tcf_signal_present_without_purpose1() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::UsState("CA".to_owned()), + gdpr_applies: true, + tcf: Some(TcfBuilder::new().with_storage(false).build()), + ..ConsentContext::default() + }; + + assert!( + !consent_allows_server_side_auction(&ctx), + "EU TCF signal without Purpose 1 should block auction even outside GDPR geo" + ); + } + + #[test] + fn auction_allowed_for_gdpr_jurisdiction_with_purpose1_consent() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + tcf: Some(TcfBuilder::new().with_storage(true).build()), + ..ConsentContext::default() + }; + + assert!( + consent_allows_server_side_auction(&ctx), + "GDPR jurisdiction with Purpose 1 consent should allow auction" + ); + } + + #[test] + fn auction_allowed_with_purpose1_via_gpp_eu_tcf_section() { + let ctx = ConsentContext { + jurisdiction: Jurisdiction::Gdpr, + gdpr_applies: true, + gpp: Some(GppConsent { + version: 1, + section_ids: vec![2], + eu_tcf: Some(TcfBuilder::new().with_storage(true).build()), + us_sale_opt_out: None, + }), + ..ConsentContext::default() + }; + + assert!( + consent_allows_server_side_auction(&ctx), + "Purpose 1 granted via GPP EU TCF section should allow auction" + ); + } + #[test] fn missing_geo_keeps_unknown_jurisdiction_and_blocks_ec_creation() { let req = build_request(); diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index b6750e453..ffcf4f034 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -1,6 +1,8 @@ use http::header::HeaderName; pub const COOKIE_TS_EC: &str = "ts-ec"; +/// Cookie written by the Trusted Server JS SDK containing a standard-base64-encoded +/// JSON array of Extended User IDs (`[{ source, uids }]`) from identity providers. pub const COOKIE_TS_EIDS: &str = "ts-eids"; pub const COOKIE_TS_TESTER: &str = "ts-tester"; pub const COOKIE_SHAREDID: &str = "sharedId"; diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs new file mode 100644 index 000000000..0fb076d69 --- /dev/null +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -0,0 +1,766 @@ +//! Configuration types and URL matching for creative opportunity slots. +//! +//! A [`CreativeOpportunitySlot`] describes a single ad placement: which pages +//! it appears on (via glob patterns), what ad formats it supports, and how it +//! maps to provider-specific identifiers such as GAM unit paths and APS slot IDs. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use glob::Pattern; + +use crate::auction::types::{AdFormat, AdSlot, MediaType}; +use crate::price_bucket::PriceGranularity; +use crate::settings::vec_from_seq_or_map; + +/// Top-level configuration for the creative opportunities system. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CreativeOpportunitiesConfig { + /// GAM network ID used to build default unit paths. + pub gam_network_id: String, + /// Maximum time in milliseconds to wait for the server-side auction before + /// closing the response body. + /// + /// The auction runs concurrently with HTML body streaming. Body content + /// above `` has already been delivered and painted before the hold + /// begins, so **FCP is not affected**. What this timeout bounds is the slip + /// on `DOMContentLoaded` and `window.load`: third-party scripts that hook + /// those events fire later by at most this duration. + /// + /// The worst case is a cache-hit page where the origin drains in <50 ms + /// but the auction takes the full timeout — the browser sits idle waiting + /// for ``. 500 ms is the recommended default and the hard upper + /// bound on DCL slip the publisher is willing to accept. + /// + /// When absent, falls back to `[auction].timeout_ms` from global config. + #[serde(default)] + pub auction_timeout_ms: Option, + /// Price granularity for header-bidding price bucketing. Defaults to `Dense`. + #[serde(default)] + pub price_granularity: PriceGranularity, + /// Slot templates. Empty vec = feature disabled (no auction fired, no globals injected). + #[serde(default, deserialize_with = "vec_from_seq_or_map")] + pub slot: Vec, +} + +impl CreativeOpportunitiesConfig { + /// Pre-compile glob patterns for all slots. Call once after deserialization. + pub fn compile_slots(&mut self) { + for slot in &mut self.slot { + slot.compile_patterns(); + } + } + + /// Validate all slot definitions after runtime preparation. + /// + /// # Errors + /// + /// Returns an error string when a slot has an invalid identifier, page + /// pattern set, format list, dimensions, or resolved GAM unit path. + pub fn validate_runtime(&self) -> Result<(), String> { + for slot in &self.slot { + slot.validate_runtime(&self.gam_network_id)?; + } + + Ok(()) + } +} + +/// A single ad placement opportunity on the publisher's site. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CreativeOpportunitySlot { + /// Unique identifier for the slot (e.g., `"atf"`, `"below-fold-sidebar"`). + pub id: String, + /// Override for the GAM ad unit path. + /// + /// When absent, the path is derived as `//`. + pub gam_unit_path: Option, + /// Override for the HTML `div` element ID that will hold the creative. + /// + /// Defaults to [`id`](Self::id) when absent. + pub div_id: Option, + /// Glob patterns for page paths this slot should appear on. + pub page_patterns: Vec, + /// Supported ad formats (size + media type combinations). + pub formats: Vec, + /// Optional floor price in CPM (USD). + pub floor_price: Option, + /// Slot-level targeting key–value pairs forwarded to the auction. + #[serde(default)] + pub targeting: HashMap, + /// Provider-specific slot identifiers. + #[serde(default)] + pub providers: SlotProviders, + /// Pre-compiled [`page_patterns`](Self::page_patterns) for hot-path matching. + /// + /// Populated by [`compile_patterns`](Self::compile_patterns) once at startup + /// via [`CreativeOpportunitiesConfig::compile_slots`]. When this is + /// empty, [`matches_path`](Self::matches_path) falls back to compiling on + /// every call so callers that build slots by hand in tests + /// still work. + /// + /// `pub(crate)` rather than private so cross-module test helpers in this + /// crate can construct slots via struct-literal syntax with an empty cache. + #[serde(skip, default)] + pub(crate) compiled_patterns: Vec, +} + +impl CreativeOpportunitySlot { + /// Validate the slot shape after [`compile_patterns`](Self::compile_patterns) has run. + /// + /// # Errors + /// + /// Returns an error string when required slot fields are empty, invalid, + /// or semantically unusable at runtime. + pub fn validate_runtime(&self, gam_network_id: &str) -> Result<(), String> { + validate_slot_id(&self.id)?; + + if self.page_patterns.is_empty() { + return Err(format!( + "slot `{}` must include at least one page pattern", + self.id + )); + } + + if self.compiled_patterns.is_empty() { + return Err(format!( + "slot `{}` must include at least one valid page pattern", + self.id + )); + } + + if self.formats.is_empty() { + return Err(format!( + "slot `{}` must include at least one format", + self.id + )); + } + + for format in &self.formats { + format.validate_runtime(&self.id)?; + } + + // A negative floor silently disables minimum-price enforcement, and a + // non-finite floor (NaN/infinity) produces surprising all-pass/all-drop + // comparisons and an invalid OpenRTB `bidfloor`. + if let Some(floor_price) = self.floor_price { + if !floor_price.is_finite() || floor_price < 0.0 { + return Err(format!( + "slot `{}` floor_price must be a finite value >= 0.0, got {floor_price}", + self.id + )); + } + } + + // An explicit empty/whitespace `div_id` override is rejected: the + // injected JS resolves slots with `candidate.id.startsWith(slot.div_id)`, + // and every element id starts with the empty string, so an empty override + // would bind the slot to the first id-bearing element in the document. + if self + .div_id + .as_deref() + .is_some_and(|div_id| div_id.trim().is_empty()) + { + return Err(format!( + "slot `{}` div_id override must not be empty", + self.id + )); + } + + if self + .resolved_gam_unit_path(gam_network_id) + .trim() + .is_empty() + { + return Err(format!( + "slot `{}` resolved GAM unit path must not be empty", + self.id + )); + } + + Ok(()) + } + + /// Returns `true` if `path` matches any of this slot's [`page_patterns`](Self::page_patterns). + /// + /// Patterns use glob syntax (e.g., `"/2024/*"` matches any path under `/2024/`, + /// `"/"` matches only the root). A single `*` matches any sequence of characters + /// including path separators because `require_literal_separator` is `false`. + /// When a pattern contains `**` in a position the glob crate considers invalid + /// (e.g., `"/20**"` or `"b**"`), the `**` is normalised to `*` before matching — + /// prefer a valid single-`*` pattern over relying on this fallback. + /// + /// Patterns that cannot be compiled even after normalisation are silently skipped. + #[must_use] + pub fn matches_path(&self, path: &str) -> bool { + // Fast path: use the pre-compiled patterns when available so we don't + // re-run `Pattern::new` on every request. The vec is non-empty iff + // [`compile_patterns`](Self::compile_patterns) succeeded at load time + // and the slot has at least one pattern. + if !self.compiled_patterns.is_empty() { + return self.compiled_patterns.iter().any(|p| p.matches(path)); + } + + // Fallback for slots constructed by hand (tests, legacy callers that + // skip `compile_patterns`). Re-compiles on every call. + self.page_patterns + .iter() + .any(|pattern| match Pattern::new(pattern) { + Ok(p) => p.matches(path), + Err(_) => { + let normalised = pattern.replace("**", "*"); + Pattern::new(&normalised) + .map(|p| p.matches(path)) + .unwrap_or(false) + } + }) + } + + /// Compile [`page_patterns`](Self::page_patterns) into the + /// [`compiled_patterns`](Self::compiled_patterns) cache. + /// + /// Patterns that fail to compile (either directly or after the `**`→`*` + /// normalisation that [`matches_path`](Self::matches_path) does) are + /// silently skipped — the slot just becomes un-matchable, matching the + /// fallback behaviour. + /// + /// Idempotent: calling twice replaces the cache, so a slot list reloaded + /// at runtime won't accumulate stale patterns. + pub fn compile_patterns(&mut self) { + self.compiled_patterns = self + .page_patterns + .iter() + .filter_map(|pattern| { + match Pattern::new(pattern).or_else(|_| Pattern::new(&pattern.replace("**", "*"))) { + Ok(compiled) => Some(compiled), + Err(_) => { + // Build-time validation only requires *one* valid pattern + // per slot, so a mixed valid/invalid set passes the build + // with the bad pattern silently dropped here. Warn so the + // operator can see the slot matches fewer pages than + // configured. + log::warn!( + "slot `{}`: dropping page pattern '{}' — it does not compile as a glob", + self.id, + pattern + ); + None + } + } + }) + .collect(); + } + + /// Returns the GAM ad unit path for this slot. + /// + /// Uses the explicit [`gam_unit_path`](Self::gam_unit_path) override when set, + /// otherwise constructs `//`. + #[must_use] + pub fn resolved_gam_unit_path(&self, gam_network_id: &str) -> String { + self.gam_unit_path + .clone() + .unwrap_or_else(|| format!("/{}/{}", gam_network_id, self.id)) + } + + /// Returns the div element ID for this slot. + /// + /// Returns the [`div_id`](Self::div_id) override when set, otherwise returns [`id`](Self::id). + #[must_use] + pub fn resolved_div_id(&self) -> &str { + self.div_id.as_deref().unwrap_or(&self.id) + } + + /// Converts this slot into an [`AdSlot`] ready for use in an auction request. + /// + /// Provider-specific params (e.g., APS `slotID`, PBS bidder params) are wired + /// into the `bidders` map keyed by provider/bidder name. + /// + /// When [`PrebidSlotParams::bidders`] is empty, a `trustedServer` entry is + /// injected so [`PrebidAuctionProvider`] expands all `config.bidders` + /// automatically. The slot's `targeting.zone` value is forwarded as + /// `trustedServer.zone` so zone-aware bid-param override rules fire correctly. + #[must_use] + pub fn to_ad_slot(&self) -> AdSlot { + let mut bidders: HashMap = HashMap::new(); + if let Some(ref aps) = self.providers.aps { + bidders.insert( + "aps".to_string(), + serde_json::json!({ "slotID": aps.slot_id }), + ); + } + if let Some(ref prebid) = self.providers.prebid { + if prebid.bidders.is_empty() { + // No explicit per-bidder override: let the Prebid provider expand + // all config.bidders. The "trustedServer" key triggers + // expand_trusted_server_bidders in PrebidAuctionProvider, giving + // each bidder an empty params object that the override engine then + // fills with zone-aware rules. + let mut ts = serde_json::json!({ "bidderParams": {} }); + if let Some(zone) = self.targeting.get("zone") { + ts["zone"] = serde_json::Value::String(zone.clone()); + } + bidders.insert("trustedServer".to_string(), ts); + } else { + for (name, params) in &prebid.bidders { + bidders.insert(name.clone(), params.clone()); + } + } + } + AdSlot { + id: self.id.clone(), + formats: self + .formats + .iter() + .map(CreativeOpportunityFormat::to_ad_format) + .collect(), + floor_price: self.floor_price, + targeting: self + .targeting + .iter() + .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) + .collect(), + bidders, + } + } +} + +/// An ad format combining a media type with pixel dimensions. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct CreativeOpportunityFormat { + /// Creative width in pixels. + pub width: u32, + /// Creative height in pixels. + pub height: u32, + /// Media type for this format. Defaults to `Banner`. + #[serde(default)] + pub media_type: MediaType, +} + +impl CreativeOpportunityFormat { + fn validate_runtime(&self, slot_id: &str) -> Result<(), String> { + if self.width == 0 || self.height == 0 { + return Err(format!( + "slot `{slot_id}` format must have positive width and height" + )); + } + + Ok(()) + } + + fn to_ad_format(&self) -> AdFormat { + AdFormat { + media_type: self.media_type.clone(), + width: self.width, + height: self.height, + } + } +} + +/// Provider-specific slot identifiers for a [`CreativeOpportunitySlot`]. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct SlotProviders { + /// Amazon Publisher Services (APS/TAM) slot parameters. + pub aps: Option, + /// Prebid Server inline bidder parameters. + /// + /// When present, these are forwarded directly as `ext.prebid.bidder.*` + /// in the `OpenRTB` request, bypassing PBS stored request lookup for this slot. + /// Useful in development environments where stored requests are not available. + pub prebid: Option, +} + +/// APS-specific parameters for a slot. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ApsSlotParams { + /// The APS slot ID string used when making TAM bid requests. + pub slot_id: String, +} + +/// Inline Prebid Server bidder parameters for a slot. +/// +/// When `bidders` is empty, `to_ad_slot` injects a `trustedServer` entry so +/// [`PrebidAuctionProvider`] expands all `config.bidders` automatically. +/// When `bidders` is non-empty the map is forwarded verbatim, bypassing +/// automatic expansion (useful for slots that need explicit per-bidder params). +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct PrebidSlotParams { + /// Per-bidder inline params map. Bidder name → params object. + /// + /// Leave empty (or omit `bidders` in config) to auto-expand all + /// `config.bidders` with zone-aware param overrides. + /// + /// Note: when this map is non-empty it is forwarded verbatim, so a slot's + /// `targeting.zone` is **not** injected for these bidders (the `trustedServer` + /// expansion key that carries it is only added when `bidders` is empty). Set + /// explicit per-bidder params only when you do not need zone-aware overrides. + #[serde(default)] + pub bidders: HashMap, +} + +/// Validates that a slot ID contains only safe characters. +/// +/// Allowed characters: ASCII alphanumerics, underscores (`_`), and hyphens (`-`). +/// +/// # Errors +/// +/// Returns an error string when the ID is empty or contains disallowed characters. +pub fn validate_slot_id(id: &str) -> Result<(), String> { + if id.is_empty() { + return Err("slot id must not be empty".to_string()); + } + if id + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { + Ok(()) + } else { + Err(format!( + "slot id '{id}' contains invalid characters; only [A-Za-z0-9_-] allowed" + )) + } +} + +/// Returns all slots whose [`page_patterns`](CreativeOpportunitySlot::page_patterns) match `path`. +#[must_use] +pub fn match_slots<'a>( + slots: &'a [CreativeOpportunitySlot], + path: &str, +) -> Vec<&'a CreativeOpportunitySlot> { + slots.iter().filter(|s| s.matches_path(path)).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_slot(id: &str, patterns: Vec<&str>) -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: id.to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: patterns.into_iter().map(String::from).collect(), + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: crate::auction::types::MediaType::Banner, + }], + floor_price: Some(0.50), + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + } + } + + #[test] + fn compile_patterns_populates_cache_and_match_uses_it() { + let mut slot = make_slot("atf", vec!["/20**", "/about"]); + assert!( + slot.compiled_patterns.is_empty(), + "freshly-built slot should have no compiled patterns" + ); + slot.compile_patterns(); + assert_eq!( + slot.compiled_patterns.len(), + 2, + "compile_patterns should populate one entry per page pattern" + ); + assert!( + slot.matches_path("/2024/01/my-article/"), + "matches_path should hit the compiled-pattern fast path" + ); + assert!( + slot.matches_path("/about"), + "matches_path should hit /about via the compiled cache" + ); + assert!( + !slot.matches_path("/contact"), + "matches_path should reject paths that match nothing in the cache" + ); + } + + #[test] + fn compile_slots_populates_every_slot() { + let mut slots = vec![make_slot("a", vec!["/a/*"]), make_slot("b", vec!["/b/*"])]; + for slot in &mut slots { + slot.compile_patterns(); + } + for slot in &slots { + assert_eq!( + slot.compiled_patterns.len(), + 1, + "every slot's patterns should be pre-compiled after compile_patterns()" + ); + } + } + + #[test] + fn glob_matches_article_path() { + let slot = make_slot("atf", vec!["/20**"]); + assert!( + slot.matches_path("/2024/01/my-article/"), + "should match article path" + ); + assert!(!slot.matches_path("/"), "should not match root"); + } + + #[test] + fn exact_match_homepage() { + let slot = make_slot("home", vec!["/"]); + assert!(slot.matches_path("/"), "should match root"); + assert!(!slot.matches_path("/about"), "should not match /about"); + } + + #[test] + fn slot_id_validates_alphanumeric() { + assert!(validate_slot_id("atf_sidebar_ad").is_ok()); + assert!(validate_slot_id("below-content-0").is_ok()); + assert!(validate_slot_id("").is_err(), "empty id should fail"); + assert!( + validate_slot_id("xss`. + /// Injected at `` open. `None` when no slots matched. + pub ad_slots_script: Option, + /// Shared auction result — written by auction task before HTML processing begins. + /// Handler reads this in `el.on_end_tag()` on the body element. + /// `None` means no auction ran; inject empty `tsjs.bids = {}` as fallback. + pub ad_bids_state: std::sync::Arc>>, /// Maximum bytes the post-processing accumulator may buffer before the /// processor aborts. Mirrors `publisher.max_buffered_body_bytes` so the - /// full-document buffering done for post-processors is bounded by the same - /// cap as the final [`crate::publisher::BoundedWriter`] sink. + /// full-document buffering done for post-processors is bounded. pub max_buffered_body_bytes: usize, } @@ -177,12 +189,37 @@ impl HtmlProcessorConfig { request_host: request_host.to_owned(), request_scheme: request_scheme.to_owned(), integrations: integrations.clone(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, } } + + /// Attach the streaming-auction `"# + .to_string(), + ), + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), + max_buffered_body_bytes: 16 * 1024 * 1024, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk( + b"Tcontent", + true, + ) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + html.contains("window.tsjs=window.tsjs||{}"), + "should inject ad slots namespace at head-open" + ); + assert!( + html.contains(".adSlots=JSON.parse"), + "should inject adSlots at head-open" + ); + assert!( + !html.contains("__ts_request_id"), + "must NOT inject request_id" + ); + } + #[test] fn golden_script_tag_injected_at_head_start() { // The trusted-server script tag must be the FIRST child of . @@ -1367,6 +1495,69 @@ mod tests { ); } + #[test] + fn injects_ts_bids_before_body_close() { + let bids_script = r#""#; + let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: Some( + r#""#.to_string(), + ), + ad_bids_state: state, + max_buffered_body_bytes: 16 * 1024 * 1024, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + html.contains("window.tsjs=window.tsjs||{}"), + "should inject _ts namespace for bids before " + ); + assert!( + html.contains(".bids=JSON.parse"), + "should inject bids before " + ); + let bids_pos = html + .find("window.tsjs=window.tsjs||{}") + .expect("bids namespace should be in output"); + let body_close_pos = html.find("").expect(" should be in output"); + assert!(bids_pos < body_close_pos, "bids must appear before "); + } + + #[test] + fn injects_ts_bids_only_once_with_multiple_body_elements() { + let bids_script = r#""#; + let state = std::sync::Arc::new(std::sync::Mutex::new(Some(bids_script.to_string()))); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: Some( + r#""#.to_string(), + ), + ad_bids_state: state, + max_buffered_body_bytes: 16 * 1024 * 1024, + }; + let mut processor = create_html_processor(config); + // Malformed HTML with two elements (common in CMS template pages) + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert_eq!( + html.matches(".bids=JSON.parse").count(), + 1, + "should inject tsjs.bids exactly once even with multiple elements" + ); + } + #[test] fn golden_url_rewriting_replaces_origin_in_href() { // href attributes pointing at origin domain must be rewritten to proxy host. @@ -1384,6 +1575,8 @@ mod tests { request_host: request_host.to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, }; let mut processor = create_html_processor(config); @@ -1422,6 +1615,59 @@ mod tests { ); } + #[test] + fn injects_empty_ts_bids_when_slots_matched_but_auction_returned_nothing() { + // Slots matched (ad_slots_script is Some) but auction task never wrote a result + // (state is None) — e.g. auction timed out with zero bids. Fallback to {}. + let state = std::sync::Arc::new(std::sync::Mutex::new(None)); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: Some( + r#""#.to_string(), + ), + ad_bids_state: state, + max_buffered_body_bytes: 16 * 1024 * 1024, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + html.contains(".bids=JSON.parse(\"{}\")"), + "should inject empty bids fallback when auction produced nothing" + ); + } + + #[test] + fn does_not_inject_ts_bids_when_no_slots_matched() { + // No slots matched this URL — ad_slots_script is None. tsjs.bids must be + // omitted entirely so the publisher's existing client-side GPT flow is + // unmodified (spec §8: "Existing client-side Prebid/GPT flow runs unmodified"). + let state = std::sync::Arc::new(std::sync::Mutex::new(None)); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + max_buffered_body_bytes: 16 * 1024 * 1024, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!( + !html.contains(".bids=JSON.parse"), + "should NOT inject tsjs.bids when no slots matched" + ); + } + #[test] fn response_size_does_not_grow_disproportionately() { // Processing must not expand HTML by more than 1.1× (accounts for the diff --git a/crates/trusted-server-core/src/integrations/adserver_mock.rs b/crates/trusted-server-core/src/integrations/adserver_mock.rs index 22a40cc98..d8b2a325f 100644 --- a/crates/trusted-server-core/src/integrations/adserver_mock.rs +++ b/crates/trusted-server-core/src/integrations/adserver_mock.rs @@ -95,6 +95,42 @@ impl IntegrationConfig for AdServerMockConfig { // Provider // ============================================================================ +/// Lookup index built from the original SSP bids, used while parsing the +/// mediation response to restore render/accounting fields that the mock +/// mediator endpoint does not echo back. +/// +/// Keyed by `(provider_name, slot_id, bidder_name)`. +type BidIndex = HashMap<(String, String, String), Bid>; + +/// Builds the SSP-bid lookup index from the orchestrator-provided +/// bidder responses on the auction context. +fn build_bid_index(bidder_responses: &[AuctionResponse]) -> BidIndex { + let mut index = BidIndex::new(); + for response in bidder_responses { + for bid in &response.bids { + let key = ( + response.provider.clone(), + bid.slot_id.clone(), + bid.bidder.clone(), + ); + // OpenRTB permits a seat to return multiple bids per imp. This index + // is last-write-wins, so a collision means an earlier bid's + // nurl/burl/cache_* are dropped and win/billing-URL restoration can + // be mis-attributed during mediation. Low severity for the mock + // mediator, but log it so the collision is visible. + if index.insert(key, bid.clone()).is_some() { + log::debug!( + "adserver_mock: duplicate bid for (provider '{}', slot '{}', bidder '{}'); keeping the last — win/billing URL restoration may be mis-attributed", + response.provider, + bid.slot_id, + bid.bidder + ); + } + } + } + index +} + /// Mock ad server mediator provider. pub struct AdServerMockProvider { config: AdServerMockConfig, @@ -140,36 +176,21 @@ impl AdServerMockProvider { .bids .iter() .map(|bid| { - // Check if this is an APS bid with encoded price (inferred from amznbid in metadata) - let encoded_price = bid - .metadata - .get("amznbid") - .and_then(|v| v.as_str()) - .map(String::from); - - if encoded_price.is_some() { - // APS bid - send encoded price for mediation to decode - json!({ - "imp_id": bid.slot_id, - "encoded_price": encoded_price, - "adm": bid.creative, - "w": bid.width, - "h": bid.height, - "crid": format!("{}-creative", bid.bidder), - "adomain": bid.adomain, - }) - } else { - // Regular bid with decoded price - json!({ - "imp_id": bid.slot_id, - "price": bid.price, - "adm": bid.creative, - "w": bid.width, - "h": bid.height, - "crid": format!("{}-creative", bid.bidder), - "adomain": bid.adomain, - }) - } + // Mocktioneer mediator always requires a numeric `price` field. + // APS bids carry price as an opaque encoded string (`amznbid`) + // that cannot be decoded client-side; use `bid.price` when set + // (a real decoded value) or fall back to a mock floor price for + // test/demo purposes. + let price = bid.price.unwrap_or(1.50); + json!({ + "imp_id": bid.slot_id, + "price": price, + "adm": bid.creative, + "w": bid.width, + "h": bid.height, + "crid": format!("{}-creative", bid.bidder), + "adomain": bid.adomain, + }) }) .collect(); @@ -234,8 +255,18 @@ impl AdServerMockProvider { /// Parse `OpenRTB` response from mediation endpoint. /// Mediation returns decoded prices for all bids (including APS bids that were encoded). - fn parse_mediation_response(&self, json: &Json, response_time_ms: u64) -> AuctionResponse { - // Parse OpenRTB response + /// + /// `bid_index` is the SSP-bid lookup built from the auction context's + /// bidder responses. The mock mediator does not echo render/accounting + /// fields back, so they are restored from the index using + /// `(seat, impid, bidder)` where bidder is recovered from the echoed `crid` + /// field (`"{bidder}-creative"` format set during request construction). + fn parse_mediation_response( + &self, + json: &Json, + response_time_ms: u64, + bid_index: &BidIndex, + ) -> AuctionResponse { let empty_array = vec![]; let seatbid = json["seatbid"].as_array().unwrap_or(&empty_array); @@ -247,22 +278,50 @@ impl AdServerMockProvider { let bids = seat["bid"].as_array().unwrap_or(&empty_bids); for bid in bids { - // Mediation layer returns decoded prices for all bids + let slot_id = bid["impid"].as_str().unwrap_or("").to_string(); + + // Recover bidder name from crid ("{bidder}-creative") to look up the + // original SSP bid and restore render/accounting fields the mediator drops. + let crid = bid["crid"].as_str().unwrap_or(""); + let bidder = crid.strip_suffix("-creative").unwrap_or_else(|| { + log::debug!( + "adserver_mock: crid '{crid}' does not match '-creative'; render/accounting fields may be missing" + ); + "" + }); + let key = (seat_name.to_string(), slot_id.clone(), bidder.to_string()); + let original = bid_index.get(&key); + let restored_bidder = + original.map_or_else(|| seat_name.to_string(), |b| b.bidder.clone()); + + let width = bid["w"].as_u64().unwrap_or(0) as u32; + let height = bid["h"].as_u64().unwrap_or(0) as u32; + if width == 0 || height == 0 { + log::debug!( + "adserver_mock: bid for slot '{slot_id}' has zero dimension ({width}×{height}), skipping" + ); + continue; + } + all_bids.push(Bid { - slot_id: bid["impid"].as_str().unwrap_or("").to_string(), - price: bid["price"].as_f64(), // Now properly decoded by mediation + slot_id, + price: bid["price"].as_f64(), currency: "USD".to_string(), creative: bid["adm"].as_str().map(String::from), - width: bid["w"].as_u64().unwrap_or(0) as u32, - height: bid["h"].as_u64().unwrap_or(0) as u32, - bidder: seat_name.to_string(), + width, + height, + bidder: restored_bidder, adomain: bid["adomain"].as_array().map(|arr| { arr.iter() .filter_map(|v| v.as_str().map(String::from)) .collect() }), - nurl: None, - burl: None, + nurl: original.and_then(|b| b.nurl.clone()), + burl: original.and_then(|b| b.burl.clone()), + ad_id: original.and_then(|b| b.ad_id.clone()), + cache_id: original.and_then(|b| b.cache_id.clone()), + cache_host: original.and_then(|b| b.cache_host.clone()), + cache_path: original.and_then(|b| b.cache_path.clone()), metadata: HashMap::new(), }); } @@ -274,6 +333,53 @@ impl AdServerMockProvider { AuctionResponse::success("adserver_mock", all_bids, response_time_ms) } } + + /// Shared parse body for the context-aware and context-less trait methods. + /// + /// # Errors + /// + /// Returns an error when the mediation response body is not valid JSON. + async fn parse_response_inner( + &self, + response: PlatformResponse, + response_time_ms: u64, + bid_index: &BidIndex, + ) -> Result> { + let response = response.response; + + if !response.status().is_success() { + log::warn!("AdServer Mock returned non-success: {}", response.status()); + return Ok(AuctionResponse::error("adserver_mock", response_time_ms)); + } + + // collect_response_bounded caps memory from misbehaving providers. + let body_bytes = collect_response_bounded( + response.into_body(), + UPSTREAM_RTB_MAX_RESPONSE_BYTES, + "adserver_mock", + ) + .await + .change_context(TrustedServerError::Auction { + message: "Failed to read AdServer Mock response body".to_string(), + })?; + let response_json: Json = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Auction { + message: "Failed to parse mediation response".to_string(), + })?; + + log::trace!("AdServer Mock response: {:?}", response_json); + + let auction_response = + self.parse_mediation_response(&response_json, response_time_ms, bid_index); + + log::info!( + "AdServer Mock returned {} bids in {}ms", + auction_response.bids.len(), + response_time_ms + ); + + Ok(auction_response) + } } #[async_trait(?Send)] @@ -378,39 +484,28 @@ impl AuctionProvider for AdServerMockProvider { response: PlatformResponse, response_time_ms: u64, ) -> Result> { - let response = response.response; - - if !response.status().is_success() { - log::warn!("AdServer Mock returned non-success: {}", response.status()); - return Ok(AuctionResponse::error("adserver_mock", response_time_ms)); - } - - // collect_response_bounded caps memory from misbehaving providers. - let body_bytes = collect_response_bounded( - response.into_body(), - UPSTREAM_RTB_MAX_RESPONSE_BYTES, - "adserver_mock", - ) - .await - .change_context(TrustedServerError::Auction { - message: "Failed to read AdServer Mock response body".to_string(), - })?; - let response_json: Json = - serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Auction { - message: "Failed to parse mediation response".to_string(), - })?; - - log::trace!("AdServer Mock response: {:?}", response_json); - - let auction_response = self.parse_mediation_response(&response_json, response_time_ms); - - log::info!( - "AdServer Mock returned {} bids in {}ms", - auction_response.bids.len(), - response_time_ms - ); + // No auction context available — nurl/burl/ad_id restoration from the + // original SSP bids is skipped. The orchestrator always calls + // [`parse_response_with_context`], so this path only serves callers + // outside the orchestration flow. + log::debug!("adserver_mock: parsing without context — SSP bid metadata unavailable"); + self.parse_response_inner(response, response_time_ms, &BidIndex::new()) + .await + } - Ok(auction_response) + async fn parse_response_with_context( + &self, + response: PlatformResponse, + response_time_ms: u64, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + // Rebuild the SSP-bid lookup from the orchestrator-provided bidder + // responses so nurl/burl/ad_id survive mediation. Request-scoped data + // travels on the context instead of provider-instance state. + let bid_index = build_bid_index(context.provider_responses.unwrap_or(&[])); + self.parse_response_inner(response, response_time_ms, &bid_index) + .await } fn supports_media_type(&self, media_type: &MediaType) -> bool { @@ -547,6 +642,10 @@ mod tests { adomain: Some(vec!["amazon.com".to_string()]), nurl: None, burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: HashMap::new(), }], response_time_ms: 150, @@ -564,8 +663,12 @@ mod tests { height: 90, bidder: "test-bidder".to_string(), adomain: None, - nurl: None, - burl: None, + nurl: Some("https://ssp.example/win?id=mock-bid-001".to_string()), + burl: Some("https://ssp.example/bill?id=mock-bid-001".to_string()), + ad_id: Some("mock-bid-001".to_string()), + cache_id: None, + cache_host: None, + cache_path: None, metadata: HashMap::new(), }], response_time_ms: 120, @@ -623,7 +726,8 @@ mod tests { "cur": "USD" }); - let auction_response = provider.parse_mediation_response(&mediation_response, 200); + let auction_response = + provider.parse_mediation_response(&mediation_response, 200, &BidIndex::new()); assert_eq!(auction_response.provider, "adserver_mock"); assert_eq!(auction_response.status, BidStatus::Success); @@ -638,6 +742,98 @@ mod tests { assert_eq!(bid.height, 90); } + #[test] + fn parse_mediation_response_restores_original_bid_render_fields() { + let provider = AdServerMockProvider::new(AdServerMockConfig::default()); + let mediation_response = json!({ + "id": "test-auction-123", + "seatbid": [ + { + "seat": "prebid", + "bid": [ + { + "id": "mediated-bid-001", + "impid": "header-banner", + "price": 0.20, + "adm": "
Mediated Ad
", + "w": 728, + "h": 90, + "crid": "mocktioneer-creative", + "adomain": ["example.com"] + } + ] + } + ], + "cur": "USD" + }); + let mut bid_index = BidIndex::new(); + bid_index.insert( + ( + "prebid".to_string(), + "header-banner".to_string(), + "mocktioneer".to_string(), + ), + Bid { + slot_id: "header-banner".to_string(), + price: Some(0.20), + currency: "USD".to_string(), + creative: Some("
Original Ad
".to_string()), + adomain: Some(vec!["example.com".to_string()]), + bidder: "mocktioneer".to_string(), + width: 728, + height: 90, + nurl: Some("https://ssp.example/win".to_string()), + burl: Some("https://ssp.example/bill".to_string()), + ad_id: Some("bid-impression-id".to_string()), + cache_id: Some("cache-uuid".to_string()), + cache_host: Some("cache.example".to_string()), + cache_path: Some("/cache".to_string()), + metadata: HashMap::new(), + }, + ); + + let auction_response = + provider.parse_mediation_response(&mediation_response, 42, &bid_index); + + assert_eq!(auction_response.status, BidStatus::Success); + assert_eq!(auction_response.bids.len(), 1); + let bid = &auction_response.bids[0]; + assert_eq!( + bid.bidder, "mocktioneer", + "should preserve underlying bidder for hb_bidder targeting" + ); + assert_eq!( + bid.nurl.as_deref(), + Some("https://ssp.example/win"), + "should restore nurl" + ); + assert_eq!( + bid.burl.as_deref(), + Some("https://ssp.example/bill"), + "should restore burl" + ); + assert_eq!( + bid.ad_id.as_deref(), + Some("bid-impression-id"), + "should restore ad_id" + ); + assert_eq!( + bid.cache_id.as_deref(), + Some("cache-uuid"), + "should restore PBS cache UUID" + ); + assert_eq!( + bid.cache_host.as_deref(), + Some("cache.example"), + "should restore PBS cache host" + ); + assert_eq!( + bid.cache_path.as_deref(), + Some("/cache"), + "should restore PBS cache path" + ); + } + #[test] fn test_parse_empty_mediation_response() { let config = AdServerMockConfig::default(); @@ -649,7 +845,8 @@ mod tests { "cur": "USD" }); - let auction_response = provider.parse_mediation_response(&mediation_response, 100); + let auction_response = + provider.parse_mediation_response(&mediation_response, 100, &BidIndex::new()); assert_eq!(auction_response.status, BidStatus::NoBid); assert_eq!(auction_response.bids.len(), 0); @@ -706,6 +903,10 @@ mod tests { adomain: Some(vec!["amazon.com".to_string()]), nurl: None, burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata: aps_metadata, }], response_time_ms: 100, @@ -726,20 +927,15 @@ mod tests { let bid = &bidder_resp["bids"][0]; assert_eq!(bid["imp_id"], "slot-1"); - // Key assertions for APS-style encoded price bids: - // 1. Should NOT have "price" field (or it should be null) - assert!( - bid["price"].is_null(), - "APS bids should not have decoded price, got: {:?}", - bid["price"] - ); - // 2. Should have "encoded_price" field + // APS bids have no decoded price (bid.price == None), so the mock floor + // price (1.50) is used. Mocktioneer requires a numeric price field and + // does not accept an opaque encoded_price string. assert_eq!( - bid["encoded_price"].as_str(), - Some("encoded-price-value"), - "APS bids should have encoded_price from metadata" + bid["price"].as_f64(), + Some(1.50), + "APS bids with no decoded price should fall back to mock floor price 1.50" ); - // 3. adm should be null (not a string) + // adm should be null (not a string) assert!( bid["adm"].is_null(), "Creative-less bids should have null adm, got: {:?}", @@ -847,7 +1043,8 @@ mod tests { "cur": "USD" }); - let auction_response = provider.parse_mediation_response(&mediation_response, 200); + let auction_response = + provider.parse_mediation_response(&mediation_response, 200, &BidIndex::new()); assert_eq!(auction_response.status, BidStatus::Success); assert_eq!(auction_response.bids.len(), 2); diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index faf2aa0a6..edee6133b 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -303,15 +303,61 @@ impl ApsAuctionProvider { Self { config } } + /// Resolve the APS-specific slot ID for a slot. + /// + /// Uses the APS slot ID from `[slot.providers.aps]` if configured; falls + /// back to the creative-opportunity slot ID otherwise. + fn aps_slot_id(slot: &crate::auction::types::AdSlot) -> String { + slot.bidders + .get("aps") + .and_then(|p| p.get("slotID")) + .and_then(|v| v.as_str()) + .unwrap_or(&slot.id) + .to_string() + } + + /// Build the APS slot ID → creative-opportunity slot ID map for `request`. + /// + /// Derived from the [`AuctionRequest`] on every call so the mapping stays + /// request-scoped: the provider instance is shared across concurrent + /// auctions on non-Fastly adapters, and instance state would let + /// overlapping auctions overwrite or consume each other's maps. + fn build_slot_id_map(request: &AuctionRequest) -> HashMap { + let mut slot_id_map: HashMap = HashMap::new(); + for slot in &request.slots { + let aps_slot_id = Self::aps_slot_id(slot); + // Last-write-wins: two slots configuring the same + // `[bidders.aps].slotID` would remap one slot's bids to the + // wrong creative slot. Log the collision so a misconfiguration + // is diagnosable, mirroring the build_bid_index collision log. + if let Some(previous_slot_id) = slot_id_map.insert(aps_slot_id.clone(), slot.id.clone()) + { + log::debug!( + "APS slot ID '{aps_slot_id}' maps to multiple creative slots \ + ('{previous_slot_id}' overwritten by '{}'); bids for this APS \ + slot will resolve to the last one", + slot.id, + ); + } + } + slot_id_map + } + /// Convert unified `AuctionRequest` to APS TAM bid request format. /// /// Populates consent fields (GDPR, US Privacy, GPP) from the /// [`ConsentContext`](crate::consent::ConsentContext) attached to the request. - fn to_aps_request(&self, request: &AuctionRequest) -> ApsBidRequest { + /// + /// `timeout_ms` is the effective auction budget for this provider (already + /// capped by the orchestrator) — advertised to APS so it never expects more + /// time than the edge will actually wait. + fn to_aps_request(&self, request: &AuctionRequest, timeout_ms: u32) -> ApsBidRequest { let slots: Vec = request .slots .iter() .map(|slot| { + let aps_slot_id = Self::aps_slot_id(slot); + // Extract sizes from banner formats let sizes: Vec<[u32; 2]> = slot .formats @@ -321,7 +367,7 @@ impl ApsAuctionProvider { .collect(); ApsSlot { - slot_id: slot.id.clone(), + slot_id: aps_slot_id, sizes, slot_name: Some(slot.id.clone()), } @@ -350,7 +396,7 @@ impl ApsAuctionProvider { slots, page_url: request.publisher.page_url.clone(), user_agent: request.device.as_ref().and_then(|d| d.user_agent.clone()), - timeout: Some(self.config.timeout_ms), + timeout: Some(timeout_ms), gdpr, us_privacy, gpp, @@ -391,7 +437,17 @@ impl ApsAuctionProvider { } // Parse size from "WxH" format - let (width, height) = Self::parse_size(&slot.size).unwrap_or((0, 0)); + let (width, height) = match Self::parse_size(&slot.size) { + Some(dims) => dims, + None => { + log::debug!( + "APS: slot '{}' has malformed size '{}', skipping", + slot.slot_id, + slot.size + ); + return Err(()); + } + }; // Build metadata from targeting keys - includes encoded price for mediation let mut metadata = HashMap::new(); @@ -425,12 +481,24 @@ impl ApsAuctionProvider { height, nurl: None, // Real APS uses client-side event tracking burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, metadata, }) } /// Parse APS TAM response into unified `AuctionResponse`. - fn parse_aps_response(&self, json: &Json, response_time_ms: u64) -> AuctionResponse { + /// + /// `slot_map` is the request-scoped APS slot ID → creative-opportunity + /// slot ID mapping built by [`Self::build_slot_id_map`]. + fn parse_aps_response( + &self, + json: &Json, + response_time_ms: u64, + slot_map: &HashMap, + ) -> AuctionResponse { let mut bids = Vec::new(); // Try to parse as ApsBidResponse with contextual wrapper @@ -442,7 +510,13 @@ impl ApsAuctionProvider { for slot in aps_response.contextual.slots { match self.parse_aps_slot(&slot) { - Ok(bid) => { + Ok(mut bid) => { + // Remap APS slot ID (e.g. "aps-slot-atf-sidebar") back to the + // creative-opportunity slot ID (e.g. "atf_sidebar_ad") so the + // mediator and bid_map can match by creative slot ID. + if let Some(creative_id) = slot_map.get(&bid.slot_id) { + bid.slot_id = creative_id.clone(); + } let encoded_price = bid .metadata .get("amznbid") @@ -474,6 +548,51 @@ impl ApsAuctionProvider { AuctionResponse::success("aps", bids, response_time_ms) } } + + /// Shared parse path for both trait entry points. + /// + /// # Errors + /// + /// Returns an error when the response body cannot be read or is not valid JSON. + async fn parse_response_inner( + &self, + response: PlatformResponse, + response_time_ms: u64, + slot_map: &HashMap, + ) -> Result> { + let response = response.response; + + // Check status code + if !response.status().is_success() { + log::warn!("APS returned non-success status: {}", response.status()); + return Ok(AuctionResponse::error("aps", response_time_ms)); + } + + // Parse response body — collect_response_bounded caps memory from misbehaving providers. + let body_bytes = + collect_response_bounded(response.into_body(), UPSTREAM_RTB_MAX_RESPONSE_BYTES, "aps") + .await + .change_context(TrustedServerError::Auction { + message: "Failed to read APS response body".to_string(), + })?; + let response_json: Json = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Auction { + message: "Failed to parse APS response JSON".to_string(), + })?; + + log::trace!("APS: received response: {:?}", response_json); + + // Transform to unified format + let auction_response = self.parse_aps_response(&response_json, response_time_ms, slot_map); + + log::info!( + "APS returned {} bids in {}ms", + auction_response.bids.len(), + response_time_ms + ); + + Ok(auction_response) + } } #[async_trait(?Send)] @@ -493,8 +612,12 @@ impl AuctionProvider for ApsAuctionProvider { self.config.pub_id ); - // Transform to APS format - let aps_request = self.to_aps_request(request); + // Transform to APS format. `context.timeout_ms` is the effective budget + // the orchestrator granted this provider — the payload must advertise + // the same deadline the edge backend enforces below. The APS-slot-ID → + // creative-slot-ID remapping is rebuilt from the request in + // `parse_response_with_context`, keeping it request-scoped. + let aps_request = self.to_aps_request(request, context.timeout_ms); // Serialize to JSON let aps_json = @@ -551,38 +674,27 @@ impl AuctionProvider for ApsAuctionProvider { response: PlatformResponse, response_time_ms: u64, ) -> Result> { - let response = response.response; - - // Check status code - if !response.status().is_success() { - log::warn!("APS returned non-success status: {}", response.status()); - return Ok(AuctionResponse::error("aps", response_time_ms)); - } - - // Parse response body — collect_response_bounded caps memory from misbehaving providers. - let body_bytes = - collect_response_bounded(response.into_body(), UPSTREAM_RTB_MAX_RESPONSE_BYTES, "aps") - .await - .change_context(TrustedServerError::Auction { - message: "Failed to read APS response body".to_string(), - })?; - let response_json: Json = - serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Auction { - message: "Failed to parse APS response JSON".to_string(), - })?; - - log::trace!("APS: received response: {:?}", response_json); - - // Transform to unified format - let auction_response = self.parse_aps_response(&response_json, response_time_ms); - - log::info!( - "APS returned {} bids in {}ms", - auction_response.bids.len(), - response_time_ms - ); + // No auction request available — bids keep their raw APS slot IDs. The + // orchestrator always calls [`Self::parse_response_with_context`], so + // this path only serves callers outside the orchestration flow. + log::debug!("APS: parsing without request context — slot ID remapping unavailable"); + self.parse_response_inner(response, response_time_ms, &HashMap::new()) + .await + } - Ok(auction_response) + async fn parse_response_with_context( + &self, + response: PlatformResponse, + response_time_ms: u64, + request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + // Rebuild the APS-slot-ID → creative-slot-ID map from the auction + // request so overlapping auctions on shared provider instances cannot + // overwrite or consume each other's mappings. + let slot_map = Self::build_slot_id_map(request); + self.parse_response_inner(response, response_time_ms, &slot_map) + .await } fn supports_media_type(&self, media_type: &MediaType) -> bool { @@ -729,7 +841,7 @@ mod tests { let provider = ApsAuctionProvider::new(config); let auction_request = create_test_auction_request(); - let aps_request = provider.to_aps_request(&auction_request); + let aps_request = provider.to_aps_request(&auction_request, 800); // Verify basic fields assert_eq!(aps_request.pub_id, "5128"); @@ -755,6 +867,82 @@ mod tests { assert_eq!(slot2.sizes[0], [300, 250]); } + #[test] + fn aps_slot_id_from_bidders_map_used_in_request_and_remapped_in_response() { + use serde_json::json; + + let config = ApsConfig { + enabled: true, + pub_id: "5128".to_string(), + endpoint: default_endpoint(), + timeout_ms: 800, + }; + let provider = ApsAuctionProvider::new(config); + + let mut bidders = HashMap::new(); + bidders.insert( + "aps".to_string(), + json!({ "slotID": "aps-slot-atf-sidebar" }), + ); + let request = AuctionRequest { + id: "test".to_string(), + slots: vec![AdSlot { + id: "atf_sidebar_ad".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: None, + targeting: HashMap::new(), + bidders, + }], + publisher: PublisherInfo { + domain: "example.com".to_string(), + page_url: None, + }, + user: UserInfo { + id: Some("user-1".to_string()), + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + }; + + let aps_request = provider.to_aps_request(&request, 800); + assert_eq!( + aps_request.slots[0].slot_id, "aps-slot-atf-sidebar", + "should send configured APS slot ID to APS" + ); + let slot_id_map = ApsAuctionProvider::build_slot_id_map(&request); + assert_eq!( + slot_id_map.get("aps-slot-atf-sidebar").map(String::as_str), + Some("atf_sidebar_ad"), + "should build reverse map from APS slot ID to creative slot ID" + ); + + let aps_response = json!({ + "contextual": { + "slots": [{ + "slotID": "aps-slot-atf-sidebar", + "size": "300x250", + "fif": "1", + "amznbid": "1gtm3q", + "meta": ["slotID"] + }] + } + }); + + let response = provider.parse_aps_response(&aps_response, 100, &slot_id_map); + assert_eq!(response.bids.len(), 1, "should parse one bid"); + assert_eq!( + response.bids[0].slot_id, "atf_sidebar_ad", + "bid slot_id should be remapped to creative slot ID" + ); + } + #[test] fn test_aps_response_parsing_success() { let config = ApsConfig { @@ -807,7 +995,7 @@ mod tests { } }); - let auction_response = provider.parse_aps_response(&aps_response, 150); + let auction_response = provider.parse_aps_response(&aps_response, 150, &HashMap::new()); // Verify response assert_eq!(auction_response.provider, "aps"); @@ -855,7 +1043,7 @@ mod tests { } }); - let auction_response = provider.parse_aps_response(&aps_response, 100); + let auction_response = provider.parse_aps_response(&aps_response, 100, &HashMap::new()); assert_eq!(auction_response.provider, "aps"); assert_eq!(auction_response.status, BidStatus::NoBid); @@ -897,7 +1085,7 @@ mod tests { } }); - let auction_response = provider.parse_aps_response(&aps_response, 100); + let auction_response = provider.parse_aps_response(&aps_response, 100, &HashMap::new()); // Should return no-bid since all slots are invalid assert_eq!(auction_response.status, BidStatus::NoBid); @@ -961,6 +1149,28 @@ mod tests { assert!(!provider.supports_media_type(&MediaType::Native)); } + #[test] + fn aps_payload_timeout_uses_effective_auction_budget_not_provider_config() { + // Provider config says 1000ms but the auction budget grants only 500ms — + // the payload must advertise the tighter effective deadline. + let config = ApsConfig { + enabled: true, + pub_id: "5128".to_string(), + endpoint: default_endpoint(), + timeout_ms: 1000, + }; + let provider = ApsAuctionProvider::new(config); + let request = create_test_auction_request(); + + let aps_request = provider.to_aps_request(&request, 500); + + assert_eq!( + aps_request.timeout, + Some(500), + "should advertise the effective auction budget, not the provider config timeout" + ); + } + #[test] fn test_aps_request_includes_consent_fields() { use crate::consent::ConsentContext; @@ -983,7 +1193,7 @@ mod tests { ..Default::default() }); - let aps_request = provider.to_aps_request(&request); + let aps_request = provider.to_aps_request(&request, 800); // Verify GDPR consent let gdpr = aps_request.gdpr.expect("should have gdpr"); @@ -1012,7 +1222,7 @@ mod tests { let provider = ApsAuctionProvider::new(config); let request = create_test_auction_request(); // consent is None - let aps_request = provider.to_aps_request(&request); + let aps_request = provider.to_aps_request(&request, 800); assert!(aps_request.gdpr.is_none()); assert!(aps_request.us_privacy.is_none()); @@ -1039,7 +1249,7 @@ mod tests { ..Default::default() }); - let aps_request = provider.to_aps_request(&request); + let aps_request = provider.to_aps_request(&request, 800); let json = serde_json::to_value(&aps_request).expect("should serialize"); // GDPR fields present @@ -1101,4 +1311,120 @@ mod tests { Some("encoded-price") ); } + + fn single_slot_request( + auction_id: &str, + aps_slot_id: &str, + creative_id: &str, + ) -> AuctionRequest { + let mut bidders = HashMap::new(); + bidders.insert("aps".to_string(), json!({ "slotID": aps_slot_id })); + AuctionRequest { + id: auction_id.to_string(), + slots: vec![AdSlot { + id: creative_id.to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: None, + targeting: HashMap::new(), + bidders, + }], + publisher: PublisherInfo { + domain: "example.com".to_string(), + page_url: None, + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: None, + site: None, + context: HashMap::new(), + } + } + + fn aps_platform_response(aps_slot_id: &str) -> PlatformResponse { + let body = json!({ + "contextual": { + "slots": [{ + "slotID": aps_slot_id, + "size": "300x250", + "fif": "1", + "amznbid": "1gtm3q", + "meta": ["slotID"] + }] + } + }); + let response = http::Response::builder() + .status(200) + .body(EdgeBody::from( + serde_json::to_vec(&body).expect("should serialize APS response body"), + )) + .expect("should build APS platform response"); + PlatformResponse::new(response) + } + + #[test] + fn concurrent_auctions_parse_out_of_order_with_correct_slot_remapping() { + // Regression: the reverse slot map used to live on the shared provider + // instance, so a second auction's request_bids overwrote the first + // auction's map and out-of-order parses remapped bids to the wrong + // creative slot. The map is now derived from each parse's own request. + let provider = ApsAuctionProvider::new(ApsConfig { + enabled: true, + pub_id: "5128".to_string(), + endpoint: default_endpoint(), + timeout_ms: 800, + }); + + let request_a = single_slot_request("auction-a", "aps-slot-a", "creative_a"); + let request_b = single_slot_request("auction-b", "aps-slot-b", "creative_b"); + + let settings = crate::test_support::tests::create_test_settings(); + let http_request = http::Request::builder() + .uri("https://example.com/") + .body(EdgeBody::empty()) + .expect("should build test request"); + let context = crate::auction::test_support::create_test_auction_context( + &settings, + &http_request, + 800, + ); + + futures::executor::block_on(async { + // Parse auction B's response first, then auction A's — the reverse + // of dispatch order — each against its own request. + let response_b = provider + .parse_response_with_context( + aps_platform_response("aps-slot-b"), + 100, + &request_b, + &context, + ) + .await + .expect("should parse auction B response"); + let response_a = provider + .parse_response_with_context( + aps_platform_response("aps-slot-a"), + 100, + &request_a, + &context, + ) + .await + .expect("should parse auction A response"); + + assert_eq!( + response_b.bids[0].slot_id, "creative_b", + "auction B bid should remap to auction B's creative slot" + ); + assert_eq!( + response_a.bids[0].slot_id, "creative_a", + "auction A bid should remap to auction A's creative slot despite parsing last" + ); + }); + } } diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 4ae15c927..717ad46e8 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -159,6 +159,7 @@ impl DataDomeIntegration { host_header_override: None, certificate_check: true, first_byte_timeout: Duration::from_millis(u64::from(self.config.timeout_ms)), + between_bytes_timeout: Duration::from_millis(u64::from(self.config.timeout_ms)), }; services.backend().ensure(&spec).change_context(Self::error( diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index b560e60f7..f2dbc3eb2 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -81,6 +81,16 @@ pub struct GptConfig { /// Whether to rewrite GPT script URLs in publisher HTML. #[serde(default = "default_rewrite_script")] pub rewrite_script: bool, + + /// URL for the slim-Prebid bundle loaded post-window.load. + /// + /// When set, `installSlimPrebidLoader()` in the GPT bundle will load this + /// script after `window.load`, enabling scroll/refresh client-side auctions + /// and userID module warm-up. Set to the publisher's tsjs-prebid bundle URL. + /// + /// Override via env var: `TRUSTED_SERVER__INTEGRATIONS__GPT__SLIM_PREBID_URL` + #[serde(default, skip_serializing_if = "Option::is_none")] + pub slim_prebid_url: Option, } impl IntegrationConfig for GptConfig { @@ -459,18 +469,57 @@ impl IntegrationHeadInjector for GptIntegration { GPT_INTEGRATION_ID } + /// Injects the `tsjs.adInit` bootstrap script into ``. + /// + /// ## Scroll / refresh handoff contract (Phase 1) + /// + /// `tsjs.adInit` handles **initial render only**: it wires server-side bid + /// targeting into GPT slots and refreshes them. Win/billing beacons fire + /// only from the TS render bridge in the JS bundle, where a matching + /// Prebid Universal Creative request proves the TS creative rendered. + /// It does **not** trigger refresh auctions or handle GPT slot refresh events. + /// + /// Post-`window.load`, slim-Prebid owns scroll and GPT refresh: it listens + /// for GPT refresh events, runs client-side auctions, and sets targeting for + /// subsequent impressions. SPA navigation is handled separately by + /// `installSpaAuctionHook()` in the GPT bundle, which re-runs the server-side + /// auction via `GET /__ts/page-bids` on pushState / replaceState / popstate + /// route changes (see `auction/endpoints.rs`). + /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { - // Set the enable flag and best-effort call the activation function - // registered by the GPT shim module. The bundle also auto-installs - // when it sees the pre-set flag, so this works regardless of whether - // the inline bootstrap runs before or after the TSJS bundle. - vec![ - "" + let mut scripts = vec![ + "" .to_string(), - ] + format!("", GPT_BOOTSTRAP_JS), + ]; + + if let Some(ref url) = self.config.slim_prebid_url { + // JSON-encode the URL, then escape `` cannot close this inline tag and + // let trailing markup execute (standard JSON-in-HTML mitigation). + let encoded = serde_json::to_string(url) + .expect("should serialize string") + .replace("window.__tsjs_slim_prebid_url={encoded};" + )); + } + + scripts } } +/// Inline `window.tsjs.adInit` bootstrap injected at `` so the bids +/// script at `` can call it before the TSJS bundle has loaded. +/// +/// The bundle's idempotent implementation in +/// `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` later overwrites this stub. +/// Both implementations guard the one-time-per-page setup with +/// `window.tsjs.servicesEnabled` so neither double-enables services if the +/// publisher's own init code also calls `googletag.enableServices()`. +const GPT_BOOTSTRAP_JS: &str = include_str!("gpt_bootstrap.js"); + // Default value functions fn default_enabled() -> bool { @@ -503,6 +552,7 @@ mod tests { script_url: default_script_url(), cache_ttl_seconds: 3600, rewrite_script: true, + slim_prebid_url: None, } } @@ -1086,7 +1136,7 @@ mod tests { let inserts = integration.head_inserts(&ctx); - assert_eq!(inserts.len(), 1, "should emit exactly one head insert"); + assert_eq!(inserts.len(), 2, "should emit exactly two head inserts"); assert_eq!( inserts[0], "", @@ -1094,6 +1144,132 @@ mod tests { ); } + #[test] + fn head_inserts_includes_ts_ad_init_with_synchronous_bids_read() { + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let inserts = integration.head_inserts(&ctx); + let combined = inserts.join(""); + assert!(combined.contains("ts.adInit"), "should define tsjs.adInit"); + assert!( + combined.contains("ts.bids"), + "should read tsjs.bids synchronously" + ); + assert!( + combined.contains("ts_initial"), + "should set ts_initial sentinel" + ); + assert!( + !combined.contains("addEventListener(\"slotRenderEnded\""), + "inline bootstrap cannot prove TS creative rendering from GPT slotRenderEnded" + ); + assert!( + !combined.contains("sendBeacon"), + "inline bootstrap must not fire win/billing beacons from GPT slotRenderEnded" + ); + assert!( + !combined.contains("getTargeting(\"hb_adid\")"), + "inline bootstrap must not treat GPT targeting as winner proof" + ); + assert!( + !combined.contains("/ts-bids"), + "must NOT fetch /ts-bids — bids are inline on the page" + ); + assert!( + !combined.contains("bidsPromise"), + "must NOT use bidsPromise — bids are synchronous" + ); + assert!( + !combined.contains("__ts_request_id"), + "must NOT reference request_id — no longer used" + ); + } + + #[test] + fn head_inserts_bootstrap_uses_css_safe_div_prefix_lookup() { + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let combined = integration.head_inserts(&ctx).join(""); + assert!( + combined.contains("querySelectorAll(\"[id]\")"), + "bootstrap should scan ID-bearing elements instead of interpolating div_id into CSS" + ); + assert!( + combined.contains(".startsWith(slot.div_id)"), + "bootstrap should match metacharacter-containing div_id prefixes with startsWith" + ); + assert!( + !combined.contains("[id^='\" + slot.div_id"), + "bootstrap must not build a CSS attribute selector from raw div_id" + ); + } + + #[test] + fn head_inserts_bootstrap_guards_enable_services_with_idempotency_flag() { + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let combined = integration.head_inserts(&ctx).join(""); + assert!( + combined.contains("ts.servicesEnabled"), + "should guard enableServices/enableSingleRequest with the tsjs.servicesEnabled flag" + ); + assert!(combined.contains("ts.adInit"), "should install tsjs.adInit"); + assert!( + !combined.contains("googletag.pubads().refresh()"), + "should never call unbounded refresh() — only refresh(newSlots)" + ); + } + + #[test] + fn head_inserts_bootstrap_refreshes_ts_slots_when_initial_load_disabled() { + // Mirrors the bundle: when the publisher calls disableInitialLoad(), + // display() only registers a TS-defined slot, so the bootstrap must also + // refresh those slots or they render blank. + let config = test_config(); + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let combined = integration.head_inserts(&ctx).join(""); + assert!( + combined.contains("disableInitialLoad"), + "bootstrap should wrap disableInitialLoad() to detect the disabled state" + ); + assert!( + combined.contains("gptInitialLoadDisabled"), + "bootstrap should record the initial-load-disabled state on window.tsjs" + ); + assert!( + combined.contains("slotsNeedingRefresh"), + "bootstrap should refresh TS-defined slots when initial load is disabled" + ); + } + #[test] fn head_injector_integration_id() { let integration = GptIntegration::new(test_config()); @@ -1102,4 +1278,97 @@ mod tests { "gpt" ); } + + #[test] + fn head_inserts_emits_slim_prebid_url_when_configured() { + let config = GptConfig { + slim_prebid_url: Some("https://cdn.example.com/tsjs-prebid.min.js".to_string()), + ..test_config() + }; + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + + let inserts = integration.head_inserts(&ctx); + + assert_eq!( + inserts.len(), + 3, + "should emit three head inserts when slim_prebid_url is set" + ); + assert_eq!( + inserts[2], + r#""#, + "should emit the slim-Prebid URL as a JSON-encoded string assignment" + ); + } + + #[test] + fn head_inserts_escapes_script_terminator_in_slim_prebid_url() { + // A configured URL containing `` must not close the inline tag. + let config = GptConfig { + slim_prebid_url: Some("https://cdn.example.com/x".to_string()), + ..test_config() + }; + let integration = GptIntegration::new(config); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + + let inserts = integration.head_inserts(&ctx); + + // The injected `` must be neutralised: the only + // `` left is the tag's own legitimate closer. + assert!( + !inserts[2].contains(" terminator, got: {}", + inserts[2] + ); + assert_eq!( + inserts[2].matches("").count(), + 1, + "only the tag's own closing should remain, got: {}", + inserts[2] + ); + assert!( + inserts[2].contains("<\\/script>"), + "should emit the escaped terminator, got: {}", + inserts[2] + ); + } + + #[test] + fn head_inserts_omits_slim_prebid_url_when_not_configured() { + let integration = GptIntegration::new(test_config()); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + + let inserts = integration.head_inserts(&ctx); + + assert_eq!( + inserts.len(), + 2, + "should emit exactly two head inserts when slim_prebid_url is absent" + ); + assert!( + inserts + .iter() + .all(|s| !s.contains("__tsjs_slim_prebid_url")), + "should not emit slim-Prebid URL tag when not configured" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js new file mode 100644 index 000000000..cc4c5c00c --- /dev/null +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -0,0 +1,171 @@ +// Edge-injected GPT auction bootstrap. +// +// This is the minimal `window.tsjs.adInit` that runs on first page load +// before the TSJS bundle has had a chance to install its richer +// idempotent implementation. The bundle in +// crates/trusted-server-js/lib/src/integrations/gpt/index.ts overwrites `tsjs.adInit` +// once it loads. +// +// Contract with the bundle: +// - Both implementations must set `window.tsjs.servicesEnabled = true` +// after calling `enableSingleRequest()`/`enableServices()` so a +// subsequent call becomes a no-op. +// - `refresh()` is called only for the slots defined in this pass, +// never the global slot list. +// +// Only installed if `window.tsjs.adInit` isn't already defined. +(function () { + if (typeof window === "undefined") return; + var ts = (window.tsjs = window.tsjs || {}); + if (ts.adInit) return; + + // Track whether the publisher disabled GPT initial load. GPT exposes no + // getter for this, so wrap pubads().disableInitialLoad() to record it. With + // initial load disabled, display() only registers a slot and the ad request + // must come from a later refresh(); adInit() reads this to refresh its own + // freshly defined slots so they are not left blank. Pushed onto the command + // queue so it runs before the publisher's own disableInitialLoad() call. + (window.googletag = window.googletag || { cmd: [] }).cmd.push(function () { + var pubads = googletag.pubads && googletag.pubads(); + if ( + !pubads || + typeof pubads.disableInitialLoad !== "function" || + pubads.__tsInitialLoadHooked + ) { + return; + } + var original = pubads.disableInitialLoad.bind(pubads); + pubads.disableInitialLoad = function () { + ts.gptInitialLoadDisabled = true; + return original(); + }; + pubads.__tsInitialLoadHooked = true; + }); + + ts.adInit = function () { + var slots = ts.adSlots || []; + var bids = ts.bids || {}; + var divToSlotId = {}; + + googletag.cmd.push(function () { + // Slots TS defined itself — tracked for SPA destroy. Publisher-owned + // slots are reused but never destroyed by TS on navigation. + var newSlots = []; + // Publisher-owned slots TS reused — refreshed to pick up server-side + // targeting. The publisher already display()ed these. + var slotsToRefresh = []; + // Element IDs of slots TS defined itself. GPT requires display() to + // register/render a freshly-defined slot; refresh() alone no-ops for a + // slot that was never displayed, so these are display()ed instead. + var slotsToDisplay = []; + slots.forEach(function (slot) { + // Resolve actual div ID: exact match first, then safe prefix scan. + // div_id in config may be a stable prefix (e.g. "ad-header-0-") when + // the suffix is dynamically generated by the framework at render time. + var el = document.getElementById(slot.div_id); + if (!el) { + var idElements = document.querySelectorAll("[id]"); + for (var i = 0; i < idElements.length; i++) { + var candidate = idElements[i]; + if ( + slot.div_id && + candidate.id.startsWith(slot.div_id) && + !candidate.id.endsWith("-container") + ) { + el = candidate; + break; + } + } + } + if (!el) return; + var actualDivId = el.id; + var b = bids[slot.id] || {}; + + var existingSlots = googletag.pubads().getSlots(); + var s = + existingSlots.find(function (gs) { + return gs.getSlotElementId() === actualDivId; + }) || null; + var tsOwned = false; + if (!s) { + // Use outer container div for TS's slot when publisher hasn't defined + // theirs yet — keeps both slots on separate divs so publisher's + // later defineSlot on the inner div doesn't conflict. + var containerEl = document.getElementById(actualDivId + "-container"); + var slotDivId = containerEl ? containerEl.id : actualDivId; + s = googletag.defineSlot(slot.gam_unit_path, slot.formats, slotDivId); + if (!s) return; + s.addService(googletag.pubads()); + tsOwned = true; + } + + Object.entries(slot.targeting || {}).forEach(function (e) { + s.setTargeting(e[0], e[1]); + }); + [ + "hb_pb", + "hb_bidder", + "hb_adid", + "hb_cache_host", + "hb_cache_path", + ].forEach(function (k) { + if (b[k]) s.setTargeting(k, b[k]); + }); + // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts + s.setTargeting("ts_initial", "1"); + // Map both the inner div and the GPT slot's element ID (the + // "-container" div when TS defined the slot there) into divToSlotId. + // This bootstrap fires no beacons and registers no slotRenderEnded + // listener; the map is consumed by the bundle's render bridge (index.ts) + // once it loads, which reports the GPT slot element ID. + divToSlotId[actualDivId] = slot.id; + var slotElementId = s.getSlotElementId(); + if (slotElementId && slotElementId !== actualDivId) { + divToSlotId[slotElementId] = slot.id; + } + if (tsOwned) { + newSlots.push(s); + var displayId = s.getSlotElementId() || actualDivId; + slotsToDisplay.push(displayId); + } else { + slotsToRefresh.push(s); + } + }); + ts.prevGptSlots = newSlots; + ts.divToSlotId = divToSlotId; + if (!ts.servicesEnabled) { + googletag.pubads().enableSingleRequest(); + googletag.enableServices(); + ts.servicesEnabled = true; + } + // Register and render TS-defined slots. GPT requires display() for a + // freshly-defined slot; without it the slot no-ops and misses its + // impression. Runs after enableServices(); on SPA navigation services are + // already enabled, so this runs unconditionally for new slots. + slotsToDisplay.forEach(function (divId) { + googletag.display(divId); + }); + // Reused publisher-owned slots always need a refresh to pick up the + // server-side targeting. TS-defined slots are fetched by display() above + // unless the publisher disabled initial load, in which case display() only + // registers them and refresh() must request the ad — otherwise they render + // blank. Only add them in that case to avoid double-requesting. + var slotsNeedingRefresh = ts.gptInitialLoadDisabled + ? slotsToRefresh.concat(newSlots) + : slotsToRefresh; + if (slotsNeedingRefresh.length > 0) { + // One-shot bypass: this internal refresh delivers the just-applied + // server-side targeting to GAM. If slim-Prebid has already wrapped + // refresh(), it must pass this call straight through — not clear the + // targeting and run a duplicate client-side auction. Mirrors the + // bundle's adInit() in crates/trusted-server-js/lib/src/integrations/gpt/index.ts. + ts.adInitRefreshInProgress = true; + try { + googletag.pubads().refresh(slotsNeedingRefresh); + } finally { + ts.adInitRefreshInProgress = false; + } + } + }); + }; +})(); diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 3678f949c..7777b79d4 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -152,6 +152,7 @@ fn integration_backend_spec( host_header_override: None, certificate_check, first_byte_timeout, + between_bytes_timeout: first_byte_timeout, }) } diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index d2bceb43a..dd73dabae 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -9,7 +9,7 @@ use http::header::HeaderValue; use http::{header, Method, StatusCode}; use serde::{Deserialize, Serialize}; use serde_json::Value as Json; -use url::Url; +use url::Url as ParsedUrl; use validator::Validate; use crate::auction::provider::AuctionProvider; @@ -29,8 +29,8 @@ use crate::integrations::{ }; use crate::openrtb::{ to_openrtb_i32, Banner, ConsentedProvidersSettings, Device, Format, Geo, Imp, ImpExt, - OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, RequestExt, Site, ToExt, - TrustedServerExt, User, UserExt, + ImpStoredRequest, OpenRtbRequest, PrebidExt, PrebidImpExt, Publisher, Regs, RegsExt, + RequestExt, Site, ToExt, TrustedServerExt, User, UserExt, }; use crate::platform::{ PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, RuntimeServices, @@ -184,6 +184,21 @@ pub struct PrebidIntegrationConfig { /// - `both` — consent in both cookies and body (default) #[serde(default)] pub consent_forwarding: ConsentForwardingMode, + /// Strip `nurl` and `burl` from PBS bids before they reach `window.tsjs.bids`. + /// + /// Set to `true` when the PBS deployment is configured to fire win/billing + /// notifications server-side (e.g. `ext.prebid.events.enabled`), so the + /// client does not double-fire them via `sendBeacon`. Default: `false`. + #[serde(default)] + pub suppress_nurl: bool, + /// Bidder seats whose `nurl` and `burl` should be stripped before they reach + /// `window.tsjs.bids`. + /// + /// Use this when only specific PBS seats fire win/billing notifications + /// internally. The global [`suppress_nurl`](Self::suppress_nurl) switch still + /// suppresses every bidder when set. + #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] + pub suppress_nurl_bidders: Vec, } impl IntegrationConfig for PrebidIntegrationConfig { @@ -308,9 +323,9 @@ impl PrebidIntegration { } let parsed = if without_query.starts_with("//") { - Url::parse(&format!("https:{without_query}")) + ParsedUrl::parse(&format!("https:{without_query}")) } else { - Url::parse(without_query) + ParsedUrl::parse(without_query) }; parsed @@ -856,6 +871,12 @@ fn non_empty_override_object( /// Copies browser headers to the outgoing Prebid Server request. /// +/// The inbound `X-Forwarded-For` header is never copied: on the server-side +/// publisher auction path `from` is the browser navigation request, so its +/// XFF value is client-supplied and spoofable. Instead the header is +/// synthesized from `client_ip` — the platform-attested client address — +/// matching the trusted IP already sent in `OpenRTB` `device.ip`. +/// /// In [`ConsentForwardingMode::OpenrtbOnly`] mode, consent cookies are /// stripped from the `Cookie` header since consent travels exclusively /// through the `OpenRTB` body. @@ -863,13 +884,9 @@ fn copy_request_headers( from: &http::Request, to: &mut http::Request, consent_forwarding: ConsentForwardingMode, + client_ip: Option, ) { - let headers_to_copy = [ - header::USER_AGENT, - header::HeaderName::from_static("x-forwarded-for"), - header::REFERER, - header::ACCEPT_LANGUAGE, - ]; + let headers_to_copy = [header::USER_AGENT, header::REFERER, header::ACCEPT_LANGUAGE]; for header_name in &headers_to_copy { if let Some(value) = from.headers().get(header_name) { @@ -877,6 +894,13 @@ fn copy_request_headers( } } + if let Some(ip) = client_ip { + if let Ok(value) = HeaderValue::from_str(&ip.to_string()) { + to.headers_mut() + .insert(header::HeaderName::from_static("x-forwarded-for"), value); + } + } + let Some(cookie_value) = from.headers().get(header::COOKIE) else { return; }; @@ -946,6 +970,21 @@ impl PrebidAuctionProvider { }) } + /// Returns the full Prebid Server `OpenRTB2` auction endpoint URL. + /// + /// Backward-compatible normalization: `server_url` may be configured as + /// either the PBS origin (path is appended here) or the full endpoint + /// already ending in `/openrtb2/auction` (used as-is, ignoring a trailing + /// slash) — both shapes produce the same request URL. + fn auction_endpoint_url(&self) -> String { + let base = self.config.server_url.trim_end_matches('/'); + if base.ends_with("/openrtb2/auction") { + base.to_string() + } else { + format!("{base}/openrtb2/auction") + } + } + /// Convert auction request to `OpenRTB` format with all enrichments. fn to_openrtb( &self, @@ -997,22 +1036,45 @@ impl PrebidAuctionProvider { // Build the bidder map for PBS. // The JS adapter sends "trustedServer" as the bidder (our orchestrator // adapter name). Replace it with the real PBS bidders from config. - // Pass through any other bidders with their params as-is. + // Only pass through keys that are known PBS bidders — skip provider-specific + // keys like "aps" which belong to their own separate auction provider. let mut bidder: HashMap = HashMap::new(); for (name, params) in &slot.bidders { if name == TRUSTED_SERVER_BIDDER { bidder.extend(expand_trusted_server_bidders(&self.config.bidders, params)); - } else { + } else if self.config.bidders.iter().any(|b| b == name) { bidder.insert(name.clone(), params.clone()); + } else if name != "aps" { + // `aps` is intentionally handled by its own provider. Any + // other unrecognized key is likely a misconfiguration (a + // slot bidder absent from `config.bidders`) that silently + // yields an empty bidder map and a stored-request no-bid — + // log it so the drop is diagnosable. + log::debug!( + "prebid: dropping slot '{}' bidder '{}' — not in config.bidders and not a known provider key", + slot.id, + name + ); } } - // Fallback to config bidders if none provided - if bidder.is_empty() { - for b in &self.config.bidders { - bidder.insert(b.clone(), Json::Object(serde_json::Map::new())); - } - } + // When no inline PBS bidder params exist (e.g. creative-opportunity slots + // whose PBS params live in stored requests), tell PBS to resolve bidder + // config from the stored request keyed by this slot ID. + // + // This cannot fire for the client /auction path: the JS adapter + // injects a `trustedServer` entry into every ad unit, so `bidder` + // is only empty for server-side creative-opportunity slots with + // no inline provider params (or when `config.bidders` is empty, + // where PBS previously received an empty bidder map and returned + // no bids — a stored-request miss is the same no-bid outcome). + let storedrequest = if bidder.is_empty() { + Some(ImpStoredRequest { + id: slot.id.clone(), + }) + } else { + None + }; // Apply canonical and compatibility-derived rules in normalized order. for (name, params) in &mut bidder { @@ -1034,7 +1096,10 @@ impl PrebidAuctionProvider { secure: Some(true), // require HTTPS creatives tagid: Some(slot.id.clone()), ext: ImpExt { - prebid: PrebidImpExt { bidder }, + prebid: PrebidImpExt { + bidder, + storedrequest, + }, } .to_ext(), ..Default::default() @@ -1053,13 +1118,13 @@ impl PrebidAuctionProvider { // Build user object — populate consent at both OpenRTB 2.6 top-level // and Prebid ext-based locations (dual placement). - // In cookies_only mode, body consent fields are omitted — consent - // travels exclusively through the forwarded Cookie header. - let consent_ctx = if self.config.consent_forwarding.includes_body_consent() { - request.user.consent.as_ref() - } else { - None - }; + // In cookies_only mode, cookie-sourced consent travels through the + // forwarded Cookie header. KV/policy-sourced consent has no inbound + // cookie to forward, so carry it in the OpenRTB body instead. + let consent_ctx = request.user.consent.as_ref().filter(|ctx| { + self.config.consent_forwarding.includes_body_consent() + || !matches!(ctx.source, crate::consent::ConsentSource::Cookie) + }); let raw_tc = consent_ctx.and_then(|c| c.raw_tc_string.clone()); let user = Some(User { id: request.user.id.clone(), @@ -1209,7 +1274,12 @@ impl PrebidAuctionProvider { .and_then(|value| value.to_str().ok()) .map(std::string::ToString::to_string); - let tmax = to_openrtb_i32(self.config.timeout_ms, "tmax", "request"); + // Advertise the effective auction budget, not the raw provider config: + // the orchestrator caps `context.timeout_ms` to the remaining auction + // budget, and the edge backend stops waiting after that long. Telling + // PBS it has more time than the edge will wait turns partial bids into + // edge timeouts. + let tmax = to_openrtb_i32(context.timeout_ms, "tmax", "request"); OpenRtbRequest { id: Some(request.id.clone()), @@ -1390,6 +1460,15 @@ impl PrebidAuctionProvider { } } + fn should_suppress_bid_notifications(&self, bidder: &str) -> bool { + self.config.suppress_nurl + || self + .config + .suppress_nurl_bidders + .iter() + .any(|suppressed_bidder| suppressed_bidder == bidder) + } + /// Parse a single bid from `OpenRTB` response. fn parse_bid(&self, bid_obj: &Json, seat: &str) -> Result { let slot_id = bid_obj @@ -1419,15 +1498,33 @@ impl PrebidAuctionProvider { .and_then(|v| u32::try_from(v).ok()) .unwrap_or(0); - let nurl = bid_obj - .get("nurl") - .and_then(|v| v.as_str()) - .map(std::string::ToString::to_string); + let suppress_bid_notifications = self.should_suppress_bid_notifications(seat); + let nurl = if suppress_bid_notifications { + None + } else { + bid_obj + .get("nurl") + .and_then(|v| v.as_str()) + .map(std::string::ToString::to_string) + }; + + let burl = if suppress_bid_notifications { + None + } else { + bid_obj + .get("burl") + .and_then(|v| v.as_str()) + .map(std::string::ToString::to_string) + }; - let burl = bid_obj - .get("burl") + // `adid` is the creative/ad identifier. The OpenRTB `id` is the bid ID, + // not an ad ID, so it is not used as a fallback: surfacing it as `ad_id` + // (which is exposed raw in the debug bid) would mislead any consumer that + // treats `ad_id` as a creative identifier. Absent `adid`, `ad_id` is None. + let ad_id = bid_obj + .get("adid") .and_then(|v| v.as_str()) - .map(std::string::ToString::to_string); + .map(String::from); let adomain = bid_obj .get("adomain") @@ -1438,6 +1535,49 @@ impl PrebidAuctionProvider { .collect() }); + // Extract PBS Cache coordinates from ext.prebid.cache.bids + let cache_entry = bid_obj + .get("ext") + .and_then(|e| e.get("prebid")) + .and_then(|p| p.get("cache")) + .and_then(|c| c.get("bids")); + + let cache_id = cache_entry + .and_then(|c| c.get("cacheId")) + .and_then(|v| v.as_str()) + .map(String::from); + + let (cache_host, cache_path) = cache_entry + .and_then(|c| c.get("url")) + .and_then(|v| v.as_str()) + .and_then(|url_str| { + ParsedUrl::parse(url_str) + .map_err(|e| log::debug!("PBS cache URL parse failed: {}", e)) + .ok() + }) + .map(|u| { + let host = u.host_str().map(String::from); + // path() returns "/" for root — only use if non-trivial + let path = u.path().to_string(); + let path = if path.is_empty() || path == "/" { + None + } else { + Some(path) + }; + (host, path) + }) + .unwrap_or((None, None)); + + // Guard: if we extracted a cache UUID but couldn't extract the host, + // the bid will have hb_adid set but no endpoint to fetch from — creative will fail. + if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — \ + creative will fail to render for slot '{}'", + slot_id + ); + } + Ok(AuctionBid { slot_id, price: Some(price), // Prebid provides decoded prices @@ -1449,6 +1589,10 @@ impl PrebidAuctionProvider { height, nurl, burl, + ad_id, + cache_id, + cache_host, + cache_path, metadata: std::collections::HashMap::new(), }) } @@ -1513,8 +1657,8 @@ impl AuctionProvider for PrebidAuctionProvider { if log::log_enabled!(log::Level::Debug) { match serde_json::to_string_pretty(&openrtb) { Ok(json) => log::debug!( - "Prebid OpenRTB request to {}/openrtb2/auction:\n{}", - self.config.server_url, + "Prebid OpenRTB request to {}:\n{}", + self.auction_endpoint_url(), json ), Err(e) => { @@ -1526,7 +1670,7 @@ impl AuctionProvider for PrebidAuctionProvider { // Create HTTP request let mut pbs_req = http::Request::builder() .method(http::Method::POST) - .uri(format!("{}/openrtb2/auction", self.config.server_url)) + .uri(self.auction_endpoint_url()) .body(EdgeBody::empty()) .change_context(TrustedServerError::Prebid { message: "Failed to build Prebid request".to_string(), @@ -1535,6 +1679,7 @@ impl AuctionProvider for PrebidAuctionProvider { context.request, &mut pbs_req, self.config.consent_forwarding, + context.services.client_info().client_ip, ); let pbs_body = serde_json::to_vec(&openrtb).change_context(TrustedServerError::Prebid { @@ -1706,7 +1851,7 @@ mod tests { AdFormat, AdSlot, AuctionContext, AuctionRequest, DeviceInfo, PublisherInfo, UserInfo, }; - use crate::consent::ConsentContext; + use crate::consent::{ConsentContext, ConsentSource}; use crate::geo::GeoInfo; use crate::html_processor::{create_html_processor, HtmlProcessorConfig}; use crate::integrations::{ @@ -1747,6 +1892,8 @@ mod tests { bid_param_overrides: HashMap::default(), bid_param_override_rules: Vec::new(), consent_forwarding: ConsentForwardingMode::Both, + suppress_nurl: false, + suppress_nurl_bidders: Vec::new(), } } @@ -1758,10 +1905,11 @@ mod tests { spec: &PlatformBackendSpec, ) -> Result> { Ok(format!( - "predicted_{}_{}_{}", + "predicted_{}_{}_{}_{}", spec.scheme, spec.host, - spec.first_byte_timeout.as_millis() + spec.first_byte_timeout.as_millis(), + spec.between_bytes_timeout.as_millis() )) } @@ -1797,8 +1945,8 @@ mod tests { .expect("should predict backend name through platform backend"); assert_eq!( - backend_name, "predicted_https_prebid.example_123", - "should use PlatformBackend::predict_name instead of duplicating the naming scheme" + backend_name, "predicted_https_prebid.example_123_123", + "should cap both first-byte and between-bytes timeouts to the auction budget" ); } @@ -2608,6 +2756,49 @@ server_url = "https://prebid.example" ); } + #[test] + fn to_openrtb_includes_kv_consent_when_cookies_only_has_no_cookie_to_forward() { + let mut config = base_config(); + config.consent_forwarding = ConsentForwardingMode::CookiesOnly; + let provider = PrebidAuctionProvider::new(config); + let mut auction_request = create_test_auction_request(); + auction_request.user.consent = Some(ConsentContext { + raw_tc_string: Some("BOkv-backed-consent-string".to_string()), + raw_us_privacy: Some("1YNN".to_string()), + gdpr_applies: true, + source: ConsentSource::KvStore, + ..Default::default() + }); + + let settings = make_settings(); + let request = build_test_request(); + assert!( + !request.headers().contains_key(header::COOKIE), + "test request should not carry a consent cookie to forward" + ); + let context = create_test_auction_context(&settings, &request); + + let openrtb = provider.to_openrtb( + &auction_request, + &context, + None, + make_request_info(&context), + ); + + assert_eq!( + openrtb.user.as_ref().and_then(|u| u.consent.as_deref()), + Some("BOkv-backed-consent-string"), + "cookies_only should fall back to body consent when consent came from KV" + ); + let regs = openrtb.regs.as_ref().expect("should include consent regs"); + assert_eq!(regs.gdpr, Some(true), "should carry GDPR applicability"); + assert_eq!( + regs.us_privacy.as_deref(), + Some("1YNN"), + "should carry non-cookie consent strings from KV" + ); + } + #[test] fn to_openrtb_sets_gdpr_true_for_non_eu_country_with_consent() { // When geo says non-GDPR but a consent string is present, the consent @@ -3156,7 +3347,7 @@ server_url = "https://prebid.example" assert_eq!( openrtb.tmax, Some(1000), - "should set tmax from config timeout_ms" + "should set tmax from the effective auction context timeout" ); assert_eq!( openrtb.cur, @@ -3166,15 +3357,72 @@ server_url = "https://prebid.example" } #[test] - fn to_openrtb_omits_tmax_when_timeout_exceeds_i32_max() { + fn auction_endpoint_url_appends_path_to_base_origin() { + let provider = PrebidAuctionProvider::new(base_config()); + assert_eq!( + provider.auction_endpoint_url(), + "https://prebid.example/openrtb2/auction", + "should append /openrtb2/auction to a base origin" + ); + } + + #[test] + fn auction_endpoint_url_does_not_double_append_full_endpoint() { let mut config = base_config(); - config.timeout_ms = i32::MAX as u32 + 1; + config.server_url = "https://prebid.example/openrtb2/auction".to_string(); + let provider = PrebidAuctionProvider::new(config); + assert_eq!( + provider.auction_endpoint_url(), + "https://prebid.example/openrtb2/auction", + "should use a full endpoint URL as-is" + ); + + let mut config = base_config(); + config.server_url = "https://prebid.example/openrtb2/auction/".to_string(); + let provider = PrebidAuctionProvider::new(config); + assert_eq!( + provider.auction_endpoint_url(), + "https://prebid.example/openrtb2/auction", + "should normalize a trailing slash on a full endpoint URL" + ); + } + + #[test] + fn to_openrtb_tmax_uses_effective_context_timeout_not_provider_config() { + // Provider config says 1000ms but the auction budget is only 500ms — + // PBS must be told the tighter effective deadline, otherwise the edge + // gives up before PBS responds. + let config = base_config(); + assert_eq!(config.timeout_ms, 1000, "should start from 1000ms config"); let provider = PrebidAuctionProvider::new(config); let auction_request = create_test_auction_request(); let settings = make_settings(); let request = build_test_request(); - let context = create_test_auction_context(&settings, &request); + let context = shared_test_auction_context(&settings, &request, 500); + + let openrtb = provider.to_openrtb( + &auction_request, + &context, + None, + make_request_info(&context), + ); + + assert_eq!( + openrtb.tmax, + Some(500), + "should set tmax from the effective auction context timeout, not provider config" + ); + } + + #[test] + fn to_openrtb_omits_tmax_when_timeout_exceeds_i32_max() { + let provider = PrebidAuctionProvider::new(base_config()); + let auction_request = create_test_auction_request(); + + let settings = make_settings(); + let request = build_test_request(); + let context = shared_test_auction_context(&settings, &request, i32::MAX as u32 + 1); let openrtb = provider.to_openrtb( &auction_request, @@ -4520,6 +4768,92 @@ set = { placementId = "explicit_header" } assert_eq!(statuses[1]["status"], "timeout"); } + // ======================================================================== + // PBS stored request tests + // ======================================================================== + + #[test] + fn to_openrtb_uses_stored_request_when_slot_has_no_pbs_bidder_params() { + // Slot only has "aps" provider — not a PBS bidder + let slot = make_slot( + "atf_sidebar_ad", + HashMap::from([("aps".to_string(), json!({"slotID": "aps-slot-atf-sidebar"}))]), + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(base_config(), &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("bidder").is_none(), + "should not send inline bidder params when using stored request" + ); + assert_eq!( + prebid["storedrequest"]["id"], "atf_sidebar_ad", + "should use slot id as stored request id" + ); + } + + #[test] + fn to_openrtb_uses_stored_request_when_slot_has_empty_bidders() { + let slot = make_slot("homepage_header_ad", HashMap::new()); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(base_config(), &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert_eq!( + prebid["storedrequest"]["id"], "homepage_header_ad", + "should use slot id as stored request id for slot with no bidder map" + ); + } + + #[test] + fn to_openrtb_uses_inline_bidder_params_not_stored_request_for_trusted_server_slots() { + let mut config = base_config(); + config.bidders = vec!["kargo".to_string()]; + + let slot = make_ts_slot( + "in_content_ad", + &json!({ "kargo": { "placementId": "client_123" } }), + None, + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config, &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("storedrequest").is_none(), + "should not use stored request when inline bidder params are present" + ); + assert_eq!( + prebid["bidder"]["kargo"]["placementId"], "client_123", + "should use inline bidder params from trustedServer expansion" + ); + } + + #[test] + fn to_openrtb_skips_aps_key_from_slot_bidders_in_pbs_request() { + let slot = make_slot( + "atf_sidebar_ad", + HashMap::from([("aps".to_string(), json!({"slotID": "aps-slot-atf-sidebar"}))]), + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(base_config(), &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("bidder").is_none(), + "should not forward aps key into PBS imp.ext.prebid.bidder" + ); + } + #[test] fn register_rejects_invalid_bid_param_override_rule() { let toml = format!( @@ -4567,4 +4901,290 @@ set = { networkId = 42 } "should fail fast when a canonical rule has no matcher fields" ); } + + #[test] + fn parse_bid_extracts_cache_id_from_ext_prebid_cache_bids() { + let bid_json = serde_json::json!({ + "id": "bid-id-123", + "impid": "atf_sidebar_ad", + "price": 1.50, + "adm": "
ad
", + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://openads.adsrvr.org/cache?uuid=f47447a0-b759-4f2f-9887-af458b79b570", + "cacheId": "f47447a0-b759-4f2f-9887-af458b79b570" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "thetradedesk") + .expect("should parse bid"); + assert_eq!( + bid.cache_id.as_deref(), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should extract cacheId as cache_id" + ); + assert_eq!( + bid.cache_host.as_deref(), + Some("openads.adsrvr.org"), + "should extract host from cache URL" + ); + assert_eq!( + bid.cache_path.as_deref(), + Some("/cache"), + "should extract path from cache URL" + ); + } + + #[test] + fn parse_bid_sets_cache_fields_to_none_when_no_cache_entry() { + let bid_json = serde_json::json!({ + "id": "bid-id-456", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250 + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert!(bid.cache_id.is_none(), "should be None when cache absent"); + assert!(bid.cache_host.is_none(), "should be None when cache absent"); + assert!(bid.cache_path.is_none(), "should be None when cache absent"); + } + + #[test] + fn parse_bid_handles_malformed_cache_url_gracefully() { + let bid_json = serde_json::json!({ + "id": "bid-id-789", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "not-a-valid-url", + "cacheId": "some-uuid" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid without panicking"); + assert_eq!( + bid.cache_id.as_deref(), + Some("some-uuid"), + "should still extract cacheId even if URL is malformed" + ); + assert!( + bid.cache_host.is_none(), + "should be None when URL parse fails" + ); + assert!( + bid.cache_path.is_none(), + "should be None when URL parse fails" + ); + } + + #[test] + fn parse_bid_includes_nurl_and_burl_by_default() { + let bid_json = serde_json::json!({ + "impid": "atf_sidebar_ad", + "price": 1.50, + "w": 300, + "h": 250, + "nurl": "https://ssp.example/win?id=abc123", + "burl": "https://ssp.example/bill?id=abc123" + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert_eq!( + bid.nurl.as_deref(), + Some("https://ssp.example/win?id=abc123"), + "should include nurl when suppress_nurl is false" + ); + assert_eq!( + bid.burl.as_deref(), + Some("https://ssp.example/bill?id=abc123"), + "should include burl when suppress_nurl is false" + ); + } + + #[test] + fn parse_bid_strips_nurl_and_burl_when_suppress_nurl_enabled() { + let bid_json = serde_json::json!({ + "impid": "atf_sidebar_ad", + "price": 1.50, + "w": 300, + "h": 250, + "nurl": "https://ssp.example/win?id=abc123", + "burl": "https://ssp.example/bill?id=abc123" + }); + let config = PrebidIntegrationConfig { + suppress_nurl: true, + ..base_config() + }; + let provider = PrebidAuctionProvider::new(config); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert_eq!( + bid.nurl, None, + "should strip nurl when suppress_nurl is true" + ); + assert_eq!( + bid.burl, None, + "should strip burl when suppress_nurl is true" + ); + } + + #[test] + fn parse_bid_strips_nurl_and_burl_for_configured_suppressed_bidder_only() { + let bid_json = serde_json::json!({ + "impid": "atf_sidebar_ad", + "price": 1.50, + "w": 300, + "h": 250, + "nurl": "https://ssp.example/win?id=abc123", + "burl": "https://ssp.example/bill?id=abc123" + }); + let config = PrebidIntegrationConfig { + suppress_nurl_bidders: vec!["appnexus".to_string()], + ..base_config() + }; + let provider = PrebidAuctionProvider::new(config); + + let suppressed_bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse suppressed bidder bid"); + let preserved_bid = provider + .parse_bid(&bid_json, "openx") + .expect("should parse unsuppressed bidder bid"); + + assert_eq!( + suppressed_bid.nurl, None, + "should strip nurl only for the configured bidder" + ); + assert_eq!( + suppressed_bid.burl, None, + "should strip burl only for the configured bidder" + ); + assert_eq!( + preserved_bid.nurl.as_deref(), + Some("https://ssp.example/win?id=abc123"), + "should preserve nurl for bidders not configured for suppression" + ); + assert_eq!( + preserved_bid.burl.as_deref(), + Some("https://ssp.example/bill?id=abc123"), + "should preserve burl for bidders not configured for suppression" + ); + } + + #[test] + fn parse_bid_preserves_ad_id_alongside_cache_id() { + let bid_json = serde_json::json!({ + "id": "bid-impression-id", + "impid": "atf_sidebar_ad", + "adid": "bidder-ad-id-abc", + "price": 1.0, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://cache.example.com/cache", + "cacheId": "cache-uuid-xyz" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert_eq!( + bid.ad_id.as_deref(), + Some("bidder-ad-id-abc"), + "should keep ad_id from adid field" + ); + assert_eq!( + bid.cache_id.as_deref(), + Some("cache-uuid-xyz"), + "should extract cache UUID separately" + ); + } + + #[test] + fn copy_request_headers_replaces_client_supplied_xff_with_attested_ip() { + let from = http::Request::builder() + .uri("https://publisher.example.com/") + .header("x-forwarded-for", "6.6.6.6") + .header(header::USER_AGENT, "test-agent") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let mut to = http::Request::builder() + .uri("https://pbs.example.com/openrtb2/auction") + .body(EdgeBody::empty()) + .expect("should build outbound request"); + + copy_request_headers( + &from, + &mut to, + ConsentForwardingMode::Both, + Some(std::net::IpAddr::from([203, 0, 113, 7])), + ); + + assert_eq!( + to.headers() + .get("x-forwarded-for") + .and_then(|v| v.to_str().ok()), + Some("203.0.113.7"), + "should synthesize XFF from the platform-attested client IP, not the spoofable inbound header" + ); + assert_eq!( + to.headers() + .get(header::USER_AGENT) + .and_then(|v| v.to_str().ok()), + Some("test-agent"), + "should still copy the browser User-Agent" + ); + } + + #[test] + fn copy_request_headers_omits_xff_without_attested_client_ip() { + let from = http::Request::builder() + .uri("https://publisher.example.com/") + .header("x-forwarded-for", "6.6.6.6") + .body(EdgeBody::empty()) + .expect("should build inbound request"); + let mut to = http::Request::builder() + .uri("https://pbs.example.com/openrtb2/auction") + .body(EdgeBody::empty()) + .expect("should build outbound request"); + + copy_request_headers(&from, &mut to, ConsentForwardingMode::Both, None); + + assert!( + !to.headers().contains_key("x-forwarded-for"), + "should not forward the client-supplied XFF when no attested IP exists" + ); + } } diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 103657e3e..22e6f4e7d 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1131,6 +1131,14 @@ impl IntegrationRegistry { .collect() } + #[cfg(test)] + #[must_use] + pub fn empty_for_tests() -> Self { + Self { + inner: Arc::new(IntegrationRegistryInner::default()), + } + } + #[cfg(test)] #[must_use] pub fn from_rewriters( diff --git a/crates/trusted-server-core/src/integrations/sourcepoint.rs b/crates/trusted-server-core/src/integrations/sourcepoint.rs index 644d9da5b..9a63b1930 100644 --- a/crates/trusted-server-core/src/integrations/sourcepoint.rs +++ b/crates/trusted-server-core/src/integrations/sourcepoint.rs @@ -1449,9 +1449,9 @@ mod tests { let integration = SourcepointIntegration::new(Arc::new(config(true))); let document_state = IntegrationDocumentState::default(); let ctx = IntegrationHtmlContext { - request_host: "ts.prospecta.com", + request_host: "ts.example.com", request_scheme: "https", - origin_host: "origin.prospecta.com", + origin_host: "origin.example.com", document_state: &document_state, }; diff --git a/crates/trusted-server-core/src/lib.rs b/crates/trusted-server-core/src/lib.rs index c177544f1..70a4d6cfd 100644 --- a/crates/trusted-server-core/src/lib.rs +++ b/crates/trusted-server-core/src/lib.rs @@ -42,6 +42,7 @@ pub mod consent_config; pub mod constants; pub mod cookies; pub mod creative; +pub mod creative_opportunities; pub mod ec; pub(crate) mod edge_cookie; pub mod error; @@ -54,10 +55,12 @@ pub mod integrations; pub mod models; pub mod openrtb; pub mod platform; +pub mod price_bucket; pub mod proxy; pub mod publisher; pub mod redacted; pub mod request_signing; +pub mod response_privacy; pub mod rsc_flight; pub(crate) mod s3_sigv4; pub mod settings; diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index a9f02adf3..c0fc5b8d7 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -65,7 +65,7 @@ pub struct ConsentedProvidersSettings { } /// An Extended User ID entry from an identity provider. -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Eid { /// Identity provider domain (e.g. `"id5-sync.com"`). pub source: String, @@ -74,7 +74,7 @@ pub struct Eid { } /// A single user identifier within an [`Eid`] entry. -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Uid { /// The identifier value. pub id: String, @@ -152,9 +152,21 @@ pub struct ImpExt { impl ToExt for ImpExt {} -#[derive(Debug, Serialize)] +#[derive(Debug, Default, Serialize)] pub struct PrebidImpExt { + #[serde(skip_serializing_if = "std::collections::HashMap::is_empty")] pub bidder: std::collections::HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub storedrequest: Option, +} + +/// PBS imp-level stored request reference. +/// +/// PBS merges the stored imp JSON (keyed by `id`) into the outgoing request, +/// populating bidder params that are not sent inline. +#[derive(Debug, Serialize)] +pub struct ImpStoredRequest { + pub id: String, } #[derive(Debug, Serialize)] diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 032859b8f..ee7201fb8 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -1,15 +1,12 @@ use std::collections::{HashMap, VecDeque}; use std::net::IpAddr; use std::sync::{Arc, Mutex}; -use std::time::Duration; use base64::{engine::general_purpose, Engine as _}; use ed25519_dalek::SigningKey; use error_stack::{Report, ResultExt as _}; use rand::rngs::OsRng; -use edgezero_core::key_value_store::{KvError, KvPage, KvStore as PlatformKvStore}; - use super::{ ClientInfo, GeoInfo, PlatformBackend, PlatformBackendSpec, PlatformConfigStore, PlatformError, PlatformGeo, PlatformHttpClient, PlatformHttpRequest, PlatformImageOptimizerOptions, @@ -559,67 +556,6 @@ impl PlatformHttpClient for StubHttpClient { } } -// --------------------------------------------------------------------------- -// RecordingKvStore -// --------------------------------------------------------------------------- - -/// Test stub for [`PlatformKvStore`] that records `delete()` keys for assertion. -/// -/// All other operations are no-ops: reads return `Ok(None)`, writes return `Ok(())`. -pub(crate) struct RecordingKvStore { - deleted: Mutex>, -} - -impl RecordingKvStore { - pub(crate) fn new() -> Self { - Self { - deleted: Mutex::new(Vec::new()), - } - } - - /// Return the keys passed to `delete()`, in call order. - pub(crate) fn deleted_keys(&self) -> Vec { - self.deleted.lock().expect("should lock deleted").clone() - } -} - -#[async_trait::async_trait(?Send)] -impl PlatformKvStore for RecordingKvStore { - async fn get_bytes(&self, _key: &str) -> Result, KvError> { - Ok(None) - } - - async fn put_bytes(&self, _key: &str, _value: bytes::Bytes) -> Result<(), KvError> { - Ok(()) - } - - async fn put_bytes_with_ttl( - &self, - _key: &str, - _value: bytes::Bytes, - _ttl: Duration, - ) -> Result<(), KvError> { - Ok(()) - } - - async fn delete(&self, key: &str) -> Result<(), KvError> { - self.deleted - .lock() - .expect("should lock deleted") - .push(key.to_owned()); - Ok(()) - } - - async fn list_keys_page( - &self, - _prefix: &str, - _cursor: Option<&str>, - _limit: usize, - ) -> Result { - Ok(KvPage::default()) - } -} - pub(crate) struct NoopGeo; impl PlatformGeo for NoopGeo { @@ -919,6 +855,7 @@ mod tests { host_header_override: None, certificate_check: true, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }; let name = stub.ensure(&spec).expect("should return a backend name"); assert_eq!(name, "stub-backend", "should return fixed name"); diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index 77f7d6c5e..23f57a580 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -139,6 +139,8 @@ pub struct PlatformBackendSpec { pub certificate_check: bool, /// Maximum time to wait for the first response byte. pub first_byte_timeout: Duration, + /// Maximum time to wait between response body bytes. + pub between_bytes_timeout: Duration, } /// Cloneable container of platform services for a single request. diff --git a/crates/trusted-server-core/src/price_bucket.rs b/crates/trusted-server-core/src/price_bucket.rs new file mode 100644 index 000000000..30b7430de --- /dev/null +++ b/crates/trusted-server-core/src/price_bucket.rs @@ -0,0 +1,171 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, Default)] +#[serde(rename_all = "lowercase")] +pub enum PriceGranularity { + Low, + Medium, + #[default] + Dense, + High, + Auto, +} + +/// Convert a CPM in dollars to whole cents, flooring to the cent. +/// +/// Multiplying by 100 and flooring directly under-buckets common CPMs because +/// many two-decimal values are not exactly representable in binary floating +/// point: `0.29 * 100.0` is `28.999…`, which would truncate to `28` ("0.28"). +/// A tiny epsilon corrects values sitting an ULP below a cent boundary without +/// promoting genuinely sub-cent values — `0.015` (`1.4999…`) still floors to +/// `1` ("0.01"), while `0.29` correctly yields `29`. +fn cpm_to_cents(cpm: f64) -> u64 { + const CENT_EPSILON: f64 = 1e-6; + (cpm * 100.0 + CENT_EPSILON).floor() as u64 +} + +#[must_use] +pub fn price_bucket(cpm: f64, granularity: PriceGranularity) -> String { + // Reject NaN / Inf early so the cast in `cpm_to_cents` can never see a + // non-finite value (the cast's behaviour for NaN/Inf is implementation- + // defined in Rust and "saturate to 0" only by convention). + if !cpm.is_finite() || cpm <= 0.0 { + return "0.00".to_string(); + } + match granularity { + PriceGranularity::Low => { + let cents = cpm_to_cents(cpm.min(5.0)); + let bucketed_cents = (cents / 50) * 50; + format!("{:.2}", bucketed_cents as f64 / 100.0) + } + PriceGranularity::Medium => { + let cents = cpm_to_cents(cpm.min(20.0)); + let bucketed_cents = (cents / 10) * 10; + format!("{:.2}", bucketed_cents as f64 / 100.0) + } + PriceGranularity::High => { + let cents = cpm_to_cents(cpm.min(20.0)); + format!("{:.2}", cents as f64 / 100.0) + } + PriceGranularity::Dense | PriceGranularity::Auto => dense_bucket(cpm), + } +} + +fn dense_bucket(cpm: f64) -> String { + if cpm >= 20.0 { + return "20.00".to_string(); + } + if cpm >= 8.0 { + let bucketed_cents = (cpm_to_cents(cpm) / 50) * 50; + return format!("{:.2}", bucketed_cents as f64 / 100.0); + } + if cpm >= 3.0 { + let bucketed_cents = (cpm_to_cents(cpm) / 5) * 5; + return format!("{:.2}", bucketed_cents as f64 / 100.0); + } + format!("{:.2}", cpm_to_cents(cpm) as f64 / 100.0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dense_below_3_increments_by_0_01() { + assert_eq!(price_bucket(0.0, PriceGranularity::Dense), "0.00"); + assert_eq!(price_bucket(0.01, PriceGranularity::Dense), "0.01"); + assert_eq!(price_bucket(0.015, PriceGranularity::Dense), "0.01"); + assert_eq!(price_bucket(1.23, PriceGranularity::Dense), "1.23"); + assert_eq!(price_bucket(2.99, PriceGranularity::Dense), "2.99"); + } + + #[test] + fn dense_3_to_8_increments_by_0_05() { + assert_eq!(price_bucket(3.00, PriceGranularity::Dense), "3.00"); + assert_eq!(price_bucket(3.03, PriceGranularity::Dense), "3.00"); + assert_eq!(price_bucket(3.05, PriceGranularity::Dense), "3.05"); + assert_eq!(price_bucket(7.99, PriceGranularity::Dense), "7.95"); + } + + #[test] + fn dense_8_to_20_increments_by_0_50() { + assert_eq!(price_bucket(8.00, PriceGranularity::Dense), "8.00"); + assert_eq!(price_bucket(8.49, PriceGranularity::Dense), "8.00"); + assert_eq!(price_bucket(8.50, PriceGranularity::Dense), "8.50"); + assert_eq!(price_bucket(19.99, PriceGranularity::Dense), "19.50"); + } + + #[test] + fn dense_above_20_caps_at_20() { + assert_eq!(price_bucket(20.00, PriceGranularity::Dense), "20.00"); + assert_eq!(price_bucket(50.00, PriceGranularity::Dense), "20.00"); + } + + #[test] + fn low_increments_by_0_50_capped_at_5() { + assert_eq!(price_bucket(0.49, PriceGranularity::Low), "0.00"); + assert_eq!(price_bucket(0.50, PriceGranularity::Low), "0.50"); + assert_eq!(price_bucket(5.01, PriceGranularity::Low), "5.00"); + } + + #[test] + fn medium_increments_by_0_10_capped_at_20() { + assert_eq!(price_bucket(1.05, PriceGranularity::Medium), "1.00"); + assert_eq!(price_bucket(1.10, PriceGranularity::Medium), "1.10"); + assert_eq!(price_bucket(20.5, PriceGranularity::Medium), "20.00"); + } + + #[test] + fn high_increments_by_0_01_capped_at_20() { + assert_eq!(price_bucket(1.234, PriceGranularity::High), "1.23"); + assert_eq!(price_bucket(20.5, PriceGranularity::High), "20.00"); + } + + #[test] + fn auto_routes_through_dense() { + assert_eq!( + price_bucket(2.53, PriceGranularity::Auto), + price_bucket(2.53, PriceGranularity::Dense) + ); + } + + #[test] + fn float_boundary_cpms_are_not_under_bucketed() { + // These two-decimal CPMs are not exactly representable in binary float + // (`0.29 * 100.0 == 28.999…`); a naive floor truncates them a cent low. + assert_eq!(price_bucket(0.29, PriceGranularity::Dense), "0.29"); + assert_eq!(price_bucket(1.15, PriceGranularity::Dense), "1.15"); + assert_eq!(price_bucket(0.29, PriceGranularity::High), "0.29"); + assert_eq!(price_bucket(1.15, PriceGranularity::High), "1.15"); + // Genuinely sub-cent values must still floor, not round up. + assert_eq!(price_bucket(0.289, PriceGranularity::High), "0.28"); + assert_eq!(price_bucket(0.015, PriceGranularity::Dense), "0.01"); + } + + #[test] + fn non_finite_cpm_returns_zero_bucket() { + for granularity in [ + PriceGranularity::Dense, + PriceGranularity::Low, + PriceGranularity::Medium, + PriceGranularity::High, + PriceGranularity::Auto, + ] { + assert_eq!( + price_bucket(f64::NAN, granularity), + "0.00", + "NaN cpm should bucket to 0.00 for granularity {granularity:?}" + ); + assert_eq!( + price_bucket(f64::INFINITY, granularity), + "0.00", + "+Inf cpm should bucket to 0.00 for granularity {granularity:?}" + ); + assert_eq!( + price_bucket(f64::NEG_INFINITY, granularity), + "0.00", + "-Inf cpm should bucket to 0.00 for granularity {granularity:?}" + ); + } + } +} diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 9bf58e630..7d830790a 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -1071,6 +1071,7 @@ pub async fn handle_asset_proxy_request( host_header_override: None, certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }) .change_context(TrustedServerError::Proxy { message: "asset backend registration failed".to_string(), @@ -1254,6 +1255,7 @@ async fn proxy_with_redirects( host_header_override: None, certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, }) .change_context(TrustedServerError::Proxy { message: "backend registration failed".to_string(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 1e41044e8..d9944ad95 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -4,34 +4,60 @@ //! the API boundary: //! - pass-through for non-processable `2xx` content //! - streamed processing for stream-safe processable responses -//! - buffered responses for unsupported encodings, `204/205`, or HTML routes -//! that require full-document post-processing +//! - buffered responses for unsupported encodings or `204/205` //! //! Unsupported `Content-Encoding` values must bypass rewriting entirely. The //! streaming processor treats unknown encodings as identity, so publisher code //! must gate them out before the body enters the rewrite pipeline. +//! +//! **Note on platform coupling:** The handler boundaries use portable HTTP +//! types: [`handle_publisher_request`] and [`stream_publisher_body`] take and +//! return `http::Request`/`http::Response` over `EdgeBody`, and platform I/O is +//! reached through `RuntimeServices` rather than `fastly::*` directly. The +//! streaming processor itself is generic: `process_response_streaming` writes +//! into any [`Write`] (a `Vec` for buffered routes, a streaming writer for +//! the streaming route). It is not a content-rewriting concern. + use std::io::Write; +use std::sync::{Arc, Mutex}; use std::time::Duration; +use cookie::CookieJar; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{header, HeaderValue, Method, Request, Response, StatusCode, Uri}; -use crate::consent::{allows_ec_creation, build_consent_context, ConsentPipelineInput}; -use crate::constants::{COOKIE_TS_EC, HEADER_X_COMPRESS_HINT, HEADER_X_TS_EC}; +use crate::auction::endpoints::{ + merge_auction_eids, resolve_auction_eids, resolve_client_auction_eids, +}; +use crate::auction::orchestrator::{AuctionOrchestrator, DispatchedAuction}; +use crate::auction::types::{ + AuctionContext, AuctionRequest, Bid, DeviceInfo, PublisherInfo, SiteInfo, UserInfo, +}; +use crate::consent::{consent_allows_server_side_auction, gate_eids_by_consent}; +use crate::constants::{COOKIE_TS_EIDS, HEADER_X_COMPRESS_HINT}; use crate::cookies::handle_request_cookies; -use crate::edge_cookie::get_or_generate_ec_id_from_http_request; +use crate::ec::kv::KvIdentityGraph; +use crate::ec::registry::PartnerRegistry; +use crate::ec::EcContext; use crate::error::TrustedServerError; -use crate::http_util::{serve_static_with_etag, RequestInfo}; +use crate::http_util::{is_navigation_request, serve_static_with_etag, RequestInfo}; use crate::integrations::IntegrationRegistry; -use crate::platform::{PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, RuntimeServices}; +use crate::price_bucket::{price_bucket, PriceGranularity}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, StreamingPipeline}; use crate::streaming_replacer::create_url_replacer; + const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); +/// Read buffer size for streaming body processing and brotli internal buffers. +/// Both the `Decompressor` and `CompressorWriter` use this value so all +/// brotli I/O layers operate on consistently-sized chunks. +const STREAM_CHUNK_SIZE: usize = 8192; + fn body_as_reader(body: EdgeBody) -> std::io::Cursor { std::io::Cursor::new(body.into_bytes().unwrap_or_default()) } @@ -187,7 +213,7 @@ fn parse_deferred_module_filename(filename: &str) -> Option<&'static str> { .find(|&id| id == stem) } -/// Parameters for processing response streaming +/// Parameters for processing response streaming. struct ProcessResponseParams<'a> { content_encoding: &'a str, origin_host: &'a str, @@ -197,6 +223,8 @@ struct ProcessResponseParams<'a> { settings: &'a Settings, content_type: &'a str, integration_registry: &'a IntegrationRegistry, + ad_slots_script: Option<&'a str>, + ad_bids_state: &'a Arc>>, } /// Process response body through the streaming pipeline. @@ -214,8 +242,9 @@ fn process_response_streaming( output: &mut W, params: &ProcessResponseParams, ) -> Result<(), Report> { - let is_html = params.content_type.contains("text/html"); - let is_rsc_flight = params.content_type.contains("text/x-component"); + let is_html = is_html_content_type(params.content_type); + let is_rsc_flight = + content_type_contains_ascii_case_insensitive(params.content_type, "text/x-component"); log::debug!( "process_response_streaming: content_type={}, content_encoding={}, is_html={}, is_rsc_flight={}", params.content_type, @@ -238,9 +267,13 @@ fn process_response_streaming( params.request_scheme, params.settings, params.integration_registry, + params.ad_slots_script.map(str::to_string), + params.ad_bids_state.clone(), )?; StreamingPipeline::new(config, processor).process(body_as_reader(body), output)?; } else if is_rsc_flight { + // RSC Flight responses are length-prefixed (T rows). A naive string replacement will + // corrupt the stream by changing byte lengths without updating the prefixes. let processor = RscFlightUrlRewriter::new( params.origin_host, params.origin_url, @@ -261,13 +294,21 @@ fn process_response_streaming( Ok(()) } -/// Create a unified HTML stream processor +/// Create a unified HTML stream processor. +/// +/// Builds the config via [`HtmlProcessorConfig::from_settings`] and then +/// layers the auction-hold streaming fields on top via +/// [`HtmlProcessorConfig::with_ad_state`], so the canonical builder stays the +/// single source of truth: a future field added to `from_settings` is +/// inherited here automatically. fn create_html_stream_processor( origin_host: &str, request_host: &str, request_scheme: &str, settings: &Settings, integration_registry: &IntegrationRegistry, + ad_slots_script: Option, + ad_bids_state: Arc>>, ) -> Result> { use crate::html_processor::{create_html_processor, HtmlProcessorConfig}; @@ -277,33 +318,39 @@ fn create_html_stream_processor( origin_host, request_host, request_scheme, - ); + ) + .with_ad_state(ad_slots_script, ad_bids_state); Ok(create_html_processor(config)) } -/// Result of publisher request handling, indicating whether the response -/// body should be streamed or has already been buffered. +/// Result of publisher request handling, indicating whether the response body +/// should be streamed or has already been buffered. pub enum PublisherResponse { /// Response is fully buffered and ready to send via `send_to_client()`. Buffered(Response), /// Response headers are ready for a streaming response. Covers processable /// content on any status (2xx or non-2xx — e.g., branded 404/500 HTML and - /// error JSON still get URL rewriting) where the encoding is supported - /// and either the content is non-HTML or no HTML post-processors are - /// registered. The caller must: + /// error JSON still get URL rewriting) where the encoding is supported. + /// Post-processors run inside the streaming processor, so processable HTML + /// is streamed regardless of whether any are registered. The caller must: /// 1. Call `finalize_response()` on the response /// 2. Call `response.stream_to_client()` to get a `StreamingBody` /// 3. Call `stream_publisher_body()` with the body and streaming writer /// 4. Call `StreamingBody::finish()` + /// + /// **Interim (PR 15):** `body` has already been fully materialised into + /// WASM heap by the platform HTTP client. `stream_publisher_body` reads + /// from an in-memory buffer, not a live origin stream. The origin-side + /// peak is bounded by `MAX_PLATFORM_RESPONSE_BODY_BYTES`. Stream { /// Response with all headers set (EC ID, cookies, etc.) /// but body not yet written. `Content-Length` already removed. response: Response, /// Origin body to be piped through the streaming pipeline. body: EdgeBody, - /// Parameters for `process_response_streaming`. - params: OwnedProcessResponseParams, + /// Parameters for [`process_response_streaming`]. + params: Box, }, /// Non-processable 2xx response (images, fonts, video). The adapter must /// reattach the body via setting the body before returning. @@ -311,6 +358,11 @@ pub enum PublisherResponse { /// response-dispatch level, not in this arm. /// /// `Content-Length` is preserved — the body is unmodified. + /// + /// **Interim (PR 15):** `body` has been fully materialised into WASM heap. + /// Previously, binary assets streamed lazily from origin with no WASM + /// buffering. This path is now bounded by `MAX_PLATFORM_RESPONSE_BODY_BYTES`; + /// assets exceeding that limit return an error instead of exhausting heap. PassThrough { /// Response with all headers set but body not yet written. response: Response, @@ -327,45 +379,30 @@ pub enum PublisherResponse { /// exercise the classifier directly so the gate formula lives in one place. #[derive(Debug, PartialEq, Eq)] pub(crate) enum ResponseRoute { - /// 2xx non-processable (images, fonts, video), not 204/205. Origin body - /// is streamed unmodified via [`PublisherResponse::PassThrough`]. + /// `2xx` non-processable content (images, fonts, video), not `204/205`. PassThrough, - /// Processable content with supported encoding and either non-HTML or no - /// HTML post-processors registered. Covers both 2xx and non-2xx (e.g., - /// branded 404/500 pages still get origin URL rewriting). Routed through - /// [`PublisherResponse::Stream`]. + /// Processable content with supported encoding. Stream, - /// Response returned unmodified via [`PublisherResponse::Buffered`] — covers - /// 204/205 (RFC-prohibited bodies), empty request host with non-processable - /// content, and unsupported encodings. + /// Response returned unmodified via [`PublisherResponse::Buffered`]. BufferedUnmodified, - /// HTML with post-processors registered. Runs the full pipeline into a - /// buffer, then returns [`PublisherResponse::Buffered`] with the processed body. - BufferedProcessed, } /// Decide how a proxied response should be routed. /// -/// Pure: no header mutation, no body consumed. All inputs are extracted -/// from the origin response at the call site. +/// Pure: no header mutation, no body consumed. All inputs are extracted from +/// the origin response at the call site. pub(crate) fn classify_response_route( status: StatusCode, content_type: &str, content_encoding: &str, request_host: &str, - has_post_processors: bool, ) -> ResponseRoute { - // 204 No Content (RFC 9110 §15.3.5) and 205 Reset Content (§15.3.6) - // prohibit a message body. Excluded first so no later arm can emit one - // regardless of Content-Type or post-processor registration. if status == StatusCode::NO_CONTENT || status == StatusCode::RESET_CONTENT { return ResponseRoute::BufferedUnmodified; } let should_process = is_processable_content_type(content_type); - // Non-processable content: 2xx streams through unchanged; non-2xx falls - // back to buffered (the origin's error body reaches the client as-is). if !should_process { if status.is_success() { return ResponseRoute::PassThrough; @@ -373,29 +410,19 @@ pub(crate) fn classify_response_route( return ResponseRoute::BufferedUnmodified; } - // Processable content (2xx or non-2xx) still needs URL rewriting against - // a known request host — without one, fall back to unmodified. if request_host.is_empty() { return ResponseRoute::BufferedUnmodified; } - // Unsupported Content-Encoding: we cannot decompress, so processing would - // treat compressed bytes as identity and produce garbled output. if !is_supported_content_encoding(content_encoding) { return ResponseRoute::BufferedUnmodified; } - let is_html = content_type.contains("text/html"); - if is_html && has_post_processors { - // HTML with post-processors: need the full document to inject. - return ResponseRoute::BufferedProcessed; - } - ResponseRoute::Stream } /// Owned version of [`ProcessResponseParams`] for returning from -/// `handle_publisher_request` without lifetime issues. +/// [`handle_publisher_request`] without lifetime issues. pub struct OwnedProcessResponseParams { pub(crate) content_encoding: String, pub(crate) origin_host: String, @@ -403,48 +430,81 @@ pub struct OwnedProcessResponseParams { pub(crate) request_host: String, pub(crate) request_scheme: String, pub(crate) content_type: String, + pub(crate) ad_slots_script: Option, + pub(crate) ad_bids_state: Arc>>, + /// In-flight SSP bids dispatched before `pending_origin.wait()`. + /// The streaming phase collects these and writes bids to `ad_bids_state` + /// before processing the last body chunk, so `` injection sees live bids. + pub(crate) dispatched_auction: Option, + /// Price granularity used to bucket bids when building `tsjs.bids`. + pub(crate) price_granularity: PriceGranularity, } -/// Buffer a [`PublisherResponse`] into a single [`Response`]. +/// Buffers a [`PublisherResponse`] into a single [`Response`], collecting the +/// dispatched server-side auction before buffering. /// /// Handles all three variants: returns [`PublisherResponse::Buffered`] unchanged, -/// pipes [`PublisherResponse::Stream`] through the streaming pipeline into memory, -/// and reattaches [`PublisherResponse::PassThrough`] bodies directly. +/// pipes [`PublisherResponse::Stream`] through the streaming pipeline into +/// memory, and reattaches [`PublisherResponse::PassThrough`] bodies directly. /// /// The buffered size is capped by `settings.publisher.max_buffered_body_bytes` -/// (16 MiB by default), so processable origin responses cannot grow the -/// buffer without bound and exhaust the Wasm heap. +/// (16 MiB by default), so processable origin responses cannot grow the buffer +/// without bound and exhaust the Wasm heap. /// -/// `method` is used to preserve metadata for bodiless responses: `HEAD` and -/// bodiless statuses (204, 304) carry no body but may advertise the `GET` -/// representation's length. `handle_publisher_request` already strips the origin -/// `Content-Length` for processable [`PublisherResponse::Stream`] responses, so -/// rewriting it here to the buffered byte count (`0`) would replace it with a -/// misleading length. Those responses skip the buffer, the length rewrite, and -/// the body replacement, mirroring the asset path's bodiless guard. +/// `method` preserves metadata for bodiless responses: `HEAD` and bodiless +/// statuses (204, 304) carry no body but may advertise the `GET` representation's +/// length, so they skip the buffer and length rewrite. +/// +/// Every adapter (Axum, Cloudflare, Spin, and the Fastly `EdgeZero` path) calls +/// this: it drives +/// [`stream_publisher_body_async`], which awaits +/// [`AuctionOrchestrator::collect_dispatched_auction`], writes the winning bids +/// into `ad_bids_state`, and injects them before ``. /// /// # Errors /// /// Returns an error if the streaming pipeline fails to process the response /// body, or if the processed body exceeds the configured buffer cap. -pub fn buffer_publisher_response( +pub async fn buffer_publisher_response_async( publisher_response: PublisherResponse, method: &Method, settings: &Settings, integration_registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, ) -> Result, Report> { match publisher_response { PublisherResponse::Buffered(response) => Ok(response), PublisherResponse::Stream { mut response, body, - params, + mut params, } => { if !response_carries_body(method, response.status()) { + if params.dispatched_auction.is_some() { + // A bodiless response (HEAD navigation, 204/304) has no + // `` to inject bids into, so the dispatched SSP + // requests are wasted — surface it for quota observability, + // matching the pass-through / buffered-unmodified arms. + log::warn!( + "Server-side auction dispatched but response is bodiless (method: {}, status: {}); in-flight SSP bid requests will not be collected", + method, + response.status(), + ); + } return Ok(response); } let mut output = BoundedWriter::new(settings.publisher.max_buffered_body_bytes); - stream_publisher_body(body, &mut output, ¶ms, settings, integration_registry)?; + stream_publisher_body_async( + body, + &mut output, + &mut params, + settings, + integration_registry, + orchestrator, + services, + ) + .await?; let bytes = output.into_inner(); response.headers_mut().insert( http::header::CONTENT_LENGTH, @@ -472,36 +532,6 @@ fn response_carries_body(method: &Method, status: StatusCode) -> bool { && status != StatusCode::NOT_MODIFIED } -/// Stream the publisher response body through the processing pipeline. -/// -/// Called by the adapter after `stream_to_client()` has committed the response -/// headers. Writes processed chunks directly to `output`. -/// -/// # Errors -/// -/// Returns an error if processing fails mid-stream. Since headers are -/// already committed, the caller should log the error and drop the -/// `StreamingBody` (client sees a truncated response). -pub fn stream_publisher_body( - body: EdgeBody, - output: &mut W, - params: &OwnedProcessResponseParams, - settings: &Settings, - integration_registry: &IntegrationRegistry, -) -> Result<(), Report> { - let borrowed = ProcessResponseParams { - content_encoding: ¶ms.content_encoding, - origin_host: ¶ms.origin_host, - origin_url: ¶ms.origin_url, - request_host: ¶ms.request_host, - request_scheme: ¶ms.request_scheme, - settings, - content_type: ¶ms.content_type, - integration_registry, - }; - process_response_streaming(body, output, &borrowed) -} - /// A [`Write`] sink that buffers into a `Vec` but fails once the configured /// byte limit would be exceeded. /// @@ -547,18 +577,602 @@ impl Write for BoundedWriter { } } +/// Stream the publisher response body through the processing pipeline. +/// +/// Called by the adapter after `stream_to_client()` has committed the response +/// headers. Runs synchronously against an already-materialised body; the async +/// I/O happened upstream in [`handle_publisher_request`]. Writes processed +/// chunks directly to `output`. +/// +/// # Errors +/// +/// Returns an error if processing fails mid-stream. Since headers are already +/// committed, the caller should log the error and drop the `StreamingBody` +/// (client sees a truncated response). +pub fn stream_publisher_body( + body: EdgeBody, + output: &mut W, + params: &OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, +) -> Result<(), Report> { + let borrowed = ProcessResponseParams { + content_encoding: ¶ms.content_encoding, + origin_host: ¶ms.origin_host, + origin_url: ¶ms.origin_url, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + settings, + content_type: ¶ms.content_type, + integration_registry, + ad_slots_script: params.ad_slots_script.as_deref(), + ad_bids_state: ¶ms.ad_bids_state, + }; + process_response_streaming(body, output, &borrowed) +} + +/// Stream publisher body with a `` handler with bids now in state. +/// +/// For non-HTML content types the auction is collected before any body bytes +/// are written (no `` to inject). If `params.dispatched_auction` is +/// `None` the function falls back to the synchronous +/// [`stream_publisher_body`] path. +/// +/// # Errors +/// +/// Returns an error if processing fails mid-stream. Headers are already +/// committed at that point; the caller logs and drops the `StreamingBody`. +pub async fn stream_publisher_body_async( + body: EdgeBody, + output: &mut W, + params: &mut OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, +) -> Result<(), Report> { + let Some(dispatched) = params.dispatched_auction.take() else { + // No auction — use the existing sync pipeline unchanged. + return stream_publisher_body(body, output, params, settings, integration_registry); + }; + + let is_html = is_html_content_type(¶ms.content_type); + + if !is_html { + // Non-HTML: collect auction first, then stream. There is no + // to hold, so delaying the entire body until collection is acceptable. + let placeholder = mediator_placeholder_request(); + let result = orchestrator + .collect_dispatched_auction( + dispatched, + services, + &make_collect_context(settings, services, &placeholder), + ) + .await; + write_bids_to_state( + &result.winning_bids, + params.price_granularity, + ¶ms.ad_bids_state, + settings.debug.inject_adm_for_testing, + ); + return stream_publisher_body(body, output, params, settings, integration_registry); + } + + // HTML: build the processor once and drive it chunk by chunk. + // One-behind buffer: stream chunk N-1 immediately; hold chunk N until origin + // EOF, then await auction and process chunk N (which contains ). + let mut processor = create_html_stream_processor( + ¶ms.origin_host, + ¶ms.request_host, + ¶ms.request_scheme, + settings, + integration_registry, + params.ad_slots_script.as_deref().map(str::to_string), + params.ad_bids_state.clone(), + )?; + + let compression = Compression::from_content_encoding(¶ms.content_encoding); + stream_html_with_auction_hold( + body, + output, + &mut processor, + compression, + AuctionCollectCtx { + dispatched, + price_granularity: params.price_granularity, + ad_bids_state: ¶ms.ad_bids_state, + orchestrator, + services, + settings, + }, + ) + .await +} + +/// Builds the canonical mediator placeholder [`Request`] passed to the collect +/// phase via [`make_collect_context`]. +/// +/// The URI is the compile-time constant +/// [`MEDIATOR_PLACEHOLDER_URL`](crate::auction::types::MEDIATOR_PLACEHOLDER_URL), +/// so the builder is infallible; a default-URI fallback would trip +/// [`make_collect_context`]'s `debug_assert_eq!`. +fn mediator_placeholder_request() -> Request { + Request::builder() + .uri(crate::auction::types::MEDIATOR_PLACEHOLDER_URL) + .body(EdgeBody::empty()) + .expect("MEDIATOR_PLACEHOLDER_URL should be a valid URI") +} + +/// Build a minimal [`AuctionContext`] for the collect phase. +/// +/// See [`AuctionContext::request`]: the orchestrator's collect path runs +/// after `send_async` has already consumed the real client request, so this +/// context carries a synthetic placeholder. The orchestrator itself +/// instantiates a fresh placeholder when it actually invokes a mediator — +/// this argument is plumbing for the (presently unused) case where the +/// orchestrator needs the caller's request shape. +fn make_collect_context<'a>( + settings: &'a Settings, + services: &'a RuntimeServices, + placeholder: &'a Request, +) -> AuctionContext<'a> { + debug_assert_eq!( + placeholder.uri().to_string(), + crate::auction::types::MEDIATOR_PLACEHOLDER_URL, + "make_collect_context must be given the canonical placeholder; \ + callers must not forward a real client request through the collect path" + ); + AuctionContext { + settings, + request: placeholder, + timeout_ms: 0, + provider_responses: None, + services, + } +} + +/// Well-known crawler User-Agent fragments. Best-effort: an attacker can +/// trivially spoof their UA, so this is for opt-out signalling to honest +/// crawlers (preventing SSP auctions burning partner quota on their behalf), +/// not security. +pub(crate) const BOT_USER_AGENT_FRAGMENTS: &[&str] = + &["Googlebot", "Bingbot", "AhrefsBot", "SemrushBot", "DotBot"]; + +/// Returns true when the request's User-Agent matches any well-known crawler +/// fragment in [`BOT_USER_AGENT_FRAGMENTS`]. +pub(crate) fn is_bot_user_agent(req: &Request) -> bool { + let ua = req + .headers() + .get("user-agent") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + BOT_USER_AGENT_FRAGMENTS + .iter() + .any(|frag| ua.contains(frag)) +} + +/// Returns true when the request advertises itself as a prefetch via either +/// the standard `Sec-Purpose` or the legacy `Purpose` header. +pub(crate) fn is_prefetch_request(req: &Request) -> bool { + let header = |name: &str| { + req.headers() + .get(name) + .and_then(|v| v.to_str().ok()) + .is_some_and(|v| v.contains("prefetch")) + }; + header("sec-purpose") || header("purpose") +} + +/// Returns true only when the publisher request should run the full +/// server-side ad stack: auction dispatch plus initial ad-slot injection. +/// +/// `auction_enabled` is the global `[auction].enabled` kill switch — when +/// false, no automatic server-side auction or ad-slot injection runs. +pub(crate) fn should_run_server_side_ad_stack( + is_get: bool, + is_navigation: bool, + is_prefetch: bool, + is_bot: bool, + has_matched_slots: bool, + consent_allows_auction: bool, + auction_enabled: bool, +) -> bool { + is_get + && is_navigation + && !is_prefetch + && !is_bot + && has_matched_slots + && consent_allows_auction + && auction_enabled +} + +/// Write winning bids from an auction result into the shared `ad_bids_state` lock. +pub(crate) fn write_bids_to_state( + winning_bids: &std::collections::HashMap, + price_granularity: PriceGranularity, + ad_bids_state: &Arc>>, + inject_adm: bool, +) { + log::debug!( + "write_bids_to_state: {} winning bid(s): [{}]", + winning_bids.len(), + winning_bids.keys().cloned().collect::>().join(", ") + ); + let bid_map = build_bid_map(winning_bids, price_granularity, inject_adm); + let bids_script = build_bids_script(&bid_map); + *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); +} + +/// Prepend an HTML comment summarising the auction result onto the shared +/// `ad_bids_state` so it lands directly before the injected bids `` injection breaking out of the script context +/// - U+2028, U+2029 — line/paragraph separators that are valid JSON but terminate +/// a JS string literal in some parsers +/// +/// All substitutions use `\uXXXX` form, which is valid inside both JSON strings +/// and JS string literals. The result is always safe to write as `JSON.parse("…")`. +fn html_escape_for_script(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for ch in s.chars() { + match ch { + '\\' => out.push_str("\\\\"), + '"' => out.push_str("\\\""), + '<' => out.push_str("\\u003C"), + '>' => out.push_str("\\u003E"), + '&' => out.push_str("\\u0026"), + '\u{2028}' => out.push_str("\\u2028"), + '\u{2029}' => out.push_str("\\u2029"), + _ => out.push(ch), } } + out +} + +/// Build a price-bucketed bid map from winning bids. +/// +/// Returns a JSON object map of slot ID → bid metadata including the bucketed +/// CPM (`hb_pb`), bidder (`hb_bidder`), and optional ad ID, nurl, and burl. +pub(crate) fn build_bid_map( + winning_bids: &std::collections::HashMap, + granularity: crate::price_bucket::PriceGranularity, + include_adm: bool, +) -> serde_json::Map { + winning_bids + .iter() + .filter_map(|(slot_id, bid)| { + bid.price.map(|cpm| { + let bucket = price_bucket(cpm, granularity); + let mut obj = serde_json::Map::new(); + obj.insert("hb_pb".to_string(), serde_json::Value::String(bucket)); + obj.insert( + "hb_bidder".to_string(), + serde_json::Value::String(bid.bidder.clone()), + ); + // hb_adid: use PBS Cache UUID when present — the Prebid Universal Creative uses + // this as the cache lookup key, NOT the OpenRTB bid ID (bid.ad_id). Fall back to + // bid.ad_id for APS and other non-PBS providers. + let hb_adid = bid.cache_id.as_deref().or(bid.ad_id.as_deref()); + if let Some(id) = hb_adid { + obj.insert( + "hb_adid".to_string(), + serde_json::Value::String(id.to_string()), + ); + } + + // Cache endpoint coordinates — only present for PBS bids with Prebid Cache enabled. + // The Prebid Universal Creative constructs: + // https://?uuid= + if let Some(ref host) = bid.cache_host { + obj.insert( + "hb_cache_host".to_string(), + serde_json::Value::String(host.clone()), + ); + } + if let Some(ref path) = bid.cache_path { + obj.insert( + "hb_cache_path".to_string(), + serde_json::Value::String(path.clone()), + ); + } + if let Some(ref nurl) = bid.nurl { + obj.insert("nurl".to_string(), serde_json::Value::String(nurl.clone())); + } + if let Some(ref burl) = bid.burl { + obj.insert("burl".to_string(), serde_json::Value::String(burl.clone())); + } + // Include raw creative markup only for explicit debug injection. + // The pbRender bridge can use it while PBS Cache is unavailable. + if include_adm { + if let Some(ref adm) = bid.creative { + obj.insert("adm".to_string(), serde_json::Value::String(adm.clone())); + } + obj.insert( + "debug_bid".to_string(), + serde_json::json!({ + "slot_id": bid.slot_id, + "price": bid.price, + "currency": bid.currency, + "creative": bid.creative, + "adomain": bid.adomain, + "bidder": bid.bidder, + "width": bid.width, + "height": bid.height, + "nurl": bid.nurl, + "burl": bid.burl, + "ad_id": bid.ad_id, + "cache_id": bid.cache_id, + "cache_host": bid.cache_host, + "cache_path": bid.cache_path, + "metadata": bid.metadata, + }), + ); + } + (slot_id.clone(), serde_json::Value::Object(obj)) + }) + }) + .collect() +} + +/// Build the `tsjs.bids` `` sequences inside the string. +pub(crate) fn build_bids_script(bid_map: &serde_json::Map) -> String { + let json = serde_json::to_string(bid_map) + .expect("serde_json::to_string of Map should be infallible"); + let escaped = html_escape_for_script(&json); + format!( + "", + escaped + ) +} + +/// Build the empty-bids `", + escaped + ) } /// Whether the content type requires processing (URL rewriting, HTML injection). @@ -829,74 +1863,298 @@ pub async fn handle_publisher_request( /// Text-based and JavaScript/JSON responses are processable; binary types /// (images, fonts, video, etc.) pass through unchanged. fn is_processable_content_type(content_type: &str) -> bool { - content_type.contains("text/") - || content_type.contains("application/javascript") - || content_type.contains("application/json") + let normalized = content_type.to_ascii_lowercase(); + normalized.contains("text/") + || normalized.contains("application/javascript") + || normalized.contains("application/json") +} + +fn is_html_content_type(content_type: &str) -> bool { + content_type_contains_ascii_case_insensitive(content_type, "text/html") +} + +fn content_type_contains_ascii_case_insensitive(content_type: &str, needle: &str) -> bool { + content_type.to_ascii_lowercase().contains(needle) } /// Whether the `Content-Encoding` is one the streaming pipeline can handle. /// /// Unsupported encodings (e.g. `zstd` from a misbehaving origin) bypass the -/// rewrite pipeline entirely and are returned unchanged. Processing such -/// bodies as identity-encoded would produce garbled output. +/// rewrite pipeline entirely and are returned unchanged. Processing such bodies +/// as identity-encoded would produce garbled output. fn is_supported_content_encoding(encoding: &str) -> bool { matches!(encoding, "" | "identity" | "gzip" | "deflate" | "br") } -/// Apply EC ID and cookie headers to the response. +/// Same-origin gate for `/__ts/page-bids`. +/// +/// The endpoint is a side-effecting GET: it dispatches real PBS/APS auctions +/// and forwards request-derived signals (IP, UA, geo, consent) to partners. +/// Without a gate, any third-party page could trigger it from a visitor's +/// browser (it cannot read the JSON, but it burns SSP quota and leaks +/// outbound partner calls). +/// +/// A request is allowed when: +/// - `Sec-Fetch-Site` is `same-origin` (the tsjs SPA hook fetches a relative +/// URL, so a genuine same-origin navigation always reports this). `same-site` +/// is intentionally rejected: it admits sibling origins under the same +/// registrable domain, which are not trusted to spend SSP quota on the +/// visitor's behalf. +/// - `Sec-Fetch-Site` is absent (legacy client predating Fetch Metadata) **and** +/// the request carries the non-simple `X-TSJS-Page-Bids` header set by the +/// tsjs SPA hook — cross-origin callers cannot attach it without a CORS +/// preflight, which this endpoint never grants. +fn page_bids_request_allowed(req: &Request) -> bool { + match req + .headers() + .get("sec-fetch-site") + .and_then(|v| v.to_str().ok()) + { + Some(site) => site == "same-origin", + None => req.headers().contains_key("x-tsjs-page-bids"), + } +} + +/// Builds the `403 Forbidden` returned for a CORS preflight (`OPTIONS`) to the +/// side-effecting `/__ts/page-bids` endpoint. +/// +/// The GET handler's [`page_bids_request_allowed`] gate trusts the +/// `X-TSJS-Page-Bids` header precisely because this endpoint never grants a +/// preflight; letting `OPTIONS` fall through to the publisher origin (which may +/// return permissive CORS) would defeat that, allowing a cross-site page to +/// trigger real PBS/APS auctions from a visitor's browser. Every adapter returns +/// this same response for `OPTIONS /__ts/page-bids`. +pub fn page_bids_preflight_denied() -> Response { + let mut response = Response::new(EdgeBody::from("Forbidden")); + *response.status_mut() = StatusCode::FORBIDDEN; + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + response +} + +/// Normalizes the client-supplied `path` query parameter before glob matching. +/// +/// The SPA hook sends `location.pathname`, but the parameter is +/// client-controlled: strip any query string or fragment and force a leading +/// `/` so slot `page_patterns` always match against a canonical path shape. +fn normalize_page_bids_path(raw: &str) -> String { + let path = raw.split(['?', '#']).next().unwrap_or(""); + if path.starts_with('/') { + path.to_string() + } else { + format!("/{path}") + } +} + +/// Handle `GET /__ts/page-bids?path=` — server-side auction for SPA navigation. +/// +/// Matches creative opportunity slots for the given path, runs a server-side +/// auction (APS + PBS), and returns the slot definitions and winning bids as JSON. +/// Called by the client-side SPA navigation hook after `pushState` / `popstate`. +/// +/// `kv` enriches the bid request with server-side EIDs from the EC identity +/// graph. Only the Fastly adapter has a KV identity store, so Axum, Cloudflare, +/// and Spin pass `None`; client cookie EIDs are still resolved and consent-gated +/// on every adapter, so no adapter forwards unconsented EIDs. /// -/// Extracted so headers can be set before streaming begins (headers must -/// be finalized before `stream_to_client()` commits them). +/// # Errors /// -/// Consent-gated EC creation: -/// - Consent given → set EC ID header + cookie. -/// - Consent absent + existing cookie → revoke (expire cookie + delete KV entry). -/// - Consent absent + no cookie → do nothing. -fn apply_ec_headers( +/// Returns [`TrustedServerError`] if cookie parsing or EC ID generation fails. +pub async fn handle_page_bids( settings: &Settings, services: &RuntimeServices, - response: &mut Response, - ec_id: &str, - ec_allowed: bool, - existing_ec_cookie: Option<&str>, - consent_context: &crate::consent::ConsentContext, -) { - if ec_allowed { - // HeaderValue::from_str rejects \r, \n, and \0, so the EC ID - // cannot inject additional response headers. - match HeaderValue::from_str(ec_id) { - Ok(v) => { - response.headers_mut().insert(HEADER_X_TS_EC, v); - } - Err(_) => { - log::warn!("Rejecting EC ID response header: value is not a valid header value"); - } - } - // Cookie persistence is skipped if the EC ID contains RFC 6265-illegal - // characters. The header is still emitted when consent allows it. - crate::ec::cookies::set_ec_cookie(settings, response, ec_id); - } else if let Some(cookie_ec_id) = existing_ec_cookie { - log::info!( - "EC revoked for '{}': consent withdrawn (jurisdiction={})", - cookie_ec_id, - consent_context.jurisdiction, + kv: Option<&KvIdentityGraph>, + auction: AuctionDispatch<'_>, + ec_context: &EcContext, + req: Request, +) -> Result, Report> { + let Some(co_config) = &settings.creative_opportunities else { + let mut response = Response::new(EdgeBody::from("Creative opportunities not configured")); + *response.status_mut() = StatusCode::NOT_FOUND; + return Ok(response); + }; + + // CSRF-style gate: refuse cross-site invocations before any auction work. + if !page_bids_request_allowed(&req) { + log::debug!( + "page-bids: rejecting request (sec-fetch-site={:?}, tsjs header present={})", + req.headers() + .get("sec-fetch-site") + .and_then(|v| v.to_str().ok()), + req.headers().contains_key("x-tsjs-page-bids") ); - crate::ec::cookies::expire_ec_cookie(settings, response); - if settings.consent.consent_store.is_some() { - crate::consent::kv::delete_consent_from_kv(services.kv_store(), cookie_ec_id); - } - } else { + let mut response = Response::new(EdgeBody::from("Forbidden")); + *response.status_mut() = StatusCode::FORBIDDEN; + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + return Ok(response); + } + + let path_param = req + .uri() + .query() + .and_then(|query| { + url::form_urlencoded::parse(query.as_bytes()) + .find(|(k, _)| k == "path") + .map(|(_, v)| normalize_page_bids_path(&v)) + }) + .unwrap_or_else(|| "/".to_string()); + + let matched_slots: Vec<_> = + crate::creative_opportunities::match_slots(auction.slots, &path_param) + .into_iter() + .cloned() + .collect(); + + let request_info = crate::http_util::RequestInfo::from_request(&req, services.client_info()); + let ec_id = ec_context.ec_value().filter(|_| ec_context.ec_allowed()); + let consent_context = ec_context.consent(); + let geo = ec_context.geo_info().cloned(); + let cookie_jar = handle_request_cookies(&req)?; + + // Same fail-closed jurisdiction-aware gate the publisher navigation path + // uses — relies on the adapter's geo-aware EC context. + let consent_allows_auction = consent_allows_server_side_auction(consent_context); + + // Same bot / prefetch guards the publisher path uses — without them this + // endpoint would fire real SSP auctions on Sec-Purpose=prefetch warm-up + // navigations and known crawler UA scans, burning partner request quota. + let is_prefetch = is_prefetch_request(&req); + let is_bot = is_bot_user_agent(&req); + + let auction_enabled = auction.orchestrator.is_enabled(); + if !auction_enabled { + log::debug!("page-bids: [auction].enabled is false — skipping auction"); + } else if matched_slots.is_empty() { + log::debug!( + "No creative opportunity slots matched path '{}' — skipping auction", + path_param + ); + } else if is_bot || is_prefetch { log::debug!( - "EC skipped: no consent and no existing cookie (jurisdiction={})", - consent_context.jurisdiction, + "page-bids: skipping auction for path '{}' (is_bot={}, is_prefetch={})", + path_param, + is_bot, + is_prefetch ); } + + // The [auction].enabled kill switch and a consent denial disable the entire + // server-side ad stack. In those states the endpoint must return no slots, + // so the SPA hook does not assign `ts.adSlots` and call `adInit()` — + // otherwise the kill switch/consent gate would stop SSP calls but still let + // the client create/refresh GPT slots. Bot/prefetch requests, by contrast, + // keep their slot definitions (the placement structure is unchanged) but + // skip the live auction, matching the existing bot/prefetch behaviour. + let ad_stack_enabled = auction_enabled && consent_allows_auction; + + let winning_bids = if ad_stack_enabled && !matched_slots.is_empty() && !is_bot && !is_prefetch { + let slots_ctx = MatchedSlotsContext { + matched_slots: &matched_slots, + request_path: &path_param, + }; + let mut auction_request = build_auction_request( + &slots_ctx, + ec_id, + consent_context, + &request_info, + req.headers() + .get("user-agent") + .and_then(|v| v.to_str().ok()), + ); + apply_auction_eids_and_device( + &mut auction_request, + &AuctionEidTargeting { + cookie_jar: cookie_jar.as_ref(), + ec_id, + kv, + partner_registry: auction.registry, + ec_context, + services, + geo: geo.as_ref(), + path_label: "Page-bids", + }, + ); + let timeout_ms = co_config + .auction_timeout_ms + .unwrap_or(settings.auction.timeout_ms); + let auction_context = AuctionContext { + settings, + request: &req, + timeout_ms, + provider_responses: None, + services, + }; + match auction + .orchestrator + .run_auction(&auction_request, &auction_context) + .await + { + Ok(result) => result.winning_bids, + Err(e) => { + log::warn!("page-bids auction failed: {e:?}"); + std::collections::HashMap::new() + } + } + } else { + std::collections::HashMap::new() + }; + + let bid_map = build_bid_map( + &winning_bids, + co_config.price_granularity, + settings.debug.inject_adm_for_testing, + ); + + // Gate slots on the ad-stack kill switch / consent: when disabled, return no + // slots so the SPA hook does not call `adInit()` / create GPT slots. + let slots_json: Vec = if ad_stack_enabled { + matched_slots + .iter() + .map(|slot| build_slot_json(slot, co_config)) + .collect() + } else { + Vec::new() + }; + + let body = serde_json::json!({ + "slots": slots_json, + "bids": bid_map, + }); + + let json_str = serde_json::to_string(&body).change_context(TrustedServerError::Proxy { + message: "Failed to serialize page-bids response".to_string(), + })?; + + let mut response = Response::new(EdgeBody::from(json_str)); + response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("application/json"), + ); + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + + Ok(response) } #[cfg(test)] mod tests { + use std::io::{self, Read as _, Write as _}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use brotli::enc::writer::CompressorWriter; + use brotli::Decompressor; + use flate2::read::GzDecoder; + use flate2::write::GzEncoder; + use super::*; - use crate::edge_cookie::get_or_generate_ec_id; + use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::integrations::IntegrationRegistry; use crate::platform::test_support::{ build_services_with_http_client, noop_services, StubHttpClient, @@ -906,6 +2164,128 @@ mod tests { use http::{header, Method, Request as HttpRequest, StatusCode}; use std::sync::Arc; + struct ChunkedReader { + chunks: std::collections::VecDeque>, + read_count: Arc, + } + + impl ChunkedReader { + fn new(chunks: &[&[u8]], read_count: Arc) -> Self { + Self { + chunks: chunks.iter().map(|chunk| chunk.to_vec()).collect(), + read_count, + } + } + } + + impl io::Read for ChunkedReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let Some(chunk) = self.chunks.pop_front() else { + return Ok(0); + }; + self.read_count.fetch_add(1, Ordering::SeqCst); + let len = chunk.len().min(buf.len()); + buf[..len].copy_from_slice(&chunk[..len]); + Ok(len) + } + } + + struct RecordingProcessor { + read_count: Arc, + body_close_processed_at: Arc, + } + + impl StreamProcessor for RecordingProcessor { + fn process_chunk(&mut self, chunk: &[u8], _is_last: bool) -> Result, io::Error> { + if find_ascii_case_insensitive(chunk, BODY_CLOSE_PREFIX).is_some() { + self.body_close_processed_at + .store(self.read_count.load(Ordering::SeqCst), Ordering::SeqCst); + } + Ok(chunk.to_vec()) + } + } + + fn gzip_encode(input: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(input) + .expect("should write gzip test input"); + encoder.finish().expect("should finish gzip encoding") + } + + fn gzip_decode(input: &[u8]) -> Vec { + let mut decoder = GzDecoder::new(input); + let mut output = Vec::new(); + decoder + .read_to_end(&mut output) + .expect("should decode gzip test output"); + output + } + + fn brotli_encode(input: &[u8]) -> Vec { + let mut encoder = CompressorWriter::new(Vec::new(), 4096, 5, 22); + encoder + .write_all(input) + .expect("should write brotli test input"); + encoder.into_inner() + } + + fn brotli_decode(input: &[u8]) -> Vec { + let mut decoder = Decompressor::new(input, 4096); + let mut output = Vec::new(); + decoder + .read_to_end(&mut output) + .expect("should decode brotli test output"); + output + } + + fn make_stream_params( + settings: &Settings, + content_encoding: &str, + ) -> OwnedProcessResponseParams { + OwnedProcessResponseParams { + content_encoding: content_encoding.to_owned(), + origin_host: settings.publisher.origin_host(), + origin_url: settings.publisher.origin_url.clone(), + request_host: settings.publisher.domain.clone(), + request_scheme: "https".to_owned(), + content_type: "application/json".to_owned(), + ad_slots_script: None, + ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), + dispatched_auction: None, + price_granularity: Default::default(), + } + } + + fn test_auction_request() -> AuctionRequest { + AuctionRequest { + id: "test-auction".to_string(), + slots: vec![AdSlot { + id: "atf".to_string(), + formats: vec![AdFormat { + media_type: MediaType::Banner, + width: 300, + height: 250, + }], + floor_price: None, + targeting: Default::default(), + bidders: Default::default(), + }], + publisher: PublisherInfo { + domain: "test-publisher.com".to_string(), + page_url: Some("https://test-publisher.com/article".to_string()), + }, + user: UserInfo { + id: None, + consent: None, + eids: None, + }, + device: None, + site: None, + context: Default::default(), + } + } + fn build_request(method: Method, uri: &str) -> HttpRequest { HttpRequest::builder() .method(method) @@ -915,66 +2295,281 @@ mod tests { } #[test] - fn response_carries_body_preserves_bodiless_metadata() { - // A processable GET 200 buffers a body and recomputes Content-Length. + fn stream_publisher_body_round_trips_gzip() { + let settings = create_test_settings(); + let integration_registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let input = b"{\"asset\":\"https://origin.test-publisher.com/path/file.js\"}"; + let compressed = gzip_encode(input); + let params = make_stream_params(&settings, "gzip"); + let mut output = Vec::new(); + + stream_publisher_body( + EdgeBody::from(compressed), + &mut output, + ¶ms, + &settings, + &integration_registry, + ) + .expect("should stream gzip response through rewrite pipeline"); + + let decoded = gzip_decode(&output); + let decoded = String::from_utf8(decoded).expect("should decode rewritten gzip payload"); assert!( - super::response_carries_body(&Method::GET, StatusCode::OK), - "a GET 200 publisher response should carry a buffered body" + decoded.contains("https://test-publisher.com/path/file.js"), + "should rewrite origin URLs to the request host" ); - // HEAD carries no body; recomputing Content-Length to 0 would mislead - // clients/caches about the GET representation length. assert!( - !super::response_carries_body(&Method::HEAD, StatusCode::OK), - "HEAD publisher responses must not get a recomputed Content-Length" + !decoded.contains("origin.test-publisher.com"), + "should remove the origin hostname from the rewritten payload" ); - // Bodiless statuses keep their metadata regardless of method. + } + + #[test] + fn stream_publisher_body_round_trips_brotli() { + let settings = create_test_settings(); + let integration_registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let input = b"{\"asset\":\"https://origin.test-publisher.com/path/file.css\"}"; + let compressed = brotli_encode(input); + let params = make_stream_params(&settings, "br"); + let mut output = Vec::new(); + + stream_publisher_body( + EdgeBody::from(compressed), + &mut output, + ¶ms, + &settings, + &integration_registry, + ) + .expect("should stream brotli response through rewrite pipeline"); + + let decoded = brotli_decode(&output); + let decoded = String::from_utf8(decoded).expect("should decode rewritten brotli payload"); assert!( - !super::response_carries_body(&Method::GET, StatusCode::NO_CONTENT), - "204 responses must not get a recomputed Content-Length" + decoded.contains("https://test-publisher.com/path/file.css"), + "should rewrite origin URLs to the request host" ); assert!( - !super::response_carries_body(&Method::GET, StatusCode::NOT_MODIFIED), - "304 responses must not get a recomputed Content-Length" + !decoded.contains("origin.test-publisher.com"), + "should remove the origin hostname from the rewritten payload" ); } - fn response_body_string(response: http::Response) -> String { - String::from_utf8( - response - .into_body() - .into_bytes() - .unwrap_or_default() - .to_vec(), - ) - .expect("response body should be valid UTF-8") - } - #[test] - fn test_content_type_detection() { - // Test which content types should be processed - let test_cases = vec![ - ("text/html", true), - ("text/html; charset=utf-8", true), - ("text/css", true), - ("text/javascript", true), - ("application/javascript", true), - ("application/json", true), - ("application/json; charset=utf-8", true), - ("image/jpeg", false), - ("image/png", false), - ("application/pdf", false), - ("video/mp4", false), - ("application/octet-stream", false), - ]; + fn request_ec_uses_cookie_not_header() { + let settings = create_test_settings(); + let header_ec = format!("{}.HdrId1", "a".repeat(64)); + let cookie_ec = format!("{}.CkId01", "b".repeat(64)); + let req = Request::builder() + .method(Method::GET) + .uri("https://test.example.com/page") + .header("x-ts-ec", &header_ec) + .header("cookie", format!("ts-ec={cookie_ec}; other=value")) + .body(EdgeBody::empty()) + .expect("should build test request"); - for (content_type, expected) in test_cases { - assert_eq!( - is_processable_content_type(content_type), - expected, - "Content-Type '{content_type}' should_process: expected {expected}", - ); - } - } + let ec_context = EcContext::read_from_request(&settings, &req, &noop_services()) + .expect("should read EC context"); + + assert_eq!( + ec_context.ec_value(), + Some(cookie_ec.as_str()), + "should resolve request EC ID from cookie" + ); + assert!( + ec_context.cookie_was_present(), + "should detect cookie was present" + ); + assert_eq!( + ec_context.existing_cookie_ec_id(), + Some(cookie_ec.as_str()), + "should return cookie EC value for revocation" + ); + } + + /// Drive `handle_publisher_request` with no creative opportunities — a plain + /// proxy with no server-side auction. Hides the auction/EC wiring so callers + /// read like a simple `(settings, services, req)` proxy. + async fn run_publisher_proxy( + settings: &Settings, + services: &RuntimeServices, + req: Request, + ) -> PublisherResponse { + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut ec_context = + EcContext::read_from_request(settings, &req, services).expect("should read EC context"); + handle_publisher_request( + settings, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + ) + .await + .expect("should proxy publisher request") + } + + #[tokio::test] + async fn publisher_request_uses_platform_http_client_with_http_types() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"origin response".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let response = match run_publisher_proxy(&settings, &services, req).await { + PublisherResponse::Buffered(r) => r, + PublisherResponse::PassThrough { mut response, body } => { + *response.body_mut() = body; + response + } + PublisherResponse::Stream { response, .. } => response, + }; + + assert_eq!(response.status(), StatusCode::OK); + assert_eq!(response_body_string(response), "origin response"); + assert_eq!( + stub.recorded_backend_names(), + vec!["stub-backend".to_string()], + "should proxy through the platform http client" + ); + } + + #[tokio::test] + async fn handle_publisher_request_does_not_self_generate_ec() { + // EC generation is the adapter's real-browser-gated responsibility. This + // handler must never mint an EC ID on its own: for a navigation from a + // client the adapter did not pre-generate for (e.g. a non-real browser), + // `ec_value` must stay `None` so no IP-derived identifier reaches the + // auction. Consent allows EC creation and a client IP is present here — + // exactly the conditions under which the old inline call would have + // generated one. + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = + EcContext::new_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); + assert!( + ec_context.ec_allowed(), + "test precondition: consent must allow EC creation" + ); + + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/article") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + + let _ = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, + }, + req, + ) + .await + .expect("should proxy publisher request"); + + assert_eq!( + ec_context.ec_value(), + None, + "handler must not self-generate an EC ID; generation is the adapter's real-browser-gated responsibility", + ); + } + + #[test] + fn response_carries_body_preserves_bodiless_metadata() { + // A processable GET 200 buffers a body and recomputes Content-Length. + assert!( + super::response_carries_body(&Method::GET, StatusCode::OK), + "a GET 200 publisher response should carry a buffered body" + ); + // HEAD carries no body; recomputing Content-Length to 0 would mislead + // clients/caches about the GET representation length. + assert!( + !super::response_carries_body(&Method::HEAD, StatusCode::OK), + "HEAD publisher responses must not get a recomputed Content-Length" + ); + // Bodiless statuses keep their metadata regardless of method. + assert!( + !super::response_carries_body(&Method::GET, StatusCode::NO_CONTENT), + "204 responses must not get a recomputed Content-Length" + ); + assert!( + !super::response_carries_body(&Method::GET, StatusCode::NOT_MODIFIED), + "304 responses must not get a recomputed Content-Length" + ); + } + + fn response_body_string(response: http::Response) -> String { + String::from_utf8( + response + .into_body() + .into_bytes() + .unwrap_or_default() + .to_vec(), + ) + .expect("response body should be valid UTF-8") + } + + #[test] + fn test_content_type_detection() { + let test_cases = vec![ + ("text/html", true), + ("text/html; charset=utf-8", true), + ("Text/HTML; Charset=utf-8", true), + ("text/css", true), + ("Text/CSS", true), + ("text/javascript", true), + ("application/javascript", true), + ("Application/JavaScript", true), + ("application/json", true), + ("application/json; charset=utf-8", true), + ("Application/JSON; Charset=UTF-8", true), + ("image/jpeg", false), + ("image/png", false), + ("application/pdf", false), + ("video/mp4", false), + ("application/octet-stream", false), + ]; + + for (content_type, expected) in test_cases { + assert_eq!( + is_processable_content_type(content_type), + expected, + "Content-Type '{content_type}' should_process: expected {expected}", + ); + } + } #[test] fn supported_content_encoding_accepts_known_values() { @@ -1004,18 +2599,138 @@ mod tests { ); } + #[test] + fn server_side_ad_stack_runs_only_when_all_auction_gates_pass() { + assert!( + should_run_server_side_ad_stack(true, true, false, false, true, true, true), + "GET, real navigation, matched slots, and consent should run TS ad stack" + ); + + assert!( + !should_run_server_side_ad_stack(false, true, false, false, true, true, true), + "non-GET requests should skip TS ad stack" + ); + assert!( + !should_run_server_side_ad_stack(true, false, false, false, true, true, true), + "non-document requests should skip TS ad stack" + ); + assert!( + !should_run_server_side_ad_stack(true, true, true, false, true, true, true), + "prefetch requests should skip TS ad stack and injection" + ); + assert!( + !should_run_server_side_ad_stack(true, true, false, true, true, true, true), + "bot requests should skip TS ad stack and injection" + ); + assert!( + !should_run_server_side_ad_stack(true, true, false, false, false, true, true), + "requests with no matching slots should skip TS ad stack" + ); + assert!( + !should_run_server_side_ad_stack(true, true, false, false, true, false, true), + "requests without required consent should skip TS ad stack and injection" + ); + assert!( + !should_run_server_side_ad_stack(true, true, false, false, true, true, false), + "disabled [auction].enabled kill switch should skip TS ad stack and injection" + ); + } + + #[tokio::test] + async fn body_close_hold_loop_processes_close_tail_before_reading_post_body_chunks() { + let settings = create_test_settings(); + let services = noop_services(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let dispatched = DispatchedAuction::empty_for_test(test_auction_request(), 500); + let read_count = Arc::new(AtomicUsize::new(0)); + let body_close_processed_at = Arc::new(AtomicUsize::new(0)); + let reader = ChunkedReader::new( + &[ + b"painted", + b"", + b"", + ], + Arc::clone(&read_count), + ); + let mut processor = RecordingProcessor { + read_count: Arc::clone(&read_count), + body_close_processed_at: Arc::clone(&body_close_processed_at), + }; + let ad_bids_state = Arc::new(Mutex::new(None)); + let ctx = AuctionCollectCtx { + dispatched, + price_granularity: PriceGranularity::default(), + ad_bids_state: &ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + }; + let mut output = Vec::new(); + + body_close_hold_loop(reader, &mut output, &mut processor, ctx) + .await + .expect("should stream body with auction hold"); + + assert_eq!( + body_close_processed_at.load(Ordering::SeqCst), + 1, + "close-body tail should be processed as soon as it is found, before later chunks are read" + ); + assert_eq!( + std::str::from_utf8(&output).expect("should be utf8"), + "painted", + "post-body chunks should still stream in order" + ); + } + + #[test] + fn body_close_hold_buffer_holds_close_body_tail_in_single_chunk() { + let mut hold = BodyCloseHoldBuffer::new(); + + let ready = hold.push(b"painted"); + let held = hold.finish(); + + assert_eq!( + std::str::from_utf8(&ready).expect("should be utf8"), + "painted", + "content before should stream before auction collection" + ); + assert_eq!( + std::str::from_utf8(&held).expect("should be utf8"), + "", + "the close-body tag and trailing bytes should be held" + ); + } + + #[test] + fn body_close_hold_buffer_holds_close_body_tail_across_chunks() { + let mut hold = BodyCloseHoldBuffer::new(); + + let first = hold.push(b"painted"); + let held = hold.finish(); + + let streamed = [first, second].concat(); + assert_eq!( + std::str::from_utf8(&streamed).expect("should be utf8"), + "painted", + "split bytes must not leak before auction collection" + ); + assert_eq!( + std::str::from_utf8(&held).expect("should be utf8"), + "", + "split close-body tag should be held intact" + ); + } + #[test] fn unsupported_encoding_response_is_returned_unmodified() { - // Processable (HTML) 2xx with unsupported encoding must route to - // BufferedUnmodified — feeding zstd-compressed bytes to the rewriter - // as identity would produce garbled output. assert_eq!( classify_response_route( StatusCode::OK, "text/html; charset=utf-8", "zstd", - "example.com", - false, + "example.com" ), ResponseRoute::BufferedUnmodified, ); @@ -1027,7 +2742,6 @@ mod tests { let origin_host = settings.publisher.origin_host(); assert_eq!(origin_host, "origin.test-publisher.com"); - // Test with port let mut settings_with_port = create_test_settings(); settings_with_port.publisher.origin_url = "origin.test-publisher.com:8080".to_string(); assert_eq!( @@ -1038,27 +2752,18 @@ mod tests { #[test] fn test_invalid_utf8_handling() { - // Test that invalid UTF-8 bytes are handled gracefully - let invalid_utf8_bytes = vec![0xFF, 0xFE, 0xFD]; // Invalid UTF-8 sequence - - // Verify these bytes cannot be converted to a valid UTF-8 string + let invalid_utf8_bytes = vec![0xFF, 0xFE, 0xFD]; assert!(String::from_utf8(invalid_utf8_bytes.clone()).is_err()); - - // In the actual function, invalid UTF-8 would be passed through unchanged - // This test verifies our approach is sound } #[test] fn test_utf8_conversion_edge_cases() { - // Test various UTF-8 edge cases let test_cases = vec![ - // Valid UTF-8 with special characters - (vec![0xE2, 0x98, 0x83], true), // ☃ (snowman) - (vec![0xF0, 0x9F, 0x98, 0x80], true), // 😀 (emoji) - // Invalid UTF-8 sequences - (vec![0xFF, 0xFE], false), // Invalid start byte - (vec![0xC0, 0x80], false), // Overlong encoding - (vec![0xED, 0xA0, 0x80], false), // Surrogate half + (vec![0xE2, 0x98, 0x83], true), + (vec![0xF0, 0x9F, 0x98, 0x80], true), + (vec![0xFF, 0xFE], false), + (vec![0xC0, 0x80], false), + (vec![0xED, 0xA0, 0x80], false), ]; for (bytes, should_be_valid) in test_cases { @@ -1072,10 +2777,6 @@ mod tests { } } - // Gate tests — exercise `classify_response_route` directly, the same - // function `handle_publisher_request` calls. If the gate formula changes, - // both production and tests are affected identically: no silent drift. - #[test] fn route_streams_2xx_html_without_post_processors() { assert_eq!( @@ -1083,112 +2784,95 @@ mod tests { StatusCode::OK, "text/html; charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, ); } #[test] - fn route_buffers_html_with_post_processors_for_processing() { + fn route_streams_mixed_case_html_content_type() { assert_eq!( classify_response_route( StatusCode::OK, - "text/html; charset=utf-8", + "Text/HTML; Charset=utf-8", "gzip", - "example.com", - true, + "example.com" ), - ResponseRoute::BufferedProcessed, + ResponseRoute::Stream, + "HTML MIME type matching must be case-insensitive", ); } #[test] - fn route_streams_non_html_even_with_post_processors_registered() { - // Post-processors only apply to HTML; JSON/JS can still stream. + fn route_streams_html_with_post_processors() { assert_eq!( classify_response_route( StatusCode::OK, - "application/json", + "text/html; charset=utf-8", "gzip", - "example.com", - true, + "example.com" ), ResponseRoute::Stream, ); } + #[test] + fn route_streams_non_html_even_with_post_processors_registered() { + assert_eq!( + classify_response_route(StatusCode::OK, "application/json", "gzip", "example.com"), + ResponseRoute::Stream, + ); + } + #[test] fn route_buffers_unmodified_on_unsupported_encoding() { - // Unsupported encoding cannot be streamed (would be fed to rewriter - // as identity and produce garbled output). assert_eq!( - classify_response_route(StatusCode::OK, "text/html", "zstd", "example.com", false,), + classify_response_route(StatusCode::OK, "text/html", "zstd", "example.com"), ResponseRoute::BufferedUnmodified, ); } #[test] fn route_passes_through_non_processable_2xx() { - // Binary content (images, fonts) on 2xx streams the origin body direct. assert_eq!( - classify_response_route(StatusCode::OK, "image/png", "", "example.com", false,), + classify_response_route(StatusCode::OK, "image/png", "", "example.com"), ResponseRoute::PassThrough, ); } #[test] fn route_buffers_non_processable_error_responses() { - // Non-2xx never pass through — response needs to reach the client - // as-is (with any error body the origin produced). assert_eq!( - classify_response_route(StatusCode::NOT_FOUND, "image/png", "", "example.com", false,), + classify_response_route(StatusCode::NOT_FOUND, "image/png", "", "example.com"), ResponseRoute::BufferedUnmodified, ); } #[test] fn route_excludes_204_from_pass_through() { - // 204 No Content (RFC 9110 §15.3.5) prohibits a message body. assert_eq!( - classify_response_route( - StatusCode::NO_CONTENT, - "image/png", - "", - "example.com", - false, - ), + classify_response_route(StatusCode::NO_CONTENT, "image/png", "", "example.com"), ResponseRoute::BufferedUnmodified, ); } #[test] fn route_excludes_205_from_pass_through() { - // 205 Reset Content (RFC 9110 §15.3.6) prohibits a message body. assert_eq!( - classify_response_route( - StatusCode::RESET_CONTENT, - "image/png", - "", - "example.com", - false, - ), + classify_response_route(StatusCode::RESET_CONTENT, "image/png", "", "example.com"), ResponseRoute::BufferedUnmodified, ); } #[test] fn route_excludes_204_for_processable_content_types() { - // 204 must stay body-less even when Content-Type would otherwise route - // to Stream or BufferedProcessed. assert_eq!( classify_response_route( StatusCode::NO_CONTENT, "text/html; charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::BufferedUnmodified, "204 + HTML must not route to Stream", @@ -1198,11 +2882,10 @@ mod tests { StatusCode::NO_CONTENT, "text/html; charset=utf-8", "gzip", - "example.com", - true, + "example.com" ), ResponseRoute::BufferedUnmodified, - "204 + HTML + post-processors must not route to BufferedProcessed", + "204 + HTML + post-processors must not route to Stream", ); } @@ -1213,8 +2896,7 @@ mod tests { StatusCode::RESET_CONTENT, "application/json", "", - "example.com", - false, + "example.com" ), ResponseRoute::BufferedUnmodified, "205 + JSON must not route to Stream", @@ -1223,15 +2905,12 @@ mod tests { #[test] fn route_streams_non_2xx_processable_content() { - // Branded 404 or 500 HTML with origin URLs must still be rewritten. - // This matches the pre-streaming behavior on main. assert_eq!( classify_response_route( StatusCode::NOT_FOUND, "text/html; charset=utf-8", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, ); @@ -1240,53 +2919,43 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR, "application/json", "gzip", - "example.com", - false, + "example.com" ), ResponseRoute::Stream, ); } #[test] - fn route_processes_non_2xx_html_with_post_processors() { - // Non-2xx HTML with post-processors still needs full-document processing - // for head injection, same as 2xx. + fn route_streams_non_2xx_html_with_post_processors() { assert_eq!( classify_response_route( StatusCode::NOT_FOUND, "text/html; charset=utf-8", "gzip", - "example.com", - true, + "example.com" ), - ResponseRoute::BufferedProcessed, + ResponseRoute::Stream, ); } #[test] fn route_passes_through_non_processable_even_with_empty_request_host() { - // Empty request_host blocks URL rewriting but pass-through does no - // rewriting, so a non-processable 2xx still streams through. assert_eq!( - classify_response_route(StatusCode::OK, "image/png", "", "", false,), + classify_response_route(StatusCode::OK, "image/png", "", ""), ResponseRoute::PassThrough, ); } #[test] fn route_buffers_processable_content_with_empty_request_host() { - // Misconfiguration case — URL rewriting needs a host, so the - // processable response falls back to unmodified pass-through. assert_eq!( - classify_response_route(StatusCode::OK, "text/html", "gzip", "", false,), + classify_response_route(StatusCode::OK, "text/html", "gzip", ""), ResponseRoute::BufferedUnmodified, ); } #[test] fn pass_through_preserves_body_and_content_length() { - // Simulate the PassThrough path: take body, reattach, send. - // Verify byte-for-byte identity and Content-Length preservation. let image_bytes: Vec = (0..=255).cycle().take(4096).collect(); let mut response = Response::builder() @@ -1321,7 +2990,6 @@ mod tests { #[test] fn test_content_encoding_detection() { - // Test that we properly handle responses with various content encodings let test_encodings = vec!["gzip", "deflate", "br", "identity", ""]; for encoding in test_encodings { @@ -1439,99 +3107,6 @@ mod tests { ); } - #[test] - fn revocation_targets_cookie_ec_id_not_header() { - let settings = create_test_settings(); - let mut req = build_request(Method::GET, "https://test.example.com/page"); - req.headers_mut().insert( - crate::constants::HEADER_X_TS_EC, - http::HeaderValue::from_static("header_id"), - ); - req.headers_mut().insert( - header::COOKIE, - http::HeaderValue::from_static("ts-ec=cookie_id; other=value"), - ); - - let cookie_jar = handle_request_cookies(&req).expect("should parse cookies"); - let existing_ec_cookie = cookie_jar - .as_ref() - .and_then(|jar| jar.get(COOKIE_TS_EC)) - .map(|cookie| cookie.value().to_owned()); - - let resolved_ec_id = - get_or_generate_ec_id(&settings, &noop_services(), &req).expect("should resolve EC ID"); - - assert_eq!( - existing_ec_cookie.as_deref(), - Some("cookie_id"), - "should read revocation target from cookie value" - ); - assert_eq!( - resolved_ec_id, "header_id", - "should still resolve request EC ID from header precedence" - ); - } - - #[test] - fn revocation_deletes_kv_entry_for_cookie_ec_id() { - use crate::platform::test_support::RecordingKvStore; - - let mut settings = create_test_settings(); - settings.consent.consent_store = Some("test-consent-store".to_string()); - - let recording = Arc::new(RecordingKvStore::new()); - let services = noop_services() - .with_kv_store(Arc::clone(&recording) as Arc); - - let mut response = Response::new(EdgeBody::empty()); - let consent_ctx = crate::consent::ConsentContext::default(); - - apply_ec_headers( - &settings, - &services, - &mut response, - "new-ec-id", - false, - Some("cookie-ec-id"), - &consent_ctx, - ); - - assert_eq!( - recording.deleted_keys(), - vec!["cookie-ec-id"], - "should delete KV entry for the revoked EC cookie ID" - ); - } - - #[test] - fn revocation_does_not_delete_kv_when_consent_store_absent() { - use crate::platform::test_support::RecordingKvStore; - - let settings = create_test_settings(); - - let recording = Arc::new(RecordingKvStore::new()); - let services = noop_services() - .with_kv_store(Arc::clone(&recording) as Arc); - - let mut response = Response::new(EdgeBody::empty()); - let consent_ctx = crate::consent::ConsentContext::default(); - - apply_ec_headers( - &settings, - &services, - &mut response, - "new-ec-id", - false, - Some("cookie-ec-id"), - &consent_ctx, - ); - - assert!( - recording.deleted_keys().is_empty(), - "should not delete KV entry when no consent_store is configured" - ); - } - #[test] fn tsjs_dynamic_returns_not_found_for_unknown_filename() { let settings = create_test_settings(); @@ -1600,7 +3175,6 @@ mod tests { #[test] fn tsjs_dynamic_serves_deferred_prebid_when_enabled() { - // Default test settings include prebid enabled let settings = create_test_settings(); let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); @@ -1663,133 +3237,48 @@ mod tests { ); } - #[test] - fn publisher_request_uses_platform_http_client_with_http_types() { - futures::executor::block_on(async { - let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"origin response".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let pub_response = handle_publisher_request(&settings, ®istry, &services, req) - .await - .expect("should proxy publisher request"); - let response = match pub_response { - PublisherResponse::Buffered(r) => r, - PublisherResponse::PassThrough { mut response, body } => { - *response.body_mut() = body; - response - } - PublisherResponse::Stream { response, .. } => response, - }; - - assert_eq!(response.status(), StatusCode::OK); - assert_eq!(response_body_string(response), "origin response"); - assert_eq!( - stub.recorded_backend_names(), - vec!["stub-backend".to_string()], - "should proxy through the platform http client" - ); - }); - } + #[tokio::test] + async fn publisher_request_sends_configured_host_header_override() { + let mut settings = create_test_settings(); + settings.publisher.origin_host_header_override = Some("www.example.com".to_string()); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"origin response".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); - #[test] - fn publisher_request_strips_fastly_ssl_before_forwarding() { - futures::executor::block_on(async { - // The EdgeZero entry point re-injects `fastly-ssl` from trusted TLS - // metadata so in-process scheme detection works. It must not leak to the - // origin: the legacy path never forwarded it. - let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"origin response".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .header("fastly-ssl", "1") - .body(EdgeBody::empty()) - .expect("should build request"); + let _ = run_publisher_proxy(&settings, &services, req).await; - handle_publisher_request(&settings, ®istry, &services, req) - .await - .expect("should proxy publisher request"); + let recorded_headers = stub.recorded_request_headers(); + let outbound_headers = recorded_headers + .first() + .expect("should record one outbound request"); + let outbound_host = outbound_headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("host")) + .map(|(_, value)| value.as_str()); - let recorded = stub.recorded_request_headers(); - let outbound = recorded - .first() - .expect("should record one outbound request"); - assert!( - !outbound - .iter() - .any(|(name, _)| name.eq_ignore_ascii_case("fastly-ssl")), - "internal fastly-ssl signal must not be forwarded to the origin, got: {outbound:?}" - ); - }); + assert_eq!( + outbound_host, + Some("www.example.com"), + "should send configured host override to outbound request" + ); } #[test] - fn publisher_request_sends_configured_host_header_override() { - futures::executor::block_on(async { - let mut settings = create_test_settings(); - settings.publisher.origin_host_header_override = Some("www.example.com".to_string()); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"origin response".to_vec()); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let _ = handle_publisher_request(&settings, ®istry, &services, req) - .await - .expect("should proxy publisher request"); + fn stream_publisher_body_preserves_gzip_round_trip() { + use flate2::write::GzEncoder; + use std::io::Write; - let recorded_headers = stub.recorded_request_headers(); - let outbound_headers = recorded_headers - .first() - .expect("should record one outbound request"); - let outbound_host = outbound_headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("host")) - .map(|(_, value)| value.as_str()); - - assert_eq!( - outbound_host, - Some("www.example.com"), - "should send configured host override to outbound request" - ); - }); - } - - #[test] - fn stream_publisher_body_preserves_gzip_round_trip() { - use flate2::write::GzEncoder; - use std::io::Write; - - let settings = create_test_settings(); - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); // Compress CSS containing an origin URL that should be rewritten. // CSS uses the text URL replacer (not lol_html), so inline URLs are rewritten. @@ -1809,6 +3298,10 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), }; let mut output = Vec::new(); @@ -1850,6 +3343,10 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), }; let mut output = Vec::new(); @@ -1868,6 +3365,51 @@ mod tests { ); } + #[test] + fn stream_publisher_body_treats_mixed_case_html_as_html() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let bids_script = + r#""#; + let state = Arc::new(Mutex::new(Some(bids_script.to_string()))); + let params = OwnedProcessResponseParams { + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "Text/HTML; Charset=utf-8".to_string(), + ad_slots_script: Some( + r#""# + .to_string(), + ), + ad_bids_state: state, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let mut output = Vec::new(); + + stream_publisher_body( + EdgeBody::from(b"content".to_vec()), + &mut output, + ¶ms, + &settings, + ®istry, + ) + .expect("should process mixed-case HTML content type"); + + let html = String::from_utf8(output).expect("should be valid UTF-8"); + assert!( + html.contains(".adSlots=JSON.parse"), + "mixed-case HTML must use the HTML processor and inject ad slots. Got: {html}" + ); + assert!( + html.contains(".bids=JSON.parse"), + "mixed-case HTML must use the HTML processor and inject bids. Got: {html}" + ); + } + /// Mid-stream decoder failure must surface as an error. The adapter /// relies on this: once headers are committed, it logs and drops the /// `StreamingBody` so the client sees a truncated response. If a decode @@ -1888,6 +3430,10 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), }; let bogus_body = EdgeBody::from(b"not gzip".to_vec()); @@ -1941,12 +3487,11 @@ mod tests { ); } - /// Buffered-processed dispatch contract: HTML with a registered post-processor - /// routes through `BufferedProcessed`, and the handler path sets - /// `Content-Length` from the processed body length. Verify that invariant - /// via the classifier + `process_response_streaming` composition. + /// Streaming dispatch contract: HTML with a registered post-processor still + /// routes through `Stream`, and the shared processor pipeline still applies + /// the post-processor rewrite. #[test] - fn buffered_processed_sets_content_length_from_processed_body() { + fn streaming_html_with_post_processors_rewrites_body() { // Configure nextjs so a post-processor is registered. let mut settings = create_test_settings(); settings @@ -1973,14 +3518,12 @@ mod tests { "text/html; charset=utf-8", "", "proxy.example.com", - registry.has_html_post_processors(), ), - ResponseRoute::BufferedProcessed, - "HTML with post-processors must route to BufferedProcessed" + ResponseRoute::Stream, + "HTML with post-processors must route to Stream" ); - // Feed a small HTML body through the same pipeline the - // BufferedProcessed arm uses (Vec output). + // Feed a small HTML body through the same pipeline the Stream arm uses. let html = b"link"; let body = EdgeBody::from(html.to_vec()); @@ -1992,14 +3535,18 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) - .expect("should process buffered HTML"); + .expect("should process streaming HTML"); assert!( !output.is_empty(), - "buffered processed output must not be empty" + "streaming processed output must not be empty" ); let as_str = std::str::from_utf8(&output).expect("output should be valid UTF-8"); assert!( @@ -2012,63 +3559,6 @@ mod tests { ); } - /// `BufferedProcessed` must enforce `publisher.max_buffered_body_bytes` so a - /// post-processed HTML body whose decoded output exceeds the cap fails instead - /// of allocating past the limit. Regression for the `EdgeZero` buffering gap - /// where only the streaming-conversion path applied the cap. - #[test] - fn buffered_processed_enforces_max_buffered_body_bytes() { - futures::executor::block_on(async { - let mut settings = create_test_settings(); - // Register an HTML post-processor so the response routes to BufferedProcessed. - settings - .integrations - .insert_config( - "nextjs", - &serde_json::json!({ - "enabled": true, - "rewrite_attributes": ["href", "link", "url"], - }), - ) - .expect("should update nextjs config"); - // Tiny cap so a modest HTML document exceeds it after processing. - settings.publisher.max_buffered_body_bytes = 64; - - let registry = - IntegrationRegistry::new(&settings).expect("should create integration registry"); - assert!( - registry.has_html_post_processors(), - "nextjs integration must register an HTML post-processor" - ); - - // Identity-encoded HTML well above the 64-byte cap once buffered. - let filler = "

padding

".repeat(64); - let html = format!("{filler}"); - let stub = Arc::new(StubHttpClient::new()); - stub.push_response_with_headers( - 200, - html.into_bytes(), - vec![("content-type", "text/html; charset=utf-8")], - ); - let services = build_services_with_http_client( - Arc::clone(&stub) as Arc - ); - let req = HttpRequest::builder() - .method(Method::GET) - .uri("https://publisher.example/page") - .header(header::HOST, "publisher.example") - .body(EdgeBody::empty()) - .expect("should build request"); - - let result = handle_publisher_request(&settings, ®istry, &services, req).await; - - assert!( - result.is_err(), - "buffered-processed body exceeding max_buffered_body_bytes must error, not allocate past the cap" - ); - }); - } - /// Document-state survives from the streaming pass into the post-processor. /// `NextJsRscPlaceholderRewriter` writes into `IntegrationDocumentState` /// during streaming; `NextJsHtmlPostProcessor` reads it and substitutes. @@ -2099,6 +3589,10 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), }; let mut output = Vec::new(); @@ -2126,6 +3620,993 @@ mod tests { ); } + #[cfg(test)] + mod creative_opportunities_tests { + use super::super::{ + build_ad_slots_script, build_auction_request, build_bid_map, build_bids_script, + html_escape_for_script, MatchedSlotsContext, + }; + use crate::auction::types::{Bid, MediaType}; + use crate::consent::ConsentContext; + use crate::creative_opportunities::{ + CreativeOpportunitiesConfig, CreativeOpportunityFormat, CreativeOpportunitySlot, + }; + use crate::http_util::RequestInfo; + use crate::price_bucket::PriceGranularity; + use std::collections::HashMap; + + fn make_config() -> CreativeOpportunitiesConfig { + CreativeOpportunitiesConfig { + gam_network_id: "21765378893".to_string(), + auction_timeout_ms: Some(500), + price_granularity: PriceGranularity::Dense, + slot: Vec::new(), + } + } + + fn make_slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: "atf_sidebar_ad".to_string(), + gam_unit_path: Some("/21765378893/publisher/atf-sidebar".to_string()), + div_id: Some("div-atf-sidebar".to_string()), + page_patterns: vec!["/20**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }], + floor_price: Some(0.50), + targeting: [("pos".to_string(), "atf".to_string())] + .into_iter() + .collect(), + providers: Default::default(), + compiled_patterns: Vec::new(), + } + } + + fn make_bid( + slot_id: &str, + price: f64, + bidder: &str, + ad_id: &str, + nurl: &str, + burl: &str, + ) -> Bid { + Bid { + slot_id: slot_id.to_string(), + price: Some(price), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: bidder.to_string(), + width: 300, + height: 250, + nurl: Some(nurl.to_string()), + burl: Some(burl.to_string()), + ad_id: Some(ad_id.to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + } + } + + #[test] + fn ad_slots_script_contains_slot_data() { + let slots = vec![make_slot()]; + let config = make_config(); + let script = build_ad_slots_script(&slots, &config); + assert!( + script.contains("window.tsjs=window.tsjs||{}"), + "should initialise tsjs namespace" + ); + assert!( + script.contains(".adSlots=JSON.parse"), + "should use JSON.parse for adSlots" + ); + assert!(script.contains("atf_sidebar_ad"), "should include slot id"); + assert!(!script.contains("adInit"), "must NOT contain adInit"); + assert!( + !script.contains("__ts_request_id"), + "must NOT contain request_id" + ); + } + + #[test] + fn ad_slots_script_is_xss_safe() { + let slots = vec![make_slot()]; + let config = make_config(); + let script = build_ad_slots_script(&slots, &config); + let inner = script + .trim_start_matches(""); + assert!(!inner.contains('<'), "no unescaped < in script content"); + assert!(!inner.contains('>'), "no unescaped > in script content"); + } + + #[test] + fn bid_map_includes_nurl_and_burl() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + make_bid( + "atf_sidebar_ad", + 1.50, + "kargo", + "abc123", + "https://ssp/win", + "https://ssp/bill", + ), + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let entry = map.get("atf_sidebar_ad").expect("should have bid entry"); + let obj = entry.as_object().expect("should be object"); + assert_eq!( + obj.get("hb_pb").and_then(|v| v.as_str()), + Some("1.50"), + "should bucket price with dense granularity" + ); + assert_eq!( + obj.get("hb_bidder").and_then(|v| v.as_str()), + Some("kargo"), + "should include bidder" + ); + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("abc123"), + "should fall back to ad_id when no cache_id present" + ); + assert_eq!( + obj.get("nurl").and_then(|v| v.as_str()), + Some("https://ssp/win"), + "should include nurl" + ); + assert_eq!( + obj.get("burl").and_then(|v| v.as_str()), + Some("https://ssp/bill"), + "should include burl" + ); + } + + #[test] + fn client_bid_map_omits_adm_by_default() { + let mut winning_bids = HashMap::new(); + let mut bid = make_bid( + "atf_sidebar_ad", + 1.50, + "kargo", + "abc123", + "https://ssp/win", + "https://ssp/bill", + ); + bid.creative = Some("
Creative
".to_string()); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + + assert!( + obj.get("adm").is_none(), + "should omit adm when debug injection is disabled" + ); + assert!( + obj.get("debug_bid").is_none(), + "should omit debug bid when debug injection is disabled" + ); + } + + #[test] + fn client_bid_map_includes_adm_when_debug_injection_enabled() { + let mut winning_bids = HashMap::new(); + let mut bid = make_bid( + "atf_sidebar_ad", + 1.50, + "kargo", + "abc123", + "https://ssp/win", + "https://ssp/bill", + ); + bid.creative = Some("
Creative
".to_string()); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, true); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + + assert_eq!( + obj.get("adm").and_then(|v| v.as_str()), + Some("
Creative
"), + "should include adm when debug injection is enabled" + ); + } + + #[test] + fn client_bid_map_includes_debug_bid_when_debug_injection_enabled() { + let mut winning_bids = HashMap::new(); + let mut bid = make_bid( + "atf_sidebar_ad", + 1.50, + "mocktioneer", + "bid-ad-id", + "https://ssp/win", + "https://ssp/bill", + ); + bid.creative = Some("
Creative
".to_string()); + bid.adomain = Some(vec!["example.com".to_string()]); + bid.cache_id = Some("cache-uuid".to_string()); + bid.cache_host = Some("cache.example".to_string()); + bid.cache_path = Some("/cache".to_string()); + bid.metadata.insert( + "raw_field".to_string(), + serde_json::Value::String("raw-value".to_string()), + ); + winning_bids.insert("atf_sidebar_ad".to_string(), bid); + + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, true); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + let debug_bid = obj + .get("debug_bid") + .and_then(|v| v.as_object()) + .expect("should include debug bid when debug injection is enabled"); + + assert_eq!( + debug_bid.get("slot_id").and_then(|v| v.as_str()), + Some("atf_sidebar_ad"), + "should expose original slot id" + ); + assert_eq!( + debug_bid.get("bidder").and_then(|v| v.as_str()), + Some("mocktioneer"), + "should expose original bidder" + ); + assert_eq!( + debug_bid.get("ad_id").and_then(|v| v.as_str()), + Some("bid-ad-id"), + "should expose original bid ad id" + ); + assert_eq!( + debug_bid.get("cache_id").and_then(|v| v.as_str()), + Some("cache-uuid"), + "should expose original PBS cache id" + ); + assert_eq!( + debug_bid.get("metadata").and_then(|v| v.get("raw_field")), + Some(&serde_json::Value::String("raw-value".to_string())), + "should expose provider metadata" + ); + } + + #[test] + fn bid_map_uses_cache_id_for_hb_adid_when_present() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "thetradedesk".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-impression-id".to_string()), + cache_id: Some("f47447a0-b759-4f2f-9887-af458b79b570".to_string()), + cache_host: Some("openads.adsrvr.org".to_string()), + cache_path: Some("/cache".to_string()), + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should use cache_id for hb_adid, not ad_id" + ); + assert_eq!( + obj.get("hb_cache_host").and_then(|v| v.as_str()), + Some("openads.adsrvr.org"), + "should emit hb_cache_host" + ); + assert_eq!( + obj.get("hb_cache_path").and_then(|v| v.as_str()), + Some("/cache"), + "should emit hb_cache_path" + ); + } + + #[test] + fn bid_map_falls_back_to_ad_id_when_cache_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "amazon-aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("aps-bid-token".to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("aps-bid-token"), + "should fall back to ad_id when cache_id absent" + ); + assert!( + obj.get("hb_cache_host").is_none(), + "should not emit hb_cache_host when absent" + ); + assert!( + obj.get("hb_cache_path").is_none(), + "should not emit hb_cache_path when absent" + ); + } + + #[test] + fn bid_map_omits_hb_adid_when_both_cache_id_and_ad_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "amazon-aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let obj = map + .get("atf_sidebar_ad") + .expect("should have bid entry") + .as_object() + .expect("should be object"); + assert!( + obj.get("hb_adid").is_none(), + "should omit hb_adid when no cache_id and no ad_id" + ); + } + + #[test] + fn bid_map_excludes_slot_when_price_is_none() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "no-price-slot".to_string(), + Bid { + slot_id: "no-price-slot".to_string(), + price: None, + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + assert!( + map.is_empty(), + "slot with no price should be excluded from bid map" + ); + } + + #[test] + fn bids_script_is_xss_safe() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + let script = build_bids_script(&map); + let inner = script + .trim_start_matches(""); + assert!(!inner.contains('<'), "no unescaped < in bids script"); + assert!(!inner.contains('>'), "no unescaped > in bids script"); + } + + #[test] + fn bids_script_calls_ad_init_without_retry_timer() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + + let script = build_bids_script(&map); + + assert!( + script.contains("window.tsjs.adInit"), + "should hand off bids to adInit" + ); + assert!( + !script.contains("setTimeout"), + "should not retry adInit on a timer" + ); + assert!( + !script.contains("prevGptSlots"), + "should not use TS-owned slots as adInit success signal" + ); + } + + #[test] + fn auction_request_without_ec_id_omits_user_id_and_uses_non_ec_request_id() { + let slot = make_slot(); + let slots = [slot]; + let slots_ctx = MatchedSlotsContext { + matched_slots: &slots, + request_path: "/2024/01/my-article/", + }; + let request_info = RequestInfo { + host: "publisher.example.com".to_string(), + scheme: "https".to_string(), + }; + + let request = build_auction_request( + &slots_ctx, + None, + &ConsentContext::default(), + &request_info, + Some("Mozilla/5.0"), + ); + + assert_eq!(request.user.id, None, "should not forward an EC user id"); + assert!( + request.id.starts_with("ts-req-"), + "should use a non-EC request id, got {}", + request.id + ); + } + + #[test] + fn auction_request_with_ec_id_sets_user_id_and_ec_request_id() { + let slot = make_slot(); + let slots = [slot]; + let slots_ctx = MatchedSlotsContext { + matched_slots: &slots, + request_path: "/2024/01/my-article/", + }; + let request_info = RequestInfo { + host: "publisher.example.com".to_string(), + scheme: "https".to_string(), + }; + + let request = build_auction_request( + &slots_ctx, + Some("ec-abc"), + &ConsentContext::default(), + &request_info, + Some("Mozilla/5.0"), + ); + + assert_eq!( + request.user.id.as_deref(), + Some("ec-abc"), + "should forward EC id when identity consent allows it" + ); + assert_eq!( + request.id, "ts-ec-abc", + "should preserve existing EC-derived request id when present" + ); + } + + #[test] + fn html_escape_encodes_special_chars() { + assert_eq!( + html_escape_for_script("text\\with\\backslash"), + "text\\\\with\\\\backslash", + "should escape backslashes" + ); + assert_eq!( + html_escape_for_script("string\"with\"quotes"), + "string\\\"with\\\"quotes", + "should escape quotes" + ); + assert_eq!( + html_escape_for_script("simple"), + "simple", + "should not change simple text" + ); + assert_eq!( + html_escape_for_script("both\\\"mixed"), + "both\\\\\\\"mixed", + "should escape both backslashes and quotes" + ); + assert_eq!( + html_escape_for_script(""), + "\\u003Cscript\\u003Ealert(1)\\u003C/script\\u003E", + "should unicode-escape angle brackets to prevent script injection" + ); + assert_eq!( + html_escape_for_script("a&b"), + "a\\u0026b", + "should unicode-escape ampersand" + ); + assert_eq!( + html_escape_for_script("line\u{2028}sep"), + "line\\u2028sep", + "should unicode-escape U+2028 line separator" + ); + assert_eq!( + html_escape_for_script("para\u{2029}sep"), + "para\\u2029sep", + "should unicode-escape U+2029 paragraph separator" + ); + } + } + + mod page_bids_no_match_tests { + use super::super::*; + use crate::auction::AuctionOrchestrator; + use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::platform::test_support::noop_services; + use crate::test_support::tests::crate_test_settings_str; + use http::Method; + + fn settings_with_co() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") + } + + fn settings_with_co_auction_disabled() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = false\n\n[creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml).expect("should parse settings with creative_opportunities") + } + + async fn run_page_bids( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> serde_json::Value { + let response = run_page_bids_response(settings, orchestrator, slots, req).await; + serde_json::from_slice(&response.into_body().into_bytes().unwrap_or_default()) + .expect("should be json") + } + + /// `run_page_bids` with an EC context whose jurisdiction allows the + /// server-side auction, so slot-counting tests isolate the variable + /// under test (bot/prefetch) from the consent gate. The default + /// request resolves to `Jurisdiction::Unknown`, which fails the + /// consent gate and now suppresses slots. + async fn run_page_bids_consent_allowed( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> serde_json::Value { + let ec_context = consent_allowing_ec_context(); + let response = + run_page_bids_response_with_ec(settings, orchestrator, slots, &ec_context, req) + .await; + serde_json::from_slice(&response.into_body().into_bytes().unwrap_or_default()) + .expect("should be json") + } + + /// Builds an [`EcContext`] whose consent context permits the server-side + /// auction (known non-GDPR jurisdiction, no EU TCF signal). + fn consent_allowing_ec_context() -> EcContext { + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + EcContext::new_for_test(None, consent) + } + + fn article_slot() -> Vec { + vec![CreativeOpportunitySlot { + id: "atf".to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: vec!["/20**".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: crate::auction::types::MediaType::Banner, + }], + floor_price: Some(0.50), + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + }] + } + + fn make_page_bids_request(path: &str) -> Request { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!( + "https://test-publisher.com/_ts/page-bids?path={path}" + )) + .body(EdgeBody::empty()) + .expect("should build test request"); + // Pass the same-origin gate the way a browser fetch from the + // publisher page does. + set_test_header(&mut req, "sec-fetch-site", "same-origin"); + req + } + + fn set_test_header(req: &mut Request, name: &'static str, value: &'static str) { + req.headers_mut().insert( + header::HeaderName::from_static(name), + HeaderValue::from_static(value), + ); + } + + async fn run_page_bids_response( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> Response { + let ec_context = EcContext::read_from_request(settings, &req, &noop_services()) + .expect("should read EC context"); + run_page_bids_response_with_ec(settings, orchestrator, slots, &ec_context, req).await + } + + async fn run_page_bids_response_with_ec( + settings: &Settings, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + ec_context: &EcContext, + req: Request, + ) -> Response { + let services = noop_services(); + handle_page_bids( + settings, + &services, + None, + AuctionDispatch { + orchestrator, + slots, + registry: None, + }, + ec_context, + req, + ) + .await + .expect("should return ok response") + } + + #[tokio::test] + async fn cross_site_fetch_metadata_is_rejected() { + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut req = make_page_bids_request("/2024/01/my-article/"); + set_test_header(&mut req, "sec-fetch-site", "cross-site"); + + let response = + run_page_bids_response(&settings, &orchestrator, &article_slot(), req).await; + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "cross-site request should be rejected before any auction work" + ); + } + + #[tokio::test] + async fn missing_fetch_metadata_without_tsjs_header_is_rejected() { + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut req = make_page_bids_request("/2024/01/my-article/"); + req.headers_mut().remove("sec-fetch-site"); + + let response = + run_page_bids_response(&settings, &orchestrator, &article_slot(), req).await; + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "request with neither fetch metadata nor tsjs header should be rejected" + ); + } + + #[tokio::test] + async fn missing_fetch_metadata_with_tsjs_header_is_allowed() { + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut req = make_page_bids_request("/2024/01/my-article/"); + req.headers_mut().remove("sec-fetch-site"); + set_test_header(&mut req, "x-tsjs-page-bids", "1"); + + let response = + run_page_bids_response(&settings, &orchestrator, &article_slot(), req).await; + + assert_eq!( + response.status(), + StatusCode::OK, + "legacy client carrying the tsjs header should pass the gate" + ); + } + + #[tokio::test] + async fn same_site_fetch_metadata_is_rejected() { + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let mut req = make_page_bids_request("/2024/01/my-article/"); + // `same-site` admits sibling origins under the same registrable + // domain — not trusted to spend SSP quota. + set_test_header(&mut req, "sec-fetch-site", "same-site"); + + let response = + run_page_bids_response(&settings, &orchestrator, &article_slot(), req).await; + + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "same-site request should be rejected; only same-origin is trusted" + ); + } + + #[tokio::test] + async fn empty_slots_file_returns_empty_slots_and_bids() { + // Spec §8 kill-switch: creative-opportunities.toml with zero slots disables + // all server-side auction activity and injection. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let req = make_page_bids_request("/2024/01/my-article/"); + + let body = run_page_bids(&settings, &orchestrator, &[], req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "empty slots should produce zero injected slots" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "empty slots should produce zero bids" + ); + } + + #[tokio::test] + async fn bot_user_agent_returns_slots_but_no_bids() { + // Crawlers should get slot definitions (so HTML structure is unchanged) + // but the server must not burn SSP request quota running a real auction + // for them. Same gate the publisher path applies. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let mut req = make_page_bids_request("/2024/01/my-article/"); + set_test_header( + &mut req, + "user-agent", + "Mozilla/5.0 (compatible; Googlebot/2.1)", + ); + + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 1, + "bot request should still get slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "bot request must not run an auction (no SSP cost burned for crawlers)" + ); + } + + #[tokio::test] + async fn prefetch_request_returns_slots_but_no_bids() { + // Navigations triggered by Sec-Purpose=prefetch should not fire real + // SSP auctions — the user has not yet visited the page. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let mut req = make_page_bids_request("/2024/01/my-article/"); + set_test_header(&mut req, "sec-purpose", "prefetch"); + + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 1, + "prefetch request should still get slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "prefetch request must not run an auction" + ); + } + + #[tokio::test] + async fn url_not_matching_any_pattern_returns_empty_response() { + // Slots exist but request path does not match — no auction, no injection. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); // slot matches /20** only + let req = make_page_bids_request("/about"); // does not match + + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "non-matching URL should produce zero injected slots" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "non-matching URL should produce zero bids" + ); + } + + #[test] + fn normalize_page_bids_path_strips_query_fragment_and_forces_leading_slash() { + assert_eq!( + normalize_page_bids_path("/2024/01/article/"), + "/2024/01/article/", + "canonical path should pass through unchanged" + ); + assert_eq!( + normalize_page_bids_path("/2024/01/article/?utm_source=x"), + "/2024/01/article/", + "query string should be stripped before glob matching" + ); + assert_eq!( + normalize_page_bids_path("/2024/01/article/#section"), + "/2024/01/article/", + "fragment should be stripped before glob matching" + ); + assert_eq!( + normalize_page_bids_path("2024/01/article/"), + "/2024/01/article/", + "missing leading slash should be added" + ); + assert_eq!( + normalize_page_bids_path(""), + "/", + "empty path should normalize to root" + ); + } + + #[tokio::test] + async fn disabled_auction_returns_no_slots_or_bids() { + // [auction].enabled = false is a global kill switch: it must disable + // the entire server-side ad stack, not just SSP calls. Returning slot + // definitions would let the SPA hook assign `ts.adSlots` and call + // `adInit()`, creating/refreshing GPT slots client-side even though + // the auction is off. Consent is allowed here so the test isolates + // the kill switch. + let settings = settings_with_co_auction_disabled(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + let body = run_page_bids_consent_allowed(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "disabled auction must not return slot definitions (kill switch stops the ad stack)" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "disabled auction must not produce bids" + ); + } + + #[tokio::test] + async fn consent_denied_returns_no_slots_or_bids() { + // When consent denies the server-side auction (here: Jurisdiction + // Unknown fails closed), the endpoint must return no slots so the SPA + // hook does not create GPT slots client-side — matching the publisher + // navigation path's `should_run_server_side_ad_stack` gate. + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + let req = make_page_bids_request("/2024/01/my-article/"); + + // run_page_bids uses the default EC context, which resolves to + // Jurisdiction::Unknown (consent denied). + let body = run_page_bids(&settings, &orchestrator, &slots, req).await; + + assert_eq!( + body["slots"] + .as_array() + .expect("slots should be array") + .len(), + 0, + "consent denial must suppress slot definitions" + ); + assert_eq!( + body["bids"] + .as_object() + .expect("bids should be object") + .len(), + 0, + "consent denial must produce no bids" + ); + } + } + #[test] fn bounded_writer_accepts_writes_within_limit() { let mut writer = BoundedWriter::new(10); diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs new file mode 100644 index 000000000..141ef3164 --- /dev/null +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -0,0 +1,222 @@ +//! Shared response cache-privacy hardening for every platform adapter. +//! +//! The server-side ad stack and EC identity lifecycle emit per-user responses +//! (assembled HTML, `page-bids` JSON, cookie-bearing navigations) that must +//! never reach a shared cache. Each adapter's `apply_finalize_headers` applies +//! operator-configured `settings.response_headers`, so the cookie-privacy +//! downgrade and the uncacheable-operator-header guard have to live in one place +//! and run byte-identically on Fastly, Cloudflare, Axum, and Spin — a shared +//! cache such as Cloudflare would otherwise serve an operator/origin +//! `Cache-Control: public` on a cookie-bearing response as-is. + +use edgezero_core::http::{header, HeaderName, HeaderValue, Response}; + +use crate::settings::Settings; + +/// Surrogate cache headers stripped from every cookie-bearing response. +/// +/// A single source of truth so the adapter copies of the privacy downgrade +/// cannot drift apart. +pub const SURROGATE_CACHE_HEADERS: &[&str] = &["surrogate-control", "fastly-surrogate-control"]; + +/// Forces cookie-bearing responses to stay private to shared caches. +/// +/// Any response that sets a per-user cookie (notably the EC identity cookie) +/// must never be shared-cached, or a shared cache could replay one user's +/// `Set-Cookie` to others. +/// +/// Idempotent: a response already marked `private`/`no-store` keeps its stricter +/// `Cache-Control`, but the surrogate cache headers are stripped regardless so a +/// `no-store` cookie response can never retain shared cacheability. +pub fn enforce_set_cookie_cache_privacy(response: &mut Response) { + if !response.headers().contains_key(header::SET_COOKIE) { + return; + } + // Surrogate cache headers must come off every cookie-bearing response, even + // one already carrying a stricter `no-store`/`private` directive — they are + // independent of Cache-Control and would otherwise let a shared cache store + // and replay one visitor's Set-Cookie. + for name in SURROGATE_CACHE_HEADERS { + response.headers_mut().remove(*name); + } + // Cache-Control directives are case-insensitive (RFC 9111 §5.2), so match + // against a lowercased copy — `No-Store` / `Private` must count. + let already_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + if !already_uncacheable { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, max-age=0"), + ); + } +} + +/// Applies operator-configured `settings.response_headers` with cookie-privacy +/// hardening. +/// +/// First downgrades cookie-bearing responses via +/// [`enforce_set_cookie_cache_privacy`], then applies operator headers — but on +/// an uncacheable (`private`/`no-store`) response the cache-controlling headers +/// (`Cache-Control` and the surrogate cache headers) are skipped so operators +/// cannot re-enable shared caching for per-user payloads. After the operator +/// headers are applied the cookie-privacy downgrade runs once more, so a +/// configured `Set-Cookie` combined with public/surrogate cache headers cannot +/// produce a shared-cacheable cookie-bearing response. +/// +/// Invalid header names/values are logged and skipped rather than panicking, so +/// a misconfigured operator header can never take down a request. +pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: &mut Response) { + enforce_set_cookie_cache_privacy(response); + + let response_is_uncacheable = response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()) + .map(str::to_ascii_lowercase) + .is_some_and(|v| v.contains("private") || v.contains("no-store")); + + for (key, value) in &settings.response_headers { + if response_is_uncacheable + && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) + || key.eq_ignore_ascii_case("surrogate-control") + || key.eq_ignore_ascii_case("fastly-surrogate-control")) + { + continue; + } + let header_name = match HeaderName::from_bytes(key.as_bytes()) { + Ok(name) => name, + Err(_) => { + log::warn!("Skipping invalid configured response header name {key}"); + continue; + } + }; + let header_value = match HeaderValue::from_str(value) { + Ok(value) => value, + Err(_) => { + log::warn!("Skipping invalid configured response header value for {key}"); + continue; + } + }; + response.headers_mut().insert(header_name, header_value); + } + + // Operator headers can themselves introduce Set-Cookie (alongside public + // or surrogate cache headers) onto a previously cookieless response, which + // the pre-apply pass could not see. Re-run the downgrade so the final + // response can never pair Set-Cookie with shared cacheability. + enforce_set_cookie_cache_privacy(response); +} + +#[cfg(test)] +mod tests { + use super::*; + + use edgezero_core::http::response_builder; + + fn settings_with_response_headers(headers: &[(&str, &str)]) -> Settings { + let mut s = Settings::from_toml( + r#" + [[handlers]] + path = "^/_ts/admin" + username = "admin" + password = "admin-pass" + + [publisher] + domain = "test-publisher.example.com" + cookie_domain = ".test-publisher.example.com" + origin_url = "https://origin.test-publisher.example.com" + proxy_secret = "unit-test-proxy-secret" + + [ec] + passphrase = "test-secret-key-32-bytes-minimum" + "#, + ) + .expect("should load test settings"); + s.response_headers = headers + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + s + } + + #[test] + fn downgrades_public_cache_control_on_cookie_response() { + let settings = settings_with_response_headers(&[("cache-control", "public, max-age=600")]); + let mut response = response_builder() + .header(header::SET_COOKIE, "id=abc") + .header("surrogate-control", "max-age=600") + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "operator public Cache-Control must not override cookie privacy downgrade" + ); + assert!( + !response.headers().contains_key("surrogate-control"), + "surrogate cache headers must be stripped on cookie responses" + ); + } + + #[test] + fn downgrades_operator_configured_set_cookie_with_public_cache_headers() { + // Operator headers that add Set-Cookie plus shared-cache directives to + // a cookieless response must be re-downgraded after they are applied. + let settings = settings_with_response_headers(&[ + ("set-cookie", "operator=abc"), + ("cache-control", "public, max-age=600"), + ("surrogate-control", "max-age=600"), + ]); + let mut response = response_builder() + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|v| v.to_str().ok()), + Some("private, max-age=0"), + "operator Set-Cookie plus public Cache-Control must be re-downgraded to private" + ); + assert!( + !response.headers().contains_key("surrogate-control"), + "surrogate cache headers must be stripped when operator headers add Set-Cookie" + ); + assert!( + response.headers().contains_key(header::SET_COOKIE), + "the operator Set-Cookie itself should still be applied" + ); + } + + #[test] + fn applies_operator_headers_on_cookieless_response() { + let settings = settings_with_response_headers(&[("x-operator", "value")]); + let mut response = response_builder() + .body(edgezero_core::body::Body::empty()) + .expect("should build response"); + + apply_response_headers_with_cache_privacy(&settings, &mut response); + + assert_eq!( + response + .headers() + .get("x-operator") + .and_then(|v| v.to_str().ok()), + Some("value"), + "operator headers should still apply to cacheable responses" + ); + } +} diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 5075569b0..1a600a042 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -13,6 +13,7 @@ use validator::{Validate, ValidationError}; use crate::auction_config_types::AuctionConfig; use crate::consent_config::ConsentConfig; +use crate::creative_opportunities::CreativeOpportunitiesConfig; use crate::error::TrustedServerError; use crate::host_header::validate_host_header_override_value; use crate::platform::PlatformImageOptimizerRegion; @@ -1731,6 +1732,21 @@ pub struct DebugConfig { /// Fastly-observed TLS details that browser JS cannot normally read. #[serde(default)] pub ja4_endpoint_enabled: bool, + + /// Inject a `` HTML comment before `` showing + /// auction pipeline stats (SSP count, mediator status, winning bid count). + /// Never enable in production — visible in page source. + #[serde(default)] + pub auction_html_comment: bool, + + /// Include raw `adm` creative markup in `window.tsjs.bids` for GPT/GAM + /// debug rendering through the Prebid Universal Creative bridge. + /// + /// Use this to validate the server-side auction→GAM targeting→creative + /// rendering pipeline while PBS Cache is unavailable. Never enable in + /// production — injects raw HTML from SSPs. + #[serde(default)] + pub inject_adm_for_testing: bool, } /// Tester-cookie endpoint configuration. @@ -1769,6 +1785,8 @@ pub struct Settings { #[serde(default)] pub proxy: Proxy, #[serde(default)] + pub creative_opportunities: Option, + #[serde(default)] pub image_optimizer: ImageOptimizerSettings, #[serde(default)] pub debug: DebugConfig, @@ -1868,8 +1886,10 @@ impl Settings { /// /// # Errors /// - /// Returns a configuration error if any handler path regex does not compile. - pub fn prepare_runtime(&self) -> Result<(), Report> { + /// Returns a configuration error if any cached runtime artifact cannot be + /// prepared, if any handler path regex does not compile, or if a creative + /// opportunity slot is invalid. + pub fn prepare_runtime(&mut self) -> Result<(), Report> { self.image_optimizer.prepare_runtime()?; self.proxy.prepare_runtime()?; self.validate_asset_image_optimizer_profile_sets()?; @@ -1878,6 +1898,18 @@ impl Settings { handler.prepare_runtime()?; } + if let Some(co) = &mut self.creative_opportunities { + co.compile_slots(); + // Slots flow into injected HTML/JS, provider payloads, and GPT + // calls. Env/private config can bypass static review, so validate + // the full runtime shape on every load path. + co.validate_runtime().map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!("Invalid creative opportunity slot config: {err}"), + }) + })?; + } + for (name, value) in &self.response_headers { http::header::HeaderName::from_bytes(name.as_bytes()).map_err(|_| { Report::new(TrustedServerError::Configuration { @@ -1894,6 +1926,17 @@ impl Settings { Ok(()) } + /// Returns compiled creative opportunity slots, or empty slice if feature is disabled. + #[must_use] + pub fn creative_opportunity_slots( + &self, + ) -> &[crate::creative_opportunities::CreativeOpportunitySlot] { + self.creative_opportunities + .as_ref() + .map(|co| co.slot.as_slice()) + .unwrap_or(&[]) + } + /// Rejects known placeholder secret values. /// /// # Errors @@ -5044,6 +5087,210 @@ origin_host_header_overide = "www.example.com""#, /// /// If this test fails, a route was added or removed in the Fastly /// router without updating `ADMIN_ENDPOINTS` (or vice versa). + #[test] + fn settings_parses_creative_opportunities_section() { + let toml = r#" +[[handlers]] +path = "^/_ts/admin" +username = "admin" +password = "unit-test-admin-secret" + +[publisher] +domain = "example.com" +cookie_domain = ".example.com" +origin_url = "https://origin.example.com" +proxy_secret = "secret" + +[ec] +passphrase = "test-secret-key-32-bytes-minimum" + +[creative_opportunities] +gam_network_id = "21765378893" +auction_timeout_ms = 500 +"#; + let settings = Settings::from_toml(toml).expect("should parse"); + let co = settings + .creative_opportunities + .expect("should have creative_opportunities"); + assert_eq!(co.gam_network_id, "21765378893"); + assert_eq!(co.auction_timeout_ms, Some(500)); + } + + #[test] + fn settings_rejects_invalid_creative_opportunity_slot_id() { + let toml = r#" +[[handlers]] +path = "^/_ts/admin" +username = "admin" +password = "unit-test-admin-secret" + +[publisher] +domain = "example.com" +cookie_domain = ".example.com" +origin_url = "https://origin.example.com" +proxy_secret = "secret" + +[ec] +passphrase = "test-secret-key-32-bytes-minimum" + +[creative_opportunities] +gam_network_id = "21765378893" + +[[creative_opportunities.slot]] +id = "xss')).toBeUndefined(); + expect(safeAdmIframeSrc('blob:https://example.com/uuid')).toBeUndefined(); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts new file mode 100644 index 000000000..2ef3a1746 --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -0,0 +1,371 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + +import type { TsjsApi } from '../../../src/core/types'; + +type TestWindow = Window & { + googletag?: unknown; + tsjs?: TsjsApi; +}; + +const originalPushState = history.pushState.bind(history); +const originalReplaceState = history.replaceState.bind(history); + +async function importGptModule() { + return import('../../../src/integrations/gpt/index'); +} + +/** Flush the microtask/timer queue so onNavigate's awaits settle. */ +async function flushAsync(): Promise { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +describe('installSpaAuctionHook', () => { + let fetchStub: ReturnType; + // popstate listeners registered by each module import. In production the hook + // installs once (guarded by `ts.spaHookInstalled`), but tests wipe + // `window.tsjs` and re-import per test, so without explicit removal the + // listeners accumulate on the shared window and all fire on every dispatch. + let popstateHandlers: EventListenerOrEventListenerObject[] = []; + const realAddEventListener = window.addEventListener.bind(window); + + beforeEach(() => { + vi.resetModules(); + delete (window as TestWindow).tsjs; + // Restore unwrapped history methods so each module import wraps exactly + // once — without this, wrappers from prior imports accumulate. + history.pushState = originalPushState; + history.replaceState = originalReplaceState; + fetchStub = vi.fn(); + vi.stubGlobal('fetch', fetchStub); + popstateHandlers = []; + vi.spyOn(window, 'addEventListener').mockImplementation((type, listener, options) => { + if (type === 'popstate' && listener) popstateHandlers.push(listener); + return realAddEventListener(type, listener, options); + }); + }); + + afterEach(() => { + history.pushState = originalPushState; + history.replaceState = originalReplaceState; + // Reset jsdom location back to root for the next test. + originalReplaceState({}, '', '/'); + // Drop any ad containers inserted by a test so DOM state does not leak. + document.body.innerHTML = ''; + // Remove this test's popstate listener(s) so they do not fire in later tests. + popstateHandlers.forEach((handler) => window.removeEventListener('popstate', handler)); + popstateHandlers = []; + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + it('fetches page-bids on pushState and applies slots/bids via adInit', async () => { + // The route's ad container already exists, so bids apply immediately. + document.body.innerHTML = '
'; + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ + slots: [{ id: 's1', div_id: 'div-s1' }], + bids: { s1: { hb_pb: '1.00' } }, + }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/next-page'); + await flushAsync(); + + expect(fetchStub).toHaveBeenCalledWith( + '/__ts/page-bids?path=%2Fnext-page', + expect.objectContaining({ + credentials: 'include', + headers: { 'X-TSJS-Page-Bids': '1' }, + }) + ); + expect(ts.adSlots).toEqual([{ id: 's1', div_id: 'div-s1' }]); + expect(ts.bids).toEqual({ s1: { hb_pb: '1.00' } }); + expect(adInit).toHaveBeenCalledTimes(1); + }); + + it('skips adInit on an empty page-bids response with no prior TS state', async () => { + // A gated page-bids response (auction kill switch or consent denial) returns + // no slots. With no prior TS state to sweep, the hook must not call adInit() + // so a consent-denied navigation cannot activate the publisher's GPT setup. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/gated-route'); + await flushAsync(); + + expect(ts.adSlots).toEqual([]); + expect(ts.bids).toEqual({}); + expect(adInit).not.toHaveBeenCalled(); + }); + + it('runs adInit on an empty page-bids response when prior TS state exists', async () => { + // When TS touched slots on a previous navigation, an empty response still + // needs adInit() to sweep the stale TS targeting from those slots. + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + ts.prevSlotTargetingKeys = { 'div-prev': ['hb_pb'] }; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/cleanup-route'); + await flushAsync(); + + expect(ts.adSlots).toEqual([]); + expect(adInit).toHaveBeenCalledTimes(1); + }); + + it('defers applying bids until the route ad container is inserted', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ + slots: [{ id: 'late', div_id: 'div-late' }], + bids: { late: { hb_pb: '2.00' } }, + }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + // Navigate before the new route's container has rendered. + history.pushState({}, '', '/late-route'); + await flushAsync(); + expect(adInit).not.toHaveBeenCalled(); + expect(ts.adSlots).toBeUndefined(); + + // Container commits — the hook should now apply bids exactly once. + document.body.innerHTML = '
'; + await flushAsync(); + + expect(ts.adSlots).toEqual([{ id: 'late', div_id: 'div-late' }]); + expect(ts.bids).toEqual({ late: { hb_pb: '2.00' } }); + expect(adInit).toHaveBeenCalledTimes(1); + }); + + it('waits for every configured route ad container before applying bids', async () => { + document.body.innerHTML = '
'; + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ + slots: [ + { id: 'first', div_id: 'div-first' }, + { id: 'second', div_id: 'div-second' }, + ], + bids: { + first: { hb_pb: '1.00' }, + second: { hb_pb: '2.00' }, + }, + }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/multi-slot-route'); + await flushAsync(); + + expect(adInit).not.toHaveBeenCalled(); + expect(ts.adSlots).toBeUndefined(); + + const second = document.createElement('div'); + second.id = 'div-second'; + document.body.appendChild(second); + await flushAsync(); + + expect(ts.adSlots).toEqual([ + { id: 'first', div_id: 'div-first' }, + { id: 'second', div_id: 'div-second' }, + ]); + expect(ts.bids).toEqual({ + first: { hb_pb: '1.00' }, + second: { hb_pb: '2.00' }, + }); + expect(adInit).toHaveBeenCalledTimes(1); + }); + + it('does not fetch when pushState targets the current path', async () => { + await importGptModule(); + + history.pushState({}, '', '/'); + await flushAsync(); + + expect(fetchStub).not.toHaveBeenCalled(); + }); + + it('fetches on replaceState navigation', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + + history.replaceState({}, '', '/replaced'); + await flushAsync(); + expect(fetchStub).toHaveBeenCalledWith( + '/__ts/page-bids?path=%2Freplaced', + expect.objectContaining({ credentials: 'include' }) + ); + }); + + it('fetches on popstate navigation to a new path', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + + // Browsers change the URL out-of-band on back/forward, then fire popstate. + // Use the unwrapped history method so the patched handler is not invoked. + originalReplaceState({}, '', '/popped'); + window.dispatchEvent(new PopStateEvent('popstate')); + await flushAsync(); + expect(fetchStub).toHaveBeenCalledWith( + '/__ts/page-bids?path=%2Fpopped', + expect.objectContaining({ credentials: 'include' }) + ); + }); + + it('does not re-fetch on popstate to the same path', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + await importGptModule(); + + history.replaceState({}, '', '/replaced'); + await flushAsync(); + expect(fetchStub).toHaveBeenCalledTimes(1); + + // popstate on the same path (hash-only change or scroll-restoration + // back/forward) must not re-request impressions. + window.dispatchEvent(new PopStateEvent('popstate')); + await flushAsync(); + expect(fetchStub).toHaveBeenCalledTimes(1); + }); + + it('drops a stale response that resolves after a newer navigation started', async () => { + let resolveFirst: ((value: unknown) => void) | undefined; + fetchStub + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFirst = resolve; + }) + ) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ slots: [{ id: 'newer', div_id: 'div-newer' }], bids: {} }), + }); + // Container for the newer route exists so its bids apply without waiting. + document.body.innerHTML = '
'; + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/first'); + history.pushState({}, '', '/second'); + await flushAsync(); + + expect(ts.adSlots).toEqual([{ id: 'newer', div_id: 'div-newer' }]); + expect(adInit).toHaveBeenCalledTimes(1); + + // First navigation's response arrives late — it must not overwrite the + // newer route's slots or trigger another adInit. + resolveFirst!({ + ok: true, + json: async () => ({ slots: [{ id: 'stale' }], bids: {} }), + }); + await flushAsync(); + + expect(ts.adSlots).toEqual([{ id: 'newer', div_id: 'div-newer' }]); + expect(adInit).toHaveBeenCalledTimes(1); + }); + + it('leaves slots and bids untouched on a non-OK response', async () => { + fetchStub.mockResolvedValue({ ok: false, status: 500 }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + ts.adSlots = [{ id: 'existing' } as never]; + const adInit = vi.fn(); + ts.adInit = adInit; + + history.pushState({}, '', '/error-page'); + await flushAsync(); + + expect(ts.adSlots).toEqual([{ id: 'existing' }]); + expect(adInit).not.toHaveBeenCalled(); + }); + + it('retries the same path after a failed page-bids fetch (currentPath rollback)', async () => { + // A failed load must roll `currentPath` back so re-navigating to the SAME + // path retries instead of being swallowed by the no-op guard at the top of + // onNavigate. Without the rollback, currentPath would already equal the + // failed path and the second navigation would return early. + document.body.innerHTML = '
'; + fetchStub.mockResolvedValueOnce({ ok: false, status: 500 }).mockResolvedValueOnce({ + ok: true, + json: async () => ({ + slots: [{ id: 's1', div_id: 'div-s1' }], + bids: { s1: { hb_pb: '1.00' } }, + }), + }); + const { installSpaAuctionHook } = await importGptModule(); + installSpaAuctionHook(); + const ts = (window as TestWindow).tsjs!; + const adInit = vi.fn(); + ts.adInit = adInit; + + // First navigation to the path fails; nothing is applied. + history.pushState({}, '', '/retry-page'); + await flushAsync(); + expect(ts.adSlots).toBeUndefined(); + + // Re-navigate to the same path — the retry must re-fetch and apply. + history.pushState({}, '', '/retry-page'); + await flushAsync(); + + expect(fetchStub).toHaveBeenCalledTimes(2); + expect(ts.adSlots).toEqual([{ id: 's1', div_id: 'div-s1' }]); + expect(adInit).toHaveBeenCalledTimes(1); + }); + + it('is idempotent — repeated install calls do not double-fetch a navigation', async () => { + fetchStub.mockResolvedValue({ + ok: true, + json: async () => ({ slots: [], bids: {} }), + }); + const { installSpaAuctionHook } = await importGptModule(); + // Module init already installed the hook; both calls must be no-ops. + installSpaAuctionHook(); + installSpaAuctionHook(); + + history.pushState({}, '', '/once'); + await flushAsync(); + + expect(fetchStub).toHaveBeenCalledTimes(1); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index f12345d25..9ad7945c4 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -62,6 +62,7 @@ import { getInjectedConfig, auctionBidsToPrebidBids, installPrebidNpm, + installRefreshHandler, } from '../../../src/integrations/prebid/index'; import type { AuctionBid } from '../../../src/core/auction'; @@ -577,6 +578,35 @@ describe('prebid/installPrebidNpm', () => { expect(adUnits[0].bids.map((b: any) => b.bidder)).toEqual(['trustedServer']); }); + it('preserves captured bidder params when requestBids runs twice on the same ad unit', () => { + const pbjs = installPrebidNpm(); + + // First auction: inline server-side params supplied by the publisher. + const adUnits = [ + { + code: 'div-1', + bids: [ + { bidder: 'appnexus', params: { placementId: 123 } }, + { bidder: 'rubicon', params: { accountId: 'abc' } }, + ], + }, + ]; + pbjs.requestBids({ adUnits } as any); + + // Second auction (refresh/re-auction) with the SAME ad unit object: the + // server-side bidder entries were already pruned, so the shim must not + // overwrite the captured params with an empty object. + pbjs.requestBids({ adUnits } as any); + + const trustedServerBid = adUnits[0].bids.find( + (b: any) => b.bidder === 'trustedServer' + ) as any; + expect(trustedServerBid.params.bidderParams).toEqual({ + appnexus: { placementId: 123 }, + rubicon: { accountId: 'abc' }, + }); + }); + it('adds bids array to ad units that have none', () => { const pbjs = installPrebidNpm(); @@ -765,6 +795,563 @@ describe('prebid/installPrebidNpm with server-injected config', () => { }); }); +describe('prebid/installRefreshHandler', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockRequestBids.mockReset(); + mockPbjs.requestBids = mockRequestBids; + mockPbjs.adUnits = []; + (window as any).tsjs = undefined; + delete (window as any).googletag; + }); + + afterEach(() => { + (window as any).tsjs = undefined; + delete (window as any).googletag; + }); + + it('builds refresh ad units from injected slot metadata', () => { + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [ + [970, 250], + [728, 90], + ], + targeting: { zone: 'homepage', pos: 'atf' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + timeout: 750, + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + mediaTypes: { + banner: { + name: 'homepage', + sizes: [ + [970, 250], + [728, 90], + ], + }, + }, + bids: [{ bidder: 'trustedServer', params: { zone: 'homepage' } }], + }), + ], + }) + ); + }); + + it('resolves the exact slot when div_ids share a prefix', () => { + // Regression: a single find() with a startsWith() clause returned the + // first slot whose div_id is a prefix of the element id. With div_ids + // "div-ad" and "div-ad-header", refreshing the "div-ad-header" element + // must resolve to the header slot, not the shorter prefix slot. + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'prefix_ad', + gam_unit_path: '/123/prefix', + div_id: 'div-ad', + formats: [[300, 250]], + targeting: { zone: 'prefix' }, + }, + { + id: 'header_ad', + gam_unit_path: '/123/header', + div_id: 'div-ad-header', + formats: [[970, 250]], + targeting: { zone: 'header' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-header', + mediaTypes: { + banner: { + name: 'header', + sizes: [[970, 250]], + }, + }, + }), + ], + }) + ); + }); + + it('scopes the GPT targeting call to the refreshed slot code', () => { + const setTargetingForGPTAsync = vi.fn(); + (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + // Run the bidsBackHandler synchronously so the targeting call fires. + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const originalRefresh = vi.fn(); + // Only the header slot is refreshed; the footer slot must be untouched. + const headerSlot = { + getSlotElementId: vi.fn(() => 'div-ad-header'), + getTargeting: vi.fn(() => []), + clearTargeting: vi.fn().mockReturnThis(), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [headerSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'header_ad', + gam_unit_path: '/123/header', + div_id: 'div-ad-header', + formats: [[728, 90]], + targeting: { zone: 'header' }, + }, + { + id: 'footer_ad', + gam_unit_path: '/123/footer', + div_id: 'div-ad-footer', + formats: [[728, 90]], + targeting: { zone: 'footer' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh([headerSlot]); + + expect(setTargetingForGPTAsync).toHaveBeenCalledTimes(1); + expect(setTargetingForGPTAsync).toHaveBeenCalledWith(['div-ad-header']); + expect(originalRefresh).toHaveBeenCalledWith([headerSlot], undefined); + + delete (mockPbjs as any).setTargetingForGPTAsync; + }); + + it('includes configured client-side bidders in refresh ad units', () => { + (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + // Original publisher ad unit carries a client-side rubicon bid. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [ + { bidder: 'trustedServer', params: {} }, + { bidder: 'rubicon', params: { accountId: 1, siteId: 2, zoneId: 3 } }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { bidder: 'trustedServer', params: { zone: 'homepage' } }, + { bidder: 'rubicon', params: { accountId: 1, siteId: 2, zoneId: 3 } }, + ], + }), + ], + }) + ); + + delete (window as any).__tsjs_prebid; + mockPbjs.adUnits = []; + }); + + it('preserves raw server-side bidder params in refresh ad units', () => { + // Original publisher ad unit carries an inline server-side appnexus bid that + // the initial auction has not yet folded into the trustedServer bid. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [{ bidder: 'appnexus', params: { placementId: 12345 } }], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + ], + }), + ], + }) + ); + + mockPbjs.adUnits = []; + }); + + it('recovers params and client-side bids for container-backed slots by injected div_id', () => { + // A TS-owned GPT slot may be defined on `${div_id}-container`, but the + // publisher's Prebid ad unit is keyed by the inner div_id. The synthetic + // refresh code stays the GPT element id (so GPT can match it), while params + // and client-side bids are recovered from the injected div_id candidate. + (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + mockPbjs.adUnits = [ + { + code: 'div-ad-x', + bids: [ + { bidder: 'appnexus', params: { placementId: 12345 } }, + { bidder: 'rubicon', params: { accountId: 1 } }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-x-container'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'x_ad', + gam_unit_path: '/123/x', + div_id: 'div-ad-x', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + // Synthetic refresh code stays the GPT element id, not the div_id. + code: 'div-ad-x-container', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + { bidder: 'rubicon', params: { accountId: 1 } }, + ], + }), + ], + }) + ); + + delete (window as any).__tsjs_prebid; + mockPbjs.adUnits = []; + }); + + it('recovers server-side bidder params already folded onto the original trustedServer bid', () => { + // After the initial auction, the requestBids shim has folded the publisher's + // server-side params into the original ad unit's trustedServer bid. A later + // refresh must still recover them by code. + mockPbjs.adUnits = [ + { + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { bidderParams: { appnexus: { placementId: 12345 } } }, + }, + ], + }, + ]; + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [[728, 90]], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh(); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + bids: [ + { + bidder: 'trustedServer', + params: { + zone: 'homepage', + bidderParams: { appnexus: { placementId: 12345 } }, + }, + }, + ], + }), + ], + }) + ); + + mockPbjs.adUnits = []; + }); + + it('auctions refreshed TS initial slots and clears stale TS targeting before refresh', () => { + const originalRefresh = vi.fn(); + const clearTargeting = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn((key: string) => { + if (key === 'ts_initial') return ['1']; + if (key === 'zone') return ['homepage']; + return []; + }), + getSizes: vi.fn(() => [ + { getWidth: () => 970, getHeight: () => 250 }, + { getWidth: () => 728, getHeight: () => 90 }, + ]), + clearTargeting, + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + const setTargetingForGPTAsync = vi.fn(); + (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { + adSlots: [ + { + id: 'homepage_header_ad', + gam_unit_path: '/123/homepage', + div_id: 'div-ad-homepage-header', + formats: [ + [970, 250], + [728, 90], + ], + targeting: { zone: 'homepage' }, + }, + ], + }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).toHaveBeenCalledWith( + expect.objectContaining({ + timeout: 750, + adUnits: [ + expect.objectContaining({ + code: 'div-ad-homepage-header', + mediaTypes: { + banner: { + name: 'homepage', + sizes: [ + [970, 250], + [728, 90], + ], + }, + }, + bids: [{ bidder: 'trustedServer', params: { zone: 'homepage' } }], + }), + ], + }) + ); + expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(originalRefresh).not.toHaveBeenCalled(); + + const bidsBackHandler = mockRequestBids.mock.calls[0][0].bidsBackHandler; + bidsBackHandler(); + + expect(setTargetingForGPTAsync).toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); + }); + + it('passes the adInit internal refresh straight to GPT without a client-side auction', () => { + const originalRefresh = vi.fn(); + const clearTargeting = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + clearTargeting, + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { adInitRefreshInProgress: true }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).not.toHaveBeenCalled(); + expect(clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([gptSlot], undefined); + }); + + it('runs a client-side auction for publisher refreshes after adInit completes', () => { + const originalRefresh = vi.fn(); + const gptSlot = { + getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), + getTargeting: vi.fn(() => []), + clearTargeting: vi.fn(), + }; + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => [gptSlot]), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + (window as any).tsjs = { adInitRefreshInProgress: false }; + + installRefreshHandler(750); + pubads.refresh([gptSlot]); + + expect(mockRequestBids).toHaveBeenCalled(); + expect(originalRefresh).not.toHaveBeenCalled(); + }); +}); + describe('prebid/client-side bidders', () => { beforeEach(() => { vi.clearAllMocks(); @@ -972,3 +1559,47 @@ describe('prebid/client-side bidders', () => { errorSpy.mockRestore(); }); }); + +describe('prebid self-init user ID module timing', () => { + const userSyncCallCount = () => + mockSetConfig.mock.calls.filter(([arg]) => arg && typeof arg === 'object' && 'userSync' in arg) + .length; + + const setReadyState = (value: DocumentReadyState) => { + Object.defineProperty(document, 'readyState', { value, configurable: true }); + }; + + beforeEach(() => { + vi.resetModules(); + mockSetConfig.mockClear(); + }); + + afterEach(() => { + setReadyState('complete'); + }); + + it('installs user ID modules immediately when the bundle loads after window load', async () => { + // The GPT slim loader appends this bundle from a window.load handler, so + // the document is already complete — a load listener would never fire. + setReadyState('complete'); + + await import('../../../src/integrations/prebid/index'); + + expect(userSyncCallCount()).toBeGreaterThan(0); + }); + + it('defers user ID modules to window load when the document is still loading', async () => { + setReadyState('loading'); + + await import('../../../src/integrations/prebid/index'); + + expect(userSyncCallCount()).toBe(0); + + window.dispatchEvent(new Event('load')); + expect(userSyncCallCount()).toBe(1); + + // { once: true } — a second load event must not reinstall. + window.dispatchEvent(new Event('load')); + expect(userSyncCallCount()).toBe(1); + }); +}); diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index 3a55bc3de..d75958812 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -373,8 +373,8 @@ This is why mediation is important when using APS: without a mediator, APS bids ```toml [integrations.aps] enabled = true -pub_id = "5128" -endpoint = "https://aax.amazon-adsystem.com/e/dtb/bid" +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" timeout_ms = 800 ``` @@ -593,8 +593,8 @@ debug = false [integrations.aps] enabled = true -pub_id = "5128" -endpoint = "https://aax.amazon-adsystem.com/e/dtb/bid" +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" timeout_ms = 800 [integrations.adserver_mock] @@ -629,12 +629,12 @@ price_floor = 0.50 #### `[integrations.aps]` -| Field | Type | Default | Description | -| ------------ | ------ | ------------------------------------------- | --------------------------- | -| `enabled` | bool | `false` | Enable APS provider | -| `pub_id` | string | — | APS publisher ID (required) | -| `endpoint` | string | `https://aax.amazon-adsystem.com/e/dtb/bid` | APS TAM endpoint | -| `timeout_ms` | u32 | `800` | Request timeout | +| Field | Type | Default | Description | +| ------------ | ------ | ----------------------------------- | --------------------------- | +| `enabled` | bool | `false` | Enable APS provider | +| `pub_id` | string | — | APS publisher ID (required) | +| `endpoint` | string | `https://aps.example.com/e/dtb/bid` | APS TAM endpoint | +| `timeout_ms` | u32 | `800` | Request timeout | #### `[integrations.adserver_mock]` diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 4dda333c2..c7d1cdf4a 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -93,7 +93,7 @@ secret_store_id = "01GYYY" [integrations.prebid] enabled = true -server_url = "https://prebid-server.com/openrtb2/auction" +server_url = "https://prebid-server.example.com/openrtb2/auction" timeout_ms = 1200 bidders = ["kargo", "appnexus", "openx"] client_side_bidders = ["rubicon"] @@ -1011,6 +1011,8 @@ apply when the integration section exists in `trusted-server.toml`. | `bid_param_overrides` | Table | `{}` | Static per-bidder param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_zone_overrides` | Table | `{}` | Per-bidder, per-zone param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_override_rules` | Array[Table] | `[]` | Canonical ordered override rules with `when` matchers and `set` objects; evaluated after compatibility fields so later rules win on conflicts | +| `suppress_nurl` | Boolean | `false` | Strip `nurl` and `burl` from every PBS bid when the PBS deployment fires win/billing notifications server-side | +| `suppress_nurl_bidders` | Array[String] | `[]` | Bidder seats whose `nurl` and `burl` should be stripped while preserving client-side win/billing pixels for other bidders | | `debug` | Boolean | `false` | Enable debug mode (sets `ext.prebid.debug` and `returnallbidstatus`; surfaces debug metadata in responses) | | `test_mode` | Boolean | `false` | Set OpenRTB `test: 1` flag for non-billable test traffic (independent of `debug`) | | `debug_query_params` | String | `None` | Extra query params appended for debugging | @@ -1207,8 +1209,8 @@ timeout_ms = 2000 [integrations.aps] enabled = true -pub_id = "5128" -endpoint = "https://aax.amazon-adsystem.com/e/dtb/bid" +pub_id = "example-publisher" +endpoint = "https://aps.example.com/e/dtb/bid" [integrations.prebid] enabled = true diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index e77774600..bea61ab09 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -58,6 +58,8 @@ set = { placementId = "_s2sHeaderPlacement" } | `bid_param_overrides` | Table | `{}` | Static per-bidder param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_zone_overrides` | Table | `{}` | Per-bidder, per-zone param overrides; normalized into the canonical override-rule engine and shallow-merged into bidder params | | `bid_param_override_rules` | Array[Table] | `[]` | Canonical ordered override rules with `when` matchers and `set` objects; evaluated after compatibility fields so later rules win on conflicts | +| `suppress_nurl` | Boolean | `false` | Strip `nurl` and `burl` from every PBS bid when the PBS deployment fires win/billing notifications server-side | +| `suppress_nurl_bidders` | Array[String] | `[]` | Bidder seats whose `nurl` and `burl` should be stripped while preserving client-side win/billing pixels for other bidders | | `debug` | Boolean | `false` | Enable Prebid debug mode (sets `ext.prebid.debug` and `ext.prebid.returnallbidstatus`; surfaces debug metadata in auction responses) | | `test_mode` | Boolean | `false` | Set the OpenRTB `test: 1` flag so bidders treat the auction as non-billable test traffic. Separate from `debug` to avoid suppressing real demand | | `debug_query_params` | String | `None` | Extra query params appended for debugging | diff --git a/docs/superpowers/plans/2026-04-30-server-side-ad-templates.md b/docs/superpowers/plans/2026-04-30-server-side-ad-templates.md new file mode 100644 index 000000000..0429bd9ac --- /dev/null +++ b/docs/superpowers/plans/2026-04-30-server-side-ad-templates.md @@ -0,0 +1,2004 @@ +# Server-Side Ad Templates Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Enable the Fastly edge to fire a full header-bidding auction (PBS + APS) in parallel with the origin fetch, inject `window.__ts_ad_slots` into ``, then inject `window.__ts_bids` inline before `` so the client drives GPT directly — without Prebid.js and without blocking FCP. + +**Architecture:** A new `creative-opportunities.toml` holds per-URL slot templates. At request time the publisher path matches the URL and fires the auction + origin fetch concurrently via `send_async()`. `window.__ts_ad_slots` is injected at `` open with no auction wait. The `` close tag is held — bounded by `A_deadline` — until the auction completes or times out; `window.__ts_bids` is then injected inline before the close tag so bids and HTML travel together in a single response. The client's `__tsAdInit` reads `window.__ts_bids` synchronously (no fetch, no Promise) and drives GPT. Both `nurl` and `burl` fire client-side from `slotRenderEnded` to avoid billing inflation. A slim-Prebid bundle lazy-loads post-`window.load` for refresh auctions and identity warm-up. + +**Tech Stack:** Rust 2024, `lol_html` 2.7.2 (existing), `glob` crate (new workspace dep), `serde`/`toml` (existing), `std::sync::{Arc, RwLock}` for within-request shared auction state, `AuctionOrchestrator::run_auction` (existing `async fn`), TypeScript for GPT shim extension. + +> **Phase 1 streaming note:** The spec describes true streaming where body content above `` paints before the auction completes. Implementing this with lol_html's synchronous callback model requires a complex outer streaming loop (emit chunks as they arrive from origin; hold only the `` chunk until auction resolves). Phase 1 uses a simpler approach: await the auction (or `A_deadline`) before processing origin HTML through lol_html, then send the fully assembled response. This still delivers the server-side auction benefit and achieves the same ad-visible latency target (~870ms cache hit). The FCP claim (~80ms) of the spec requires the streaming approach and is tracked as a Phase 2 optimization. The shared `Arc>>` is the correct coordination primitive either way — the Phase 2 upgrade path only changes when the auction is awaited relative to lol_html processing. + +--- + +## File Map + +### New files + +| File | Responsibility | +| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------- | +| `creative-opportunities.toml` | Slot template definitions (page patterns, formats, floor prices, per-provider params) | +| `crates/trusted-server-core/src/creative_opportunities.rs` | Config types, TOML parsing, URL glob matching, slot→`AdSlot` conversion, startup validation | +| `crates/trusted-server-core/src/price_bucket.rs` | Prebid price granularity tables; converts `f64` CPM to `hb_pb` string | + +### Modified files + +| File | Change summary | +| ------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `Cargo.toml` | Add `glob = "0.3"` to `[workspace.dependencies]` | +| `crates/trusted-server-core/Cargo.toml` | Add `glob = { workspace = true }` | +| `crates/trusted-server-core/Cargo.toml` (build-deps) | Verify `regex` is listed under `[build-dependencies]` (needed for slot-ID validation in `build.rs`) | +| `crates/trusted-server-core/src/auction/types.rs` | Add `MediaType::banner()` constructor; add `ad_id: Option` to `Bid` | +| `crates/trusted-server-core/src/settings.rs` | Add `creative_opportunities: Option` to `Settings` | +| `trusted-server.toml` | Add `[creative_opportunities]` section | +| `crates/trusted-server-core/build.rs` | Validate slot IDs at build time using inline TOML parse; `rerun-if-changed` for `creative-opportunities.toml` | +| `crates/trusted-server-core/src/html_processor.rs` | Add `ad_slots_script: Option` (head) and `ad_bids_script: Arc>>` (body) to `HtmlProcessorConfig`; inject at `` open and via `el.on_end_tag()` on body element | +| `crates/trusted-server-core/src/publisher.rs` | Convert to `async fn`; add eligibility gates (bot UA, prefetch, HEAD); EID decoration; shared `AuctionBidState`; inject `__ts_bids` before ``; `Cache-Control: private, max-age=0`; strip `Surrogate-Control` | +| `crates/trusted-server-adapter-fastly/src/main.rs` | Await the now-async handler; pass orchestrator reference; no `/ts-bids` route | +| `crates/trusted-server-core/src/integrations/gpt.rs` | Extend `head_inserts()` to emit `__tsAdInit` that reads `window.__ts_bids` synchronously; `ts_initial=1` sentinel; nurl+burl from `slotRenderEnded` | +| `crates/js/lib/src/integrations/gpt/index.ts` | Synchronous `__tsAdInit` with `window.__ts_bids` read; nurl+burl `sendBeacon`; lazy slim-Prebid loader post-`window.load` | +| `crates/trusted-server-core/src/integrations/prebid.rs` | Add `suppress_nurl: bool` config (default `false`) as per-bidder escape hatch; remove server-side nurl firing | + +### Deleted (relative to prior revision of this spec) + +- `crates/trusted-server-core/src/bid_cache.rs` — never created; in-process cache rejected because Fastly Compute per-request Wasm isolates are not pinned across requests +- `/ts-bids` endpoint — never created; body-injection replaces the fetch pattern + +--- + +## Task 1: Add `glob` workspace dependency + +**Files:** + +- Modify: `Cargo.toml` +- Modify: `crates/trusted-server-core/Cargo.toml` + +- [ ] **Step 1: Write a failing compile test** + + In `crates/trusted-server-core/src/lib.rs`, temporarily add: + + ```rust + // Compilation test — remove after Step 4 + use glob::Pattern as _; + ``` + + Run: `cargo check --package trusted-server-core` + Expected: error `use of undeclared crate or module 'glob'` + +- [ ] **Step 2: Add glob to workspace `Cargo.toml`** + + Under `[workspace.dependencies]`, add: + + ```toml + glob = "0.3" + ``` + +- [ ] **Step 3: Add glob to core crate** + + In `crates/trusted-server-core/Cargo.toml` under `[dependencies]`, add: + + ```toml + glob = { workspace = true } + ``` + +- [ ] **Step 4: Remove temp import, verify compile** + + Remove the temp `use glob::Pattern as _;` from `lib.rs`. + Run: `cargo check --package trusted-server-core` + Expected: clean compile + +- [ ] **Step 5: Commit** + + ```bash + git add Cargo.toml crates/trusted-server-core/Cargo.toml + git commit -m "Add glob workspace dependency for URL pattern matching" + ``` + +--- + +## Task 2: `price_bucket.rs` — Prebid price granularity + +**Files:** + +- Create: `crates/trusted-server-core/src/price_bucket.rs` +- Modify: `crates/trusted-server-core/src/lib.rs` + +The `hb_pb` value in bid responses is a discretized bucket string from Prebid's granularity tables. "Dense" is the default used in most Prebid deployments. + +- [ ] **Step 1: Write failing tests** + + Create `crates/trusted-server-core/src/price_bucket.rs` with only the tests: + + ```rust + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn dense_below_3_increments_by_0_01() { + assert_eq!(price_bucket(0.0, PriceGranularity::Dense), "0.00"); + assert_eq!(price_bucket(0.01, PriceGranularity::Dense), "0.01"); + assert_eq!(price_bucket(0.015, PriceGranularity::Dense), "0.01"); + assert_eq!(price_bucket(1.23, PriceGranularity::Dense), "1.23"); + assert_eq!(price_bucket(2.99, PriceGranularity::Dense), "2.99"); + } + + #[test] + fn dense_3_to_8_increments_by_0_05() { + assert_eq!(price_bucket(3.00, PriceGranularity::Dense), "3.00"); + assert_eq!(price_bucket(3.03, PriceGranularity::Dense), "3.00"); + assert_eq!(price_bucket(3.05, PriceGranularity::Dense), "3.05"); + assert_eq!(price_bucket(7.99, PriceGranularity::Dense), "7.95"); + } + + #[test] + fn dense_8_to_20_increments_by_0_50() { + assert_eq!(price_bucket(8.00, PriceGranularity::Dense), "8.00"); + assert_eq!(price_bucket(8.49, PriceGranularity::Dense), "8.00"); + assert_eq!(price_bucket(8.50, PriceGranularity::Dense), "8.50"); + assert_eq!(price_bucket(19.99, PriceGranularity::Dense), "19.50"); + } + + #[test] + fn dense_above_20_caps_at_20() { + assert_eq!(price_bucket(20.00, PriceGranularity::Dense), "20.00"); + assert_eq!(price_bucket(50.00, PriceGranularity::Dense), "20.00"); + } + + #[test] + fn low_increments_by_0_50_capped_at_5() { + assert_eq!(price_bucket(0.49, PriceGranularity::Low), "0.00"); + assert_eq!(price_bucket(0.50, PriceGranularity::Low), "0.50"); + assert_eq!(price_bucket(5.01, PriceGranularity::Low), "5.00"); + } + + #[test] + fn medium_increments_by_0_10_capped_at_20() { + assert_eq!(price_bucket(1.05, PriceGranularity::Medium), "1.00"); + assert_eq!(price_bucket(1.10, PriceGranularity::Medium), "1.10"); + assert_eq!(price_bucket(20.5, PriceGranularity::Medium), "20.00"); + } + + #[test] + fn high_increments_by_0_01_capped_at_20() { + assert_eq!(price_bucket(1.234, PriceGranularity::High), "1.23"); + assert_eq!(price_bucket(20.5, PriceGranularity::High), "20.00"); + } + + #[test] + fn auto_routes_through_dense() { + assert_eq!( + price_bucket(2.53, PriceGranularity::Auto), + price_bucket(2.53, PriceGranularity::Dense) + ); + } + } + ``` + + Run: `cargo test -p trusted-server-core price_bucket` + Expected: compile error (module not yet exported from lib.rs) + +- [ ] **Step 2: Implement price_bucket.rs** + + ```rust + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, Default)] + #[serde(rename_all = "lowercase")] + pub enum PriceGranularity { + Low, + Medium, + #[default] + Dense, + High, + Auto, + } + + impl PriceGranularity { + pub fn dense() -> Self { + Self::Dense + } + } + + pub fn price_bucket(cpm: f64, granularity: PriceGranularity) -> String { + if cpm <= 0.0 { + return "0.00".to_string(); + } + match granularity { + PriceGranularity::Low => { + let capped = cpm.min(5.0); + format!("{:.2}", (capped / 0.50).floor() * 0.50) + } + PriceGranularity::Medium => { + let capped = cpm.min(20.0); + format!("{:.2}", (capped / 0.10).floor() * 0.10) + } + PriceGranularity::High => { + let capped = cpm.min(20.0); + format!("{:.2}", (capped / 0.01).floor() * 0.01) + } + PriceGranularity::Dense | PriceGranularity::Auto => dense_bucket(cpm), + } + } + + fn dense_bucket(cpm: f64) -> String { + if cpm >= 20.0 { + return "20.00".to_string(); + } + if cpm >= 8.0 { + return format!("{:.2}", (cpm / 0.50).floor() * 0.50); + } + if cpm >= 3.0 { + return format!("{:.2}", (cpm / 0.05).floor() * 0.05); + } + format!("{:.2}", (cpm / 0.01).floor() * 0.01) + } + ``` + +- [ ] **Step 3: Export from lib.rs** + + ```rust + pub mod price_bucket; + ``` + +- [ ] **Step 4: Run tests** + + Run: `cargo test -p trusted-server-core price_bucket` + Expected: all tests pass + +- [ ] **Step 5: Commit** + + ```bash + git add crates/trusted-server-core/src/price_bucket.rs \ + crates/trusted-server-core/src/lib.rs + git commit -m "Add Prebid price granularity bucketing (dense default, auto = dense)" + ``` + +--- + +## Task 3: Extend `auction::types` — `MediaType::banner()` and `Bid::ad_id` + +**Files:** + +- Modify: `crates/trusted-server-core/src/auction/types.rs` +- Modify: `crates/trusted-server-core/src/integrations/prebid.rs` + +`CreativeOpportunityFormat` uses `#[serde(default = "MediaType::banner")]` which requires a free function. Add `ad_id: Option` to `Bid` for `hb_adid` targeting. The `suppress_nurl` config escape hatch is also added here to `PrebidConfig`. + +- [ ] **Step 1: Write failing tests** + + In `auction/types.rs` test module: + + ```rust + #[test] + fn media_type_banner_fn_returns_banner() { + assert_eq!(MediaType::banner(), MediaType::Banner); + } + + #[test] + fn bid_has_ad_id_field() { + let bid = Bid { + slot_id: "s".to_string(), + price: Some(1.0), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("prebid-ad-id-abc".to_string()), + metadata: Default::default(), + }; + assert_eq!(bid.ad_id.as_deref(), Some("prebid-ad-id-abc")); + } + ``` + + Run: `cargo test -p trusted-server-core auction::types::tests` + Expected: compile error (`MediaType::banner` not found, `ad_id` field missing) + +- [ ] **Step 2: Add `MediaType::banner()` and `Bid::ad_id`** + + In `auction/types.rs`, add to `MediaType`: + + ```rust + impl MediaType { + pub fn banner() -> Self { + Self::Banner + } + } + ``` + + Add `ad_id: Option` field to `Bid`. Update the `make_bid` test helper to include `ad_id: None`. + +- [ ] **Step 3: Run tests** + + Run: `cargo test -p trusted-server-core auction` + Expected: all tests pass + +- [ ] **Step 4: Populate `ad_id` in prebid.rs** + + In `crates/trusted-server-core/src/integrations/prebid.rs`, in the `Bid` construction, find where `nurl` and `burl` are set and add: + + ```rust + ad_id: bid_obj.get("adid") + .or_else(|| bid_obj.get("id")) + .and_then(|v| v.as_str()) + .map(String::from), + ``` + + (Prebid Server uses lowercase `adid`. Fall back to `id` if absent.) + +- [ ] **Step 5: Add `suppress_nurl` to `PrebidConfig`** + + In `prebid.rs`, add to the `PrebidConfig` struct: + + ```rust + #[serde(default)] + pub suppress_nurl: bool, + ``` + + Write a test: + + ```rust + #[test] + fn prebid_config_suppress_nurl_defaults_to_false() { + let config = PrebidConfig::default(); + assert!(!config.suppress_nurl, "should not suppress nurl by default"); + } + ``` + + Run: `cargo test -p trusted-server-core integrations::prebid` + Expected: passes + +- [ ] **Step 6: Commit** + + ```bash + git add crates/trusted-server-core/src/auction/types.rs \ + crates/trusted-server-core/src/integrations/prebid.rs + git commit -m "Add MediaType::banner(), Bid::ad_id, and suppress_nurl config to PrebidConfig" + ``` + +--- + +## Task 4: `creative_opportunities.rs` — Config types and URL matching + +**Files:** + +- Create: `crates/trusted-server-core/src/creative_opportunities.rs` +- Modify: `crates/trusted-server-core/src/lib.rs` + +- [ ] **Step 1: Write failing tests** + + Create `crates/trusted-server-core/src/creative_opportunities.rs` with only tests: + + ```rust + #[cfg(test)] + mod tests { + use super::*; + + fn make_slot(id: &str, patterns: Vec<&str>) -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: id.to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: patterns.into_iter().map(String::from).collect(), + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: crate::auction::types::MediaType::Banner, + }], + floor_price: Some(0.50), + targeting: Default::default(), + providers: Default::default(), + } + } + + #[test] + fn glob_matches_article_path() { + let slot = make_slot("atf", vec!["/20**"]); + assert!(slot.matches_path("/2024/01/my-article/"), "should match article path"); + assert!(!slot.matches_path("/"), "should not match root"); + } + + #[test] + fn exact_match_homepage() { + let slot = make_slot("home", vec!["/"]); + assert!(slot.matches_path("/"), "should match root"); + assert!(!slot.matches_path("/about"), "should not match /about"); + } + + #[test] + fn slot_id_validates_alphanumeric() { + assert!(validate_slot_id("atf_sidebar_ad").is_ok()); + assert!(validate_slot_id("below-content-0").is_ok()); + assert!(validate_slot_id("").is_err(), "empty id should fail"); + assert!(validate_slot_id("xss"#.to_string() + ), + ad_bids_state: std::sync::Arc::new(std::sync::RwLock::new(None)), + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"Tcontent", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!(html.contains("window.__ts_ad_slots"), "should inject ad slots at head-open"); + assert!(!html.contains("__ts_request_id"), "must NOT inject request_id — body-injection arch has no request_id"); + } + + #[test] + fn injects_ts_bids_before_body_close() { + let bids_script = r#""#; + let state = std::sync::Arc::new(std::sync::RwLock::new( + Some(bids_script.to_string()) + )); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!(html.contains("window.__ts_bids"), "should inject bids before "); + let bids_pos = html.find("window.__ts_bids").expect("bids should be in output"); + let body_close_pos = html.find("").expect(" should be in output"); + assert!(bids_pos < body_close_pos, "bids must appear before "); + } + + #[test] + fn injects_empty_ts_bids_when_state_is_none() { + let state = std::sync::Arc::new(std::sync::RwLock::new(None)); + let config = HtmlProcessorConfig { + origin_host: "origin.example.com".to_string(), + request_host: "example.com".to_string(), + request_scheme: "https".to_string(), + integrations: IntegrationRegistry::empty_for_tests(), + ad_slots_script: None, + ad_bids_state: state, + }; + let mut processor = create_html_processor(config); + let output = processor + .process_chunk(b"content", true) + .expect("should process"); + let html = std::str::from_utf8(&output).expect("should be utf8"); + assert!(html.contains("__ts_bids=JSON.parse(\"{}\""), "should inject empty bids on None state"); + } + ``` + + Run: `cargo test -p trusted-server-core html_processor` + Expected: compile error (no `ad_bids_state` field yet) + +- [ ] **Step 2: Add `empty_for_tests()` to `IntegrationRegistry`** + + In `registry.rs`, add: + + ```rust + #[cfg(test)] + impl IntegrationRegistry { + pub fn empty_for_tests() -> Self { + Self { + inner: Arc::new(RegistryInner { + proxies: Default::default(), + attribute_rewriters: Default::default(), + script_rewriters: Vec::new(), + html_post_processors: Vec::new(), + head_injectors: Vec::new(), + metadata: Default::default(), + }) + } + } + } + ``` + + (Adjust field names to match the actual `RegistryInner` struct.) + +- [ ] **Step 3: Update `HtmlProcessorConfig`** + + ```rust + pub struct HtmlProcessorConfig { + pub origin_host: String, + pub request_host: String, + pub request_scheme: String, + pub integrations: IntegrationRegistry, + /// Pre-computed ``. + /// Injected at `` open. `None` when no slots matched. + pub ad_slots_script: Option, + /// Shared auction result script — written by the auction task before HTML processing + /// begins. Handler reads this in `el.on_end_tag()` on the body element. + /// `None` means no auction ran (consent denied, bot UA, no slot match, etc.); + /// inject empty `__ts_bids = {}` as graceful fallback. + pub ad_bids_state: std::sync::Arc>>, + } + ``` + + Update `from_settings` (or wherever `HtmlProcessorConfig` is constructed) to initialize `ad_bids_state: Arc::new(RwLock::new(None))`. + +- [ ] **Step 4: Inject `ad_slots_script` at head-open** + + In `create_html_processor`, within the existing `element!("head", ...)` handler: + + ```rust + let ad_slots_script = config.ad_slots_script.clone(); + // existing captures... + + element!("head", |el| { + let mut snippet = String::new(); + if let Some(ref slots_script) = ad_slots_script { + snippet.push_str(slots_script); + } + // existing integration head inserts... + if !snippet.is_empty() { + el.prepend(&snippet, ContentType::Html); + } + // DO NOT register on_end_tag — flushes immediately + Ok(()) + }) + ``` + +- [ ] **Step 5: Inject `__ts_bids` before `` via `el.on_end_tag()`** + + Add a new handler in `create_html_processor`. The shared state is already populated by the time lol_html reaches `` (Task 9 awaits the auction before starting HTML processing): + + ```rust + let ad_bids_state = config.ad_bids_state.clone(); + + element!("body", |el| { + let state = ad_bids_state.clone(); + el.on_end_tag(move |end_tag| { + let script = state.read().expect("should read bid state"); + let bids_script = match &*script { + Some(s) => s.clone(), + None => { + r#""#.to_string() + } + }; + end_tag.before(&bids_script, ContentType::Html); + Ok(()) + })?; + Ok(()) + }) + ``` + +- [ ] **Step 6: Run tests** + + Run: `cargo test -p trusted-server-core html_processor` + Expected: all tests pass + +- [ ] **Step 7: Run full suite** + + Run: `cargo test --workspace` + Expected: clean + +- [ ] **Step 8: Commit** + + ```bash + git add crates/trusted-server-core/src/html_processor.rs \ + crates/trusted-server-core/src/integrations/registry.rs + git commit -m "Inject __ts_ad_slots at head-open and __ts_bids before via shared auction state" + ``` + +--- + +## Task 8: `handle_publisher_request` async restructuring + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` + +> **Key constraint from spec §4.3 and §3:** No `bid_cache`. No `/ts-bids`. No `request_id`. Bids travel inline with the HTML response via body injection. The `Arc>>` is the coordination mechanism within a single request's lifetime — it is written before HTML processing and read by the lol_html `` handler. + +> **Eligibility gating (spec §4.3):** Auctions fire only for real GET requests from non-bot, non-prefetch clients with TCF Purpose 1 consent and at least one matching slot. All other requests proceed with no auction and no `__ts_bids` injection. + +> **Cache-Control (spec §4.7):** Set `Cache-Control: private, max-age=0` (not `no-store`) to preserve BFCache eligibility. Strip `Surrogate-Control` and `Fastly-Surrogate-Control`. + +- [ ] **Step 1: Update function signature** + + Change `handle_publisher_request` in `publisher.rs`: + + > **Existing context:** The existing `publisher.rs` function body already computes `consent_context`, `ec_id`, `request_info`, `origin_host`, and `backend_name` before the origin fetch. Steps below insert new logic between those existing computations and the origin fetch — they do not replace them. + + ```rust + pub async fn handle_publisher_request( + settings: &Settings, + integration_registry: &IntegrationRegistry, + services: &RuntimeServices, + orchestrator: &crate::auction::orchestrator::AuctionOrchestrator, + slots_file: &crate::creative_opportunities::CreativeOpportunitiesFile, + mut req: Request, + ) -> Result> + ``` + + Add imports at top of file: + + ```rust + use std::sync::{Arc, RwLock}; + use fastly::http::header; + use crate::auction::orchestrator::AuctionOrchestrator; + use crate::auction::types::{AuctionContext, AuctionRequest, PublisherInfo, UserInfo, SiteInfo}; + use crate::creative_opportunities::{CreativeOpportunitiesFile, match_slots}; + use crate::price_bucket::price_bucket; + ``` + + > **`send_async` return type:** `req.send_async()` returns `fastly::handle::PendingRequestHandle` (re-exported as `fastly::PendingRequest` in recent versions). Confirm the exact type from the `fastly` crate version in `Cargo.toml`; `.wait()` is the blocking resolve method on whichever type is returned. + +- [ ] **Step 2: Apply auction-eligibility gates** + + At the top of the function body, before origin fetch: + + ```rust + let request_path = req.get_path().to_string(); + let request_method = req.get_method().clone(); + + // Gate 1: Only GET triggers auctions. HEAD skips everything. + let is_get = request_method == fastly::http::Method::GET; + + // Gate 2: Skip prefetch hints (Sec-Purpose: prefetch or Purpose: prefetch). + let is_prefetch = req.get_header_str("sec-purpose") + .map_or(false, |v| v.contains("prefetch")) + || req.get_header_str("purpose") + .map_or(false, |v| v.contains("prefetch")); + + // Gate 3: Skip well-known crawler UAs (protects SSP QPS budget). + let user_agent = req.get_header_str("user-agent").unwrap_or(""); + let is_bot = ["Googlebot", "Bingbot", "AhrefsBot", "SemrushBot", "DotBot"] + .iter() + .any(|bot| user_agent.contains(bot)); + + // Gate 4: Slot match. + let matched_slots: Vec<_> = if settings.creative_opportunities.is_some() && is_get { + match_slots(&slots_file.slots, &request_path) + .into_iter() + .cloned() + .collect() + } else { + Vec::new() + }; + + // Gate 5: TCF Purpose 1 consent. + let consent_allows_auction = consent_context + .tcf + .as_ref() + .map_or(false, |tcf| tcf.has_purpose_consent(1)); + + let should_run_auction = is_get + && !is_prefetch + && !is_bot + && !matched_slots.is_empty() + && consent_allows_auction; + + let auction_timeout_ms = settings + .creative_opportunities + .as_ref() + .and_then(|co| co.auction_timeout_ms) + .unwrap_or(settings.auction.timeout_ms); + ``` + +- [ ] **Step 3: Create shared bid state, fire origin + auction concurrently** + + ```rust + // Shared state: auction task writes the ready-to-inject script; lol_html + // handler reads it. Both within the same request — no cross-request sharing. + let ad_bids_state: Arc>> = Arc::new(RwLock::new(None)); + + restrict_accept_encoding(&mut req); + req.set_header("host", &origin_host); + + // Fire origin immediately — both origin and auction SSP calls overlap on the network. + let pending_origin = req + .send_async(&backend_name) + .change_context(TrustedServerError::Proxy { + message: "Failed to dispatch async origin request".to_string(), + })?; + + // Run auction. Internal SSP calls use send_async and overlap with origin fetch. + let auction_result = if should_run_auction { + let co_config = settings.creative_opportunities.as_ref() + .expect("should be present when should_run_auction is true"); + let auction_request = build_auction_request( + &matched_slots, + &ec_id, + &consent_context, + &request_info, + co_config, + ); + let placeholder_req = fastly::Request::new(); + let auction_context = AuctionContext { + settings, + request: &placeholder_req, + client_info: &services.client_info, + timeout_ms: auction_timeout_ms, + provider_responses: None, + }; + match orchestrator.run_auction(&auction_request, &auction_context, services).await { + Ok(result) => Some(result), + Err(e) => { + log::warn!("server-side auction failed, proceeding without bids: {e:?}"); + None + } + } + } else { + None + }; + + // Write auction result to shared state before HTML processing begins. + // The lol_html handler reads this synchronously — it is always populated here. + // `build_bid_map` returns `serde_json::Map`. + if should_run_auction { + let co_config = settings.creative_opportunities.as_ref() + .expect("should be present"); + let empty_bids: std::collections::HashMap = + std::collections::HashMap::new(); + let winning_bids = auction_result.as_ref() + .map(|r| &r.winning_bids) + .unwrap_or(&empty_bids); + let bid_map = build_bid_map(winning_bids, co_config.price_granularity); + let bids_script = build_bids_script(&bid_map); + *ad_bids_state.write().expect("should write bid state") = Some(bids_script); + } + + // Await origin response (may already be buffered since we started it before the auction). + let mut response = pending_origin + .wait() + .change_context(TrustedServerError::Proxy { + message: "Failed to await origin response".to_string(), + })?; + ``` + +- [ ] **Step 4: Build head injection script, set cache headers, force chunked encoding** + + After acquiring `response`: + + ```rust + let ad_slots_script = if let Some(co_config) = &settings.creative_opportunities { + if !matched_slots.is_empty() { + Some(build_ad_slots_script(&matched_slots, co_config)) + } else { + None + } + } else { + None + }; + + // Set cache headers when slots matched. private, max-age=0 (not no-store) preserves + // BFCache eligibility — browser back/forward cache restores the already-rendered ad + // without firing a new GAM call, which is the desired behavior. + if ad_slots_script.is_some() { + response.set_header(header::CACHE_CONTROL, "private, max-age=0"); + response.remove_header("surrogate-control"); + response.remove_header("fastly-surrogate-control"); + } + + // Force chunked encoding so reaches the browser immediately as chunks arrive. + // Sending both Content-Length and Transfer-Encoding is invalid HTTP/1.1. + response.remove_header(header::CONTENT_LENGTH); + response.set_header("transfer-encoding", "chunked"); + ``` + +- [ ] **Step 5: Thread shared state into `OwnedProcessResponseParams`** + + Update `OwnedProcessResponseParams`: + + ```rust + pub struct OwnedProcessResponseParams { + // existing fields... + pub(crate) ad_slots_script: Option, + pub(crate) ad_bids_state: Arc>>, + } + ``` + + Pass both through to `create_html_stream_processor` and into `HtmlProcessorConfig`. + +- [ ] **Step 6: Add `pub(crate)` helper functions** + + > **`BidMap` type:** Use `serde_json::Map` directly — no separate module needed. + + Add helpers in this order (each function is used by the one below it, so define leaf functions first): + + ```rust + /// HTML-escape a JSON string for safe inline `"#) + } + + /// Build the `"#) + } + + fn build_auction_request( + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + ec_id: &str, + consent_context: &crate::consent::ConsentContext, + request_info: &crate::http_util::RequestInfo, + co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, + ) -> AuctionRequest { + AuctionRequest { + id: uuid::Uuid::new_v4().to_string(), + slots: matched_slots.iter() + .map(|s| s.to_ad_slot(&co_config.gam_network_id)) + .collect(), + publisher: PublisherInfo { + domain: request_info.host.clone(), + page_url: Some(format!("{}://{}", request_info.scheme, request_info.host)), + }, + user: UserInfo { + id: ec_id.to_string(), + fresh_id: uuid::Uuid::new_v4().to_string(), + consent: Some(consent_context.clone()), + }, + device: None, + site: Some(SiteInfo { + domain: request_info.host.clone(), + page: request_info.host.clone(), + }), + context: Default::default(), + } + } + ``` + + > **Type note:** All helper signatures use `serde_json::Map` directly. Do not create a `BidMap` type alias or `bid_types.rs` module. + +- [ ] **Step 7: Update `main.rs` call site** + + In `crates/trusted-server-adapter-fastly/src/main.rs`: + + ```rust + // At startup (top of main() / request handler setup, before the request dispatch loop). + // include_str! embeds the file at compile time — no runtime file I/O. + const CREATIVE_OPPORTUNITIES_TOML: &str = + include_str!("../../../creative-opportunities.toml"); + + let slots_file: trusted_server_core::creative_opportunities::CreativeOpportunitiesFile = + toml::from_str(CREATIVE_OPPORTUNITIES_TOML) + .expect("should parse creative-opportunities.toml"); + ``` + + `slots_file` is a local in the startup/handler scope and passed by reference into `handle_publisher_request` on each request — no `Arc` needed since it's immutable and the handler borrows it. + + Update the call to `handle_publisher_request`: + + ```rust + match handle_publisher_request( + settings, + integration_registry, + &publisher_services, + orchestrator, // existing + &slots_file, // new + req, + ).await { + // existing match arms unchanged + } + ``` + + There is **no `/ts-bids` route** to add. The body injection is complete within `handle_publisher_request`. + +- [ ] **Step 8: Compile check** + + Run: `cargo check --workspace` + Expected: clean compile + +- [ ] **Step 9: Run full tests** + + Run: `cargo test --workspace` + Expected: all pass + +- [ ] **Step 10: Commit** + + ```bash + git add crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-adapter-fastly/src/main.rs + git commit -m "Convert handle_publisher_request to async; body-inject __ts_bids; eligibility gates; max-age=0" + ``` + +--- + +## Task 9: GPT head injector — emit `__tsAdInit` with synchronous bid read + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/gpt.rs` + +> **Critical:** `__tsAdInit` reads `window.__ts_bids` **synchronously** — no fetch, no Promise. `window.__ts_bids` is already on the page (injected before ``) when `__tsAdInit` runs (it executes post-DCL, after `` is received). Both `nurl` and `burl` fire client-side from `slotRenderEnded`; neither is fired server-side. + +- [ ] **Step 1: Write failing test** + + ```rust + #[test] + fn head_inserts_includes_ts_ad_init_with_synchronous_bids_read() { + let config = test_config(); + let integration = GptIntegration::new(config); + let ctx = make_test_context(); + let inserts = integration.head_inserts(&ctx); + let combined = inserts.join(""); + assert!(combined.contains("__tsAdInit"), "should define __tsAdInit"); + assert!(combined.contains("window.__ts_bids"), "should read window.__ts_bids synchronously"); + assert!(combined.contains("ts_initial"), "should set ts_initial sentinel"); + assert!(combined.contains("slotRenderEnded"), "should register slotRenderEnded"); + assert!(combined.contains("sendBeacon"), "should fire nurl and burl via sendBeacon"); + assert!(combined.contains("nurl"), "should fire nurl on confirmed render"); + assert!(!combined.contains("/ts-bids"), "must NOT fetch /ts-bids — bids are inline on the page"); + assert!(!combined.contains("bidsPromise"), "must NOT use bidsPromise — bids are synchronous"); + assert!(!combined.contains("__ts_request_id"), "must NOT reference request_id — no longer used"); + } + ``` + + Run: `cargo test -p trusted-server-core integrations::gpt` + Expected: FAIL + +- [ ] **Step 2: Replace `head_inserts()` in gpt.rs** + + ```rust + impl IntegrationHeadInjector for GptIntegration { + fn integration_id(&self) -> &'static str { + GPT_INTEGRATION_ID + } + + fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { + vec![ + "" + .to_string(), + // __tsAdInit: reads window.__ts_bids synchronously (injected before ). + // No fetch, no Promise. Executes post-DCL when has already arrived. + // Both nurl and burl fire client-side from slotRenderEnded — never server-side. + // Note: window.__tsjs_installGptShim above is an EXISTING function in the + // tsjs-core bundle that stubs googletag.cmd before the real GPT loads. + concat!( + "" + ).to_string(), + ] + } + } + ``` + +- [ ] **Step 3: Run tests** + + Run: `cargo test -p trusted-server-core integrations::gpt` + Expected: all pass including new test + +- [ ] **Step 4: Commit** + + ```bash + git add crates/trusted-server-core/src/integrations/gpt.rs + git commit -m "Emit __tsAdInit with synchronous window.__ts_bids read; nurl+burl from slotRenderEnded" + ``` + +--- + +## Task 10: `gpt/index.ts` — TypeScript `__tsAdInit` with slim-Prebid lazy loader + +**Files:** + +- Modify: `crates/js/lib/src/integrations/gpt/index.ts` + +The TypeScript version mirrors the Rust inline string from Task 9 and adds the lazy slim-Prebid loader. Slim-Prebid loads post-`window.load` and handles two things: refresh auctions (via existing GPT refresh triggers) and userID module warm-up to enrich the EC graph for the next request. + +- [ ] **Step 1: Write failing tests** + + In `crates/js/lib/src/integrations/gpt/index.test.ts`: + + ```typescript + import { describe, it, expect, vi, beforeEach } from 'vitest' + + describe('installTsAdInit', () => { + beforeEach(() => { + delete (window as any).__ts_ad_slots + delete (window as any).__ts_bids + delete (window as any).__tsAdInit + }) + + it('reads window.__ts_bids synchronously and applies bid targeting before refresh', async () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('atf'), + getTargeting: vi.fn().mockReturnValue(['abc']), + } + const mockPubads = { + enableSingleRequest: vi.fn(), + addEventListener: vi.fn(), + refresh: vi.fn(), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: { pos: 'atf' }, + }, + ] + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + nurl: 'https://ssp/win', + burl: 'https://ssp/bill', + }, + } + + const fetchSpy = vi.spyOn(global, 'fetch') + + const { installTsAdInit } = await import('./index') + installTsAdInit() + ;(window as any).__tsAdInit() + + expect(fetchSpy).not.toHaveBeenCalled() + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00') + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'kargo') + expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1') + expect(mockPubads.refresh).toHaveBeenCalled() + + fetchSpy.mockRestore() + }) + + it('fires both nurl and burl via sendBeacon on slotRenderEnded when our bid won', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) + let capturedListener: ((e: any) => void) | undefined + + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('atf'), + getTargeting: vi.fn().mockReturnValue(['abc']), + } + const mockPubads = { + enableSingleRequest: vi.fn(), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: any) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn + }), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: {}, + }, + ] + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + nurl: 'https://ssp/win', + burl: 'https://ssp/bill', + }, + } + + const { installTsAdInit } = await import('./index') + installTsAdInit() + ;(window as any).__tsAdInit() + + expect(capturedListener).toBeDefined() + capturedListener!({ isEmpty: false, slot: mockSlot }) + + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/win') + expect(beaconSpy).toHaveBeenCalledWith('https://ssp/bill') + beaconSpy.mockRestore() + }) + + it('does not fire nurl/burl when bid did not win GAM line item', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true) + let capturedListener: ((e: any) => void) | undefined + + const mockSlotNoMatch = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('atf'), + getTargeting: vi.fn().mockReturnValue(['OTHER_BID_ID']), + } + const mockPubads = { + enableSingleRequest: vi.fn(), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: any) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn + }), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlotNoMatch), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [ + { + id: 'atf', + gam_unit_path: '/123/atf', + div_id: 'atf', + formats: [[300, 250]], + targeting: {}, + }, + ] + ;(window as any).__ts_bids = { + atf: { + hb_pb: '1.00', + hb_bidder: 'kargo', + hb_adid: 'abc', + nurl: 'https://ssp/win', + burl: 'https://ssp/bill', + }, + } + + const { installTsAdInit } = await import('./index') + installTsAdInit() + ;(window as any).__tsAdInit() + capturedListener!({ isEmpty: false, slot: mockSlotNoMatch }) + + expect(beaconSpy).not.toHaveBeenCalled() + beaconSpy.mockRestore() + }) + + it('calls refresh even when __ts_bids is empty (graceful fallback)', () => { + const mockPubads = { + enableSingleRequest: vi.fn(), + addEventListener: vi.fn(), + refresh: vi.fn(), + } + ;(window as any).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue({ + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + }), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + } + ;(window as any).__ts_ad_slots = [] + ;(window as any).__ts_bids = {} + + const { installTsAdInit } = require('./index') + installTsAdInit() + ;(window as any).__tsAdInit() + + expect(mockPubads.refresh).toHaveBeenCalled() + }) + }) + ``` + + Run: `cd crates/js/lib && npx vitest run` + Expected: FAIL — `installTsAdInit` not defined or assertions fail + +- [ ] **Step 2: Implement `installTsAdInit` in `index.ts`** + + Replace the old `/ts-bids` fetch implementation with: + + ```typescript + interface TsAdSlot { + id: string + gam_unit_path: string + div_id: string + formats: Array + targeting: Record + } + + interface TsBidData { + hb_pb?: string + hb_bidder?: string + hb_adid?: string + nurl?: string + burl?: string + } + + type TsWindow = Window & { + __ts_ad_slots?: TsAdSlot[] + __ts_bids?: Record + __tsAdInit?: () => void + } + + /** + * Install `window.__tsAdInit`. + * + * Reads `window.__ts_ad_slots` (injected at head-open) and `window.__ts_bids` + * (injected before ) synchronously — no fetch, no Promise. Applies bid + * targeting to GPT slots, sets the `ts_initial` sentinel, registers + * `slotRenderEnded` to fire both nurl and burl via sendBeacon when our + * specific Prebid bid wins the GAM line item match, then calls refresh(). + */ + export function installTsAdInit(): void { + const w = window as TsWindow + w.__tsAdInit = function () { + const slots = w.__ts_ad_slots ?? [] + const bids = w.__ts_bids ?? {} + const g = (window as GptWindow).googletag + if (!g) return + + g.cmd.push(() => { + const gptSlots = slots + .map((slot) => { + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats, + slot.div_id + ) + if (!gptSlot) return null + gptSlot.addService(g.pubads()) + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => + gptSlot.setTargeting(k, v) + ) + const bid = bids[slot.id] ?? {} + ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + }) + gptSlot.setTargeting('ts_initial', '1') + return { id: slot.id, gptSlot } + }) + .filter(Boolean) as Array<{ + id: string + gptSlot: NonNullable> + }> + + g.pubads().enableSingleRequest() + g.enableServices() + + g.pubads().addEventListener?.('slotRenderEnded', (event: any) => { + const slotId: string = event.slot?.getSlotElementId?.() ?? '' + const bid = bids[slotId] ?? {} + const ourBidWon = + !event.isEmpty && + bid.hb_adid && + event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + if (ourBidWon) { + if (bid.nurl) navigator.sendBeacon(bid.nurl) + if (bid.burl) navigator.sendBeacon(bid.burl) + } + }) + + g.pubads().refresh() + }) + } + } + ``` + +- [ ] **Step 3: Add lazy slim-Prebid loader (post-`window.load`)** + + After `installTsAdInit`, add: + + ```typescript + /** + * Register the slim-Prebid lazy loader. Fires after window.load — off the + * critical path. slim-Prebid handles refresh auctions and userID module + * warm-up (ID5, sharedID, LiveRamp ATS, Lockr). It skips initial-render slots + * (ts_initial=1) and registers as the GPT refresh handler for scroll/sticky auctions. + * + * Phase 1: no-op unless window.__tsjs_slim_prebid_url is set (it won't be until + * the slim-Prebid bundle build target ships in a later phase). + */ + export function installSlimPrebidLoader(): void { + const url = (window as any).__tsjs_slim_prebid_url as string | undefined + if (!url) return + window.addEventListener('load', () => { + const script = document.createElement('script') + script.src = url + script.defer = true + document.head.appendChild(script) + }) + } + ``` + + Call `installTsAdInit()` from the integration's existing initialization path — wherever the module's init function runs at page load (look for the existing `init()` or module-level call that sets up the GPT integration). Add: + + ```typescript + // In the integration's init / module entry point: + installTsAdInit() + ``` + + `window.__tsAdInit()` itself is called by `__tsAdInit` being invoked from the `"); + assert!(!inner.contains('<'), "no unescaped < in script content"); + assert!(!inner.contains('>'), "no unescaped > in script content"); + } + + #[test] + fn bid_map_includes_nurl_and_burl() { + let mut winning_bids = HashMap::new(); + winning_bids.insert("atf_sidebar_ad".to_string(), Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(2.53), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, + height: 250, + nurl: Some("https://ssp/win".to_string()), + burl: Some("https://ssp/bill".to_string()), + ad_id: Some("abc123".to_string()), + metadata: Default::default(), + }); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let entry = map.get("atf_sidebar_ad").expect("should have bid entry"); + assert_eq!(entry.get("hb_pb").and_then(|v| v.as_str()), Some("2.50")); + assert_eq!(entry.get("hb_bidder").and_then(|v| v.as_str()), Some("kargo")); + assert_eq!(entry.get("hb_adid").and_then(|v| v.as_str()), Some("abc123")); + assert_eq!(entry.get("nurl").and_then(|v| v.as_str()), Some("https://ssp/win")); + assert_eq!(entry.get("burl").and_then(|v| v.as_str()), Some("https://ssp/bill")); + } + + #[test] + fn bid_map_excludes_slot_when_price_is_none() { + let mut winning_bids = HashMap::new(); + winning_bids.insert("no-price-slot".to_string(), Bid { + slot_id: "no-price-slot".to_string(), + price: None, + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "kargo".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + metadata: Default::default(), + }); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + assert!(map.is_empty(), "slot with no price should be excluded from bid map"); + } + + #[test] + fn bids_script_is_xss_safe() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + let script = build_bids_script(&map); + let inner = script + .trim_start_matches(""); + assert!(!inner.contains('<'), "no unescaped < in bids script"); + assert!(!inner.contains('>'), "no unescaped > in bids script"); + } + + #[test] + fn html_escape_encodes_special_chars() { + assert_eq!(html_escape_for_script("", escaped) + +// After — initialise _ts if absent, then set adSlots +format!("", escaped) +``` + +`build_bids_script` generates the script injected before ``. Change: + +```rust +// Before +format!( + "", + escaped +) + +// After +format!( + "", + escaped +) +``` + +Note: `{{}}` is the Rust format-string escape for a literal `{}`. + +Update any test assertions in `publisher.rs` that check for the old global names. + +- [ ] **Step 2: Update `gpt_bootstrap.js`** + +Replace all `window.__ts*` references. The bootstrap IIFE runs before the TS bundle, so it must initialise `window._ts` if absent: + +```js +;(function () { + if (typeof window === 'undefined') return + // Initialise namespace; adInit guard prevents double-install. + var ts = (window._ts = window._ts || {}) + if (ts.adInit) return + + ts.adInit = function () { + var slots = ts.adSlots || [] + var bids = ts.bids || {} + var divToSlotId = {} + googletag.cmd.push(function () { + var newSlots = [] + slots.forEach(function (slot) { + var s = googletag.defineSlot( + slot.gam_unit_path, + slot.formats, + slot.div_id + ) + if (!s) return + s.addService(googletag.pubads()) + Object.entries(slot.targeting || {}).forEach(function (e) { + s.setTargeting(e[0], e[1]) + }) + var b = bids[slot.id] || {} + ;['hb_pb', 'hb_bidder', 'hb_adid'].forEach(function (k) { + if (b[k]) s.setTargeting(k, b[k]) + }) + s.setTargeting('ts_initial', '1') + divToSlotId[slot.div_id] = slot.id + newSlots.push(s) + }) + ts.prevGptSlots = newSlots + ts.divToSlotId = divToSlotId + if (!ts.servicesEnabled) { + googletag.pubads().enableSingleRequest() + googletag.enableServices() + ts.servicesEnabled = true + googletag.pubads().addEventListener('slotRenderEnded', function (ev) { + var divId = ev.slot.getSlotElementId() + var slotId = (ts.divToSlotId || {})[divId] + if (!slotId) return + var b = (ts.bids || {})[slotId] || {} + var ourBidWon = + !ev.isEmpty && + (b.hb_adid + ? ev.slot.getTargeting('hb_adid')[0] === b.hb_adid + : !!b.hb_bidder) + if (ourBidWon) { + if (b.nurl) navigator.sendBeacon(b.nurl) + if (b.burl) navigator.sendBeacon(b.burl) + } + }) + } + if (newSlots.length > 0) { + googletag.pubads().refresh(newSlots) + } + }) + } +})() +``` + +- [ ] **Step 3: Update `index.ts` — rename `TsWindow` type** + +Replace the `TsWindow` interface: + +```typescript +type TsNamespace = { + adSlots?: TsAdSlot[] + bids?: Record + adInit?: () => void + prevGptSlots?: GoogleTagSlot[] + servicesEnabled?: boolean + divToSlotId?: Record + spaHookInstalled?: boolean +} + +type TsWindow = Window & { + _ts?: TsNamespace +} +``` + +- [ ] **Step 4: Update `installTsAdInit` in `index.ts`** + +Update all properties to live under `window.tsjs`. Use `window.tsjs` directly: + +```typescript +export function installTsAdInit(): void { + const w = window as TsWindow + const ts = (w._ts = w._ts ?? {}) + ts.adInit = function () { + const slots = ts.adSlots ?? [] + const bids = ts.bids ?? {} + const g = (window as GptWindow).googletag + if (!g) return + + g.cmd?.push(() => { + if (ts.prevGptSlots && ts.prevGptSlots.length > 0) { + g.destroySlots?.(ts.prevGptSlots) + ts.prevGptSlots = [] + } + const newSlots: GoogleTagSlot[] = [] + const divToSlotId: Record = {} + + slots.forEach((slot) => { + const gptSlot = g.defineSlot?.( + slot.gam_unit_path, + slot.formats as Array, + slot.div_id + ) + if (!gptSlot) return + gptSlot.addService(g.pubads!()) + Object.entries(slot.targeting ?? {}).forEach(([k, v]) => + gptSlot.setTargeting(k, v) + ) + const bid = bids[slot.id] ?? {} + ;(['hb_pb', 'hb_bidder', 'hb_adid'] as const).forEach((key) => { + if (bid[key]) gptSlot.setTargeting(key, bid[key]!) + }) + gptSlot.setTargeting('ts_initial', '1') + divToSlotId[slot.div_id] = slot.id + newSlots.push(gptSlot) + }) + + ts.prevGptSlots = newSlots + ts.divToSlotId = divToSlotId + + if (!ts.servicesEnabled) { + g.pubads!().enableSingleRequest() + g.enableServices?.() + ts.servicesEnabled = true + g.pubads!().addEventListener?.( + 'slotRenderEnded', + (event: SlotRenderEndedEvent) => { + const divId: string = event.slot?.getSlotElementId?.() ?? '' + const slotId = (ts.divToSlotId ?? {})[divId] + if (!slotId) return + const bid = (ts.bids ?? {})[slotId] ?? {} + const ourBidWon = + !event.isEmpty && + (bid.hb_adid + ? event.slot?.getTargeting?.('hb_adid')?.[0] === bid.hb_adid + : !!bid.hb_bidder) + if (ourBidWon) { + if (bid.nurl) navigator.sendBeacon(bid.nurl) + if (bid.burl) navigator.sendBeacon(bid.burl) + } + } + ) + } + if (newSlots.length > 0) { + g.pubads!().refresh(newSlots) + } + }) + } +} +``` + +- [ ] **Step 5: Update `installSpaHook` in `index.ts`** + +Replace `__tsSpaHookInstalled` and `__ts_ad_slots`/`__ts_bids` reads: + +```typescript +export function installSpaHook(): void { + const win = window as TsWindow + const ts = (win._ts = win._ts ?? {}) + if (ts.spaHookInstalled) return + ts.spaHookInstalled = true + // ... rest of SPA hook logic uses ts.adSlots, ts.bids, ts.adInit +} +``` + +- [ ] **Step 6: Update tests in `index.test.ts`** + +Find all test assertions that reference `window.__ts_ad_slots`, `window.__ts_bids`, `window.__tsAdInit`, etc. and update to `window.tsjs.adSlots`, `window.tsjs.bids`, `window.tsjs.adInit` etc. + +Run tests first to see what fails: + +```bash +cd crates/js/lib && npx vitest run +``` + +Fix each failing assertion. + +- [ ] **Step 7: Run JS tests and format** + +```bash +cd crates/js/lib && npx vitest run +cd crates/js/lib && npm run format +``` + +Expected: all tests pass, no format errors. + +- [ ] **Step 8: Run Rust tests** + +```bash +cargo test --workspace +``` + +Update any test assertions in `publisher.rs` that check for old global names (e.g. `script.contains("window.__ts_ad_slots")`). + +- [ ] **Step 9: Run clippy and fmt** + +```bash +cargo fmt --all -- --check +cargo clippy --workspace --all-targets --all-features -- -D warnings +``` + +- [ ] **Step 10: Commit** + +```bash +git commit -m "Namespace window globals under window._ts" +``` + +--- + +## Task 3: Fix `formats` type and extract `ts_initial` constant + +**What:** Two small TypeScript/JS cleanups. `TsAdSlot.formats` should be typed as `Array<[number, number]>` (tuple, not array-of-array) to match GPT's actual input. The string `'ts_initial'` is hardcoded in both `gpt_bootstrap.js` and `index.ts` — extract as a named constant in `index.ts` (no JS equivalent needed since the bootstrap is vanilla JS). + +**Files:** + +- Modify: `crates/js/lib/src/integrations/gpt/index.ts` +- Modify: `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` (comment only — JS can't share TS constants) + +**Steps:** + +- [ ] **Step 1: Fix `TsAdSlot.formats` type** + +In `index.ts`, change: + +```typescript +// Before +interface TsAdSlot { + ... + formats: Array; +} + +// After +interface TsAdSlot { + ... + formats: Array<[number, number]>; +} +``` + +Update the cast at the GPT `defineSlot` call site — `[number, number]` satisfies `number | number[]` so the cast can be removed or simplified: + +```typescript +// Before +slot.formats as Array + +// After — [number, number][] already satisfies Array +slot.formats +``` + +- [ ] **Step 2: Extract `ts_initial` constant in `index.ts`** + +Near the top of `index.ts`, add: + +```typescript +const TS_INITIAL_TARGETING_KEY = 'ts_initial' +``` + +Replace both occurrences of `'ts_initial'` in `installTsAdInit` with `TS_INITIAL_TARGETING_KEY`. + +Add a comment in `gpt_bootstrap.js` where `'ts_initial'` appears: + +```js +// Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts +s.setTargeting('ts_initial', '1') +``` + +- [ ] **Step 3: Run JS tests and format** + +```bash +cd crates/js/lib && npx vitest run +cd crates/js/lib && npm run format +``` + +- [ ] **Step 4: Commit** + +```bash +git commit -m "Fix TsAdSlot formats type and extract ts_initial constant" +``` + +--- + +## Final verification + +- [ ] `cargo fmt --all -- --check` +- [ ] `cargo clippy --workspace --all-targets --all-features -- -D warnings` +- [ ] `cargo test --workspace` +- [ ] `cd crates/js/lib && npx vitest run` +- [ ] `cd crates/js/lib && npm run format` +- [ ] `cd docs && npm run format` diff --git a/docs/superpowers/plans/2026-05-29-prebid-creative-rendering-fix.md b/docs/superpowers/plans/2026-05-29-prebid-creative-rendering-fix.md new file mode 100644 index 000000000..7a3f34207 --- /dev/null +++ b/docs/superpowers/plans/2026-05-29-prebid-creative-rendering-fix.md @@ -0,0 +1,760 @@ +# Prebid Creative Rendering Fix Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix `hb_adid` to carry the PBS Cache UUID (not the OpenRTB bid ID) so the Prebid Universal Creative in GAM can fetch and render the correct creative markup. + +**Architecture:** Three-file change: add `cache_id`/`cache_host`/`cache_path` fields to the shared `Bid` struct in `types.rs`, extract these from `ext.prebid.cache.bids` in `prebid.rs`'s `parse_bid`, then emit them as `hb_adid`/`hb_cache_host`/`hb_cache_path` in `publisher.rs`'s `build_bid_map`. `AuctionBid` in `prebid.rs` is a type alias for `Bid` (`use ... Bid as AuctionBid`), so only one struct needs the new fields. + +**Tech Stack:** Rust 2024, `serde`, `url` crate (already in workspace deps at v2.5.8), `cargo test --workspace` + +--- + +## Context for all tasks + +- **Branch:** `fix/server-side-ad-template-entrypoint` (already checked out) +- **Spec:** `docs/superpowers/specs/2026-05-29-prebid-creative-rendering-fix.md` +- **Error handling:** `error-stack` (`Report`), not anyhow. Use `expect("should ...")` not `unwrap()`. +- **No `println!`/`eprintln!`** — use `log::` macros. +- **All public items must have doc comments.** +- CI gates: `cargo fmt --all -- --check`, `cargo clippy --workspace --all-targets --all-features -- -D warnings`, `cargo test --workspace` + +--- + +## Task 1: Add cache fields to `Bid` struct and fix all construction sites + +**What:** Add three new `Option` fields to `Bid`. Since Rust struct literals are exhaustive, every place that constructs a `Bid { ... }` in the codebase will fail to compile until the new fields are added. Fix all of them with `None` defaults (except the APS provider which constructs a real `Bid` — also `None` since APS doesn't use PBS Cache). + +**Files:** + +- Modify: `crates/trusted-server-core/src/auction/types.rs:200` (after `ad_id` field) +- Modify (test helpers/literals — add `None` fields): + - `crates/trusted-server-core/src/auction/types.rs:314` (`make_bid` helper) + - `crates/trusted-server-core/src/auction/types.rs:445` (inline `Bid` literal) + - `crates/trusted-server-core/src/publisher.rs:2616` (`make_bid` helper) + - `crates/trusted-server-core/src/publisher.rs:2714` (inline `Bid` literal) + - `crates/trusted-server-core/src/auction/orchestrator.rs:1121,1138,1278,1325,1358` (test `Bid` literals) + - `crates/trusted-server-core/src/integrations/aps.rs:442` (production `Bid` construction) + +**Steps:** + +- [ ] **Step 1: Add three fields to `Bid` struct in `types.rs`** + + In `crates/trusted-server-core/src/auction/types.rs`, after line 200 (`pub ad_id: Option,`), add: + + ```rust + /// Prebid Cache UUID for this bid. + /// + /// Populated from `ext.prebid.cache.bids.cacheId` in the PBS response. + /// Used as `hb_adid` targeting value in `window._ts.bids`. `None` for + /// non-PBS providers (e.g., APS) and PBS bids without Prebid Cache enabled. + pub cache_id: Option, + /// Prebid Cache host (e.g., `"openads.adsrvr.org"`). + /// + /// Populated from the host of `ext.prebid.cache.bids.url`. Used as + /// `hb_cache_host` targeting value. `None` when cache is absent. + pub cache_host: Option, + /// Prebid Cache path (e.g., `"/cache"`). + /// + /// Populated from the path of `ext.prebid.cache.bids.url`. Used as + /// `hb_cache_path` targeting value. `None` when cache is absent. + pub cache_path: Option, + ``` + +- [ ] **Step 2: Verify compile fails as expected** + + ```bash + cargo check --package trusted-server-core 2>&1 | grep "missing field" + ``` + + Expected: multiple errors about missing `cache_id`, `cache_host`, `cache_path` in `Bid` struct literals. This confirms every construction site will be found. + +- [ ] **Step 3: Fix `make_bid` helper in `types.rs` (line ~314)** + + Add three `None` fields to the `Bid {}` literal inside the `make_bid` test helper: + + ```rust + fn make_bid(bidder: &str) -> Bid { + Bid { + slot_id: "slot-1".to_string(), + price: Some(1.0), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: bidder.to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: HashMap::new(), + } + } + ``` + +- [ ] **Step 4: Fix inline `Bid` literal in `types.rs` (line ~445)** + + Find the `Bid {` literal around line 445 in the test section of `types.rs`. Add: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 5: Fix `make_bid` helper in `publisher.rs` (line ~2616)** + + In the `make_bid` test helper function in `publisher.rs`, add to the `Bid {}` literal: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 6: Fix inline `Bid` literal in `publisher.rs` (line ~2714)** + + Find the `Bid {` literal around line 2714 in `publisher.rs` tests. Add: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 7: Fix five `Bid` literals in `orchestrator.rs` (lines ~1121,1138,1278,1325,1358)** + + Add to each of the five `Bid {}` literals in the test section of `orchestrator.rs`: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + +- [ ] **Step 8: Fix APS production `Bid` construction in `aps.rs` (line ~442)** + + In `aps.rs`, inside `parse_aps_response` (or wherever the `Ok(Bid { ... })` is around line 442), add: + + ```rust + cache_id: None, + cache_host: None, + cache_path: None, + ``` + + APS does not use PBS Cache — these fields are intentionally `None` for APS bids. + +- [ ] **Step 9: Verify compile succeeds** + + ```bash + cargo check --package trusted-server-core 2>&1 | grep -E "^error" + ``` + + Expected: no output (clean compile). + +- [ ] **Step 10: Run tests to confirm nothing regressed** + + ```bash + cargo test --workspace 2>&1 | tail -5 + ``` + + Expected: all tests pass. + +- [ ] **Step 11: Run clippy and fmt** + + ```bash + cargo fmt --all + cargo clippy --workspace --all-targets --all-features -- -D warnings 2>&1 | tail -5 + ``` + + Expected: clean. + +- [ ] **Step 12: Commit** + + ```bash + git add crates/trusted-server-core/src/auction/types.rs \ + crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-core/src/auction/orchestrator.rs \ + crates/trusted-server-core/src/integrations/aps.rs + git commit -m "Add cache_id, cache_host, cache_path fields to Bid struct" + ``` + +--- + +## Task 2: Extract PBS Cache fields in `prebid.rs` `parse_bid` + tests + +**What:** After extracting `ad_id` in `parse_bid`, extract `ext.prebid.cache.bids.cacheId` as `cache_id` and split `ext.prebid.cache.bids.url` into `cache_host` + `cache_path`. Populate all three new fields on the returned `AuctionBid`. Add TDD tests first. + +**Files:** + +- Modify: `crates/trusted-server-core/src/integrations/prebid.rs:1362–1391` (extraction + struct literal) +- Test: `crates/trusted-server-core/src/integrations/prebid.rs` (test module near bottom) + +**Steps:** + +- [ ] **Step 1: Write the failing tests** + + Find the `#[cfg(test)]` module in `prebid.rs`. Add these tests (they will fail because extraction doesn't exist yet): + + ```rust + #[test] + fn parse_bid_extracts_cache_id_from_ext_prebid_cache_bids() { + // Real PBS response shape from auction_response.json + let bid_json = serde_json::json!({ + "id": "bid-id-123", + "impid": "atf_sidebar_ad", + "price": 1.50, + "adm": "
ad
", + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://openads.adsrvr.org/cache?uuid=f47447a0-b759-4f2f-9887-af458b79b570", + "cacheId": "f47447a0-b759-4f2f-9887-af458b79b570" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "thetradedesk") + .expect("should parse bid"); + assert_eq!( + bid.cache_id.as_deref(), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should extract cacheId as cache_id" + ); + assert_eq!( + bid.cache_host.as_deref(), + Some("openads.adsrvr.org"), + "should extract host from cache URL" + ); + assert_eq!( + bid.cache_path.as_deref(), + Some("/cache"), + "should extract path from cache URL" + ); + } + + #[test] + fn parse_bid_sets_cache_fields_to_none_when_no_cache_entry() { + let bid_json = serde_json::json!({ + "id": "bid-id-456", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250 + // no ext.prebid.cache + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert!(bid.cache_id.is_none(), "should be None when cache absent"); + assert!(bid.cache_host.is_none(), "should be None when cache absent"); + assert!(bid.cache_path.is_none(), "should be None when cache absent"); + } + + #[test] + fn parse_bid_handles_malformed_cache_url_gracefully() { + let bid_json = serde_json::json!({ + "id": "bid-id-789", + "impid": "atf_sidebar_ad", + "price": 0.50, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "not-a-valid-url", + "cacheId": "some-uuid" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid without panicking"); + assert_eq!( + bid.cache_id.as_deref(), + Some("some-uuid"), + "should still extract cacheId even if URL is malformed" + ); + assert!(bid.cache_host.is_none(), "should be None when URL parse fails"); + assert!(bid.cache_path.is_none(), "should be None when URL parse fails"); + } + + #[test] + fn parse_bid_preserves_ad_id_alongside_cache_id() { + let bid_json = serde_json::json!({ + "id": "bid-impression-id", + "impid": "atf_sidebar_ad", + "adid": "bidder-ad-id-abc", + "price": 1.0, + "w": 300, + "h": 250, + "ext": { + "prebid": { + "cache": { + "bids": { + "url": "https://cache.example.com/cache", + "cacheId": "cache-uuid-xyz" + } + } + } + } + }); + let provider = PrebidAuctionProvider::new(base_config()); + let bid = provider + .parse_bid(&bid_json, "appnexus") + .expect("should parse bid"); + assert_eq!( + bid.ad_id.as_deref(), + Some("bidder-ad-id-abc"), + "should keep ad_id from adid field" + ); + assert_eq!( + bid.cache_id.as_deref(), + Some("cache-uuid-xyz"), + "should extract cache UUID separately" + ); + } + ``` + + Note: `base_config()` and `PrebidAuctionProvider::new()` are the standard test construction pattern used throughout the existing `prebid.rs` test module. `parse_bid` is a private method but is accessible from the `#[cfg(test)]` module in the same file. + +- [ ] **Step 2: Run tests to verify they fail** + + ```bash + cargo test --package trusted-server-core parse_bid_extracts_cache_id 2>&1 | tail -15 + ``` + + Expected: compile error (`no field 'cache_id' on type 'Bid'`) or test failure. Either confirms the extraction code is missing. + +- [ ] **Step 3: Add cache extraction to `parse_bid` in `prebid.rs`** + + In `parse_bid` (around line 1362), after the `ad_id` extraction block and before the `Ok(AuctionBid { ... })`, add: + + ```rust + // Extract PBS Cache coordinates from ext.prebid.cache.bids. + // The Prebid Universal Creative uses cacheId as hb_adid and the host/path + // to construct the fetch URL: https://?uuid= + let cache_entry = bid_obj + .get("ext") + .and_then(|e| e.get("prebid")) + .and_then(|p| p.get("cache")) + .and_then(|c| c.get("bids")); + + let cache_id = cache_entry + .and_then(|c| c.get("cacheId")) + .and_then(|v| v.as_str()) + .map(String::from); + + let (cache_host, cache_path) = cache_entry + .and_then(|c| c.get("url")) + .and_then(|v| v.as_str()) + .and_then(|url_str| { + url::Url::parse(url_str) + .map_err(|e| log::debug!("PBS cache URL parse failed: {e}")) + .ok() + }) + .map(|u| { + let host = u.host_str().map(String::from); + let path = u.path().to_string(); + let path = if path.is_empty() || path == "/" { + None + } else { + Some(path) + }; + (host, path) + }) + .unwrap_or((None, None)); + + if cache_id.is_some() && cache_host.is_none() { + log::warn!( + "PBS bid has cache UUID but cache URL could not be parsed — \ + creative will fail to render for slot '{slot_id}'" + ); + } + ``` + + Then add the three fields to the `Ok(AuctionBid { ... })` struct literal (around line 1377): + + ```rust + Ok(AuctionBid { + slot_id, + price: Some(price), + currency: DEFAULT_CURRENCY.to_string(), + creative, + adomain, + bidder: seat.to_string(), + width, + height, + nurl, + burl, + ad_id, + cache_id, + cache_host, + cache_path, + metadata: std::collections::HashMap::new(), + }) + ``` + +- [ ] **Step 4: Run tests to verify they pass** + + ```bash + cargo test --package trusted-server-core parse_bid 2>&1 | tail -20 + ``` + + Expected: all 4 new tests pass. + +- [ ] **Step 5: Run full test suite** + + ```bash + cargo test --workspace 2>&1 | tail -5 + ``` + + Expected: all tests pass. + +- [ ] **Step 6: Run clippy and fmt** + + ```bash + cargo fmt --all + cargo clippy --workspace --all-targets --all-features -- -D warnings 2>&1 | tail -5 + ``` + + Expected: clean. If clippy warns about the `log::debug!` return value being unused inside `map_err`, suppress with `let _ = ...` or restructure. + +- [ ] **Step 7: Commit** + + ```bash + git add crates/trusted-server-core/src/integrations/prebid.rs + git commit -m "Extract PBS Cache UUID and endpoint from bid ext into Bid fields" + ``` + +--- + +## Task 3: Emit cache fields in `build_bid_map` + update tests + +**What:** Change `build_bid_map` to use `bid.cache_id` for `hb_adid` (falling back to `bid.ad_id` for APS/other providers), and emit `hb_cache_host`/`hb_cache_path` when present. Update the existing `bid_map_includes_nurl_and_burl` test (which currently passes `"abc123"` as `ad_id` and asserts `hb_adid = "abc123"`) to use a cache-based bid. Add new tests covering cache fields and fallback path. + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs:1311–1342` (`build_bid_map`) +- Modify: `crates/trusted-server-core/src/publisher.rs:2608–2630` (`make_bid` helper — add cache params) +- Modify: `crates/trusted-server-core/src/publisher.rs:2666–2707` (existing `bid_map_includes_nurl_and_burl` test) +- Test: `crates/trusted-server-core/src/publisher.rs` (new tests in the existing test module) + +**Steps:** + +- [ ] **Step 1: Write new failing tests for cache field emission** + + Add these tests to the `#[cfg(test)]` module in `publisher.rs`, near the existing `bid_map_includes_nurl_and_burl` test: + + ```rust + #[test] + fn bid_map_uses_cache_id_for_hb_adid_when_present() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "thetradedesk".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-impression-id".to_string()), + cache_id: Some("f47447a0-b759-4f2f-9887-af458b79b570".to_string()), + cache_host: Some("openads.adsrvr.org".to_string()), + cache_path: Some("/cache".to_string()), + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let obj = map + .get("atf_sidebar_ad") + .expect("should have entry") + .as_object() + .expect("should be object"); + + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("f47447a0-b759-4f2f-9887-af458b79b570"), + "should use cache_id for hb_adid, not ad_id" + ); + assert_eq!( + obj.get("hb_cache_host").and_then(|v| v.as_str()), + Some("openads.adsrvr.org"), + "should emit hb_cache_host" + ); + assert_eq!( + obj.get("hb_cache_path").and_then(|v| v.as_str()), + Some("/cache"), + "should emit hb_cache_path" + ); + } + + #[test] + fn bid_map_falls_back_to_ad_id_when_cache_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "aps-amazon".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("aps-bid-token".to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let obj = map + .get("atf_sidebar_ad") + .expect("should have entry") + .as_object() + .expect("should be object"); + + assert_eq!( + obj.get("hb_adid").and_then(|v| v.as_str()), + Some("aps-bid-token"), + "should fall back to ad_id when cache_id absent" + ); + assert!( + obj.get("hb_cache_host").is_none(), + "should not emit hb_cache_host when absent" + ); + assert!( + obj.get("hb_cache_path").is_none(), + "should not emit hb_cache_path when absent" + ); + } + + #[test] + fn bid_map_omits_hb_adid_when_both_cache_id_and_ad_id_absent() { + let mut winning_bids = HashMap::new(); + winning_bids.insert( + "atf_sidebar_ad".to_string(), + Bid { + slot_id: "atf_sidebar_ad".to_string(), + price: Some(0.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "amazon-aps".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + }, + ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense); + let obj = map + .get("atf_sidebar_ad") + .expect("should have entry") + .as_object() + .expect("should be object"); + + assert!( + obj.get("hb_adid").is_none(), + "should omit hb_adid when no cache_id and no ad_id" + ); + } + ``` + +- [ ] **Step 2: Run tests to verify they fail** + + ```bash + cargo test --package trusted-server-core bid_map_uses_cache_id 2>&1 | tail -15 + ``` + + Expected: test fails — `hb_adid` returns `"bid-impression-id"` (the wrong value) instead of the cache UUID, and `hb_cache_host`/`hb_cache_path` are not emitted. + +- [ ] **Step 3: Update `build_bid_map` in `publisher.rs`** + + Replace the current `hb_adid` emission block (lines ~1326–1331) and the `nurl`/`burl` block with: + + ```rust + // hb_adid: PBS Cache UUID when present (Prebid Universal Creative uses this + // as the cache lookup key). Falls back to ad_id for APS and other non-PBS + // providers. Note: ad_id (OpenRTB bid ID) is NOT the same as the cache UUID. + let hb_adid = bid.cache_id.as_deref().or(bid.ad_id.as_deref()); + if let Some(id) = hb_adid { + obj.insert( + "hb_adid".to_string(), + serde_json::Value::String(id.to_string()), + ); + } + + // Cache endpoint coordinates — only present for PBS bids with Prebid Cache. + // The Prebid Universal Creative constructs: + // https://?uuid= + if let Some(ref host) = bid.cache_host { + obj.insert( + "hb_cache_host".to_string(), + serde_json::Value::String(host.clone()), + ); + } + if let Some(ref path) = bid.cache_path { + obj.insert( + "hb_cache_path".to_string(), + serde_json::Value::String(path.clone()), + ); + } + + if let Some(ref nurl) = bid.nurl { + obj.insert("nurl".to_string(), serde_json::Value::String(nurl.clone())); + } + if let Some(ref burl) = bid.burl { + obj.insert("burl".to_string(), serde_json::Value::String(burl.clone())); + } + ``` + +- [ ] **Step 4: Update the existing `bid_map_includes_nurl_and_burl` test** + + The existing test at line ~2666 constructs a bid via `make_bid("atf_sidebar_ad", 1.50, "kargo", "abc123", ...)` and asserts `hb_adid = "abc123"`. Update `make_bid` to accept optional `cache_id`, `cache_host`, `cache_path`, OR create a separate variant. The simplest fix: update the assertion in the existing test to reflect the new priority logic. + + The test currently passes `ad_id = "abc123"` and `cache_id = None`. After the fix, `hb_adid` should still be `"abc123"` (fallback path). So the existing assertion is correct — just verify it still passes. No change needed to that test body. Just update `make_bid` to set the new fields to `None`: + + ```rust + fn make_bid( + slot_id: &str, + price: f64, + bidder: &str, + ad_id: &str, + nurl: &str, + burl: &str, + ) -> Bid { + Bid { + slot_id: slot_id.to_string(), + price: Some(price), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: bidder.to_string(), + width: 300, + height: 250, + nurl: Some(nurl.to_string()), + burl: Some(burl.to_string()), + ad_id: Some(ad_id.to_string()), + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + } + } + ``` + + Also update the assertion comment at line ~2694 from `"should include ad_id"` to `"should fall back to ad_id when no cache_id"`. + +- [ ] **Step 5: Run all new tests** + + ```bash + cargo test --package trusted-server-core bid_map 2>&1 | tail -20 + ``` + + Expected: all `bid_map_*` tests pass, including both new and existing. + +- [ ] **Step 6: Add round-trip serialization test for `Bid`** + + Add this test to the `#[cfg(test)]` module in `types.rs`: + + ```rust + #[test] + fn bid_with_cache_fields_round_trips_through_json() { + let bid = Bid { + slot_id: "atf".to_string(), + price: Some(1.50), + currency: "USD".to_string(), + creative: None, + adomain: None, + bidder: "thetradedesk".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: Some("bid-id".to_string()), + cache_id: Some("cache-uuid".to_string()), + cache_host: Some("cache.example.com".to_string()), + cache_path: Some("/pbc/v1/cache".to_string()), + metadata: HashMap::new(), + }; + let json = serde_json::to_string(&bid).expect("should serialize Bid"); + let restored: Bid = serde_json::from_str(&json).expect("should deserialize Bid"); + assert_eq!(restored.cache_id.as_deref(), Some("cache-uuid"), "should round-trip cache_id"); + assert_eq!(restored.cache_host.as_deref(), Some("cache.example.com"), "should round-trip cache_host"); + assert_eq!(restored.cache_path.as_deref(), Some("/pbc/v1/cache"), "should round-trip cache_path"); + } + ``` + + Run: + + ```bash + cargo test --package trusted-server-core bid_with_cache_fields_round_trips 2>&1 | tail -5 + ``` + + Expected: PASS. + +- [ ] **Step 7: Run full CI suite** + + ```bash + cargo test --workspace 2>&1 | tail -5 + cargo fmt --all -- --check + cargo clippy --workspace --all-targets --all-features -- -D warnings 2>&1 | tail -5 + ``` + + Expected: all pass, no warnings. + +- [ ] **Step 8: Commit** + + ```bash + git add crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-core/src/auction/types.rs + git commit -m "Emit hb_adid from PBS Cache UUID and add hb_cache_host/hb_cache_path to bid map" + ``` + +--- + +## Final verification + +- [ ] Run `cargo test --workspace` — all pass +- [ ] Run `cargo clippy --workspace --all-targets --all-features -- -D warnings` — clean +- [ ] Run `cargo fmt --all -- --check` — clean +- [ ] In browser devtools after deploy: `window._ts.bids` shows `hb_cache_host`, `hb_cache_path`, and `hb_adid` matching the UUID in `ext.prebid.cache.bids.cacheId` from the raw PBS response + +--- + +## Rollout reminder (from spec §8) + +1. TS: this branch deployed +2. GAM: ad ops updates Prebid line item creatives to server-side cache-fetch variant (see spec §4.6) +3. PBS: Prebid Cache already enabled (confirmed from real response) +4. Verify in devtools diff --git a/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md new file mode 100644 index 000000000..8617ef877 --- /dev/null +++ b/docs/superpowers/specs/2026-04-15-server-side-ad-templates-design.md @@ -0,0 +1,1198 @@ +# Server-Side Ad Templates Design + +_Author · 2026-04-15_ +_Revised · 2026-05-04 (body-injection rework)_ + +--- + +## 1. Problem Statement + +Today's display ad pipeline on most publisher sites is structurally sequential and +browser-bound: + +1. Page HTML arrives at browser +2. Prebid.js (~300KB) downloads and parses +3. Smart Slots SDK scans the DOM to discover ad placements +4. `addAdUnits()` registers slot definitions +5. Prebid auction fires from the browser (~80–150ms RTT to SSPs) +6. Bids return (~1,000–1,500ms window) +7. GPT `setTargeting()` + `refresh()` fires +8. GAM creative renders + +**Total time to ad visible: ~3,100ms.** + +The browser is the slowest possible place to run an auction. It must first download and +parse multiple SDKs, scan the DOM to discover what ad slots exist, and then fire SSP +requests over a consumer internet connection with high and variable latency. + +Trusted Server sits at the Fastly edge — milliseconds from the user, with +data-center-to-data-center RTT to Prebid Server (~20–30ms vs ~80–150ms from a browser). +The server knows, from the request URL alone, exactly which ad slots are available on any +given page. There is no reason to wait for the browser. + +--- + +## 2. Goal + +Enable Trusted Server to: + +1. Match an incoming page request URL against a set of pre-configured slot templates. +2. Immediately fire the full server-side auction (all providers: PBS, APS, future + wrappers) in parallel with the origin HTML fetch — before the browser receives a + single byte. +3. Inject GPT slot definitions into `` so the client can define slots without any + SDK. +4. Inject the auction result inline before ``, so bids and HTML travel together + in a single response. The auction runs in parallel with origin fetch and body + streaming; the close-of-body tag is held only until the auction completes or + `A_deadline` fires, whichever is first. **FCP is unaffected** — body content above + `` has already streamed and painted by the time the close tag is held. +5. Eliminate Prebid.js from the page-load critical path. A stripped-down Prebid bundle + (_slim-Prebid_) is lazy-loaded post-`window.load` to handle scroll/refresh auctions + and to run userID modules that enrich the EC identity graph for the next request. + +**Target time to ad visible: ~870ms (cache hit) / ~1,020ms (cache miss). FCP unchanged +from a no-TS baseline.** Net saving on ad-visible: ~2,200–2,400ms vs today, compounded +across every navigation in the user's clickstream rather than once per session. + +> **Note:** The latency numbers in this document are modeled estimates based on known +> edge→PBS RTT ranges and typical origin response times. They should be validated with +> production measurements after Phase 1 ships. Quantitative claims in §6 are tagged with +> the assumptions they depend on. + +--- + +## 3. Non-Goals + +- Eliminating client-side GPT / Google Ad Manager — GAM remains in the rendering + pipeline. The GAM call (`securepubads.g.doubleclick.net`) moving server-side is + aspirational, contingent on Google agreement, and is not committed for any phase + (see §9.6). +- Eliminating Prebid entirely — a stripped-down Prebid bundle (_slim-Prebid_) is + lazy-loaded post-`window.load` to handle scroll/refresh auctions and userID + enrichment. **TS owns the first impression; Prebid owns subsequent refresh + auctions.** +- Dynamic slot discovery (reading the DOM) — this design commits to pre-defined, + URL-matched slot templates. Smart Slots' dynamic injection behavior is replaced by + server knowledge. +- Changing the `AuctionOrchestrator` internally — the orchestrator already handles + parallel provider fan-out. This design adds a new trigger point, not new auction + logic. +- A separate `/ts-bids` endpoint — an earlier revision of this spec proposed a fetch + endpoint backed by an in-process `bid_cache`. That design was rejected because + in-process caches don't survive Fastly Compute's per-request Wasm isolate model: + isolates are not pinned across requests, and there is no documented contract for + cross-request shared state. Body-injection achieves the same FCP property with a + single response and no shared-state requirement. + +--- + +## 4. Architecture + +### 4.1 Slot configuration in `trusted-server.toml` + +Slot templates live in `trusted-server.toml` under `[[creative_opportunities.slot]]` +(consolidated from the original `creative-opportunities.toml`). Each entry holds page +pattern matching rules, ad formats, floor prices, GAM targeting key-values, and +per-provider bidder params. PBS bidder-level params (placement IDs, account IDs) live +in Prebid Server stored requests, keyed by slot ID. APS params are specified inline per +slot under `[slot.providers.aps]`. + +Loaded at build time via `include_str!()` and compiled into the WASM binary. Slot +changes require a redeploy; this is intentional (fast reads, no KV overhead, no +per-request cost). A migration path to KV-backed config is tracked in §9.5. + +`floor_price` is the publisher-owned hard floor per slot — the source of truth for the +minimum acceptable bid price, enforced at the edge before bids reach the ad server. Any +bid below the floor is discarded at the orchestrator level before it enters `tsjs.bids`. +SSPs may apply their own dynamic floors independently within their platforms; this floor +is the publisher's baseline that supersedes all other floor logic by virtue of being +enforced earliest in the pipeline. + +#### Top-level config (in `trusted-server.toml`) + +```toml +[creative_opportunities] +# GAM network ID used to construct default ad-unit paths. +gam_network_id = "21765378893" + +# Optional. Defaults to [auction].timeout_ms if not set. +# Recommended: 500ms (vs client-side 1000–1500ms) due to lower edge→PBS RTT. +# This value is also the upper bound on the -close hold; once A_deadline +# fires, TS injects an empty tsjs.bids and emits regardless. +auction_timeout_ms = 500 + +# Granularity table for hb_pb price bucket strings. +# Options: "low" | "medium" | "high" | "auto" | "dense" | "custom" +# Defaults to "dense" if not set. +price_granularity = "dense" +``` + +#### `[creative_opportunities]` schema + +```toml +[[slot]] +id = "atf_sidebar_ad" +# Optional. Defaults to "/{gam_network_id}/{id}". +# Override for non-standard GAM ad-unit paths. +gam_unit_path = "/21765378893/publisher/atf-sidebar" +# Optional. DOM container element ID. Defaults to slot id. +div_id = "div-atf-sidebar" +page_patterns = ["/20**"] +formats = [{ width = 300, height = 250 }] +floor_price = 0.50 + +[slot.targeting] +pos = "atf" +zone = "atfSidebar" + +[slot.providers.aps] +slot_id = "aps-slot-atf-sidebar" + +[[slot]] +id = "below-content-ad" +page_patterns = ["/20**"] +formats = [{ width = 300, height = 250 }, { width = 728, height = 90 }] +floor_price = 0.25 + +[slot.targeting] +pos = "btf" +zone = "belowContent" + +[slot.providers.aps] +slot_id = "aps-slot-below-content" + +[[slot]] +id = "ad-homepage-0" +page_patterns = ["/", "/index.html"] +formats = [{ width = 970, height = 250 }, { width = 728, height = 90 }] +floor_price = 1.00 + +[slot.targeting] +pos = "atf" +zone = "homepage" +slot_index = "0" + +[slot.providers.aps] +slot_id = "aps-slot-homepage-0" +``` + +#### Rust types + +```rust +#[derive(Debug, Clone, serde::Deserialize)] +pub struct CreativeOpportunitiesConfig { + pub gam_network_id: String, + #[serde(default = "default_auction_timeout_ms")] + pub auction_timeout_ms: Option, + #[serde(default = "PriceGranularity::dense")] + pub price_granularity: PriceGranularity, +} + +#[derive(Debug, Clone, serde::Deserialize)] +pub struct CreativeOpportunitySlot { + pub id: String, + pub gam_unit_path: Option, // defaults to /{gam_network_id}/{id} + pub div_id: Option, // defaults to id + pub page_patterns: Vec, + pub formats: Vec, + pub floor_price: Option, + #[serde(default)] + pub targeting: HashMap, // strings only — validated at startup + #[serde(default)] + pub providers: SlotProviders, +} + +/// Separate from auction::AdFormat so media_type can default to Banner +/// without requiring it in the TOML. Converted to AdFormat at auction time. +#[derive(Debug, Clone, serde::Deserialize)] +pub struct CreativeOpportunityFormat { + pub width: u32, + pub height: u32, + #[serde(default = "MediaType::banner")] + pub media_type: MediaType, +} + +#[derive(Debug, Clone, Default, serde::Deserialize)] +pub struct SlotProviders { + pub aps: Option, +} + +#[derive(Debug, Clone, serde::Deserialize)] +pub struct ApsSlotParams { + pub slot_id: String, +} +``` + +> **Targeting value types:** `targeting` values are `String`-only (not +> `serde_json::Value`). GPT's `setTargeting()` only accepts `string | string[]`; +> non-string values are silently dropped by the browser. Validated at startup — a +> non-string targeting value is a startup error. + +> **Slot ID validation:** Slot IDs are validated at startup against a strict allowlist +> (`[A-Za-z0-9_-]+`). IDs outside this set fail startup. This prevents XSS via crafted +> IDs appearing in the injected `, ContentType::Html)`. + +> **Security:** All string values are JSON-serialized via `serde_json` and HTML-escaped +> before insertion into the ` +``` + +Alternatively, publishers using the Prebid Universal Creative package can use: + +```html + + +``` + +> **This creative configuration is a publisher/ad ops action, not a TS code change.** +> Document it in the integration guide and verify during onboarding. + +> **Cache TTL:** PBS Cache entries expire per the `bid.exp` field (default 300–3600s; +> the real response has `"exp": 3600`). Creative fetch must complete within this window. +> BFCache page restores after long idle sessions may hit expired cache entries — the +> creative will silently fail to render in that case. This is acceptable for Phase 1; +> the probability is low for typical session lengths. + +--- + +## 5. APS — Out of Scope + +APS does not use PBS Cache. APS bids will have `cache_id = None`, `cache_host = None`, +`cache_path = None`. The existing `ad_id` fallback path remains for APS. APS creative +rendering depends on Amazon's own GAM creative tag — separate from the Prebid path. + +APS win detection over-fires on the `!!bid.hb_bidder` fallback remain a known +limitation tracked separately. + +--- + +## 6. Files Changed + +| File | Change | +| ------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/auction/types.rs` | Add `cache_id`, `cache_host`, `cache_path` to `Bid` struct | +| `crates/trusted-server-core/src/integrations/prebid.rs` | Extract `ext.prebid.cache.bids.{cacheId,url}` in `parse_bid_object`; update `AuctionBid` → `Bid` conversion to carry the three new fields | +| `crates/trusted-server-core/src/publisher.rs` | `build_bid_map`: use `cache_id` for `hb_adid`, emit `hb_cache_host`/`hb_cache_path` | + +> **Implementer note — `AuctionBid` → `Bid` conversion:** `prebid.rs` constructs an +> intermediate `AuctionBid` type that is later converted to the shared `Bid` type from +> `types.rs`. The new `cache_id`, `cache_host`, `cache_path` fields must be added to +> **both** types and the conversion must map them explicitly. Verify by grepping for +> where `AuctionBid` is constructed and where it is converted to `Bid`; if they are the +> same type (a type alias), only one struct needs the new fields. If they differ, both +> need updating or the fields will silently be `None` in `build_bid_map`. + +Test files: +| File | Change | +|---|---| +| `crates/trusted-server-core/src/integrations/prebid.rs` tests | Add test: PBS response with cache entry → correct `hb_adid`, `hb_cache_host`, `hb_cache_path` injected | +| `crates/trusted-server-core/src/publisher.rs` tests | Add test: `build_bid_map` emits cache fields when present; falls back to `ad_id` when absent | + +--- + +## 7. Testing + +**Unit tests:** + +1. `prebid.rs`: bid with `ext.prebid.cache.bids.cacheId` → `bid.cache_id = Some(uuid)`, `bid.cache_host = Some("openads.adsrvr.org")`, `bid.cache_path = Some("/cache")` +2. `prebid.rs`: bid without `ext.prebid.cache` → `bid.cache_id = None`, `bid.cache_host = None`, `bid.cache_path = None` +3. `prebid.rs`: bid with only `adid` (no cache) → `bid.ad_id = Some(...)`, `bid.cache_id = None` +4. `prebid.rs`: bid with malformed cache URL → `cache_host = None`, `cache_path = None`, no panic +5. `publisher.rs` `build_bid_map`: bid with `cache_id` → `hb_adid` uses `cache_id`, `hb_cache_host`/`hb_cache_path` emitted +6. `publisher.rs` `build_bid_map`: bid with no `cache_id` but has `ad_id` → `hb_adid` falls back to `ad_id`, no cache keys emitted +7. `publisher.rs` `build_bid_map`: APS bid (no `cache_id`, no `ad_id`) → no `hb_adid` emitted +8. `types.rs`: `Bid` with all three cache fields round-trips through `serde_json::to_string` / `from_str` + +> **Note for implementer:** `make_bid()` or equivalent `Bid` construction helpers in test modules +> must be updated to initialise `cache_id`, `cache_host`, `cache_path` to `None` +> (they will fail to compile otherwise once the fields are added to the struct). + +**Integration verification (manual):** + +After deploying, verify `window.tsjs.bids` in browser devtools shows `hb_cache_host` +and `hb_cache_path` present. Verify `hb_adid` matches the UUID in +`ext.prebid.cache.bids.cacheId` from the raw PBS response. + +--- + +## 8. Rollout Dependency Checklist + +Before this fix has end-to-end effect: + +- [ ] TS: this PR merged and deployed +- [ ] GAM: publisher ad ops updates all Prebid line item creatives to the server-side + cache-fetch variant (see §4.6) +- [ ] PBS: Prebid Cache enabled and populated (confirmed from real response — already + working) +- [ ] Verify: `window.tsjs.bids` shows correct cache UUID in `hb_adid` after deploy + +--- + +## 9. Known Remaining Gaps (not in scope) + +| Gap | Severity | Tracking | +| ----------------------------------------------------------------- | -------- | ------------------ | +| APS win detection over-fires nurl/burl | P1 | Separate issue | +| Dual bootstrap (`gpt_bootstrap.js` + `installTsAdInit`) sync risk | P2 | Separate issue | +| Slim-Prebid bundle not yet built | Phase 2 | §9.8 of design doc | diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 0e8226efb..1c3f3cf55 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -127,3 +127,19 @@ example_segments = "segments" [debug] ja4_endpoint_enabled = false + +[creative_opportunities] +gam_network_id = "123456789" +# FCP is not affected by this value — body content above has already +# streamed and painted before the hold begins. What this caps is the slip on +# DOMContentLoaded and window.load. Worst case: a cache-hit page where origin +# drains in <50 ms but the auction runs to the limit. 500 ms is the recommended +# default; raise only if your SSPs need more headroom and your analytics confirm +# the DCL slip is acceptable. +auction_timeout_ms = 500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__AUCTION_TIMEOUT_MS +price_granularity = "dense" + +# No slot templates are enabled in the checked-in default config. Add +# `[[creative_opportunities.slot]]` entries via private config or override the +# entire array via: +# TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT='[{"id":"...","gam_unit_path":"...",...}]'