diff --git a/prometheus/README.md b/prometheus/README.md index 46bb1f6..eae825d 100644 --- a/prometheus/README.md +++ b/prometheus/README.md @@ -30,6 +30,12 @@ directory, and its shared `prometheus_client` source parser, runs conventionally and writes disabled, and compares generated stdout byte-for-byte with the committed registry. Generators use only the Python standard library. +FastAPI, LiteLLM, and vLLM extraction fingerprints the normalized AST of each complete Python module whose executable shape +is an input, including imports and other global bindings. Formatting and comments do not affect the fingerprint; any +executable source-shape change fails closed until the pinned upgrade is reviewed and the expected fingerprint is replaced. +The shared `prometheus_client` fingerprint also closes every consumer over the `_created` emitter implementation and its +positive feature gate. Other generators use source-language-specific closed grammars and negative tests instead. + ## Consumer contract Prometheus profile validation uses the latest `netdata/testdata` `master`. Each profile has one stable directory; proof diff --git a/prometheus/profiles/fastapi/SOURCE-REGISTRY.generator.yaml b/prometheus/profiles/fastapi/SOURCE-REGISTRY.generator.yaml index 3068865..fc5a957 100644 --- a/prometheus/profiles/fastapi/SOURCE-REGISTRY.generator.yaml +++ b/prometheus/profiles/fastapi/SOURCE-REGISTRY.generator.yaml @@ -9,6 +9,7 @@ upstreams: commit: 2f841527277a21ac9ea622a9f923a5f9078234c4 paths: - src/prometheus_fastapi_instrumentator/metrics.py + - src/prometheus_fastapi_instrumentator/middleware.py prometheus_client_python: repository: prometheus/client_python commit: f417f6ea8f058165a1934e368fed245e91aafc14 diff --git a/prometheus/profiles/fastapi/SOURCE-REGISTRY.yaml b/prometheus/profiles/fastapi/SOURCE-REGISTRY.yaml index 823c0ca..38e4ef1 100644 --- a/prometheus/profiles/fastapi/SOURCE-REGISTRY.yaml +++ b/prometheus/profiles/fastapi/SOURCE-REGISTRY.yaml @@ -98,3 +98,18 @@ groups: - upstream: prometheus_client_python path: prometheus_client/metrics.py range: {start: 692, end: 703} + middleware_http: + registrations: + http_requests_inprogress: + family: {exact: http_requests_inprogress} + when: + any: + - all: + - {axis: inprogress, op: in, values: [unlabeled, labeled]} + prometheus: {type: gauge, shape: scalar} + components: + value: {wire_role: scalar} + source_locations: + - upstream: fastapi_instrumentator + path: src/prometheus_fastapi_instrumentator/middleware.py + range: {start: 117, end: 122} diff --git a/prometheus/profiles/fastapi/SOURCE-SEMANTICS.yaml b/prometheus/profiles/fastapi/SOURCE-SEMANTICS.yaml index 3fa375d..5d9ef39 100644 --- a/prometheus/profiles/fastapi/SOURCE-SEMANTICS.yaml +++ b/prometheus/profiles/fastapi/SOURCE-SEMANTICS.yaml @@ -13,8 +13,10 @@ evidence: request_population: kind: population upstream: fastapi_instrumentator - locations: [src/prometheus_fastapi_instrumentator/metrics.py:779-820] - claim: One instrumentation call records one request outcome, both size observations, and both enabled duration observations. + locations: + - src/prometheus_fastapi_instrumentator/middleware.py:173-220 + - src/prometheus_fastapi_instrumentator/metrics.py:779-820 + claim: After one request completes, one instrumentation call records its outcome, both size observations, the endpoint duration, and the configured service-wide duration population. component_lifecycle: kind: lifecycle upstream: prometheus_client_python @@ -49,25 +51,153 @@ evidence: kind: label upstream: fastapi_instrumentator locations: [src/prometheus_fastapi_instrumentator/metrics.py:706-777, src/prometheus_fastapi_instrumentator/metrics.py:786-820] - claim: Default request outcomes use method, status, and handler; size summaries use handler; endpoint duration uses method and handler; service duration is global. + claim: Request outcomes use method, configured status representation, and handler; size summaries use handler; endpoint duration uses method and handler; service duration is global. status_domain: kind: label upstream: fastapi_instrumentator locations: [src/prometheus_fastapi_instrumentator/middleware.py:23-24, src/prometheus_fastapi_instrumentator/middleware.py:178-214] - claim: Default middleware replaces the exact HTTP response code with its one-digit status class. + claim: Middleware emits one-digit response classes by default and exact HTTP response codes when status grouping is disabled. + status_grouping_mode: + kind: availability + upstream: fastapi_instrumentator + locations: [src/prometheus_fastapi_instrumentator/middleware.py:23-24, src/prometheus_fastapi_instrumentator/middleware.py:202-204] + claim: should_group_status_codes selects grouped one-digit classes or exact response codes without changing the metric family. + highr_population_mode: + kind: availability + upstream: fastapi_instrumentator + locations: [src/prometheus_fastapi_instrumentator/metrics.py:616-620, src/prometheus_fastapi_instrumentator/metrics.py:811-815] + claim: should_only_respect_2xx_for_highr selects all handled requests or only successful responses for the same service-wide histogram family. + streaming_duration_mode: + kind: availability + upstream: fastapi_instrumentator + locations: [src/prometheus_fastapi_instrumentator/metrics.py:616-620, src/prometheus_fastapi_instrumentator/metrics.py:779-784] + claim: should_exclude_streaming_duration selects full request duration or duration with response-body streaming time removed for both duration histograms. + custom_labels_mode: + kind: availability + upstream: fastapi_instrumentator + locations: [src/prometheus_fastapi_instrumentator/metrics.py:616-646, src/prometheus_fastapi_instrumentator/metrics.py:706-820] + claim: The default metric factory accepts arbitrary additional constant label keys; this finite proof contract is limited to its default empty custom_labels mapping. + metric_naming_mode: + kind: availability + upstream: fastapi_instrumentator + locations: [src/prometheus_fastapi_instrumentator/metrics.py:613-614, src/prometheus_fastapi_instrumentator/metrics.py:706-777] + claim: The default metric factory accepts namespace and subsystem prefixes; this stock contract covers the default empty values that preserve the exact http_* families. + inprogress_naming_mode: + kind: availability + upstream: fastapi_instrumentator + locations: [src/prometheus_fastapi_instrumentator/middleware.py:28-35, src/prometheus_fastapi_instrumentator/middleware.py:79-80, src/prometheus_fastapi_instrumentator/middleware.py:107-122] + claim: The middleware accepts an arbitrary in-progress Gauge name; this stock contract covers its default http_requests_inprogress family. count_projection: kind: relationship upstream: fastapi_instrumentator locations: [src/prometheus_fastapi_instrumentator/metrics.py:786-809] claim: Each handled request increments the outcome counter once and observes each size summary once, so each handler-level summary count equals outcomes summed over method and status. - request_concurrency_display: - kind: display_convention + inprogress_availability: + kind: availability + upstream: fastapi_instrumentator + locations: [src/prometheus_fastapi_instrumentator/middleware.py:28-35, src/prometheus_fastapi_instrumentator/middleware.py:107-122] + claim: In-progress instrumentation defaults to disabled; when enabled it registers the default http_requests_inprogress Gauge with either no labels or method and handler labels. + inprogress_population: + kind: population + upstream: fastapi_instrumentator + locations: [src/prometheus_fastapi_instrumentator/middleware.py:137-142, src/prometheus_fastapi_instrumentator/middleware.py:173-194] + claim: The Gauge counts non-excluded HTTP requests after entry and removes each request in the completion path. + inprogress_lifecycle: + kind: lifecycle upstream: fastapi_instrumentator - locations: [src/prometheus_fastapi_instrumentator/metrics.py:779-820] - claim: Summing request duration and taking its per-second increase yields the average number of requests in flight during the interval. + locations: [src/prometheus_fastapi_instrumentator/middleware.py:137-142, src/prometheus_fastapi_instrumentator/middleware.py:173-194] + claim: The Gauge is a current value incremented before awaiting the application and decremented in the request completion path. + inprogress_unit: + kind: unit + upstream: fastapi_instrumentator + locations: [src/prometheus_fastapi_instrumentator/middleware.py:117-122] + claim: The Gauge documentation defines its value as the number of HTTP requests in progress. + inprogress_labels: + kind: label + upstream: fastapi_instrumentator + locations: [src/prometheus_fastapi_instrumentator/middleware.py:107-122, src/prometheus_fastapi_instrumentator/middleware.py:137-142] + claim: The optional labeled mode identifies each current population by request method and normalized handler; unlabeled mode exposes a service-wide value. environment: - axes: {} - policies: {} + axes: + inprogress: + kind: enum + values: [disabled, unlabeled, labeled] + meaning: Availability and label shape of the optional in-progress request Gauge using its default metric name. + evidence: [inprogress_availability] + status_grouping: + kind: enum + values: [grouped, exact] + meaning: Response status label representation for the default request counter. + evidence: [status_grouping_mode] + highr_population: + kind: enum + values: [all, 2xx_only] + meaning: Request population admitted to the service-wide high-resolution duration histogram. + evidence: [highr_population_mode] + streaming_duration: + kind: enum + values: [included, excluded] + meaning: Whether response-body streaming time contributes to recorded request durations. + evidence: [streaming_duration_mode] + custom_labels: + kind: enum + values: [none] + meaning: Finite proof scope for the default empty custom_labels mapping; arbitrary configured label keys are not claimed by this contract. + evidence: [custom_labels_mode] + metric_naming: + kind: enum + values: [default] + meaning: Finite stock-profile scope for empty metric_namespace and metric_subsystem values that preserve exact http_* families. + evidence: [metric_naming_mode] + inprogress_naming: + kind: enum + values: [default] + meaning: Finite stock-profile scope for the default http_requests_inprogress Gauge name. + evidence: [inprogress_naming_mode] + policies: + inprogress_enabled: + when: + any: + - all: + - {axis: inprogress, op: in, values: [unlabeled, labeled]} + - {axis: inprogress_naming, op: eq, value: default} + evidence: [inprogress_availability, inprogress_naming_mode] + inprogress_labeled: + when: + any: + - all: [{axis: inprogress, op: eq, value: labeled}] + evidence: [inprogress_availability] + request_outcomes_supported: + when: + any: + - all: + - {axis: status_grouping, op: in, values: [grouped, exact]} + - {axis: custom_labels, op: eq, value: none} + - {axis: metric_naming, op: eq, value: default} + evidence: [status_grouping_mode, custom_labels_mode, metric_naming_mode] + default_labeled_metrics_supported: + when: + any: + - all: + - {axis: custom_labels, op: eq, value: none} + - {axis: metric_naming, op: eq, value: default} + evidence: [custom_labels_mode, metric_naming_mode] + endpoint_duration_supported: + when: + any: + - all: + - {axis: streaming_duration, op: in, values: [included, excluded]} + - {axis: custom_labels, op: eq, value: none} + - {axis: metric_naming, op: eq, value: default} + evidence: [streaming_duration_mode, custom_labels_mode, metric_naming_mode] + service_duration_supported: + when: + any: + - all: + - {axis: highr_population, op: in, values: [all, 2xx_only]} + - {axis: streaming_duration, op: in, values: [included, excluded]} + - {axis: metric_naming, op: eq, value: default} + evidence: [highr_population_mode, streaming_duration_mode, metric_naming_mode] component_policies: duration_histogram: bucket: @@ -103,6 +233,7 @@ component_policies: label_policies: {} signals: request_outcomes: + availability: request_outcomes_supported source: generated: registry_groups: [default_http] @@ -138,14 +269,15 @@ signals: stability: stable evidence: [request_labels] status: - meaning: One-digit HTTP response status class. + meaning: Producer-configured HTTP response status class or exact status code. presence: required - domain: {kind: closed, values: [1xx, 2xx, 3xx, 4xx, 5xx]} - endpoint_cardinality: {kind: closed_domain} + domain: {kind: open} + endpoint_cardinality: {kind: bounded_configuration} stability: stable evidence: [request_labels, status_domain] functional_dependencies: {} request_size: + availability: default_labeled_metrics_supported source: generated: registry_groups: [default_http] @@ -185,6 +317,7 @@ signals: evidence: [request_labels] functional_dependencies: {} response_size: + availability: default_labeled_metrics_supported source: generated: registry_groups: [default_http] @@ -224,13 +357,14 @@ signals: evidence: [request_labels] functional_dependencies: {} endpoint_duration: + availability: endpoint_duration_supported source: generated: registry_groups: [default_http] scope: {registrations: [http_request_duration_seconds]} population: id: handled_requests - meaning: Requests handled by the instrumented FastAPI application. + meaning: Requests handled by the instrumented FastAPI application using the producer-configured streaming-duration basis. evidence: [request_population] component_policy: duration_histogram labels: @@ -250,18 +384,57 @@ signals: evidence: [request_labels] functional_dependencies: {} service_duration: + availability: service_duration_supported source: generated: registry_groups: [default_http] scope: {registrations: [http_request_duration_highr_seconds]} population: - id: handled_requests - meaning: Requests handled by the instrumented FastAPI application. + id: configured_high_resolution_requests + meaning: Producer-configured service-wide population and streaming-duration basis. evidence: [request_population] component_policy: duration_histogram labels: {} functional_dependencies: {} + requests_in_progress: + availability: inprogress_enabled + source: + generated: + registry_groups: [middleware_http] + scope: {registrations: [http_requests_inprogress]} + population: + id: active_http_requests + meaning: Non-excluded HTTP requests currently executing in the instrumented FastAPI application. + evidence: [inprogress_population] + components: + value: + wire_role: scalar + lifecycle: {kind: current, evidence: [inprogress_lifecycle]} + unit: + quantity: count + base: one + rate: none + object: requests + aspect: in_progress + evidence: [inprogress_unit] + labels: + handler: + meaning: Normalized FastAPI route handler when in-progress labels are enabled. + presence: {when: inprogress_labeled} + domain: {kind: open} + endpoint_cardinality: {kind: operational_population} + stability: stable + evidence: [inprogress_labels] + method: + meaning: HTTP request method when in-progress labels are enabled. + presence: {when: inprogress_labeled} + domain: {kind: open} + endpoint_cardinality: {kind: operational_population} + stability: stable + evidence: [inprogress_labels] + functional_dependencies: {} registration_timestamps: + availability: default_labeled_metrics_supported source: generated: registry_groups: [default_http] @@ -297,10 +470,10 @@ signals: stability: stable evidence: [request_labels] status: - meaning: HTTP response status class when the registered instrument has outcome identity. + meaning: Producer-configured HTTP response status class or exact status code when the registered instrument has outcome identity. presence: optional - domain: {kind: closed, values: [1xx, 2xx, 3xx, 4xx, 5xx]} - endpoint_cardinality: {kind: closed_domain} + domain: {kind: open} + endpoint_cardinality: {kind: bounded_configuration} stability: stable evidence: [request_labels, status_domain] functional_dependencies: {} diff --git a/prometheus/profiles/fastapi/fixtures/fastapi_configured_modes.prom b/prometheus/profiles/fastapi/fixtures/fastapi_configured_modes.prom new file mode 100644 index 0000000..417b618 --- /dev/null +++ b/prometheus/profiles/fastapi/fixtures/fastapi_configured_modes.prom @@ -0,0 +1,85 @@ +# Synthetic FastAPI fixture for exact status codes, 2xx-only high-resolution observations, and excluded streaming time. +# Labels and values are non-production placeholders derived from public exporter source. +# HELP http_requests_total Total number of requests by method, status and handler. +# TYPE http_requests_total counter +http_requests_total{handler="/v1/example",method="POST",status="200"} 3 +http_requests_total{handler="/v1/example",method="POST",status="101"} 1 +http_requests_total{handler="/v1/example",method="POST",status="302"} 1 +http_requests_total{handler="/v1/example",method="POST",status="404"} 1 +http_requests_total{handler="/v1/example",method="POST",status="500"} 1 +http_requests_total{handler="/health",method="GET",status="200"} 3 +# HELP http_requests_created Total number of requests by method, status and handler. +# TYPE http_requests_created gauge +http_requests_created{handler="/v1/example",method="POST",status="200"} 1 +http_requests_created{handler="/v1/example",method="POST",status="101"} 1 +http_requests_created{handler="/v1/example",method="POST",status="302"} 1 +http_requests_created{handler="/v1/example",method="POST",status="404"} 1 +http_requests_created{handler="/v1/example",method="POST",status="500"} 1 +http_requests_created{handler="/health",method="GET",status="200"} 1 +# HELP http_request_size_bytes Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_request_size_bytes summary +http_request_size_bytes_count{handler="/v1/example"} 7 +http_request_size_bytes_sum{handler="/v1/example"} 6 +http_request_size_bytes_count{handler="/health"} 3 +http_request_size_bytes_sum{handler="/health"} 6 +# HELP http_request_size_bytes_created Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_request_size_bytes_created gauge +http_request_size_bytes_created{handler="/v1/example"} 1 +http_request_size_bytes_created{handler="/health"} 1 +# HELP http_response_size_bytes Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_response_size_bytes summary +http_response_size_bytes_count{handler="/v1/example"} 7 +http_response_size_bytes_sum{handler="/v1/example"} 6 +http_response_size_bytes_count{handler="/health"} 3 +http_response_size_bytes_sum{handler="/health"} 6 +# HELP http_response_size_bytes_created Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_response_size_bytes_created gauge +http_response_size_bytes_created{handler="/v1/example"} 1 +http_response_size_bytes_created{handler="/health"} 1 +# HELP http_request_duration_highr_seconds Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. +# TYPE http_request_duration_highr_seconds histogram +http_request_duration_highr_seconds_bucket{le="0.01"} 0 +http_request_duration_highr_seconds_bucket{le="0.025"} 0 +http_request_duration_highr_seconds_bucket{le="0.05"} 1 +http_request_duration_highr_seconds_bucket{le="0.075"} 1 +http_request_duration_highr_seconds_bucket{le="0.1"} 2 +http_request_duration_highr_seconds_bucket{le="0.25"} 3 +http_request_duration_highr_seconds_bucket{le="0.5"} 4 +http_request_duration_highr_seconds_bucket{le="0.75"} 4 +http_request_duration_highr_seconds_bucket{le="1.0"} 5 +http_request_duration_highr_seconds_bucket{le="1.5"} 5 +http_request_duration_highr_seconds_bucket{le="2.0"} 6 +http_request_duration_highr_seconds_bucket{le="2.5"} 6 +http_request_duration_highr_seconds_bucket{le="3.0"} 6 +http_request_duration_highr_seconds_bucket{le="3.5"} 6 +http_request_duration_highr_seconds_bucket{le="4.0"} 6 +http_request_duration_highr_seconds_bucket{le="4.5"} 6 +http_request_duration_highr_seconds_bucket{le="5.0"} 6 +http_request_duration_highr_seconds_bucket{le="7.5"} 6 +http_request_duration_highr_seconds_bucket{le="10.0"} 6 +http_request_duration_highr_seconds_bucket{le="30.0"} 6 +http_request_duration_highr_seconds_bucket{le="60.0"} 6 +http_request_duration_highr_seconds_bucket{le="+Inf"} 6 +http_request_duration_highr_seconds_count 6 +http_request_duration_highr_seconds_sum 4 +# HELP http_request_duration_highr_seconds_created Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. +# TYPE http_request_duration_highr_seconds_created gauge +http_request_duration_highr_seconds_created 1 +# HELP http_request_duration_seconds Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{handler="/v1/example",le="0.1",method="POST"} 1 +http_request_duration_seconds_bucket{handler="/v1/example",le="0.5",method="POST"} 3 +http_request_duration_seconds_bucket{handler="/v1/example",le="1.0",method="POST"} 7 +http_request_duration_seconds_bucket{handler="/v1/example",le="+Inf",method="POST"} 7 +http_request_duration_seconds_count{handler="/v1/example",method="POST"} 7 +http_request_duration_seconds_sum{handler="/v1/example",method="POST"} 4 +http_request_duration_seconds_bucket{handler="/health",le="0.1",method="GET"} 1 +http_request_duration_seconds_bucket{handler="/health",le="0.5",method="GET"} 2 +http_request_duration_seconds_bucket{handler="/health",le="1.0",method="GET"} 3 +http_request_duration_seconds_bucket{handler="/health",le="+Inf",method="GET"} 3 +http_request_duration_seconds_count{handler="/health",method="GET"} 3 +http_request_duration_seconds_sum{handler="/health",method="GET"} 3 +# HELP http_request_duration_seconds_created Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. +# TYPE http_request_duration_seconds_created gauge +http_request_duration_seconds_created{handler="/v1/example",method="POST"} 1 +http_request_duration_seconds_created{handler="/health",method="GET"} 1 diff --git a/prometheus/profiles/fastapi/fixtures/fastapi_inprogress_labeled.prom b/prometheus/profiles/fastapi/fixtures/fastapi_inprogress_labeled.prom new file mode 100644 index 0000000..04e02e8 --- /dev/null +++ b/prometheus/profiles/fastapi/fixtures/fastapi_inprogress_labeled.prom @@ -0,0 +1,89 @@ +# Synthetic FastAPI HTTP instrumentation Prometheus structural fixture; this is not a captured scrape. +# Labels and values are non-production placeholders derived from public exporter source. +# HELP http_requests_total Total number of requests by method, status and handler. +# TYPE http_requests_total counter +http_requests_total{handler="/v1/example",method="POST",status="2xx"} 3 +http_requests_total{handler="/v1/example",method="POST",status="1xx"} 1 +http_requests_total{handler="/v1/example",method="POST",status="3xx"} 1 +http_requests_total{handler="/v1/example",method="POST",status="4xx"} 1 +http_requests_total{handler="/v1/example",method="POST",status="5xx"} 1 +http_requests_total{handler="/health",method="GET",status="2xx"} 3 +# HELP http_requests_created Total number of requests by method, status and handler. +# TYPE http_requests_created gauge +http_requests_created{handler="/v1/example",method="POST",status="2xx"} 1 +http_requests_created{handler="/v1/example",method="POST",status="1xx"} 1 +http_requests_created{handler="/v1/example",method="POST",status="3xx"} 1 +http_requests_created{handler="/v1/example",method="POST",status="4xx"} 1 +http_requests_created{handler="/v1/example",method="POST",status="5xx"} 1 +http_requests_created{handler="/health",method="GET",status="2xx"} 1 +# HELP http_request_size_bytes Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_request_size_bytes summary +http_request_size_bytes_count{handler="/v1/example"} 7 +http_request_size_bytes_sum{handler="/v1/example"} 6 +http_request_size_bytes_count{handler="/health"} 3 +http_request_size_bytes_sum{handler="/health"} 6 +# HELP http_request_size_bytes_created Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_request_size_bytes_created gauge +http_request_size_bytes_created{handler="/v1/example"} 1 +http_request_size_bytes_created{handler="/health"} 1 +# HELP http_response_size_bytes Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_response_size_bytes summary +http_response_size_bytes_count{handler="/v1/example"} 7 +http_response_size_bytes_sum{handler="/v1/example"} 6 +http_response_size_bytes_count{handler="/health"} 3 +http_response_size_bytes_sum{handler="/health"} 6 +# HELP http_response_size_bytes_created Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_response_size_bytes_created gauge +http_response_size_bytes_created{handler="/v1/example"} 1 +http_response_size_bytes_created{handler="/health"} 1 +# HELP http_request_duration_highr_seconds Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. +# TYPE http_request_duration_highr_seconds histogram +http_request_duration_highr_seconds_bucket{le="0.01"} 0 +http_request_duration_highr_seconds_bucket{le="0.025"} 0 +http_request_duration_highr_seconds_bucket{le="0.05"} 1 +http_request_duration_highr_seconds_bucket{le="0.075"} 1 +http_request_duration_highr_seconds_bucket{le="0.1"} 2 +http_request_duration_highr_seconds_bucket{le="0.25"} 3 +http_request_duration_highr_seconds_bucket{le="0.5"} 4 +http_request_duration_highr_seconds_bucket{le="0.75"} 5 +http_request_duration_highr_seconds_bucket{le="1.0"} 6 +http_request_duration_highr_seconds_bucket{le="1.5"} 7 +http_request_duration_highr_seconds_bucket{le="2.0"} 8 +http_request_duration_highr_seconds_bucket{le="2.5"} 8 +http_request_duration_highr_seconds_bucket{le="3.0"} 9 +http_request_duration_highr_seconds_bucket{le="3.5"} 9 +http_request_duration_highr_seconds_bucket{le="4.0"} 9 +http_request_duration_highr_seconds_bucket{le="4.5"} 9 +http_request_duration_highr_seconds_bucket{le="5.0"} 9 +http_request_duration_highr_seconds_bucket{le="7.5"} 10 +http_request_duration_highr_seconds_bucket{le="10.0"} 10 +http_request_duration_highr_seconds_bucket{le="30.0"} 10 +http_request_duration_highr_seconds_bucket{le="60.0"} 10 +http_request_duration_highr_seconds_bucket{le="+Inf"} 10 +http_request_duration_highr_seconds_count 10 +http_request_duration_highr_seconds_sum 9 +# HELP http_request_duration_highr_seconds_created Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. +# TYPE http_request_duration_highr_seconds_created gauge +http_request_duration_highr_seconds_created 1 +# HELP http_request_duration_seconds Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{handler="/v1/example",le="0.1",method="POST"} 1 +http_request_duration_seconds_bucket{handler="/v1/example",le="0.5",method="POST"} 3 +http_request_duration_seconds_bucket{handler="/v1/example",le="1.0",method="POST"} 7 +http_request_duration_seconds_bucket{handler="/v1/example",le="+Inf",method="POST"} 7 +http_request_duration_seconds_count{handler="/v1/example",method="POST"} 7 +http_request_duration_seconds_sum{handler="/v1/example",method="POST"} 5 +http_request_duration_seconds_bucket{handler="/health",le="0.1",method="GET"} 1 +http_request_duration_seconds_bucket{handler="/health",le="0.5",method="GET"} 2 +http_request_duration_seconds_bucket{handler="/health",le="1.0",method="GET"} 3 +http_request_duration_seconds_bucket{handler="/health",le="+Inf",method="GET"} 3 +http_request_duration_seconds_count{handler="/health",method="GET"} 3 +http_request_duration_seconds_sum{handler="/health",method="GET"} 4 +# HELP http_request_duration_seconds_created Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. +# TYPE http_request_duration_seconds_created gauge +http_request_duration_seconds_created{handler="/v1/example",method="POST"} 1 +http_request_duration_seconds_created{handler="/health",method="GET"} 1 +# HELP http_requests_inprogress Number of HTTP requests in progress. +# TYPE http_requests_inprogress gauge +http_requests_inprogress{handler="/v1/example",method="POST"} 2 +http_requests_inprogress{handler="/health",method="GET"} 1 diff --git a/prometheus/profiles/fastapi/fixtures/fastapi_inprogress_unlabeled.prom b/prometheus/profiles/fastapi/fixtures/fastapi_inprogress_unlabeled.prom new file mode 100644 index 0000000..20255fd --- /dev/null +++ b/prometheus/profiles/fastapi/fixtures/fastapi_inprogress_unlabeled.prom @@ -0,0 +1,88 @@ +# Synthetic FastAPI HTTP instrumentation Prometheus structural fixture; this is not a captured scrape. +# Labels and values are non-production placeholders derived from public exporter source. +# HELP http_requests_total Total number of requests by method, status and handler. +# TYPE http_requests_total counter +http_requests_total{handler="/v1/example",method="POST",status="2xx"} 3 +http_requests_total{handler="/v1/example",method="POST",status="1xx"} 1 +http_requests_total{handler="/v1/example",method="POST",status="3xx"} 1 +http_requests_total{handler="/v1/example",method="POST",status="4xx"} 1 +http_requests_total{handler="/v1/example",method="POST",status="5xx"} 1 +http_requests_total{handler="/health",method="GET",status="2xx"} 3 +# HELP http_requests_created Total number of requests by method, status and handler. +# TYPE http_requests_created gauge +http_requests_created{handler="/v1/example",method="POST",status="2xx"} 1 +http_requests_created{handler="/v1/example",method="POST",status="1xx"} 1 +http_requests_created{handler="/v1/example",method="POST",status="3xx"} 1 +http_requests_created{handler="/v1/example",method="POST",status="4xx"} 1 +http_requests_created{handler="/v1/example",method="POST",status="5xx"} 1 +http_requests_created{handler="/health",method="GET",status="2xx"} 1 +# HELP http_request_size_bytes Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_request_size_bytes summary +http_request_size_bytes_count{handler="/v1/example"} 7 +http_request_size_bytes_sum{handler="/v1/example"} 6 +http_request_size_bytes_count{handler="/health"} 3 +http_request_size_bytes_sum{handler="/health"} 6 +# HELP http_request_size_bytes_created Content length of incoming requests by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_request_size_bytes_created gauge +http_request_size_bytes_created{handler="/v1/example"} 1 +http_request_size_bytes_created{handler="/health"} 1 +# HELP http_response_size_bytes Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_response_size_bytes summary +http_response_size_bytes_count{handler="/v1/example"} 7 +http_response_size_bytes_sum{handler="/v1/example"} 6 +http_response_size_bytes_count{handler="/health"} 3 +http_response_size_bytes_sum{handler="/health"} 6 +# HELP http_response_size_bytes_created Content length of outgoing responses by handler. Only value of header is respected. Otherwise ignored. No percentile calculated. +# TYPE http_response_size_bytes_created gauge +http_response_size_bytes_created{handler="/v1/example"} 1 +http_response_size_bytes_created{handler="/health"} 1 +# HELP http_request_duration_highr_seconds Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. +# TYPE http_request_duration_highr_seconds histogram +http_request_duration_highr_seconds_bucket{le="0.01"} 0 +http_request_duration_highr_seconds_bucket{le="0.025"} 0 +http_request_duration_highr_seconds_bucket{le="0.05"} 1 +http_request_duration_highr_seconds_bucket{le="0.075"} 1 +http_request_duration_highr_seconds_bucket{le="0.1"} 2 +http_request_duration_highr_seconds_bucket{le="0.25"} 3 +http_request_duration_highr_seconds_bucket{le="0.5"} 4 +http_request_duration_highr_seconds_bucket{le="0.75"} 5 +http_request_duration_highr_seconds_bucket{le="1.0"} 6 +http_request_duration_highr_seconds_bucket{le="1.5"} 7 +http_request_duration_highr_seconds_bucket{le="2.0"} 8 +http_request_duration_highr_seconds_bucket{le="2.5"} 8 +http_request_duration_highr_seconds_bucket{le="3.0"} 9 +http_request_duration_highr_seconds_bucket{le="3.5"} 9 +http_request_duration_highr_seconds_bucket{le="4.0"} 9 +http_request_duration_highr_seconds_bucket{le="4.5"} 9 +http_request_duration_highr_seconds_bucket{le="5.0"} 9 +http_request_duration_highr_seconds_bucket{le="7.5"} 10 +http_request_duration_highr_seconds_bucket{le="10.0"} 10 +http_request_duration_highr_seconds_bucket{le="30.0"} 10 +http_request_duration_highr_seconds_bucket{le="60.0"} 10 +http_request_duration_highr_seconds_bucket{le="+Inf"} 10 +http_request_duration_highr_seconds_count 10 +http_request_duration_highr_seconds_sum 9 +# HELP http_request_duration_highr_seconds_created Latency with many buckets but no API specific labels. Made for more accurate percentile calculations. +# TYPE http_request_duration_highr_seconds_created gauge +http_request_duration_highr_seconds_created 1 +# HELP http_request_duration_seconds Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{handler="/v1/example",le="0.1",method="POST"} 1 +http_request_duration_seconds_bucket{handler="/v1/example",le="0.5",method="POST"} 3 +http_request_duration_seconds_bucket{handler="/v1/example",le="1.0",method="POST"} 7 +http_request_duration_seconds_bucket{handler="/v1/example",le="+Inf",method="POST"} 7 +http_request_duration_seconds_count{handler="/v1/example",method="POST"} 7 +http_request_duration_seconds_sum{handler="/v1/example",method="POST"} 5 +http_request_duration_seconds_bucket{handler="/health",le="0.1",method="GET"} 1 +http_request_duration_seconds_bucket{handler="/health",le="0.5",method="GET"} 2 +http_request_duration_seconds_bucket{handler="/health",le="1.0",method="GET"} 3 +http_request_duration_seconds_bucket{handler="/health",le="+Inf",method="GET"} 3 +http_request_duration_seconds_count{handler="/health",method="GET"} 3 +http_request_duration_seconds_sum{handler="/health",method="GET"} 4 +# HELP http_request_duration_seconds_created Latency with only few buckets by handler. Made to be only used if aggregation by handler is important. +# TYPE http_request_duration_seconds_created gauge +http_request_duration_seconds_created{handler="/v1/example",method="POST"} 1 +http_request_duration_seconds_created{handler="/health",method="GET"} 1 +# HELP http_requests_inprogress Number of HTTP requests in progress. +# TYPE http_requests_inprogress gauge +http_requests_inprogress 2 diff --git a/prometheus/profiles/fastapi/generator/generate.py b/prometheus/profiles/fastapi/generator/generate.py index c966e63..fd5c56e 100644 --- a/prometheus/profiles/fastapi/generator/generate.py +++ b/prometheus/profiles/fastapi/generator/generate.py @@ -8,15 +8,22 @@ from dataclasses import dataclass from pathlib import Path -from source_registry_client_python import parse_created_emitters +from source_registry_client_python import ast_fingerprint, parse_created_emitters FASTAPI_SOURCE = Path( "upstreams/fastapi_instrumentator/" "src/prometheus_fastapi_instrumentator/metrics.py" ) +MIDDLEWARE_SOURCE = Path( + "upstreams/fastapi_instrumentator/" + "src/prometheus_fastapi_instrumentator/middleware.py" +) CLIENT_SOURCE = Path("upstreams/prometheus_client_python/prometheus_client/metrics.py") +FASTAPI_METRICS_AST_FINGERPRINT = "ac64e70eeea8ccd77c118997571bc86c1fc0214a0827b099a68448a0b7d6e69e" +FASTAPI_MIDDLEWARE_AST_FINGERPRINT = "68a4fa3ca22ffb170328fe1c757a14ba1cb6d51f243627c2cf6a8f845f1b5e4e" + @dataclass(frozen=True) class MetricRegistration: @@ -27,6 +34,7 @@ class MetricRegistration: line_start: int line_end: int emits_created: bool + source_path: str CONSTRUCTORS = { @@ -54,6 +62,11 @@ class MetricRegistration: def parse_default_metrics(source: str) -> list[MetricRegistration]: + _require_reviewed_source_shape( + source, + FASTAPI_METRICS_AST_FINGERPRINT, + "FastAPI metrics module", + ) tree = ast.parse(source) functions = [ node @@ -86,6 +99,7 @@ def parse_default_metrics(source: str) -> list[MetricRegistration]: line_start=node.lineno, line_end=node.end_lineno or node.lineno, emits_created=emits_created, + source_path="src/prometheus_fastapi_instrumentator/metrics.py", ) ) @@ -97,8 +111,89 @@ def parse_default_metrics(source: str) -> list[MetricRegistration]: return sorted(registrations, key=lambda registration: registration.family) -def generate_registry(fastapi_source: str, client_source: str) -> str: +def parse_inprogress_metric(source: str) -> MetricRegistration: + _require_reviewed_source_shape( + source, + FASTAPI_MIDDLEWARE_AST_FINGERPRINT, + "FastAPI middleware module", + ) + tree = ast.parse(source) + classes = [ + node + for node in tree.body + if isinstance(node, ast.ClassDef) + and node.name == "PrometheusInstrumentatorMiddleware" + ] + if len(classes) != 1: + raise ValueError(f"expected exactly one middleware class, found {len(classes)}") + initializers = [ + node + for node in classes[0].body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name == "__init__" + ] + if len(initializers) != 1: + raise ValueError(f"expected exactly one middleware initializer, found {len(initializers)}") + initializer = initializers[0] + defaults = { + argument.arg: default + for argument, default in zip(initializer.args.kwonlyargs, initializer.args.kw_defaults) + if default is not None + } + if _literal_value(defaults.get("should_instrument_requests_inprogress")) is not False: + raise ValueError("in-progress instrumentation must default to disabled") + if _literal_value(defaults.get("inprogress_labels")) is not False: + raise ValueError("in-progress labels must default to disabled") + family = _literal_string(defaults.get("inprogress_name"), "inprogress_name default") + if family != "http_requests_inprogress": + raise ValueError(f"unexpected inprogress_name default {family!r}") + + registrations = [] + for node in ast.walk(initializer): + if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Name) or node.func.id != "Gauge": + continue + name_keywords = [keyword for keyword in node.keywords if keyword.arg == "name"] + if len(name_keywords) != 1: + continue + name = name_keywords[0].value + if ( + isinstance(name, ast.Attribute) + and isinstance(name.value, ast.Name) + and name.value.id == "self" + and name.attr == "inprogress_name" + ): + registrations.append(node) + if len(registrations) != 1: + raise ValueError(f"expected exactly one in-progress Gauge registration, found {len(registrations)}") + registration = registrations[0] + return MetricRegistration( + family=family, + prometheus_type="gauge", + shape="scalar", + components=(("value", "scalar"),), + line_start=registration.lineno, + line_end=registration.end_lineno or registration.lineno, + emits_created=False, + source_path="src/prometheus_fastapi_instrumentator/middleware.py", + ) + + +def _require_reviewed_source_shape( + source: str, + expected: str, + description: str, +) -> None: + actual = ast_fingerprint(source) + if actual != expected: + raise ValueError( + f"{description} source shape fingerprint {actual} does not match " + f"reviewed fingerprint {expected}" + ) + + +def generate_registry(fastapi_source: str, middleware_source: str, client_source: str) -> str: registrations = parse_default_metrics(fastapi_source) + inprogress = parse_inprogress_metric(middleware_source) created_emitters = parse_created_emitters(client_source) created = [registration for registration in registrations if registration.emits_created] if not created: @@ -135,25 +230,49 @@ def generate_registry(fastapi_source: str, client_source: str) -> str: for registration in registrations: lines.extend(_render_registration(registration)) lines.extend(_render_created_registration(created, created_emitters)) + lines.extend( + [ + " middleware_http:", + " registrations:", + ] + ) + lines.extend(_render_registration(inprogress, when=("inprogress", ("unlabeled", "labeled")))) return "\n".join(lines) + "\n" -def _render_registration(registration: MetricRegistration) -> list[str]: +def _render_registration( + registration: MetricRegistration, + when: tuple[str, tuple[str, ...]] | None = None, +) -> list[str]: registration_id = _registration_id(registration.family) result = [ f" {registration_id}:", f" family: {{exact: {registration.family}}}", - " prometheus: " - f"{{type: {registration.prometheus_type}, shape: {registration.shape}}}", - " components:", ] + if when: + axis, values = when + result.extend( + [ + " when:", + " any:", + " - all:", + f" - {{axis: {axis}, op: in, values: [{', '.join(values)}]}}", + ] + ) + result.extend( + [ + " prometheus: " + f"{{type: {registration.prometheus_type}, shape: {registration.shape}}}", + " components:", + ] + ) for component_id, wire_role in registration.components: result.append(f" {component_id}: {{wire_role: {wire_role}}}") result.extend( [ " source_locations:", " - upstream: fastapi_instrumentator", - " path: src/prometheus_fastapi_instrumentator/metrics.py", + f" path: {registration.source_path}", " range: " f"{{start: {registration.line_start}, end: {registration.line_end}}}", ] @@ -201,6 +320,12 @@ def _literal_string(node: ast.AST, field: str) -> str: return node.value +def _literal_value(node: ast.AST | None) -> object: + if not isinstance(node, ast.Constant): + raise ValueError("middleware default must be a literal") + return node.value + + def _single_token_namespace(families: list[str]) -> str: first_tokens = {family.split("_", 1)[0] for family in families} if len(first_tokens) != 1 or any("_" not in family for family in families): @@ -225,6 +350,7 @@ def main() -> None: print( generate_registry( FASTAPI_SOURCE.read_text(encoding="utf-8"), + MIDDLEWARE_SOURCE.read_text(encoding="utf-8"), CLIENT_SOURCE.read_text(encoding="utf-8"), ), end="", diff --git a/prometheus/profiles/fastapi/generator/test_negative_parser.py b/prometheus/profiles/fastapi/generator/test_negative_parser.py index 17e75c7..82fbeac 100644 --- a/prometheus/profiles/fastapi/generator/test_negative_parser.py +++ b/prometheus/profiles/fastapi/generator/test_negative_parser.py @@ -15,12 +15,39 @@ class FailClosedParserTest(unittest.TestCase): + def test_rejects_rebound_default_metric_constructor(self): + if not GENERATOR.FASTAPI_SOURCE.is_file(): + self.skipTest("pinned upstream source is staged only by the hermetic runner") + source = GENERATOR.FASTAPI_SOURCE.read_text(encoding="utf-8") + source = source.replace( + "from prometheus_client import REGISTRY, CollectorRegistry, Counter, Histogram, Summary", + "from prometheus_client import REGISTRY, CollectorRegistry, Gauge as Counter, Histogram, Summary", + 1, + ) + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_default_metrics(source) + + def test_rejects_rebound_inprogress_metric_constructor(self): + if not GENERATOR.MIDDLEWARE_SOURCE.is_file(): + self.skipTest("pinned upstream source is staged only by the hermetic runner") + source = GENERATOR.MIDDLEWARE_SOURCE.read_text(encoding="utf-8") + source = source.replace( + "from prometheus_client import REGISTRY, CollectorRegistry, Gauge", + "from prometheus_client import REGISTRY, CollectorRegistry, Counter as Gauge", + 1, + ) + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_inprogress_metric(source) + def test_rejects_dynamic_metric_name(self): source = """ def default(name): - Counter(name=name, documentation="requests") + try: + TOTAL = Counter(name=name, documentation="requests") + except ValueError: + pass """ - with self.assertRaisesRegex(ValueError, "nonempty string literal"): + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): GENERATOR.parse_default_metrics(source) def test_rejects_missing_created_emitter(self): @@ -36,12 +63,192 @@ def _child_samples(self): return () """ with self.assertRaisesRegex(ValueError, "gated _created sample"): - GENERATOR.parse_created_emitters(source) + GENERATOR.parse_created_emitters( + source, + expected_ast_fingerprint=GENERATOR.ast_fingerprint(source), + ) def test_rejects_mixed_metric_namespaces(self): with self.assertRaisesRegex(ValueError, "one token namespace"): GENERATOR._single_token_namespace(["http_requests", "rpc_requests"]) + def test_rejects_changed_inprogress_default(self): + source = """ +class PrometheusInstrumentatorMiddleware: + def __init__(self, *, should_instrument_requests_inprogress=False, inprogress_name="custom", inprogress_labels=False): + self.inprogress_name = inprogress_name + Gauge(name=self.inprogress_name) +""" + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_inprogress_metric(source) + + def test_rejects_unguarded_inprogress_registration(self): + source = """ +class PrometheusInstrumentatorMiddleware: + def __init__(self, *, should_instrument_requests_inprogress=False, inprogress_name="http_requests_inprogress", inprogress_labels=False): + self.should_instrument_requests_inprogress = should_instrument_requests_inprogress + self.inprogress_name = inprogress_name + self.inprogress_labels = inprogress_labels + Gauge( + name=self.inprogress_name, + labelnames=("method", "handler") if self.inprogress_labels else (), + ) +""" + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_inprogress_metric(source) + + def test_rejects_changed_inprogress_label_modes(self): + source = """ +class PrometheusInstrumentatorMiddleware: + def __init__(self, *, should_instrument_requests_inprogress=False, inprogress_name="http_requests_inprogress", inprogress_labels=False): + self.should_instrument_requests_inprogress = should_instrument_requests_inprogress + self.inprogress_name = inprogress_name + self.inprogress_labels = inprogress_labels + if self.should_instrument_requests_inprogress: + labels = ("method",) if self.inprogress_labels else () + Gauge( + name=self.inprogress_name, + labelnames=labels, + ) +""" + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_inprogress_metric(source) + + def test_rejects_rebound_inprogress_configuration(self): + source = """ +class PrometheusInstrumentatorMiddleware: + def __init__(self, *, should_instrument_requests_inprogress=False, inprogress_name="http_requests_inprogress", inprogress_labels=False): + self.should_instrument_requests_inprogress = True + self.inprogress_name = "different_runtime_family" + self.inprogress_labels = True + if self.should_instrument_requests_inprogress: + labels = ("method", "handler") if self.inprogress_labels else () + Gauge(name=self.inprogress_name, labelnames=labels) +""" + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_inprogress_metric(source) + + def test_rejects_inprogress_registration_in_negative_guard_branch(self): + source = """ +class PrometheusInstrumentatorMiddleware: + def __init__(self, *, should_instrument_requests_inprogress=False, inprogress_name="http_requests_inprogress", inprogress_labels=False): + self.should_instrument_requests_inprogress = should_instrument_requests_inprogress + self.inprogress_name = inprogress_name + self.inprogress_labels = inprogress_labels + if self.should_instrument_requests_inprogress: + pass + else: + labels = ("method", "handler") if self.inprogress_labels else () + Gauge(name=self.inprogress_name, labelnames=labels) +""" + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_inprogress_metric(source) + + def test_rejects_alternate_inprogress_attribute_writes(self): + mutations = ( + 'self.inprogress_name += "_changed"', + 'self.inprogress_name: str = "other"', + 'setattr(self, "inprogress_name", "other")', + 'self.__setattr__("inprogress_name", "other")', + ) + for mutation in mutations: + with self.subTest(mutation=mutation): + source = f'''\ +class PrometheusInstrumentatorMiddleware: + def __init__(self, *, should_instrument_requests_inprogress=False, inprogress_name="http_requests_inprogress", inprogress_labels=False): + self.should_instrument_requests_inprogress = should_instrument_requests_inprogress + self.inprogress_name = inprogress_name + self.inprogress_labels = inprogress_labels + {mutation} + if self.should_instrument_requests_inprogress: + labels = ("method", "handler") if self.inprogress_labels else () + Gauge(name=self.inprogress_name, labelnames=labels) +''' + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_inprogress_metric(source) + + def test_rejects_alternate_inprogress_labelnames_write(self): + source = ''' +class PrometheusInstrumentatorMiddleware: + def __init__(self, *, should_instrument_requests_inprogress=False, inprogress_name="http_requests_inprogress", inprogress_labels=False): + self.should_instrument_requests_inprogress = should_instrument_requests_inprogress + self.inprogress_name = inprogress_name + self.inprogress_labels = inprogress_labels + if self.should_instrument_requests_inprogress: + labels = ("method", "handler") if self.inprogress_labels else () + labels += ("extra",) + Gauge(name=self.inprogress_name, labelnames=labels) +''' + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_inprogress_metric(source) + + def test_rejects_conditional_default_registration(self): + source = ''' +def default(): + if False: + Counter(name="http_never_registered", documentation="never") +''' + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_default_metrics(source) + + def test_rejects_unreachable_direct_default_registration(self): + source = ''' +def default(): + try: + return + TOTAL = Counter(name="http_never_registered", documentation="never") + except ValueError: + pass +''' + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_default_metrics(source) + + def test_rejects_changed_default_metric_naming(self): + source = ''' +def default(metric_namespace="custom", metric_subsystem=""): + try: + TOTAL = Counter( + name="http_requests_total", + documentation="requests", + namespace=metric_namespace, + subsystem=metric_subsystem, + ) + except ValueError: + pass +''' + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_default_metrics(source) + + def test_rejects_unreachable_nested_inprogress_registration(self): + source = ''' +class PrometheusInstrumentatorMiddleware: + def __init__(self, *, should_instrument_requests_inprogress=False, inprogress_name="http_requests_inprogress", inprogress_labels=False): + self.should_instrument_requests_inprogress = should_instrument_requests_inprogress + self.inprogress_name = inprogress_name + self.inprogress_labels = inprogress_labels + if self.should_instrument_requests_inprogress: + labels = ("method", "handler") if self.inprogress_labels else () + if False: + Gauge(name=self.inprogress_name, labelnames=labels) +''' + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_inprogress_metric(source) + + def test_rejects_mapping_inprogress_mutation(self): + source = ''' +class PrometheusInstrumentatorMiddleware: + def __init__(self, *, should_instrument_requests_inprogress=False, inprogress_name="http_requests_inprogress", inprogress_labels=False): + self.should_instrument_requests_inprogress = should_instrument_requests_inprogress + self.inprogress_name = inprogress_name + self.inprogress_labels = inprogress_labels + self.__dict__["inprogress_name"] = "other" + if self.should_instrument_requests_inprogress: + labels = ("method", "handler") if self.inprogress_labels else () + Gauge(name=self.inprogress_name, labelnames=labels) +''' + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.parse_inprogress_metric(source) + if __name__ == "__main__": unittest.main() diff --git a/prometheus/profiles/litellm/generator/generate.py b/prometheus/profiles/litellm/generator/generate.py index ea04c75..00f5a81 100644 --- a/prometheus/profiles/litellm/generator/generate.py +++ b/prometheus/profiles/litellm/generator/generate.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from pathlib import Path -from source_registry_client_python import parse_created_emitters +from source_registry_client_python import parse_created_emitters, require_ast_fingerprints PROMETHEUS_SOURCE = Path("upstreams/litellm/litellm/integrations/prometheus.py") @@ -17,6 +17,13 @@ IN_FLIGHT_SOURCE = Path("upstreams/litellm/litellm/proxy/middleware/in_flight_requests_middleware.py") CLIENT_SOURCE = Path("upstreams/prometheus_client_python/prometheus_client/metrics.py") +LITELLM_SOURCE_AST_FINGERPRINTS = { + "litellm/integrations/prometheus.py": "c034f316adbd27b4dfc1c73e2459ca79d901833158ba91052fb9e70471b70c58", + "litellm/integrations/prometheus_services.py": "ef7d3491fcb384fe48eee32b284a9648bfc8db726c47826bc714db361ccc0154", + "litellm/proxy/middleware/in_flight_requests_middleware.py": "80971bf4705e7add3a425b4a1067d23d1d9c79cb54f6168019136354ac09e5e2", + "litellm/types/services.py": "7ead64cb0fce85cf98d5a8bedf5223f8f9d935ab9f12aad7f523b626e4f67789", +} + @dataclass(frozen=True) class SourceLocation: @@ -227,6 +234,16 @@ def generate_registry( in_flight_source: str, client_source: str, ) -> str: + require_ast_fingerprints( + { + "litellm/integrations/prometheus.py": prometheus_source, + "litellm/integrations/prometheus_services.py": services_source, + "litellm/proxy/middleware/in_flight_requests_middleware.py": in_flight_source, + "litellm/types/services.py": service_types_source, + }, + LITELLM_SOURCE_AST_FINGERPRINTS, + "LiteLLM registration modules", + ) callback = parse_callback_metrics(prometheus_source) services = parse_service_metrics(services_source, service_types_source) in_flight = parse_in_flight_metric(in_flight_source) diff --git a/prometheus/profiles/litellm/generator/test_negative_parser.py b/prometheus/profiles/litellm/generator/test_negative_parser.py index 0d209d5..bf80caa 100644 --- a/prometheus/profiles/litellm/generator/test_negative_parser.py +++ b/prometheus/profiles/litellm/generator/test_negative_parser.py @@ -5,6 +5,8 @@ import sys import unittest +from source_registry_client_python import ast_fingerprint + MODULE_PATH = pathlib.Path(__file__).with_name("generate.py") SPEC = importlib.util.spec_from_file_location("litellm_registry_generator", MODULE_PATH) @@ -15,6 +17,10 @@ class FailClosedParserTest(unittest.TestCase): + def test_rejects_unreviewed_registration_source_shapes(self): + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.generate_registry("", "", "", "", "") + def test_rejects_dynamic_callback_metric_name(self): source = """ class PrometheusLogger: @@ -53,7 +59,10 @@ def _child_samples(self): return () """ with self.assertRaisesRegex(ValueError, "gated _created sample"): - GENERATOR.parse_created_emitters(source) + GENERATOR.parse_created_emitters( + source, + expected_ast_fingerprint=ast_fingerprint(source), + ) if __name__ == "__main__": diff --git a/prometheus/profiles/vllm/SOURCE-SEMANTICS.yaml b/prometheus/profiles/vllm/SOURCE-SEMANTICS.yaml index 7699d4f..a3c459a 100644 --- a/prometheus/profiles/vllm/SOURCE-SEMANTICS.yaml +++ b/prometheus/profiles/vllm/SOURCE-SEMANTICS.yaml @@ -174,7 +174,7 @@ evidence: - vllm/distributed/kv_transfer/kv_connector/v1/offloading/metrics.py:21-150 - vllm/entrypoints/speech_to_text/realtime/metrics.py:21-78 - vllm/parser/metrics.py:15-108 - claim: The registered metric meanings support the profile's exact percentage, concurrency, and per-GPU display conversions. + claim: The registered metric meanings support the profile's exact percentage and per-GPU display conversions. environment: axes: transport: diff --git a/prometheus/profiles/vllm/generator/generate.py b/prometheus/profiles/vllm/generator/generate.py index 6074908..c0769b8 100644 --- a/prometheus/profiles/vllm/generator/generate.py +++ b/prometheus/profiles/vllm/generator/generate.py @@ -8,7 +8,10 @@ from dataclasses import dataclass from pathlib import Path -from source_registry_client_python import parse_created_emitters as parse_client_created_emitters +from source_registry_client_python import ( + parse_created_emitters as parse_client_created_emitters, + require_ast_fingerprints, +) VLLM_ROOT = Path("upstreams/vllm") @@ -16,6 +19,7 @@ RAY_AGENT_SOURCE = Path("upstreams/ray/python/ray/_private/metrics_agent.py") RAY_TAG_SOURCE = Path("upstreams/ray/src/ray/stats/tag_defs.cc") RAY_WRAPPER_PATH = "vllm/v1/metrics/ray_wrappers.py" +RAY_AGENT_PATH = "python/ray/_private/metrics_agent.py" REGISTRATION_PATHS = ( "vllm/v1/metrics/loggers.py", @@ -33,6 +37,27 @@ "vllm/v1/kv_offload/tiering/spec.py", ) +VLLM_SOURCE_AST_FINGERPRINTS = { + "vllm/v1/metrics/loggers.py": "1afc2c7c922a796299543eaffd4bbc12d8ae9786347fa569f45298fc83d90dbe", + "vllm/v1/metrics/perf.py": "24874c09180752512468c75c56c74d3328f8e62ff518abbcaca6e1f3c77f36cd", + "vllm/v1/spec_decode/metrics.py": "c71b1957ebac328e59f1a1f5301c1e2d38552b3bfa22489fb88baa9f99ca459f", + "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py": "37e5c2a90d5457b658197813587226971ad051e9dc0d17f584859104b8915c84", + "vllm/distributed/kv_transfer/kv_connector/v1/hf3fs/hf3fs_connector.py": "7ede827f0b0d3f43403558427f2858eeaa31ac5a844ac8fe62a078bc54f3181e", + "vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/metrics.py": "c641a72ccb8c74745c5e68e8ce260e62b4cdf55bb3b1e62d3f74b386e27b559d", + "vllm/distributed/kv_transfer/kv_connector/v1/offloading/metrics.py": "04ba764211864e688175a12a655ec099852d4892e4478779683eeef95226fe00", + "vllm/entrypoints/speech_to_text/realtime/metrics.py": "b510a4c0362b321d48d940df2c4f5e981f5e45e718e9c885b1b1881dc95b01a5", + "vllm/parser/metrics.py": "66fea8e6880458bf72d14f31359bc2b620aca527034d9420187d6cdc14f50deb", + "vllm/v1/kv_offload/cpu/common.py": "8f5da827569e2fcd2df08101ba0b483328f7ed8bbf466f5255cefbff1086fc9e", + "vllm/v1/kv_offload/cpu/spec.py": "47de1b3683f4e84a60f3e5e44249c537eebfbdcc2b618b1d7b24ebad042e6e7c", + "vllm/v1/kv_offload/tiering/base.py": "ed96667431d44f0eb3b05e7caf2b674b54e3bc023db092fc3ec1fdaefe58b3f6", + "vllm/v1/kv_offload/tiering/spec.py": "9a88971a1371354aa1674f86b60887f7338ae5f9b9cdacd2fb26153de65aaa82", +} + +VLLM_TRANSPORT_AST_FINGERPRINTS = { + RAY_WRAPPER_PATH: "8288a3dad228d3bbdc65d3170395af3ffad6706f27338788dabadc208991acf2", + RAY_AGENT_PATH: "424c3c13c63a2953aead5c83ca42048a799adda4eca70da80364c03582635a68", +} + RAY_UNSUPPORTED_PATH_PREFIXES = ( "vllm/entrypoints/speech_to_text/realtime/", "vllm/parser/", @@ -329,6 +354,16 @@ def generate_registry( ray_agent: str, ray_tags: str, ) -> str: + require_ast_fingerprints( + sources, + VLLM_SOURCE_AST_FINGERPRINTS, + "vLLM registration modules", + ) + require_ast_fingerprints( + {RAY_WRAPPER_PATH: ray_wrapper, RAY_AGENT_PATH: ray_agent}, + VLLM_TRANSPORT_AST_FINGERPRINTS, + "vLLM Ray transport modules", + ) native = parse_registrations(sources) created_emitters = parse_created_emitters(client_source) validate_ray_transport(ray_wrapper, ray_agent, ray_tags) diff --git a/prometheus/profiles/vllm/generator/test_negative_parser.py b/prometheus/profiles/vllm/generator/test_negative_parser.py index 0faee76..3db0f8f 100644 --- a/prometheus/profiles/vllm/generator/test_negative_parser.py +++ b/prometheus/profiles/vllm/generator/test_negative_parser.py @@ -15,6 +15,11 @@ class FailClosedParserTest(unittest.TestCase): + def test_rejects_unreviewed_registration_source_shapes(self): + sources = {path: "" for path in GENERATOR.REGISTRATION_PATHS} + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + GENERATOR.generate_registry(sources, "", "", "", "") + def test_rejects_untyped_metric_declaration(self): sources = {"metrics.py": 'MISSING = "vllm:missing"\n'} with self.assertRaisesRegex(ValueError, "no typed registration"): diff --git a/prometheus/tools/source_registry_client_python.py b/prometheus/tools/source_registry_client_python.py index 791ed46..5078c99 100644 --- a/prometheus/tools/source_registry_client_python.py +++ b/prometheus/tools/source_registry_client_python.py @@ -4,15 +4,60 @@ from __future__ import annotations import ast +import hashlib +import json + + +CLIENT_METRICS_AST_FINGERPRINT = "da91514df6b25f19f34ae9e99ca85a47a49201e9d8187f6422a0571397e56744" + + +def ast_fingerprint(source: str) -> str: + """Return a line-independent fingerprint of one complete Python module.""" + canonical = json.dumps( + _canonical_ast(ast.parse(source)), + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(canonical).hexdigest() + + +def require_ast_fingerprints( + sources: dict[str, str], + expected: dict[str, str], + description: str, +) -> None: + """Require the exact reviewed executable shapes for a Python source closure.""" + if set(sources) != set(expected): + missing = sorted(set(expected) - set(sources)) + unexpected = sorted(set(sources) - set(expected)) + raise ValueError( + f"{description} source closure differs: missing={missing}, unexpected={unexpected}" + ) + for path in sorted(expected): + actual = ast_fingerprint(sources[path]) + if actual != expected[path]: + raise ValueError( + f"{description} source shape fingerprint for {path} is {actual}; " + f"reviewed fingerprint is {expected[path]}" + ) def parse_created_emitters( source: str, class_names: tuple[str, ...] = ("Counter", "Summary", "Histogram"), + *, + expected_ast_fingerprint: str = CLIENT_METRICS_AST_FINGERPRINT, ) -> dict[str, tuple[int, int]]: """Return source ranges for classes that emit one gated _created sample.""" if not class_names or len(class_names) != len(set(class_names)): raise ValueError("created-emitter class names must be nonempty and unique") + actual_fingerprint = ast_fingerprint(source) + if actual_fingerprint != expected_ast_fingerprint: + raise ValueError( + "prometheus_client metrics source shape fingerprint " + f"{actual_fingerprint} does not match reviewed fingerprint " + f"{expected_ast_fingerprint}" + ) tree = ast.parse(source) result: dict[str, tuple[int, int]] = {} @@ -42,11 +87,7 @@ def parse_created_emitters( and isinstance(node.args[0], ast.Constant) and node.args[0].value == "_created" ] - uses_gate = any( - isinstance(node, ast.Name) and node.id == "_use_created" - for node in ast.walk(method) - ) - if len(created_samples) != 1 or not uses_gate: + if len(created_samples) != 1 or not _guarded_by_created_gate(created_samples[0], method): raise ValueError(f"{class_name} does not have one gated _created sample") result[class_name] = (method.lineno, method.end_lineno or method.lineno) return result @@ -58,3 +99,47 @@ def _terminal_name(node: ast.AST) -> str | None: if isinstance(node, ast.Attribute): return node.attr return None + + +def _guarded_by_created_gate(node: ast.AST, method: ast.AST) -> bool: + parents: dict[ast.AST, ast.AST] = {} + for parent in ast.walk(method): + for child in ast.iter_child_nodes(parent): + parents[child] = parent + + current = node + while current is not method: + parent = parents.get(current) + if parent is None: + return False + if ( + isinstance(parent, ast.If) + and current in parent.body + and isinstance(parent.test, ast.Name) + and parent.test.id == "_use_created" + ): + return True + current = parent + return False + + +def _canonical_ast(value: object) -> object: + if isinstance(value, ast.AST): + fields = [] + for name, field in ast.iter_fields(value): + normalized = _canonical_ast(field) + if normalized is None or normalized == []: + continue + fields.append([name, normalized]) + return [type(value).__name__, fields] + if isinstance(value, list): + return [_canonical_ast(item) for item in value] + if isinstance(value, bytes): + return ["bytes", value.hex()] + if value is Ellipsis: + return ["ellipsis"] + if isinstance(value, complex): + return ["complex", value.real, value.imag] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + raise TypeError(f"unsupported AST field value {type(value).__name__}") diff --git a/prometheus/tools/test_source_registry_client_python.py b/prometheus/tools/test_source_registry_client_python.py index 7153053..3c69099 100644 --- a/prometheus/tools/test_source_registry_client_python.py +++ b/prometheus/tools/test_source_registry_client_python.py @@ -2,7 +2,11 @@ import unittest -from source_registry_client_python import parse_created_emitters +from source_registry_client_python import ( + ast_fingerprint, + parse_created_emitters, + require_ast_fingerprints, +) VALID_SOURCE = """ @@ -22,22 +26,53 @@ async def _child_samples(self): class CreatedEmitterParserTest(unittest.TestCase): + def test_requires_the_complete_reviewed_source_closure(self): + expected = {"one.py": ast_fingerprint("value = 1\n")} + require_ast_fingerprints({"one.py": "value = 1\n"}, expected, "test modules") + with self.assertRaisesRegex(ValueError, "source closure differs"): + require_ast_fingerprints({}, expected, "test modules") + with self.assertRaisesRegex(ValueError, "source shape fingerprint"): + require_ast_fingerprints({"one.py": "value = 2\n"}, expected, "test modules") + def test_returns_exact_method_ranges(self): self.assertEqual( - parse_created_emitters(VALID_SOURCE), + parse_created_emitters( + VALID_SOURCE, + expected_ast_fingerprint=ast_fingerprint(VALID_SOURCE), + ), {"Counter": (3, 5), "Summary": (7, 9), "Histogram": (11, 13)}, ) def test_supports_an_explicit_class_subset(self): self.assertEqual( - set(parse_created_emitters(VALID_SOURCE, ("Counter", "Histogram"))), + set( + parse_created_emitters( + VALID_SOURCE, + ("Counter", "Histogram"), + expected_ast_fingerprint=ast_fingerprint(VALID_SOURCE), + ) + ), {"Counter", "Histogram"}, ) def test_rejects_missing_gate(self): source = VALID_SOURCE.replace("if _use_created:\n return [samples.Sample", "return [samples.Sample", 1) with self.assertRaisesRegex(ValueError, "gated _created sample"): - parse_created_emitters(source) + parse_created_emitters( + source, + expected_ast_fingerprint=ast_fingerprint(source), + ) + + def test_rejects_created_samples_behind_unrelated_false_guards(self): + source = VALID_SOURCE.replace( + "if _use_created:\n return", + "enabled = _use_created\n if False:\n return", + ) + with self.assertRaisesRegex(ValueError, "gated _created sample"): + parse_created_emitters( + source, + expected_ast_fingerprint=ast_fingerprint(source), + ) def test_rejects_duplicate_class_contract(self): with self.assertRaisesRegex(ValueError, "nonempty and unique"):