diff --git a/changelog.d/http_server_encoding.breaking.md b/changelog.d/http_server_encoding.breaking.md new file mode 100644 index 0000000000000..82485637bfb8e --- /dev/null +++ b/changelog.d/http_server_encoding.breaking.md @@ -0,0 +1,19 @@ +# HTTP server `encoding` option removed {#http-server-encoding-removed} + +## Summary + +The deprecated `encoding` option has been removed from the `http` and +`http_server` sources. Configurations using it now fail validation. + +## Migration + +Replace `encoding` with `decoding` and `framing`: + +| Previous `encoding` | `decoding.codec` | `framing.method` | +| --- | --- | --- | +| `text` | `text` | `newline_delimited` | +| `json` | `json` | `bytes` | +| `ndjson` | `json` | `newline_delimited` | +| `binary` | `bytes` | `bytes` | + +authors: pront diff --git a/deprecation.d/http-server-encoding.md b/deprecation.d/http-server-encoding.md deleted file mode 100644 index 9753bac0ab5f8..0000000000000 --- a/deprecation.d/http-server-encoding.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -what: "`encoding` field on HTTP server sources" -deprecated_since: "0.50.0" ---- - -The `encoding` field will be removed. Use `decoding` and `framing` instead. diff --git a/src/sources/http_server.rs b/src/sources/http_server.rs index 0c99f12c7d130..cf1ae69f2921e 100644 --- a/src/sources/http_server.rs +++ b/src/sources/http_server.rs @@ -6,11 +6,7 @@ use http::StatusCode; use http_serde; use tokio_util::codec::Decoder as _; use vector_lib::{ - codecs::{ - BytesDecoderConfig, BytesDeserializerConfig, JsonDeserializerConfig, - NewlineDelimitedDecoderConfig, - decoding::{DeserializerConfig, FramingConfig}, - }, + codecs::decoding::{DeserializerConfig, FramingConfig}, config::{DataType, LegacyKey, LogNamespace}, configurable::configurable_component, lookup::{lookup_v2::OptionalValuePath, owned_value_path, path}, @@ -30,7 +26,7 @@ use crate::{ http::KeepaliveConfig, serde::{bool_or_struct, default_decoding}, sources::util::{ - Encoding, HttpSource, + HttpSource, http::{HttpMethod, add_headers, add_query_parameters}, }, tls::TlsEnableableConfig, @@ -71,6 +67,7 @@ impl SourceConfig for HttpConfig { /// Configuration for the `http_server` source. #[configurable_component(source("http_server", "Host an HTTP endpoint to receive logs."))] #[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] pub struct SimpleHttpConfig { /// The socket address to listen for connections on. /// @@ -79,13 +76,6 @@ pub struct SimpleHttpConfig { #[configurable(metadata(docs::examples = "localhost:80"))] address: SocketAddr, - /// The expected encoding of received data. - /// - /// For `json` and `ndjson` encodings, the fields of the JSON objects are output as separate fields. - #[configurable(deprecated)] - #[serde(default)] - encoding: Option, - /// A list of HTTP headers to include in the log event. /// /// Accepts the wildcard (`*`) character for headers matching a specified pattern. @@ -241,37 +231,11 @@ impl SimpleHttpConfig { } fn get_decoding_config(&self) -> crate::Result { - if self.encoding.is_some() && (self.framing.is_some() || self.decoding.is_some()) { - return Err("Using `encoding` is deprecated and does not have any effect when `decoding` or `framing` is provided. Configure `framing` and `decoding` instead.".into()); - } - - let (framing, decoding) = if let Some(encoding) = self.encoding { - match encoding { - Encoding::Text => ( - NewlineDelimitedDecoderConfig::new().into(), - BytesDeserializerConfig::new().into(), - ), - Encoding::Json => ( - BytesDecoderConfig::new().into(), - JsonDeserializerConfig::default().into(), - ), - Encoding::Ndjson => ( - NewlineDelimitedDecoderConfig::new().into(), - JsonDeserializerConfig::default().into(), - ), - Encoding::Binary => ( - BytesDecoderConfig::new().into(), - BytesDeserializerConfig::new().into(), - ), - } - } else { - let decoding = self.decoding.clone().unwrap_or_else(default_decoding); - let framing = self - .framing - .clone() - .unwrap_or_else(|| decoding.default_stream_framing()); - (framing, decoding) - }; + let decoding = self.decoding.clone().unwrap_or_else(default_decoding); + let framing = self + .framing + .clone() + .unwrap_or_else(|| decoding.default_stream_framing()); Ok(DecodingConfig::new( framing, @@ -285,7 +249,6 @@ impl Default for SimpleHttpConfig { fn default() -> Self { Self { address: "0.0.0.0:8080".parse().unwrap(), - encoding: None, headers: Vec::new(), query_parameters: Vec::new(), tls: None, @@ -596,6 +559,19 @@ mod tests { crate::test_util::test_generate_config::(); } + #[test] + fn rejects_removed_encoding_field() { + let error = serde_yaml::from_str::( + r#" +address: "0.0.0.0:8080" +encoding: text +"#, + ) + .unwrap_err(); + + assert!(error.to_string().contains("unknown field `encoding`")); + } + #[allow(clippy::too_many_arguments)] async fn source<'a>( headers: Vec, @@ -628,7 +604,6 @@ mod tests { SimpleHttpConfig { address, headers, - encoding: None, query_parameters, response_code, tls: None, diff --git a/website/cue/reference/components/sources/generated/http.cue b/website/cue/reference/components/sources/generated/http.cue index 9024dcde6d039..d612bf02edaa6 100644 --- a/website/cue/reference/components/sources/generated/http.cue +++ b/website/cue/reference/components/sources/generated/http.cue @@ -393,21 +393,6 @@ generated: components: sources: http: configuration: { } } } - encoding: { - deprecated: true - description: """ - The expected encoding of received data. - - For `json` and `ndjson` encodings, the fields of the JSON objects are output as separate fields. - """ - required: false - type: string: enum: { - binary: "Binary." - json: "JSON." - ndjson: "Newline-delimited JSON." - text: "Plaintext." - } - } framing: { description: """ Framing configuration. diff --git a/website/cue/reference/components/sources/generated/http_server.cue b/website/cue/reference/components/sources/generated/http_server.cue index 9f2fd3b59393f..6d48e2a8a5c7d 100644 --- a/website/cue/reference/components/sources/generated/http_server.cue +++ b/website/cue/reference/components/sources/generated/http_server.cue @@ -393,21 +393,6 @@ generated: components: sources: http_server: configuration: { } } } - encoding: { - deprecated: true - description: """ - The expected encoding of received data. - - For `json` and `ndjson` encodings, the fields of the JSON objects are output as separate fields. - """ - required: false - type: string: enum: { - binary: "Binary." - json: "JSON." - ndjson: "Newline-delimited JSON." - text: "Plaintext." - } - } framing: { description: """ Framing configuration. diff --git a/website/cue/reference/components/sources/http_server.cue b/website/cue/reference/components/sources/http_server.cue index e2ff0100c61bd..4f2880648cfe2 100644 --- a/website/cue/reference/components/sources/http_server.cue +++ b/website/cue/reference/components/sources/http_server.cue @@ -63,7 +63,7 @@ components: sources: http_server: { fields: { message: { description: "The raw line from the incoming payload." - relevant_when: "encoding == \"text\"" + relevant_when: "decoding.codec == \"text\"" required: true type: string: { examples: ["Hello world"] @@ -91,7 +91,7 @@ components: sources: http_server: { fields: { "*": { description: "Any field contained in your JSON payload" - relevant_when: "encoding != \"text\"" + relevant_when: "decoding.codec != \"text\"" required: false type: "*": {} } diff --git a/website/data/deprecations.json b/website/data/deprecations.json index 645f6b49788c1..08e99f13f797d 100644 --- a/website/data/deprecations.json +++ b/website/data/deprecations.json @@ -20,11 +20,6 @@ "deprecated_since": "0.56.0", "description": "The `series_api_version: v1` option is deprecated in favor of `v2` (the default).\nThe v1 series endpoint (`/api/v1/series`) is a legacy endpoint.\n\nUsers should remove `series_api_version: v1` from their configuration or set it to `v2`." }, - { - "what": "`encoding` field on HTTP server sources", - "deprecated_since": "0.50.0", - "description": "The `encoding` field will be removed. Use `decoding` and `framing` instead." - }, { "what": "Environment-variable and secret placeholders in non-string positions", "deprecated_since": "0.57.0", @@ -37,6 +32,12 @@ "deprecated_since": "0.55.0", "removed_in": "0.56.0", "description": "The `greptimedb_metrics` and `greptimedb_logs` sinks drop support for GreptimeDB v0.x.\nUsers must upgrade their GreptimeDB instance to v1.x before upgrading Vector." + }, + { + "what": "`encoding` field on HTTP server sources", + "deprecated_since": "0.50.0", + "removed_in": "0.58.0", + "description": "The `encoding` field will be removed. Use `decoding` and `framing` instead." } ] }