diff --git a/include/datadog/sampling_decision.h b/include/datadog/sampling_decision.h index cfccbf74..fa79d548 100644 --- a/include/datadog/sampling_decision.h +++ b/include/datadog/sampling_decision.h @@ -39,6 +39,9 @@ struct SamplingDecision { // The per-second maximum allowed number of "keeps" configured for the limiter // consulted in this decision, if any. Optional limiter_max_per_second; + // Whether the sample rate alone, before the rate limiter, would keep this + // trace. + Optional was_probability_sampled; // The provenance of this decision. Origin origin; }; diff --git a/include/datadog/trace_segment.h b/include/datadog/trace_segment.h index f7fa958a..e4c0e24d 100644 --- a/include/datadog/trace_segment.h +++ b/include/datadog/trace_segment.h @@ -77,6 +77,7 @@ class TraceSegment { std::vector> spans_; std::size_t num_finished_spans_; Optional sampling_decision_; + const Optional otel_w3c_tracestate_; const Optional additional_w3c_tracestate_; const Optional additional_datadog_w3c_tracestate_; @@ -99,6 +100,7 @@ class TraceSegment { Optional origin, std::size_t tags_header_max_size, std::vector> trace_tags, Optional sampling_decision, + Optional otel_w3c_tracestate, Optional additional_w3c_tracestate, Optional additional_datadog_w3c_tracestate, std::unique_ptr local_root, diff --git a/src/datadog/extracted_data.h b/src/datadog/extracted_data.h index a26274d7..83e12a5a 100644 --- a/src/datadog/extracted_data.h +++ b/src/datadog/extracted_data.h @@ -26,11 +26,13 @@ struct ExtractedData { // refering to the latest datadog parent ID. Optional datadog_w3c_parent_id; // If this `ExtractedData` was created on account of `PropagationStyle::W3C`, - // then `additional_w3c_tracestate` contains the parts of the "tracestate" - // header that are not the "dd" (Datadog) entry. If there are no other parts, - // then `additional_w3c_tracestate` is null. + // then `additional_w3c_tracestate` contains the entries of the "tracestate" + // header other than the "dd" (Datadog) and "ot" (OpenTelemetry) entries. + // If there are no such entries, then `additional_w3c_tracestate` is null. // `additional_w3c_tracestate` is used for the `W3C` injection style. Optional additional_w3c_tracestate; + // The raw value of the OpenTelemetry `ot` tracestate member, if present. + Optional otel_w3c_tracestate; // If this `ExtractedData` was created on account of `PropagationStyle::W3C`, // and if the "tracestate" header contained a "dd" (Datadog) entry, then // `additional_datadog_w3c_tracestate` contains fields from within the "dd" diff --git a/src/datadog/extraction_util.cpp b/src/datadog/extraction_util.cpp index b5405ece..e9a7c27e 100644 --- a/src/datadog/extraction_util.cpp +++ b/src/datadog/extraction_util.cpp @@ -283,6 +283,7 @@ ExtractedData merge( if (w3c != contexts.end() && w3c->second.trace_id == result.trace_id) { result.additional_w3c_tracestate = w3c->second.additional_w3c_tracestate; + result.otel_w3c_tracestate = w3c->second.otel_w3c_tracestate; result.additional_datadog_w3c_tracestate = w3c->second.additional_datadog_w3c_tracestate; result.headers_examined.insert(result.headers_examined.end(), diff --git a/src/datadog/sampling_util.h b/src/datadog/sampling_util.h index 19f424f4..3d6500c6 100644 --- a/src/datadog/sampling_util.h +++ b/src/datadog/sampling_util.h @@ -4,6 +4,7 @@ // `TraceSampler` and `SpanSampler`. #include +#include #include #include @@ -11,6 +12,22 @@ namespace datadog { namespace tracing { +inline bool is_probability_mechanism(int mechanism) { + switch (static_cast(mechanism)) { + case SamplingMechanism::DEFAULT: + case SamplingMechanism::AGENT_RATE: + case SamplingMechanism::REMOTE_RATE_AUTO: + case SamplingMechanism::RULE: + case SamplingMechanism::REMOTE_RATE_USER_DEFINED: + case SamplingMechanism::REMOTE_RATE_EMERGENCY: + case SamplingMechanism::REMOTE_RULE: + case SamplingMechanism::REMOTE_ADAPTIVE_RULE: + return true; + default: + return false; + } +} + // Return a hash value for the specified `value`. `value` is one of the // following: // diff --git a/src/datadog/trace_sampler.cpp b/src/datadog/trace_sampler.cpp index d455b9c4..9784cde5 100644 --- a/src/datadog/trace_sampler.cpp +++ b/src/datadog/trace_sampler.cpp @@ -50,8 +50,10 @@ SamplingDecision TraceSampler::decide(const SpanData& span) { decision.mechanism = int(rule.mechanism); decision.limiter_max_per_second = limiter_max_per_second_; decision.configured_rate = rule.rate; - const std::uint64_t threshold = max_id_from_rate(rule.rate); - if (knuth_hash(span.trace_id.low) <= threshold) { + const std::uint64_t threshold = max_id_from_rate(*decision.configured_rate); + decision.was_probability_sampled = + knuth_hash(span.trace_id.low) <= threshold; + if (*decision.was_probability_sampled) { if (rule.bypass_limiter) { decision.priority = int(SamplingPriority::USER_KEEP); return decision; @@ -91,7 +93,8 @@ SamplingDecision TraceSampler::decide(const SpanData& span) { } const std::uint64_t threshold = max_id_from_rate(*decision.configured_rate); - if (knuth_hash(span.trace_id.low) <= threshold) { + decision.was_probability_sampled = knuth_hash(span.trace_id.low) <= threshold; + if (*decision.was_probability_sampled) { decision.priority = int(SamplingPriority::AUTO_KEEP); } else { decision.priority = int(SamplingPriority::AUTO_DROP); diff --git a/src/datadog/trace_segment.cpp b/src/datadog/trace_segment.cpp index 555f648c..5c044fd5 100644 --- a/src/datadog/trace_segment.cpp +++ b/src/datadog/trace_segment.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include "endpoint_inferral.h" #include "hex.h" #include "platform_util.h" +#include "sampling_util.h" #include "span_data.h" #include "span_sampler.h" #include "tag_propagation.h" @@ -151,6 +153,39 @@ Optional format_rate(double rate, Logger& logger) { return std::string(begin, end); } +Optional resolve_otel_tracestate( + TraceID trace_id, const SamplingDecision& decision, + const Optional& inherited) { + if (decision.origin != SamplingDecision::Origin::LOCAL) { + return inherited ? sanitize_otel_tracestate(StringView(*inherited)) + : nullopt; + } + + const StringView raw = inherited ? StringView(*inherited) : StringView{}; + if (!decision.mechanism || !decision.configured_rate || + !decision.was_probability_sampled || + !is_probability_mechanism(*decision.mechanism) || + (*decision.was_probability_sampled && decision.priority <= 0)) { + return rewrite_otel_tracestate(raw, extract_otel_random_value(raw), + nullopt); + } + + constexpr std::uint64_t max_value = UINT64_C(1) << 56; + std::uint64_t threshold = static_cast( + std::round((1.0 - decision.configured_rate->value()) * + static_cast(max_value))); + threshold = std::min(threshold, max_value - 1); + + std::uint64_t random_value = (~knuth_hash(trace_id.low)) >> 8; + if (*decision.was_probability_sampled && random_value < threshold) { + random_value = threshold; + } else if (!*decision.was_probability_sampled && random_value >= threshold) { + random_value = threshold == 0 ? 0 : threshold - 1; + } + + return rewrite_otel_tracestate(raw, random_value, threshold); +} + } // anonymous namespace TraceSegment::TraceSegment( @@ -166,6 +201,7 @@ TraceSegment::TraceSegment( std::size_t tags_header_max_size, std::vector> trace_tags, Optional sampling_decision, + Optional otel_w3c_tracestate, Optional additional_w3c_tracestate, Optional additional_datadog_w3c_tracestate, std::unique_ptr local_root, @@ -184,6 +220,7 @@ TraceSegment::TraceSegment( trace_tags_(std::move(trace_tags)), num_finished_spans_(0), sampling_decision_(std::move(sampling_decision)), + otel_w3c_tracestate_(std::move(otel_w3c_tracestate)), additional_w3c_tracestate_(std::move(additional_w3c_tracestate)), additional_datadog_w3c_tracestate_( std::move(additional_datadog_w3c_tracestate)), @@ -214,16 +251,16 @@ Optional TraceSegment::sampling_decision() const { return sampling_decision_; } -Optional> TraceSegment::w3c_link_context( +Optional TraceSegment::w3c_link_context( const SpanData& span) const { - int sampling_priority; + SamplingDecision sampling_decision; std::vector> trace_tags; { std::lock_guard lock(mutex_); if (!sampling_decision_) { return nullopt; } - sampling_priority = sampling_decision_->priority; + sampling_decision = *sampling_decision_; trace_tags = trace_tags_; const Optional trace_source_tag = @@ -234,10 +271,13 @@ Optional> TraceSegment::w3c_link_context( } return std::make_pair( - encode_tracestate(span.span_id, sampling_priority, origin_, trace_tags, - additional_datadog_w3c_tracestate_, - additional_w3c_tracestate_), - sampling_priority > 0 ? 1u : 0u); + encode_tracestate( + span.span_id, sampling_decision.priority, origin_, trace_tags, + additional_datadog_w3c_tracestate_, + resolve_otel_tracestate(span.trace_id, sampling_decision, + otel_w3c_tracestate_), + additional_w3c_tracestate_), + sampling_decision.priority > 0 ? 1u : 0u); } Logger& TraceSegment::logger() const { return *logger_; } @@ -443,13 +483,13 @@ bool TraceSegment::inject(DictWriter& writer, const SpanData& span, // and trace tags might change when that happens ("_dd.p.dm"). // So, we lock here, make a sampling decision if necessary, and then copy the // decision and trace tags before unlocking. - int sampling_priority; + SamplingDecision sampling_decision; std::vector> trace_tags; { std::lock_guard lock(mutex_); make_sampling_decision_if_null(); assert(sampling_decision_); - sampling_priority = sampling_decision_->priority; + sampling_decision = *sampling_decision_; trace_tags = trace_tags_; } @@ -464,7 +504,7 @@ bool TraceSegment::inject(DictWriter& writer, const SpanData& span, // - the local root span is NOT created by another product (no `_dd.p.ts`) // - sampling priority is DROP if (!tracing_enabled_) { - if (!trace_source_tag && sampling_priority <= 0) { + if (!trace_source_tag && sampling_decision.priority <= 0) { writer.erase("x-datadog-trace-id"); writer.erase("x-datadog-parent-id"); writer.erase("x-datadog-sampling-priority"); @@ -492,7 +532,7 @@ bool TraceSegment::inject(DictWriter& writer, const SpanData& span, writer.set("x-datadog-trace-id", std::to_string(span.trace_id.low)); writer.set("x-datadog-parent-id", std::to_string(span.span_id)); writer.set("x-datadog-sampling-priority", - std::to_string(sampling_priority)); + std::to_string(sampling_decision.priority)); if (origin_) { writer.set("x-datadog-origin", *origin_); } @@ -509,7 +549,8 @@ bool TraceSegment::inject(DictWriter& writer, const SpanData& span, writer.set("x-b3-traceid", hex_padded(span.trace_id.low)); } writer.set("x-b3-spanid", hex_padded(span.span_id)); - writer.set("x-b3-sampled", std::to_string(int(sampling_priority > 0))); + writer.set("x-b3-sampled", + std::to_string(int(sampling_decision.priority > 0))); if (origin_) { writer.set("x-datadog-origin", *origin_); } @@ -519,14 +560,16 @@ bool TraceSegment::inject(DictWriter& writer, const SpanData& span, {"header_style:b3multi"}); break; case PropagationStyle::W3C: - writer.set( - "traceparent", - encode_traceparent(span.trace_id, span.span_id, sampling_priority)); - writer.set( - "tracestate", - encode_tracestate(span.span_id, sampling_priority, origin_, - trace_tags, additional_datadog_w3c_tracestate_, - additional_w3c_tracestate_)); + writer.set("traceparent", + encode_traceparent(span.trace_id, span.span_id, + sampling_decision.priority)); + writer.set("tracestate", + encode_tracestate( + span.span_id, sampling_decision.priority, origin_, + trace_tags, additional_datadog_w3c_tracestate_, + resolve_otel_tracestate(span.trace_id, sampling_decision, + otel_w3c_tracestate_), + additional_w3c_tracestate_)); telemetry::counter::increment(metrics::tracer::trace_context::injected, {"header_style:tracecontext"}); break; diff --git a/src/datadog/tracer.cpp b/src/datadog/tracer.cpp index 8c0e6bd7..4872dcea 100644 --- a/src/datadog/tracer.cpp +++ b/src/datadog/tracer.cpp @@ -247,7 +247,8 @@ Span Tracer::create_span(const SpanConfig& config) { logger_, collector_, config_manager_->trace_sampler(), span_sampler_, defaults, config_manager_, runtime_id_, injection_styles_, hostname_, nullopt /* origin */, tags_header_max_size_, std::move(trace_tags), - nullopt /* sampling_decision */, nullopt /* additional_w3c_tracestate */, + nullopt /* sampling_decision */, nullopt /* otel_w3c_tracestate */, + nullopt /* additional_w3c_tracestate */, nullopt /* additional_datadog_w3c_tracestate*/, std::move(span_data), resource_renaming_mode_, tracing_enabled_); Span span{span_data_ptr, segment, @@ -483,6 +484,7 @@ Expected Tracer::extract_span(const DictReader& reader, injection_styles_, hostname_, std::move(merged_context.origin), tags_header_max_size_, std::move(merged_context.trace_tags), std::move(sampling_decision), + std::move(merged_context.otel_w3c_tracestate), std::move(merged_context.additional_w3c_tracestate), std::move(merged_context.additional_datadog_w3c_tracestate), std::move(span_data), resource_renaming_mode_, tracing_enabled_); diff --git a/src/datadog/w3c_propagation.cpp b/src/datadog/w3c_propagation.cpp index acba657f..670963c4 100644 --- a/src/datadog/w3c_propagation.cpp +++ b/src/datadog/w3c_propagation.cpp @@ -33,11 +33,109 @@ auto verboten(int lowest_ascii, int highest_ascii, }; } -constexpr bool is_hexdiglc(const char c) { +constexpr bool is_hexdig(const char c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } +constexpr bool is_lowercase_hexdig(const char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); +} + +constexpr std::size_t otel_random_value_size = 14; +constexpr std::size_t min_otel_threshold_size = 1; +constexpr std::size_t max_otel_threshold_size = otel_random_value_size; +constexpr std::size_t max_w3c_tracestate_member_value_size = 256; +constexpr std::size_t max_datadog_tracestate_value_size = 512; +constexpr std::size_t max_w3c_tracestate_members = 32; + +Optional parse_otel_value(StringView value, + std::size_t minimum_size, + std::size_t maximum_size) { + if (value.size() < minimum_size || value.size() > maximum_size || + !std::all_of(value.begin(), value.end(), is_lowercase_hexdig)) { + return nullopt; + } + + const Expected parsed = parse_uint64(value, 16); + if (parsed.if_error()) { + return nullopt; + } + return *parsed; +} + +Optional parse_otel_random_value(StringView value) { + return parse_otel_value(value, otel_random_value_size, + otel_random_value_size); +} + +Optional parse_otel_threshold(StringView value) { + return parse_otel_value(value, min_otel_threshold_size, + max_otel_threshold_size); +} + +template +void for_each_otel_item(StringView raw, Function&& function) { + std::size_t begin = 0; + while (begin < raw.size()) { + const std::size_t end = raw.find(';', begin); + const StringView item = raw.substr(begin, end - begin); + const std::size_t separator = item.find(':'); + const StringView key = item.substr(0, separator); + const StringView value = separator == StringView::npos + ? StringView{} + : item.substr(separator + 1); + function(item, key, value); + if (end == StringView::npos) { + return; + } + begin = end + 1; + } +} + +template +void for_each_w3c_tracestate_member(StringView tracestate, + Function&& function) { + std::size_t begin = 0; + while (begin < tracestate.size()) { + const std::size_t end = tracestate.find(',', begin); + const StringView member = trim(tracestate.substr(begin, end - begin)); + if (!function(member) || end == StringView::npos) { + return; + } + begin = end + 1; + } +} + +void append_otel_item(std::string& result, StringView item) { + if (item.empty()) { + return; + } + + const std::size_t separator_size = result.empty() ? 0 : 1; + if (result.size() + separator_size + item.size() > + max_w3c_tracestate_member_value_size) { + return; + } + + if (separator_size) { + result += ';'; + } + append(result, item); +} + +std::string format_otel_hex(std::uint64_t value) { + return hex_padded(value).substr(2); +} + +std::string format_otel_threshold(std::uint64_t threshold) { + std::string result = format_otel_hex(threshold); + while (result.size() > 1 && result.back() == '0') { + result.pop_back(); + } + return result; +} + // Populate the specified `result` with data extracted from the "traceparent" // entry of the specified `headers`. Return `nullopt` on success. Return a value // for the `tags::internal::w3c_extraction_error` tag if an error occurs. @@ -64,7 +162,7 @@ Optional extract_traceparent(ExtractedData& result, beg = i + 1; internal_state = state::trace_id; - } else if (!is_hexdiglc(traceparent[i])) { + } else if (!is_hexdig(traceparent[i])) { return "invalid_version"; } } break; @@ -124,86 +222,25 @@ Optional extract_traceparent(ExtractedData& result, return nullopt; } -// `struct PartiallyParsedTracestat` contains the separated Datadog-specific and -// non-Datadog-specific portions of tracestate. -struct PartiallyParsedTracestate { - StringView datadog_value; - std::string other_entries; -}; - -// Return the separate Datadog-specific and non-Datadog-specific portions of the -// specified `tracestate`. If `tracestate` does not have a Datadog-specific -// portion, return `nullopt`. -Optional parse_tracestate(StringView tracestate) { - const std::size_t begin = 0; - const std::size_t end = tracestate.size(); - std::size_t pair_begin = begin; - while (pair_begin < end) { - const std::size_t pair_end = tracestate.find(',', pair_begin); - // Note that since this `pair` is `strip`ped, `pair_begin` is not - // necessarily equal to `pair.begin()` (similarly for the ends). - const auto pair = - trim(tracestate.substr(pair_begin, pair_end - pair_begin)); - if (pair.empty()) { - pair_begin = (pair_end == StringView::npos) ? end : pair_end + 1; - continue; - } - - const auto kv_separator = pair.find('='); - if (kv_separator == StringView::npos) { - // This is an invalid entry because it contains a non-whitespace character - // but not a "=". - // Let's move on to the next entry. - pair_begin = (pair_end == StringView::npos) ? end : pair_end + 1; - continue; - } - - const auto key = pair.substr(0, kv_separator); - if (key != "dd") { - // On to the next. - pair_begin = (pair_end == StringView::npos) ? end : pair_end + 1; - continue; - } - - PartiallyParsedTracestate result; - result.datadog_value = pair.substr(kv_separator + 1); - // `result->other_entries` is whatever was before the "dd" entry and - // whatever is after the "dd" entry, but without an extra comma in the - // middle. - if (pair_begin != 0) { - // There's a prefix - append(result.other_entries, tracestate.substr(0, pair_begin - 1)); - if (pair_end != StringView::npos && pair_end + 1 < end) { - // and a suffix - append(result.other_entries, tracestate.substr(pair_end)); - } - } else if (pair_end != StringView::npos && pair_end + 1 < end) { - // There's just a suffix - append(result.other_entries, tracestate.substr(pair_end + 1)); - } - - return result; - } - - return nullopt; -} // Fill the specified `result` with information parsed from the specified -// `datadog_value`. `datadog_value` is the value of the "dd" entry in the -// "tracestate" header. +// `datadog_trace_state`. `datadog_trace_state` is the value of the "dd" entry +// in the W3C "tracestate" header. // -// `parse_datadog_tracestate` populates the following `ExtractedData` fields: +// `parse_datadog_trace_state` populates the following `ExtractedData` fields: // // - `origin` // - `trace_tags` // - `sampling_priority` // - `datadog_w3c_parent_id` // - `additional_datadog_w3c_tracestate` -void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { - const std::size_t end = datadog_value.size(); +void parse_datadog_trace_state(ExtractedData& result, + StringView datadog_trace_state) { + const std::size_t end = datadog_trace_state.size(); std::size_t pair_begin = 0; while (pair_begin < end) { - const std::size_t pair_end = datadog_value.find(';', pair_begin); - const auto pair = datadog_value.substr(pair_begin, pair_end - pair_begin); + const std::size_t pair_end = datadog_trace_state.find(';', pair_begin); + const auto pair = + datadog_trace_state.substr(pair_begin, pair_end - pair_begin); pair_begin = (pair_end == StringView::npos) ? end : pair_end + 1; if (pair.empty()) { continue; @@ -272,45 +309,65 @@ void parse_datadog_tracestate(ExtractedData& result, StringView datadog_value) { } } -// Fill the specified `result` with information parsed from the "tracestate" -// element of the specified `headers`, if present. +// Fill the specified `result` with information parsed from the specified +// `ot_tracestate`. `ot_tracestate` is the value of the "ot" entry in the W3C +// "tracestate" header. // -// `extract_tracestate` populates the `additional_w3c_tracestate` field of -// `ExtractedData`, in addition to those populated by -// `parse_datadog_tracestate`. -void extract_tracestate( - ExtractedData& result, const DictReader& headers, - std::unordered_map& span_tags) { - const auto maybe_tracestate = headers.lookup("tracestate"); - if (!maybe_tracestate || maybe_tracestate->empty()) { - return; +// `parse_ot_tracestate` preserves the first OpenTelemetry state in a header. +void parse_ot_tracestate(ExtractedData& result, StringView ot_tracestate) { + if (!result.otel_w3c_tracestate) { + result.otel_w3c_tracestate = std::string(ot_tracestate); } +} - const auto tracestate = trim(*maybe_tracestate); - result.tracestate_full = tracestate; - - auto maybe_parsed = parse_tracestate(tracestate); - if (!maybe_parsed) { - // No "dd" entry in `tracestate`, so there's nothing to extract. - if (!tracestate.empty()) { - result.additional_w3c_tracestate = std::string{tracestate}; +void parse_w3c_tracestate_member( + ExtractedData& result, StringView member, + std::unordered_map& span_tags, + std::string& other_w3c_tracestate) { + const std::size_t separator = member.find('='); + const StringView key = member.substr(0, separator); + const StringView member_value = separator == StringView::npos + ? StringView{} + : member.substr(separator + 1); + if (key == "dd") { + if (member_value.size() > max_datadog_tracestate_value_size) { + span_tags[tags::internal::propagation_error] = "extract_max_size"; + } else { + parse_datadog_trace_state(result, member_value); } - return; - } - - auto& [datadog_value, other_entries] = *maybe_parsed; - if (!other_entries.empty()) { - result.additional_w3c_tracestate = std::move(other_entries); + } else if (key == "ot") { + if (member_value.size() > max_w3c_tracestate_member_value_size) { + span_tags[tags::internal::propagation_error] = "extract_max_size"; + } else { + parse_ot_tracestate(result, member_value); + } + } else { + if (!other_w3c_tracestate.empty()) { + other_w3c_tracestate += ','; + } + append(other_w3c_tracestate, member); } +} - // If the "dd" vendor entry's value exceeds 512 bytes, drop it and record a - // propagation error tag. - if (datadog_value.size() > 512) { - span_tags[tags::internal::propagation_error] = "extract_max_size"; - return; +// Fill the specified `result` with information parsed from the specified W3C +// `tracestate`. +// +// `parse_w3c_tracestate` populates `additional_w3c_tracestate`, in addition to +// the fields populated by `parse_datadog_trace_state` and +// `parse_ot_tracestate`. +void parse_w3c_tracestate( + ExtractedData& result, StringView w3c_tracestate, + std::unordered_map& span_tags) { + std::string other_w3c_tracestate; + for_each_w3c_tracestate_member(w3c_tracestate, [&](StringView member) { + parse_w3c_tracestate_member(result, member, span_tags, + other_w3c_tracestate); + return true; + }); + + if (!other_w3c_tracestate.empty()) { + result.additional_w3c_tracestate = std::move(other_w3c_tracestate); } - - parse_datadog_tracestate(result, datadog_value); } } // namespace @@ -340,7 +397,12 @@ Expected extract_w3c( } result.datadog_w3c_parent_id = "0000000000000000"; - extract_tracestate(result, headers, span_tags); + const Optional maybe_tracestate = headers.lookup("tracestate"); + if (maybe_tracestate && !maybe_tracestate->empty()) { + const StringView tracestate = trim(*maybe_tracestate); + result.tracestate_full = tracestate; + parse_w3c_tracestate(result, tracestate, span_tags); + } return result; } @@ -411,12 +473,11 @@ std::string encode_datadog_tracestate( result += *additional_datadog_w3c_tracestate; } - const std::size_t max_size = 256; - while (result.size() > max_size) { + while (result.size() > max_w3c_tracestate_member_value_size) { const auto last_semicolon_index = result.rfind(';'); // This assumption is safe, because `result` always begins with - // "dd=s:", and that's fewer than `max_size` characters for any - // ``. + // "dd=s:", and that's fewer than + // `max_w3c_tracestate_member_value_size` characters for any ``. assert(last_semicolon_index != std::string::npos); result.resize(last_semicolon_index); } @@ -424,19 +485,89 @@ std::string encode_datadog_tracestate( return result; } +Optional sanitize_otel_tracestate(StringView raw) { + std::string result; + for_each_otel_item(raw, + [&](StringView item, StringView key, StringView value) { + if ((key == "rv" && !parse_otel_random_value(value)) || + (key == "th" && !parse_otel_threshold(value))) { + return; + } + append_otel_item(result, item); + }); + if (result.empty()) { + return nullopt; + } + return result; +} + +Optional extract_otel_random_value(StringView raw) { + Optional result; + for_each_otel_item(raw, [&](StringView, StringView key, StringView value) { + if (!result && key == "rv") { + result = parse_otel_random_value(value); + } + }); + return result; +} + +Optional rewrite_otel_tracestate( + StringView raw, Optional random_value, + Optional threshold) { + std::string result; + if (random_value) { + const std::string item = "rv:" + format_otel_hex(*random_value); + append_otel_item(result, item); + } + if (threshold) { + const std::string item = "th:" + format_otel_threshold(*threshold); + append_otel_item(result, item); + } + + for_each_otel_item(raw, [&](StringView item, StringView key, StringView) { + if (key != "rv" && key != "th") { + append_otel_item(result, item); + } + }); + + if (result.empty()) { + return nullopt; + } + return result; +} + +void append_tracestate_entries(std::string& result, StringView entries, + std::size_t& member_count) { + for_each_w3c_tracestate_member(entries, [&](StringView entry) { + if (!entry.empty()) { + result += ','; + append(result, entry); + ++member_count; + } + return member_count < max_w3c_tracestate_members; + }); +} + std::string encode_tracestate( uint64_t span_id, int sampling_priority, const Optional& origin, const std::vector>& trace_tags, const Optional& additional_datadog_w3c_tracestate, + const Optional& otel_w3c_tracestate, const Optional& additional_w3c_tracestate) { std::string result = encode_datadog_tracestate(span_id, sampling_priority, origin, trace_tags, additional_datadog_w3c_tracestate); + std::size_t member_count = 1; + if (otel_w3c_tracestate) { + result += ",ot="; + result += *otel_w3c_tracestate; + ++member_count; + } + if (additional_w3c_tracestate) { - result += ','; - result += *additional_w3c_tracestate; + append_tracestate_entries(result, *additional_w3c_tracestate, member_count); } return result; diff --git a/src/datadog/w3c_propagation.h b/src/datadog/w3c_propagation.h index 823cce39..7c1e6616 100644 --- a/src/datadog/w3c_propagation.h +++ b/src/datadog/w3c_propagation.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -37,12 +38,21 @@ Expected extract_w3c( std::string encode_traceparent(TraceID trace_id, std::uint64_t span_id, int sampling_priority); +Optional sanitize_otel_tracestate(StringView raw); + +Optional extract_otel_random_value(StringView raw); + +Optional rewrite_otel_tracestate( + StringView raw, Optional random_value, + Optional threshold); + // Return a value for the "tracestate" header containing the specified fields. std::string encode_tracestate( uint64_t span_id, int sampling_priority, const Optional& origin, const std::vector>& trace_tags, const Optional& additional_datadog_w3c_tracestate, + const Optional& otel_w3c_tracestate, const Optional& additional_w3c_tracestate); } // namespace tracing diff --git a/test/test_span.cpp b/test/test_span.cpp index 6608e60b..1304c099 100644 --- a/test/test_span.cpp +++ b/test/test_span.cpp @@ -792,7 +792,8 @@ TEST_SPAN("injecting W3C tracestate header") { {"x-datadog-origin", "France"}, }, // The "s:-1" and "t.ksr:0" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id;o:France;t.ksr:0"}, + "dd=s:-1;p:$parent_id;o:France;t.ksr:0,ot=rv:f0948a54d43b8e;th:" + "ffffffffffffff"}, {__LINE__, "trace tags", @@ -802,7 +803,8 @@ TEST_SPAN("injecting W3C tracestate header") { {"x-datadog-tags", "_dd.p.foo=x,_dd.p.bar=y,ignored=wrong_prefix"}, }, // The "s:-1" and "t.ksr:0" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id;t.foo:x;t.bar:y;t.ksr:0"}, + "dd=s:-1;p:$parent_id;t.foo:x;t.bar:y;t.ksr:0,ot=rv:f0948a54d43b8e;" + "th:ffffffffffffff"}, {__LINE__, "extra fields", @@ -832,7 +834,7 @@ TEST_SPAN("injecting W3C tracestate header") { }, // The "s:-1" comes from the 0% sample rate. "dd=s:-1;p:$parent_id;o:France_ is a country~nation_ so is " - "______.;t.ksr:0", + "______.;t.ksr:0,ot=rv:f0948a54d43b8e;th:ffffffffffffff", }, {__LINE__, @@ -843,7 +845,8 @@ TEST_SPAN("injecting W3C tracestate header") { {"x-datadog-tags", "_dd.p.a;d台北x =foo,_dd.p.ok=bar"}, }, // The "s:-1" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id;t.a_d______x_:foo;t.ok:bar;t.ksr:0"}, + "dd=s:-1;p:$parent_id;t.a_d______x_:foo;t.ok:bar;t.ksr:0,ot=rv:" + "f0948a54d43b8e;th:ffffffffffffff"}, {__LINE__, "replace invalid characters in trace tag value", @@ -854,7 +857,7 @@ TEST_SPAN("injecting W3C tracestate header") { }, // The "s:-1" comes from the 0% sample rate. "dd=s:-1;p:$parent_id;t.wacky:hello fr_d_ how are " - "_________?;t.ksr:0"}, + "_________?;t.ksr:0,ot=rv:f0948a54d43b8e;th:ffffffffffffff"}, {__LINE__, "replace equal signs with tildes in trace tag value", @@ -864,7 +867,8 @@ TEST_SPAN("injecting W3C tracestate header") { {"x-datadog-tags", "_dd.p.base64_thingy=d2Fra2EhIHdhaw=="}, }, // The "s:-1" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id;t.base64_thingy:d2Fra2EhIHdhaw~~;t.ksr:0"}, + "dd=s:-1;p:$parent_id;t.base64_thingy:d2Fra2EhIHdhaw~~;t.ksr:0,ot=" + "rv:f0948a54d43b8e;th:ffffffffffffff"}, {__LINE__, "oversized origin truncates it and subsequent fields", @@ -883,7 +887,7 @@ TEST_SPAN("injecting W3C tracestate header") { {"x-datadog-tags", "_dd.p.foo=bar,_dd.p.honk=honk"}, }, // The "s:-1" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id"}, + "dd=s:-1;p:$parent_id,ot=rv:f0948a54d43b8e;th:ffffffffffffff"}, {__LINE__, "oversized trace tag truncates it and subsequent fields", @@ -901,7 +905,8 @@ TEST_SPAN("injecting W3C tracestate header") { "ooooooooooooooooooong,_dd.p.lost=forever"}, }, // The "s:-1" comes from the 0% sample rate. - "dd=s:-1;p:$parent_id;t.foo:bar"}, + "dd=s:-1;p:$parent_id;t.foo:bar,ot=rv:f0948a54d43b8e;" + "th:ffffffffffffff"}, {__LINE__, "oversized extra field truncates itself and subsequent fields", @@ -955,6 +960,127 @@ TEST_SPAN("injecting W3C tracestate header") { REQUIRE(logger->error_count() == 0); } +TEST_SPAN("OpenTelemetry consistent probability sampling") { + class Generator : public IDGenerator { + const TraceID trace_id_; + + public: + explicit Generator(TraceID trace_id) : trace_id_(trace_id) {} + TraceID trace_id(const TimePoint&) const override { return trace_id_; } + std::uint64_t span_id() const override { return trace_id_.low; } + }; + + SECTION("local probability decisions emit a consistent rv and th") { + struct TestCase { + double rate; + std::uint64_t trace_id; + bool sampled; + std::string expected_ot; + }; + + const auto test_case = GENERATE(values({ + {0.01, 1, false, "rv:f0948a54d43b8e;th:fd70a3d70a3d7"}, + {0.1, 1, true, "rv:f0948a54d43b8e;th:e6666666666668"}, + {0.2, 1, true, "rv:f0948a54d43b8e;th:ccccccccccccd"}, + {0.5, 1, true, "rv:f0948a54d43b8e;th:8"}, + {0.99, 1, true, "rv:f0948a54d43b8e;th:028f5c28f5c29"}, + {0.1, UINT64_C(0x03A93EE8B1999F00), true, + "rv:e6666666666668;th:e6666666666668"}, + {0.05, UINT64_C(5401449561355763072), false, + "rv:f333333333332f;th:f333333333333"}, + })); + + CAPTURE(test_case.rate); + CAPTURE(test_case.trace_id); + CAPTURE(test_case.expected_ot); + + TracerConfig config; + config.service = "testsvc"; + config.collector = std::make_shared(); + config.logger = std::make_shared(); + config.telemetry.enabled = false; + config.injection_styles = {PropagationStyle::W3C}; + config.trace_sampler.sample_rate = test_case.rate; + config.trace_sampler.max_per_second = 100; + + const Expected finalized = finalize_config(config); + REQUIRE(finalized); + Tracer tracer{*finalized, + std::make_shared(TraceID(test_case.trace_id))}; + + Span span = tracer.create_span(); + MockDictWriter writer; + span.inject(writer); + + const auto tracestate = writer.items.find("tracestate"); + REQUIRE(tracestate != writer.items.end()); + REQUIRE(tracestate->second.find("dd=") == 0); + REQUIRE(tracestate->second.find("ot=" + test_case.expected_ot) != + std::string::npos); + const std::string& traceparent = writer.items.at("traceparent"); + REQUIRE(traceparent.substr(traceparent.size() - 3) == + (test_case.sampled ? "-01" : "-00")); + } + + SECTION("non-probability decisions retain inherited rv but erase th") { + TracerConfig config; + config.service = "testsvc"; + config.collector = std::make_shared(); + config.logger = std::make_shared(); + config.telemetry.enabled = false; + config.extraction_styles = {PropagationStyle::W3C}; + config.injection_styles = {PropagationStyle::W3C}; + + const Expected finalized = finalize_config(config); + REQUIRE(finalized); + Tracer tracer{*finalized}; + + const std::unordered_map input_headers{ + {"traceparent", + "00-00000000000000000000000000000001-0000000000000001-00"}, + {"tracestate", "ot=rv:1234567890abcd;th:e6666666666668"}, + }; + MockDictReader reader{input_headers}; + Expected span = tracer.extract_span(reader); + REQUIRE(span); + span->trace_segment().override_sampling_priority( + int(SamplingPriority::USER_KEEP)); + + MockDictWriter writer; + span->inject(writer); + REQUIRE(writer.items.at("tracestate").find("ot=rv:1234567890abcd") != + std::string::npos); + REQUIRE(writer.items.at("tracestate").find("th:") == std::string::npos); + } + + SECTION("rate-limiter demotion clears locally generated sampling values") { + TracerConfig config; + config.service = "testsvc"; + config.collector = std::make_shared(); + config.logger = std::make_shared(); + config.telemetry.enabled = false; + config.injection_styles = {PropagationStyle::W3C}; + config.trace_sampler.sample_rate = 1.0; + config.trace_sampler.max_per_second = 0.1; + + const Expected finalized = finalize_config(config); + REQUIRE(finalized); + Tracer tracer{*finalized, std::make_shared(TraceID(1))}; + + { + Span span = tracer.create_span(); + MockDictWriter writer; + span.inject(writer); + REQUIRE(writer.items.at("tracestate").find("ot=") != std::string::npos); + } + + Span span = tracer.create_span(); + MockDictWriter writer; + span.inject(writer); + REQUIRE(writer.items.at("tracestate").find("ot=") == std::string::npos); + } +} + TEST_SPAN("128-bit trace ID injection") { TracerConfig config; config.service = "testsvc"; diff --git a/test/test_tracer.cpp b/test/test_tracer.cpp index 55f8d15c..4cd26aef 100644 --- a/test/test_tracer.cpp +++ b/test/test_tracer.cpp @@ -1291,6 +1291,33 @@ TEST_TRACER("span extraction") { "extract_max_size"); } + SECTION( + "'extract_max_size' propagation error if tracestate \"ot\" vendor " + "value is oversized on extract") { + constexpr std::size_t max_w3c_tracestate_member_value_size = 256; + const std::string ot_value(max_w3c_tracestate_member_value_size + 1, 'a'); + std::unordered_map span_tags; + MockLogger logger; + CAPTURE(logger.entries); + CAPTURE(span_tags); + + std::unordered_map headers{ + {"traceparent", + "00-00000000000000000000000000000001-0000000000000001-00"}, + {"tracestate", "dd=s:1,ot=" + ot_value + ",vendorx=keepme"}}; + MockDictReader reader{headers}; + + const auto extracted = extract_w3c(reader, span_tags, logger); + REQUIRE(extracted); + REQUIRE(extracted->otel_w3c_tracestate == nullopt); + REQUIRE(extracted->additional_w3c_tracestate == "vendorx=keepme"); + + REQUIRE(logger.entries.empty()); + REQUIRE(span_tags.count(tags::internal::propagation_error) == 1); + REQUIRE(span_tags.at(tags::internal::propagation_error) == + "extract_max_size"); + } + SECTION("W3C Phase 3 support - Preferring tracecontext") { // Tests behavior from system-test // test_headers_tracecontext.py::test_tracestate_w3c_p_extract_datadog_w3c @@ -1727,6 +1754,67 @@ TEST_TRACER("restart extraction link uses metadata from the selected context") { REQUIRE(link.context.flags == Optional(1u)); } +TEST_TRACER("OpenTelemetry tracestate sampling values") { + SECTION("malformed sampling values are removed") { + const Optional normalized = + sanitize_otel_tracestate("rv:1234567890abcd;th:ABC;future:value"); + REQUIRE(normalized); + REQUIRE(*normalized == "rv:1234567890abcd;future:value"); + + REQUIRE(!sanitize_otel_tracestate("rv:1234567890ABCD;th:A")); + } + + SECTION("sampling values are replaced without altering other values") { + const Optional rewritten = rewrite_otel_tracestate( + "rv:bad;future:value;th:bad", UINT64_C(0xf0948a54d43b8e), + UINT64_C(0xe6666666666668)); + REQUIRE(rewritten); + REQUIRE(*rewritten == "rv:f0948a54d43b8e;th:e6666666666668;future:value"); + + const Optional no_threshold = + rewrite_otel_tracestate("rv:1234567890abcd;th:e6666666666668", + UINT64_C(0x1234567890abcd), nullopt); + REQUIRE(no_threshold); + REQUIRE(*no_threshold == "rv:1234567890abcd"); + } + + SECTION("the OpenTelemetry member is separated from other vendors") { + const std::unordered_map headers{ + {"traceparent", + "00-00000000000000000000000000000001-0000000000000001-01"}, + {"tracestate", + "dd=s:2,ot=rv:1234567890abcd;th:e6666666666668;future:value," + "congo=t61rcWkgMzE"}, + }; + MockDictReader reader{headers}; + std::unordered_map span_tags; + MockLogger logger; + + const Expected extracted = + extract_w3c(reader, span_tags, logger); + REQUIRE(extracted); + REQUIRE(extracted->otel_w3c_tracestate == + "rv:1234567890abcd;th:e6666666666668;future:value"); + REQUIRE(extracted->additional_w3c_tracestate == "congo=t61rcWkgMzE"); + } + + SECTION("the first OpenTelemetry member is retained") { + const std::unordered_map headers{ + {"traceparent", + "00-00000000000000000000000000000001-0000000000000001-01"}, + {"tracestate", "ot=first,ot=second"}, + }; + MockDictReader reader{headers}; + std::unordered_map span_tags; + MockLogger logger; + + const Expected extracted = + extract_w3c(reader, span_tags, logger); + REQUIRE(extracted); + REQUIRE(extracted->otel_w3c_tracestate == "first"); + } +} + TEST_TRACER("baggage usage") { TracerConfig config; config.logger = std::make_shared(); @@ -2037,9 +2125,12 @@ TEST_TRACER("heterogeneous extraction") { {{"x-datadog-trace-id", "48"}, {"x-datadog-parent-id", "64"}, {"x-datadog-origin", "Kansas"}, {"x-datadog-sampling-priority", "2"}, {"traceparent", "00-00000000000000000000000000000030-0000000000000040-01"}, - {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo"}}, // origin is different + {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo," + "ot=rv:1234567890abcd;th:e6666666666668;future:value"}}, // origin is different {{"traceparent", "00-00000000000000000000000000000030-000000000000002a-01"}, - {"tracestate", "dd=s:2;p:000000000000002a;o:Kansas;ah:choo,competitor=stuff"}}}, + {"tracestate", "dd=s:2;p:000000000000002a;o:Kansas;ah:choo," + "ot=rv:1234567890abcd;th:e6666666666668;future:value," + "competitor=stuff"}}}, {__LINE__, "ignore interlopers", {PropagationStyle::DATADOG, PropagationStyle::B3, PropagationStyle::W3C}, @@ -2060,7 +2151,8 @@ TEST_TRACER("heterogeneous extraction") { {{"x-datadog-trace-id", "48"}, {"x-datadog-parent-id", "64"}, {"x-datadog-origin", "Kansas"}, {"x-datadog-sampling-priority", "2"}, {"traceparent", "00-00000000000000000000000000000031-0000000000000040-01"}, - {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo"}}, + {"tracestate", "competitor=stuff,dd=o:Nebraska;s:1;ah:choo," + "ot=rv:1234567890abcd;th:e6666666666668;future:value"}}, {{"traceparent", "00-00000000000000000000000000000030-000000000000002a-01"}, {"tracestate", "dd=s:2;p:000000000000002a;o:Kansas"}}},