diff --git a/include/fluent-bit/multiline/flb_ml.h b/include/fluent-bit/multiline/flb_ml.h index c9c10e0ee43..2d132d29b4e 100644 --- a/include/fluent-bit/multiline/flb_ml.h +++ b/include/fluent-bit/multiline/flb_ml.h @@ -254,6 +254,9 @@ struct flb_ml_parser_ins { /* Link to struct flb_ml_group->parsers */ struct mk_list _head; + + /* drop a flush whose key_content is an empty string (off) */ + int drop_empty_content; }; struct flb_ml_group { diff --git a/plugins/in_tail/tail_config.c b/plugins/in_tail/tail_config.c index 3f325989615..8f41990f2bf 100644 --- a/plugins/in_tail/tail_config.c +++ b/plugins/in_tail/tail_config.c @@ -69,6 +69,8 @@ static int multiline_load_parsers(struct flb_tail_config *ctx) if (!parser_i) { return -1; } + + parser_i->drop_empty_content = ctx->skip_empty_lines; } } diff --git a/src/multiline/flb_ml.c b/src/multiline/flb_ml.c index 413808d9a01..2afaf2b8971 100644 --- a/src/multiline/flb_ml.c +++ b/src/multiline/flb_ml.c @@ -1596,6 +1596,7 @@ int flb_ml_flush_stream_group(struct flb_ml_parser *ml_parser, int ret; int size; int len; + int key_id; size_t off = 0; msgpack_object map; msgpack_object k; @@ -1682,8 +1683,17 @@ int flb_ml_flush_stream_group(struct flb_ml_parser *ml_parser, } } else { - /* The buffer is empty, so just pack the original map from the context */ - msgpack_pack_object(&mp_pck, map); + key_id = -1; + if (parser_i->drop_empty_content) { + key_id = get_key_id(&map, parser_i->key_content); + } + + if (key_id == -1 || + map.via.map.ptr[key_id].val.type != MSGPACK_OBJECT_STR || + map.via.map.ptr[key_id].val.via.str.size > 0) { + /* The buffer is empty, so just pack the original map from the context */ + msgpack_pack_object(&mp_pck, map); + } } msgpack_unpacked_destroy(&result); diff --git a/tests/internal/multiline.c b/tests/internal/multiline.c index fd13a252be7..3220b72a54b 100644 --- a/tests/internal/multiline.c +++ b/tests/internal/multiline.c @@ -82,6 +82,37 @@ struct record_check cri_output[] = { {"4b. non multiline 2"} }; +/* CRI, empty payload dropped via drop_empty_content */ +struct record_check issue_6703_input[] = { + {"2025-01-01T00:00:00.000000000Z stdout F hello world"}, + {"2025-01-01T00:00:00.000000001Z stdout F "}, + {"2025-01-01T00:00:00.000000002Z stdout F goodbye world"} +}; + +struct record_check issue_6703_output[] = { + {"hello world"}, + {"goodbye world"} +}; + +/* Same input, drop_empty_content left off (default): behavior is unchanged */ +struct record_check issue_6703_disabled_output[] = { + {"hello world"}, + {""}, + {"goodbye world"} +}; + +/* CRI, every line in isolation is an empty payload */ +struct record_check issue_6703_all_empty_input[] = { + {"2025-01-01T00:00:00.000000000Z stdout F "}, + {"2025-01-01T00:00:00.000000001Z stdout F "}, + {"2025-01-01T00:00:00.000000002Z stdout F "} +}; + +/* Sink for issue_6703_all_empty: only read if the fix regresses */ +struct record_check issue_6703_all_empty_output[] = { + {""}, {""}, {""} +}; + /* ENDSWITH */ struct record_check endswith_input[] = { {"1a. some multiline log \\"}, @@ -585,6 +616,150 @@ static void test_parser_cri() flb_config_exit(config); } +static void test_issue_6703() +{ + int i; + int len; + int ret; + int entries; + uint64_t stream_id; + struct record_check *r; + struct flb_config *config; + struct flb_time tm; + struct flb_ml *ml; + struct flb_ml_parser_ins *mlp_i; + struct expected_result res = {0}; + + res.key = "log"; + res.out_records = issue_6703_output; + + config = flb_config_init(); + + ml = flb_ml_create(config, "cri-drop-empty-test"); + TEST_CHECK(ml != NULL); + + mlp_i = flb_ml_parser_instance_create(ml, "cri"); + TEST_CHECK(mlp_i != NULL); + mlp_i->drop_empty_content = FLB_TRUE; + + ret = flb_ml_stream_create(ml, "cri-drop-empty", -1, flush_callback, + (void *) &res, &stream_id); + TEST_CHECK(ret == 0); + + entries = sizeof(issue_6703_input) / sizeof(struct record_check); + for (i = 0; i < entries; i++) { + r = &issue_6703_input[i]; + len = strlen(r->buf); + flb_time_get(&tm); + ret = flb_ml_append_text(ml, stream_id, &tm, r->buf, len); + TEST_CHECK(ret == FLB_MULTILINE_OK); + } + + TEST_CHECK(res.current_record == 2); + + if (ml) { + flb_ml_destroy(ml); + } + + flb_config_exit(config); +} + +static void test_issue_6703_disabled() +{ + int i; + int len; + int ret; + int entries; + uint64_t stream_id; + struct record_check *r; + struct flb_config *config; + struct flb_time tm; + struct flb_ml *ml; + struct flb_ml_parser_ins *mlp_i; + struct expected_result res = {0}; + + res.key = "log"; + res.out_records = issue_6703_disabled_output; + + config = flb_config_init(); + + ml = flb_ml_create(config, "cri-drop-empty-disabled-test"); + TEST_CHECK(ml != NULL); + + mlp_i = flb_ml_parser_instance_create(ml, "cri"); + TEST_CHECK(mlp_i != NULL); + TEST_CHECK(mlp_i->drop_empty_content == FLB_FALSE); + + ret = flb_ml_stream_create(ml, "cri-drop-empty-disabled", -1, flush_callback, + (void *) &res, &stream_id); + TEST_CHECK(ret == 0); + + entries = sizeof(issue_6703_input) / sizeof(struct record_check); + for (i = 0; i < entries; i++) { + r = &issue_6703_input[i]; + len = strlen(r->buf); + flb_time_get(&tm); + ret = flb_ml_append_text(ml, stream_id, &tm, r->buf, len); + TEST_CHECK(ret == FLB_MULTILINE_OK); + } + + TEST_CHECK(res.current_record == 3); + + if (ml) { + flb_ml_destroy(ml); + } + + flb_config_exit(config); +} + +static void test_issue_6703_all_empty() +{ + int i; + int len; + int ret; + int entries; + uint64_t stream_id; + struct record_check *r; + struct flb_config *config; + struct flb_time tm; + struct flb_ml *ml; + struct flb_ml_parser_ins *mlp_i; + struct expected_result res = {0}; + + res.key = "log"; + res.out_records = issue_6703_all_empty_output; + + config = flb_config_init(); + + ml = flb_ml_create(config, "cri-drop-empty-all-empty-test"); + TEST_CHECK(ml != NULL); + + mlp_i = flb_ml_parser_instance_create(ml, "cri"); + TEST_CHECK(mlp_i != NULL); + mlp_i->drop_empty_content = FLB_TRUE; + + ret = flb_ml_stream_create(ml, "cri-drop-empty-all-empty", -1, flush_callback, + (void *) &res, &stream_id); + TEST_CHECK(ret == 0); + + entries = sizeof(issue_6703_all_empty_input) / sizeof(struct record_check); + for (i = 0; i < entries; i++) { + r = &issue_6703_all_empty_input[i]; + len = strlen(r->buf); + flb_time_get(&tm); + ret = flb_ml_append_text(ml, stream_id, &tm, r->buf, len); + TEST_CHECK(ret == FLB_MULTILINE_OK); + } + + TEST_CHECK(res.current_record == 0); + + if (ml) { + flb_ml_destroy(ml); + } + + flb_config_exit(config); +} + static void test_container_mix() { int i; @@ -2149,5 +2324,8 @@ TEST_LIST = { { "issue_5504" , test_issue_5504}, { "issue_10576" , test_issue_10576}, { "issue_truncation_10576", test_issue_truncation_10576 }, + { "issue_6703" , test_issue_6703}, + { "issue_6703_disabled", test_issue_6703_disabled}, + { "issue_6703_all_empty", test_issue_6703_all_empty}, { 0 } };