diff --git a/plugins/out_gcs/gcs.c b/plugins/out_gcs/gcs.c index e9062359d13..6f3eac1b62d 100644 --- a/plugins/out_gcs/gcs.c +++ b/plugins/out_gcs/gcs.c @@ -593,6 +593,110 @@ int gcs_jwt_encode(struct flb_gcs *ctx, char *payload, char *secret, return -1; } +static int gcs_fetch_metadata_token(struct flb_gcs *ctx, flb_sds_t *payload) +{ + int ret; + int result; + size_t bytes_sent; + const char *test_response; + flb_sds_t tmp; + struct flb_connection *connection; + struct flb_http_client *client; + + if (gcs_under_test_mode() == FLB_TRUE) { + test_response = getenv("TEST_GCS_METADATA_RESPONSE"); + if (!test_response) { + return -1; + } + + tmp = flb_sds_copy(*payload, test_response, strlen(test_response)); + if (!tmp) { + return -1; + } + *payload = tmp; + mock_gcs_call_increment_counter("MetadataToken"); + gcs_setenv("TEST_GCS_LAST_METADATA_URI", FLB_GCS_METADATA_TOKEN_URI); + return 0; + } + + connection = flb_upstream_conn_get(ctx->metadata_u); + if (!connection) { + flb_plg_error(ctx->ins, + "failed to connect to metadata server at '%s'; " + "provide google_service_credentials when not running on GCE/GKE", + ctx->metadata_server); + return -1; + } + + client = flb_http_client(connection, FLB_HTTP_GET, + FLB_GCS_METADATA_TOKEN_URI, + "", 0, NULL, 0, NULL, 0); + if (!client) { + flb_upstream_conn_release(connection); + return -1; + } + + flb_http_buffer_size(client, FLB_GCS_METADATA_TOKEN_SIZE_MAX); + flb_http_add_header(client, "User-Agent", 10, "Fluent-Bit", 10); + flb_http_add_header(client, "Metadata-Flavor", 15, "Google", 6); + + ret = flb_http_do(client, &bytes_sent); + if (ret != 0) { + flb_plg_warn(ctx->ins, "metadata token request failed: http_do=%i", ret); + result = -1; + } + else if (client->resp.status == 200) { + tmp = flb_sds_copy(*payload, client->resp.payload, + client->resp.payload_size); + if (tmp) { + *payload = tmp; + result = 0; + } + else { + result = -1; + } + } + else { + flb_plg_warn(ctx->ins, + "metadata token request failed with status=%i response='%.*s'", + client->resp.status, + (int) client->resp.payload_size, + client->resp.payload ? client->resp.payload : ""); + result = -1; + } + + flb_http_client_destroy(client); + flb_upstream_conn_release(connection); + + return result; +} + +static int gcs_get_metadata_token(struct flb_gcs *ctx) +{ + int ret; + flb_sds_t payload; + + payload = flb_sds_create_size(FLB_GCS_METADATA_TOKEN_SIZE_MAX); + if (!payload) { + return -1; + } + + ret = gcs_fetch_metadata_token(ctx, &payload); + if (ret == 0) { + ret = flb_oauth2_parse_json_response(payload, flb_sds_len(payload), + ctx->o); + } + flb_sds_destroy(payload); + + if (ret != 0) { + flb_plg_error(ctx->ins, "could not retrieve a metadata server token"); + return -1; + } + + ctx->o->expires_at = time(NULL) + ctx->o->expires_in; + return 0; +} + static int gcs_get_oauth2_token(struct flb_gcs *ctx) { int ret; @@ -603,6 +707,10 @@ static int gcs_get_oauth2_token(struct flb_gcs *ctx) char payload[1024]; flb_oauth2_payload_clear(ctx->o); + if (ctx->metadata_server_auth == FLB_TRUE) { + return gcs_get_metadata_token(ctx); + } + issued = time(NULL); expires = issued + FLB_GCS_TOKEN_REFRESH; snprintf(payload, sizeof(payload) - 1, @@ -865,7 +973,8 @@ static int upload_data(struct flb_gcs *ctx, size_t upload_size; char random_hex[9]; - if (gcs_under_test_mode() == FLB_TRUE) { + if (gcs_under_test_mode() == FLB_TRUE && + ctx->metadata_server_auth == FLB_FALSE) { auth = flb_sds_create("Bearer test-token"); } else { @@ -1220,6 +1329,7 @@ static int cb_gcs_init(struct flb_output_instance *ins, struct flb_config *confi int ret; struct flb_gcs *ctx; const char *tmp; + const char *legacy_credentials; (void) data; ctx = flb_calloc(1, sizeof(*ctx)); @@ -1275,7 +1385,16 @@ static int cb_gcs_init(struct flb_output_instance *ins, struct flb_config *confi goto error; } - tmp = getenv("GOOGLE_SERVICE_CREDENTIALS"); + tmp = getenv("GOOGLE_APPLICATION_CREDENTIALS"); + legacy_credentials = getenv("GOOGLE_SERVICE_CREDENTIALS"); + if (!ctx->credentials_file && tmp && legacy_credentials) { + flb_plg_warn(ins, "GOOGLE_APPLICATION_CREDENTIALS and " + "GOOGLE_SERVICE_CREDENTIALS are both set; using " + "GOOGLE_APPLICATION_CREDENTIALS"); + } + if (!ctx->credentials_file && !tmp) { + tmp = legacy_credentials; + } if (!ctx->credentials_file && tmp) { ctx->credentials_file = flb_sds_create(tmp); if (!ctx->credentials_file) { @@ -1284,16 +1403,21 @@ static int cb_gcs_init(struct flb_output_instance *ins, struct flb_config *confi ctx->credentials_file_owned = FLB_TRUE; } - ctx->oauth_credentials = flb_calloc(1, sizeof(struct flb_gcs_oauth_credentials)); - if (!ctx->oauth_credentials) { - flb_errno(); - goto error; - } + if (ctx->credentials_file) { + ctx->oauth_credentials = flb_calloc(1, sizeof(struct flb_gcs_oauth_credentials)); + if (!ctx->oauth_credentials) { + flb_errno(); + goto error; + } - if (!ctx->credentials_file || - flb_gcs_read_credentials_file(ctx, ctx->credentials_file, ctx->oauth_credentials) == -1) { - flb_errno(); - goto error; + if (flb_gcs_read_credentials_file(ctx, ctx->credentials_file, + ctx->oauth_credentials) == -1) { + goto error; + } + } + else { + ctx->metadata_server_auth = FLB_TRUE; + flb_plg_info(ins, "using GCE/GKE metadata server authentication"); } ctx->o = flb_oauth2_create(config, FLB_GCS_AUTH_URL, FLB_GCS_TOKEN_REFRESH); @@ -1311,6 +1435,15 @@ static int cb_gcs_init(struct flb_output_instance *ins, struct flb_config *confi if (!ctx->u) { goto error; } + if (ctx->metadata_server_auth == FLB_TRUE) { + ctx->metadata_u = flb_upstream_create_url(config, ctx->metadata_server, + FLB_IO_TCP, NULL); + if (!ctx->metadata_u) { + flb_plg_error(ins, "metadata upstream creation failed"); + goto error; + } + flb_stream_disable_async_mode(&ctx->metadata_u->base); + } ctx->out_format = FLB_PACK_JSON_FORMAT_LINES; ctx->json_date_format = FLB_PACK_JSON_DATE_DOUBLE; if (ctx->content_type == NULL) { @@ -1432,6 +1565,10 @@ static int gcs_ctx_destroy(void *data, struct flb_config *config) flb_upstream_destroy(ctx->u); } + if (ctx->metadata_u) { + flb_upstream_destroy(ctx->metadata_u); + } + if (ctx->o) { flb_oauth2_destroy(ctx->o); } @@ -1522,6 +1659,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct flb_gcs, credentials_file), "Service account JSON file." }, + { + FLB_CONFIG_MAP_STR, "metadata_server", FLB_GCS_METADATA_SERVER, + 0, FLB_TRUE, offsetof(struct flb_gcs, metadata_server), + "GCE/GKE metadata server used when no credentials file is configured." + }, { FLB_CONFIG_MAP_STR, "store_dir", "/tmp/fluent-bit/gcs", 0, FLB_TRUE, offsetof(struct flb_gcs, store_dir), diff --git a/plugins/out_gcs/gcs.h b/plugins/out_gcs/gcs.h index 57f9de845f9..91f9bebfd05 100644 --- a/plugins/out_gcs/gcs.h +++ b/plugins/out_gcs/gcs.h @@ -31,6 +31,10 @@ #define FLB_GCS_SCOPE "https://www.googleapis.com/auth/devstorage.read_write" #define FLB_GCS_AUTH_URL "https://oauth2.googleapis.com/token" #define FLB_GCS_TOKEN_REFRESH 3000 +#define FLB_GCS_METADATA_SERVER "http://metadata.google.internal" +#define FLB_GCS_METADATA_TOKEN_URI \ + "/computeMetadata/v1/instance/service-accounts/default/token" +#define FLB_GCS_METADATA_TOKEN_SIZE_MAX 14336 #define FLB_GCS_COMPRESSION_NONE 0 #define FLB_GCS_COMPRESSION_GZIP 1 @@ -59,14 +63,17 @@ struct flb_gcs { struct flb_output_instance *ins; struct flb_config *config; struct flb_upstream *u; + struct flb_upstream *metadata_u; struct flb_oauth2 *o; pthread_mutex_t token_mutex; int token_mutex_initialized; + int metadata_server_auth; flb_sds_t bucket; flb_sds_t content_type; flb_sds_t credentials_file; int credentials_file_owned; + flb_sds_t metadata_server; flb_sds_t store_dir; flb_sds_t gcs_key_format; flb_sds_t tag_delimiters; diff --git a/tests/runtime/out_gcs.c b/tests/runtime/out_gcs.c index f58a25bf262..3b33f60d70f 100644 --- a/tests/runtime/out_gcs.c +++ b/tests/runtime/out_gcs.c @@ -248,6 +248,181 @@ void flb_test_gcs_accepts_extra_credential_fields(void) flb_free(store_dir); } +void flb_test_gcs_application_default_credentials_env(void) +{ + int ret; + int in_ffd; + int out_ffd; + char *store_dir; + flb_ctx_t *ctx; + + store_dir = create_test_store_directory("/flb-gcs-test-adc-env-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (!store_dir) { + return; + } + + setenv("GOOGLE_APPLICATION_CREDENTIALS", SERVICE_CREDENTIALS, 1); + setenv("GOOGLE_SERVICE_CREDENTIALS", "/does/not/exist", 1); + + ctx = flb_create(); + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "gcs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret == 0) { + flb_stop(ctx); + } + flb_destroy(ctx); + + unsetenv("GOOGLE_APPLICATION_CREDENTIALS"); + unsetenv("GOOGLE_SERVICE_CREDENTIALS"); + flb_free(store_dir); +} + +void flb_test_gcs_metadata_server_authentication(void) +{ + int ret; + int in_ffd; + int out_ffd; + int metadata_calls; + int upload_calls; + char *store_dir; + char *value; + flb_ctx_t *ctx; + + store_dir = create_test_store_directory("/flb-gcs-test-metadata-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (!store_dir) { + return; + } + + unsetenv("GOOGLE_APPLICATION_CREDENTIALS"); + unsetenv("GOOGLE_SERVICE_CREDENTIALS"); + unsetenv("TEST_GCS_MetadataToken_CALL_COUNT"); + unsetenv("TEST_GCS_UploadObject_CALL_COUNT"); + setenv("FLB_GCS_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_GCS_METADATA_RESPONSE", + "{\"access_token\":\"metadata-token\",\"expires_in\":3600," + "\"token_type\":\"Bearer\"}", 1); + + ctx = flb_create(); + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "gcs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd, "upload_timeout", "1s", NULL); + flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret == 0) { + flb_lib_push(ctx, in_ffd, (char *) JSON_TD, (int) sizeof(JSON_TD) - 1); + sleep(3); + flb_stop(ctx); + } + flb_destroy(ctx); + + value = getenv("TEST_GCS_MetadataToken_CALL_COUNT"); + metadata_calls = value ? atoi(value) : 0; + TEST_CHECK_(metadata_calls == 1, + "Expected 1 metadata token call, got %d", metadata_calls); + value = getenv("TEST_GCS_UploadObject_CALL_COUNT"); + upload_calls = value ? atoi(value) : 0; + TEST_CHECK_(upload_calls == 1, + "Expected 1 UploadObject call, got %d", upload_calls); + value = getenv("TEST_GCS_LAST_METADATA_URI"); + TEST_CHECK_(value != NULL, "Expected the metadata URI to be captured"); + if (value) { + TEST_CHECK(strcmp(value, FLB_GCS_METADATA_TOKEN_URI) == 0); + } + + unsetenv("FLB_GCS_PLUGIN_UNDER_TEST"); + unsetenv("TEST_GCS_METADATA_RESPONSE"); + unsetenv("TEST_GCS_MetadataToken_CALL_COUNT"); + unsetenv("TEST_GCS_UploadObject_CALL_COUNT"); + unsetenv("TEST_GCS_LAST_METADATA_URI"); + unsetenv("TEST_GCS_LAST_URI"); + unsetenv("TEST_GCS_LAST_BODY_GZIP"); + flb_free(store_dir); +} + +void flb_test_gcs_rejects_invalid_metadata_response(void) +{ + int ret; + int in_ffd; + int out_ffd; + int metadata_calls; + int upload_calls; + char *store_dir; + char *value; + flb_ctx_t *ctx; + + store_dir = create_test_store_directory("/flb-gcs-test-metadata-invalid-XXXXXX"); + TEST_CHECK(store_dir != NULL); + if (!store_dir) { + return; + } + + unsetenv("GOOGLE_APPLICATION_CREDENTIALS"); + unsetenv("GOOGLE_SERVICE_CREDENTIALS"); + unsetenv("TEST_GCS_MetadataToken_CALL_COUNT"); + unsetenv("TEST_GCS_UploadObject_CALL_COUNT"); + setenv("FLB_GCS_PLUGIN_UNDER_TEST", "true", 1); + setenv("TEST_GCS_METADATA_RESPONSE", "{\"invalid\":true}", 1); + + ctx = flb_create(); + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "gcs", NULL); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, "match", "*", NULL); + flb_output_set(ctx, out_ffd, "bucket", "fluent", NULL); + flb_output_set(ctx, out_ffd, "upload_timeout", "1s", NULL); + flb_output_set(ctx, out_ffd, "store_dir", store_dir, NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + if (ret == 0) { + flb_lib_push(ctx, in_ffd, (char *) JSON_TD, (int) sizeof(JSON_TD) - 1); + sleep(3); + flb_stop(ctx); + } + flb_destroy(ctx); + + value = getenv("TEST_GCS_MetadataToken_CALL_COUNT"); + metadata_calls = value ? atoi(value) : 0; + TEST_CHECK_(metadata_calls >= 1, + "Expected at least 1 metadata token call, got %d", metadata_calls); + value = getenv("TEST_GCS_UploadObject_CALL_COUNT"); + upload_calls = value ? atoi(value) : 0; + TEST_CHECK_(upload_calls == 0, + "Expected no UploadObject calls, got %d", upload_calls); + + unsetenv("FLB_GCS_PLUGIN_UNDER_TEST"); + unsetenv("TEST_GCS_METADATA_RESPONSE"); + unsetenv("TEST_GCS_MetadataToken_CALL_COUNT"); + unsetenv("TEST_GCS_UploadObject_CALL_COUNT"); + unsetenv("TEST_GCS_LAST_METADATA_URI"); + unsetenv("TEST_GCS_LAST_URI"); + unsetenv("TEST_GCS_LAST_BODY_GZIP"); + flb_free(store_dir); +} + void flb_test_gcs_upload_error(void) { int ret; @@ -397,6 +572,9 @@ TEST_LIST = { {"rejects_invalid_configuration", flb_test_gcs_rejects_invalid_configuration}, {"rejects_invalid_compression", flb_test_gcs_rejects_invalid_compression}, {"accepts_extra_credential_fields", flb_test_gcs_accepts_extra_credential_fields}, + {"application_default_credentials_env", flb_test_gcs_application_default_credentials_env}, + {"metadata_server_authentication", flb_test_gcs_metadata_server_authentication}, + {"rejects_invalid_metadata_response", flb_test_gcs_rejects_invalid_metadata_response}, {"upload_error", flb_test_gcs_upload_error}, {"shutdown_preserves_pending_upload", flb_test_gcs_shutdown_preserves_pending_upload}, {NULL, NULL}