diff --git a/CMakeLists.txt b/CMakeLists.txt index 3d38b2569f7..0ce53b64675 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -388,6 +388,7 @@ if(FLB_ALL) set(FLB_OUT_LIB 1) set(FLB_OUT_FLOWCOUNTER 1) set(FLB_OUT_WEBSOCKET 1) + set(FLB_OUT_ARVANCLOUD_CLOUDLOGS 1) endif() if(FLB_DEV) diff --git a/cmake/plugins_options.cmake b/cmake/plugins_options.cmake index aa8bfd05bda..9aad810532c 100644 --- a/cmake/plugins_options.cmake +++ b/cmake/plugins_options.cmake @@ -156,3 +156,4 @@ DEFINE_OPTION(FLB_OUT_TCP "Enable TCP output plugin" DEFINE_OPTION(FLB_OUT_UDP "Enable UDP output plugin" ON) DEFINE_OPTION(FLB_OUT_VIVO_EXPORTER "Enable Vivo exporter output plugin" ON) DEFINE_OPTION(FLB_OUT_WEBSOCKET "Enable Websocket output plugin" ON) +DEFINE_OPTION(FLB_OUT_ARVANCLOUD_CLOUDLOGS "Enable ArvanCloud CloudLogs output plugin" ON) diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index c885f51fda3..ce7271878fe 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -414,6 +414,7 @@ REGISTER_OUT_PLUGIN("out_prometheus_remote_write") REGISTER_OUT_PLUGIN("out_s3") REGISTER_OUT_PLUGIN("out_vivo_exporter") REGISTER_OUT_PLUGIN("out_chronicle") +REGISTER_OUT_PLUGIN("out_arvancloud_cloudlogs") if(FLB_ZIG) REGISTER_OUT_PLUGIN("out_zig_demo" "zig") diff --git a/plugins/out_arvancloud_cloudlogs/CMakeLists.txt b/plugins/out_arvancloud_cloudlogs/CMakeLists.txt new file mode 100644 index 00000000000..9c067e7758f --- /dev/null +++ b/plugins/out_arvancloud_cloudlogs/CMakeLists.txt @@ -0,0 +1,5 @@ +set(src + arvancloud_cloudlogs.c + arvancloud_cloudlogs_conf.c) + +FLB_PLUGIN(out_arvancloud_cloudlogs "${src}" "") \ No newline at end of file diff --git a/plugins/out_arvancloud_cloudlogs/arvancloud_cloudlogs.c b/plugins/out_arvancloud_cloudlogs/arvancloud_cloudlogs.c new file mode 100644 index 00000000000..3945abd86f7 --- /dev/null +++ b/plugins/out_arvancloud_cloudlogs/arvancloud_cloudlogs.c @@ -0,0 +1,500 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arvancloud_cloudlogs.h" + +/* + * Format event time to RFC3339 with microseconds in UTC. + * String parsing belongs in a parser (Time_Key / Time_Format / %L / %z); + * this output only shapes the API date-time field. + */ +static size_t format_timestamp_rfc3339(char *buffer, size_t buffer_size, + time_t seconds, long nsec) +{ + struct tm tm; + size_t s; + int len; + + gmtime_r(&seconds, &tm); + s = strftime(buffer, buffer_size - 1, "%Y-%m-%dT%H:%M:%S", &tm); + len = snprintf(buffer + s, buffer_size - 1 - s, + ".%06ldZ", (long) (nsec / 1000)); + return s + len; +} + +static int arvancloud_format(struct flb_config *config, + struct flb_input_instance *ins, + void *plugin_context, + void *flush_context, + int event_type, + const char *tag, int tag_len, + const void *data, size_t bytes, + void **out_data, size_t *out_size) +{ + int ret; + size_t final_log_type_len; + struct flb_log_event_decoder log_decoder; + struct flb_log_event log_event; + struct flb_out_arvancloud_cloudlogs *ctx; + struct flb_event_chunk *event_chunk; + msgpack_sbuffer mp_sbuf; + msgpack_packer mp_pck; + size_t count; + size_t s; + struct flb_time t; + char time_formatted[64]; + flb_sds_t log_type_value; + const char *final_log_type; + + ctx = plugin_context; + event_chunk = flush_context; + count = 0; + + (void) ins; + (void) event_type; + + /* Get event count from event_chunk if available, otherwise count manually */ + if (event_chunk != NULL) { + count = event_chunk->total_events; + } + else { + count = flb_mp_count(data, bytes); + } + + ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); + if (ret != FLB_EVENT_DECODER_SUCCESS) { + flb_plg_error(ctx->ins, "log event decoder init error: %d", ret); + return -1; + } + + msgpack_sbuffer_init(&mp_sbuf); + msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); + + /* Build request object: { logs: [ ... ] } */ + msgpack_pack_map(&mp_pck, 1); + + /* logs */ + msgpack_pack_str(&mp_pck, 4); + msgpack_pack_str_body(&mp_pck, "logs", 4); + + msgpack_pack_array(&mp_pck, count); + + while ((ret = flb_log_event_decoder_next( + &log_decoder, &log_event)) == FLB_EVENT_DECODER_SUCCESS) { + t = log_event.timestamp; + log_type_value = NULL; + + msgpack_pack_map(&mp_pck, ctx->include_tag_key ? 6 : 5); + + /* logType: priority order is log_type_key > log_type */ + if (ctx->log_type_key && ctx->ra_log_type_key) { + /* Try to extract from record using log_type_key */ + log_type_value = flb_ra_translate(ctx->ra_log_type_key, + (char *) tag, tag_len, + *log_event.body, NULL); + if (log_type_value && !flb_sds_is_empty(log_type_value)) { + final_log_type = log_type_value; + final_log_type_len = flb_sds_len(log_type_value); + } + else { + /* Clean up failed extraction and fall back to log_type */ + if (log_type_value) { + flb_sds_destroy(log_type_value); + log_type_value = NULL; + } + final_log_type = ctx->log_type; + final_log_type_len = flb_sds_len(ctx->log_type); + } + } + else { + /* Use configured log_type (has default value) */ + final_log_type = ctx->log_type; + final_log_type_len = flb_sds_len(ctx->log_type); + } + + msgpack_pack_str(&mp_pck, 7); + msgpack_pack_str_body(&mp_pck, "logType", 7); + msgpack_pack_str(&mp_pck, final_log_type_len); + msgpack_pack_str_body(&mp_pck, final_log_type, final_log_type_len); + + /* Clean up if we allocated log_type_value */ + if (log_type_value) { + flb_sds_destroy(log_type_value); + } + + /* + * timestamp: prefer Fluent Bit event time (set upstream via parser + * Time_Key / Time_Format). Optional timestamp_key is pass-through + * only when the field is already an OpenAPI date-time string. + */ + if (ctx->timestamp_key && ctx->ra_timestamp_key) { + flb_sds_t timestamp_value; + int use_record_value; + + use_record_value = FLB_FALSE; + timestamp_value = flb_ra_translate(ctx->ra_timestamp_key, + (char *) tag, tag_len, + *log_event.body, NULL); + + if (timestamp_value && !flb_sds_is_empty(timestamp_value)) { + s = flb_sds_len(timestamp_value); + if (s >= sizeof(time_formatted)) { + s = sizeof(time_formatted) - 1; + } + + memcpy(time_formatted, timestamp_value, s); + time_formatted[s] = '\0'; + use_record_value = FLB_TRUE; + } + + if (timestamp_value) { + flb_sds_destroy(timestamp_value); + } + + if (!use_record_value) { + s = format_timestamp_rfc3339(time_formatted, + sizeof(time_formatted), + t.tm.tv_sec, t.tm.tv_nsec); + } + } + else { + s = format_timestamp_rfc3339(time_formatted, sizeof(time_formatted), + t.tm.tv_sec, t.tm.tv_nsec); + } + + msgpack_pack_str(&mp_pck, 9); + msgpack_pack_str_body(&mp_pck, "timestamp", 9); + msgpack_pack_str(&mp_pck, s); + msgpack_pack_str_body(&mp_pck, time_formatted, s); + + msgpack_pack_str(&mp_pck, 8); + msgpack_pack_str_body(&mp_pck, "severity", 8); + msgpack_pack_str(&mp_pck, 4); + msgpack_pack_str_body(&mp_pck, "INFO", 4); + + /* resource: minimal object */ + msgpack_pack_str(&mp_pck, 8); + msgpack_pack_str_body(&mp_pck, "resource", 8); + msgpack_pack_map(&mp_pck, 1); + msgpack_pack_str(&mp_pck, 4); + msgpack_pack_str_body(&mp_pck, "type", 4); + msgpack_pack_str(&mp_pck, 7); + msgpack_pack_str_body(&mp_pck, "general", 7); + + /* payload: original record map */ + msgpack_pack_str(&mp_pck, 7); + msgpack_pack_str_body(&mp_pck, "payload", 7); + msgpack_pack_object(&mp_pck, *log_event.body); + + /* Include tag key within the log object if configured */ + if (ctx->include_tag_key) { + msgpack_pack_str(&mp_pck, flb_sds_len(ctx->tag_key)); + msgpack_pack_str_body(&mp_pck, ctx->tag_key, flb_sds_len(ctx->tag_key)); + msgpack_pack_str(&mp_pck, tag_len); + msgpack_pack_str_body(&mp_pck, tag, tag_len); + } + } + + flb_log_event_decoder_destroy(&log_decoder); + + *out_data = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size, + config->json_escape_unicode); + msgpack_sbuffer_destroy(&mp_sbuf); + if (!*out_data) { + return -1; + } + *out_size = flb_sds_len((flb_sds_t) *out_data); + return 0; +} + +static void cb_arvancloud_flush(struct flb_event_chunk *event_chunk, + struct flb_output_flush *out_flush, + struct flb_input_instance *i_ins, + void *out_context, + struct flb_config *config) +{ + int compressed; + int ret; + size_t b_sent; + size_t payload_size; + size_t final_payload_size; + size_t prefix_len; + void *payload_buf; + void *final_payload; + const char *prefix; + flb_sds_t header; + struct flb_out_arvancloud_cloudlogs *ctx; + struct flb_connection *u_conn; + struct flb_http_client *c; + + compressed = FLB_FALSE; + payload_size = 0; + final_payload_size = 0; + payload_buf = NULL; + final_payload = NULL; + ctx = out_context; + + (void) out_flush; + (void) i_ins; + + u_conn = flb_upstream_conn_get(ctx->u); + if (!u_conn) { + FLB_OUTPUT_RETURN(FLB_RETRY); + } + + ret = arvancloud_format(config, i_ins, ctx, event_chunk, + event_chunk->type, + event_chunk->tag, flb_sds_len(event_chunk->tag), + event_chunk->data, event_chunk->size, + &payload_buf, &payload_size); + if (ret == -1) { + flb_upstream_conn_release(u_conn); + FLB_OUTPUT_RETURN(FLB_ERROR); + } + + final_payload = payload_buf; + final_payload_size = payload_size; + if (ctx->compress_gzip == FLB_TRUE) { + ret = flb_gzip_compress((void *) payload_buf, payload_size, + &final_payload, &final_payload_size); + if (ret == 0) { + compressed = FLB_TRUE; + } + else { + flb_plg_error(ctx->ins, "cannot gzip payload, disabling compression"); + final_payload = payload_buf; + final_payload_size = payload_size; + } + } + + /* Debug: dump request JSON payload */ + flb_plg_debug(ctx->ins, "request payload (%zu bytes): %.*s", + final_payload_size, + (int) final_payload_size, + (const char *) final_payload); + + c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri, + final_payload, final_payload_size, + NULL, 0, + NULL, 0); + if (!c) { + if (final_payload != payload_buf) { + flb_free(final_payload); + } + flb_sds_destroy((flb_sds_t) payload_buf); + flb_upstream_conn_release(u_conn); + FLB_OUTPUT_RETURN(FLB_RETRY); + } + + /* Add standard headers */ + flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10); + flb_http_add_header(c, "Content-Type", 12, "application/json", 16); + + /* Content Encoding: gzip */ + if (compressed == FLB_TRUE) { + flb_http_set_content_encoding_gzip(c); + } + + /* API Key Authorization header: "apikey " */ + if (ctx->api_key) { + prefix = "apikey "; + prefix_len = 7; + header = flb_sds_create_size(prefix_len + flb_sds_len(ctx->api_key)); + if (header) { + header = flb_sds_cat(header, prefix, prefix_len); + header = flb_sds_cat(header, ctx->api_key, + flb_sds_len(ctx->api_key)); + flb_http_add_header(c, "Authorization", 13, header, + flb_sds_len(header)); + flb_sds_destroy(header); + } + } + + ret = flb_http_do(c, &b_sent); + if (ret != 0) { + flb_plg_warn(ctx->ins, "http_do=%i", ret); + ret = FLB_RETRY; + } + else { + /* Handle HTTP status codes */ + if (c->resp.status >= 200 && c->resp.status <= 205) { + /* Success: 200 OK, 201 Created, 202 Accepted, 203-205 */ + flb_plg_debug(ctx->ins, "HTTP status=%i", c->resp.status); + if (c->resp.payload && c->resp.payload_size > 0) { + flb_plg_debug(ctx->ins, "response body (%zu bytes): %.*s", + c->resp.payload_size, + (int) c->resp.payload_size, + (const char *) c->resp.payload); + } + ret = FLB_OK; + } + else if (c->resp.status == 400) { + /* Bad Request - usually a client error, don't retry */ + flb_plg_error(ctx->ins, "HTTP status=400 (Bad Request)"); + if (c->resp.payload && c->resp.payload_size > 0) { + flb_plg_error(ctx->ins, "response body: %.*s", + (int) c->resp.payload_size, + (const char *) c->resp.payload); + } + ret = FLB_ERROR; + } + else if (c->resp.status == 401 || c->resp.status == 403) { + /* Unauthorized or Forbidden - auth issue, don't retry */ + flb_plg_error(ctx->ins, + "HTTP status=%i (Authentication/Authorization failed)", + c->resp.status); + if (c->resp.payload && c->resp.payload_size > 0) { + flb_plg_error(ctx->ins, "response body: %.*s", + (int) c->resp.payload_size, + (const char *) c->resp.payload); + } + ret = FLB_ERROR; + } + else if (c->resp.status == 429) { + /* Too Many Requests - rate limit, retry */ + flb_plg_warn(ctx->ins, + "HTTP status=429 (Rate Limited), will retry"); + ret = FLB_RETRY; + } + else if (c->resp.status >= 500) { + /* Server errors - retry */ + flb_plg_warn(ctx->ins, + "HTTP status=%i (Server Error), will retry", + c->resp.status); + if (c->resp.payload && c->resp.payload_size > 0) { + flb_plg_warn(ctx->ins, "response body: %.*s", + (int) c->resp.payload_size, + (const char *) c->resp.payload); + } + ret = FLB_RETRY; + } + else { + /* Other client errors - don't retry */ + flb_plg_error(ctx->ins, "HTTP status=%i (Client Error)", c->resp.status); + if (c->resp.payload && c->resp.payload_size > 0) { + flb_plg_error(ctx->ins, "response body: %.*s", + (int) c->resp.payload_size, + (const char *) c->resp.payload); + } + ret = FLB_ERROR; + } + } + + if (final_payload != payload_buf) { + flb_free(final_payload); + } + flb_sds_destroy((flb_sds_t) payload_buf); + flb_http_client_destroy(c); + flb_upstream_conn_release(u_conn); + + FLB_OUTPUT_RETURN(ret); +} + +static int cb_arvancloud_init(struct flb_output_instance *ins, + struct flb_config *config, + void *data) +{ + struct flb_out_arvancloud_cloudlogs *ctx; + (void) data; + + ctx = flb_arvancloud_conf_create(ins, config); + if (!ctx) { + return -1; + } + flb_output_set_context(ins, ctx); + return 0; +} + +static int cb_arvancloud_exit(void *data, struct flb_config *config) +{ + (void) config; + return flb_arvancloud_conf_destroy(data); +} + +/* Configuration properties map */ +static struct flb_config_map config_map[] = { + { + FLB_CONFIG_MAP_STR, "apikey", NULL, + 0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, api_key), + "API key for authorization (will be sent as 'apikey ')" + }, + { + FLB_CONFIG_MAP_STR, "log_type", FLB_ARVANCLOUD_LOG_TYPE, + 0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, log_type), + "Log type value to use. Defaults to 'fluentbit'." + }, + { + FLB_CONFIG_MAP_STR, "log_type_key", NULL, + 0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, log_type_key), + "Field in the record to use as log type. " + "Takes priority over 'log_type' and tag prefix." + }, + { + FLB_CONFIG_MAP_BOOL, "gzip", "false", + 0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, compress_gzip), + "Enable gzip compression" + }, + { + FLB_CONFIG_MAP_BOOL, "include_tag_key", "false", + 0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, include_tag_key), + "Include original tag in each record" + }, + { + FLB_CONFIG_MAP_STR, "tag_key", "tag", + 0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, tag_key), + "Tag key name when include_tag_key=true" + }, + { + FLB_CONFIG_MAP_STR, "timestamp_key", NULL, + 0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, timestamp_key), + "Optional record field to forward as CloudLogs timestamp when the " + "value is already an OpenAPI date-time string. Prefer setting event " + "time with a parser (Time_Key / Time_Format). If unset or missing, " + "uses the Fluent Bit event timestamp formatted as RFC3339 UTC." + }, + /* EOF */ + {0} +}; + +struct flb_output_plugin out_arvancloud_cloudlogs_plugin = { + .name = "arvancloud_cloudlogs", + .description = "Send events to ArvanCloud CloudLogs", + .cb_init = cb_arvancloud_init, + .cb_flush = cb_arvancloud_flush, + .cb_exit = cb_arvancloud_exit, + .test_formatter.callback = arvancloud_format, + .config_map = config_map, + .flags = FLB_OUTPUT_NET | FLB_IO_TLS, +}; + + diff --git a/plugins/out_arvancloud_cloudlogs/arvancloud_cloudlogs.h b/plugins/out_arvancloud_cloudlogs/arvancloud_cloudlogs.h new file mode 100644 index 00000000000..b7ee5e1d4c1 --- /dev/null +++ b/plugins/out_arvancloud_cloudlogs/arvancloud_cloudlogs.h @@ -0,0 +1,60 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_OUT_ARVANCLOUD_CLOUDLOGS_H +#define FLB_OUT_ARVANCLOUD_CLOUDLOGS_H + +#define FLB_ARVANCLOUD_LOG_TYPE "fluentbit" + +#include +#include +#include +#include + +struct flb_out_arvancloud_cloudlogs { + /* Network */ + struct flb_upstream *u; + flb_sds_t host; + int port; + flb_sds_t uri; + flb_sds_t scheme; + + /* Config */ + flb_sds_t api_key; + int compress_gzip; + int include_tag_key; + flb_sds_t tag_key; + flb_sds_t log_type; + flb_sds_t log_type_key; + struct flb_record_accessor *ra_log_type_key; + flb_sds_t timestamp_key; + struct flb_record_accessor *ra_timestamp_key; + + /* Instance */ + struct flb_output_instance *ins; +}; + +struct flb_out_arvancloud_cloudlogs *flb_arvancloud_conf_create( + struct flb_output_instance *ins, + struct flb_config *config); +int flb_arvancloud_conf_destroy(struct flb_out_arvancloud_cloudlogs *ctx); + +#endif + + diff --git a/plugins/out_arvancloud_cloudlogs/arvancloud_cloudlogs_conf.c b/plugins/out_arvancloud_cloudlogs/arvancloud_cloudlogs_conf.c new file mode 100644 index 00000000000..aa63b22ea9a --- /dev/null +++ b/plugins/out_arvancloud_cloudlogs/arvancloud_cloudlogs_conf.c @@ -0,0 +1,151 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "arvancloud_cloudlogs.h" + +struct flb_out_arvancloud_cloudlogs *flb_arvancloud_conf_create( + struct flb_output_instance *ins, + struct flb_config *config) +{ + int io_flags; + struct flb_upstream *upstream; + struct flb_out_arvancloud_cloudlogs *ctx; + + io_flags = 0; + + ctx = flb_calloc(1, sizeof(struct flb_out_arvancloud_cloudlogs)); + if (!ctx) { + flb_errno(); + return NULL; + } + ctx->ins = ins; + + if (flb_output_config_map_set(ins, (void *) ctx) == -1) { + flb_plg_error(ins, "flb_output_config_map_set failed"); + flb_arvancloud_conf_destroy(ctx); + return NULL; + } + + + /* Force HTTPS */ + io_flags = FLB_IO_TLS; + ctx->scheme = flb_sds_create("https://"); + if (!ctx->scheme) { + flb_errno(); + flb_arvancloud_conf_destroy(ctx); + return NULL; + } + + if (!ctx->api_key) { + flb_plg_error(ins, "missing required 'apikey'"); + flb_arvancloud_conf_destroy(ctx); + return NULL; + } + + /* Initialize record accessor for log_type_key if configured */ + if (ctx->log_type_key) { + ctx->ra_log_type_key = flb_ra_create(ctx->log_type_key, FLB_TRUE); + if (!ctx->ra_log_type_key) { + flb_plg_error(ins, "invalid log_type_key pattern '%s'", + ctx->log_type_key); + flb_arvancloud_conf_destroy(ctx); + return NULL; + } + } + + /* Initialize record accessor for timestamp_key if configured */ + if (ctx->timestamp_key) { + ctx->ra_timestamp_key = flb_ra_create(ctx->timestamp_key, FLB_TRUE); + if (!ctx->ra_timestamp_key) { + flb_plg_error(ins, "invalid timestamp_key pattern '%s'", + ctx->timestamp_key); + flb_arvancloud_conf_destroy(ctx); + return NULL; + } + } + + /* Hardcode host */ + ctx->host = flb_sds_create("napi.arvancloud.ir"); + if (!ctx->host) { + flb_errno(); + flb_arvancloud_conf_destroy(ctx); + return NULL; + } + + /* Hardcode port */ + ctx->port = 443; + + /* Hardcode uri */ + if (ctx->uri) { + flb_sds_destroy(ctx->uri); + } + ctx->uri = flb_sds_create("/logging/v1/entries/write"); + if (!ctx->uri) { + flb_errno(); + flb_arvancloud_conf_destroy(ctx); + return NULL; + } + + upstream = flb_upstream_create(config, ctx->host, ctx->port, + io_flags, ins->tls); + if (!upstream) { + flb_plg_error(ins, "cannot create upstream context"); + flb_arvancloud_conf_destroy(ctx); + return NULL; + } + + ctx->u = upstream; + flb_output_upstream_set(ctx->u, ins); + + return ctx; +} + +int flb_arvancloud_conf_destroy(struct flb_out_arvancloud_cloudlogs *ctx) +{ + if (!ctx) { + return -1; + } + if (ctx->scheme) { + flb_sds_destroy(ctx->scheme); + } + if (ctx->host) { + flb_sds_destroy(ctx->host); + } + if (ctx->uri) { + flb_sds_destroy(ctx->uri); + } + if (ctx->ra_log_type_key) { + flb_ra_destroy(ctx->ra_log_type_key); + } + if (ctx->ra_timestamp_key) { + flb_ra_destroy(ctx->ra_timestamp_key); + } + if (ctx->u) { + flb_upstream_destroy(ctx->u); + } + flb_free(ctx); + return 0; +} + + diff --git a/tests/runtime/CMakeLists.txt b/tests/runtime/CMakeLists.txt index f7aa58f7e73..2501f30efad 100644 --- a/tests/runtime/CMakeLists.txt +++ b/tests/runtime/CMakeLists.txt @@ -248,6 +248,7 @@ if(FLB_IN_LIB) FLB_RT_TEST(FLB_OUT_COUNTER "out_counter.c") FLB_RT_TEST(FLB_OUT_AZURE_BLOB "out_azure_blob_compression.c") FLB_RT_TEST(FLB_OUT_AZURE_KUSTO "out_azure_kusto.c") + FLB_RT_TEST(FLB_OUT_ARVANCLOUD_CLOUDLOGS "out_arvancloud_cloudlogs.c") FLB_RT_TEST(FLB_OUT_DATADOG "out_datadog.c") FLB_RT_TEST(FLB_OUT_SKYWALKING "out_skywalking.c") FLB_RT_TEST(FLB_OUT_ES "out_elasticsearch.c") diff --git a/tests/runtime/out_arvancloud_cloudlogs.c b/tests/runtime/out_arvancloud_cloudlogs.c new file mode 100644 index 00000000000..38fb33170eb --- /dev/null +++ b/tests/runtime/out_arvancloud_cloudlogs.c @@ -0,0 +1,525 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include "flb_tests_runtime.h" + +/* + * The ArvanCloud CloudLogs output plugin formats events as a JSON object: + * + * { + * "logs": [ + * { + * "logType": "", + * "timestamp": "", + * "severity": "INFO", + * "resource": { "type": "general" }, + * "payload": { ...original record... } + * (optional) "": "" if include_tag_key=true + * }, + * ... + * ] + * } + * + * Tests use the formatter test mode (cb_check_*) to intercept the formatted + * JSON payload before any HTTP request is sent. This means we don't need a + * live endpoint to validate the plugin's behavior. + */ + +#define JSON_BASIC \ + "[1448403340, {\"key\":\"value\",\"foo\":\"bar\"}]" + +#define JSON_WITH_LOG_TYPE \ + "[1448403340, {\"key\":\"value\",\"category\":\"security\"}]" + +#define JSON_WITH_TIMESTAMP \ + "[1448403340, {\"key\":\"value\",\"ts\":\"2024-01-15T10:30:45.123456+03:30\"}]" + +/* Event time 1448403340 == 2015-11-24T22:15:40Z */ +#define EXPECTED_EVENT_TIMESTAMP "2015-11-24T22:15:40.000000Z" + +/* + * Convert a formatted JSON payload into msgpack and validate that the value + * referenced by `key_accessor` equals `val`. + * + * Returns FLB_TRUE on match, FLB_FALSE otherwise. + */ +static int mp_kv_cmp(char *json_data, size_t json_len, + char *key_accessor, char *val) +{ + int ret; + int type; + char *mp_buf = NULL; + size_t mp_size; + size_t off = 0; + msgpack_object map; + msgpack_unpacked result; + struct flb_ra_value *rval = NULL; + struct flb_record_accessor *ra = NULL; + + ret = flb_pack_json((const char *) json_data, json_len, &mp_buf, &mp_size, + &type, NULL); + TEST_CHECK(ret != -1); + + ret = FLB_FALSE; + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, mp_buf, mp_size, &off); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + map = result.data; + + ra = flb_ra_create(key_accessor, FLB_TRUE); + if (!ra) { + flb_error("invalid record accessor key '%s', aborting test", + key_accessor); + goto out; + } + + rval = flb_ra_get_value_object(ra, map); + TEST_CHECK(rval != NULL); + msgpack_unpacked_destroy(&result); + if (!rval) { + goto out; + } + + TEST_CHECK(rval->type == FLB_RA_STRING); + if (rval->type == FLB_RA_STRING && strcmp(rval->val.string, val) == 0) { + ret = FLB_TRUE; + } + + out: + if (rval) { + flb_ra_key_value_destroy(rval); + } + if (ra) { + flb_ra_destroy(ra); + } + if (mp_buf) { + flb_free(mp_buf); + } + return ret; +} + +/* + * Basic shape check: the payload must contain the expected top-level keys + * (`logs` array) and per-record fields (`logType`, `timestamp`, `severity`, + * `resource`, `payload`). + */ +static void cb_check_basic_shape(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + flb_sds_t out_js = res_data; + + /* Top-level "logs" array */ + if (!TEST_CHECK(strstr(out_js, "\"logs\":[") != NULL)) { + TEST_MSG("missing top-level \"logs\" array. Given:%s", out_js); + } + + /* Required per-record fields */ + if (!TEST_CHECK(strstr(out_js, "\"logType\":") != NULL)) { + TEST_MSG("missing logType. Given:%s", out_js); + } + if (!TEST_CHECK(strstr(out_js, "\"timestamp\":") != NULL)) { + TEST_MSG("missing timestamp. Given:%s", out_js); + } + if (!TEST_CHECK(strstr(out_js, "\"severity\":\"INFO\"") != NULL)) { + TEST_MSG("missing severity. Given:%s", out_js); + } + if (!TEST_CHECK(strstr(out_js, "\"resource\":{\"type\":\"general\"}") != NULL)) { + TEST_MSG("missing/wrong resource. Given:%s", out_js); + } + if (!TEST_CHECK(strstr(out_js, "\"payload\":") != NULL)) { + TEST_MSG("missing payload. Given:%s", out_js); + } + + /* Original record fields must be preserved verbatim inside payload */ + if (!TEST_CHECK(strstr(out_js, "\"key\":\"value\"") != NULL)) { + TEST_MSG("original record key not preserved. Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +/* Default log_type is "fluentbit" when not configured. */ +static void cb_check_default_log_type(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + flb_sds_t out_js = res_data; + + /* res_data is a JSON map: { "logs": [ {...} ] } */ + ret = mp_kv_cmp((char *) out_js, res_size, + "$logs[0]['logType']", "fluentbit"); + if (!TEST_CHECK(ret == FLB_TRUE)) { + TEST_MSG("expected default logType=fluentbit. Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +/* When log_type is configured, it must override the default. */ +static void cb_check_configured_log_type(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + flb_sds_t out_js = res_data; + + ret = mp_kv_cmp((char *) out_js, res_size, + "$logs[0]['logType']", "myapp"); + if (!TEST_CHECK(ret == FLB_TRUE)) { + TEST_MSG("expected logType=myapp. Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +/* + * When log_type_key is configured and the referenced field exists in the + * record, it takes priority over the static log_type value. + */ +static void cb_check_log_type_key(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + flb_sds_t out_js = res_data; + + ret = mp_kv_cmp((char *) out_js, res_size, + "$logs[0]['logType']", "security"); + if (!TEST_CHECK(ret == FLB_TRUE)) { + TEST_MSG("expected logType extracted from record. Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +/* + * If log_type_key is configured but the field is missing from the record, + * the plugin must fall back to the configured/default log_type. + */ +static void cb_check_log_type_key_fallback(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + flb_sds_t out_js = res_data; + + ret = mp_kv_cmp((char *) out_js, res_size, + "$logs[0]['logType']", "fallback"); + if (!TEST_CHECK(ret == FLB_TRUE)) { + TEST_MSG("expected fallback logType. Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +/* With include_tag_key=true, the tag must appear under the configured key. */ +static void cb_check_include_tag_key(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + flb_sds_t out_js = res_data; + + if (!TEST_CHECK(strstr(out_js, "\"fluentbit_tag\":\"test\"") != NULL)) { + TEST_MSG("expected tag included with custom key. Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +/* + * Default path (design A): format Fluent Bit event time as RFC3339 UTC. + * String parsing belongs in an upstream parser, not this output. + */ +static void cb_check_event_timestamp(void *ctx, int ffd, + int res_ret, void *res_data, size_t res_size, + void *data) +{ + int ret; + flb_sds_t out_js = res_data; + + ret = mp_kv_cmp((char *) out_js, res_size, + "$logs[0]['timestamp']", + EXPECTED_EVENT_TIMESTAMP); + if (!TEST_CHECK(ret == FLB_TRUE)) { + TEST_MSG("expected event-time RFC3339 UTC. Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +/* + * Optional timestamp_key is pass-through only: the record value is forwarded + * as-is when present (must already be OpenAPI date-time). + */ +static void cb_check_timestamp_passthrough(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + int ret; + flb_sds_t out_js = res_data; + + ret = mp_kv_cmp((char *) out_js, res_size, + "$logs[0]['timestamp']", + "2024-01-15T10:30:45.123456+03:30"); + if (!TEST_CHECK(ret == FLB_TRUE)) { + TEST_MSG("expected timestamp_key pass-through. Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +/* Missing timestamp_key field falls back to event timestamp. */ +static void cb_check_timestamp_key_fallback(void *ctx, int ffd, + int res_ret, void *res_data, + size_t res_size, void *data) +{ + int ret; + flb_sds_t out_js = res_data; + + ret = mp_kv_cmp((char *) out_js, res_size, + "$logs[0]['timestamp']", + EXPECTED_EVENT_TIMESTAMP); + if (!TEST_CHECK(ret == FLB_TRUE)) { + TEST_MSG("expected event-time fallback. Given:%s", out_js); + } + + flb_sds_destroy(out_js); +} + +/* + * Helper: create a fluent-bit context wired to the arvancloud_cloudlogs + * output in formatter test mode. The caller provides any additional + * key/value config pairs (terminated by NULL). + * + * Returns a started context, plus *in_ffd_out for the lib input handle. + */ +static flb_ctx_t *create_ctx(int *in_ffd_out, + void (*cb)(void *, int, int, void *, size_t, void *), + ...) +{ + int ret; + int in_ffd; + int out_ffd; + flb_ctx_t *ctx; + va_list ap; + const char *k; + const char *v; + + ctx = flb_create(); + flb_service_set(ctx, "flush", "1", "grace", "1", + "log_level", "error", NULL); + + in_ffd = flb_input(ctx, (char *) "lib", NULL); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + out_ffd = flb_output(ctx, (char *) "arvancloud_cloudlogs", NULL); + + /* Required: apikey. Match all events tagged "test". */ + flb_output_set(ctx, out_ffd, + "match", "test", + "apikey", "test-api-key", + NULL); + + /* Apply caller-provided extra config pairs. */ + va_start(ap, cb); + while ((k = va_arg(ap, const char *)) != NULL) { + v = va_arg(ap, const char *); + flb_output_set(ctx, out_ffd, k, v, NULL); + } + va_end(ap); + + ret = flb_output_set_test(ctx, out_ffd, "formatter", cb, NULL, NULL); + TEST_CHECK(ret == 0); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + *in_ffd_out = in_ffd; + return ctx; +} + +void flb_test_basic_shape() +{ + int in_ffd; + flb_ctx_t *ctx; + + ctx = create_ctx(&in_ffd, cb_check_basic_shape, NULL); + + flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, sizeof(JSON_BASIC) - 1); + sleep(2); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_default_log_type() +{ + int in_ffd; + flb_ctx_t *ctx; + + ctx = create_ctx(&in_ffd, cb_check_default_log_type, NULL); + + flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, sizeof(JSON_BASIC) - 1); + sleep(2); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_configured_log_type() +{ + int in_ffd; + flb_ctx_t *ctx; + + ctx = create_ctx(&in_ffd, cb_check_configured_log_type, + "log_type", "myapp", + NULL); + + flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, sizeof(JSON_BASIC) - 1); + sleep(2); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_log_type_key() +{ + int in_ffd; + flb_ctx_t *ctx; + + ctx = create_ctx(&in_ffd, cb_check_log_type_key, + "log_type", "should-be-overridden", + "log_type_key", "$category", + NULL); + + flb_lib_push(ctx, in_ffd, + (char *) JSON_WITH_LOG_TYPE, + sizeof(JSON_WITH_LOG_TYPE) - 1); + sleep(2); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_log_type_key_fallback() +{ + int in_ffd; + flb_ctx_t *ctx; + + /* The record has no "missing_field"; expect fallback to log_type. */ + ctx = create_ctx(&in_ffd, cb_check_log_type_key_fallback, + "log_type", "fallback", + "log_type_key", "$missing_field", + NULL); + + flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, sizeof(JSON_BASIC) - 1); + sleep(2); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_include_tag_key() +{ + int in_ffd; + flb_ctx_t *ctx; + + ctx = create_ctx(&in_ffd, cb_check_include_tag_key, + "include_tag_key", "true", + "tag_key", "fluentbit_tag", + NULL); + + flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, sizeof(JSON_BASIC) - 1); + sleep(2); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_event_timestamp() +{ + int in_ffd; + flb_ctx_t *ctx; + + ctx = create_ctx(&in_ffd, cb_check_event_timestamp, NULL); + + flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, sizeof(JSON_BASIC) - 1); + sleep(2); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_timestamp_key_passthrough() +{ + int in_ffd; + flb_ctx_t *ctx; + + ctx = create_ctx(&in_ffd, cb_check_timestamp_passthrough, + "timestamp_key", "$ts", + NULL); + + flb_lib_push(ctx, in_ffd, + (char *) JSON_WITH_TIMESTAMP, + sizeof(JSON_WITH_TIMESTAMP) - 1); + sleep(2); + + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_timestamp_key_fallback() +{ + int in_ffd; + flb_ctx_t *ctx; + + ctx = create_ctx(&in_ffd, cb_check_timestamp_key_fallback, + "timestamp_key", "$missing_ts", + NULL); + + flb_lib_push(ctx, in_ffd, (char *) JSON_BASIC, sizeof(JSON_BASIC) - 1); + sleep(2); + + flb_stop(ctx); + flb_destroy(ctx); +} + +TEST_LIST = { + { "basic_shape", flb_test_basic_shape }, + { "default_log_type", flb_test_default_log_type }, + { "configured_log_type", flb_test_configured_log_type }, + { "log_type_key", flb_test_log_type_key }, + { "log_type_key_fallback", flb_test_log_type_key_fallback }, + { "include_tag_key", flb_test_include_tag_key }, + { "event_timestamp", flb_test_event_timestamp }, + { "timestamp_key_passthrough", flb_test_timestamp_key_passthrough }, + { "timestamp_key_fallback", flb_test_timestamp_key_fallback }, + { NULL, NULL } +};