Coverage Report

Created: 2026-11-12 07:29

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/fluent-bit/plugins/out_arvancloud_cloudlogs/arvancloud_cloudlogs.c
Line
Count
Source
1
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2
3
/*  Fluent Bit
4
 *  ==========
5
 *  Copyright (C) 2015-2024 The Fluent Bit Authors
6
 *
7
 *  Licensed under the Apache License, Version 2.0 (the "License");
8
 *  you may not use this file except in compliance with the License.
9
 *  You may obtain a copy of the License at
10
 *
11
 *      http://www.apache.org/licenses/LICENSE-2.0
12
 *
13
 *  Unless required by applicable law or agreed to in writing, software
14
 *  distributed under the License is distributed on an "AS IS" BASIS,
15
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
 *  See the License for the specific language governing permissions and
17
 *  limitations under the License.
18
 */
19
20
#include <fluent-bit/flb_output_plugin.h>
21
#include <fluent-bit/flb_compat.h>
22
#include <fluent-bit/flb_http_client.h>
23
#include <fluent-bit/flb_utils.h>
24
#include <fluent-bit/flb_pack.h>
25
#include <fluent-bit/flb_gzip.h>
26
#include <fluent-bit/flb_sds.h>
27
#include <fluent-bit/flb_record_accessor.h>
28
#include <fluent-bit/flb_time.h>
29
#include <time.h>
30
#include <stdio.h>
31
#include <string.h>
32
#include <fluent-bit/flb_log_event_decoder.h>
33
34
#include "arvancloud_cloudlogs.h"
35
36
/*
37
 * Format event time to RFC3339 with microseconds in UTC.
38
 * String parsing belongs in a parser (Time_Key / Time_Format / %L / %z);
39
 * this output only shapes the API date-time field.
40
 */
41
static size_t format_timestamp_rfc3339(char *buffer, size_t buffer_size,
42
                                        time_t seconds, long nsec)
43
0
{
44
0
    struct tm tm;
45
0
    size_t s;
46
0
    int len;
47
48
0
    gmtime_r(&seconds, &tm);
49
0
    s = strftime(buffer, buffer_size - 1, "%Y-%m-%dT%H:%M:%S", &tm);
50
0
    len = snprintf(buffer + s, buffer_size - 1 - s,
51
0
                   ".%06ldZ", (long) (nsec / 1000));
52
0
    return s + len;
53
0
}
54
55
static int arvancloud_format(struct flb_config *config,
56
                        struct flb_input_instance *ins,
57
                        void *plugin_context,
58
                        void *flush_context,
59
                        int event_type,
60
                        const char *tag, int tag_len,
61
                        const void *data, size_t bytes,
62
                        void **out_data, size_t *out_size)
63
0
{
64
0
    int ret;
65
0
    size_t final_log_type_len;
66
0
    struct flb_log_event_decoder log_decoder;
67
0
    struct flb_log_event log_event;
68
0
    struct flb_out_arvancloud_cloudlogs *ctx;
69
0
    struct flb_event_chunk *event_chunk;
70
0
    msgpack_sbuffer mp_sbuf;
71
0
    msgpack_packer mp_pck;
72
0
    size_t count;
73
0
    size_t s;
74
0
    struct flb_time t;
75
0
    char time_formatted[64];
76
0
    flb_sds_t log_type_value;
77
0
    const char *final_log_type;
78
79
0
    ctx = plugin_context;
80
0
    event_chunk = flush_context;
81
0
    count = 0;
82
83
0
    (void) ins;
84
0
    (void) event_type;
85
86
    /* Get event count from event_chunk if available, otherwise count manually */
87
0
    if (event_chunk != NULL) {
88
0
        count = event_chunk->total_events;
89
0
    }
90
0
    else {
91
0
        count = flb_mp_count(data, bytes);
92
0
    }
93
94
0
    ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes);
95
0
    if (ret != FLB_EVENT_DECODER_SUCCESS) {
96
0
        flb_plg_error(ctx->ins, "log event decoder init error: %d", ret);
97
0
        return -1;
98
0
    }
99
100
0
    msgpack_sbuffer_init(&mp_sbuf);
101
0
    msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write);
102
103
    /* Build request object: { logs: [ ... ] } */
104
0
    msgpack_pack_map(&mp_pck, 1);
105
106
    /* logs */
107
0
    msgpack_pack_str(&mp_pck, 4);
108
0
    msgpack_pack_str_body(&mp_pck, "logs", 4);
109
110
0
    msgpack_pack_array(&mp_pck, count);
111
112
0
    while ((ret = flb_log_event_decoder_next(
113
0
                &log_decoder, &log_event)) == FLB_EVENT_DECODER_SUCCESS) {
114
0
        t = log_event.timestamp;
115
0
        log_type_value = NULL;
116
117
0
        msgpack_pack_map(&mp_pck, ctx->include_tag_key ? 6 : 5);
118
119
        /* logType: priority order is log_type_key > log_type */
120
0
        if (ctx->log_type_key && ctx->ra_log_type_key) {
121
            /* Try to extract from record using log_type_key */
122
0
            log_type_value = flb_ra_translate(ctx->ra_log_type_key,
123
0
                                              (char *) tag, tag_len,
124
0
                                              *log_event.body, NULL);
125
0
            if (log_type_value && !flb_sds_is_empty(log_type_value)) {
126
0
                final_log_type = log_type_value;
127
0
                final_log_type_len = flb_sds_len(log_type_value);
128
0
            }
129
0
            else {
130
                /* Clean up failed extraction and fall back to log_type */
131
0
                if (log_type_value) {
132
0
                    flb_sds_destroy(log_type_value);
133
0
                    log_type_value = NULL;
134
0
                }
135
0
                final_log_type = ctx->log_type;
136
0
                final_log_type_len = flb_sds_len(ctx->log_type);
137
0
            }
138
0
        }
139
0
        else {
140
            /* Use configured log_type (has default value) */
141
0
            final_log_type = ctx->log_type;
142
0
            final_log_type_len = flb_sds_len(ctx->log_type);
143
0
        }
144
145
0
        msgpack_pack_str(&mp_pck, 7);
146
0
        msgpack_pack_str_body(&mp_pck, "logType", 7);
147
0
        msgpack_pack_str(&mp_pck, final_log_type_len);
148
0
        msgpack_pack_str_body(&mp_pck, final_log_type, final_log_type_len);
149
150
        /* Clean up if we allocated log_type_value */
151
0
        if (log_type_value) {
152
0
            flb_sds_destroy(log_type_value);
153
0
        }
154
155
        /*
156
         * timestamp: prefer Fluent Bit event time (set upstream via parser
157
         * Time_Key / Time_Format). Optional timestamp_key is pass-through
158
         * only when the field is already an OpenAPI date-time string.
159
         */
160
0
        if (ctx->timestamp_key && ctx->ra_timestamp_key) {
161
0
            flb_sds_t timestamp_value;
162
0
            int use_record_value;
163
164
0
            use_record_value = FLB_FALSE;
165
0
            timestamp_value = flb_ra_translate(ctx->ra_timestamp_key,
166
0
                                               (char *) tag, tag_len,
167
0
                                               *log_event.body, NULL);
168
169
0
            if (timestamp_value && !flb_sds_is_empty(timestamp_value)) {
170
0
                s = flb_sds_len(timestamp_value);
171
0
                if (s >= sizeof(time_formatted)) {
172
0
                    s = sizeof(time_formatted) - 1;
173
0
                }
174
175
0
                memcpy(time_formatted, timestamp_value, s);
176
0
                time_formatted[s] = '\0';
177
0
                use_record_value = FLB_TRUE;
178
0
            }
179
180
0
            if (timestamp_value) {
181
0
                flb_sds_destroy(timestamp_value);
182
0
            }
183
184
0
            if (!use_record_value) {
185
0
                s = format_timestamp_rfc3339(time_formatted,
186
0
                                             sizeof(time_formatted),
187
0
                                             t.tm.tv_sec, t.tm.tv_nsec);
188
0
            }
189
0
        }
190
0
        else {
191
0
            s = format_timestamp_rfc3339(time_formatted, sizeof(time_formatted),
192
0
                                         t.tm.tv_sec, t.tm.tv_nsec);
193
0
        }
194
195
0
        msgpack_pack_str(&mp_pck, 9);
196
0
        msgpack_pack_str_body(&mp_pck, "timestamp", 9);
197
0
        msgpack_pack_str(&mp_pck, s);
198
0
        msgpack_pack_str_body(&mp_pck, time_formatted, s);
199
200
0
        msgpack_pack_str(&mp_pck, 8);
201
0
        msgpack_pack_str_body(&mp_pck, "severity", 8);
202
0
        msgpack_pack_str(&mp_pck, 4);
203
0
        msgpack_pack_str_body(&mp_pck, "INFO", 4);
204
205
        /* resource: minimal object */
206
0
        msgpack_pack_str(&mp_pck, 8);
207
0
        msgpack_pack_str_body(&mp_pck, "resource", 8);
208
0
        msgpack_pack_map(&mp_pck, 1);
209
0
        msgpack_pack_str(&mp_pck, 4);
210
0
        msgpack_pack_str_body(&mp_pck, "type", 4);
211
0
        msgpack_pack_str(&mp_pck, 7);
212
0
        msgpack_pack_str_body(&mp_pck, "general", 7);
213
214
        /* payload: original record map */
215
0
        msgpack_pack_str(&mp_pck, 7);
216
0
        msgpack_pack_str_body(&mp_pck, "payload", 7);
217
0
        msgpack_pack_object(&mp_pck, *log_event.body);
218
219
        /* Include tag key within the log object if configured */
220
0
        if (ctx->include_tag_key) {
221
0
            msgpack_pack_str(&mp_pck, flb_sds_len(ctx->tag_key));
222
0
            msgpack_pack_str_body(&mp_pck, ctx->tag_key, flb_sds_len(ctx->tag_key));
223
0
            msgpack_pack_str(&mp_pck, tag_len);
224
0
            msgpack_pack_str_body(&mp_pck, tag, tag_len);
225
0
        }
226
0
    }
227
228
0
    flb_log_event_decoder_destroy(&log_decoder);
229
230
0
    *out_data = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size,
231
0
                                             config->json_escape_unicode);
232
0
    msgpack_sbuffer_destroy(&mp_sbuf);
233
0
    if (!*out_data) {
234
0
        return -1;
235
0
    }
236
0
    *out_size = flb_sds_len((flb_sds_t) *out_data);
237
0
    return 0;
238
0
}
239
240
static void cb_arvancloud_flush(struct flb_event_chunk *event_chunk,
241
                           struct flb_output_flush *out_flush,
242
                           struct flb_input_instance *i_ins,
243
                           void *out_context,
244
                           struct flb_config *config)
245
0
{
246
0
    int compressed;
247
0
    int ret;
248
0
    size_t b_sent;
249
0
    size_t payload_size;
250
0
    size_t final_payload_size;
251
0
    size_t prefix_len;
252
0
    void *payload_buf;
253
0
    void *final_payload;
254
0
    const char *prefix;
255
0
    flb_sds_t header;
256
0
    struct flb_out_arvancloud_cloudlogs *ctx;
257
0
    struct flb_connection *u_conn;
258
0
    struct flb_http_client *c;
259
260
0
    compressed = FLB_FALSE;
261
0
    payload_size = 0;
262
0
    final_payload_size = 0;
263
0
    payload_buf = NULL;
264
0
    final_payload = NULL;
265
0
    ctx = out_context;
266
267
0
    (void) out_flush;
268
0
    (void) i_ins;
269
270
0
    ret = arvancloud_format(config, i_ins, ctx, event_chunk,
271
0
                       event_chunk->type,
272
0
                       event_chunk->tag, flb_sds_len(event_chunk->tag),
273
0
                       event_chunk->data, event_chunk->size,
274
0
                       &payload_buf, &payload_size);
275
0
    if (ret == -1) {
276
0
        FLB_OUTPUT_RETURN(FLB_ERROR);
277
0
    }
278
279
0
    final_payload = payload_buf;
280
0
    final_payload_size = payload_size;
281
0
    if (ctx->compress_gzip == FLB_TRUE) {
282
0
        ret = flb_gzip_compress((void *) payload_buf, payload_size,
283
0
                                &final_payload, &final_payload_size);
284
0
        if (ret == 0) {
285
0
            compressed = FLB_TRUE;
286
0
        }
287
0
        else {
288
0
            flb_plg_error(ctx->ins, "cannot gzip payload, disabling compression");
289
0
            final_payload = payload_buf;
290
0
            final_payload_size = payload_size;
291
0
        }
292
0
    }
293
294
    /* Debug: dump request JSON payload */
295
0
    flb_plg_debug(ctx->ins, "request payload (%zu bytes): %.*s",
296
0
                  final_payload_size,
297
0
                  (int) final_payload_size,
298
0
                  (const char *) final_payload);
299
300
0
    u_conn = flb_upstream_conn_get(ctx->u);
301
0
    if (!u_conn) {
302
0
        flb_plg_error(ctx->ins, "no upstream connections available to %s:%i",
303
0
                      ctx->u->tcp_host, ctx->u->tcp_port);
304
0
        if (final_payload != payload_buf) {
305
0
            flb_free(final_payload);
306
0
        }
307
0
        flb_sds_destroy((flb_sds_t) payload_buf);
308
0
        FLB_OUTPUT_RETURN(FLB_RETRY);
309
0
    }
310
311
0
    c = flb_http_client(u_conn, FLB_HTTP_POST, ctx->uri,
312
0
                        final_payload, final_payload_size,
313
0
                        NULL, 0,
314
0
                        NULL, 0);
315
0
    if (!c) {
316
0
        if (final_payload != payload_buf) {
317
0
            flb_free(final_payload);
318
0
        }
319
0
        flb_sds_destroy((flb_sds_t) payload_buf);
320
0
        flb_upstream_conn_release(u_conn);
321
0
        FLB_OUTPUT_RETURN(FLB_RETRY);
322
0
    }
323
324
    /* Add standard headers */
325
0
    flb_http_add_header(c, "User-Agent", 10, "Fluent-Bit", 10);
326
0
    flb_http_add_header(c, "Content-Type", 12, "application/json", 16);
327
    
328
    /* Content Encoding: gzip */
329
0
    if (compressed == FLB_TRUE) {
330
0
        flb_http_set_content_encoding_gzip(c);
331
0
    }
332
    
333
    /* API Key Authorization header: "apikey <api_key>" */
334
0
    if (ctx->api_key) {
335
0
        prefix = "apikey ";
336
0
        prefix_len = 7;
337
0
        header = flb_sds_create_size(prefix_len + flb_sds_len(ctx->api_key));
338
0
        if (header) {
339
0
            header = flb_sds_cat(header, prefix, prefix_len);
340
0
            header = flb_sds_cat(header, ctx->api_key,
341
0
                                 flb_sds_len(ctx->api_key));
342
0
            flb_http_add_header(c, "Authorization", 13, header,
343
0
                                flb_sds_len(header));
344
0
            flb_sds_destroy(header);
345
0
        }
346
0
    }
347
348
0
    ret = flb_http_do(c, &b_sent);
349
0
    if (ret != 0) {
350
0
        flb_plg_warn(ctx->ins, "http_do=%i", ret);
351
0
        ret = FLB_RETRY;
352
0
    }
353
0
    else {
354
        /* Handle HTTP status codes */
355
0
        if (c->resp.status >= 200 && c->resp.status <= 205) {
356
            /* Success: 200 OK, 201 Created, 202 Accepted, 203-205 */
357
0
            flb_plg_debug(ctx->ins, "HTTP status=%i", c->resp.status);
358
0
            if (c->resp.payload && c->resp.payload_size > 0) {
359
0
                flb_plg_debug(ctx->ins, "response body (%zu bytes): %.*s",
360
0
                              c->resp.payload_size,
361
0
                              (int) c->resp.payload_size,
362
0
                              (const char *) c->resp.payload);
363
0
            }
364
0
            ret = FLB_OK;
365
0
        }
366
0
        else if (c->resp.status == 400) {
367
            /* Bad Request - usually a client error, don't retry */
368
0
            flb_plg_error(ctx->ins, "HTTP status=400 (Bad Request)");
369
0
            if (c->resp.payload && c->resp.payload_size > 0) {
370
0
                flb_plg_error(ctx->ins, "response body: %.*s",
371
0
                              (int) c->resp.payload_size,
372
0
                              (const char *) c->resp.payload);
373
0
            }
374
0
            ret = FLB_ERROR;
375
0
        }
376
0
        else if (c->resp.status == 401 || c->resp.status == 403) {
377
            /* Unauthorized or Forbidden - auth issue, don't retry */
378
0
            flb_plg_error(ctx->ins,
379
0
                          "HTTP status=%i (Authentication/Authorization failed)",
380
0
                          c->resp.status);
381
0
            if (c->resp.payload && c->resp.payload_size > 0) {
382
0
                flb_plg_error(ctx->ins, "response body: %.*s",
383
0
                              (int) c->resp.payload_size,
384
0
                              (const char *) c->resp.payload);
385
0
            }
386
0
            ret = FLB_ERROR;
387
0
        }
388
0
        else if (c->resp.status == 429) {
389
            /* Too Many Requests - rate limit, retry */
390
0
            flb_plg_warn(ctx->ins,
391
0
                         "HTTP status=429 (Rate Limited), will retry");
392
0
            ret = FLB_RETRY;
393
0
        }
394
0
        else if (c->resp.status >= 500) {
395
            /* Server errors - retry */
396
0
            flb_plg_warn(ctx->ins,
397
0
                         "HTTP status=%i (Server Error), will retry",
398
0
                         c->resp.status);
399
0
            if (c->resp.payload && c->resp.payload_size > 0) {
400
0
                flb_plg_warn(ctx->ins, "response body: %.*s",
401
0
                             (int) c->resp.payload_size,
402
0
                             (const char *) c->resp.payload);
403
0
            }
404
0
            ret = FLB_RETRY;
405
0
        }
406
0
        else {
407
            /* Other client errors - don't retry */
408
0
            flb_plg_error(ctx->ins, "HTTP status=%i (Client Error)", c->resp.status);
409
0
            if (c->resp.payload && c->resp.payload_size > 0) {
410
0
                flb_plg_error(ctx->ins, "response body: %.*s",
411
0
                              (int) c->resp.payload_size,
412
0
                              (const char *) c->resp.payload);
413
0
            }
414
0
            ret = FLB_ERROR;
415
0
        }
416
0
    }
417
418
0
    if (final_payload != payload_buf) {
419
0
        flb_free(final_payload);
420
0
    }
421
0
    flb_sds_destroy((flb_sds_t) payload_buf);
422
0
    flb_http_client_destroy(c);
423
0
    flb_upstream_conn_release(u_conn);
424
425
0
    FLB_OUTPUT_RETURN(ret);
426
0
}
427
428
static int cb_arvancloud_init(struct flb_output_instance *ins,
429
                              struct flb_config *config,
430
                              void *data)
431
0
{
432
0
    struct flb_out_arvancloud_cloudlogs *ctx;
433
0
    (void) data;
434
435
0
    ctx = flb_arvancloud_conf_create(ins, config);
436
0
    if (!ctx) {
437
0
        return -1;
438
0
    }
439
0
    flb_output_set_context(ins, ctx);
440
0
    return 0;
441
0
}
442
443
static int cb_arvancloud_exit(void *data, struct flb_config *config)
444
0
{
445
0
    (void) config;
446
0
    return flb_arvancloud_conf_destroy(data);
447
0
}
448
449
/* Configuration properties map */
450
static struct flb_config_map config_map[] = {
451
    {
452
        FLB_CONFIG_MAP_STR, "apikey", NULL,
453
        0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, api_key),
454
        "API key for authorization (will be sent as 'apikey <value>')"
455
    },
456
    {
457
        FLB_CONFIG_MAP_STR, "log_type", FLB_ARVANCLOUD_LOG_TYPE,
458
        0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, log_type),
459
        "Log type value to use. Defaults to 'fluentbit'."
460
    },
461
    {
462
        FLB_CONFIG_MAP_STR, "log_type_key", NULL,
463
        0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, log_type_key),
464
        "Field in the record to use as log type. "
465
        "Takes priority over 'log_type' and tag prefix."
466
    },
467
    {
468
        FLB_CONFIG_MAP_BOOL, "gzip", "false",
469
        0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, compress_gzip),
470
        "Enable gzip compression"
471
    },
472
    {
473
        FLB_CONFIG_MAP_BOOL, "include_tag_key", "false",
474
        0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, include_tag_key),
475
        "Include original tag in each record"
476
    },
477
    {
478
        FLB_CONFIG_MAP_STR, "tag_key", "tag",
479
        0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, tag_key),
480
        "Tag key name when include_tag_key=true"
481
    },
482
    {
483
        FLB_CONFIG_MAP_STR, "timestamp_key", NULL,
484
        0, FLB_TRUE, offsetof(struct flb_out_arvancloud_cloudlogs, timestamp_key),
485
        "Optional record field to forward as CloudLogs timestamp when the "
486
        "value is already an OpenAPI date-time string. Prefer setting event "
487
        "time with a parser (Time_Key / Time_Format). If unset or missing, "
488
        "uses the Fluent Bit event timestamp formatted as RFC3339 UTC."
489
    },
490
    /* EOF */
491
    {0}
492
};
493
494
struct flb_output_plugin out_arvancloud_cloudlogs_plugin = {
495
    .name         = "arvancloud_cloudlogs",
496
    .description  = "Send events to ArvanCloud CloudLogs",
497
    .cb_init      = cb_arvancloud_init,
498
    .cb_flush     = cb_arvancloud_flush,
499
    .cb_exit      = cb_arvancloud_exit,
500
    .test_formatter.callback = arvancloud_format,
501
    .config_map   = config_map,
502
    .workers      = 2,
503
    .flags        = FLB_OUTPUT_NET | FLB_IO_TLS,
504
};
505
506