Coverage Report

Created: 2025-09-04 07:51

/src/fluent-bit/plugins/out_td/td.c
Line
Count
Source (jump to first uncovered line)
1
/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2
3
/*  Fluent Bit
4
 *  ==========
5
 *  Copyright (C) 2015-2024 The Fluent Bit Authors
6
 *
7
 *  Licensed under the Apache License, Version 2.0 (the "License");
8
 *  you may not use this file except in compliance with the License.
9
 *  You may obtain a copy of the License at
10
 *
11
 *      http://www.apache.org/licenses/LICENSE-2.0
12
 *
13
 *  Unless required by applicable law or agreed to in writing, software
14
 *  distributed under the License is distributed on an "AS IS" BASIS,
15
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
 *  See the License for the specific language governing permissions and
17
 *  limitations under the License.
18
 */
19
20
#include <fluent-bit/flb_output_plugin.h>
21
#include <fluent-bit/flb_network.h>
22
#include <fluent-bit/flb_pack.h>
23
#include <fluent-bit/flb_http_client.h>
24
#include <fluent-bit/flb_time.h>
25
#include <fluent-bit/flb_log_event_decoder.h>
26
#include <msgpack.h>
27
28
#include "td.h"
29
#include "td_http.h"
30
#include "td_config.h"
31
32
#include <stdio.h>
33
#include <stdlib.h>
34
#include <assert.h>
35
#include <errno.h>
36
37
/*
38
 * Convert the internal Fluent Bit data representation to the required
39
 * one by Treasure Data cloud service.
40
 *
41
 * This function returns a new msgpack buffer and store the bytes length
42
 * in the out_size variable.
43
 */
44
static char *td_format(struct flb_td *ctx, const void *data, size_t bytes, int *out_size)
45
0
{
46
0
    int i;
47
0
    int ret;
48
0
    int n_size;
49
0
    time_t atime;
50
0
    char *buf;
51
0
    struct msgpack_sbuffer mp_sbuf;
52
0
    struct msgpack_packer mp_pck;
53
0
    msgpack_object map;
54
0
    msgpack_sbuffer *sbuf;
55
0
    struct flb_log_event_decoder log_decoder;
56
0
    struct flb_log_event log_event;
57
58
    /* Initialize contexts for new output */
59
0
    msgpack_sbuffer_init(&mp_sbuf);
60
0
    msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write);
61
62
0
    ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes);
63
64
0
    if (ret != FLB_EVENT_DECODER_SUCCESS) {
65
0
        flb_plg_error(ctx->ins,
66
0
                      "Log event decoder initialization error : %d", ret);
67
68
0
        return NULL;
69
0
    }
70
71
0
    while ((ret = flb_log_event_decoder_next(
72
0
                    &log_decoder,
73
0
                    &log_event)) == FLB_EVENT_DECODER_SUCCESS) {
74
0
        atime = log_event.timestamp.tm.tv_sec;
75
0
        map   = *log_event.body;
76
77
0
        n_size = map.via.map.size + 1;
78
0
        msgpack_pack_map(&mp_pck, n_size);
79
0
        msgpack_pack_str(&mp_pck, 4);
80
0
        msgpack_pack_str_body(&mp_pck, "time", 4);
81
0
        msgpack_pack_int32(&mp_pck, atime);
82
83
0
        for (i = 0; i < n_size - 1; i++) {
84
0
            msgpack_pack_object(&mp_pck, map.via.map.ptr[i].key);
85
0
            msgpack_pack_object(&mp_pck, map.via.map.ptr[i].val);
86
0
        }
87
0
    }
88
89
0
    flb_log_event_decoder_destroy(&log_decoder);
90
91
    /* Create new buffer */
92
0
    sbuf = &mp_sbuf;
93
0
    *out_size = sbuf->size;
94
0
    buf = flb_malloc(sbuf->size);
95
0
    if (!buf) {
96
0
        flb_errno();
97
0
        return NULL;
98
0
    }
99
100
    /* set a new buffer and re-initialize our MessagePack context */
101
0
    memcpy(buf, sbuf->data, sbuf->size);
102
0
    msgpack_sbuffer_destroy(&mp_sbuf);
103
104
0
    return buf;
105
0
}
106
107
static int cb_td_init(struct flb_output_instance *ins, struct flb_config *config,
108
                      void *data)
109
0
{
110
0
    struct flb_td *ctx;
111
0
    struct flb_upstream *upstream;
112
0
    (void) data;
113
114
0
    ctx = td_config_init(ins);
115
0
    if (!ctx) {
116
0
        flb_plg_warn(ins, "Error reading configuration");
117
0
        return -1;
118
0
    }
119
120
0
    if (ctx->region == FLB_TD_REGION_US) {
121
0
        flb_output_net_default("api.treasuredata.com", 443, ins);
122
0
    }
123
0
    else if (ctx->region == FLB_TD_REGION_JP) {
124
0
        flb_output_net_default("api.treasuredata.co.jp", 443, ins);
125
0
    }
126
127
0
    upstream = flb_upstream_create(config,
128
0
                                   ins->host.name,
129
0
                                   ins->host.port,
130
0
                                   FLB_IO_TLS, ins->tls);
131
0
    if (!upstream) {
132
0
        flb_free(ctx);
133
0
        return -1;
134
0
    }
135
0
    ctx->u = upstream;
136
0
    flb_output_upstream_set(ctx->u, ins);
137
138
0
    flb_output_set_context(ins, ctx);
139
0
    return 0;
140
0
}
141
142
static void cb_td_flush(struct flb_event_chunk *event_chunk,
143
                        struct flb_output_flush *out_flush,
144
                        struct flb_input_instance *i_ins,
145
                        void *out_context,
146
                        struct flb_config *config)
147
0
{
148
0
    int ret;
149
0
    int bytes_out;
150
0
    char *pack;
151
0
    size_t b_sent;
152
0
    char *body = NULL;
153
0
    struct flb_td *ctx = out_context;
154
0
    struct flb_connection *u_conn;
155
0
    struct flb_http_client *c;
156
0
    (void) i_ins;
157
158
    /* Convert format */
159
0
    pack = td_format(ctx, event_chunk->data, event_chunk->size, &bytes_out);
160
0
    if (!pack) {
161
0
        FLB_OUTPUT_RETURN(FLB_ERROR);
162
0
    }
163
164
    /* Lookup an available connection context */
165
0
    u_conn = flb_upstream_conn_get(ctx->u);
166
0
    if (!u_conn) {
167
0
        flb_plg_error(ctx->ins, "no upstream connections available");
168
0
        flb_free(pack);
169
0
        FLB_OUTPUT_RETURN(FLB_RETRY);
170
0
    }
171
172
    /* Compose request */
173
0
    c = td_http_client(u_conn, pack, bytes_out, &body, ctx, config);
174
0
    if (!c) {
175
0
        flb_free(pack);
176
0
        flb_upstream_conn_release(u_conn);
177
0
        FLB_OUTPUT_RETURN(FLB_RETRY);
178
0
    }
179
180
    /* Issue HTTP request */
181
0
    ret = flb_http_do(c, &b_sent);
182
183
    /* Release Resources */
184
0
    flb_free(pack);
185
0
    flb_free(body);
186
187
    /* Validate HTTP status */
188
0
    if (ret == 0) {
189
        /* We expect a HTTP 200 OK */
190
0
        if (c->resp.status != 200) {
191
0
            if (c->resp.payload_size > 0) {
192
0
                flb_plg_warn(ctx->ins, "HTTP status %i\n%s",
193
0
                             c->resp.status, c->resp.payload);
194
0
            }
195
0
            else {
196
0
                flb_plg_warn(ctx->ins, "HTTP status %i", c->resp.status);
197
0
            }
198
0
            goto retry;
199
0
        }
200
0
        else {
201
0
            flb_plg_info(ctx->ins, "HTTP status 200 OK");
202
0
        }
203
0
    }
204
0
    else {
205
0
        flb_plg_error(ctx->ins, "http_do=%i", ret);
206
0
        goto retry;
207
0
    }
208
209
    /* release */
210
0
    flb_upstream_conn_release(u_conn);
211
0
    flb_http_client_destroy(c);
212
213
0
    FLB_OUTPUT_RETURN(FLB_OK);
214
215
0
 retry:
216
0
    flb_upstream_conn_release(u_conn);
217
0
    flb_http_client_destroy(c);
218
219
0
    FLB_OUTPUT_RETURN(FLB_RETRY);
220
0
}
221
222
static int cb_td_exit(void *data, struct flb_config *config)
223
0
{
224
0
    struct flb_td *ctx = data;
225
226
0
    if (!ctx) {
227
0
        return 0;
228
0
    }
229
230
0
    flb_upstream_destroy(ctx->u);
231
0
    flb_free(ctx);
232
233
0
    return 0;
234
0
}
235
236
static struct flb_config_map config_map[] = {
237
    {
238
      FLB_CONFIG_MAP_STR, "API", (char *)NULL,
239
      0, FLB_TRUE, offsetof(struct flb_td, api),
240
      "Set the API key"
241
    },
242
    {
243
      FLB_CONFIG_MAP_STR, "Database", (char *)NULL,
244
      0, FLB_TRUE, offsetof(struct flb_td, db_name),
245
      "Set the Database file"
246
    },
247
    {
248
      FLB_CONFIG_MAP_STR, "Table", (char *)NULL,
249
      0, FLB_TRUE, offsetof(struct flb_td, db_table),
250
      "Set the Database Table"
251
    },
252
    {
253
      FLB_CONFIG_MAP_STR, "Region", (char *)NULL,
254
      0, FLB_TRUE, offsetof(struct flb_td, region_str),
255
      "Set the Region: us or jp"
256
    },
257
    /* EOF */
258
    {0}
259
};
260
261
/* Plugin reference */
262
struct flb_output_plugin out_td_plugin = {
263
    .name           = "td",
264
    .description    = "Treasure Data",
265
    .cb_init        = cb_td_init,
266
    .cb_pre_run     = NULL,
267
    .cb_flush       = cb_td_flush,
268
    .cb_exit        = cb_td_exit,
269
    .config_map     = config_map,
270
    .flags          = FLB_IO_TLS,
271
};