/src/fluent-bit/plugins/out_td/td.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ |
2 | | |
3 | | /* Fluent Bit |
4 | | * ========== |
5 | | * Copyright (C) 2015-2024 The Fluent Bit Authors |
6 | | * |
7 | | * Licensed under the Apache License, Version 2.0 (the "License"); |
8 | | * you may not use this file except in compliance with the License. |
9 | | * You may obtain a copy of the License at |
10 | | * |
11 | | * http://www.apache.org/licenses/LICENSE-2.0 |
12 | | * |
13 | | * Unless required by applicable law or agreed to in writing, software |
14 | | * distributed under the License is distributed on an "AS IS" BASIS, |
15 | | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
16 | | * See the License for the specific language governing permissions and |
17 | | * limitations under the License. |
18 | | */ |
19 | | |
20 | | #include <fluent-bit/flb_output_plugin.h> |
21 | | #include <fluent-bit/flb_network.h> |
22 | | #include <fluent-bit/flb_pack.h> |
23 | | #include <fluent-bit/flb_http_client.h> |
24 | | #include <fluent-bit/flb_time.h> |
25 | | #include <fluent-bit/flb_log_event_decoder.h> |
26 | | #include <msgpack.h> |
27 | | |
28 | | #include "td.h" |
29 | | #include "td_http.h" |
30 | | #include "td_config.h" |
31 | | |
32 | | #include <stdio.h> |
33 | | #include <stdlib.h> |
34 | | #include <assert.h> |
35 | | #include <errno.h> |
36 | | |
37 | | /* |
38 | | * Convert the internal Fluent Bit data representation to the required |
39 | | * one by Treasure Data cloud service. |
40 | | * |
41 | | * This function returns a new msgpack buffer and store the bytes length |
42 | | * in the out_size variable. |
43 | | */ |
44 | | static char *td_format(struct flb_td *ctx, const void *data, size_t bytes, int *out_size) |
45 | 0 | { |
46 | 0 | int i; |
47 | 0 | int ret; |
48 | 0 | int n_size; |
49 | 0 | time_t atime; |
50 | 0 | char *buf; |
51 | 0 | struct msgpack_sbuffer mp_sbuf; |
52 | 0 | struct msgpack_packer mp_pck; |
53 | 0 | msgpack_object map; |
54 | 0 | msgpack_sbuffer *sbuf; |
55 | 0 | struct flb_log_event_decoder log_decoder; |
56 | 0 | struct flb_log_event log_event; |
57 | | |
58 | | /* Initialize contexts for new output */ |
59 | 0 | msgpack_sbuffer_init(&mp_sbuf); |
60 | 0 | msgpack_packer_init(&mp_pck, &mp_sbuf, msgpack_sbuffer_write); |
61 | |
|
62 | 0 | ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes); |
63 | |
|
64 | 0 | if (ret != FLB_EVENT_DECODER_SUCCESS) { |
65 | 0 | flb_plg_error(ctx->ins, |
66 | 0 | "Log event decoder initialization error : %d", ret); |
67 | |
|
68 | 0 | return NULL; |
69 | 0 | } |
70 | | |
71 | 0 | while ((ret = flb_log_event_decoder_next( |
72 | 0 | &log_decoder, |
73 | 0 | &log_event)) == FLB_EVENT_DECODER_SUCCESS) { |
74 | 0 | atime = log_event.timestamp.tm.tv_sec; |
75 | 0 | map = *log_event.body; |
76 | |
|
77 | 0 | n_size = map.via.map.size + 1; |
78 | 0 | msgpack_pack_map(&mp_pck, n_size); |
79 | 0 | msgpack_pack_str(&mp_pck, 4); |
80 | 0 | msgpack_pack_str_body(&mp_pck, "time", 4); |
81 | 0 | msgpack_pack_int32(&mp_pck, atime); |
82 | |
|
83 | 0 | for (i = 0; i < n_size - 1; i++) { |
84 | 0 | msgpack_pack_object(&mp_pck, map.via.map.ptr[i].key); |
85 | 0 | msgpack_pack_object(&mp_pck, map.via.map.ptr[i].val); |
86 | 0 | } |
87 | 0 | } |
88 | |
|
89 | 0 | flb_log_event_decoder_destroy(&log_decoder); |
90 | | |
91 | | /* Create new buffer */ |
92 | 0 | sbuf = &mp_sbuf; |
93 | 0 | *out_size = sbuf->size; |
94 | 0 | buf = flb_malloc(sbuf->size); |
95 | 0 | if (!buf) { |
96 | 0 | flb_errno(); |
97 | 0 | return NULL; |
98 | 0 | } |
99 | | |
100 | | /* set a new buffer and re-initialize our MessagePack context */ |
101 | 0 | memcpy(buf, sbuf->data, sbuf->size); |
102 | 0 | msgpack_sbuffer_destroy(&mp_sbuf); |
103 | |
|
104 | 0 | return buf; |
105 | 0 | } |
106 | | |
107 | | static int cb_td_init(struct flb_output_instance *ins, struct flb_config *config, |
108 | | void *data) |
109 | 0 | { |
110 | 0 | struct flb_td *ctx; |
111 | 0 | struct flb_upstream *upstream; |
112 | 0 | (void) data; |
113 | |
|
114 | 0 | ctx = td_config_init(ins); |
115 | 0 | if (!ctx) { |
116 | 0 | flb_plg_warn(ins, "Error reading configuration"); |
117 | 0 | return -1; |
118 | 0 | } |
119 | | |
120 | 0 | if (ctx->region == FLB_TD_REGION_US) { |
121 | 0 | flb_output_net_default("api.treasuredata.com", 443, ins); |
122 | 0 | } |
123 | 0 | else if (ctx->region == FLB_TD_REGION_JP) { |
124 | 0 | flb_output_net_default("api.treasuredata.co.jp", 443, ins); |
125 | 0 | } |
126 | |
|
127 | 0 | upstream = flb_upstream_create(config, |
128 | 0 | ins->host.name, |
129 | 0 | ins->host.port, |
130 | 0 | FLB_IO_TLS, ins->tls); |
131 | 0 | if (!upstream) { |
132 | 0 | flb_free(ctx); |
133 | 0 | return -1; |
134 | 0 | } |
135 | 0 | ctx->u = upstream; |
136 | 0 | flb_output_upstream_set(ctx->u, ins); |
137 | |
|
138 | 0 | flb_output_set_context(ins, ctx); |
139 | 0 | return 0; |
140 | 0 | } |
141 | | |
142 | | static void cb_td_flush(struct flb_event_chunk *event_chunk, |
143 | | struct flb_output_flush *out_flush, |
144 | | struct flb_input_instance *i_ins, |
145 | | void *out_context, |
146 | | struct flb_config *config) |
147 | 0 | { |
148 | 0 | int ret; |
149 | 0 | int bytes_out; |
150 | 0 | char *pack; |
151 | 0 | size_t b_sent; |
152 | 0 | char *body = NULL; |
153 | 0 | struct flb_td *ctx = out_context; |
154 | 0 | struct flb_connection *u_conn; |
155 | 0 | struct flb_http_client *c; |
156 | 0 | (void) i_ins; |
157 | | |
158 | | /* Convert format */ |
159 | 0 | pack = td_format(ctx, event_chunk->data, event_chunk->size, &bytes_out); |
160 | 0 | if (!pack) { |
161 | 0 | FLB_OUTPUT_RETURN(FLB_ERROR); |
162 | 0 | } |
163 | | |
164 | | /* Lookup an available connection context */ |
165 | 0 | u_conn = flb_upstream_conn_get(ctx->u); |
166 | 0 | if (!u_conn) { |
167 | 0 | flb_plg_error(ctx->ins, "no upstream connections available"); |
168 | 0 | flb_free(pack); |
169 | 0 | FLB_OUTPUT_RETURN(FLB_RETRY); |
170 | 0 | } |
171 | | |
172 | | /* Compose request */ |
173 | 0 | c = td_http_client(u_conn, pack, bytes_out, &body, ctx, config); |
174 | 0 | if (!c) { |
175 | 0 | flb_free(pack); |
176 | 0 | flb_upstream_conn_release(u_conn); |
177 | 0 | FLB_OUTPUT_RETURN(FLB_RETRY); |
178 | 0 | } |
179 | | |
180 | | /* Issue HTTP request */ |
181 | 0 | ret = flb_http_do(c, &b_sent); |
182 | | |
183 | | /* Release Resources */ |
184 | 0 | flb_free(pack); |
185 | 0 | flb_free(body); |
186 | | |
187 | | /* Validate HTTP status */ |
188 | 0 | if (ret == 0) { |
189 | | /* We expect a HTTP 200 OK */ |
190 | 0 | if (c->resp.status != 200) { |
191 | 0 | if (c->resp.payload_size > 0) { |
192 | 0 | flb_plg_warn(ctx->ins, "HTTP status %i\n%s", |
193 | 0 | c->resp.status, c->resp.payload); |
194 | 0 | } |
195 | 0 | else { |
196 | 0 | flb_plg_warn(ctx->ins, "HTTP status %i", c->resp.status); |
197 | 0 | } |
198 | 0 | goto retry; |
199 | 0 | } |
200 | 0 | else { |
201 | 0 | flb_plg_info(ctx->ins, "HTTP status 200 OK"); |
202 | 0 | } |
203 | 0 | } |
204 | 0 | else { |
205 | 0 | flb_plg_error(ctx->ins, "http_do=%i", ret); |
206 | 0 | goto retry; |
207 | 0 | } |
208 | | |
209 | | /* release */ |
210 | 0 | flb_upstream_conn_release(u_conn); |
211 | 0 | flb_http_client_destroy(c); |
212 | |
|
213 | 0 | FLB_OUTPUT_RETURN(FLB_OK); |
214 | | |
215 | 0 | retry: |
216 | 0 | flb_upstream_conn_release(u_conn); |
217 | 0 | flb_http_client_destroy(c); |
218 | |
|
219 | 0 | FLB_OUTPUT_RETURN(FLB_RETRY); |
220 | 0 | } |
221 | | |
222 | | static int cb_td_exit(void *data, struct flb_config *config) |
223 | 0 | { |
224 | 0 | struct flb_td *ctx = data; |
225 | |
|
226 | 0 | if (!ctx) { |
227 | 0 | return 0; |
228 | 0 | } |
229 | | |
230 | 0 | flb_upstream_destroy(ctx->u); |
231 | 0 | flb_free(ctx); |
232 | |
|
233 | 0 | return 0; |
234 | 0 | } |
235 | | |
236 | | static struct flb_config_map config_map[] = { |
237 | | { |
238 | | FLB_CONFIG_MAP_STR, "API", (char *)NULL, |
239 | | 0, FLB_TRUE, offsetof(struct flb_td, api), |
240 | | "Set the API key" |
241 | | }, |
242 | | { |
243 | | FLB_CONFIG_MAP_STR, "Database", (char *)NULL, |
244 | | 0, FLB_TRUE, offsetof(struct flb_td, db_name), |
245 | | "Set the Database file" |
246 | | }, |
247 | | { |
248 | | FLB_CONFIG_MAP_STR, "Table", (char *)NULL, |
249 | | 0, FLB_TRUE, offsetof(struct flb_td, db_table), |
250 | | "Set the Database Table" |
251 | | }, |
252 | | { |
253 | | FLB_CONFIG_MAP_STR, "Region", (char *)NULL, |
254 | | 0, FLB_TRUE, offsetof(struct flb_td, region_str), |
255 | | "Set the Region: us or jp" |
256 | | }, |
257 | | /* EOF */ |
258 | | {0} |
259 | | }; |
260 | | |
261 | | /* Plugin reference */ |
262 | | struct flb_output_plugin out_td_plugin = { |
263 | | .name = "td", |
264 | | .description = "Treasure Data", |
265 | | .cb_init = cb_td_init, |
266 | | .cb_pre_run = NULL, |
267 | | .cb_flush = cb_td_flush, |
268 | | .cb_exit = cb_td_exit, |
269 | | .config_map = config_map, |
270 | | .flags = FLB_IO_TLS, |
271 | | }; |