/src/fluent-bit/src/multiline/flb_ml_parser.c
Line | Count | Source |
1 | | /* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ |
2 | | |
3 | | /* Fluent Bit |
4 | | * ========== |
5 | | * Copyright (C) 2015-2026 The Fluent Bit Authors |
6 | | * |
7 | | * Licensed under the Apache License, Version 2.0 (the "License"); |
8 | | * you may not use this file except in compliance with the License. |
9 | | * You may obtain a copy of the License at |
10 | | * |
11 | | * http://www.apache.org/licenses/LICENSE-2.0 |
12 | | * |
13 | | * Unless required by applicable law or agreed to in writing, software |
14 | | * distributed under the License is distributed on an "AS IS" BASIS, |
15 | | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
16 | | * See the License for the specific language governing permissions and |
17 | | * limitations under the License. |
18 | | */ |
19 | | |
20 | | #include <fluent-bit/flb_info.h> |
21 | | #include <fluent-bit/flb_log.h> |
22 | | #include <fluent-bit/multiline/flb_ml.h> |
23 | | #include <fluent-bit/multiline/flb_ml_parser.h> |
24 | | #include <fluent-bit/multiline/flb_ml_rule.h> |
25 | | #include <fluent-bit/multiline/flb_ml_mode.h> |
26 | | #include <fluent-bit/multiline/flb_ml_group.h> |
27 | | |
28 | | #include <stddef.h> |
29 | | #include <string.h> |
30 | | |
31 | | /* ---------------- params + defaults ---------------- */ |
32 | | struct flb_ml_parser_params flb_ml_parser_params_default(const char *name) |
33 | 588k | { |
34 | 588k | struct flb_ml_parser_params p; |
35 | 588k | memset(&p, 0, sizeof(p)); |
36 | | |
37 | 588k | p.size = sizeof(p); |
38 | 588k | p.name = (char *) name; |
39 | 588k | p.type = FLB_ML_REGEX; /* sane default */ |
40 | 588k | p.negate = 0; |
41 | 588k | p.flush_ms = FLB_ML_FLUSH_TIMEOUT; /* header constant */ |
42 | | /* other pointers remain NULL by default */ |
43 | 588k | return p; |
44 | 588k | } |
45 | | |
46 | | /* New canonical creator that mirrors old behavior using params */ |
47 | | struct flb_ml_parser *flb_ml_parser_create_params(struct flb_config *ctx, |
48 | | const struct flb_ml_parser_params *p) |
49 | 588k | { |
50 | 588k | struct flb_ml_parser *ml_parser; |
51 | 588k | size_t min = offsetof(struct flb_ml_parser_params, flags) + sizeof(uint32_t); |
52 | | |
53 | 588k | if (!ctx || !p || p->size < min || !p->name) { |
54 | 0 | return NULL; |
55 | 0 | } |
56 | | |
57 | 588k | ml_parser = flb_calloc(1, sizeof(struct flb_ml_parser)); |
58 | 588k | if (!ml_parser) { |
59 | 16 | flb_errno(); |
60 | 16 | return NULL; |
61 | 16 | } |
62 | | |
63 | | /* prepare rules list */ |
64 | 588k | mk_list_init(&ml_parser->_head); |
65 | 588k | mk_list_init(&ml_parser->regex_rules); |
66 | | |
67 | | /* name/type */ |
68 | 588k | ml_parser->name = flb_sds_create(p->name); |
69 | 588k | if (!ml_parser->name) { |
70 | 30 | flb_ml_parser_destroy(ml_parser); |
71 | 30 | return NULL; |
72 | 30 | } |
73 | 588k | ml_parser->type = p->type; |
74 | | |
75 | | /* ENDSWITH/EQ optimization string */ |
76 | 588k | if (p->match_str) { |
77 | 183k | ml_parser->match_str = flb_sds_create(p->match_str); |
78 | 183k | if (!ml_parser->match_str) { |
79 | 7 | flb_ml_parser_destroy(ml_parser); |
80 | 7 | return NULL; |
81 | 7 | } |
82 | 183k | } |
83 | | |
84 | | /* sub-parser (immediate / delayed) */ |
85 | 588k | ml_parser->parser = p->parser_ctx; |
86 | 588k | if (p->parser_name) { |
87 | 555 | ml_parser->parser_name = flb_sds_create(p->parser_name); |
88 | 555 | if (!ml_parser->parser_name) { |
89 | 0 | flb_ml_parser_destroy(ml_parser); |
90 | 0 | return NULL; |
91 | 0 | } |
92 | 555 | } |
93 | | |
94 | | /* basic props */ |
95 | 588k | ml_parser->negate = p->negate; |
96 | 588k | ml_parser->flush_ms = (p->flush_ms > 0) ? p->flush_ms : FLB_ML_FLUSH_TIMEOUT; |
97 | | |
98 | | |
99 | | /* optional keys */ |
100 | 588k | if (p->key_content) { |
101 | 183k | ml_parser->key_content = flb_sds_create(p->key_content); |
102 | 183k | if (!ml_parser->key_content) { |
103 | 10 | flb_ml_parser_destroy(ml_parser); |
104 | 10 | return NULL; |
105 | 10 | } |
106 | 183k | } |
107 | 588k | if (p->key_group) { |
108 | 182k | ml_parser->key_group = flb_sds_create(p->key_group); |
109 | 182k | if (!ml_parser->key_group) { |
110 | 10 | flb_ml_parser_destroy(ml_parser); |
111 | 10 | return NULL; |
112 | 10 | } |
113 | 182k | } |
114 | 588k | if (p->key_pattern) { |
115 | 91.6k | ml_parser->key_pattern = flb_sds_create(p->key_pattern); |
116 | 91.6k | if (!ml_parser->key_pattern) { |
117 | 7 | flb_ml_parser_destroy(ml_parser); |
118 | 7 | return NULL; |
119 | 7 | } |
120 | 91.6k | } |
121 | | |
122 | | /* keep back-pointer to config for later rule init */ |
123 | 588k | ml_parser->config = ctx; |
124 | | |
125 | | /* link into registry after all initialization succeeds */ |
126 | 588k | mk_list_add(&ml_parser->_head, &ctx->multiline_parsers); |
127 | | |
128 | 588k | return ml_parser; |
129 | 588k | } |
130 | | |
131 | | int flb_ml_parser_init(struct flb_ml_parser *ml_parser) |
132 | 395k | { |
133 | 395k | int ret; |
134 | | |
135 | 395k | ret = flb_ml_rule_init(ml_parser); |
136 | 395k | if (ret == -1) { |
137 | 303 | return -1; |
138 | 303 | } |
139 | | |
140 | 395k | return 0; |
141 | 395k | } |
142 | | |
143 | | /* Create built-in multiline parsers */ |
144 | | int flb_ml_parser_builtin_create(struct flb_config *config) |
145 | 91.4k | { |
146 | 91.4k | struct flb_ml_parser *mlp; |
147 | 91.4k | int ret = -1; |
148 | | |
149 | | /* Docker */ |
150 | 91.4k | mlp = flb_ml_parser_docker(config); |
151 | 91.4k | if (!mlp) { |
152 | 46 | flb_error("[multiline] could not init 'docker' built-in parser"); |
153 | 46 | goto error; |
154 | 46 | } |
155 | | |
156 | | /* CRI */ |
157 | 91.3k | mlp = flb_ml_parser_cri(config); |
158 | 91.3k | if (!mlp) { |
159 | 49 | flb_error("[multiline] could not init 'cri' built-in parser"); |
160 | 49 | goto error; |
161 | 49 | } |
162 | | |
163 | | /* Java */ |
164 | 91.3k | mlp = flb_ml_parser_java(config, NULL); |
165 | 91.3k | if (!mlp) { |
166 | 238 | flb_error("[multiline] could not init 'java' built-in parser"); |
167 | 238 | goto error; |
168 | 238 | } |
169 | | |
170 | | /* Go */ |
171 | 91.0k | mlp = flb_ml_parser_go(config, NULL); |
172 | 91.0k | if (!mlp) { |
173 | 108 | flb_error("[multiline] could not init 'go' built-in parser"); |
174 | 108 | goto error; |
175 | 108 | } |
176 | | |
177 | | /* Ruby */ |
178 | 90.9k | mlp = flb_ml_parser_ruby(config, NULL); |
179 | 90.9k | if (!mlp) { |
180 | 38 | flb_error("[multiline] could not init 'ruby' built-in parser"); |
181 | 38 | goto error; |
182 | 38 | } |
183 | | |
184 | | /* Python */ |
185 | 90.9k | mlp = flb_ml_parser_python(config, NULL); |
186 | 90.9k | if (!mlp) { |
187 | 59 | flb_error("[multiline] could not init 'python' built-in parser"); |
188 | 59 | goto error; |
189 | 59 | } |
190 | | |
191 | 90.8k | ret = 0; |
192 | 90.8k | return ret; |
193 | | |
194 | 538 | error: |
195 | 538 | flb_ml_parser_destroy_all(&config->multiline_parsers); |
196 | 538 | return ret; |
197 | 90.9k | } |
198 | | |
199 | | /* Legacy positional-args API -> thin wrapper to params */ |
200 | | struct flb_ml_parser *flb_ml_parser_create(struct flb_config *ctx, |
201 | | char *name, |
202 | | int type, char *match_str, int negate, |
203 | | int flush_ms, |
204 | | char *key_content, |
205 | | char *key_group, |
206 | | char *key_pattern, |
207 | | struct flb_parser *parser_ctx, |
208 | | char *parser_name) |
209 | 588k | { |
210 | 588k | struct flb_ml_parser_params p = flb_ml_parser_params_default(name); |
211 | | |
212 | | /* override with legacy parameters */ |
213 | 588k | p.type = type; |
214 | 588k | p.match_str = match_str; |
215 | 588k | p.negate = negate; |
216 | 588k | p.flush_ms = flush_ms; |
217 | 588k | p.key_content = key_content; |
218 | 588k | p.key_group = key_group; |
219 | 588k | p.key_pattern = key_pattern; |
220 | 588k | p.parser_ctx = parser_ctx; |
221 | 588k | p.parser_name = parser_name; |
222 | | |
223 | 588k | return flb_ml_parser_create_params(ctx, &p); |
224 | 588k | } |
225 | | |
226 | | struct flb_ml_parser *flb_ml_parser_get(struct flb_config *ctx, char *name) |
227 | 55.4k | { |
228 | 55.4k | struct mk_list *head; |
229 | 55.4k | struct flb_ml_parser *ml_parser; |
230 | | |
231 | 177k | mk_list_foreach(head, &ctx->multiline_parsers) { |
232 | 177k | ml_parser = mk_list_entry(head, struct flb_ml_parser, _head); |
233 | 177k | if (strcasecmp(ml_parser->name, name) == 0) { |
234 | 55.4k | return ml_parser; |
235 | 55.4k | } |
236 | 177k | } |
237 | | |
238 | 0 | return NULL; |
239 | 55.4k | } |
240 | | |
241 | | int flb_ml_parser_instance_has_data(struct flb_ml_parser_ins *ins) |
242 | 0 | { |
243 | 0 | struct mk_list *head; |
244 | 0 | struct mk_list *head_group; |
245 | 0 | struct flb_ml_stream *st; |
246 | 0 | struct flb_ml_stream_group *st_group; |
247 | |
|
248 | 0 | mk_list_foreach(head, &ins->streams) { |
249 | 0 | st = mk_list_entry(head, struct flb_ml_stream, _head); |
250 | 0 | mk_list_foreach(head_group, &st->groups) { |
251 | 0 | st_group = mk_list_entry(head_group, struct flb_ml_stream_group, _head); |
252 | 0 | if (st_group->mp_sbuf.size > 0) { |
253 | 0 | return FLB_TRUE; |
254 | 0 | } |
255 | 0 | } |
256 | 0 | } |
257 | | |
258 | 0 | return FLB_FALSE; |
259 | 0 | } |
260 | | |
261 | | struct flb_ml_parser_ins *flb_ml_parser_instance_create(struct flb_ml *ml, |
262 | | char *name) |
263 | 55.4k | { |
264 | 55.4k | int ret; |
265 | 55.4k | struct flb_ml_parser_ins *ins; |
266 | 55.4k | struct flb_ml_parser *parser; |
267 | | |
268 | 55.4k | parser = flb_ml_parser_get(ml->config, name); |
269 | 55.4k | if (!parser) { |
270 | 0 | flb_error("[multiline] parser '%s' not registered", name); |
271 | 0 | return NULL; |
272 | 0 | } |
273 | | |
274 | 55.4k | ins = flb_calloc(1, sizeof(struct flb_ml_parser_ins)); |
275 | 55.4k | if (!ins) { |
276 | 0 | flb_errno(); |
277 | 0 | return NULL; |
278 | 0 | } |
279 | 55.4k | ins->last_stream_id = 0; |
280 | 55.4k | ins->ml_parser = parser; |
281 | 55.4k | mk_list_init(&ins->streams); |
282 | | |
283 | | /* Copy parent configuration */ |
284 | 55.4k | if (parser->key_content) { |
285 | 22.1k | ins->key_content = flb_sds_create(parser->key_content); |
286 | 22.1k | } |
287 | 55.4k | if (parser->key_pattern) { |
288 | 11.0k | ins->key_pattern = flb_sds_create(parser->key_pattern); |
289 | 11.0k | } |
290 | 55.4k | if (parser->key_group) { |
291 | 22.1k | ins->key_group = flb_sds_create(parser->key_group); |
292 | 22.1k | } |
293 | | |
294 | | /* Append this multiline parser instance to the active multiline group */ |
295 | 55.4k | ret = flb_ml_group_add_parser(ml, ins); |
296 | 55.4k | if (ret != 0) { |
297 | 0 | flb_error("[multiline] could not register parser '%s' on " |
298 | 0 | "multiline '%s 'group", name, ml->name); |
299 | 0 | flb_free(ins); |
300 | 0 | return NULL; |
301 | 0 | } |
302 | | |
303 | | /* |
304 | | * Update flush_interval for pending records on multiline context. We always |
305 | | * use the greater value found. |
306 | | */ |
307 | 55.4k | if (parser->flush_ms > ml->flush_ms) { |
308 | 11.0k | ml->flush_ms = parser->flush_ms; |
309 | 11.0k | } |
310 | | |
311 | 55.4k | return ins; |
312 | 55.4k | } |
313 | | |
314 | | /* Override a fixed parser property for the instance only*/ |
315 | | int flb_ml_parser_instance_set(struct flb_ml_parser_ins *p, char *prop, char *val) |
316 | 11.0k | { |
317 | 11.0k | if (strcasecmp(prop, "key_content") == 0) { |
318 | 11.0k | if (p->key_content) { |
319 | 0 | flb_sds_destroy(p->key_content); |
320 | 0 | } |
321 | 11.0k | p->key_content = flb_sds_create(val); |
322 | 11.0k | } |
323 | 0 | else if (strcasecmp(prop, "key_pattern") == 0) { |
324 | 0 | if (p->key_pattern) { |
325 | 0 | flb_sds_destroy(p->key_pattern); |
326 | 0 | } |
327 | 0 | p->key_pattern = flb_sds_create(val); |
328 | 0 | } |
329 | 0 | else if (strcasecmp(prop, "key_group") == 0) { |
330 | 0 | if (p->key_group) { |
331 | 0 | flb_sds_destroy(p->key_group); |
332 | 0 | } |
333 | 0 | p->key_group = flb_sds_create(val); |
334 | 0 | } |
335 | 0 | else { |
336 | 0 | return -1; |
337 | 0 | } |
338 | | |
339 | 11.0k | return 0; |
340 | 11.0k | } |
341 | | |
342 | | int flb_ml_parser_destroy(struct flb_ml_parser *ml_parser) |
343 | 588k | { |
344 | 588k | if (!ml_parser) { |
345 | 0 | return 0; |
346 | 0 | } |
347 | | |
348 | 588k | if (ml_parser->name) { |
349 | 588k | flb_sds_destroy(ml_parser->name); |
350 | 588k | } |
351 | | |
352 | 588k | if (ml_parser->parser_name) { |
353 | 555 | flb_sds_destroy(ml_parser->parser_name); |
354 | 555 | } |
355 | | |
356 | 588k | if (ml_parser->match_str) { |
357 | 183k | flb_sds_destroy(ml_parser->match_str); |
358 | 183k | } |
359 | 588k | if (ml_parser->key_content) { |
360 | 183k | flb_sds_destroy(ml_parser->key_content); |
361 | 183k | } |
362 | 588k | if (ml_parser->key_group) { |
363 | 182k | flb_sds_destroy(ml_parser->key_group); |
364 | 182k | } |
365 | 588k | if (ml_parser->key_pattern) { |
366 | 91.6k | flb_sds_destroy(ml_parser->key_pattern); |
367 | 91.6k | } |
368 | | |
369 | | /* Regex rules */ |
370 | 588k | flb_ml_rule_destroy_all(ml_parser); |
371 | | |
372 | | /* Unlink from struct flb_config->multiline_parsers */ |
373 | 588k | mk_list_del(&ml_parser->_head); |
374 | | |
375 | 588k | flb_free(ml_parser); |
376 | 588k | return 0; |
377 | 588k | } |
378 | | |
379 | | int flb_ml_parser_instance_destroy(struct flb_ml_parser_ins *ins) |
380 | 55.4k | { |
381 | 55.4k | struct mk_list *tmp; |
382 | 55.4k | struct mk_list *head; |
383 | 55.4k | struct flb_ml_stream *stream; |
384 | | |
385 | | /* Destroy streams */ |
386 | 148k | mk_list_foreach_safe(head, tmp, &ins->streams) { |
387 | 148k | stream = mk_list_entry(head, struct flb_ml_stream, _head); |
388 | 148k | flb_ml_stream_destroy(stream); |
389 | 148k | } |
390 | | |
391 | 55.4k | if (ins->key_content) { |
392 | 33.2k | flb_sds_destroy(ins->key_content); |
393 | 33.2k | } |
394 | 55.4k | if (ins->key_pattern) { |
395 | 11.0k | flb_sds_destroy(ins->key_pattern); |
396 | 11.0k | } |
397 | 55.4k | if (ins->key_group) { |
398 | 22.1k | flb_sds_destroy(ins->key_group); |
399 | 22.1k | } |
400 | | |
401 | 55.4k | flb_free(ins); |
402 | | |
403 | 55.4k | return 0; |
404 | 55.4k | } |
405 | | |
406 | | void flb_ml_parser_destroy_all(struct mk_list *list) |
407 | 123k | { |
408 | 123k | struct mk_list *tmp; |
409 | 123k | struct mk_list *head; |
410 | 123k | struct flb_ml_parser *parser; |
411 | | |
412 | 587k | mk_list_foreach_safe(head, tmp, list) { |
413 | | parser = mk_list_entry(head, struct flb_ml_parser, _head); |
414 | 587k | flb_ml_parser_destroy(parser); |
415 | 587k | } |
416 | 123k | } |