/src/suricata7/src/output-json-http.c
Line | Count | Source |
1 | | /* Copyright (C) 2007-2021 Open Information Security Foundation |
2 | | * |
3 | | * You can copy, redistribute or modify this Program under the terms of |
4 | | * the GNU General Public License version 2 as published by the Free |
5 | | * Software Foundation. |
6 | | * |
7 | | * This program is distributed in the hope that it will be useful, |
8 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
9 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
10 | | * GNU General Public License for more details. |
11 | | * |
12 | | * You should have received a copy of the GNU General Public License |
13 | | * version 2 along with this program; if not, write to the Free Software |
14 | | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA |
15 | | * 02110-1301, USA. |
16 | | */ |
17 | | |
18 | | /** |
19 | | * \file |
20 | | * |
21 | | * \author Tom DeCanio <td@npulsetech.com> |
22 | | * |
23 | | * Implements HTTP JSON logging portion of the engine. |
24 | | */ |
25 | | |
26 | | #include "suricata-common.h" |
27 | | #include "detect.h" |
28 | | #include "pkt-var.h" |
29 | | #include "conf.h" |
30 | | |
31 | | #include "threads.h" |
32 | | #include "threadvars.h" |
33 | | #include "tm-threads.h" |
34 | | |
35 | | #include "util-print.h" |
36 | | #include "util-unittest.h" |
37 | | |
38 | | #include "util-debug.h" |
39 | | |
40 | | #include "output.h" |
41 | | #include "app-layer-htp.h" |
42 | | #include "app-layer-htp-file.h" |
43 | | #include "app-layer-htp-xff.h" |
44 | | #include "app-layer.h" |
45 | | #include "app-layer-parser.h" |
46 | | #include "util-privs.h" |
47 | | #include "util-buffer.h" |
48 | | #include "util-proto-name.h" |
49 | | #include "util-logopenfile.h" |
50 | | #include "util-time.h" |
51 | | #include "output-json.h" |
52 | | #include "output-json-alert.h" |
53 | | #include "output-json-http.h" |
54 | | #include "util-byte.h" |
55 | | |
56 | | typedef struct LogHttpFileCtx_ { |
57 | | uint32_t flags; /** Store mode */ |
58 | | uint64_t fields;/** Store fields */ |
59 | | HttpXFFCfg *xff_cfg; |
60 | | HttpXFFCfg *parent_xff_cfg; |
61 | | OutputJsonCtx *eve_ctx; |
62 | | } LogHttpFileCtx; |
63 | | |
64 | | typedef struct JsonHttpLogThread_ { |
65 | | LogHttpFileCtx *httplog_ctx; |
66 | | uint32_t uri_cnt; |
67 | | OutputJsonThreadCtx *ctx; |
68 | | } JsonHttpLogThread; |
69 | | |
70 | 298k | #define MAX_SIZE_HEADER_NAME 256 |
71 | 298k | #define MAX_SIZE_HEADER_VALUE 2048 |
72 | | |
73 | 4 | #define LOG_HTTP_DEFAULT 0 |
74 | 161k | #define LOG_HTTP_EXTENDED 1 |
75 | | #define LOG_HTTP_REQUEST 2 /* request field */ |
76 | | #define LOG_HTTP_ARRAY 4 /* require array handling */ |
77 | 805k | #define LOG_HTTP_REQ_HEADERS 8 |
78 | 485k | #define LOG_HTTP_RES_HEADERS 16 |
79 | | |
80 | | typedef enum { |
81 | | HTTP_FIELD_ACCEPT = 0, |
82 | | HTTP_FIELD_ACCEPT_CHARSET, |
83 | | HTTP_FIELD_ACCEPT_ENCODING, |
84 | | HTTP_FIELD_ACCEPT_LANGUAGE, |
85 | | HTTP_FIELD_ACCEPT_DATETIME, |
86 | | HTTP_FIELD_AUTHORIZATION, |
87 | | HTTP_FIELD_CACHE_CONTROL, |
88 | | HTTP_FIELD_COOKIE, |
89 | | HTTP_FIELD_FROM, |
90 | | HTTP_FIELD_MAX_FORWARDS, |
91 | | HTTP_FIELD_ORIGIN, |
92 | | HTTP_FIELD_PRAGMA, |
93 | | HTTP_FIELD_PROXY_AUTHORIZATION, |
94 | | HTTP_FIELD_RANGE, |
95 | | HTTP_FIELD_TE, |
96 | | HTTP_FIELD_VIA, |
97 | | HTTP_FIELD_X_REQUESTED_WITH, |
98 | | HTTP_FIELD_DNT, |
99 | | HTTP_FIELD_X_FORWARDED_PROTO, |
100 | | HTTP_FIELD_X_AUTHENTICATED_USER, |
101 | | HTTP_FIELD_X_FLASH_VERSION, |
102 | | HTTP_FIELD_ACCEPT_RANGES, |
103 | | HTTP_FIELD_AGE, |
104 | | HTTP_FIELD_ALLOW, |
105 | | HTTP_FIELD_CONNECTION, |
106 | | HTTP_FIELD_CONTENT_ENCODING, |
107 | | HTTP_FIELD_CONTENT_LANGUAGE, |
108 | | HTTP_FIELD_CONTENT_LENGTH, |
109 | | HTTP_FIELD_CONTENT_LOCATION, |
110 | | HTTP_FIELD_CONTENT_MD5, |
111 | | HTTP_FIELD_CONTENT_RANGE, |
112 | | HTTP_FIELD_CONTENT_TYPE, |
113 | | HTTP_FIELD_DATE, |
114 | | HTTP_FIELD_ETAG, |
115 | | HTTP_FIELD_EXPIRES, |
116 | | HTTP_FIELD_LAST_MODIFIED, |
117 | | HTTP_FIELD_LINK, |
118 | | HTTP_FIELD_LOCATION, |
119 | | HTTP_FIELD_PROXY_AUTHENTICATE, |
120 | | HTTP_FIELD_REFERRER, |
121 | | HTTP_FIELD_REFRESH, |
122 | | HTTP_FIELD_RETRY_AFTER, |
123 | | HTTP_FIELD_SERVER, |
124 | | HTTP_FIELD_SET_COOKIE, |
125 | | HTTP_FIELD_TRAILER, |
126 | | HTTP_FIELD_TRANSFER_ENCODING, |
127 | | HTTP_FIELD_UPGRADE, |
128 | | HTTP_FIELD_VARY, |
129 | | HTTP_FIELD_WARNING, |
130 | | HTTP_FIELD_WWW_AUTHENTICATE, |
131 | | HTTP_FIELD_TRUE_CLIENT_IP, |
132 | | HTTP_FIELD_ORG_SRC_IP, |
133 | | HTTP_FIELD_X_BLUECOAT_VIA, |
134 | | HTTP_FIELD_SIZE |
135 | | } HttpField; |
136 | | |
137 | | struct { |
138 | | const char *config_field; |
139 | | const char *htp_field; |
140 | | uint32_t flags; |
141 | | } http_fields[] = { |
142 | | { "accept", "accept", LOG_HTTP_REQUEST }, |
143 | | { "accept_charset", "accept-charset", LOG_HTTP_REQUEST }, |
144 | | { "accept_encoding", "accept-encoding", LOG_HTTP_REQUEST }, |
145 | | { "accept_language", "accept-language", LOG_HTTP_REQUEST }, |
146 | | { "accept_datetime", "accept-datetime", LOG_HTTP_REQUEST }, |
147 | | { "authorization", "authorization", LOG_HTTP_REQUEST }, |
148 | | { "cache_control", "cache-control", LOG_HTTP_REQUEST }, |
149 | | { "cookie", "cookie", LOG_HTTP_REQUEST | LOG_HTTP_ARRAY }, |
150 | | { "from", "from", LOG_HTTP_REQUEST }, |
151 | | { "max_forwards", "max-forwards", LOG_HTTP_REQUEST }, |
152 | | { "origin", "origin", LOG_HTTP_REQUEST }, |
153 | | { "pragma", "pragma", LOG_HTTP_REQUEST }, |
154 | | { "proxy_authorization", "proxy-authorization", LOG_HTTP_REQUEST }, |
155 | | { "range", "range", LOG_HTTP_REQUEST }, |
156 | | { "te", "te", LOG_HTTP_REQUEST }, |
157 | | { "via", "via", LOG_HTTP_REQUEST }, |
158 | | { "x_requested_with", "x-requested-with", LOG_HTTP_REQUEST }, |
159 | | { "dnt", "dnt", LOG_HTTP_REQUEST }, |
160 | | { "x_forwarded_proto", "x-forwarded-proto", LOG_HTTP_REQUEST }, |
161 | | { "x_authenticated_user", "x-authenticated-user", LOG_HTTP_REQUEST }, |
162 | | { "x_flash_version", "x-flash-version", LOG_HTTP_REQUEST }, |
163 | | { "accept_range", "accept-range", 0 }, |
164 | | { "age", "age", 0 }, |
165 | | { "allow", "allow", 0 }, |
166 | | { "connection", "connection", 0 }, |
167 | | { "content_encoding", "content-encoding", 0 }, |
168 | | { "content_language", "content-language", 0 }, |
169 | | { "content_length", "content-length", 0 }, |
170 | | { "content_location", "content-location", 0 }, |
171 | | { "content_md5", "content-md5", 0 }, |
172 | | { "content_range", "content-range", 0 }, |
173 | | { "content_type", "content-type", 0 }, |
174 | | { "date", "date", 0 }, |
175 | | { "etag", "etags", 0 }, |
176 | | { "expires", "expires", 0 }, |
177 | | { "last_modified", "last-modified", 0 }, |
178 | | { "link", "link", 0 }, |
179 | | { "location", "location", 0 }, |
180 | | { "proxy_authenticate", "proxy-authenticate", 0 }, |
181 | | { "referer", "referer", LOG_HTTP_EXTENDED }, |
182 | | { "refresh", "refresh", 0 }, |
183 | | { "retry_after", "retry-after", 0 }, |
184 | | { "server", "server", 0 }, |
185 | | { "set_cookie", "set-cookie", 0 }, |
186 | | { "trailer", "trailer", 0 }, |
187 | | { "transfer_encoding", "transfer-encoding", 0 }, |
188 | | { "upgrade", "upgrade", 0 }, |
189 | | { "vary", "vary", 0 }, |
190 | | { "warning", "warning", 0 }, |
191 | | { "www_authenticate", "www-authenticate", 0 }, |
192 | | { "true_client_ip", "true-client-ip", LOG_HTTP_REQUEST }, |
193 | | { "org_src_ip", "org-src-ip", LOG_HTTP_REQUEST }, |
194 | | { "x_bluecoat_via", "x-bluecoat-via", LOG_HTTP_REQUEST }, |
195 | | }; |
196 | | |
197 | | static void EveHttpLogJSONBasic(JsonBuilder *js, htp_tx_t *tx) |
198 | 172k | { |
199 | | /* hostname */ |
200 | 172k | if (tx->request_hostname != NULL) { |
201 | 38.8k | jb_set_string_from_bytes( |
202 | 38.8k | js, "hostname", bstr_ptr(tx->request_hostname), bstr_len(tx->request_hostname)); |
203 | 38.8k | } |
204 | | |
205 | | /* port */ |
206 | | /* NOTE: this field will be set ONLY if the port is present in the |
207 | | * hostname. It may be present in the header "Host" or in the URL. |
208 | | * There is no connection (from the suricata point of view) between this |
209 | | * port and the TCP destination port of the flow. |
210 | | */ |
211 | 172k | if (tx->request_port_number >= 0) { |
212 | 85.6k | jb_set_uint(js, "http_port", tx->request_port_number); |
213 | 85.6k | } |
214 | | |
215 | | /* uri */ |
216 | 172k | if (tx->request_uri != NULL) { |
217 | 154k | jb_set_string_from_bytes(js, "url", bstr_ptr(tx->request_uri), bstr_len(tx->request_uri)); |
218 | 154k | } |
219 | | |
220 | 172k | if (tx->request_headers != NULL) { |
221 | | /* user agent */ |
222 | 172k | htp_header_t *h_user_agent = htp_table_get_c(tx->request_headers, "user-agent"); |
223 | 172k | if (h_user_agent != NULL) { |
224 | 26.5k | jb_set_string_from_bytes(js, "http_user_agent", bstr_ptr(h_user_agent->value), |
225 | 26.5k | bstr_len(h_user_agent->value)); |
226 | 26.5k | } |
227 | | |
228 | | /* x-forwarded-for */ |
229 | 172k | htp_header_t *h_x_forwarded_for = htp_table_get_c(tx->request_headers, "x-forwarded-for"); |
230 | 172k | if (h_x_forwarded_for != NULL) { |
231 | 693 | jb_set_string_from_bytes(js, "xff", bstr_ptr(h_x_forwarded_for->value), |
232 | 693 | bstr_len(h_x_forwarded_for->value)); |
233 | 693 | } |
234 | 172k | } |
235 | | |
236 | | /* content-type */ |
237 | 172k | if (tx->response_headers != NULL) { |
238 | 172k | htp_header_t *h_content_type = htp_table_get_c(tx->response_headers, "content-type"); |
239 | 172k | if (h_content_type != NULL) { |
240 | 27.0k | uint32_t len = (uint32_t)bstr_len(h_content_type->value); |
241 | 27.0k | const uint8_t *p = memchr(bstr_ptr(h_content_type->value), ';', len); |
242 | 27.0k | if (p != NULL) |
243 | 8.97k | len = (uint32_t)(p - bstr_ptr(h_content_type->value)); |
244 | 27.0k | jb_set_string_from_bytes( |
245 | 27.0k | js, "http_content_type", bstr_ptr(h_content_type->value), len); |
246 | 27.0k | } |
247 | 172k | htp_header_t *h_content_range = htp_table_get_c(tx->response_headers, "content-range"); |
248 | 172k | if (h_content_range != NULL) { |
249 | 5.67k | jb_open_object(js, "content_range"); |
250 | 5.67k | jb_set_string_from_bytes( |
251 | 5.67k | js, "raw", bstr_ptr(h_content_range->value), bstr_len(h_content_range->value)); |
252 | 5.67k | HTTPContentRange crparsed; |
253 | 5.67k | if (HTPParseContentRange(h_content_range->value, &crparsed) == 0) { |
254 | 5.38k | if (crparsed.start >= 0) |
255 | 5.38k | jb_set_uint(js, "start", crparsed.start); |
256 | 5.38k | if (crparsed.end >= 0) |
257 | 5.38k | jb_set_uint(js, "end", crparsed.end); |
258 | 5.38k | if (crparsed.size >= 0) |
259 | 5.38k | jb_set_uint(js, "size", crparsed.size); |
260 | 5.38k | } |
261 | 5.67k | jb_close(js); |
262 | 5.67k | } |
263 | 172k | } |
264 | 172k | } |
265 | | |
266 | | static void EveHttpLogJSONExtended(JsonBuilder *js, htp_tx_t *tx) |
267 | 172k | { |
268 | | /* referer */ |
269 | 172k | htp_header_t *h_referer = NULL; |
270 | 172k | if (tx->request_headers != NULL) { |
271 | 172k | h_referer = htp_table_get_c(tx->request_headers, "referer"); |
272 | 172k | } |
273 | 172k | if (h_referer != NULL) { |
274 | 7.65k | jb_set_string_from_bytes( |
275 | 7.65k | js, "http_refer", bstr_ptr(h_referer->value), bstr_len(h_referer->value)); |
276 | 7.65k | } |
277 | | |
278 | | /* method */ |
279 | 172k | if (tx->request_method != NULL) { |
280 | 105k | jb_set_string_from_bytes( |
281 | 105k | js, "http_method", bstr_ptr(tx->request_method), bstr_len(tx->request_method)); |
282 | 105k | } |
283 | | |
284 | | /* protocol */ |
285 | 172k | if (tx->request_protocol != NULL) { |
286 | 76.1k | jb_set_string_from_bytes( |
287 | 76.1k | js, "protocol", bstr_ptr(tx->request_protocol), bstr_len(tx->request_protocol)); |
288 | 76.1k | } |
289 | | |
290 | | /* response status */ |
291 | 172k | if (tx->response_status != NULL) { |
292 | 53.7k | const size_t status_size = bstr_len(tx->response_status) * 2 + 1; |
293 | 53.7k | char status_string[status_size]; |
294 | 53.7k | BytesToStringBuffer(bstr_ptr(tx->response_status), bstr_len(tx->response_status), |
295 | 53.7k | status_string, status_size); |
296 | 53.7k | unsigned int val = strtoul(status_string, NULL, 10); |
297 | 53.7k | jb_set_uint(js, "status", val); |
298 | | |
299 | 53.7k | htp_header_t *h_location = htp_table_get_c(tx->response_headers, "location"); |
300 | 53.7k | if (h_location != NULL) { |
301 | 721 | jb_set_string_from_bytes( |
302 | 721 | js, "redirect", bstr_ptr(h_location->value), bstr_len(h_location->value)); |
303 | 721 | } |
304 | 53.7k | } |
305 | | |
306 | | /* length */ |
307 | 172k | jb_set_uint(js, "length", tx->response_message_len); |
308 | 172k | } |
309 | | |
310 | | static void EveHttpLogJSONHeaders( |
311 | | JsonBuilder *js, uint32_t direction, htp_tx_t *tx, LogHttpFileCtx *http_ctx) |
312 | 160k | { |
313 | 160k | htp_table_t * headers = direction & LOG_HTTP_REQ_HEADERS ? |
314 | 80.0k | tx->request_headers : tx->response_headers; |
315 | 160k | char name[MAX_SIZE_HEADER_NAME] = {0}; |
316 | 160k | char value[MAX_SIZE_HEADER_VALUE] = {0}; |
317 | 160k | size_t n = htp_table_size(headers); |
318 | 160k | JsonBuilderMark mark = { 0, 0, 0 }; |
319 | 160k | jb_get_mark(js, &mark); |
320 | 160k | bool array_empty = true; |
321 | 160k | jb_open_array(js, direction & LOG_HTTP_REQ_HEADERS ? "request_headers" : "response_headers"); |
322 | 456k | for (size_t i = 0; i < n; i++) { |
323 | 296k | htp_header_t *h = htp_table_get_index(headers, i, NULL); |
324 | 296k | if ((http_ctx->flags & direction) == 0 && http_ctx->fields != 0) { |
325 | 0 | bool tolog = false; |
326 | 0 | for (HttpField f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++) { |
327 | 0 | if ((http_ctx->fields & (1ULL << f)) != 0) { |
328 | | /* prevent logging a field twice if extended logging is |
329 | | enabled */ |
330 | 0 | if (((http_ctx->flags & LOG_HTTP_EXTENDED) == 0) || |
331 | 0 | ((http_ctx->flags & LOG_HTTP_EXTENDED) != |
332 | 0 | (http_fields[f].flags & LOG_HTTP_EXTENDED))) { |
333 | 0 | if (bstr_cmp_c_nocase(h->name, http_fields[f].htp_field) == 0) { |
334 | 0 | tolog = true; |
335 | 0 | break; |
336 | 0 | } |
337 | 0 | } |
338 | 0 | } |
339 | 0 | } |
340 | 0 | if (!tolog) { |
341 | 0 | continue; |
342 | 0 | } |
343 | 0 | } |
344 | 296k | array_empty = false; |
345 | 296k | jb_start_object(js); |
346 | 296k | size_t size_name = bstr_len(h->name) < MAX_SIZE_HEADER_NAME - 1 ? |
347 | 295k | bstr_len(h->name) : MAX_SIZE_HEADER_NAME - 1; |
348 | 296k | memcpy(name, bstr_ptr(h->name), size_name); |
349 | 296k | name[size_name] = '\0'; |
350 | 296k | jb_set_string(js, "name", name); |
351 | 296k | size_t size_value = bstr_len(h->value) < MAX_SIZE_HEADER_VALUE - 1 ? |
352 | 295k | bstr_len(h->value) : MAX_SIZE_HEADER_VALUE - 1; |
353 | 296k | memcpy(value, bstr_ptr(h->value), size_value); |
354 | 296k | value[size_value] = '\0'; |
355 | 296k | jb_set_string(js, "value", value); |
356 | 296k | jb_close(js); |
357 | 296k | } |
358 | 160k | if (array_empty) { |
359 | 102k | jb_restore_mark(js, &mark); |
360 | 102k | } else { |
361 | | // Close array. |
362 | 57.7k | jb_close(js); |
363 | 57.7k | } |
364 | 160k | } |
365 | | |
366 | | static void BodyPrintableBuffer(JsonBuilder *js, HtpBody *body, const char *key) |
367 | 183k | { |
368 | 183k | if (body->sb != NULL && body->sb->region.buf != NULL) { |
369 | 90.2k | const uint8_t *body_data; |
370 | 90.2k | uint32_t body_data_len; |
371 | 90.2k | uint64_t body_offset; |
372 | | |
373 | 90.2k | if (StreamingBufferGetData(body->sb, &body_data, |
374 | 90.2k | &body_data_len, &body_offset) == 0) { |
375 | 0 | return; |
376 | 0 | } |
377 | | |
378 | 90.2k | SCJbSetPrintAsciiString(js, key, body_data, body_data_len); |
379 | 90.2k | } |
380 | 183k | } |
381 | | |
382 | | void EveHttpLogJSONBodyPrintable(JsonBuilder *js, Flow *f, uint64_t tx_id) |
383 | 40.9k | { |
384 | 40.9k | HtpState *htp_state = (HtpState *)FlowGetAppState(f); |
385 | 40.9k | if (htp_state) { |
386 | 40.9k | htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, htp_state, tx_id); |
387 | 40.9k | if (tx) { |
388 | 40.9k | HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); |
389 | 40.9k | if (htud != NULL) { |
390 | 40.9k | BodyPrintableBuffer(js, &htud->request_body, "http_request_body_printable"); |
391 | 40.9k | BodyPrintableBuffer(js, &htud->response_body, "http_response_body_printable"); |
392 | 40.9k | } |
393 | 40.9k | } |
394 | 40.9k | } |
395 | 40.9k | } |
396 | | |
397 | | static void BodyBase64Buffer(JsonBuilder *js, HtpBody *body, const char *key) |
398 | 183k | { |
399 | 183k | if (body->sb != NULL && body->sb->region.buf != NULL) { |
400 | 90.2k | const uint8_t *body_data; |
401 | 90.2k | uint32_t body_data_len; |
402 | 90.2k | uint64_t body_offset; |
403 | | |
404 | 90.2k | if (StreamingBufferGetData(body->sb, &body_data, |
405 | 90.2k | &body_data_len, &body_offset) == 0) { |
406 | 0 | return; |
407 | 0 | } |
408 | | |
409 | 90.2k | jb_set_base64(js, key, body_data, body_data_len); |
410 | 90.2k | } |
411 | 183k | } |
412 | | |
413 | | void EveHttpLogJSONBodyBase64(JsonBuilder *js, Flow *f, uint64_t tx_id) |
414 | 40.9k | { |
415 | 40.9k | HtpState *htp_state = (HtpState *)FlowGetAppState(f); |
416 | 40.9k | if (htp_state) { |
417 | 40.9k | htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, htp_state, tx_id); |
418 | 40.9k | if (tx) { |
419 | 40.9k | HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); |
420 | 40.9k | if (htud != NULL) { |
421 | 40.9k | BodyBase64Buffer(js, &htud->request_body, "http_request_body"); |
422 | 40.9k | BodyBase64Buffer(js, &htud->response_body, "http_response_body"); |
423 | 40.9k | } |
424 | 40.9k | } |
425 | 40.9k | } |
426 | 40.9k | } |
427 | | |
428 | | /* JSON format logging */ |
429 | | static void EveHttpLogJSON(JsonHttpLogThread *aft, JsonBuilder *js, htp_tx_t *tx, uint64_t tx_id) |
430 | 161k | { |
431 | 161k | LogHttpFileCtx *http_ctx = aft->httplog_ctx; |
432 | 161k | jb_open_object(js, "http"); |
433 | | |
434 | 161k | EveHttpLogJSONBasic(js, tx); |
435 | 161k | if (http_ctx->flags & LOG_HTTP_EXTENDED) |
436 | 161k | EveHttpLogJSONExtended(js, tx); |
437 | 161k | if (http_ctx->flags & LOG_HTTP_REQ_HEADERS || http_ctx->fields != 0) |
438 | 161k | EveHttpLogJSONHeaders(js, LOG_HTTP_REQ_HEADERS, tx, http_ctx); |
439 | 161k | if (http_ctx->flags & LOG_HTTP_RES_HEADERS || http_ctx->fields != 0) |
440 | 161k | EveHttpLogJSONHeaders(js, LOG_HTTP_RES_HEADERS, tx, http_ctx); |
441 | | |
442 | 161k | jb_close(js); |
443 | 161k | } |
444 | | |
445 | | static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id) |
446 | 161k | { |
447 | 161k | SCEnter(); |
448 | | |
449 | 161k | htp_tx_t *tx = txptr; |
450 | 161k | JsonHttpLogThread *jhl = (JsonHttpLogThread *)thread_data; |
451 | | |
452 | 161k | JsonBuilder *js = CreateEveHeaderWithTxId( |
453 | 161k | p, LOG_DIR_FLOW, "http", NULL, tx_id, jhl->httplog_ctx->eve_ctx); |
454 | 161k | if (unlikely(js == NULL)) |
455 | 0 | return TM_ECODE_OK; |
456 | | |
457 | 161k | SCLogDebug("got a HTTP request and now logging !!"); |
458 | | |
459 | 161k | EveHttpLogJSON(jhl, js, tx, tx_id); |
460 | 161k | HttpXFFCfg *xff_cfg = jhl->httplog_ctx->xff_cfg != NULL ? |
461 | 161k | jhl->httplog_ctx->xff_cfg : jhl->httplog_ctx->parent_xff_cfg; |
462 | | |
463 | | /* xff header */ |
464 | 161k | if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) { |
465 | 161k | int have_xff_ip = 0; |
466 | 161k | char buffer[XFF_MAXLEN]; |
467 | | |
468 | 161k | have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, buffer, XFF_MAXLEN); |
469 | | |
470 | 161k | if (have_xff_ip) { |
471 | 485 | if (xff_cfg->flags & XFF_EXTRADATA) { |
472 | 485 | jb_set_string(js, "xff", buffer); |
473 | 485 | } |
474 | 0 | else if (xff_cfg->flags & XFF_OVERWRITE) { |
475 | 0 | if (p->flowflags & FLOW_PKT_TOCLIENT) { |
476 | 0 | jb_set_string(js, "dest_ip", buffer); |
477 | 0 | } else { |
478 | 0 | jb_set_string(js, "src_ip", buffer); |
479 | 0 | } |
480 | 0 | } |
481 | 485 | } |
482 | 161k | } |
483 | | |
484 | 161k | OutputJsonBuilderBuffer(js, jhl->ctx); |
485 | 161k | jb_free(js); |
486 | | |
487 | 161k | SCReturnInt(TM_ECODE_OK); |
488 | 161k | } |
489 | | |
490 | | bool EveHttpAddMetadata(const Flow *f, uint64_t tx_id, JsonBuilder *js) |
491 | 207k | { |
492 | 207k | HtpState *htp_state = (HtpState *)FlowGetAppState(f); |
493 | 207k | if (htp_state) { |
494 | 207k | htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, htp_state, tx_id); |
495 | | |
496 | 207k | if (tx) { |
497 | 205k | EveHttpLogJSONBasic(js, tx); |
498 | 205k | EveHttpLogJSONExtended(js, tx); |
499 | 205k | return true; |
500 | 205k | } |
501 | 207k | } |
502 | | |
503 | 2.76k | return false; |
504 | 207k | } |
505 | | |
506 | | static void OutputHttpLogDeinitSub(OutputCtx *output_ctx) |
507 | 0 | { |
508 | 0 | LogHttpFileCtx *http_ctx = output_ctx->data; |
509 | 0 | if (http_ctx->xff_cfg) { |
510 | 0 | SCFree(http_ctx->xff_cfg); |
511 | 0 | } |
512 | 0 | SCFree(http_ctx); |
513 | 0 | SCFree(output_ctx); |
514 | 0 | } |
515 | | |
516 | | static OutputInitResult OutputHttpLogInitSub(ConfNode *conf, OutputCtx *parent_ctx) |
517 | 4 | { |
518 | 4 | OutputInitResult result = { NULL, false }; |
519 | 4 | OutputJsonCtx *ojc = parent_ctx->data; |
520 | | |
521 | 4 | LogHttpFileCtx *http_ctx = SCCalloc(1, sizeof(LogHttpFileCtx)); |
522 | 4 | if (unlikely(http_ctx == NULL)) |
523 | 0 | return result; |
524 | 4 | memset(http_ctx, 0x00, sizeof(*http_ctx)); |
525 | | |
526 | 4 | OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); |
527 | 4 | if (unlikely(output_ctx == NULL)) { |
528 | 0 | SCFree(http_ctx); |
529 | 0 | return result; |
530 | 0 | } |
531 | | |
532 | 4 | http_ctx->flags = LOG_HTTP_DEFAULT; |
533 | 4 | http_ctx->eve_ctx = ojc; |
534 | | |
535 | 4 | if (conf) { |
536 | 4 | const char *extended = ConfNodeLookupChildValue(conf, "extended"); |
537 | | |
538 | 4 | if (extended != NULL) { |
539 | 4 | if (ConfValIsTrue(extended)) { |
540 | 4 | http_ctx->flags = LOG_HTTP_EXTENDED; |
541 | 4 | } |
542 | 4 | } |
543 | | |
544 | 4 | const char *all_headers = ConfNodeLookupChildValue(conf, "dump-all-headers"); |
545 | 4 | if (all_headers != NULL) { |
546 | 4 | if (strncmp(all_headers, "both", 4) == 0) { |
547 | 4 | http_ctx->flags |= LOG_HTTP_REQ_HEADERS; |
548 | 4 | http_ctx->flags |= LOG_HTTP_RES_HEADERS; |
549 | 4 | } else if (strncmp(all_headers, "request", 7) == 0) { |
550 | 0 | http_ctx->flags |= LOG_HTTP_REQ_HEADERS; |
551 | 0 | } else if (strncmp(all_headers, "response", 8) == 0) { |
552 | 0 | http_ctx->flags |= LOG_HTTP_RES_HEADERS; |
553 | 0 | } |
554 | 4 | } |
555 | 4 | ConfNode *custom; |
556 | 4 | if ((custom = ConfNodeLookupChild(conf, "custom")) != NULL) { |
557 | 0 | if ((http_ctx->flags & (LOG_HTTP_REQ_HEADERS | LOG_HTTP_RES_HEADERS)) == |
558 | 0 | (LOG_HTTP_REQ_HEADERS | LOG_HTTP_RES_HEADERS)) { |
559 | 0 | SCLogWarning("No need for custom as dump-all-headers is already present"); |
560 | 0 | } |
561 | 0 | ConfNode *field; |
562 | 0 | TAILQ_FOREACH (field, &custom->head, next) { |
563 | 0 | HttpField f; |
564 | 0 | for (f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++) { |
565 | 0 | if ((strcmp(http_fields[f].config_field, field->val) == 0) || |
566 | 0 | (strcasecmp(http_fields[f].htp_field, field->val) == 0)) { |
567 | 0 | http_ctx->fields |= (1ULL << f); |
568 | 0 | break; |
569 | 0 | } |
570 | 0 | } |
571 | 0 | } |
572 | 0 | } |
573 | 4 | } |
574 | | |
575 | 4 | if (conf != NULL && ConfNodeLookupChild(conf, "xff") != NULL) { |
576 | 0 | http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg)); |
577 | 0 | if (http_ctx->xff_cfg != NULL) { |
578 | 0 | HttpXFFGetCfg(conf, http_ctx->xff_cfg); |
579 | 0 | } |
580 | 4 | } else if (ojc->xff_cfg) { |
581 | 4 | http_ctx->parent_xff_cfg = ojc->xff_cfg; |
582 | 4 | } |
583 | | |
584 | 4 | output_ctx->data = http_ctx; |
585 | 4 | output_ctx->DeInit = OutputHttpLogDeinitSub; |
586 | | |
587 | | /* enable the logger for the app layer */ |
588 | 4 | AppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_HTTP1); |
589 | | |
590 | 4 | result.ctx = output_ctx; |
591 | 4 | result.ok = true; |
592 | 4 | return result; |
593 | 4 | } |
594 | | |
595 | | static TmEcode JsonHttpLogThreadInit(ThreadVars *t, const void *initdata, void **data) |
596 | 4 | { |
597 | 4 | JsonHttpLogThread *aft = SCCalloc(1, sizeof(JsonHttpLogThread)); |
598 | 4 | if (unlikely(aft == NULL)) |
599 | 0 | return TM_ECODE_FAILED; |
600 | | |
601 | 4 | if(initdata == NULL) |
602 | 0 | { |
603 | 0 | SCLogDebug("Error getting context for EveLogHTTP. \"initdata\" argument NULL"); |
604 | 0 | goto error_exit; |
605 | 0 | } |
606 | | |
607 | | /* Use the Output Context (file pointer and mutex) */ |
608 | 4 | aft->httplog_ctx = ((OutputCtx *)initdata)->data; //TODO |
609 | | |
610 | 4 | aft->ctx = CreateEveThreadCtx(t, aft->httplog_ctx->eve_ctx); |
611 | 4 | if (!aft->ctx) { |
612 | 0 | goto error_exit; |
613 | 0 | } |
614 | | |
615 | 4 | *data = (void *)aft; |
616 | 4 | return TM_ECODE_OK; |
617 | | |
618 | 0 | error_exit: |
619 | 0 | SCFree(aft); |
620 | 0 | return TM_ECODE_FAILED; |
621 | 4 | } |
622 | | |
623 | | static TmEcode JsonHttpLogThreadDeinit(ThreadVars *t, void *data) |
624 | 0 | { |
625 | 0 | JsonHttpLogThread *aft = (JsonHttpLogThread *)data; |
626 | 0 | if (aft == NULL) { |
627 | 0 | return TM_ECODE_OK; |
628 | 0 | } |
629 | | |
630 | 0 | FreeEveThreadCtx(aft->ctx); |
631 | | |
632 | | /* clear memory */ |
633 | 0 | memset(aft, 0, sizeof(JsonHttpLogThread)); |
634 | |
|
635 | 0 | SCFree(aft); |
636 | 0 | return TM_ECODE_OK; |
637 | 0 | } |
638 | | |
639 | | void JsonHttpLogRegister (void) |
640 | 71 | { |
641 | | /* register as child of eve-log */ |
642 | 71 | OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonHttpLog", "eve-log.http", |
643 | 71 | OutputHttpLogInitSub, ALPROTO_HTTP1, JsonHttpLogger, JsonHttpLogThreadInit, |
644 | | JsonHttpLogThreadDeinit, NULL); |
645 | 71 | } |