/src/suricata8/src/output-json-http.c
Line | Count | Source |
1 | | /* Copyright (C) 2007-2021 Open Information Security Foundation |
2 | | * |
3 | | * You can copy, redistribute or modify this Program under the terms of |
4 | | * the GNU General Public License version 2 as published by the Free |
5 | | * Software Foundation. |
6 | | * |
7 | | * This program is distributed in the hope that it will be useful, |
8 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
9 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
10 | | * GNU General Public License for more details. |
11 | | * |
12 | | * You should have received a copy of the GNU General Public License |
13 | | * version 2 along with this program; if not, write to the Free Software |
14 | | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA |
15 | | * 02110-1301, USA. |
16 | | */ |
17 | | |
18 | | /** |
19 | | * \file |
20 | | * |
21 | | * \author Tom DeCanio <td@npulsetech.com> |
22 | | * |
23 | | * Implements HTTP JSON logging portion of the engine. |
24 | | */ |
25 | | |
26 | | #include "suricata-common.h" |
27 | | #include "detect.h" |
28 | | #include "pkt-var.h" |
29 | | #include "conf.h" |
30 | | |
31 | | #include "threads.h" |
32 | | #include "threadvars.h" |
33 | | #include "tm-threads.h" |
34 | | |
35 | | #include "util-print.h" |
36 | | #include "util-unittest.h" |
37 | | |
38 | | #include "util-debug.h" |
39 | | |
40 | | #include "output.h" |
41 | | #include "app-layer-htp.h" |
42 | | #include "app-layer-htp-file.h" |
43 | | #include "app-layer-htp-xff.h" |
44 | | #include "app-layer.h" |
45 | | #include "app-layer-parser.h" |
46 | | #include "util-privs.h" |
47 | | #include "util-buffer.h" |
48 | | #include "util-proto-name.h" |
49 | | #include "util-logopenfile.h" |
50 | | #include "util-time.h" |
51 | | #include "output-json.h" |
52 | | #include "output-json-alert.h" |
53 | | #include "output-json-http.h" |
54 | | #include "util-byte.h" |
55 | | |
56 | | typedef struct LogHttpFileCtx_ { |
57 | | uint32_t flags; /** Store mode */ |
58 | | uint64_t fields;/** Store fields */ |
59 | | HttpXFFCfg *xff_cfg; |
60 | | HttpXFFCfg *parent_xff_cfg; |
61 | | OutputJsonCtx *eve_ctx; |
62 | | } LogHttpFileCtx; |
63 | | |
64 | | typedef struct JsonHttpLogThread_ { |
65 | | LogHttpFileCtx *httplog_ctx; |
66 | | uint32_t uri_cnt; |
67 | | OutputJsonThreadCtx *ctx; |
68 | | } JsonHttpLogThread; |
69 | | |
70 | 533k | #define MAX_SIZE_HEADER_NAME 256 |
71 | 533k | #define MAX_SIZE_HEADER_VALUE 2048 |
72 | | |
73 | 4 | #define LOG_HTTP_DEFAULT 0 |
74 | 133k | #define LOG_HTTP_EXTENDED 1 |
75 | | #define LOG_HTTP_REQUEST 2 /* request field */ |
76 | | #define LOG_HTTP_ARRAY 4 /* require array handling */ |
77 | 934k | #define LOG_HTTP_REQ_HEADERS 8 |
78 | 400k | #define LOG_HTTP_RES_HEADERS 16 |
79 | | |
80 | | typedef enum { |
81 | | HTTP_FIELD_ACCEPT = 0, |
82 | | HTTP_FIELD_ACCEPT_CHARSET, |
83 | | HTTP_FIELD_ACCEPT_ENCODING, |
84 | | HTTP_FIELD_ACCEPT_LANGUAGE, |
85 | | HTTP_FIELD_ACCEPT_DATETIME, |
86 | | HTTP_FIELD_AUTHORIZATION, |
87 | | HTTP_FIELD_CACHE_CONTROL, |
88 | | HTTP_FIELD_COOKIE, |
89 | | HTTP_FIELD_FROM, |
90 | | HTTP_FIELD_MAX_FORWARDS, |
91 | | HTTP_FIELD_ORIGIN, |
92 | | HTTP_FIELD_PRAGMA, |
93 | | HTTP_FIELD_PROXY_AUTHORIZATION, |
94 | | HTTP_FIELD_RANGE, |
95 | | HTTP_FIELD_TE, |
96 | | HTTP_FIELD_VIA, |
97 | | HTTP_FIELD_X_REQUESTED_WITH, |
98 | | HTTP_FIELD_DNT, |
99 | | HTTP_FIELD_X_FORWARDED_PROTO, |
100 | | HTTP_FIELD_X_AUTHENTICATED_USER, |
101 | | HTTP_FIELD_X_FLASH_VERSION, |
102 | | HTTP_FIELD_ACCEPT_RANGES, |
103 | | HTTP_FIELD_AGE, |
104 | | HTTP_FIELD_ALLOW, |
105 | | HTTP_FIELD_CONNECTION, |
106 | | HTTP_FIELD_CONTENT_ENCODING, |
107 | | HTTP_FIELD_CONTENT_LANGUAGE, |
108 | | HTTP_FIELD_CONTENT_LENGTH, |
109 | | HTTP_FIELD_CONTENT_LOCATION, |
110 | | HTTP_FIELD_CONTENT_MD5, |
111 | | HTTP_FIELD_CONTENT_RANGE, |
112 | | HTTP_FIELD_CONTENT_TYPE, |
113 | | HTTP_FIELD_DATE, |
114 | | HTTP_FIELD_ETAG, |
115 | | HTTP_FIELD_EXPIRES, |
116 | | HTTP_FIELD_LAST_MODIFIED, |
117 | | HTTP_FIELD_LINK, |
118 | | HTTP_FIELD_LOCATION, |
119 | | HTTP_FIELD_PROXY_AUTHENTICATE, |
120 | | HTTP_FIELD_REFERRER, |
121 | | HTTP_FIELD_REFRESH, |
122 | | HTTP_FIELD_RETRY_AFTER, |
123 | | HTTP_FIELD_SERVER, |
124 | | HTTP_FIELD_SET_COOKIE, |
125 | | HTTP_FIELD_TRAILER, |
126 | | HTTP_FIELD_TRANSFER_ENCODING, |
127 | | HTTP_FIELD_UPGRADE, |
128 | | HTTP_FIELD_VARY, |
129 | | HTTP_FIELD_WARNING, |
130 | | HTTP_FIELD_WWW_AUTHENTICATE, |
131 | | HTTP_FIELD_TRUE_CLIENT_IP, |
132 | | HTTP_FIELD_ORG_SRC_IP, |
133 | | HTTP_FIELD_X_BLUECOAT_VIA, |
134 | | HTTP_FIELD_SIZE |
135 | | } HttpField; |
136 | | |
137 | | struct { |
138 | | const char *config_field; |
139 | | const char *htp_field; |
140 | | uint32_t flags; |
141 | | } http_fields[] = { |
142 | | { "accept", "accept", LOG_HTTP_REQUEST }, |
143 | | { "accept_charset", "accept-charset", LOG_HTTP_REQUEST }, |
144 | | { "accept_encoding", "accept-encoding", LOG_HTTP_REQUEST }, |
145 | | { "accept_language", "accept-language", LOG_HTTP_REQUEST }, |
146 | | { "accept_datetime", "accept-datetime", LOG_HTTP_REQUEST }, |
147 | | { "authorization", "authorization", LOG_HTTP_REQUEST }, |
148 | | { "cache_control", "cache-control", LOG_HTTP_REQUEST }, |
149 | | { "cookie", "cookie", LOG_HTTP_REQUEST | LOG_HTTP_ARRAY }, |
150 | | { "from", "from", LOG_HTTP_REQUEST }, |
151 | | { "max_forwards", "max-forwards", LOG_HTTP_REQUEST }, |
152 | | { "origin", "origin", LOG_HTTP_REQUEST }, |
153 | | { "pragma", "pragma", LOG_HTTP_REQUEST }, |
154 | | { "proxy_authorization", "proxy-authorization", LOG_HTTP_REQUEST }, |
155 | | { "range", "range", LOG_HTTP_REQUEST }, |
156 | | { "te", "te", LOG_HTTP_REQUEST }, |
157 | | { "via", "via", LOG_HTTP_REQUEST }, |
158 | | { "x_requested_with", "x-requested-with", LOG_HTTP_REQUEST }, |
159 | | { "dnt", "dnt", LOG_HTTP_REQUEST }, |
160 | | { "x_forwarded_proto", "x-forwarded-proto", LOG_HTTP_REQUEST }, |
161 | | { "x_authenticated_user", "x-authenticated-user", LOG_HTTP_REQUEST }, |
162 | | { "x_flash_version", "x-flash-version", LOG_HTTP_REQUEST }, |
163 | | { "accept_range", "accept-range", 0 }, |
164 | | { "age", "age", 0 }, |
165 | | { "allow", "allow", 0 }, |
166 | | { "connection", "connection", 0 }, |
167 | | { "content_encoding", "content-encoding", 0 }, |
168 | | { "content_language", "content-language", 0 }, |
169 | | { "content_length", "content-length", 0 }, |
170 | | { "content_location", "content-location", 0 }, |
171 | | { "content_md5", "content-md5", 0 }, |
172 | | { "content_range", "content-range", 0 }, |
173 | | { "content_type", "content-type", 0 }, |
174 | | { "date", "date", 0 }, |
175 | | { "etag", "etags", 0 }, |
176 | | { "expires", "expires", 0 }, |
177 | | { "last_modified", "last-modified", 0 }, |
178 | | { "link", "link", 0 }, |
179 | | { "location", "location", 0 }, |
180 | | { "proxy_authenticate", "proxy-authenticate", 0 }, |
181 | | { "referer", "referer", LOG_HTTP_EXTENDED }, |
182 | | { "refresh", "refresh", 0 }, |
183 | | { "retry_after", "retry-after", 0 }, |
184 | | { "server", "server", 0 }, |
185 | | { "set_cookie", "set-cookie", 0 }, |
186 | | { "trailer", "trailer", 0 }, |
187 | | { "transfer_encoding", "transfer-encoding", 0 }, |
188 | | { "upgrade", "upgrade", 0 }, |
189 | | { "vary", "vary", 0 }, |
190 | | { "warning", "warning", 0 }, |
191 | | { "www_authenticate", "www-authenticate", 0 }, |
192 | | { "true_client_ip", "true-client-ip", LOG_HTTP_REQUEST }, |
193 | | { "org_src_ip", "org-src-ip", LOG_HTTP_REQUEST }, |
194 | | { "x_bluecoat_via", "x-bluecoat-via", LOG_HTTP_REQUEST }, |
195 | | }; |
196 | | |
197 | | static void EveHttpLogJSONBasic(SCJsonBuilder *js, htp_tx_t *tx) |
198 | 302k | { |
199 | | /* hostname */ |
200 | 302k | if (htp_tx_request_hostname(tx) != NULL) { |
201 | 77.2k | SCJbSetStringFromBytes(js, "hostname", bstr_ptr(htp_tx_request_hostname(tx)), |
202 | 77.2k | (uint32_t)bstr_len(htp_tx_request_hostname(tx))); |
203 | 77.2k | } |
204 | | |
205 | | /* port */ |
206 | | /* NOTE: this field will be set ONLY if the port is present in the |
207 | | * hostname. It may be present in the header "Host" or in the URL. |
208 | | * There is no connection (from the suricata point of view) between this |
209 | | * port and the TCP destination port of the flow. |
210 | | */ |
211 | 302k | if (htp_tx_request_port_number(tx) >= 0) { |
212 | 11.0k | SCJbSetUint(js, "http_port", htp_tx_request_port_number(tx)); |
213 | 11.0k | } |
214 | | |
215 | | /* uri */ |
216 | 302k | if (htp_tx_request_uri(tx) != NULL) { |
217 | 291k | SCJbSetStringFromBytes(js, "url", bstr_ptr(htp_tx_request_uri(tx)), |
218 | 291k | (uint32_t)bstr_len(htp_tx_request_uri(tx))); |
219 | 291k | } |
220 | | |
221 | 302k | if (htp_tx_request_headers(tx) != NULL) { |
222 | | /* user agent */ |
223 | 302k | const htp_header_t *h_user_agent = htp_tx_request_header(tx, "user-agent"); |
224 | 302k | if (h_user_agent != NULL) { |
225 | 41.0k | SCJbSetStringFromBytes(js, "http_user_agent", htp_header_value_ptr(h_user_agent), |
226 | 41.0k | (uint32_t)htp_header_value_len(h_user_agent)); |
227 | 41.0k | } |
228 | | |
229 | | /* x-forwarded-for */ |
230 | 302k | const htp_header_t *h_x_forwarded_for = htp_tx_request_header(tx, "x-forwarded-for"); |
231 | 302k | if (h_x_forwarded_for != NULL) { |
232 | 1.10k | SCJbSetStringFromBytes(js, "xff", htp_header_value_ptr(h_x_forwarded_for), |
233 | 1.10k | (uint32_t)htp_header_value_len(h_x_forwarded_for)); |
234 | 1.10k | } |
235 | 302k | } |
236 | | |
237 | | /* content-type */ |
238 | 302k | if (htp_tx_response_headers(tx) != NULL) { |
239 | 302k | const htp_header_t *h_content_type = htp_tx_response_header(tx, "content-type"); |
240 | 302k | if (h_content_type != NULL) { |
241 | 57.0k | uint32_t len = (uint32_t)htp_header_value_len(h_content_type); |
242 | 57.0k | const uint8_t *p = memchr(htp_header_value_ptr(h_content_type), ';', len); |
243 | 57.0k | if (p != NULL) |
244 | 9.84k | len = (uint32_t)(p - htp_header_value_ptr(h_content_type)); |
245 | 57.0k | SCJbSetStringFromBytes( |
246 | 57.0k | js, "http_content_type", htp_header_value_ptr(h_content_type), len); |
247 | 57.0k | } |
248 | 302k | const htp_header_t *h_content_range = htp_tx_response_header(tx, "content-range"); |
249 | 302k | if (h_content_range != NULL) { |
250 | 40.9k | SCJbOpenObject(js, "content_range"); |
251 | 40.9k | SCJbSetStringFromBytes(js, "raw", htp_header_value_ptr(h_content_range), |
252 | 40.9k | (uint32_t)htp_header_value_len(h_content_range)); |
253 | 40.9k | HTTPContentRange crparsed; |
254 | 40.9k | if (HTPParseContentRange(htp_header_value(h_content_range), &crparsed) == 0) { |
255 | 38.4k | if (crparsed.start >= 0) |
256 | 38.4k | SCJbSetUint(js, "start", crparsed.start); |
257 | 38.4k | if (crparsed.end >= 0) |
258 | 38.4k | SCJbSetUint(js, "end", crparsed.end); |
259 | 38.4k | if (crparsed.size >= 0) |
260 | 38.4k | SCJbSetUint(js, "size", crparsed.size); |
261 | 38.4k | } |
262 | 40.9k | SCJbClose(js); |
263 | 40.9k | } |
264 | 302k | } |
265 | 302k | } |
266 | | |
267 | | static void EveHttpLogJSONExtended(SCJsonBuilder *js, htp_tx_t *tx) |
268 | 302k | { |
269 | | /* referer */ |
270 | 302k | const htp_header_t *h_referer = NULL; |
271 | 302k | if (htp_tx_request_headers(tx) != NULL) { |
272 | 302k | h_referer = htp_tx_request_header(tx, "referer"); |
273 | 302k | } |
274 | 302k | if (h_referer != NULL) { |
275 | 6.81k | SCJbSetStringFromBytes(js, "http_refer", htp_header_value_ptr(h_referer), |
276 | 6.81k | (uint32_t)htp_header_value_len(h_referer)); |
277 | 6.81k | } |
278 | | |
279 | | /* method */ |
280 | 302k | if (htp_tx_request_method(tx) != NULL) { |
281 | 145k | SCJbSetStringFromBytes(js, "http_method", bstr_ptr(htp_tx_request_method(tx)), |
282 | 145k | (uint32_t)bstr_len(htp_tx_request_method(tx))); |
283 | 145k | } |
284 | | |
285 | | /* protocol */ |
286 | 302k | if (htp_tx_request_protocol(tx) != NULL) { |
287 | 109k | SCJbSetStringFromBytes(js, "protocol", bstr_ptr(htp_tx_request_protocol(tx)), |
288 | 109k | (uint32_t)bstr_len(htp_tx_request_protocol(tx))); |
289 | 109k | } |
290 | | |
291 | | /* response status */ |
292 | 302k | const int resp = htp_tx_response_status_number(tx); |
293 | 302k | if (resp > 0) { |
294 | 95.4k | SCJbSetUint(js, "status", (uint32_t)resp); |
295 | 206k | } else if (htp_tx_response_status(tx) != NULL) { |
296 | 21.3k | SCJbSetStringFromBytes(js, "status_string", bstr_ptr(htp_tx_response_status(tx)), |
297 | 21.3k | (uint32_t)bstr_len(htp_tx_response_status(tx))); |
298 | 21.3k | } |
299 | | |
300 | 302k | const htp_header_t *h_location = htp_tx_response_header(tx, "location"); |
301 | 302k | if (h_location != NULL) { |
302 | 1.18k | SCJbSetStringFromBytes(js, "redirect", htp_header_value_ptr(h_location), |
303 | 1.18k | (uint32_t)htp_header_value_len(h_location)); |
304 | 1.18k | } |
305 | | |
306 | | /* length */ |
307 | 302k | SCJbSetUint(js, "length", htp_tx_response_message_len(tx)); |
308 | 302k | } |
309 | | |
310 | | static void EveHttpLogJSONHeaders( |
311 | | SCJsonBuilder *js, uint32_t direction, htp_tx_t *tx, LogHttpFileCtx *http_ctx) |
312 | 267k | { |
313 | 267k | const htp_headers_t *headers = direction & LOG_HTTP_REQ_HEADERS ? htp_tx_request_headers(tx) |
314 | 267k | : htp_tx_response_headers(tx); |
315 | 267k | char name[MAX_SIZE_HEADER_NAME] = {0}; |
316 | 267k | char value[MAX_SIZE_HEADER_VALUE] = {0}; |
317 | 267k | size_t n = htp_headers_size(headers); |
318 | 267k | SCJsonBuilderMark mark = { 0, 0, 0 }; |
319 | 267k | SCJbGetMark(js, &mark); |
320 | 267k | bool array_empty = true; |
321 | 267k | SCJbOpenArray(js, direction & LOG_HTTP_REQ_HEADERS ? "request_headers" : "response_headers"); |
322 | 798k | for (size_t i = 0; i < n; i++) { |
323 | 531k | const htp_header_t *h = htp_headers_get_index(headers, i); |
324 | 531k | if ((http_ctx->flags & direction) == 0 && http_ctx->fields != 0) { |
325 | 0 | bool tolog = false; |
326 | 0 | for (HttpField f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++) { |
327 | 0 | if ((http_ctx->fields & (1ULL << f)) != 0) { |
328 | | /* prevent logging a field twice if extended logging is |
329 | | enabled */ |
330 | 0 | if (((http_ctx->flags & LOG_HTTP_EXTENDED) == 0) || |
331 | 0 | ((http_ctx->flags & LOG_HTTP_EXTENDED) != |
332 | 0 | (http_fields[f].flags & LOG_HTTP_EXTENDED))) { |
333 | 0 | if (bstr_cmp_c_nocase(htp_header_name(h), http_fields[f].htp_field)) { |
334 | 0 | tolog = true; |
335 | 0 | break; |
336 | 0 | } |
337 | 0 | } |
338 | 0 | } |
339 | 0 | } |
340 | 0 | if (!tolog) { |
341 | 0 | continue; |
342 | 0 | } |
343 | 0 | } |
344 | 531k | array_empty = false; |
345 | 531k | SCJbStartObject(js); |
346 | 531k | size_t size_name = htp_header_name_len(h) < MAX_SIZE_HEADER_NAME - 1 |
347 | 531k | ? htp_header_name_len(h) |
348 | 531k | : MAX_SIZE_HEADER_NAME - 1; |
349 | 531k | memcpy(name, htp_header_name_ptr(h), size_name); |
350 | 531k | name[size_name] = '\0'; |
351 | 531k | SCJbSetString(js, "name", name); |
352 | 531k | size_t size_value = htp_header_value_len(h) < MAX_SIZE_HEADER_VALUE - 1 |
353 | 531k | ? htp_header_value_len(h) |
354 | 531k | : MAX_SIZE_HEADER_VALUE - 1; |
355 | 531k | memcpy(value, htp_header_value_ptr(h), size_value); |
356 | 531k | value[size_value] = '\0'; |
357 | 531k | SCJbSetString(js, "value", value); |
358 | 531k | SCJbClose(js); |
359 | 531k | } |
360 | 267k | if (array_empty) { |
361 | 151k | SCJbRestoreMark(js, &mark); |
362 | 151k | } else { |
363 | | // Close array. |
364 | 115k | SCJbClose(js); |
365 | 115k | } |
366 | 267k | } |
367 | | |
368 | | static void BodyPrintableBuffer(SCJsonBuilder *js, HtpBody *body, const char *key) |
369 | 137k | { |
370 | 137k | if (body->sb != NULL && body->sb->region.buf != NULL) { |
371 | 68.6k | const uint8_t *body_data; |
372 | 68.6k | uint32_t body_data_len; |
373 | 68.6k | uint64_t body_offset; |
374 | | |
375 | 68.6k | if (StreamingBufferGetData(body->sb, &body_data, |
376 | 68.6k | &body_data_len, &body_offset) == 0) { |
377 | 0 | return; |
378 | 0 | } |
379 | | |
380 | 68.6k | SCJbSetPrintAsciiString(js, key, body_data, body_data_len); |
381 | 68.6k | } |
382 | 137k | } |
383 | | |
384 | | void EveHttpLogJSONBodyPrintable(SCJsonBuilder *js, Flow *f, uint64_t tx_id) |
385 | 68.5k | { |
386 | 68.5k | HtpState *htp_state = (HtpState *)FlowGetAppState(f); |
387 | 68.5k | if (htp_state) { |
388 | 68.5k | htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, htp_state, tx_id); |
389 | 68.5k | if (tx) { |
390 | 68.5k | HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); |
391 | 68.5k | BodyPrintableBuffer(js, &htud->request_body, "http_request_body_printable"); |
392 | 68.5k | BodyPrintableBuffer(js, &htud->response_body, "http_response_body_printable"); |
393 | 68.5k | } |
394 | 68.5k | } |
395 | 68.5k | } |
396 | | |
397 | | static void BodyBase64Buffer(SCJsonBuilder *js, HtpBody *body, const char *key) |
398 | 137k | { |
399 | 137k | if (body->sb != NULL && body->sb->region.buf != NULL) { |
400 | 68.6k | const uint8_t *body_data; |
401 | 68.6k | uint32_t body_data_len; |
402 | 68.6k | uint64_t body_offset; |
403 | | |
404 | 68.6k | if (StreamingBufferGetData(body->sb, &body_data, |
405 | 68.6k | &body_data_len, &body_offset) == 0) { |
406 | 0 | return; |
407 | 0 | } |
408 | | |
409 | 68.6k | SCJbSetBase64(js, key, body_data, body_data_len); |
410 | 68.6k | } |
411 | 137k | } |
412 | | |
413 | | void EveHttpLogJSONBodyBase64(SCJsonBuilder *js, Flow *f, uint64_t tx_id) |
414 | 68.5k | { |
415 | 68.5k | HtpState *htp_state = (HtpState *)FlowGetAppState(f); |
416 | 68.5k | if (htp_state) { |
417 | 68.5k | htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, htp_state, tx_id); |
418 | 68.5k | if (tx) { |
419 | 68.5k | HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); |
420 | 68.5k | BodyBase64Buffer(js, &htud->request_body, "http_request_body"); |
421 | 68.5k | BodyBase64Buffer(js, &htud->response_body, "http_response_body"); |
422 | 68.5k | } |
423 | 68.5k | } |
424 | 68.5k | } |
425 | | |
426 | | /* JSON format logging */ |
427 | | static void EveHttpLogJSON(JsonHttpLogThread *aft, SCJsonBuilder *js, htp_tx_t *tx, uint64_t tx_id) |
428 | 133k | { |
429 | 133k | LogHttpFileCtx *http_ctx = aft->httplog_ctx; |
430 | 133k | SCJbOpenObject(js, "http"); |
431 | | |
432 | 133k | EveHttpLogJSONBasic(js, tx); |
433 | 133k | if (http_ctx->flags & LOG_HTTP_EXTENDED) |
434 | 133k | EveHttpLogJSONExtended(js, tx); |
435 | 133k | if (http_ctx->flags & LOG_HTTP_REQ_HEADERS || http_ctx->fields != 0) |
436 | 133k | EveHttpLogJSONHeaders(js, LOG_HTTP_REQ_HEADERS, tx, http_ctx); |
437 | 133k | if (http_ctx->flags & LOG_HTTP_RES_HEADERS || http_ctx->fields != 0) |
438 | 133k | EveHttpLogJSONHeaders(js, LOG_HTTP_RES_HEADERS, tx, http_ctx); |
439 | | |
440 | 133k | SCJbClose(js); |
441 | 133k | } |
442 | | |
443 | | static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flow *f, void *alstate, void *txptr, uint64_t tx_id) |
444 | 133k | { |
445 | 133k | SCEnter(); |
446 | | |
447 | 133k | htp_tx_t *tx = txptr; |
448 | 133k | JsonHttpLogThread *jhl = (JsonHttpLogThread *)thread_data; |
449 | | |
450 | 133k | SCJsonBuilder *js = CreateEveHeaderWithTxId( |
451 | 133k | p, LOG_DIR_FLOW, "http", NULL, tx_id, jhl->httplog_ctx->eve_ctx); |
452 | 133k | if (unlikely(js == NULL)) |
453 | 0 | return TM_ECODE_OK; |
454 | | |
455 | 133k | SCLogDebug("got a HTTP request and now logging !!"); |
456 | | |
457 | 133k | EveHttpLogJSON(jhl, js, tx, tx_id); |
458 | 133k | HttpXFFCfg *xff_cfg = jhl->httplog_ctx->xff_cfg != NULL ? |
459 | 133k | jhl->httplog_ctx->xff_cfg : jhl->httplog_ctx->parent_xff_cfg; |
460 | | |
461 | | /* xff header */ |
462 | 133k | if ((xff_cfg != NULL) && !(xff_cfg->flags & XFF_DISABLED) && p->flow != NULL) { |
463 | 133k | int have_xff_ip = 0; |
464 | 133k | char buffer[XFF_MAXLEN]; |
465 | | |
466 | 133k | have_xff_ip = HttpXFFGetIPFromTx(p->flow, tx_id, xff_cfg, buffer, XFF_MAXLEN); |
467 | | |
468 | 133k | if (have_xff_ip) { |
469 | 285 | if (xff_cfg->flags & XFF_EXTRADATA) { |
470 | 285 | SCJbSetString(js, "xff", buffer); |
471 | 285 | } |
472 | 0 | else if (xff_cfg->flags & XFF_OVERWRITE) { |
473 | 0 | if (p->flowflags & FLOW_PKT_TOCLIENT) { |
474 | 0 | SCJbSetString(js, "dest_ip", buffer); |
475 | 0 | } else { |
476 | 0 | SCJbSetString(js, "src_ip", buffer); |
477 | 0 | } |
478 | 0 | } |
479 | 285 | } |
480 | 133k | } |
481 | | |
482 | 133k | OutputJsonBuilderBuffer(tv, p, p->flow, js, jhl->ctx); |
483 | 133k | SCJbFree(js); |
484 | | |
485 | 133k | SCReturnInt(TM_ECODE_OK); |
486 | 133k | } |
487 | | |
488 | | bool EveHttpAddMetadata(const Flow *f, uint64_t tx_id, SCJsonBuilder *js) |
489 | 168k | { |
490 | 168k | HtpState *htp_state = (HtpState *)FlowGetAppState(f); |
491 | 168k | if (htp_state) { |
492 | 168k | htp_tx_t *tx = AppLayerParserGetTx(IPPROTO_TCP, ALPROTO_HTTP1, htp_state, tx_id); |
493 | | |
494 | 168k | if (tx) { |
495 | 168k | EveHttpLogJSONBasic(js, tx); |
496 | 168k | EveHttpLogJSONExtended(js, tx); |
497 | 168k | return true; |
498 | 168k | } |
499 | 168k | } |
500 | | |
501 | 0 | return false; |
502 | 168k | } |
503 | | |
504 | | static void OutputHttpLogDeinitSub(OutputCtx *output_ctx) |
505 | 0 | { |
506 | 0 | LogHttpFileCtx *http_ctx = output_ctx->data; |
507 | 0 | if (http_ctx->xff_cfg) { |
508 | 0 | SCFree(http_ctx->xff_cfg); |
509 | 0 | } |
510 | 0 | SCFree(http_ctx); |
511 | 0 | SCFree(output_ctx); |
512 | 0 | } |
513 | | |
514 | | static OutputInitResult OutputHttpLogInitSub(SCConfNode *conf, OutputCtx *parent_ctx) |
515 | 4 | { |
516 | 4 | OutputInitResult result = { NULL, false }; |
517 | 4 | OutputJsonCtx *ojc = parent_ctx->data; |
518 | | |
519 | 4 | LogHttpFileCtx *http_ctx = SCCalloc(1, sizeof(LogHttpFileCtx)); |
520 | 4 | if (unlikely(http_ctx == NULL)) |
521 | 0 | return result; |
522 | 4 | memset(http_ctx, 0x00, sizeof(*http_ctx)); |
523 | | |
524 | 4 | OutputCtx *output_ctx = SCCalloc(1, sizeof(OutputCtx)); |
525 | 4 | if (unlikely(output_ctx == NULL)) { |
526 | 0 | SCFree(http_ctx); |
527 | 0 | return result; |
528 | 0 | } |
529 | | |
530 | 4 | http_ctx->flags = LOG_HTTP_DEFAULT; |
531 | 4 | http_ctx->eve_ctx = ojc; |
532 | | |
533 | 4 | if (conf) { |
534 | 4 | const char *extended = SCConfNodeLookupChildValue(conf, "extended"); |
535 | | |
536 | 4 | if (extended != NULL) { |
537 | 4 | if (SCConfValIsTrue(extended)) { |
538 | 4 | http_ctx->flags = LOG_HTTP_EXTENDED; |
539 | 4 | } |
540 | 4 | } |
541 | | |
542 | 4 | const char *all_headers = SCConfNodeLookupChildValue(conf, "dump-all-headers"); |
543 | 4 | if (all_headers != NULL) { |
544 | 4 | if (strncmp(all_headers, "both", 4) == 0) { |
545 | 4 | http_ctx->flags |= LOG_HTTP_REQ_HEADERS; |
546 | 4 | http_ctx->flags |= LOG_HTTP_RES_HEADERS; |
547 | 4 | } else if (strncmp(all_headers, "request", 7) == 0) { |
548 | 0 | http_ctx->flags |= LOG_HTTP_REQ_HEADERS; |
549 | 0 | } else if (strncmp(all_headers, "response", 8) == 0) { |
550 | 0 | http_ctx->flags |= LOG_HTTP_RES_HEADERS; |
551 | 0 | } |
552 | 4 | } |
553 | 4 | SCConfNode *custom; |
554 | 4 | if ((custom = SCConfNodeLookupChild(conf, "custom")) != NULL) { |
555 | 0 | if ((http_ctx->flags & (LOG_HTTP_REQ_HEADERS | LOG_HTTP_RES_HEADERS)) == |
556 | 0 | (LOG_HTTP_REQ_HEADERS | LOG_HTTP_RES_HEADERS)) { |
557 | 0 | SCLogWarning("No need for custom as dump-all-headers is already present"); |
558 | 0 | } |
559 | 0 | SCConfNode *field; |
560 | 0 | TAILQ_FOREACH (field, &custom->head, next) { |
561 | 0 | HttpField f; |
562 | 0 | for (f = HTTP_FIELD_ACCEPT; f < HTTP_FIELD_SIZE; f++) { |
563 | 0 | if ((strcmp(http_fields[f].config_field, field->val) == 0) || |
564 | 0 | (strcasecmp(http_fields[f].htp_field, field->val) == 0)) { |
565 | 0 | http_ctx->fields |= (1ULL << f); |
566 | 0 | break; |
567 | 0 | } |
568 | 0 | } |
569 | 0 | } |
570 | 0 | } |
571 | 4 | } |
572 | | |
573 | 4 | if (conf != NULL && SCConfNodeLookupChild(conf, "xff") != NULL) { |
574 | 0 | http_ctx->xff_cfg = SCCalloc(1, sizeof(HttpXFFCfg)); |
575 | 0 | if (http_ctx->xff_cfg != NULL) { |
576 | 0 | HttpXFFGetCfg(conf, http_ctx->xff_cfg); |
577 | 0 | } |
578 | 4 | } else if (ojc->xff_cfg) { |
579 | 4 | http_ctx->parent_xff_cfg = ojc->xff_cfg; |
580 | 4 | } |
581 | | |
582 | 4 | output_ctx->data = http_ctx; |
583 | 4 | output_ctx->DeInit = OutputHttpLogDeinitSub; |
584 | | |
585 | | /* enable the logger for the app layer */ |
586 | 4 | SCAppLayerParserRegisterLogger(IPPROTO_TCP, ALPROTO_HTTP1); |
587 | | |
588 | 4 | result.ctx = output_ctx; |
589 | 4 | result.ok = true; |
590 | 4 | return result; |
591 | 4 | } |
592 | | |
593 | | static TmEcode JsonHttpLogThreadInit(ThreadVars *t, const void *initdata, void **data) |
594 | 4 | { |
595 | 4 | JsonHttpLogThread *aft = SCCalloc(1, sizeof(JsonHttpLogThread)); |
596 | 4 | if (unlikely(aft == NULL)) |
597 | 0 | return TM_ECODE_FAILED; |
598 | | |
599 | 4 | if(initdata == NULL) |
600 | 0 | { |
601 | 0 | SCLogDebug("Error getting context for EveLogHTTP. \"initdata\" argument NULL"); |
602 | 0 | goto error_exit; |
603 | 0 | } |
604 | | |
605 | | /* Use the Output Context (file pointer and mutex) */ |
606 | 4 | aft->httplog_ctx = ((OutputCtx *)initdata)->data; //TODO |
607 | | |
608 | 4 | aft->ctx = CreateEveThreadCtx(t, aft->httplog_ctx->eve_ctx); |
609 | 4 | if (!aft->ctx) { |
610 | 0 | goto error_exit; |
611 | 0 | } |
612 | | |
613 | 4 | *data = (void *)aft; |
614 | 4 | return TM_ECODE_OK; |
615 | | |
616 | 0 | error_exit: |
617 | 0 | SCFree(aft); |
618 | 0 | return TM_ECODE_FAILED; |
619 | 4 | } |
620 | | |
621 | | static TmEcode JsonHttpLogThreadDeinit(ThreadVars *t, void *data) |
622 | 0 | { |
623 | 0 | JsonHttpLogThread *aft = (JsonHttpLogThread *)data; |
624 | 0 | if (aft == NULL) { |
625 | 0 | return TM_ECODE_OK; |
626 | 0 | } |
627 | | |
628 | 0 | FreeEveThreadCtx(aft->ctx); |
629 | | |
630 | | /* clear memory */ |
631 | 0 | memset(aft, 0, sizeof(JsonHttpLogThread)); |
632 | |
|
633 | 0 | SCFree(aft); |
634 | 0 | return TM_ECODE_OK; |
635 | 0 | } |
636 | | |
637 | | void JsonHttpLogRegister (void) |
638 | 78 | { |
639 | | /* register as child of eve-log */ |
640 | 78 | OutputRegisterTxSubModule(LOGGER_JSON_TX, "eve-log", "JsonHttpLog", "eve-log.http", |
641 | 78 | OutputHttpLogInitSub, ALPROTO_HTTP1, JsonHttpLogger, JsonHttpLogThreadInit, |
642 | 78 | JsonHttpLogThreadDeinit); |
643 | 78 | } |