/src/suricata8/src/detect-file-data.c
Line | Count | Source |
1 | | /* Copyright (C) 2007-2022 Open Information Security Foundation |
2 | | * |
3 | | * You can copy, redistribute or modify this Program under the terms of |
4 | | * the GNU General Public License version 2 as published by the Free |
5 | | * Software Foundation. |
6 | | * |
7 | | * This program is distributed in the hope that it will be useful, |
8 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
9 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
10 | | * GNU General Public License for more details. |
11 | | * |
12 | | * You should have received a copy of the GNU General Public License |
13 | | * version 2 along with this program; if not, write to the Free Software |
14 | | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA |
15 | | * 02110-1301, USA. |
16 | | */ |
17 | | |
18 | | /** |
19 | | * \file |
20 | | * |
21 | | * \author Victor Julien <victor@inliniac.net> |
22 | | * |
23 | | */ |
24 | | |
25 | | #include "suricata-common.h" |
26 | | #include "threads.h" |
27 | | #include "decode.h" |
28 | | |
29 | | #include "detect.h" |
30 | | #include "detect-parse.h" |
31 | | |
32 | | #include "detect-engine.h" |
33 | | #include "detect-engine-buffer.h" |
34 | | #include "detect-engine-mpm.h" |
35 | | #include "detect-engine-state.h" |
36 | | #include "detect-engine-prefilter.h" |
37 | | #include "detect-engine-content-inspection.h" |
38 | | #include "detect-engine-file.h" |
39 | | #include "detect-file-data.h" |
40 | | |
41 | | #include "app-layer.h" |
42 | | #include "app-layer-parser.h" |
43 | | #include "app-layer-htp.h" |
44 | | #include "app-layer-smtp.h" |
45 | | |
46 | | #include "flow.h" |
47 | | #include "flow-var.h" |
48 | | #include "flow-util.h" |
49 | | |
50 | | #include "util-debug.h" |
51 | | #include "util-spm-bm.h" |
52 | | #include "util-unittest.h" |
53 | | #include "util-unittest-helper.h" |
54 | | #include "util-file-decompression.h" |
55 | | #include "util-profiling.h" |
56 | | |
57 | | static int DetectFiledataSetup (DetectEngineCtx *, Signature *, const char *); |
58 | | #ifdef UNITTESTS |
59 | | static void DetectFiledataRegisterTests(void); |
60 | | #endif |
61 | | static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx, |
62 | | Signature *s); |
63 | | static int g_file_data_buffer_id = 0; |
64 | | |
65 | | /* file API */ |
66 | | int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, |
67 | | const DetectBufferMpmRegistry *mpm_reg, int list_id); |
68 | | |
69 | | // file protocols with common file handling |
70 | | typedef struct { |
71 | | AppProto alproto; |
72 | | int direction; |
73 | | int to_client_progress; |
74 | | int to_server_progress; |
75 | | } DetectFileHandlerProtocol_t; |
76 | | |
77 | | /* Table with all filehandler registrations */ |
78 | | DetectFileHandlerTableElmt filehandler_table[DETECT_TBLSIZE_STATIC]; |
79 | | |
80 | 0 | #define ALPROTO_WITHFILES_MAX 16 |
81 | | |
82 | | // file protocols with common file handling |
83 | | DetectFileHandlerProtocol_t al_protocols[ALPROTO_WITHFILES_MAX] = { |
84 | | { .alproto = ALPROTO_NFS, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, |
85 | | { .alproto = ALPROTO_SMB, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, |
86 | | { .alproto = ALPROTO_FTP, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, |
87 | | { .alproto = ALPROTO_FTPDATA, .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT }, |
88 | | { .alproto = ALPROTO_HTTP1, |
89 | | .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT, |
90 | | .to_client_progress = HTP_RESPONSE_PROGRESS_BODY, |
91 | | .to_server_progress = HTP_REQUEST_PROGRESS_BODY }, |
92 | | { .alproto = ALPROTO_HTTP2, |
93 | | .direction = SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT, |
94 | | .to_client_progress = HTTP2StateDataServer, |
95 | | .to_server_progress = HTTP2StateDataClient }, |
96 | | { .alproto = ALPROTO_SMTP, |
97 | | .direction = SIG_FLAG_TOSERVER, |
98 | | .to_server_progress = SMTP_REQUEST_DATA }, |
99 | | { .alproto = ALPROTO_UNKNOWN } |
100 | | }; |
101 | | |
102 | | void DetectFileRegisterProto( |
103 | | AppProto alproto, int direction, int to_client_progress, int to_server_progress) |
104 | 0 | { |
105 | 0 | size_t i = 0; |
106 | 0 | while (i < ALPROTO_WITHFILES_MAX && al_protocols[i].alproto != ALPROTO_UNKNOWN) { |
107 | 0 | i++; |
108 | 0 | } |
109 | 0 | if (i == ALPROTO_WITHFILES_MAX) { |
110 | 0 | return; |
111 | 0 | } |
112 | 0 | al_protocols[i].alproto = alproto; |
113 | 0 | al_protocols[i].direction = direction; |
114 | 0 | al_protocols[i].to_client_progress = to_client_progress; |
115 | 0 | al_protocols[i].to_server_progress = to_server_progress; |
116 | 0 | if (i + 1 < ALPROTO_WITHFILES_MAX) { |
117 | 0 | al_protocols[i + 1].alproto = ALPROTO_UNKNOWN; |
118 | 0 | } |
119 | 0 | } |
120 | | |
121 | | void DetectFileRegisterFileProtocols(DetectFileHandlerTableElmt *reg) |
122 | 237 | { |
123 | 1.89k | for (size_t i = 0; i < g_alproto_max; i++) { |
124 | 1.89k | if (al_protocols[i].alproto == ALPROTO_UNKNOWN) { |
125 | 237 | break; |
126 | 237 | } |
127 | 1.65k | int direction = al_protocols[i].direction == 0 |
128 | 1.65k | ? (int)(SIG_FLAG_TOSERVER | SIG_FLAG_TOCLIENT) |
129 | 1.65k | : al_protocols[i].direction; |
130 | | |
131 | 1.65k | if (direction & SIG_FLAG_TOCLIENT) { |
132 | 1.42k | DetectAppLayerMpmRegister(reg->name, SIG_FLAG_TOCLIENT, reg->priority, reg->PrefilterFn, |
133 | 1.42k | reg->GetData, al_protocols[i].alproto, al_protocols[i].to_client_progress); |
134 | 1.42k | DetectAppLayerInspectEngineRegister(reg->name, al_protocols[i].alproto, |
135 | 1.42k | SIG_FLAG_TOCLIENT, al_protocols[i].to_client_progress, reg->Callback, |
136 | 1.42k | reg->GetData); |
137 | 1.42k | } |
138 | 1.65k | if (direction & SIG_FLAG_TOSERVER) { |
139 | 1.65k | DetectAppLayerMpmRegister(reg->name, SIG_FLAG_TOSERVER, reg->priority, reg->PrefilterFn, |
140 | 1.65k | reg->GetData, al_protocols[i].alproto, al_protocols[i].to_server_progress); |
141 | 1.65k | DetectAppLayerInspectEngineRegister(reg->name, al_protocols[i].alproto, |
142 | 1.65k | SIG_FLAG_TOSERVER, al_protocols[i].to_server_progress, reg->Callback, |
143 | 1.65k | reg->GetData); |
144 | 1.65k | } |
145 | 1.65k | } |
146 | 237 | } |
147 | | |
148 | | /** |
149 | | * \brief Registration function for keyword: file_data |
150 | | */ |
151 | | void DetectFiledataRegister(void) |
152 | 79 | { |
153 | 79 | sigmatch_table[DETECT_FILE_DATA].name = "file.data"; |
154 | 79 | sigmatch_table[DETECT_FILE_DATA].alias = "file_data"; |
155 | 79 | sigmatch_table[DETECT_FILE_DATA].desc = "make content keywords match on file data"; |
156 | 79 | sigmatch_table[DETECT_FILE_DATA].url = "/rules/file-keywords.html#file-data"; |
157 | 79 | sigmatch_table[DETECT_FILE_DATA].Setup = DetectFiledataSetup; |
158 | | #ifdef UNITTESTS |
159 | | sigmatch_table[DETECT_FILE_DATA].RegisterTests = DetectFiledataRegisterTests; |
160 | | #endif |
161 | 79 | sigmatch_table[DETECT_FILE_DATA].flags = SIGMATCH_OPTIONAL_OPT | SIGMATCH_SUPPORT_DIR; |
162 | | |
163 | 79 | filehandler_table[DETECT_FILE_DATA].name = "file_data"; |
164 | 79 | filehandler_table[DETECT_FILE_DATA].priority = 2; |
165 | 79 | filehandler_table[DETECT_FILE_DATA].PrefilterFn = PrefilterMpmFiledataRegister; |
166 | 79 | filehandler_table[DETECT_FILE_DATA].Callback = DetectEngineInspectFiledata; |
167 | | |
168 | 79 | DetectBufferTypeRegisterSetupCallback("file_data", DetectFiledataSetupCallback); |
169 | | |
170 | 79 | DetectBufferTypeSetDescriptionByName("file_data", "data from tracked files"); |
171 | 79 | DetectBufferTypeSupportsMultiInstance("file_data"); |
172 | | |
173 | 79 | g_file_data_buffer_id = DetectBufferTypeGetByName("file_data"); |
174 | 79 | } |
175 | | |
176 | 8.69k | static void SetupDetectEngineConfig(DetectEngineCtx *de_ctx) { |
177 | 8.69k | if (de_ctx->filedata_config) |
178 | 8.35k | return; |
179 | | |
180 | 349 | de_ctx->filedata_config = SCMalloc(g_alproto_max * sizeof(DetectFileDataCfg)); |
181 | 349 | if (unlikely(de_ctx->filedata_config == NULL)) |
182 | 0 | return; |
183 | | /* initialize default */ |
184 | 14.3k | for (AppProto i = 0; i < g_alproto_max; i++) { |
185 | 13.9k | de_ctx->filedata_config[i].content_limit = FILEDATA_CONTENT_LIMIT; |
186 | 13.9k | de_ctx->filedata_config[i].content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE; |
187 | 13.9k | } |
188 | | |
189 | | /* add protocol specific settings here */ |
190 | | |
191 | | /* SMTP */ |
192 | 349 | de_ctx->filedata_config[ALPROTO_SMTP].content_limit = smtp_config.content_limit; |
193 | 349 | de_ctx->filedata_config[ALPROTO_SMTP].content_inspect_min_size = |
194 | 349 | smtp_config.content_inspect_min_size; |
195 | 349 | } |
196 | | |
197 | | /** |
198 | | * \brief this function is used to parse filedata options |
199 | | * \brief into the current signature |
200 | | * |
201 | | * \param de_ctx pointer to the Detection Engine Context |
202 | | * \param s pointer to the Current Signature |
203 | | * \param str pointer to the user provided "filestore" option |
204 | | * |
205 | | * \retval 0 on Success |
206 | | * \retval -1 on Failure |
207 | | */ |
208 | | static int DetectFiledataSetup (DetectEngineCtx *de_ctx, Signature *s, const char *str) |
209 | 60.3k | { |
210 | 60.3k | SCEnter(); |
211 | | |
212 | 60.3k | if (s->alproto != ALPROTO_UNKNOWN && !AppLayerParserSupportsFiles(IPPROTO_TCP, s->alproto) && |
213 | 259 | !AppLayerParserSupportsFiles(IPPROTO_UDP, s->alproto)) { |
214 | 259 | SCLogError("The 'file_data' keyword cannot be used with protocol %s", |
215 | 259 | AppLayerGetProtoName(s->alproto)); |
216 | 259 | return -1; |
217 | 259 | } |
218 | | |
219 | 60.0k | if (s->alproto == ALPROTO_SMTP && (s->init_data->init_flags & SIG_FLAG_INIT_FLOW) && |
220 | 211 | !(s->flags & SIG_FLAG_TOSERVER) && (s->flags & SIG_FLAG_TOCLIENT)) { |
221 | 9 | SCLogError("The 'file-data' keyword cannot be used with SMTP flow:to_client or " |
222 | 9 | "flow:from_server."); |
223 | 9 | return -1; |
224 | 9 | } |
225 | | |
226 | 60.0k | if (SCDetectBufferSetActiveList(de_ctx, s, DetectBufferTypeGetByName("file_data")) < 0) |
227 | 53 | return -1; |
228 | | |
229 | 60.0k | s->init_data->init_flags |= SIG_FLAG_INIT_FILEDATA; |
230 | 60.0k | if ((s->init_data->init_flags & SIG_FLAG_INIT_FORCE_TOCLIENT) == 0) { |
231 | | // we cannot use a transactional rule with a fast pattern to client and this |
232 | 59.8k | if (s->init_data->init_flags & SIG_FLAG_INIT_TXDIR_FAST_TOCLIENT) { |
233 | 1 | SCLogError("fast_pattern cannot be used on to_client keyword for " |
234 | 1 | "transactional rule with a streaming buffer to server %u", |
235 | 1 | s->id); |
236 | 1 | return -1; |
237 | 1 | } |
238 | 59.8k | s->init_data->init_flags |= SIG_FLAG_INIT_TXDIR_STREAMING_TOSERVER; |
239 | 59.8k | } |
240 | | |
241 | 60.0k | SetupDetectEngineConfig(de_ctx); |
242 | 60.0k | return 0; |
243 | 60.0k | } |
244 | | |
245 | | static void DetectFiledataSetupCallback(const DetectEngineCtx *de_ctx, |
246 | | Signature *s) |
247 | 39.4k | { |
248 | 39.4k | if (s->alproto == ALPROTO_HTTP1 || s->alproto == ALPROTO_UNKNOWN || |
249 | 36.6k | s->alproto == ALPROTO_HTTP) { |
250 | 36.6k | AppLayerHtpEnableResponseBodyCallback(); |
251 | 36.6k | } |
252 | | |
253 | | /* server body needs to be inspected in sync with stream if possible */ |
254 | 39.4k | s->init_data->init_flags |= SIG_FLAG_INIT_NEED_FLUSH; |
255 | | |
256 | 39.4k | SCLogDebug("callback invoked by %u", s->id); |
257 | 39.4k | } |
258 | | |
259 | | /* common */ |
260 | | |
261 | | static void PrefilterMpmFiledataFree(void *ptr) |
262 | 73.7k | { |
263 | 73.7k | SCFree(ptr); |
264 | 73.7k | } |
265 | | |
266 | | /* file API based inspection */ |
267 | | |
268 | | static inline InspectionBuffer *FiledataWithXformsGetDataCallback(DetectEngineThreadCtx *det_ctx, |
269 | | const DetectEngineTransforms *transforms, const int list_id, int local_file_id, |
270 | | InspectionBuffer *base_buffer) |
271 | 19.0k | { |
272 | 19.0k | InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, list_id, local_file_id); |
273 | 19.0k | if (buffer == NULL) { |
274 | 0 | SCLogDebug("list_id: %d: no buffer", list_id); |
275 | 0 | return NULL; |
276 | 0 | } |
277 | 19.0k | if (buffer->initialized) { |
278 | 141 | SCLogDebug("list_id: %d: returning %p", list_id, buffer); |
279 | 141 | return buffer; |
280 | 141 | } |
281 | | |
282 | 18.9k | InspectionBufferSetupMulti( |
283 | 18.9k | det_ctx, buffer, transforms, base_buffer->inspect, base_buffer->inspect_len); |
284 | 18.9k | buffer->inspect_offset = base_buffer->inspect_offset; |
285 | 18.9k | SCLogDebug("xformed buffer %p size %u", buffer, buffer->inspect_len); |
286 | 18.9k | SCReturnPtr(buffer, "InspectionBuffer"); |
287 | 19.0k | } |
288 | | |
289 | | static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx, |
290 | | const DetectEngineTransforms *transforms, Flow *f, uint8_t flow_flags, File *cur_file, |
291 | | const int list_id, const int base_id, int local_file_id, void *txv) |
292 | 31.8k | { |
293 | 31.8k | SCEnter(); |
294 | 31.8k | SCLogDebug("starting: list_id %d base_id %d", list_id, base_id); |
295 | | |
296 | 31.8k | InspectionBuffer *buffer = InspectionBufferMultipleForListGet(det_ctx, base_id, local_file_id); |
297 | 31.8k | SCLogDebug("base: buffer %p", buffer); |
298 | 31.8k | if (buffer == NULL) |
299 | 0 | return NULL; |
300 | 31.8k | if (base_id != list_id && buffer->inspect != NULL) { |
301 | 533 | SCLogDebug("handle xform %s", (list_id != base_id) ? "true" : "false"); |
302 | 533 | return FiledataWithXformsGetDataCallback( |
303 | 533 | det_ctx, transforms, list_id, local_file_id, buffer); |
304 | 533 | } |
305 | 31.3k | if (buffer->initialized) { |
306 | 2.22k | SCLogDebug("base_id: %d, not first: use %p", base_id, buffer); |
307 | 2.22k | return buffer; |
308 | 2.22k | } |
309 | | |
310 | 29.1k | const uint64_t file_size = FileDataSize(cur_file); |
311 | 29.1k | const DetectEngineCtx *de_ctx = det_ctx->de_ctx; |
312 | 29.1k | uint32_t content_limit = FILEDATA_CONTENT_LIMIT; |
313 | 29.1k | uint32_t content_inspect_min_size = FILEDATA_CONTENT_INSPECT_MIN_SIZE; |
314 | 29.1k | if (de_ctx->filedata_config) { |
315 | 28.1k | content_limit = de_ctx->filedata_config[f->alproto].content_limit; |
316 | 28.1k | content_inspect_min_size = de_ctx->filedata_config[f->alproto].content_inspect_min_size; |
317 | 28.1k | } |
318 | | |
319 | 29.1k | SCLogDebug("[list %d] content_limit %u, content_inspect_min_size %u", list_id, content_limit, |
320 | 29.1k | content_inspect_min_size); |
321 | | |
322 | 29.1k | SCLogDebug("[list %d] file %p size %" PRIu64 ", state %d", list_id, cur_file, file_size, |
323 | 29.1k | cur_file->state); |
324 | | |
325 | | /* no new data */ |
326 | 29.1k | if (cur_file->content_inspected == file_size) { |
327 | 266 | SCLogDebug("no new data"); |
328 | 266 | goto empty_return; |
329 | 266 | } |
330 | | |
331 | 28.8k | if (file_size == 0) { |
332 | 0 | SCLogDebug("no data to inspect for this transaction"); |
333 | 0 | goto empty_return; |
334 | 0 | } |
335 | | |
336 | 28.8k | SCLogDebug("offset %" PRIu64, StreamingBufferGetOffset(cur_file->sb)); |
337 | 28.8k | SCLogDebug("size %" PRIu64, cur_file->size); |
338 | 28.8k | SCLogDebug("content_inspected %" PRIu64, cur_file->content_inspected); |
339 | 28.8k | SCLogDebug("inspect_window %" PRIu32, cur_file->inspect_window); |
340 | 28.8k | SCLogDebug("inspect_min_size %" PRIu32, cur_file->inspect_min_size); |
341 | | |
342 | 28.8k | bool ips = false; |
343 | 28.8k | uint64_t offset = 0; |
344 | 28.8k | if (f->alproto == ALPROTO_HTTP1) { |
345 | | |
346 | 26.3k | htp_tx_t *tx = txv; |
347 | 26.3k | HtpState *htp_state = f->alstate; |
348 | 26.3k | ips = htp_state->cfg->http_body_inline; |
349 | | |
350 | 26.3k | const bool body_done = AppLayerParserGetStateProgress(IPPROTO_TCP, ALPROTO_HTTP1, tx, |
351 | 26.3k | flow_flags) > HTP_RESPONSE_PROGRESS_BODY; |
352 | | |
353 | 26.3k | SCLogDebug("response.body_limit %u file_size %" PRIu64 |
354 | 26.3k | ", cur_file->inspect_min_size %" PRIu32 ", EOF %s, progress > body? %s", |
355 | 26.3k | htp_state->cfg->response.body_limit, file_size, cur_file->inspect_min_size, |
356 | 26.3k | flow_flags & STREAM_EOF ? "true" : "false", BOOL2STR(body_done)); |
357 | | |
358 | 26.3k | if (!htp_state->cfg->http_body_inline) { |
359 | | /* inspect the body if the transfer is complete or we have hit |
360 | | * our body size limit */ |
361 | 26.3k | if ((htp_state->cfg->response.body_limit == 0 || |
362 | 26.3k | file_size < htp_state->cfg->response.body_limit) && |
363 | 26.3k | file_size < cur_file->inspect_min_size && !body_done && |
364 | 6.20k | !(flow_flags & STREAM_EOF)) { |
365 | 5.94k | SCLogDebug("we still haven't seen the entire response body. " |
366 | 5.94k | "Let's defer body inspection till we see the " |
367 | 5.94k | "entire body."); |
368 | 5.94k | goto empty_return; |
369 | 5.94k | } |
370 | 20.3k | SCLogDebug("inline and we're continuing"); |
371 | 20.3k | } |
372 | | |
373 | 26.3k | bool force = (flow_flags & STREAM_EOF) || (cur_file->state > FILE_STATE_OPENED) || |
374 | 5.29k | body_done || htp_state->cfg->http_body_inline; |
375 | | /* get the inspect buffer |
376 | | * |
377 | | * make sure that we have at least the configured inspect_win size. |
378 | | * If we have more, take at least 1/4 of the inspect win size before |
379 | | * the new data. |
380 | | */ |
381 | 20.3k | if (cur_file->content_inspected == 0) { |
382 | 15.2k | if (!force && file_size < cur_file->inspect_min_size) { |
383 | 0 | SCLogDebug("skip as file_size %" PRIu64 " < inspect_min_size %u", file_size, |
384 | 0 | cur_file->inspect_min_size); |
385 | 0 | goto empty_return; |
386 | 0 | } |
387 | 15.2k | } else { |
388 | 5.16k | uint64_t new_data = file_size - cur_file->content_inspected; |
389 | 5.16k | DEBUG_VALIDATE_BUG_ON(new_data == 0); |
390 | 5.16k | if (new_data < cur_file->inspect_window) { |
391 | 4.87k | uint64_t inspect_short = cur_file->inspect_window - new_data; |
392 | 4.87k | if (cur_file->content_inspected < inspect_short) { |
393 | 0 | offset = 0; |
394 | 0 | SCLogDebug("offset %" PRIu64, offset); |
395 | 4.87k | } else { |
396 | 4.87k | offset = cur_file->content_inspected - inspect_short; |
397 | 4.87k | SCLogDebug("offset %" PRIu64, offset); |
398 | 4.87k | } |
399 | 4.87k | } else { |
400 | 289 | BUG_ON(cur_file->content_inspected == 0); |
401 | 289 | uint32_t margin = cur_file->inspect_window / 4; |
402 | 289 | if ((uint64_t)margin <= cur_file->content_inspected) { |
403 | 289 | offset = cur_file->content_inspected - (cur_file->inspect_window / 4); |
404 | 289 | } else { |
405 | 0 | offset = 0; |
406 | 0 | } |
407 | 289 | SCLogDebug("offset %" PRIu64 " (data from offset %" PRIu64 ")", offset, |
408 | 289 | file_size - offset); |
409 | 289 | } |
410 | 5.16k | } |
411 | | |
412 | 20.3k | } else { |
413 | 2.54k | if ((content_limit == 0 || file_size < content_limit) && |
414 | 2.31k | file_size < content_inspect_min_size && !(flow_flags & STREAM_EOF) && |
415 | 2.10k | !(cur_file->state > FILE_STATE_OPENED)) { |
416 | 612 | SCLogDebug("we still haven't seen the entire content. " |
417 | 612 | "Let's defer content inspection till we see the " |
418 | 612 | "entire content. We've seen %ld and need at least %d", |
419 | 612 | file_size, content_inspect_min_size); |
420 | 612 | goto empty_return; |
421 | 612 | } |
422 | 1.93k | offset = cur_file->content_inspected; |
423 | 1.93k | } |
424 | | |
425 | 22.2k | const uint8_t *data; |
426 | 22.2k | uint32_t data_len; |
427 | | |
428 | 22.2k | SCLogDebug("Fetching data at offset: %ld", offset); |
429 | 22.2k | StreamingBufferGetDataAtOffset(cur_file->sb, &data, &data_len, offset); |
430 | 22.2k | SCLogDebug("data_len %u", data_len); |
431 | | /* update inspected tracker */ |
432 | 22.2k | buffer->inspect_offset = offset; |
433 | | |
434 | 22.2k | if (ips && file_size < cur_file->inspect_min_size) { |
435 | | // don't update content_inspected yet |
436 | 22.2k | } else { |
437 | 22.2k | SCLogDebug("content inspected: %" PRIu64, cur_file->content_inspected); |
438 | 22.2k | cur_file->content_inspected = MAX(cur_file->content_inspected, offset + data_len); |
439 | 22.2k | SCLogDebug("content inspected: %" PRIu64, cur_file->content_inspected); |
440 | 22.2k | } |
441 | | |
442 | 22.2k | InspectionBufferSetupMulti(det_ctx, buffer, NULL, data, data_len); |
443 | 22.2k | SCLogDebug("[list %d] [before] buffer offset %" PRIu64 "; buffer len %" PRIu32 |
444 | 22.2k | "; data_len %" PRIu32 "; file_size %" PRIu64, |
445 | 22.2k | list_id, buffer->inspect_offset, buffer->inspect_len, data_len, file_size); |
446 | | |
447 | 22.2k | if (f->alproto == ALPROTO_HTTP1 && flow_flags & STREAM_TOCLIENT) { |
448 | 19.6k | HtpState *htp_state = f->alstate; |
449 | | /* built-in 'transformation' */ |
450 | 19.6k | if (htp_state->cfg->swf_decompression_enabled) { |
451 | 19.6k | int swf_file_type = FileIsSwfFile(data, data_len); |
452 | 19.6k | if (swf_file_type == FILE_SWF_ZLIB_COMPRESSION || |
453 | 19.6k | swf_file_type == FILE_SWF_LZMA_COMPRESSION) { |
454 | 2 | SCLogDebug("decompressing ..."); |
455 | 2 | (void)FileSwfDecompression(data, data_len, det_ctx, buffer, |
456 | 2 | htp_state->cfg->swf_compression_type, htp_state->cfg->swf_decompress_depth, |
457 | 2 | htp_state->cfg->swf_compress_depth); |
458 | 2 | SCLogDebug("uncompressed buffer %p size %u; buf: \"%s\"", buffer, |
459 | 2 | buffer->inspect_len, (char *)buffer->inspect); |
460 | 2 | } |
461 | 19.6k | } |
462 | 19.6k | } |
463 | | |
464 | 22.2k | SCLogDebug("content inspected: %" PRIu64, cur_file->content_inspected); |
465 | | |
466 | | /* get buffer for the list id if it is different from the base id */ |
467 | 22.2k | if (list_id != base_id) { |
468 | 18.5k | SCLogDebug("regular %d has been set up: now handle xforms id %d", base_id, list_id); |
469 | 18.5k | InspectionBuffer *tbuffer = FiledataWithXformsGetDataCallback( |
470 | 18.5k | det_ctx, transforms, list_id, local_file_id, buffer); |
471 | 18.5k | SCReturnPtr(tbuffer, "InspectionBuffer"); |
472 | 18.5k | } |
473 | 22.2k | SCReturnPtr(buffer, "InspectionBuffer"); |
474 | | |
475 | 6.82k | empty_return: |
476 | 6.82k | InspectionBufferSetupMultiEmpty(buffer); |
477 | 6.82k | return NULL; |
478 | 22.2k | } |
479 | | |
480 | | uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, |
481 | | const DetectEngineAppInspectionEngine *engine, const Signature *s, Flow *f, uint8_t flags, |
482 | | void *alstate, void *txv, uint64_t tx_id) |
483 | 46.0k | { |
484 | 46.0k | const DetectEngineTransforms *transforms = NULL; |
485 | 46.0k | if (!engine->mpm) { |
486 | 43.7k | transforms = engine->v2.transforms; |
487 | 43.7k | } |
488 | | |
489 | 46.0k | AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); |
490 | 46.0k | FileContainer *ffc = files.fc; |
491 | 46.0k | if (ffc == NULL) { |
492 | 817 | return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; |
493 | 817 | } |
494 | 45.2k | if (ffc->head == NULL) { |
495 | 27.2k | const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > |
496 | 27.2k | engine->progress); |
497 | 27.2k | if (eof && engine->match_on_null) { |
498 | 48 | return DETECT_ENGINE_INSPECT_SIG_MATCH; |
499 | 48 | } |
500 | 27.1k | return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; |
501 | 27.2k | } |
502 | | |
503 | 18.0k | int local_file_id = 0; |
504 | 18.0k | File *file = ffc->head; |
505 | 24.6k | for (; file != NULL; file = file->next) { |
506 | 18.0k | InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, transforms, f, flags, file, |
507 | 18.0k | engine->sm_list, engine->sm_list_base, local_file_id, txv); |
508 | 18.0k | if (buffer == NULL) { |
509 | 2.89k | local_file_id++; |
510 | 2.89k | continue; |
511 | 2.89k | } |
512 | | |
513 | 18.0k | bool eof = (file->state == FILE_STATE_CLOSED); |
514 | 15.1k | uint8_t ciflags = eof ? DETECT_CI_FLAGS_END : 0; |
515 | 15.1k | if (buffer->inspect_offset == 0) |
516 | 13.6k | ciflags |= DETECT_CI_FLAGS_START; |
517 | | |
518 | 15.1k | const bool match = DetectEngineContentInspection(de_ctx, det_ctx, s, engine->smd, NULL, f, |
519 | 15.1k | buffer->inspect, buffer->inspect_len, buffer->inspect_offset, ciflags, |
520 | 15.1k | DETECT_ENGINE_CONTENT_INSPECTION_MODE_STATE); |
521 | 15.1k | if (match) { |
522 | 11.4k | return DETECT_ENGINE_INSPECT_SIG_MATCH; |
523 | 11.4k | } |
524 | 3.73k | local_file_id++; |
525 | 3.73k | } |
526 | | |
527 | 6.61k | return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; |
528 | 18.0k | } |
529 | | |
530 | | /** \brief Filedata Filedata Mpm prefilter callback |
531 | | * |
532 | | * \param det_ctx detection engine thread ctx |
533 | | * \param pectx inspection context |
534 | | * \param p packet to inspect |
535 | | * \param f flow to inspect |
536 | | * \param txv tx to inspect |
537 | | * \param idx transaction id |
538 | | * \param flags STREAM_* flags including direction |
539 | | */ |
540 | | static void PrefilterTxFiledata(DetectEngineThreadCtx *det_ctx, const void *pectx, Packet *p, |
541 | | Flow *f, void *txv, const uint64_t idx, const AppLayerTxData *txd, const uint8_t flags) |
542 | 48.2k | { |
543 | 48.2k | SCEnter(); |
544 | | |
545 | 48.2k | if (!AppLayerParserHasFilesInDir(txd, flags)) |
546 | 32.9k | return; |
547 | | |
548 | 15.3k | const PrefilterMpmFiledata *ctx = (const PrefilterMpmFiledata *)pectx; |
549 | 15.3k | const MpmCtx *mpm_ctx = ctx->mpm_ctx; |
550 | 15.3k | const int list_id = ctx->list_id; |
551 | | |
552 | 15.3k | AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); |
553 | 15.3k | FileContainer *ffc = files.fc; |
554 | 15.3k | if (ffc != NULL) { |
555 | 15.3k | int local_file_id = 0; |
556 | 29.1k | for (File *file = ffc->head; file != NULL; file = file->next) { |
557 | 13.8k | InspectionBuffer *buffer = FiledataGetDataCallback(det_ctx, ctx->transforms, f, flags, |
558 | 13.8k | file, list_id, ctx->base_list_id, local_file_id, txv); |
559 | 13.8k | if (buffer == NULL) { |
560 | 3.92k | local_file_id++; |
561 | 3.92k | continue; |
562 | 3.92k | } |
563 | 9.89k | SCLogDebug("[%" PRIu64 "] buffer size %u", p->pcap_cnt, buffer->inspect_len); |
564 | | |
565 | 9.89k | if (buffer->inspect_len >= mpm_ctx->minlen) { |
566 | 7.32k | uint32_t prev_rule_id_array_cnt = det_ctx->pmq.rule_id_array_cnt; |
567 | 7.32k | (void)mpm_table[mpm_ctx->mpm_type].Search(mpm_ctx, &det_ctx->mtc, &det_ctx->pmq, |
568 | 7.32k | buffer->inspect, buffer->inspect_len); |
569 | 7.32k | PREFILTER_PROFILING_ADD_BYTES(det_ctx, buffer->inspect_len); |
570 | | |
571 | 7.32k | if (det_ctx->pmq.rule_id_array_cnt > prev_rule_id_array_cnt) { |
572 | 1.00k | SCLogDebug( |
573 | 1.00k | "%u matches", det_ctx->pmq.rule_id_array_cnt - prev_rule_id_array_cnt); |
574 | 1.00k | } |
575 | 7.32k | } |
576 | 9.89k | local_file_id++; |
577 | 9.89k | } |
578 | 15.3k | } |
579 | 15.3k | } |
580 | | |
581 | | int PrefilterMpmFiledataRegister(DetectEngineCtx *de_ctx, SigGroupHead *sgh, MpmCtx *mpm_ctx, |
582 | | const DetectBufferMpmRegistry *mpm_reg, int list_id) |
583 | 74.4k | { |
584 | 74.4k | PrefilterMpmFiledata *pectx = SCCalloc(1, sizeof(*pectx)); |
585 | 74.4k | if (pectx == NULL) |
586 | 0 | return -1; |
587 | 74.4k | pectx->list_id = list_id; |
588 | 74.4k | pectx->base_list_id = mpm_reg->sm_list_base; |
589 | 74.4k | pectx->mpm_ctx = mpm_ctx; |
590 | 74.4k | pectx->transforms = &mpm_reg->transforms; |
591 | | |
592 | 74.4k | return PrefilterAppendTxEngine(de_ctx, sgh, PrefilterTxFiledata, |
593 | 74.4k | mpm_reg->app_v2.alproto, mpm_reg->app_v2.tx_min_progress, |
594 | 74.4k | pectx, PrefilterMpmFiledataFree, mpm_reg->pname); |
595 | 74.4k | } |
596 | | |
597 | | #ifdef UNITTESTS |
598 | | #include "tests/detect-file-data.c" |
599 | | #endif |