/src/suricata7/rust/src/filetracker.rs
Line | Count | Source |
1 | | /* Copyright (C) 2017 Open Information Security Foundation |
2 | | * |
3 | | * You can copy, redistribute or modify this Program under the terms of |
4 | | * the GNU General Public License version 2 as published by the Free |
5 | | * Software Foundation. |
6 | | * |
7 | | * This program is distributed in the hope that it will be useful, |
8 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
9 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
10 | | * GNU General Public License for more details. |
11 | | * |
12 | | * You should have received a copy of the GNU General Public License |
13 | | * version 2 along with this program; if not, write to the Free Software |
14 | | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA |
15 | | * 02110-1301, USA. |
16 | | */ |
17 | | |
18 | | //! Gap handling and Chunk-based file transfer tracker module. |
19 | | //! |
20 | | //! GAP handling. If a data gap is encountered, the file is truncated |
21 | | //! and new data is no longer pushed down to the lower level APIs. |
22 | | //! The tracker does continue to follow the file |
23 | | // |
24 | | //! Tracks chunk based file transfers. Chunks may be transferred out |
25 | | //! of order, but cannot be transferred in parallel. So only one |
26 | | //! chunk at a time. |
27 | | //! |
28 | | //! Author: Victor Julien <victor@inliniac.net> |
29 | | |
30 | | use crate::core::*; |
31 | | use std::collections::HashMap; |
32 | | use std::collections::hash_map::Entry::{Occupied, Vacant}; |
33 | | use crate::filecontainer::*; |
34 | | |
35 | | #[derive(Debug)] |
36 | | struct FileChunk { |
37 | | contains_gap: bool, |
38 | | chunk: Vec<u8>, |
39 | | } |
40 | | |
41 | | impl FileChunk { |
42 | 104k | pub fn new(size: u32) -> FileChunk { |
43 | 104k | FileChunk { |
44 | 104k | contains_gap: false, |
45 | 104k | chunk: Vec::with_capacity(size as usize), |
46 | 104k | } |
47 | 104k | } |
48 | | } |
49 | | |
50 | | #[derive(Debug)] |
51 | | #[derive(Default)] |
52 | | pub struct FileTransferTracker { |
53 | | pub tracked: u64, |
54 | | cur_ooo: u64, // how many bytes do we have queued from ooo chunks |
55 | | track_id: u32, |
56 | | chunk_left: u32, |
57 | | |
58 | | pub file: FileContainer, |
59 | | pub file_flags: u16, |
60 | | |
61 | | pub tx_id: u64, |
62 | | |
63 | | fill_bytes: u8, |
64 | | pub file_open: bool, |
65 | | file_closed: bool, |
66 | | chunk_is_last: bool, |
67 | | chunk_is_ooo: bool, |
68 | | file_is_truncated: bool, |
69 | | |
70 | | chunks: HashMap<u64, FileChunk>, |
71 | | cur_ooo_chunk_offset: u64, |
72 | | |
73 | | in_flight: u64, |
74 | | } |
75 | | |
76 | | impl FileTransferTracker { |
77 | 2.53M | pub fn new() -> FileTransferTracker { |
78 | 2.53M | FileTransferTracker { |
79 | 2.53M | chunks:HashMap::new(), |
80 | 2.53M | ..Default::default() |
81 | 2.53M | } |
82 | 2.53M | } |
83 | | |
84 | 203k | pub fn is_done(&self) -> bool { |
85 | 203k | !self.file_open |
86 | 203k | } |
87 | | |
88 | 207k | pub fn is_initialized(&self) -> bool { |
89 | 207k | return self.file_open || self.file_is_truncated || self.file_closed; |
90 | 207k | } |
91 | | |
92 | 353k | fn open(&mut self, config: &'static SuricataFileContext, name: &[u8]) -> i32 |
93 | | { |
94 | 353k | let r = self.file.file_open(config, self.track_id, name, self.file_flags); |
95 | 353k | if r == 0 { |
96 | 353k | self.file_open = true; |
97 | 353k | } |
98 | 353k | r |
99 | 353k | } |
100 | | |
101 | 254k | pub fn close(&mut self, config: &'static SuricataFileContext) |
102 | | { |
103 | 254k | if !self.file_is_truncated { |
104 | 252k | SCLogDebug!("closing file with id {}", self.track_id); |
105 | 252k | self.file.file_close(config, &self.track_id, self.file_flags); |
106 | 252k | } |
107 | 254k | self.file_open = false; |
108 | 254k | self.file_closed = true; |
109 | 254k | self.tracked = 0; |
110 | 254k | } |
111 | | |
112 | 999k | pub fn trunc (&mut self, config: &'static SuricataFileContext) |
113 | | { |
114 | 999k | if self.file_is_truncated || !self.file_open { |
115 | 992k | return; |
116 | 7.46k | } |
117 | 7.46k | let myflags = self.file_flags | 1; // TODO util-file.c::FILE_TRUNCATED |
118 | 7.46k | self.file.file_close(config, &self.track_id, myflags); |
119 | | SCLogDebug!("truncated file"); |
120 | 7.46k | self.file_is_truncated = true; |
121 | 7.46k | self.chunks.clear(); |
122 | 7.46k | self.in_flight = 0; |
123 | 7.46k | self.cur_ooo = 0; |
124 | 999k | } |
125 | | |
126 | 1.65M | pub fn new_chunk(&mut self, config: &'static SuricataFileContext, |
127 | 1.65M | name: &[u8], data: &[u8], chunk_offset: u64, chunk_size: u32, |
128 | 1.65M | fill_bytes: u8, is_last: bool, xid: &u32) -> u32 |
129 | | { |
130 | 1.65M | if self.chunk_left != 0 || self.fill_bytes != 0 { |
131 | 987k | SCLogDebug!("current chunk incomplete: truncating"); |
132 | 987k | self.trunc(config); |
133 | 987k | } |
134 | | |
135 | | SCLogDebug!("NEW CHUNK: chunk_size {} fill_bytes {}", chunk_size, fill_bytes); |
136 | | |
137 | | // for now assume that is_last means its really the last chunk |
138 | | // so no out of order chunks coming after. This means that if |
139 | | // the last chunk is out or order, we've missed chunks before. |
140 | 1.65M | if chunk_offset != self.tracked { |
141 | | SCLogDebug!("NEW CHUNK IS OOO: expected {}, got {}", self.tracked, chunk_offset); |
142 | 228k | if is_last { |
143 | 12.2k | SCLogDebug!("last chunk is out of order, this means we missed data before"); |
144 | 12.2k | self.trunc(config); |
145 | 216k | } |
146 | 228k | self.chunk_is_ooo = true; |
147 | 228k | self.cur_ooo_chunk_offset = chunk_offset; |
148 | 1.42M | } |
149 | | |
150 | 1.65M | self.chunk_left = chunk_size; |
151 | 1.65M | self.fill_bytes = fill_bytes; |
152 | 1.65M | self.chunk_is_last = is_last; |
153 | | |
154 | 1.65M | if self.file_is_truncated || self.file_closed { |
155 | 1.02M | return 0; |
156 | 633k | } |
157 | 633k | if !self.file_open { |
158 | 353k | SCLogDebug!("NEW CHUNK: FILE OPEN"); |
159 | 353k | self.track_id = *xid; |
160 | 353k | self.open(config, name); |
161 | 353k | } |
162 | | |
163 | 633k | if self.file_open { |
164 | 633k | let res = self.update(config, data, 0); |
165 | | SCLogDebug!("NEW CHUNK: update res {:?}", res); |
166 | 633k | return res; |
167 | 0 | } |
168 | | |
169 | 0 | 0 |
170 | 1.65M | } |
171 | | |
172 | | /// update the file tracker |
173 | | /// If gap_size > 0 'data' should not be used. |
174 | | /// return how much we consumed of data |
175 | 801k | pub fn update(&mut self, config: &'static SuricataFileContext, data: &[u8], gap_size: u32) -> u32 |
176 | | { |
177 | 801k | if self.file_is_truncated { |
178 | 2.19k | let consumed = std::cmp::min(data.len() as u32, self.chunk_left); |
179 | 2.19k | self.chunk_left = self.chunk_left.saturating_sub(data.len() as u32); |
180 | 2.19k | return consumed; |
181 | 799k | } |
182 | 799k | let mut consumed = 0_usize; |
183 | 799k | let is_gap = gap_size > 0; |
184 | 799k | if is_gap || gap_size > 0 { |
185 | 244 | SCLogDebug!("is_gap {} size {} ooo? {}", is_gap, gap_size, self.chunk_is_ooo); |
186 | 798k | } |
187 | | |
188 | 799k | if self.chunk_left == 0 && self.fill_bytes == 0 { |
189 | | //SCLogDebug!("UPDATE: nothing to do"); |
190 | 362k | if self.chunk_is_last { |
191 | 165k | SCLogDebug!("last empty chunk, closing"); |
192 | 165k | self.close(config); |
193 | 165k | self.chunk_is_last = false; |
194 | 197k | } |
195 | 362k | return 0 |
196 | 436k | } else if self.chunk_left == 0 { |
197 | | SCLogDebug!("FILL BYTES {} from prev run", self.fill_bytes); |
198 | 5.43k | if data.len() >= self.fill_bytes as usize { |
199 | 1.63k | consumed += self.fill_bytes as usize; |
200 | 1.63k | self.fill_bytes = 0; |
201 | 1.63k | SCLogDebug!("CHUNK(pre) fill bytes now 0"); |
202 | 3.79k | } else { |
203 | 3.79k | consumed += data.len(); |
204 | 3.79k | self.fill_bytes -= data.len() as u8; |
205 | 3.79k | SCLogDebug!("CHUNK(pre) fill bytes now still {}", self.fill_bytes); |
206 | 3.79k | } |
207 | | SCLogDebug!("FILL BYTES: returning {}", consumed); |
208 | 5.43k | return consumed as u32 |
209 | 431k | } |
210 | | SCLogDebug!("UPDATE: data {} chunk_left {}", data.len(), self.chunk_left); |
211 | | |
212 | 431k | if self.chunk_left > 0 { |
213 | 431k | if self.chunk_left <= data.len() as u32 { |
214 | 300k | let d = &data[0..self.chunk_left as usize]; |
215 | | |
216 | 300k | if !self.chunk_is_ooo { |
217 | 163k | let res = self.file.file_append(config, &self.track_id, d, is_gap); |
218 | 163k | match res { |
219 | 162k | 0 => { }, |
220 | 0 | -2 => { |
221 | 0 | self.file_is_truncated = true; |
222 | 0 | }, |
223 | 409 | _ => { |
224 | 409 | SCLogDebug!("got error so truncating file"); |
225 | 409 | self.file_is_truncated = true; |
226 | 409 | }, |
227 | | } |
228 | | |
229 | 163k | self.tracked += self.chunk_left as u64; |
230 | | } else { |
231 | | SCLogDebug!("UPDATE: appending data {} to ooo chunk at offset {}/{}", |
232 | | d.len(), self.cur_ooo_chunk_offset, self.tracked); |
233 | 137k | let c = match self.chunks.entry(self.cur_ooo_chunk_offset) { |
234 | 97.9k | Vacant(entry) => { |
235 | 97.9k | entry.insert(FileChunk::new(self.chunk_left)) |
236 | | }, |
237 | 39.5k | Occupied(entry) => entry.into_mut(), |
238 | | }; |
239 | 137k | self.cur_ooo += d.len() as u64; |
240 | 137k | c.contains_gap |= is_gap; |
241 | 137k | c.chunk.extend(d); |
242 | | |
243 | 137k | self.in_flight += d.len() as u64; |
244 | | SCLogDebug!("{:p} in_flight {}", self, self.in_flight); |
245 | | } |
246 | | |
247 | 300k | consumed += self.chunk_left as usize; |
248 | 300k | if self.fill_bytes > 0 { |
249 | 11.5k | let extra = data.len() - self.chunk_left as usize; |
250 | 11.5k | if extra >= self.fill_bytes as usize { |
251 | 310 | consumed += self.fill_bytes as usize; |
252 | 310 | self.fill_bytes = 0; |
253 | 310 | SCLogDebug!("CHUNK(post) fill bytes now 0"); |
254 | 11.2k | } else { |
255 | 11.2k | consumed += extra; |
256 | 11.2k | self.fill_bytes -= extra as u8; |
257 | 11.2k | SCLogDebug!("CHUNK(post) fill bytes now still {}", self.fill_bytes); |
258 | 11.2k | } |
259 | 11.5k | self.chunk_left = 0; |
260 | | } else { |
261 | 289k | self.chunk_left = 0; |
262 | | |
263 | 289k | if !self.chunk_is_ooo { |
264 | | loop { |
265 | 162k | let _offset = self.tracked; |
266 | 162k | match self.chunks.remove(&self.tracked) { |
267 | 6.07k | Some(c) => { |
268 | 6.07k | self.in_flight -= c.chunk.len() as u64; |
269 | | |
270 | 6.07k | let res = self.file.file_append(config, &self.track_id, &c.chunk, c.contains_gap); |
271 | 6.07k | match res { |
272 | 5.67k | 0 => { }, |
273 | 0 | -2 => { |
274 | 0 | self.file_is_truncated = true; |
275 | 0 | }, |
276 | 397 | _ => { |
277 | 397 | SCLogDebug!("got error so truncating file"); |
278 | 397 | self.file_is_truncated = true; |
279 | 397 | }, |
280 | | } |
281 | | |
282 | 6.07k | self.tracked += c.chunk.len() as u64; |
283 | 6.07k | self.cur_ooo -= c.chunk.len() as u64; |
284 | | |
285 | | SCLogDebug!("STORED OOO CHUNK at offset {}, tracked now {}, stored len {}", _offset, self.tracked, c.chunk.len()); |
286 | | }, |
287 | | _ => { |
288 | | SCLogDebug!("NO STORED CHUNK found at _offset {}", self.tracked); |
289 | 156k | break; |
290 | | }, |
291 | | }; |
292 | | } |
293 | 132k | } else { |
294 | 132k | SCLogDebug!("UPDATE: complete ooo chunk. Offset {}", self.cur_ooo_chunk_offset); |
295 | 132k | |
296 | 132k | self.chunk_is_ooo = false; |
297 | 132k | self.cur_ooo_chunk_offset = 0; |
298 | 132k | } |
299 | | } |
300 | 300k | if self.chunk_is_last { |
301 | 38.1k | SCLogDebug!("last chunk, closing"); |
302 | 38.1k | self.close(config); |
303 | 38.1k | self.chunk_is_last = false; |
304 | 262k | } else { |
305 | 262k | SCLogDebug!("NOT last chunk, keep going"); |
306 | 262k | } |
307 | | |
308 | | } else { |
309 | 130k | if !self.chunk_is_ooo { |
310 | 111k | let res = self.file.file_append(config, &self.track_id, data, is_gap); |
311 | 111k | match res { |
312 | 111k | 0 => { }, |
313 | 0 | -2 => { |
314 | 0 | self.file_is_truncated = true; |
315 | 0 | }, |
316 | 220 | _ => { |
317 | 220 | SCLogDebug!("got error so truncating file"); |
318 | 220 | self.file_is_truncated = true; |
319 | 220 | }, |
320 | | } |
321 | 111k | self.tracked += data.len() as u64; |
322 | | } else { |
323 | 18.5k | let c = match self.chunks.entry(self.cur_ooo_chunk_offset) { |
324 | 6.44k | Vacant(entry) => entry.insert(FileChunk::new(32768)), |
325 | 12.1k | Occupied(entry) => entry.into_mut(), |
326 | | }; |
327 | 18.5k | c.chunk.extend(data); |
328 | 18.5k | c.contains_gap |= is_gap; |
329 | 18.5k | self.cur_ooo += data.len() as u64; |
330 | 18.5k | self.in_flight += data.len() as u64; |
331 | | } |
332 | | |
333 | 130k | self.chunk_left -= data.len() as u32; |
334 | 130k | consumed += data.len(); |
335 | | } |
336 | 0 | } |
337 | 431k | consumed as u32 |
338 | 801k | } |
339 | | |
340 | 1.91k | pub fn get_queued_size(&self) -> u64 { |
341 | 1.91k | self.cur_ooo |
342 | 1.91k | } |
343 | | |
344 | 36.1k | pub fn get_inflight_size(&self) -> u64 { |
345 | 36.1k | self.in_flight |
346 | 36.1k | } |
347 | 36.1k | pub fn get_inflight_cnt(&self) -> usize { |
348 | 36.1k | self.chunks.len() |
349 | 36.1k | } |
350 | | } |