/rust/registry/src/index.crates.io-1949cf8c6b5b557f/ureq-3.3.0/src/body/lossy.rs
Line | Count | Source |
1 | | use std::io; |
2 | | |
3 | | use utf8_zero::DecodeError; |
4 | | |
5 | | use crate::util::ConsumeBuf; |
6 | | |
7 | | const REPLACEMENT_CHAR: u8 = b'?'; |
8 | | const MIN_BUF: usize = 8; |
9 | | |
10 | | pub struct LossyUtf8Reader<R> { |
11 | | reader: R, |
12 | | ended: bool, |
13 | | input: ConsumeBuf, |
14 | | valid_len: usize, |
15 | | } |
16 | | impl<R> LossyUtf8Reader<R> { |
17 | 0 | pub(crate) fn new(reader: R) -> Self { |
18 | 0 | Self { |
19 | 0 | reader, |
20 | 0 | ended: false, |
21 | 0 | input: ConsumeBuf::new(8), |
22 | 0 | valid_len: 0, |
23 | 0 | } |
24 | 0 | } |
25 | | |
26 | 0 | fn process_input(&mut self) -> usize { |
27 | 0 | match utf8_zero::decode(self.input.unconsumed()) { |
28 | | Ok(_) => { |
29 | | // Entire input is valid |
30 | 0 | self.input.unconsumed().len() |
31 | | } |
32 | 0 | Err(e) => match e { |
33 | | DecodeError::Invalid { |
34 | 0 | valid_prefix, |
35 | 0 | invalid_sequence, |
36 | | .. |
37 | | } => { |
38 | 0 | let valid_len = valid_prefix.len(); |
39 | 0 | let invalid_len = invalid_sequence.len(); |
40 | | |
41 | | // Switch out the problem input chars |
42 | 0 | let replace_in = self.input.unconsumed_mut(); |
43 | 0 | for i in 0..invalid_len { |
44 | 0 | replace_in[valid_len + i] = REPLACEMENT_CHAR; |
45 | 0 | } |
46 | | |
47 | 0 | valid_len + invalid_len |
48 | | } |
49 | 0 | DecodeError::Incomplete { valid_prefix, .. } => { |
50 | 0 | let valid_len = valid_prefix.len(); |
51 | | |
52 | 0 | if self.ended { |
53 | | // blank the rest |
54 | 0 | let replace_in = self.input.unconsumed_mut(); |
55 | 0 | let invalid_len = replace_in.len() - valid_len; |
56 | 0 | for i in 0..invalid_len { |
57 | 0 | replace_in[valid_len + i] = REPLACEMENT_CHAR; |
58 | 0 | } |
59 | 0 | valid_len + invalid_len |
60 | | } else { |
61 | 0 | valid_len |
62 | | } |
63 | | } |
64 | | }, |
65 | | } |
66 | 0 | } |
67 | | } |
68 | | |
69 | | impl<R: io::Read> io::Read for LossyUtf8Reader<R> { |
70 | 0 | fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { |
71 | | // Match the input buffer size |
72 | 0 | if !self.ended { |
73 | 0 | let total_len = self.input.unconsumed().len() + self.input.free_mut().len(); |
74 | 0 | let wanted_len = buf.len().max(MIN_BUF); |
75 | 0 | if wanted_len < total_len { |
76 | 0 | self.input.add_space(total_len - wanted_len); |
77 | 0 | } |
78 | 0 | } |
79 | | |
80 | | // Fill up to a point where we definitely will make progress. |
81 | 0 | while !self.ended && self.input.unconsumed().len() < MIN_BUF { |
82 | 0 | let amount = self.reader.read(self.input.free_mut())?; |
83 | 0 | self.input.add_filled(amount); |
84 | | |
85 | 0 | if amount == 0 { |
86 | 0 | self.ended = true; |
87 | 0 | } |
88 | | } |
89 | | |
90 | 0 | if self.ended && self.input.unconsumed().is_empty() { |
91 | 0 | return Ok(0); |
92 | 0 | } |
93 | | |
94 | 0 | if self.valid_len == 0 { |
95 | 0 | self.valid_len = self.process_input(); |
96 | 0 | assert!(self.valid_len > 0); |
97 | 0 | } |
98 | | |
99 | 0 | let src = &self.input.unconsumed()[..self.valid_len]; |
100 | 0 | let max = src.len().min(buf.len()); |
101 | 0 | buf[..max].copy_from_slice(&src[..max]); |
102 | 0 | self.input.consume(max); |
103 | | |
104 | 0 | self.valid_len -= max; |
105 | | |
106 | 0 | Ok(max) |
107 | 0 | } |
108 | | } |
109 | | |
110 | | #[cfg(test)] |
111 | | mod test { |
112 | | use std::io::Read; |
113 | | |
114 | | use super::*; |
115 | | |
116 | | fn do_reader<'a>(bytes: &'a mut [&'a [u8]]) -> String { |
117 | | let mut r = LossyUtf8Reader::new(TestReader(bytes)); |
118 | | let mut buf = String::new(); |
119 | | r.read_to_string(&mut buf).unwrap(); |
120 | | buf |
121 | | } |
122 | | |
123 | | #[test] |
124 | | fn ascii() { |
125 | | assert_eq!(do_reader(&mut [b"abc123"]), "abc123"); |
126 | | } |
127 | | |
128 | | #[test] |
129 | | fn utf8_one_read() { |
130 | | assert_eq!(do_reader(&mut ["åiåaäeö".as_bytes()]), "åiåaäeö"); |
131 | | } |
132 | | |
133 | | #[test] |
134 | | fn utf8_chopped_single_char() { |
135 | | assert_eq!(do_reader(&mut [&[195], &[165]]), "å"); |
136 | | } |
137 | | |
138 | | #[test] |
139 | | fn utf8_chopped_prefix_ascii() { |
140 | | assert_eq!(do_reader(&mut [&[97, 97, 97, 195], &[165]]), "aaaå"); |
141 | | } |
142 | | |
143 | | #[test] |
144 | | fn utf8_chopped_suffix_ascii() { |
145 | | assert_eq!(do_reader(&mut [&[195], &[165, 97, 97, 97]]), "åaaa"); |
146 | | } |
147 | | |
148 | | #[test] |
149 | | fn utf8_broken_single() { |
150 | | assert_eq!(do_reader(&mut [&[195]]), "?"); |
151 | | } |
152 | | |
153 | | #[test] |
154 | | fn utf8_broken_suffix_ascii() { |
155 | | assert_eq!(do_reader(&mut [&[195, 97, 97, 97]]), "?aaa"); |
156 | | } |
157 | | |
158 | | #[test] |
159 | | fn utf8_broken_prefix_ascii() { |
160 | | assert_eq!(do_reader(&mut [&[97, 97, 97, 195]]), "aaa?"); |
161 | | } |
162 | | |
163 | | #[test] |
164 | | fn hiragana() { |
165 | | assert_eq!(do_reader(&mut ["あいうえお".as_bytes()]), "あいうえお"); |
166 | | } |
167 | | |
168 | | #[test] |
169 | | fn emoji() { |
170 | | assert_eq!(do_reader(&mut ["✅✅✅".as_bytes()]), "✅✅✅"); |
171 | | } |
172 | | |
173 | | #[test] |
174 | | fn leftover() { |
175 | | let s = "あ"; |
176 | | assert_eq!(s.as_bytes(), &[227, 129, 130]); |
177 | | |
178 | | let mut buf = [0; 2]; |
179 | | let mut r = LossyUtf8Reader::new(s.as_bytes()); |
180 | | |
181 | | assert_eq!(r.read(&mut buf).unwrap(), 2); |
182 | | assert_eq!(&buf[..], &[227, 129]); |
183 | | |
184 | | assert_eq!(r.read(&mut buf).unwrap(), 1); |
185 | | assert_eq!(&buf[..1], &[130]); |
186 | | |
187 | | assert_eq!(r.read(&mut buf).unwrap(), 0); |
188 | | } |
189 | | |
190 | | struct TestReader<'a>(&'a mut [&'a [u8]]); |
191 | | |
192 | | impl<'a> io::Read for TestReader<'a> { |
193 | | fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { |
194 | | if self.0.iter().all(|c| c.is_empty()) { |
195 | | return Ok(0); |
196 | | } |
197 | | |
198 | | let pos = self.0.iter().position(|c| !c.is_empty()).unwrap(); |
199 | | let cur = &self.0[pos]; |
200 | | |
201 | | let max = cur.len().min(buf.len()); |
202 | | buf[..max].copy_from_slice(&cur[..max]); |
203 | | |
204 | | self.0[pos] = &cur[max..]; |
205 | | |
206 | | Ok(max) |
207 | | } |
208 | | } |
209 | | } |