/rust/registry/src/index.crates.io-1949cf8c6b5b557f/ureq-3.4.2/src/pool.rs
Line | Count | Source |
1 | | use std::collections::VecDeque; |
2 | | use std::fmt; |
3 | | use std::sync::{Arc, Mutex, Weak}; |
4 | | |
5 | | use http::Uri; |
6 | | use http::uri::{Authority, Scheme}; |
7 | | |
8 | | use crate::Error; |
9 | | use crate::config::Config; |
10 | | use crate::http; |
11 | | use crate::proxy::Proxy; |
12 | | use crate::transport::time::{Duration, Instant}; |
13 | | use crate::transport::{Buffers, ConnectionDetails, Connector, NextTimeout, Transport}; |
14 | | use crate::util::DebugAuthority; |
15 | | |
16 | | pub(crate) struct ConnectionPool { |
17 | | connector: Box<dyn Connector<Out = Box<dyn Transport>>>, |
18 | | pool: Arc<Mutex<Pool>>, |
19 | | } |
20 | | |
21 | | impl ConnectionPool { |
22 | 0 | pub fn new(connector: Box<dyn Connector<Out = Box<dyn Transport>>>, config: &Config) -> Self { |
23 | 0 | ConnectionPool { |
24 | 0 | connector, |
25 | 0 | pool: Arc::new(Mutex::new(Pool::new(config))), |
26 | 0 | } |
27 | 0 | } |
28 | | |
29 | 0 | pub fn connect( |
30 | 0 | &self, |
31 | 0 | details: &ConnectionDetails, |
32 | 0 | max_idle_age: Duration, |
33 | 0 | use_pool: bool, |
34 | 0 | ) -> Result<Connection, Error> { |
35 | 0 | let key = details.into(); |
36 | | |
37 | 0 | if use_pool { |
38 | 0 | let mut pool = self.pool.lock().unwrap(); |
39 | 0 | pool.purge(details.now); |
40 | | |
41 | 0 | if let Some(conn) = pool.get(&key, max_idle_age, details.now) { |
42 | 0 | debug!("Use pooled: {:?}", key); |
43 | 0 | return Ok(conn); |
44 | 0 | } |
45 | 0 | } |
46 | | |
47 | 0 | let transport = self.run_connector(details)?; |
48 | | |
49 | 0 | let conn = Connection { |
50 | 0 | transport, |
51 | 0 | key, |
52 | 0 | last_use: details.now, |
53 | 0 | pool: if use_pool { |
54 | 0 | Arc::downgrade(&self.pool) |
55 | | } else { |
56 | | // An incompatible request must neither borrow from nor return |
57 | | // its newly established connection to the Agent's pool. |
58 | 0 | Weak::new() |
59 | | }, |
60 | 0 | position_per_host: None, |
61 | | }; |
62 | | |
63 | 0 | Ok(conn) |
64 | 0 | } |
65 | | |
66 | 0 | pub fn run_connector(&self, details: &ConnectionDetails) -> Result<Box<dyn Transport>, Error> { |
67 | 0 | let transport = self |
68 | 0 | .connector |
69 | 0 | .connect(details, None)? |
70 | 0 | .ok_or(Error::ConnectionFailed)?; |
71 | | |
72 | 0 | Ok(transport) |
73 | 0 | } |
74 | | |
75 | | #[cfg(test)] |
76 | | /// Exposed for testing the pool count. |
77 | | pub fn pool_count(&self) -> usize { |
78 | | let lock = self.pool.lock().unwrap(); |
79 | | lock.lru.len() |
80 | | } |
81 | | } |
82 | | |
83 | | pub(crate) struct Connection { |
84 | | transport: Box<dyn Transport>, |
85 | | key: PoolKey, |
86 | | last_use: Instant, |
87 | | pool: Weak<Mutex<Pool>>, |
88 | | |
89 | | /// Used to prune max_idle_connections_by_host. |
90 | | /// |
91 | | /// # Example |
92 | | /// |
93 | | /// If we have a max idle per hosts set to 3, and we have the following LRU: |
94 | | /// |
95 | | /// ```text |
96 | | /// [B, A, A, B, A, B, A] |
97 | | /// ``` |
98 | | /// |
99 | | /// This field is used to enumerate the elements per host reverse: |
100 | | /// |
101 | | /// ```text |
102 | | /// [B2, A3, A2, B1, A1, B0, A0] |
103 | | /// ``` |
104 | | /// |
105 | | /// Once we have that enumeration, we can drop elements from the front where there |
106 | | /// position_per_host >= idle_per_host. |
107 | | position_per_host: Option<usize>, |
108 | | } |
109 | | |
110 | | impl Connection { |
111 | 0 | pub fn buffers(&mut self) -> &mut dyn Buffers { |
112 | 0 | self.transport.buffers() |
113 | 0 | } |
114 | | |
115 | 0 | pub fn transmit_output(&mut self, amount: usize, timeout: NextTimeout) -> Result<(), Error> { |
116 | | // An already expired budget must fail here. Transports can't set a zero |
117 | | // socket timeout and would instead grant a short grace period per call. |
118 | 0 | if timeout.after.is_zero() { |
119 | 0 | return Err(Error::Timeout(timeout.reason)); |
120 | 0 | } |
121 | 0 | self.transport.transmit_output(amount, timeout) |
122 | 0 | } |
123 | | |
124 | 0 | pub fn maybe_await_input(&mut self, timeout: NextTimeout) -> Result<bool, Error> { |
125 | 0 | if timeout.after.is_zero() { |
126 | 0 | return Err(Error::Timeout(timeout.reason)); |
127 | 0 | } |
128 | 0 | self.transport.maybe_await_input(timeout) |
129 | 0 | } |
130 | | |
131 | 0 | pub fn consume_input(&mut self, amount: usize) { |
132 | 0 | self.transport.buffers().input_consume(amount) |
133 | 0 | } |
134 | | |
135 | 0 | pub fn close(self) { |
136 | 0 | debug!("Close: {:?}", self.key); |
137 | | // Just consume self. |
138 | 0 | } |
139 | | |
140 | 0 | pub fn reuse(mut self, now: Instant) { |
141 | 0 | if !self.transport.buffers().input().is_empty() { |
142 | | // Unconsumed input means the server sent more bytes than the body |
143 | | // we read. Same condition as the probe below, only the bytes are |
144 | | // already in our buffer instead of still in the socket. |
145 | 0 | debug!("Unconsumed input. Closing connection"); |
146 | 0 | return; |
147 | 0 | } |
148 | | |
149 | 0 | if !self.transport.is_open() { |
150 | | // The purpose of probing is that is_open() for tcp connector attempts |
151 | | // to read some more bytes. If that succeeds, the connection is considered |
152 | | // _NOT_ open, since that means we either failed to read the previous |
153 | | // body to end, or the server sent bogus data after the body. Either |
154 | | // is a condition where we mustn't reuse the connection. |
155 | 0 | return; |
156 | 0 | } |
157 | 0 | self.last_use = now; |
158 | | |
159 | 0 | let Some(arc) = self.pool.upgrade() else { |
160 | 0 | debug!("Pool gone: {:?}", self.key); |
161 | 0 | return; |
162 | | }; |
163 | | |
164 | 0 | debug!("Return to pool: {:?}", self.key); |
165 | | |
166 | 0 | let mut pool = arc.lock().unwrap(); |
167 | | |
168 | 0 | pool.add(self); |
169 | 0 | pool.purge(now); |
170 | 0 | } |
171 | | |
172 | 0 | pub fn is_tls(&self) -> bool { |
173 | 0 | self.transport.is_tls() |
174 | 0 | } |
175 | | |
176 | 0 | fn age(&self, now: Instant) -> Duration { |
177 | 0 | now.duration_since(self.last_use) |
178 | 0 | } |
179 | | |
180 | 0 | fn is_open(&mut self) -> bool { |
181 | 0 | self.transport.is_open() |
182 | 0 | } |
183 | | } |
184 | | |
185 | | /// The pool key is the Scheme, Authority from the uri and the Proxy setting |
186 | | /// |
187 | | /// |
188 | | /// ```notrust |
189 | | /// abc://username:password@example.com:123/path/data?key=value&key2=value2#fragid1 |
190 | | /// |-| |-------------------------------||--------| |-------------------| |-----| |
191 | | /// | | | | | |
192 | | /// scheme authority path query fragment |
193 | | /// ``` |
194 | | /// |
195 | | /// It's correct to include username/password since connections with differing such and |
196 | | /// the same host/port must not be mixed up. |
197 | | /// |
198 | | #[derive(Clone, PartialEq, Eq)] |
199 | | struct PoolKey(Arc<PoolKeyInner>); |
200 | | |
201 | | impl PoolKey { |
202 | 0 | fn new(uri: &Uri, proxy: Option<&Proxy>) -> Self { |
203 | 0 | let inner = PoolKeyInner( |
204 | 0 | uri.scheme().expect("uri with scheme").clone(), |
205 | 0 | uri.authority().expect("uri with authority").clone(), |
206 | 0 | proxy.cloned(), |
207 | 0 | ); |
208 | | |
209 | 0 | PoolKey(Arc::new(inner)) |
210 | 0 | } |
211 | | } |
212 | | |
213 | | #[derive(PartialEq, Eq)] |
214 | | struct PoolKeyInner(Scheme, Authority, Option<Proxy>); |
215 | | |
216 | | #[derive(Debug)] |
217 | | struct Pool { |
218 | | lru: VecDeque<Connection>, |
219 | | max_idle_connections: usize, |
220 | | max_idle_connections_per_host: usize, |
221 | | max_idle_age: Duration, |
222 | | } |
223 | | |
224 | | impl Pool { |
225 | 0 | fn new(config: &Config) -> Self { |
226 | 0 | Pool { |
227 | 0 | lru: VecDeque::new(), |
228 | 0 | max_idle_connections: config.max_idle_connections(), |
229 | 0 | max_idle_connections_per_host: config.max_idle_connections_per_host(), |
230 | 0 | max_idle_age: config.max_idle_age().into(), |
231 | 0 | } |
232 | 0 | } |
233 | | |
234 | 0 | fn purge(&mut self, now: Instant) { |
235 | 0 | while self.lru.len() > self.max_idle_connections || self.front_is_too_old(now) { |
236 | 0 | self.lru.pop_front(); |
237 | 0 | } |
238 | | |
239 | 0 | self.update_position_per_host(); |
240 | | |
241 | 0 | let max = self.max_idle_connections_per_host; |
242 | | |
243 | | // unwrap is ok because update_position_per_host() should have set all |
244 | 0 | self.lru.retain(|c| c.position_per_host.unwrap() < max); |
245 | 0 | } |
246 | | |
247 | 0 | fn front_is_too_old(&self, now: Instant) -> bool { |
248 | 0 | self.lru.front().map(|c| c.age(now)) > Some(self.max_idle_age) |
249 | 0 | } |
250 | | |
251 | 0 | fn update_position_per_host(&mut self) { |
252 | | // Reset position counters |
253 | 0 | for c in &mut self.lru { |
254 | 0 | c.position_per_host = None; |
255 | 0 | } |
256 | | |
257 | | loop { |
258 | 0 | let maybe_uncounted = self |
259 | 0 | .lru |
260 | 0 | .iter() |
261 | 0 | .rev() |
262 | 0 | .find(|c| c.position_per_host.is_none()); |
263 | | |
264 | 0 | let Some(uncounted) = maybe_uncounted else { |
265 | 0 | break; // nothing more to count. |
266 | | }; |
267 | | |
268 | 0 | let key_to_count = uncounted.key.clone(); |
269 | | |
270 | 0 | for (position, c) in self |
271 | 0 | .lru |
272 | 0 | .iter_mut() |
273 | 0 | .rev() |
274 | 0 | .filter(|c| c.key == key_to_count) |
275 | 0 | .enumerate() |
276 | 0 | { |
277 | 0 | c.position_per_host = Some(position); |
278 | 0 | } |
279 | | } |
280 | 0 | } |
281 | | |
282 | 0 | fn add(&mut self, conn: Connection) { |
283 | 0 | self.lru.push_back(conn) |
284 | 0 | } |
285 | | |
286 | 0 | fn get(&mut self, key: &PoolKey, max_idle_age: Duration, now: Instant) -> Option<Connection> { |
287 | 0 | while let Some(i) = self.lru.iter().position(|c| c.key == *key) { |
288 | 0 | let mut conn = self.lru.remove(i).unwrap(); // unwrap ok since we just got the position |
289 | | |
290 | | // Before we release the connection, we probe that it appears to still work. |
291 | 0 | if !conn.is_open() { |
292 | | // This connection is broken. Try find another one. |
293 | 0 | continue; |
294 | 0 | } |
295 | | |
296 | 0 | if conn.age(now) >= max_idle_age { |
297 | | // A max_duration that is shorter in the request than the pool. |
298 | | // This connection survives in the pool, but is not used for this |
299 | | // specific connection. |
300 | 0 | continue; |
301 | 0 | } |
302 | | |
303 | 0 | return Some(conn); |
304 | | } |
305 | 0 | None |
306 | 0 | } |
307 | | } |
308 | | |
309 | | impl fmt::Debug for ConnectionPool { |
310 | 0 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
311 | 0 | f.debug_struct("ConnectionPool") |
312 | 0 | .field("connector", &self.connector) |
313 | 0 | .finish() |
314 | 0 | } |
315 | | } |
316 | | |
317 | | impl fmt::Debug for Connection { |
318 | 0 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
319 | 0 | f.debug_struct("Connection") |
320 | 0 | .field("key", &self.key) |
321 | 0 | .field("conn", &self.transport) |
322 | 0 | .finish() |
323 | 0 | } |
324 | | } |
325 | | |
326 | | impl fmt::Debug for PoolKey { |
327 | 0 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
328 | 0 | f.debug_struct("PoolKey") |
329 | 0 | .field("scheme", &self.0.0) |
330 | 0 | .field("authority", &DebugAuthority(&self.0.1)) |
331 | 0 | .field("proxy", &self.0.2) |
332 | 0 | .finish() |
333 | 0 | } |
334 | | } |
335 | | |
336 | | impl<'a, 'b> From<&'a ConnectionDetails<'b>> for PoolKey { |
337 | 0 | fn from(details: &'a ConnectionDetails) -> Self { |
338 | 0 | PoolKey::new(details.uri, details.config.proxy()) |
339 | 0 | } |
340 | | } |
341 | | |
342 | | #[cfg(all(test, feature = "_test"))] |
343 | | mod test { |
344 | | use super::*; |
345 | | |
346 | | #[test] |
347 | | fn poolkey_new() { |
348 | | // Test that PoolKey::new() does not panic on unrecognized schemes. |
349 | | PoolKey::new(&Uri::from_static("zzz://example.com"), None); |
350 | | } |
351 | | |
352 | | #[test] |
353 | | fn no_reuse_with_unconsumed_input() { |
354 | | use crate::test::init_test_log; |
355 | | use crate::transport::set_handler; |
356 | | |
357 | | init_test_log(); |
358 | | |
359 | | // The body is 5 bytes, but the server sends 9. The 4 extra bytes end |
360 | | // up in the input buffer. A connection with unconsumed input must not |
361 | | // go back into the pool. |
362 | | set_handler("/trailing", 200, &[("content-length", "5")], b"hellojunk"); |
363 | | |
364 | | let agent = crate::Agent::new_with_defaults(); |
365 | | let mut res = agent.get("https://example.test/trailing").call().unwrap(); |
366 | | assert_eq!(res.body_mut().read_to_string().unwrap(), "hello"); |
367 | | |
368 | | assert_eq!(agent.pool_count(), 0); |
369 | | } |
370 | | } |
371 | | |
372 | | #[cfg(test)] |
373 | | mod config_pooling_tests { |
374 | | use super::*; |
375 | | use crate::Agent; |
376 | | use crate::transport::LazyBuffers; |
377 | | use crate::unversioned::resolver::DefaultResolver; |
378 | | use std::sync::atomic::{AtomicUsize, Ordering}; |
379 | | |
380 | | #[derive(Debug)] |
381 | | struct CountingConnector(Arc<AtomicUsize>); |
382 | | |
383 | | impl Connector for CountingConnector { |
384 | | type Out = TestTransport; |
385 | | |
386 | | fn connect( |
387 | | &self, |
388 | | _: &ConnectionDetails, |
389 | | _: Option<()>, |
390 | | ) -> Result<Option<Self::Out>, Error> { |
391 | | let id = self.0.fetch_add(1, Ordering::SeqCst) + 1; |
392 | | Ok(Some(TestTransport { |
393 | | id, |
394 | | buffers: LazyBuffers::new(1024, 1024), |
395 | | })) |
396 | | } |
397 | | } |
398 | | |
399 | | #[derive(Debug)] |
400 | | struct TestTransport { |
401 | | id: usize, |
402 | | buffers: LazyBuffers, |
403 | | } |
404 | | |
405 | | impl Transport for TestTransport { |
406 | | fn buffers(&mut self) -> &mut dyn Buffers { |
407 | | &mut self.buffers |
408 | | } |
409 | | fn transmit_output(&mut self, _: usize, _: NextTimeout) -> Result<(), Error> { |
410 | | Ok(()) |
411 | | } |
412 | | fn await_input(&mut self, _: NextTimeout) -> Result<bool, Error> { |
413 | | let body = self.id.to_string(); |
414 | | let response = format!( |
415 | | "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{}", |
416 | | body.len(), |
417 | | body |
418 | | ); |
419 | | self.buffers.input_append_buf()[..response.len()].copy_from_slice(response.as_bytes()); |
420 | | self.buffers.input_appended(response.len()); |
421 | | Ok(true) |
422 | | } |
423 | | fn is_open(&mut self) -> bool { |
424 | | true |
425 | | } |
426 | | fn is_tls(&self) -> bool { |
427 | | true |
428 | | } |
429 | | } |
430 | | |
431 | | fn agent(config: Config) -> Agent { |
432 | | Agent::with_parts( |
433 | | config, |
434 | | CountingConnector(Arc::new(AtomicUsize::new(0))), |
435 | | DefaultResolver::default(), |
436 | | ) |
437 | | } |
438 | | |
439 | | fn request(agent: &Agent, config: Config) -> String { |
440 | | let mut req = http::Request::get("https://127.0.0.1/").body(()).unwrap(); |
441 | | req.extensions_mut() |
442 | | .insert(crate::config::RequestLevelConfig(config)); |
443 | | agent.run(req).unwrap().body_mut().read_to_string().unwrap() |
444 | | } |
445 | | |
446 | | fn check_override(config: Config) { |
447 | | let base = Agent::config_builder().proxy(None).build(); |
448 | | check_configs(base, config); |
449 | | } |
450 | | |
451 | | fn check_configs(base: Config, config: Config) { |
452 | | let agent = agent(base.clone()); |
453 | | assert_eq!(request(&agent, base.clone()), "1"); |
454 | | assert_eq!( |
455 | | request(&agent, config.clone()), |
456 | | "2", |
457 | | "override must bypass existing connection" |
458 | | ); |
459 | | assert_eq!(request(&agent, config), "3", "override must not enter pool"); |
460 | | assert_eq!( |
461 | | request(&agent, base), |
462 | | "1", |
463 | | "Agent connection must remain reusable" |
464 | | ); |
465 | | } |
466 | | |
467 | | #[test] |
468 | | fn connection_overrides_bypass_pool() { |
469 | | check_override(Agent::config_builder().proxy(None).no_delay(false).build()); |
470 | | check_override( |
471 | | Agent::config_builder() |
472 | | .proxy(None) |
473 | | .ip_family(crate::config::IpFamily::Ipv4Only) |
474 | | .build(), |
475 | | ); |
476 | | check_override( |
477 | | Agent::config_builder() |
478 | | .proxy(None) |
479 | | .input_buffer_size(4096) |
480 | | .build(), |
481 | | ); |
482 | | check_override( |
483 | | Agent::config_builder() |
484 | | .proxy(None) |
485 | | .output_buffer_size(4096) |
486 | | .build(), |
487 | | ); |
488 | | check_override( |
489 | | Agent::config_builder() |
490 | | .proxy(None) |
491 | | .user_agent("custom") |
492 | | .build(), |
493 | | ); |
494 | | } |
495 | | |
496 | | #[test] |
497 | | fn request_settings_preserve_pooling() { |
498 | | let base = Agent::config_builder().proxy(None).build(); |
499 | | let agent = agent(base.clone()); |
500 | | assert_eq!(request(&agent, base), "1"); |
501 | | let config = Agent::config_builder() |
502 | | .proxy(None) |
503 | | .https_only(true) |
504 | | .http_status_as_error(false) |
505 | | .timeout_global(Some(std::time::Duration::from_secs(10))) |
506 | | .max_response_header_size(4096) |
507 | | .build(); |
508 | | assert_eq!(request(&agent, config), "1"); |
509 | | } |
510 | | |
511 | | #[test] |
512 | | #[cfg(feature = "_tls")] |
513 | | fn client_identity_isolation() { |
514 | | use crate::tls::{Certificate, ClientCert, PrivateKey, TlsConfig}; |
515 | | // The transport is synthetic: these bytes identify credentials without |
516 | | // performing a handshake. Response bodies identify actual connections. |
517 | | let identity = |bytes: &'static [u8]| { |
518 | | ClientCert::new_with_certs( |
519 | | &[Certificate::from_der(bytes)], |
520 | | PrivateKey::from_pem( |
521 | | b"-----BEGIN PRIVATE KEY-----\nQQ==\n-----END PRIVATE KEY-----\n", |
522 | | ) |
523 | | .unwrap(), |
524 | | ) |
525 | | }; |
526 | | let config = |cert| { |
527 | | Agent::config_builder() |
528 | | .proxy(None) |
529 | | .tls_config(TlsConfig::builder().client_cert(cert).build()) |
530 | | .build() |
531 | | }; |
532 | | let a = config(Some(identity(b"A"))); |
533 | | let b = config(Some(identity(b"B"))); |
534 | | let none = config(None); |
535 | | check_configs(a.clone(), b); |
536 | | check_configs(a.clone(), none.clone()); |
537 | | check_configs(none, a.clone()); |
538 | | let agent = agent(a.clone()); |
539 | | assert_eq!(request(&agent, a.clone()), "1"); |
540 | | assert_eq!( |
541 | | request(&agent, a), |
542 | | "1", |
543 | | "cloned credentials can reuse connections" |
544 | | ); |
545 | | } |
546 | | |
547 | | #[test] |
548 | | #[cfg(feature = "_tls")] |
549 | | fn tls_overrides_bypass_pool() { |
550 | | use crate::tls::{RootCerts, TlsConfig, TlsProvider}; |
551 | | for tls in [ |
552 | | TlsConfig::builder().disable_verification(true).build(), |
553 | | TlsConfig::builder().use_sni(false).build(), |
554 | | TlsConfig::builder() |
555 | | .provider(TlsProvider::NativeTls) |
556 | | .build(), |
557 | | TlsConfig::builder() |
558 | | .root_certs(RootCerts::PlatformVerifier) |
559 | | .build(), |
560 | | ] { |
561 | | check_override(Agent::config_builder().proxy(None).tls_config(tls).build()); |
562 | | } |
563 | | } |
564 | | } |