/rust/registry/src/index.crates.io-1949cf8c6b5b557f/reqwest-0.13.4/src/retry.rs
Line | Count | Source |
1 | | //! Retry requests |
2 | | //! |
3 | | //! A `Client` has the ability to retry requests, by sending additional copies |
4 | | //! to the server if a response is considered retryable. |
5 | | //! |
6 | | //! The [`Builder`] makes it easier to configure what requests to retry, along |
7 | | //! with including best practices by default, such as a retry budget. |
8 | | //! |
9 | | //! # Defaults |
10 | | //! |
11 | | //! The default retry behavior of a `Client` is to only retry requests where an |
12 | | //! error or low-level protocol NACK is encountered that is known to be safe to |
13 | | //! retry. Note however that providing a specific retry policy will override |
14 | | //! the default, and you will need to explicitly include that behavior. |
15 | | //! |
16 | | //! All policies default to including a retry budget that permits 20% extra |
17 | | //! requests to be sent. |
18 | | //! |
19 | | //! # Scoped |
20 | | //! |
21 | | //! A client's retry policy is scoped. That means that the policy doesn't |
22 | | //! apply to all requests, but only those within a user-defined scope. |
23 | | //! |
24 | | //! Since all policies include a budget by default, it doesn't make sense to |
25 | | //! apply it on _all_ requests. Rather, the retry history applied by a budget |
26 | | //! should likely only be applied to the same host. |
27 | | //! |
28 | | //! # Classifiers |
29 | | //! |
30 | | //! A retry policy needs to be configured with a classifier that determines |
31 | | //! if a request should be retried. Knowledge of the destination server's |
32 | | //! behavior is required to make a safe classifier. **Requests should not be |
33 | | //! retried** if the server cannot safely handle the same request twice, or if |
34 | | //! it causes side effects. |
35 | | //! |
36 | | //! Some common properties to check include if the request method is |
37 | | //! idempotent, or if the response status code indicates a transient error. |
38 | | |
39 | | use std::sync::Arc; |
40 | | use std::time::Duration; |
41 | | |
42 | | use tower::retry::budget::{Budget as _, TpsBudget as Budget}; |
43 | | |
44 | | #[cfg(docsrs)] |
45 | | pub use classify::ReqRep; |
46 | | |
47 | | /// Builder to configure retries |
48 | | /// |
49 | | /// Construct with [`for_host()`]. |
50 | | #[derive(Debug)] |
51 | | pub struct Builder { |
52 | | //backoff: Backoff, |
53 | | budget: Option<f32>, |
54 | | classifier: classify::Classifier, |
55 | | max_retries_per_request: u32, |
56 | | scope: scope::Scoped, |
57 | | } |
58 | | |
59 | | /// The internal type that we convert the builder into, that implements |
60 | | /// tower::retry::Policy privately. |
61 | | #[derive(Clone, Debug)] |
62 | | pub(crate) struct Policy { |
63 | | budget: Option<Arc<Budget>>, |
64 | | classifier: classify::Classifier, |
65 | | max_retries_per_request: u32, |
66 | | retry_cnt: u32, |
67 | | scope: scope::Scoped, |
68 | | } |
69 | | |
70 | | //#[derive(Debug)] |
71 | | //struct Backoff; |
72 | | |
73 | | /// Create a retry builder with a request scope. |
74 | | /// |
75 | | /// To provide a scope that isn't a closure, use the more general |
76 | | /// [`Builder::scoped()`]. |
77 | 0 | pub fn for_host<S>(host: S) -> Builder |
78 | 0 | where |
79 | 0 | S: for<'a> PartialEq<&'a str> + Send + Sync + 'static, |
80 | | { |
81 | 0 | scoped(move |req| host == req.uri().host().unwrap_or("")) |
82 | 0 | } |
83 | | |
84 | | /// Create a retry policy that will never retry any request. |
85 | | /// |
86 | | /// This is useful for disabling the `Client`s default behavior of retrying |
87 | | /// protocol nacks. |
88 | 0 | pub fn never() -> Builder { |
89 | 0 | scoped(|_| false).no_budget() |
90 | 0 | } |
91 | | |
92 | 0 | fn scoped<F>(func: F) -> Builder |
93 | 0 | where |
94 | 0 | F: Fn(&Req) -> bool + Send + Sync + 'static, |
95 | | { |
96 | 0 | Builder::scoped(scope::ScopeFn(func)) |
97 | 0 | } |
98 | | |
99 | | // ===== impl Builder ===== |
100 | | |
101 | | impl Builder { |
102 | | /// Create a scoped retry policy. |
103 | | /// |
104 | | /// For a more convenient constructor, see [`for_host()`]. |
105 | 0 | pub fn scoped(scope: impl scope::Scope) -> Self { |
106 | 0 | Self { |
107 | 0 | budget: Some(0.2), |
108 | 0 | classifier: classify::Classifier::Never, |
109 | 0 | max_retries_per_request: 2, // on top of the original |
110 | 0 | scope: scope::Scoped::Dyn(Arc::new(scope)), |
111 | 0 | } |
112 | 0 | } |
113 | | |
114 | | /// Set no retry budget. |
115 | | /// |
116 | | /// Sets that no budget will be enforced. This could also be considered |
117 | | /// to be an infinite budget. |
118 | | /// |
119 | | /// This is NOT recommended. Disabling the budget can make your system more |
120 | | /// susceptible to retry storms. |
121 | 0 | pub fn no_budget(mut self) -> Self { |
122 | 0 | self.budget = None; |
123 | 0 | self |
124 | 0 | } |
125 | | |
126 | | /// Sets the max extra load the budget will allow. |
127 | | /// |
128 | | /// Think of the amount of requests your client generates, and how much |
129 | | /// load that puts on the server. This option configures as a percentage |
130 | | /// how much extra load is allowed via retries. |
131 | | /// |
132 | | /// For example, if you send 1,000 requests per second, setting a maximum |
133 | | /// extra load value of `0.3` would allow 300 more requests per second |
134 | | /// in retries. A value of `2.5` would allow 2,500 more requests. |
135 | | /// |
136 | | /// # Panics |
137 | | /// |
138 | | /// The `extra_percent` value must be within reasonable values for a |
139 | | /// percentage. This method will panic if it is less than `0.0`, or greater |
140 | | /// than `1000.0`. |
141 | 0 | pub fn max_extra_load(mut self, extra_percent: f32) -> Self { |
142 | 0 | assert!(extra_percent >= 0.0); |
143 | 0 | assert!(extra_percent <= 1000.0); |
144 | 0 | self.budget = Some(extra_percent); |
145 | 0 | self |
146 | 0 | } |
147 | | |
148 | | // pub fn max_replay_body |
149 | | |
150 | | /// Set the max retries allowed per request. |
151 | | /// |
152 | | /// For each logical (initial) request, only retry up to `max` times. |
153 | | /// |
154 | | /// This value is used in combination with a token budget that is applied |
155 | | /// to all requests. Even if the budget would allow more requests, this |
156 | | /// limit will prevent. Likewise, the budget may prevent retrying up to |
157 | | /// `max` times. This setting prevents a single request from consuming |
158 | | /// the entire budget. |
159 | | /// |
160 | | /// Default is currently 2 retries. |
161 | 0 | pub fn max_retries_per_request(mut self, max: u32) -> Self { |
162 | 0 | self.max_retries_per_request = max; |
163 | 0 | self |
164 | 0 | } |
165 | | |
166 | | /// Provide a classifier to determine if a request should be retried. |
167 | | /// |
168 | | /// # Example |
169 | | /// |
170 | | /// ```rust |
171 | | /// # fn with_builder(builder: reqwest::retry::Builder) -> reqwest::retry::Builder { |
172 | | /// builder.classify_fn(|req_rep| { |
173 | | /// match (req_rep.method(), req_rep.status()) { |
174 | | /// (&http::Method::GET, Some(http::StatusCode::SERVICE_UNAVAILABLE)) => { |
175 | | /// req_rep.retryable() |
176 | | /// }, |
177 | | /// _ => req_rep.success() |
178 | | /// } |
179 | | /// }) |
180 | | /// # } |
181 | | /// ``` |
182 | 0 | pub fn classify_fn<F>(self, func: F) -> Self |
183 | 0 | where |
184 | 0 | F: Fn(classify::ReqRep<'_>) -> classify::Action + Send + Sync + 'static, |
185 | | { |
186 | 0 | self.classify(classify::ClassifyFn(func)) |
187 | 0 | } |
188 | | |
189 | | /// Provide a classifier to determine if a request should be retried. |
190 | 0 | pub fn classify(mut self, classifier: impl classify::Classify) -> Self { |
191 | 0 | self.classifier = classify::Classifier::Dyn(Arc::new(classifier)); |
192 | 0 | self |
193 | 0 | } |
194 | | |
195 | 0 | pub(crate) fn default() -> Builder { |
196 | 0 | Self { |
197 | 0 | // unscoped protocols nacks doesn't need a budget |
198 | 0 | budget: None, |
199 | 0 | classifier: classify::Classifier::ProtocolNacks, |
200 | 0 | max_retries_per_request: 2, // on top of the original |
201 | 0 | scope: scope::Scoped::Unscoped, |
202 | 0 | } |
203 | 0 | } |
204 | | |
205 | 0 | pub(crate) fn into_policy(self) -> Policy { |
206 | 0 | let budget = self |
207 | 0 | .budget |
208 | 0 | .map(|p| Arc::new(Budget::new(Duration::from_secs(10), 10, p))); |
209 | 0 | Policy { |
210 | 0 | budget, |
211 | 0 | classifier: self.classifier, |
212 | 0 | max_retries_per_request: self.max_retries_per_request, |
213 | 0 | retry_cnt: 0, |
214 | 0 | scope: self.scope, |
215 | 0 | } |
216 | 0 | } |
217 | | } |
218 | | |
219 | | // ===== internal ====== |
220 | | |
221 | | type Req = http::Request<crate::async_impl::body::Body>; |
222 | | |
223 | | impl<B> tower::retry::Policy<Req, http::Response<B>, crate::Error> for Policy { |
224 | | // TODO? backoff futures... |
225 | | type Future = std::future::Ready<()>; |
226 | | |
227 | 0 | fn retry( |
228 | 0 | &mut self, |
229 | 0 | req: &mut Req, |
230 | 0 | result: &mut crate::Result<http::Response<B>>, |
231 | 0 | ) -> Option<Self::Future> { |
232 | 0 | match self.classifier.classify(req, result) { |
233 | | classify::Action::Success => { |
234 | 0 | log::trace!("shouldn't retry!"); |
235 | 0 | if let Some(ref budget) = self.budget { |
236 | 0 | budget.deposit(); |
237 | 0 | } |
238 | 0 | None |
239 | | } |
240 | | classify::Action::Retryable => { |
241 | 0 | log::trace!("could retry!"); |
242 | 0 | if self.budget.as_ref().map(|b| b.withdraw()).unwrap_or(true) { |
243 | 0 | self.retry_cnt += 1; |
244 | 0 | Some(std::future::ready(())) |
245 | | } else { |
246 | 0 | log::debug!("retryable but could not withdraw from budget"); |
247 | 0 | None |
248 | | } |
249 | | } |
250 | | } |
251 | 0 | } |
252 | | |
253 | 0 | fn clone_request(&mut self, req: &Req) -> Option<Req> { |
254 | 0 | if self.retry_cnt > 0 && !self.scope.applies_to(req) { |
255 | 0 | return None; |
256 | 0 | } |
257 | 0 | if self.retry_cnt >= self.max_retries_per_request { |
258 | 0 | log::trace!("max_retries_per_request hit"); |
259 | 0 | return None; |
260 | 0 | } |
261 | 0 | let body = req.body().try_clone()?; |
262 | 0 | let mut new = http::Request::new(body); |
263 | 0 | *new.method_mut() = req.method().clone(); |
264 | 0 | *new.uri_mut() = req.uri().clone(); |
265 | 0 | *new.version_mut() = req.version(); |
266 | 0 | *new.headers_mut() = req.headers().clone(); |
267 | 0 | *new.extensions_mut() = req.extensions().clone(); |
268 | | |
269 | 0 | Some(new) |
270 | 0 | } |
271 | | } |
272 | | |
273 | 0 | fn is_retryable_error(err: &crate::Error) -> bool { |
274 | | use std::error::Error as _; |
275 | | |
276 | | // pop the reqwest::Error |
277 | 0 | let err = if let Some(err) = err.source() { |
278 | 0 | err |
279 | | } else { |
280 | 0 | return false; |
281 | | }; |
282 | | // pop the legacy::Error |
283 | 0 | let err = if let Some(err) = err.source() { |
284 | 0 | err |
285 | | } else { |
286 | 0 | return false; |
287 | | }; |
288 | | |
289 | | #[cfg(not(any(feature = "http3", feature = "http2")))] |
290 | 0 | let _err = err; |
291 | | |
292 | | #[cfg(feature = "http3")] |
293 | | if let Some(cause) = err.source() { |
294 | | if let Some(err) = cause.downcast_ref::<h3::error::ConnectionError>() { |
295 | | log::trace!("determining if HTTP/3 error {err} can be retried"); |
296 | | // TODO: Does h3 provide an API for checking the error? |
297 | | return err.to_string().as_str() == "timeout"; |
298 | | } |
299 | | } |
300 | | |
301 | | #[cfg(feature = "http2")] |
302 | | if let Some(cause) = err.source() { |
303 | | if let Some(err) = cause.downcast_ref::<h2::Error>() { |
304 | | // They sent us a graceful shutdown, try with a new connection! |
305 | | if err.is_go_away() && err.is_remote() && err.reason() == Some(h2::Reason::NO_ERROR) { |
306 | | return true; |
307 | | } |
308 | | |
309 | | // REFUSED_STREAM was sent from the server, which is safe to retry. |
310 | | // https://www.rfc-editor.org/rfc/rfc9113.html#section-8.7-3.2 |
311 | | if err.is_reset() && err.is_remote() && err.reason() == Some(h2::Reason::REFUSED_STREAM) |
312 | | { |
313 | | return true; |
314 | | } |
315 | | } |
316 | | } |
317 | 0 | false |
318 | 0 | } |
319 | | |
320 | | // sealed types and traits on purpose while exploring design space |
321 | | mod scope { |
322 | | pub trait Scope: Send + Sync + 'static { |
323 | | fn applies_to(&self, req: &super::Req) -> bool; |
324 | | } |
325 | | |
326 | | // I think scopes likely make the most sense being to hosts. |
327 | | // If that's the case, then it should probably be easiest to check for |
328 | | // the host. Perhaps also considering the ability to add more things |
329 | | // to scope off in the future... |
330 | | |
331 | | // For Future Whoever: making a blanket impl for any closure sounds nice, |
332 | | // but it causes inference issues at the call site. Every closure would |
333 | | // need to include `: ReqRep` in the arguments. |
334 | | // |
335 | | // An alternative is to make things like `ScopeFn`. Slightly more annoying, |
336 | | // but also more forwards-compatible. :shrug: |
337 | | |
338 | | pub struct ScopeFn<F>(pub(super) F); |
339 | | |
340 | | impl<F> Scope for ScopeFn<F> |
341 | | where |
342 | | F: Fn(&super::Req) -> bool + Send + Sync + 'static, |
343 | | { |
344 | 0 | fn applies_to(&self, req: &super::Req) -> bool { |
345 | 0 | (self.0)(req) |
346 | 0 | } |
347 | | } |
348 | | |
349 | | #[derive(Clone)] |
350 | | pub(super) enum Scoped { |
351 | | Unscoped, |
352 | | Dyn(std::sync::Arc<dyn Scope>), |
353 | | } |
354 | | |
355 | | impl Scoped { |
356 | 0 | pub(super) fn applies_to(&self, req: &super::Req) -> bool { |
357 | 0 | let ret = match self { |
358 | 0 | Self::Unscoped => true, |
359 | 0 | Self::Dyn(s) => s.applies_to(req), |
360 | | }; |
361 | 0 | log::trace!("retry in scope: {ret}"); |
362 | 0 | ret |
363 | 0 | } |
364 | | } |
365 | | |
366 | | impl std::fmt::Debug for Scoped { |
367 | 0 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
368 | 0 | match self { |
369 | 0 | Self::Unscoped => f.write_str("Unscoped"), |
370 | 0 | Self::Dyn(_) => f.write_str("Scoped"), |
371 | | } |
372 | 0 | } |
373 | | } |
374 | | } |
375 | | |
376 | | // sealed types and traits on purpose while exploring design space |
377 | | mod classify { |
378 | | pub trait Classify: Send + Sync + 'static { |
379 | | fn classify(&self, req_rep: ReqRep<'_>) -> Action; |
380 | | } |
381 | | |
382 | | // For Future Whoever: making a blanket impl for any closure sounds nice, |
383 | | // but it causes inference issues at the call site. Every closure would |
384 | | // need to include `: ReqRep` in the arguments. |
385 | | // |
386 | | // An alternative is to make things like `ClassifyFn`. Slightly more |
387 | | // annoying, but also more forwards-compatible. :shrug: |
388 | | pub struct ClassifyFn<F>(pub(super) F); |
389 | | |
390 | | impl<F> Classify for ClassifyFn<F> |
391 | | where |
392 | | F: Fn(ReqRep<'_>) -> Action + Send + Sync + 'static, |
393 | | { |
394 | 0 | fn classify(&self, req_rep: ReqRep<'_>) -> Action { |
395 | 0 | (self.0)(req_rep) |
396 | 0 | } |
397 | | } |
398 | | |
399 | | /// A request/response result to inspect for possible retries. |
400 | | /// |
401 | | /// This is passed to a `classify` function. |
402 | | #[derive(Debug)] |
403 | | pub struct ReqRep<'a>(&'a super::Req, Result<http::StatusCode, &'a crate::Error>); |
404 | | |
405 | | impl ReqRep<'_> { |
406 | | /// Access the request method. |
407 | 0 | pub fn method(&self) -> &http::Method { |
408 | 0 | self.0.method() |
409 | 0 | } |
410 | | |
411 | | /// Access the request URI. |
412 | 0 | pub fn uri(&self) -> &http::Uri { |
413 | 0 | self.0.uri() |
414 | 0 | } |
415 | | |
416 | | /// Access the response status, if it did not error. |
417 | 0 | pub fn status(&self) -> Option<http::StatusCode> { |
418 | 0 | self.1.ok() |
419 | 0 | } |
420 | | |
421 | | /// Access the error, if a response was not received. |
422 | 0 | pub fn error(&self) -> Option<&(dyn std::error::Error + 'static)> { |
423 | 0 | self.1.as_ref().err().map(|e| &**e as _) |
424 | 0 | } |
425 | | |
426 | | /// Classify this attempt as retryable. |
427 | 0 | pub fn retryable(self) -> Action { |
428 | 0 | Action::Retryable |
429 | 0 | } |
430 | | |
431 | | /// Classify this attempt as success. |
432 | | /// |
433 | | /// Even if it was a domain error, a "success" means it will not retry. |
434 | 0 | pub fn success(self) -> Action { |
435 | 0 | Action::Success |
436 | 0 | } |
437 | | |
438 | 0 | fn is_protocol_nack(&self) -> bool { |
439 | 0 | self.1 |
440 | 0 | .as_ref() |
441 | 0 | .err() |
442 | 0 | .map(|&e| super::is_retryable_error(e)) |
443 | 0 | .unwrap_or(false) |
444 | 0 | } |
445 | | } |
446 | | |
447 | | #[must_use] |
448 | | #[derive(Debug)] |
449 | | pub enum Action { |
450 | | Success, |
451 | | Retryable, |
452 | | } |
453 | | |
454 | | #[derive(Clone)] |
455 | | pub(super) enum Classifier { |
456 | | Never, |
457 | | ProtocolNacks, |
458 | | Dyn(std::sync::Arc<dyn Classify>), |
459 | | } |
460 | | |
461 | | impl Classifier { |
462 | 0 | pub(super) fn classify<B>( |
463 | 0 | &self, |
464 | 0 | req: &super::Req, |
465 | 0 | res: &Result<http::Response<B>, crate::Error>, |
466 | 0 | ) -> Action { |
467 | 0 | let req_rep = ReqRep(req, res.as_ref().map(|r| r.status())); |
468 | 0 | match self { |
469 | 0 | Self::Never => Action::Success, |
470 | | Self::ProtocolNacks => { |
471 | 0 | if req_rep.is_protocol_nack() { |
472 | 0 | Action::Retryable |
473 | | } else { |
474 | 0 | Action::Success |
475 | | } |
476 | | } |
477 | 0 | Self::Dyn(c) => c.classify(req_rep), |
478 | | } |
479 | 0 | } |
480 | | } |
481 | | |
482 | | impl std::fmt::Debug for Classifier { |
483 | 0 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
484 | 0 | match self { |
485 | 0 | Self::Never => f.write_str("Never"), |
486 | 0 | Self::ProtocolNacks => f.write_str("ProtocolNacks"), |
487 | 0 | Self::Dyn(_) => f.write_str("Classifier"), |
488 | | } |
489 | 0 | } |
490 | | } |
491 | | } |