/src/gdal/curl/lib/ratelimit.c

Source
/***************************************************************************
 *                                  _   _ ____  _
 *  Project                     ___| | | |  _ \| |
 *                             / __| | | | |_) | |
 *                            | (__| |_| |  _ <| |___
 *                             \___|\___/|_| \_\_____|
 *
 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution. The terms
 * are also available at https://curl.se/docs/copyright.html.
 *
 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
 * copies of the Software, and permit persons to whom the Software is
 * furnished to do so, under the terms of the COPYING file.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 * SPDX-License-Identifier: curl
 *
 ***************************************************************************/
#include "curl_setup.h"

#include "urldata.h"
#include "ratelimit.h"

#define CURL_US_PER_SEC         1000000
#define CURL_RLIMIT_MIN_RATE    (4 * 1024)  /* minimum step rate */
#define CURL_RLIMIT_STEP_MIN_MS 2  /* minimum step duration */

static void rlimit_update(struct Curl_rlimit *r,
                          const struct curltime *pts)
{
  timediff_t elapsed_us, elapsed_steps;
  int64_t token_gain;

  DEBUGASSERT(r->rate_per_step);
  if((r->ts.tv_sec == pts->tv_sec) && (r->ts.tv_usec == pts->tv_usec))
    return;

  elapsed_us = curlx_ptimediff_us(pts, &r->ts);
  if(elapsed_us < 0) { /* not going back in time */
    DEBUGASSERT(0);
    return;
  }

  elapsed_us += r->spare_us;
  if(elapsed_us < r->step_us)
    return;

  /* we do the update */
  r->ts = *pts;
  elapsed_steps = elapsed_us / r->step_us;
  r->spare_us = elapsed_us % r->step_us;

  /* How many tokens did we gain since the last update? */
  if(r->rate_per_step > (INT64_MAX / elapsed_steps))
    token_gain = INT64_MAX;
  else {
    token_gain = r->rate_per_step * elapsed_steps;
  }

  if((INT64_MAX - token_gain) > r->tokens)
    r->tokens += token_gain;
  else
    r->tokens = INT64_MAX;

  /* Limit the token again by the burst rate (if set), so we
   * do not suddenly have a huge number of tokens after inactivity. */
  if(r->burst_per_step && (r->tokens > r->burst_per_step)) {
    r->tokens = r->burst_per_step;
  }
}

static void rlimit_tune_steps(struct Curl_rlimit *r,
                              int64_t tokens_total)
{
  int64_t tokens_last, tokens_main, msteps;

  /* Tune the ratelimit at the start *if* we know how many tokens
   * are expected to be consumed in total.
   * The reason for tuning is that rlimit provides tokens to be consumed
   * per "step" which starts out to be a second. The tokens may be consumed
   * in full at the beginning of a step. The remainder of the second will
   * have no tokens available, effectively blocking the consumption and
   * so keeping the "step average" in line.
   * This works will up to the last step. When no more tokens are needed,
   * no wait will happen and the last step would be too fast. This is
   * especially noticeable when only a few steps are needed.
   *
   * Example: downloading 1.5kb with a ratelimit of 1k could be done in
   * roughly 1 second (1k in the first second and the 0.5 at the start of
   * the second one).
   *
   * The tuning tries to make the last step small, using only
   * 1 percent of the total tokens (at least 1). The rest of the tokens
   * are to be consumed in the steps before by adjusting the duration of
   * the step and the amount of tokens it provides. */
  if(!r->rate_per_step ||
     (tokens_total <= 1) ||
     (tokens_total > (INT64_MAX / 1000)))
    return;

  /* Calculate tokens for the last step and the ones before. */
  tokens_last = tokens_total / 100;
  if(!tokens_last) /* less than 100 total, use 1 */
    tokens_last = 1;
  else if(tokens_last > CURL_RLIMIT_MIN_RATE)
    tokens_last = CURL_RLIMIT_MIN_RATE;
  DEBUGASSERT(tokens_last);
  tokens_main = tokens_total - tokens_last;
  DEBUGASSERT(tokens_main);

  /* how many milli-steps will it take to consume those, give the
  * original rate limit per second? */
  DEBUGASSERT(r->step_us == CURL_US_PER_SEC);

  msteps = (tokens_main * 1000 / r->rate_per_step);
  if(msteps < CURL_RLIMIT_STEP_MIN_MS) {
    /* Steps this small will not work. Do not tune. */
    return;
  }
  else if(msteps < 1000) {
    /* It needs less than one step to provide the needed tokens.
     * Make it exactly that long and with exactly those tokens. */
    r->step_us = (timediff_t)msteps * 1000;
    r->rate_per_step = tokens_main;
    r->tokens = r->rate_per_step;
  }
  else {
    /* More than 1 step. Spread the remainder milli steps and
     * the tokens they need to provide across all steps. If integer
     * arithmetic can do it. */
    curl_off_t ms_unaccounted = (msteps % 1000);
    curl_off_t mstep_inc = (ms_unaccounted / (msteps / 1000));
    if(mstep_inc) {
      curl_off_t rate_inc = ((r->rate_per_step * mstep_inc) / 1000);
      if(rate_inc) {
        r->step_us = CURL_US_PER_SEC + ((timediff_t)mstep_inc * 1000);
        r->rate_per_step += rate_inc;
        r->tokens = r->rate_per_step;
        if(r->burst_per_step) {
          curl_off_t burst_inc = ((r->burst_per_step * mstep_inc) / 1000);
          if(burst_inc)
            r->burst_per_step += burst_inc;
        }
      }
    }
  }
}

void Curl_rlimit_init(struct Curl_rlimit *r,
                      int64_t rate_per_sec,
                      int64_t burst_per_sec,
                      const struct curltime *pts)
{
  DEBUGASSERT(rate_per_sec >= 0);
  DEBUGASSERT(burst_per_sec >= rate_per_sec || !burst_per_sec);
  DEBUGASSERT(pts);
  r->rate_per_step = r->rate_per_sec = rate_per_sec;
  r->burst_per_step = r->burst_per_sec = burst_per_sec;
  r->step_us = CURL_US_PER_SEC;
  r->spare_us = 0;
  r->tokens = r->rate_per_step;
  r->ts = *pts;
  r->blocked = FALSE;
}

void Curl_rlimit_start(struct Curl_rlimit *r, const struct curltime *pts,
                       int64_t total_tokens)
{
  /* A start always resets the values to initial defaults, then
   * fine tunes the intervals for the total_tokens expected. */
  r->rate_per_step = r->rate_per_sec;
  r->burst_per_step = r->burst_per_sec;
  r->step_us = CURL_US_PER_SEC;
  r->spare_us = 0;
  r->tokens = r->rate_per_step;
  r->ts = *pts;
  rlimit_tune_steps(r, total_tokens);
}

int64_t Curl_rlimit_per_step(struct Curl_rlimit *r)
{
  return r->rate_per_step;
}

bool Curl_rlimit_active(struct Curl_rlimit *r)
{
  return (r->rate_per_step > 0) || r->blocked;
}

bool Curl_rlimit_is_blocked(struct Curl_rlimit *r)
{
  return (bool)r->blocked;
}

int64_t Curl_rlimit_avail(struct Curl_rlimit *r,
                          const struct curltime *pts)
{
  if(r->blocked)
    return 0;
  else if(r->rate_per_step) {
    rlimit_update(r, pts);
    return r->tokens;
  }
  else
    return INT64_MAX;
}

void Curl_rlimit_drain(struct Curl_rlimit *r,
                       size_t tokens,
                       const struct curltime *pts)
{
  if(r->blocked || !r->rate_per_step)
    return;

  rlimit_update(r, pts);
#if 8 <= SIZEOF_SIZE_T
  if(tokens > INT64_MAX) {
    r->tokens = INT64_MAX;
  }
  else
#endif
  {
    int64_t val = (int64_t)tokens;
    if((INT64_MIN + val) < r->tokens)
      r->tokens -= val;
    else
      r->tokens = INT64_MIN;
  }
}

timediff_t Curl_rlimit_wait_ms(struct Curl_rlimit *r,
                               const struct curltime *pts)
{
  timediff_t wait_us, elapsed_us;

  if(r->blocked || !r->rate_per_step)
    return 0;
  rlimit_update(r, pts);
  if(r->tokens > 0)
    return 0;

  /* How much time will it take tokens to become positive again?
   * Deduct `spare_us` and check against already elapsed time */
  wait_us = r->step_us - r->spare_us;
  if(r->tokens < 0) {
    curl_off_t debt_pct = ((-r->tokens) * 100 / r->rate_per_step);
    if(debt_pct)
      wait_us += (r->step_us * debt_pct / 100);
  }

  elapsed_us = curlx_ptimediff_us(pts, &r->ts);
  if(elapsed_us >= wait_us)
    return 0;
  wait_us -= elapsed_us;
  return (wait_us + 999) / 1000; /* in milliseconds */
}

timediff_t Curl_rlimit_next_step_ms(struct Curl_rlimit *r,
                                    const struct curltime *pts)
{
  if(!r->blocked && r->rate_per_step) {
    timediff_t elapsed_us, next_us;

    elapsed_us = curlx_ptimediff_us(pts, &r->ts) + r->spare_us;
    if(r->step_us > elapsed_us) {
      next_us = r->step_us - elapsed_us;
      return (next_us + 999) / 1000; /* in milliseconds */
    }
  }
  return 0;
}

void Curl_rlimit_block(struct Curl_rlimit *r,
                       bool activate,
                       const struct curltime *pts)
{
  if(!activate == !r->blocked)
    return;

  r->ts = *pts;
  r->blocked = activate;
  if(!r->blocked) {
    /* Start rate limiting fresh. The amount of time this was blocked
     * does not generate extra tokens. */
    Curl_rlimit_start(r, pts, -1);
  }
  else {
    r->tokens = 0;
  }
}

Coverage Report

Created: 2026-06-30 08:33

Line	Count	Source
1		/***************************************************************************
2		* _ _ ____ _
3		* Project ___\| \| \| \| _ \\| \|
4		* / __\| \| \| \| \|_) \| \|
5		* \| (__\| \|_\| \| _ <\| \|___
6		* \___\|\___/\|_\| \_\_____\|
7		*
8		* Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.
9		*
10		* This software is licensed as described in the file COPYING, which
11		* you should have received as part of this distribution. The terms
12		* are also available at https://curl.se/docs/copyright.html.
13		*
14		* You may opt to use, copy, modify, merge, publish, distribute and/or sell
15		* copies of the Software, and permit persons to whom the Software is
16		* furnished to do so, under the terms of the COPYING file.
17		*
18		* This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19		* KIND, either express or implied.
20		*
21		* SPDX-License-Identifier: curl
22		*
23		***************************************************************************/
24		#include "curl_setup.h"
25
26		#include "urldata.h"
27		#include "ratelimit.h"
28
29	13.0k	#define CURL_US_PER_SEC 1000000
30	0	#define CURL_RLIMIT_MIN_RATE (4 * 1024) /* minimum step rate */
31	0	#define CURL_RLIMIT_STEP_MIN_MS 2 /* minimum step duration */
32
33		static void rlimit_update(struct Curl_rlimit *r,
34		const struct curltime *pts)
35	0	{
36	0	timediff_t elapsed_us, elapsed_steps;
37	0	int64_t token_gain;
38
39	0	DEBUGASSERT(r->rate_per_step);
40	0	if((r->ts.tv_sec == pts->tv_sec) && (r->ts.tv_usec == pts->tv_usec))
41	0	return;
42
43	0	elapsed_us = curlx_ptimediff_us(pts, &r->ts);
44	0	if(elapsed_us < 0) { /* not going back in time */
45	0	DEBUGASSERT(0);
46	0	return;
47	0	}
48
49	0	elapsed_us += r->spare_us;
50	0	if(elapsed_us < r->step_us)
51	0	return;
52
53		/* we do the update */
54	0	r->ts = *pts;
55	0	elapsed_steps = elapsed_us / r->step_us;
56	0	r->spare_us = elapsed_us % r->step_us;
57
58		/* How many tokens did we gain since the last update? */
59	0	if(r->rate_per_step > (INT64_MAX / elapsed_steps))
60	0	token_gain = INT64_MAX;
61	0	else {
62	0	token_gain = r->rate_per_step * elapsed_steps;
63	0	}
64
65	0	if((INT64_MAX - token_gain) > r->tokens)
66	0	r->tokens += token_gain;
67	0	else
68	0	r->tokens = INT64_MAX;
69
70		/* Limit the token again by the burst rate (if set), so we
71		* do not suddenly have a huge number of tokens after inactivity. */
72	0	if(r->burst_per_step && (r->tokens > r->burst_per_step)) {
73	0	r->tokens = r->burst_per_step;
74	0	}
75	0	}
76
77		static void rlimit_tune_steps(struct Curl_rlimit *r,
78		int64_t tokens_total)
79	13.0k	{
80	13.0k	int64_t tokens_last, tokens_main, msteps;
81
82		/* Tune the ratelimit at the start if we know how many tokens
83		* are expected to be consumed in total.
84		* The reason for tuning is that rlimit provides tokens to be consumed
85		* per "step" which starts out to be a second. The tokens may be consumed
86		* in full at the beginning of a step. The remainder of the second will
87		* have no tokens available, effectively blocking the consumption and
88		* so keeping the "step average" in line.
89		* This works will up to the last step. When no more tokens are needed,
90		* no wait will happen and the last step would be too fast. This is
91		* especially noticeable when only a few steps are needed.
92		*
93		* Example: downloading 1.5kb with a ratelimit of 1k could be done in
94		* roughly 1 second (1k in the first second and the 0.5 at the start of
95		* the second one).
96		*
97		* The tuning tries to make the last step small, using only
98		* 1 percent of the total tokens (at least 1). The rest of the tokens
99		* are to be consumed in the steps before by adjusting the duration of
100		* the step and the amount of tokens it provides. */
101	13.0k	if(!r->rate_per_step \|\|
102	0	(tokens_total <= 1) \|\|
103	0	(tokens_total > (INT64_MAX / 1000)))
104	13.0k	return;
105
106		/* Calculate tokens for the last step and the ones before. */
107	0	tokens_last = tokens_total / 100;
108	0	if(!tokens_last) /* less than 100 total, use 1 */
109	0	tokens_last = 1;
110	0	else if(tokens_last > CURL_RLIMIT_MIN_RATE)
111	0	tokens_last = CURL_RLIMIT_MIN_RATE;
112	0	DEBUGASSERT(tokens_last);
113	0	tokens_main = tokens_total - tokens_last;
114	0	DEBUGASSERT(tokens_main);
115
116		/* how many milli-steps will it take to consume those, give the
117		* original rate limit per second? */
118	0	DEBUGASSERT(r->step_us == CURL_US_PER_SEC);
119
120	0	msteps = (tokens_main * 1000 / r->rate_per_step);
121	0	if(msteps < CURL_RLIMIT_STEP_MIN_MS) {
122		/* Steps this small will not work. Do not tune. */
123	0	return;
124	0	}
125	0	else if(msteps < 1000) {
126		/* It needs less than one step to provide the needed tokens.
127		* Make it exactly that long and with exactly those tokens. */
128	0	r->step_us = (timediff_t)msteps * 1000;
129	0	r->rate_per_step = tokens_main;
130	0	r->tokens = r->rate_per_step;
131	0	}
132	0	else {
133		/* More than 1 step. Spread the remainder milli steps and
134		* the tokens they need to provide across all steps. If integer
135		* arithmetic can do it. */
136	0	curl_off_t ms_unaccounted = (msteps % 1000);
137	0	curl_off_t mstep_inc = (ms_unaccounted / (msteps / 1000));
138	0	if(mstep_inc) {
139	0	curl_off_t rate_inc = ((r->rate_per_step * mstep_inc) / 1000);
140	0	if(rate_inc) {
141	0	r->step_us = CURL_US_PER_SEC + ((timediff_t)mstep_inc * 1000);
142	0	r->rate_per_step += rate_inc;
143	0	r->tokens = r->rate_per_step;
144	0	if(r->burst_per_step) {
145	0	curl_off_t burst_inc = ((r->burst_per_step * mstep_inc) / 1000);
146	0	if(burst_inc)
147	0	r->burst_per_step += burst_inc;
148	0	}
149	0	}
150	0	}
151	0	}
152	0	}
153
154		void Curl_rlimit_init(struct Curl_rlimit *r,
155		int64_t rate_per_sec,
156		int64_t burst_per_sec,
157		const struct curltime *pts)
158	0	{
159	0	DEBUGASSERT(rate_per_sec >= 0);
160	0	DEBUGASSERT(burst_per_sec >= rate_per_sec \|\| !burst_per_sec);
161	0	DEBUGASSERT(pts);
162	0	r->rate_per_step = r->rate_per_sec = rate_per_sec;
163	0	r->burst_per_step = r->burst_per_sec = burst_per_sec;
164	0	r->step_us = CURL_US_PER_SEC;
165	0	r->spare_us = 0;
166	0	r->tokens = r->rate_per_step;
167	0	r->ts = *pts;
168	0	r->blocked = FALSE;
169	0	}
170
171		void Curl_rlimit_start(struct Curl_rlimit r, const struct curltime pts,
172		int64_t total_tokens)
173	13.0k	{
174		/* A start always resets the values to initial defaults, then
175		* fine tunes the intervals for the total_tokens expected. */
176	13.0k	r->rate_per_step = r->rate_per_sec;
177	13.0k	r->burst_per_step = r->burst_per_sec;
178	13.0k	r->step_us = CURL_US_PER_SEC;
179	13.0k	r->spare_us = 0;
180	13.0k	r->tokens = r->rate_per_step;
181	13.0k	r->ts = *pts;
182	13.0k	rlimit_tune_steps(r, total_tokens);
183	13.0k	}
184
185		int64_t Curl_rlimit_per_step(struct Curl_rlimit *r)
186	0	{
187	0	return r->rate_per_step;
188	0	}
189
190		bool Curl_rlimit_active(struct Curl_rlimit *r)
191	172k	{
192	172k	return (r->rate_per_step > 0) \|\| r->blocked;
193	172k	}
194
195		bool Curl_rlimit_is_blocked(struct Curl_rlimit *r)
196	487k	{
197	487k	return (bool)r->blocked;
198	487k	}
199
200		int64_t Curl_rlimit_avail(struct Curl_rlimit *r,
201		const struct curltime *pts)
202	51.4k	{
203	51.4k	if(r->blocked)
204	0	return 0;
205	51.4k	else if(r->rate_per_step) {
206	0	rlimit_update(r, pts);
207	0	return r->tokens;
208	0	}
209	51.4k	else
210	51.4k	return INT64_MAX;
211	51.4k	}
212
213		void Curl_rlimit_drain(struct Curl_rlimit *r,
214		size_t tokens,
215		const struct curltime *pts)
216	56.7k	{
217	56.7k	if(r->blocked \|\| !r->rate_per_step)
218	56.7k	return;
219
220	0	rlimit_update(r, pts);
221	0	#if 8 <= SIZEOF_SIZE_T
222	0	if(tokens > INT64_MAX) {
223	0	r->tokens = INT64_MAX;
224	0	}
225	0	else
226	0	#endif
227	0	{
228	0	int64_t val = (int64_t)tokens;
229	0	if((INT64_MIN + val) < r->tokens)
230	0	r->tokens -= val;
231	0	else
232	0	r->tokens = INT64_MIN;
233	0	}
234	0	}
235
236		timediff_t Curl_rlimit_wait_ms(struct Curl_rlimit *r,
237		const struct curltime *pts)
238	0	{
239	0	timediff_t wait_us, elapsed_us;
240
241	0	if(r->blocked \|\| !r->rate_per_step)
242	0	return 0;
243	0	rlimit_update(r, pts);
244	0	if(r->tokens > 0)
245	0	return 0;
246
247		/* How much time will it take tokens to become positive again?
248		* Deduct `spare_us` and check against already elapsed time */
249	0	wait_us = r->step_us - r->spare_us;
250	0	if(r->tokens < 0) {
251	0	curl_off_t debt_pct = ((-r->tokens) * 100 / r->rate_per_step);
252	0	if(debt_pct)
253	0	wait_us += (r->step_us * debt_pct / 100);
254	0	}
255
256	0	elapsed_us = curlx_ptimediff_us(pts, &r->ts);
257	0	if(elapsed_us >= wait_us)
258	0	return 0;
259	0	wait_us -= elapsed_us;
260	0	return (wait_us + 999) / 1000; /* in milliseconds */
261	0	}
262
263		timediff_t Curl_rlimit_next_step_ms(struct Curl_rlimit *r,
264		const struct curltime *pts)
265	0	{
266	0	if(!r->blocked && r->rate_per_step) {
267	0	timediff_t elapsed_us, next_us;
268
269	0	elapsed_us = curlx_ptimediff_us(pts, &r->ts) + r->spare_us;
270	0	if(r->step_us > elapsed_us) {
271	0	next_us = r->step_us - elapsed_us;
272	0	return (next_us + 999) / 1000; /* in milliseconds */
273	0	}
274	0	}
275	0	return 0;
276	0	}
277
278		void Curl_rlimit_block(struct Curl_rlimit *r,
279		bool activate,
280		const struct curltime *pts)
281	751k	{
282	751k	if(!activate == !r->blocked)
283	751k	return;
284
285	0	r->ts = *pts;
286	0	r->blocked = activate;
287	0	if(!r->blocked) {
288		/* Start rate limiting fresh. The amount of time this was blocked
289		* does not generate extra tokens. */
290	0	Curl_rlimit_start(r, pts, -1);
291	0	}
292	0	else {
293	0	r->tokens = 0;
294	0	}
295	0	}