Line | Count | Source |
1 | | /* |
2 | | * Health-checks functions. |
3 | | * |
4 | | * Copyright 2000-2009 Willy Tarreau <w@1wt.eu> |
5 | | * Copyright 2007-2009 Krzysztof Piotr Oledzki <ole@ans.pl> |
6 | | * |
7 | | * This program is free software; you can redistribute it and/or |
8 | | * modify it under the terms of the GNU General Public License |
9 | | * as published by the Free Software Foundation; either version |
10 | | * 2 of the License, or (at your option) any later version. |
11 | | * |
12 | | */ |
13 | | |
14 | | #include <assert.h> |
15 | | #include <ctype.h> |
16 | | #include <errno.h> |
17 | | #include <stdarg.h> |
18 | | #include <stdio.h> |
19 | | #include <stdlib.h> |
20 | | #include <string.h> |
21 | | #include <time.h> |
22 | | #include <unistd.h> |
23 | | #include <sys/resource.h> |
24 | | #include <sys/socket.h> |
25 | | #include <sys/types.h> |
26 | | #include <sys/wait.h> |
27 | | #include <netinet/in.h> |
28 | | #include <netinet/tcp.h> |
29 | | #include <arpa/inet.h> |
30 | | |
31 | | #include <haproxy/action.h> |
32 | | #include <haproxy/api.h> |
33 | | #include <haproxy/arg.h> |
34 | | #include <haproxy/cfgparse.h> |
35 | | #include <haproxy/check.h> |
36 | | #include <haproxy/chunk.h> |
37 | | #include <haproxy/counters.h> |
38 | | #include <haproxy/dgram.h> |
39 | | #include <haproxy/dynbuf.h> |
40 | | #include <haproxy/extcheck.h> |
41 | | #include <haproxy/fd.h> |
42 | | #include <haproxy/global.h> |
43 | | #include <haproxy/h1.h> |
44 | | #include <haproxy/http.h> |
45 | | #include <haproxy/http_htx.h> |
46 | | #include <haproxy/htx.h> |
47 | | #include <haproxy/istbuf.h> |
48 | | #include <haproxy/list.h> |
49 | | #include <haproxy/log.h> |
50 | | #include <haproxy/mailers.h> |
51 | | #include <haproxy/port_range.h> |
52 | | #include <haproxy/proto_tcp.h> |
53 | | #include <haproxy/protocol.h> |
54 | | #include <haproxy/proxy.h> |
55 | | #include <haproxy/queue.h> |
56 | | #include <haproxy/regex.h> |
57 | | #include <haproxy/resolvers.h> |
58 | | #include <haproxy/sample.h> |
59 | | #include <haproxy/server.h> |
60 | | #include <haproxy/ssl_sock.h> |
61 | | #include <haproxy/stats-t.h> |
62 | | #include <haproxy/task.h> |
63 | | #include <haproxy/tcpcheck.h> |
64 | | #include <haproxy/thread.h> |
65 | | #include <haproxy/time.h> |
66 | | #include <haproxy/tools.h> |
67 | | #include <haproxy/trace.h> |
68 | | #include <haproxy/vars.h> |
69 | | |
70 | | /* trace source and events */ |
71 | | static void check_trace(enum trace_level level, uint64_t mask, |
72 | | const struct trace_source *src, |
73 | | const struct ist where, const struct ist func, |
74 | | const void *a1, const void *a2, const void *a3, const void *a4); |
75 | | |
76 | | /* The event representation is split like this : |
77 | | * check - check |
78 | | * |
79 | | * CHECK_EV_* macros are defined in <haproxy/check.h> |
80 | | */ |
81 | | static const struct trace_event check_trace_events[] = { |
82 | | { .mask = CHK_EV_TASK_WAKE, .name = "task_wake", .desc = "Check task woken up" }, |
83 | | { .mask = CHK_EV_HCHK_START, .name = "hchck_start", .desc = "Health-check started" }, |
84 | | { .mask = CHK_EV_HCHK_WAKE, .name = "hchck_wake", .desc = "Health-check woken up" }, |
85 | | { .mask = CHK_EV_HCHK_RUN, .name = "hchck_run", .desc = "Health-check running" }, |
86 | | { .mask = CHK_EV_HCHK_END, .name = "hchck_end", .desc = "Health-check terminated" }, |
87 | | { .mask = CHK_EV_HCHK_SUCC, .name = "hchck_succ", .desc = "Health-check success" }, |
88 | | { .mask = CHK_EV_HCHK_ERR, .name = "hchck_err", .desc = "Health-check failure" }, |
89 | | |
90 | | { .mask = CHK_EV_TCPCHK_EVAL, .name = "tcp_check_eval", .desc = "tcp-check rules evaluation" }, |
91 | | { .mask = CHK_EV_TCPCHK_ERR, .name = "tcp_check_err", .desc = "tcp-check evaluation error" }, |
92 | | { .mask = CHK_EV_TCPCHK_CONN, .name = "tcp_check_conn", .desc = "tcp-check connection rule" }, |
93 | | { .mask = CHK_EV_TCPCHK_SND, .name = "tcp_check_send", .desc = "tcp-check send rule" }, |
94 | | { .mask = CHK_EV_TCPCHK_EXP, .name = "tcp_check_expect", .desc = "tcp-check expect rule" }, |
95 | | { .mask = CHK_EV_TCPCHK_ACT, .name = "tcp_check_action", .desc = "tcp-check action rule" }, |
96 | | |
97 | | { .mask = CHK_EV_RX_DATA, .name = "rx_data", .desc = "receipt of data" }, |
98 | | { .mask = CHK_EV_RX_BLK, .name = "rx_blk", .desc = "receipt blocked" }, |
99 | | { .mask = CHK_EV_RX_ERR, .name = "rx_err", .desc = "receipt error" }, |
100 | | |
101 | | { .mask = CHK_EV_TX_DATA, .name = "tx_data", .desc = "transmission of data" }, |
102 | | { .mask = CHK_EV_TX_BLK, .name = "tx_blk", .desc = "transmission blocked" }, |
103 | | { .mask = CHK_EV_TX_ERR, .name = "tx_err", .desc = "transmission error" }, |
104 | | |
105 | | {} |
106 | | }; |
107 | | |
108 | | static const struct name_desc check_trace_lockon_args[4] = { |
109 | | /* arg1 */ { /* already used by the check */ }, |
110 | | /* arg2 */ { }, |
111 | | /* arg3 */ { }, |
112 | | /* arg4 */ { } |
113 | | }; |
114 | | |
115 | | static const struct name_desc check_trace_decoding[] = { |
116 | 0 | #define CHK_VERB_CLEAN 1 |
117 | | { .name="clean", .desc="only user-friendly stuff, generally suitable for level \"user\"" }, |
118 | 0 | #define CHK_VERB_MINIMAL 2 |
119 | | { .name="minimal", .desc="report info on streams and connectors" }, |
120 | | #define CHK_VERB_SIMPLE 3 |
121 | | { .name="simple", .desc="add info on request and response channels" }, |
122 | 0 | #define CHK_VERB_ADVANCED 4 |
123 | | { .name="advanced", .desc="add info on channel's buffer for data and developer levels only" }, |
124 | 0 | #define CHK_VERB_COMPLETE 5 |
125 | | { .name="complete", .desc="add info on channel's buffer" }, |
126 | | { /* end */ } |
127 | | }; |
128 | | |
129 | | struct trace_source trace_check = { |
130 | | .name = IST("check"), |
131 | | .desc = "Health-check", |
132 | | .arg_def = TRC_ARG1_CHK, // TRACE()'s first argument is always a stream |
133 | | .default_cb = check_trace, |
134 | | .known_events = check_trace_events, |
135 | | .lockon_args = check_trace_lockon_args, |
136 | | .decoding = check_trace_decoding, |
137 | | .report_events = ~0, // report everything by default |
138 | | }; |
139 | | |
140 | | #define TRACE_SOURCE &trace_check |
141 | | INITCALL1(STG_REGISTER, trace_register_source, TRACE_SOURCE); |
142 | | |
143 | | |
144 | | /* Dummy frontend used to create all checks sessions. */ |
145 | | struct proxy checks_fe; |
146 | | |
147 | | |
148 | | static inline void check_trace_buf(const struct buffer *buf, size_t ofs, size_t len) |
149 | 0 | { |
150 | 0 | size_t block1, block2; |
151 | 0 | int line, ptr, newptr; |
152 | |
|
153 | 0 | block1 = b_contig_data(buf, ofs); |
154 | 0 | block2 = 0; |
155 | 0 | if (block1 > len) |
156 | 0 | block1 = len; |
157 | 0 | block2 = len - block1; |
158 | |
|
159 | 0 | ofs = b_peek_ofs(buf, ofs); |
160 | |
|
161 | 0 | line = 0; |
162 | 0 | ptr = ofs; |
163 | 0 | while (ptr < ofs + block1) { |
164 | 0 | newptr = dump_text_line(&trace_buf, b_orig(buf), b_size(buf), ofs + block1, &line, ptr); |
165 | 0 | if (newptr == ptr) |
166 | 0 | break; |
167 | 0 | ptr = newptr; |
168 | 0 | } |
169 | |
|
170 | 0 | line = ptr = 0; |
171 | 0 | while (ptr < block2) { |
172 | 0 | newptr = dump_text_line(&trace_buf, b_orig(buf), b_size(buf), block2, &line, ptr); |
173 | 0 | if (newptr == ptr) |
174 | 0 | break; |
175 | 0 | ptr = newptr; |
176 | 0 | } |
177 | 0 | } |
178 | | |
179 | | /* trace source and events */ |
180 | | static void check_trace(enum trace_level level, uint64_t mask, |
181 | | const struct trace_source *src, |
182 | | const struct ist where, const struct ist func, |
183 | | const void *a1, const void *a2, const void *a3, const void *a4) |
184 | 0 | { |
185 | 0 | const struct check *check = a1; |
186 | 0 | const struct server *srv = (check ? check->server : NULL); |
187 | 0 | const size_t *val = a4; |
188 | 0 | const char *res; |
189 | |
|
190 | 0 | if (!check || src->verbosity < CHK_VERB_CLEAN) |
191 | 0 | return; |
192 | | |
193 | 0 | BUG_ON(!srv); |
194 | 0 | chunk_appendf(&trace_buf, " : [%c] SRV=%s", |
195 | 0 | ((check->type == PR_O2_EXT_CHK) ? 'E' : (check->state & CHK_ST_AGENT ? 'A' : 'H')), |
196 | 0 | srv->id); |
197 | |
|
198 | 0 | chunk_appendf(&trace_buf, " status=%d/%d %s exp=%d", |
199 | 0 | (check->health >= check->rise) ? check->health - check->rise + 1 : check->health, |
200 | 0 | (check->health >= check->rise) ? check->fall : check->rise, |
201 | 0 | (check->health >= check->rise) ? (srv->uweight ? "UP" : "DRAIN") : "DOWN", |
202 | 0 | (check->task->expire ? TICKS_TO_MS(check->task->expire - now_ms) : 0)); |
203 | |
|
204 | 0 | switch (check->result) { |
205 | 0 | case CHK_RES_NEUTRAL: res = "-"; break; |
206 | 0 | case CHK_RES_FAILED: res = "FAIL"; break; |
207 | 0 | case CHK_RES_PASSED: res = "PASS"; break; |
208 | 0 | case CHK_RES_CONDPASS: res = "COND"; break; |
209 | 0 | default: res = "UNK"; break; |
210 | 0 | } |
211 | | |
212 | 0 | if (src->verbosity == CHK_VERB_CLEAN) |
213 | 0 | return; |
214 | | |
215 | 0 | chunk_appendf(&trace_buf, " - last=%s(%d)/%s(%d)", |
216 | 0 | get_check_status_info(check->status), check->status, |
217 | 0 | res, check->result); |
218 | | |
219 | | /* Display the value to the 4th argument (level > STATE) */ |
220 | 0 | if (src->level > TRACE_LEVEL_STATE && val) |
221 | 0 | chunk_appendf(&trace_buf, " - VAL=%lu", (long)*val); |
222 | |
|
223 | 0 | chunk_appendf(&trace_buf, " check=%p(0x%08x)", check, check->state); |
224 | |
|
225 | 0 | if (src->verbosity == CHK_VERB_MINIMAL) |
226 | 0 | return; |
227 | | |
228 | | |
229 | 0 | if (check->sc) { |
230 | 0 | struct connection *conn = sc_conn(check->sc); |
231 | |
|
232 | 0 | chunk_appendf(&trace_buf, " - conn=%p(0x%08x)", conn, conn ? conn->flags : 0); |
233 | 0 | chunk_appendf(&trace_buf, " sc=%p(0x%08x)", check->sc, check->sc->flags); |
234 | 0 | } |
235 | |
|
236 | 0 | if (mask & CHK_EV_TCPCHK) { |
237 | 0 | const char *type; |
238 | |
|
239 | 0 | switch (check->tcpcheck_rules->flags & TCPCHK_RULES_PROTO_CHK) { |
240 | 0 | case TCPCHK_RULES_PGSQL_CHK: type = "PGSQL"; break; |
241 | 0 | case TCPCHK_RULES_REDIS_CHK: type = "REDIS"; break; |
242 | 0 | case TCPCHK_RULES_SMTP_CHK: type = "SMTP"; break; |
243 | 0 | case TCPCHK_RULES_HTTP_CHK: type = "HTTP"; break; |
244 | 0 | case TCPCHK_RULES_MYSQL_CHK: type = "MYSQL"; break; |
245 | 0 | case TCPCHK_RULES_LDAP_CHK: type = "LDAP"; break; |
246 | 0 | case TCPCHK_RULES_SSL3_CHK: type = "SSL3"; break; |
247 | 0 | case TCPCHK_RULES_AGENT_CHK: type = "AGENT"; break; |
248 | 0 | case TCPCHK_RULES_SPOP_CHK: type = "SPOP"; break; |
249 | 0 | case TCPCHK_RULES_TCP_CHK: type = "TCP"; break; |
250 | 0 | default: type = "???"; break; |
251 | 0 | } |
252 | 0 | if (check->current_step) |
253 | 0 | chunk_appendf(&trace_buf, " - tcp-check=(%s,%d)", type, tcpcheck_get_step_id(check, NULL)); |
254 | 0 | else |
255 | 0 | chunk_appendf(&trace_buf, " - tcp-check=(%s,-)", type); |
256 | 0 | } |
257 | | |
258 | | /* Display bi and bo buffer info (level > USER & verbosity > SIMPLE) */ |
259 | 0 | if (src->level > TRACE_LEVEL_USER) { |
260 | 0 | const struct buffer *buf = NULL; |
261 | |
|
262 | 0 | chunk_appendf(&trace_buf, " bi=%u@%p+%u/%u", |
263 | 0 | (unsigned int)b_data(&check->bi), b_orig(&check->bi), |
264 | 0 | (unsigned int)b_head_ofs(&check->bi), (unsigned int)b_size(&check->bi)); |
265 | 0 | chunk_appendf(&trace_buf, " bo=%u@%p+%u/%u", |
266 | 0 | (unsigned int)b_data(&check->bo), b_orig(&check->bo), |
267 | 0 | (unsigned int)b_head_ofs(&check->bo), (unsigned int)b_size(&check->bo)); |
268 | |
|
269 | 0 | if (src->verbosity >= CHK_VERB_ADVANCED && (mask & (CHK_EV_RX))) |
270 | 0 | buf = (b_is_null(&check->bi) ? NULL : &check->bi); |
271 | 0 | else if (src->verbosity >= CHK_VERB_ADVANCED && (mask & (CHK_EV_TX))) |
272 | 0 | buf = (b_is_null(&check->bo) ? NULL : &check->bo); |
273 | |
|
274 | 0 | if (buf) { |
275 | 0 | if ((check->tcpcheck_rules->flags & TCPCHK_RULES_PROTO_CHK) == TCPCHK_RULES_HTTP_CHK) { |
276 | 0 | int full = (src->verbosity == CHK_VERB_COMPLETE); |
277 | |
|
278 | 0 | chunk_memcat(&trace_buf, "\n\t", 2); |
279 | 0 | htx_dump(&trace_buf, htxbuf(buf), full); |
280 | 0 | } |
281 | 0 | else { |
282 | 0 | int max = ((src->verbosity == CHK_VERB_COMPLETE) ? 1024 : 256); |
283 | |
|
284 | 0 | chunk_memcat(&trace_buf, "\n", 1); |
285 | 0 | if (b_data(buf) > max) { |
286 | 0 | check_trace_buf(buf, 0, max); |
287 | 0 | chunk_memcat(&trace_buf, " ...\n", 6); |
288 | 0 | } |
289 | 0 | else |
290 | 0 | check_trace_buf(buf, 0, b_data(buf)); |
291 | 0 | } |
292 | |
|
293 | 0 | } |
294 | 0 | } |
295 | |
|
296 | 0 | } |
297 | | |
298 | | |
299 | | /**************************************************************************/ |
300 | | /************************ Handle check results ****************************/ |
301 | | /**************************************************************************/ |
302 | | struct check_status { |
303 | | short result; /* one of SRV_CHK_* */ |
304 | | char *info; /* human readable short info */ |
305 | | char *desc; /* long description */ |
306 | | }; |
307 | | |
308 | | struct analyze_status { |
309 | | char *desc; /* description */ |
310 | | unsigned char lr[HANA_OBS_SIZE]; /* result for l4/l7: 0 = ignore, 1 - error, 2 - OK */ |
311 | | }; |
312 | | |
313 | | static const struct check_status check_statuses[HCHK_STATUS_SIZE] = { |
314 | | [HCHK_STATUS_UNKNOWN] = { CHK_RES_UNKNOWN, "UNK", "Unknown" }, |
315 | | [HCHK_STATUS_INI] = { CHK_RES_UNKNOWN, "INI", "Initializing" }, |
316 | | [HCHK_STATUS_START] = { /* SPECIAL STATUS*/ }, |
317 | | |
318 | | /* Below we have finished checks */ |
319 | | [HCHK_STATUS_CHECKED] = { CHK_RES_NEUTRAL, "CHECKED", "No status change" }, |
320 | | [HCHK_STATUS_HANA] = { CHK_RES_FAILED, "HANA", "Health analyze" }, |
321 | | |
322 | | [HCHK_STATUS_SOCKERR] = { CHK_RES_FAILED, "SOCKERR", "Socket error" }, |
323 | | |
324 | | [HCHK_STATUS_L4OK] = { CHK_RES_PASSED, "L4OK", "Layer4 check passed" }, |
325 | | [HCHK_STATUS_L4TOUT] = { CHK_RES_FAILED, "L4TOUT", "Layer4 timeout" }, |
326 | | [HCHK_STATUS_L4CON] = { CHK_RES_FAILED, "L4CON", "Layer4 connection problem" }, |
327 | | |
328 | | [HCHK_STATUS_L6OK] = { CHK_RES_PASSED, "L6OK", "Layer6 check passed" }, |
329 | | [HCHK_STATUS_L6TOUT] = { CHK_RES_FAILED, "L6TOUT", "Layer6 timeout" }, |
330 | | [HCHK_STATUS_L6RSP] = { CHK_RES_FAILED, "L6RSP", "Layer6 invalid response" }, |
331 | | |
332 | | [HCHK_STATUS_L7TOUT] = { CHK_RES_FAILED, "L7TOUT", "Layer7 timeout" }, |
333 | | [HCHK_STATUS_L7RSP] = { CHK_RES_FAILED, "L7RSP", "Layer7 invalid response" }, |
334 | | |
335 | | [HCHK_STATUS_L57DATA] = { /* DUMMY STATUS */ }, |
336 | | |
337 | | [HCHK_STATUS_L7OKD] = { CHK_RES_PASSED, "L7OK", "Layer7 check passed" }, |
338 | | [HCHK_STATUS_L7OKCD] = { CHK_RES_CONDPASS, "L7OKC", "Layer7 check conditionally passed" }, |
339 | | [HCHK_STATUS_L7STS] = { CHK_RES_FAILED, "L7STS", "Layer7 wrong status" }, |
340 | | |
341 | | [HCHK_STATUS_PROCERR] = { CHK_RES_FAILED, "PROCERR", "External check error" }, |
342 | | [HCHK_STATUS_PROCTOUT] = { CHK_RES_FAILED, "PROCTOUT", "External check timeout" }, |
343 | | [HCHK_STATUS_PROCOK] = { CHK_RES_PASSED, "PROCOK", "External check passed" }, |
344 | | }; |
345 | | |
346 | | static const struct analyze_status analyze_statuses[HANA_STATUS_SIZE] = { /* 0: ignore, 1: error, 2: OK */ |
347 | | [HANA_STATUS_UNKNOWN] = { "Unknown", { 0, 0 }}, |
348 | | |
349 | | [HANA_STATUS_L4_OK] = { "L4 successful connection", { 2, 0 }}, |
350 | | [HANA_STATUS_L4_ERR] = { "L4 unsuccessful connection", { 1, 1 }}, |
351 | | |
352 | | [HANA_STATUS_HTTP_OK] = { "Correct http response", { 0, 2 }}, |
353 | | [HANA_STATUS_HTTP_STS] = { "Wrong http response", { 0, 1 }}, |
354 | | [HANA_STATUS_HTTP_HDRRSP] = { "Invalid http response (headers)", { 0, 1 }}, |
355 | | [HANA_STATUS_HTTP_RSP] = { "Invalid http response", { 0, 1 }}, |
356 | | |
357 | | [HANA_STATUS_HTTP_READ_ERROR] = { "Read error (http)", { 0, 1 }}, |
358 | | [HANA_STATUS_HTTP_READ_TIMEOUT] = { "Read timeout (http)", { 0, 1 }}, |
359 | | [HANA_STATUS_HTTP_BROKEN_PIPE] = { "Close from server (http)", { 0, 1 }}, |
360 | | }; |
361 | | |
362 | | /* checks if <err> is a real error for errno or one that can be ignored, and |
363 | | * return 0 for these ones or <err> for real ones. |
364 | | */ |
365 | | static inline int unclean_errno(int err) |
366 | 0 | { |
367 | 0 | if (err == EAGAIN || err == EWOULDBLOCK || err == EINPROGRESS || |
368 | 0 | err == EISCONN || err == EALREADY) |
369 | 0 | return 0; |
370 | 0 | return err; |
371 | 0 | } |
372 | | |
373 | | /* Converts check_status code to result code */ |
374 | | short get_check_status_result(short check_status) |
375 | 0 | { |
376 | 0 | if (check_status < HCHK_STATUS_SIZE) |
377 | 0 | return check_statuses[check_status].result; |
378 | 0 | else |
379 | 0 | return check_statuses[HCHK_STATUS_UNKNOWN].result; |
380 | 0 | } |
381 | | |
382 | | /* Converts check_status code to description */ |
383 | 0 | const char *get_check_status_description(short check_status) { |
384 | |
|
385 | 0 | const char *desc; |
386 | |
|
387 | 0 | if (check_status < HCHK_STATUS_SIZE) |
388 | 0 | desc = check_statuses[check_status].desc; |
389 | 0 | else |
390 | 0 | desc = NULL; |
391 | |
|
392 | 0 | if (desc && *desc) |
393 | 0 | return desc; |
394 | 0 | else |
395 | 0 | return check_statuses[HCHK_STATUS_UNKNOWN].desc; |
396 | 0 | } |
397 | | |
398 | | /* Converts check_status code to short info */ |
399 | | const char *get_check_status_info(short check_status) |
400 | 0 | { |
401 | 0 | const char *info; |
402 | |
|
403 | 0 | if (check_status < HCHK_STATUS_SIZE) |
404 | 0 | info = check_statuses[check_status].info; |
405 | 0 | else |
406 | 0 | info = NULL; |
407 | |
|
408 | 0 | if (info && *info) |
409 | 0 | return info; |
410 | 0 | else |
411 | 0 | return check_statuses[HCHK_STATUS_UNKNOWN].info; |
412 | 0 | } |
413 | | |
414 | | /* Convert analyze_status to description */ |
415 | 0 | const char *get_analyze_status(short analyze_status) { |
416 | |
|
417 | 0 | const char *desc; |
418 | |
|
419 | 0 | if (analyze_status < HANA_STATUS_SIZE) |
420 | 0 | desc = analyze_statuses[analyze_status].desc; |
421 | 0 | else |
422 | 0 | desc = NULL; |
423 | |
|
424 | 0 | if (desc && *desc) |
425 | 0 | return desc; |
426 | 0 | else |
427 | 0 | return analyze_statuses[HANA_STATUS_UNKNOWN].desc; |
428 | 0 | } |
429 | | |
430 | | /* append check info to buffer msg */ |
431 | | void check_append_info(struct buffer *msg, struct check *check) |
432 | 0 | { |
433 | 0 | if (!check) |
434 | 0 | return; |
435 | 0 | chunk_appendf(msg, ", reason: %s", get_check_status_description(check->status)); |
436 | |
|
437 | 0 | if (check->status >= HCHK_STATUS_L57DATA) |
438 | 0 | chunk_appendf(msg, ", code: %d", check->code); |
439 | |
|
440 | 0 | if (check->desc[0]) { |
441 | 0 | struct buffer src; |
442 | |
|
443 | 0 | chunk_appendf(msg, ", info: \""); |
444 | |
|
445 | 0 | chunk_initlen(&src, check->desc, 0, strlen(check->desc)); |
446 | 0 | chunk_asciiencode(msg, &src, '"'); |
447 | |
|
448 | 0 | chunk_appendf(msg, "\""); |
449 | 0 | } |
450 | |
|
451 | 0 | if (check->duration >= 0) |
452 | 0 | chunk_appendf(msg, ", check duration: %ldms", check->duration); |
453 | 0 | } |
454 | | |
455 | | /* Sets check->status, update check->duration and fill check->result with an |
456 | | * adequate CHK_RES_* value. The new check->health is computed based on the |
457 | | * result. |
458 | | * |
459 | | * Shows information in logs about failed health check if server is UP or |
460 | | * succeeded health checks if server is DOWN. |
461 | | */ |
462 | | void set_server_check_status(struct check *check, short status, const char *desc) |
463 | 0 | { |
464 | 0 | struct server *s = check->server; |
465 | 0 | short prev_status = check->status; |
466 | 0 | int report = (status != prev_status) ? 1 : 0; |
467 | |
|
468 | 0 | TRACE_POINT(CHK_EV_HCHK_RUN, check); |
469 | |
|
470 | 0 | if (status == HCHK_STATUS_START) { |
471 | 0 | check->result = CHK_RES_UNKNOWN; /* no result yet */ |
472 | 0 | check->desc[0] = '\0'; |
473 | 0 | check->start = now_ns; |
474 | 0 | return; |
475 | 0 | } |
476 | | |
477 | 0 | if (!check->status) |
478 | 0 | return; |
479 | | |
480 | 0 | if (desc && *desc) { |
481 | 0 | strncpy(check->desc, desc, HCHK_DESC_LEN-1); |
482 | 0 | check->desc[HCHK_DESC_LEN-1] = '\0'; |
483 | 0 | } else |
484 | 0 | check->desc[0] = '\0'; |
485 | |
|
486 | 0 | check->status = status; |
487 | 0 | if (check_statuses[status].result) |
488 | 0 | check->result = check_statuses[status].result; |
489 | |
|
490 | 0 | if (status == HCHK_STATUS_HANA) |
491 | 0 | check->duration = -1; |
492 | 0 | else if (check->start) { |
493 | | /* set_server_check_status() may be called more than once */ |
494 | 0 | check->duration = ns_to_ms(now_ns - check->start); |
495 | 0 | check->start = 0; |
496 | 0 | } |
497 | | |
498 | | /* no change is expected if no state change occurred */ |
499 | 0 | if (check->result == CHK_RES_NEUTRAL) |
500 | 0 | return; |
501 | | |
502 | | /* If the check was really just sending a mail, it won't have an |
503 | | * associated server, so we're done now. |
504 | | */ |
505 | 0 | if (!s) |
506 | 0 | return; |
507 | | |
508 | 0 | switch (check->result) { |
509 | 0 | case CHK_RES_FAILED: |
510 | | /* Failure to connect to the agent as a secondary check should not |
511 | | * cause the server to be marked down. |
512 | | */ |
513 | 0 | if ((!(check->state & CHK_ST_AGENT) || |
514 | 0 | (check->status >= HCHK_STATUS_L57DATA)) && |
515 | 0 | (check->health > 0)) { |
516 | 0 | if (s->counters.shared.tg[tgid - 1]) |
517 | 0 | _HA_ATOMIC_INC(&s->counters.shared.tg[tgid - 1]->failed_checks); |
518 | 0 | report = 1; |
519 | 0 | check->health--; |
520 | 0 | if (check->health < check->rise) |
521 | 0 | check->health = 0; |
522 | 0 | } |
523 | 0 | break; |
524 | | |
525 | 0 | case CHK_RES_PASSED: |
526 | 0 | case CHK_RES_CONDPASS: |
527 | 0 | if (check->health < check->rise + check->fall - 1) { |
528 | 0 | report = 1; |
529 | 0 | check->health++; |
530 | |
|
531 | 0 | if (check->health >= check->rise) |
532 | 0 | check->health = check->rise + check->fall - 1; /* OK now */ |
533 | 0 | } |
534 | | |
535 | | /* clear consecutive_errors if observing is enabled */ |
536 | 0 | if (s->onerror) |
537 | 0 | HA_ATOMIC_STORE(&s->consecutive_errors, 0); |
538 | 0 | break; |
539 | | |
540 | 0 | default: |
541 | 0 | break; |
542 | 0 | } |
543 | | |
544 | 0 | if (report) |
545 | 0 | srv_event_hdl_publish_check(s, check); |
546 | |
|
547 | 0 | if (s->proxy->options2 & PR_O2_LOGHCHKS && report) { |
548 | 0 | chunk_printf(&trash, |
549 | 0 | "%s check for %sserver %s/%s %s%s", |
550 | 0 | (check->state & CHK_ST_AGENT) ? "Agent" : "Health", |
551 | 0 | s->flags & SRV_F_BACKUP ? "backup " : "", |
552 | 0 | s->proxy->id, s->id, |
553 | 0 | (check->result == CHK_RES_CONDPASS) ? "conditionally ":"", |
554 | 0 | (check->result >= CHK_RES_PASSED) ? "succeeded" : "failed"); |
555 | |
|
556 | 0 | check_append_info(&trash, check); |
557 | |
|
558 | 0 | chunk_appendf(&trash, ", status: %d/%d %s", |
559 | 0 | (check->health >= check->rise) ? check->health - check->rise + 1 : check->health, |
560 | 0 | (check->health >= check->rise) ? check->fall : check->rise, |
561 | 0 | (check->health >= check->rise) ? (s->uweight ? "UP" : "DRAIN") : "DOWN"); |
562 | |
|
563 | 0 | ha_warning("%s.\n", trash.area); |
564 | 0 | send_log(s->proxy, LOG_NOTICE, "%s.\n", trash.area); |
565 | 0 | } |
566 | 0 | } |
567 | | |
568 | | static inline enum srv_op_st_chg_cause check_notify_cause(struct check *check) |
569 | 0 | { |
570 | 0 | struct server *s = check->server; |
571 | | |
572 | | /* We only report a cause for the check if we did not do so previously */ |
573 | 0 | if (!s->track && !(s->proxy->options2 & PR_O2_LOGHCHKS)) |
574 | 0 | return (check->state & CHK_ST_AGENT) ? SRV_OP_STCHGC_AGENT : SRV_OP_STCHGC_HEALTH; |
575 | 0 | return SRV_OP_STCHGC_NONE; |
576 | 0 | } |
577 | | |
578 | | /* Marks the check <check>'s server down if the current check is already failed |
579 | | * and the server is not down yet nor in maintenance. |
580 | | */ |
581 | | void check_notify_failure(struct check *check) |
582 | 0 | { |
583 | 0 | struct server *s = check->server; |
584 | | |
585 | | /* The agent secondary check should only cause a server to be marked |
586 | | * as down if check->status is HCHK_STATUS_L7STS, which indicates |
587 | | * that the agent returned "fail", "stopped" or "down". |
588 | | * The implication here is that failure to connect to the agent |
589 | | * as a secondary check should not cause the server to be marked |
590 | | * down. */ |
591 | 0 | if ((check->state & CHK_ST_AGENT) && check->status != HCHK_STATUS_L7STS) |
592 | 0 | return; |
593 | | |
594 | 0 | if (check->health > 0) |
595 | 0 | return; |
596 | | |
597 | 0 | TRACE_STATE("health-check failed, set server DOWN", CHK_EV_HCHK_END|CHK_EV_HCHK_ERR, check); |
598 | 0 | srv_set_stopped(s, check_notify_cause(check)); |
599 | 0 | } |
600 | | |
601 | | /* Marks the check <check> as valid and tries to set its server up, provided |
602 | | * it isn't in maintenance, it is not tracking a down server and other checks |
603 | | * comply. The rule is simple : by default, a server is up, unless any of the |
604 | | * following conditions is true : |
605 | | * - health check failed (check->health < rise) |
606 | | * - agent check failed (agent->health < rise) |
607 | | * - the server tracks a down server (track && track->state == STOPPED) |
608 | | * Note that if the server has a slowstart, it will switch to STARTING instead |
609 | | * of RUNNING. Also, only the health checks support the nolb mode, so the |
610 | | * agent's success may not take the server out of this mode. |
611 | | */ |
612 | | void check_notify_success(struct check *check) |
613 | 0 | { |
614 | 0 | struct server *s = check->server; |
615 | |
|
616 | 0 | if (s->next_admin & SRV_ADMF_MAINT) |
617 | 0 | return; |
618 | | |
619 | 0 | if (s->track && s->track->next_state == SRV_ST_STOPPED) |
620 | 0 | return; |
621 | | |
622 | 0 | if ((s->check.state & CHK_ST_ENABLED) && (s->check.health < s->check.rise)) |
623 | 0 | return; |
624 | | |
625 | 0 | if ((s->agent.state & CHK_ST_ENABLED) && (s->agent.health < s->agent.rise)) |
626 | 0 | return; |
627 | | |
628 | 0 | if ((check->state & CHK_ST_AGENT) && s->next_state == SRV_ST_STOPPING) |
629 | 0 | return; |
630 | | |
631 | 0 | TRACE_STATE("health-check succeeded, set server RUNNING", CHK_EV_HCHK_END|CHK_EV_HCHK_SUCC, check); |
632 | 0 | srv_set_running(s, check_notify_cause(check)); |
633 | 0 | } |
634 | | |
635 | | /* Marks the check <check> as valid and tries to set its server into stopping mode |
636 | | * if it was running or starting, and provided it isn't in maintenance and other |
637 | | * checks comply. The conditions for the server to be marked in stopping mode are |
638 | | * the same as for it to be turned up. Also, only the health checks support the |
639 | | * nolb mode. |
640 | | */ |
641 | | void check_notify_stopping(struct check *check) |
642 | 0 | { |
643 | 0 | struct server *s = check->server; |
644 | |
|
645 | 0 | if (s->next_admin & SRV_ADMF_MAINT) |
646 | 0 | return; |
647 | | |
648 | 0 | if (check->state & CHK_ST_AGENT) |
649 | 0 | return; |
650 | | |
651 | 0 | if (s->track && s->track->next_state == SRV_ST_STOPPED) |
652 | 0 | return; |
653 | | |
654 | 0 | if ((s->check.state & CHK_ST_ENABLED) && (s->check.health < s->check.rise)) |
655 | 0 | return; |
656 | | |
657 | 0 | if ((s->agent.state & CHK_ST_ENABLED) && (s->agent.health < s->agent.rise)) |
658 | 0 | return; |
659 | | |
660 | 0 | TRACE_STATE("health-check condionnaly succeeded, set server STOPPING", CHK_EV_HCHK_END|CHK_EV_HCHK_SUCC, check); |
661 | 0 | srv_set_stopping(s, check_notify_cause(check)); |
662 | 0 | } |
663 | | |
664 | | /* note: use health_adjust() only, which first checks that the observe mode is |
665 | | * enabled. This will take the server lock if needed. |
666 | | */ |
667 | | void __health_adjust(struct server *s, short status) |
668 | 0 | { |
669 | 0 | int failed; |
670 | |
|
671 | 0 | if (s->observe >= HANA_OBS_SIZE) |
672 | 0 | return; |
673 | | |
674 | 0 | if (status >= HANA_STATUS_SIZE || !analyze_statuses[status].desc) |
675 | 0 | return; |
676 | | |
677 | 0 | switch (analyze_statuses[status].lr[s->observe - 1]) { |
678 | 0 | case 1: |
679 | 0 | failed = 1; |
680 | 0 | break; |
681 | | |
682 | 0 | case 2: |
683 | 0 | failed = 0; |
684 | 0 | break; |
685 | | |
686 | 0 | default: |
687 | 0 | return; |
688 | 0 | } |
689 | | |
690 | 0 | if (!failed) { |
691 | | /* good: clear consecutive_errors */ |
692 | 0 | HA_ATOMIC_STORE(&s->consecutive_errors, 0); |
693 | 0 | return; |
694 | 0 | } |
695 | | |
696 | 0 | if (HA_ATOMIC_ADD_FETCH(&s->consecutive_errors, 1) < s->consecutive_errors_limit) |
697 | 0 | return; |
698 | | |
699 | 0 | chunk_printf(&trash, "Detected %d consecutive errors, last one was: %s", |
700 | 0 | HA_ATOMIC_LOAD(&s->consecutive_errors), get_analyze_status(status)); |
701 | |
|
702 | 0 | HA_SPIN_LOCK(SERVER_LOCK, &s->lock); |
703 | | |
704 | | /* force fastinter for upcoming check |
705 | | * (does nothing if fastinter is not enabled) |
706 | | */ |
707 | 0 | s->check.state |= CHK_ST_FASTINTER; |
708 | |
|
709 | 0 | switch (s->onerror) { |
710 | 0 | case HANA_ONERR_FASTINTER: |
711 | | /* force fastinter - nothing to do here as all modes force it */ |
712 | 0 | break; |
713 | | |
714 | 0 | case HANA_ONERR_SUDDTH: |
715 | | /* simulate a pre-fatal failed health check */ |
716 | 0 | if (s->check.health > s->check.rise) |
717 | 0 | s->check.health = s->check.rise + 1; |
718 | |
|
719 | 0 | __fallthrough; |
720 | |
|
721 | 0 | case HANA_ONERR_FAILCHK: |
722 | | /* simulate a failed health check */ |
723 | 0 | set_server_check_status(&s->check, HCHK_STATUS_HANA, |
724 | 0 | trash.area); |
725 | 0 | check_notify_failure(&s->check); |
726 | 0 | break; |
727 | | |
728 | 0 | case HANA_ONERR_MARKDWN: |
729 | | /* mark server down */ |
730 | 0 | s->check.health = s->check.rise; |
731 | 0 | set_server_check_status(&s->check, HCHK_STATUS_HANA, |
732 | 0 | trash.area); |
733 | 0 | check_notify_failure(&s->check); |
734 | 0 | break; |
735 | | |
736 | 0 | default: |
737 | | /* write a warning? */ |
738 | 0 | break; |
739 | 0 | } |
740 | | |
741 | 0 | HA_SPIN_UNLOCK(SERVER_LOCK, &s->lock); |
742 | |
|
743 | 0 | HA_ATOMIC_STORE(&s->consecutive_errors, 0); |
744 | 0 | if (s->counters.shared.tg[tgid - 1]) |
745 | 0 | _HA_ATOMIC_INC(&s->counters.shared.tg[tgid - 1]->failed_hana); |
746 | |
|
747 | 0 | if (s->check.fastinter) { |
748 | | /* timer might need to be advanced, it might also already be |
749 | | * running in another thread. Let's just wake the task up, it |
750 | | * will automatically adjust its timer. |
751 | | */ |
752 | 0 | task_wakeup(s->check.task, TASK_WOKEN_MSG); |
753 | 0 | } |
754 | 0 | } |
755 | | |
756 | | /* Checks the connection. If an error has already been reported or the socket is |
757 | | * closed, keep errno intact as it is supposed to contain the valid error code. |
758 | | * If no error is reported, check the socket's error queue using getsockopt(). |
759 | | * Warning, this must be done only once when returning from poll, and never |
760 | | * after an I/O error was attempted, otherwise the error queue might contain |
761 | | * inconsistent errors. If an error is detected, the CO_FL_ERROR is set on the |
762 | | * socket. Returns non-zero if an error was reported, zero if everything is |
763 | | * clean (including a properly closed socket). |
764 | | */ |
765 | | static int retrieve_errno_from_socket(struct connection *conn) |
766 | 0 | { |
767 | 0 | int skerr; |
768 | 0 | socklen_t lskerr = sizeof(skerr); |
769 | |
|
770 | 0 | if (conn->flags & CO_FL_ERROR && (unclean_errno(errno) || !conn->ctrl)) |
771 | 0 | return 1; |
772 | | |
773 | 0 | if (!conn_ctrl_ready(conn)) |
774 | 0 | return 0; |
775 | | |
776 | 0 | BUG_ON(conn->flags & CO_FL_FDLESS); |
777 | |
|
778 | 0 | if (getsockopt(conn->handle.fd, SOL_SOCKET, SO_ERROR, &skerr, &lskerr) == 0) |
779 | 0 | errno = skerr; |
780 | |
|
781 | 0 | errno = unclean_errno(errno); |
782 | |
|
783 | 0 | if (!errno) { |
784 | | /* we could not retrieve an error, that does not mean there is |
785 | | * none. Just don't change anything and only report the prior |
786 | | * error if any. |
787 | | */ |
788 | 0 | if (conn->flags & CO_FL_ERROR) |
789 | 0 | return 1; |
790 | 0 | else |
791 | 0 | return 0; |
792 | 0 | } |
793 | | |
794 | 0 | conn->flags |= CO_FL_ERROR | CO_FL_SOCK_WR_SH | CO_FL_SOCK_RD_SH; |
795 | 0 | return 1; |
796 | 0 | } |
797 | | |
798 | | /* Tries to collect as much information as possible on the connection status, |
799 | | * and adjust the server status accordingly. It may make use of <errno_bck> |
800 | | * if non-null when the caller is absolutely certain of its validity (eg: |
801 | | * checked just after a syscall). If the caller doesn't have a valid errno, |
802 | | * it can pass zero, and retrieve_errno_from_socket() will be called to try |
803 | | * to extract errno from the socket. If no error is reported, it will consider |
804 | | * the <expired> flag. This is intended to be used when a connection error was |
805 | | * reported in conn->flags or when a timeout was reported in <expired>. The |
806 | | * function takes care of not updating a server status which was already set. |
807 | | * All situations where at least one of <expired> or CO_FL_ERROR are set |
808 | | * produce a status. |
809 | | */ |
810 | | void chk_report_conn_err(struct check *check, int errno_bck, int expired) |
811 | 0 | { |
812 | 0 | struct stconn *sc = check->sc; |
813 | 0 | struct connection *conn = sc_conn(sc); |
814 | 0 | const char *err_msg; |
815 | 0 | struct buffer *chk; |
816 | 0 | int step; |
817 | |
|
818 | 0 | if (check->result != CHK_RES_UNKNOWN) { |
819 | 0 | return; |
820 | 0 | } |
821 | | |
822 | 0 | errno = unclean_errno(errno_bck); |
823 | 0 | if (conn && errno) |
824 | 0 | retrieve_errno_from_socket(conn); |
825 | |
|
826 | 0 | TRACE_ENTER(CHK_EV_HCHK_END|CHK_EV_HCHK_ERR, check, 0, 0, (size_t[]){expired}); |
827 | | |
828 | | /* we'll try to build a meaningful error message depending on the |
829 | | * context of the error possibly present in conn->err_code, and the |
830 | | * socket error possibly collected above. This is useful to know the |
831 | | * exact step of the L6 layer (eg: SSL handshake). |
832 | | */ |
833 | 0 | chk = get_trash_chunk(); |
834 | |
|
835 | 0 | if (check->type == PR_O2_TCPCHK_CHK && |
836 | 0 | (check->tcpcheck_rules->flags & TCPCHK_RULES_PROTO_CHK) == TCPCHK_RULES_TCP_CHK) { |
837 | 0 | step = tcpcheck_get_step_id(check, NULL); |
838 | 0 | if (!step) { |
839 | 0 | TRACE_DEVEL("initial connection failure", CHK_EV_HCHK_END|CHK_EV_HCHK_ERR, check); |
840 | 0 | chunk_printf(chk, " at initial connection step of tcp-check"); |
841 | 0 | } |
842 | 0 | else { |
843 | 0 | chunk_printf(chk, " at step %d of tcp-check", step); |
844 | | /* we were looking for a string */ |
845 | 0 | if (check->current_step && check->current_step->action == TCPCHK_ACT_CONNECT) { |
846 | 0 | if (check->current_step->connect.port) |
847 | 0 | chunk_appendf(chk, " (connect port %d)" ,check->current_step->connect.port); |
848 | 0 | else |
849 | 0 | chunk_appendf(chk, " (connect)"); |
850 | 0 | TRACE_DEVEL("connection failure", CHK_EV_HCHK_END|CHK_EV_HCHK_ERR, check); |
851 | 0 | } |
852 | 0 | else if (check->current_step && check->current_step->action == TCPCHK_ACT_EXPECT) { |
853 | 0 | struct tcpcheck_expect *expect = &check->current_step->expect; |
854 | |
|
855 | 0 | switch (expect->type) { |
856 | 0 | case TCPCHK_EXPECT_STRING: |
857 | 0 | chunk_appendf(chk, " (expect string '%.*s')", (unsigned int)istlen(expect->data), istptr(expect->data)); |
858 | 0 | break; |
859 | 0 | case TCPCHK_EXPECT_BINARY: |
860 | 0 | chunk_appendf(chk, " (expect binary '"); |
861 | 0 | dump_binary(chk, istptr(expect->data), (int)istlen(expect->data)); |
862 | 0 | chunk_appendf(chk, "')"); |
863 | 0 | break; |
864 | 0 | case TCPCHK_EXPECT_STRING_REGEX: |
865 | 0 | chunk_appendf(chk, " (expect regex)"); |
866 | 0 | break; |
867 | 0 | case TCPCHK_EXPECT_BINARY_REGEX: |
868 | 0 | chunk_appendf(chk, " (expect binary regex)"); |
869 | 0 | break; |
870 | 0 | case TCPCHK_EXPECT_STRING_LF: |
871 | 0 | chunk_appendf(chk, " (expect log-format string)"); |
872 | 0 | break; |
873 | 0 | case TCPCHK_EXPECT_BINARY_LF: |
874 | 0 | chunk_appendf(chk, " (expect log-format binary)"); |
875 | 0 | break; |
876 | 0 | case TCPCHK_EXPECT_HTTP_STATUS: |
877 | 0 | chunk_appendf(chk, " (expect HTTP status codes)"); |
878 | 0 | break; |
879 | 0 | case TCPCHK_EXPECT_HTTP_STATUS_REGEX: |
880 | 0 | chunk_appendf(chk, " (expect HTTP status regex)"); |
881 | 0 | break; |
882 | 0 | case TCPCHK_EXPECT_HTTP_HEADER: |
883 | 0 | chunk_appendf(chk, " (expect HTTP header pattern)"); |
884 | 0 | break; |
885 | 0 | case TCPCHK_EXPECT_HTTP_BODY: |
886 | 0 | chunk_appendf(chk, " (expect HTTP body content '%.*s')", (unsigned int)istlen(expect->data), istptr(expect->data)); |
887 | 0 | break; |
888 | 0 | case TCPCHK_EXPECT_HTTP_BODY_REGEX: |
889 | 0 | chunk_appendf(chk, " (expect HTTP body regex)"); |
890 | 0 | break; |
891 | 0 | case TCPCHK_EXPECT_HTTP_BODY_LF: |
892 | 0 | chunk_appendf(chk, " (expect log-format HTTP body)"); |
893 | 0 | break; |
894 | 0 | case TCPCHK_EXPECT_CUSTOM: |
895 | 0 | chunk_appendf(chk, " (expect custom function)"); |
896 | 0 | break; |
897 | 0 | case TCPCHK_EXPECT_UNDEF: |
898 | 0 | chunk_appendf(chk, " (undefined expect!)"); |
899 | 0 | break; |
900 | 0 | } |
901 | 0 | TRACE_DEVEL("expect rule failed", CHK_EV_HCHK_END|CHK_EV_HCHK_ERR, check); |
902 | 0 | } |
903 | 0 | else if (check->current_step && check->current_step->action == TCPCHK_ACT_SEND) { |
904 | 0 | chunk_appendf(chk, " (send)"); |
905 | 0 | TRACE_DEVEL("send rule failed", CHK_EV_HCHK_END|CHK_EV_HCHK_ERR, check); |
906 | 0 | } |
907 | | |
908 | 0 | if (check->current_step && check->current_step->comment) |
909 | 0 | chunk_appendf(chk, " comment: '%s'", check->current_step->comment); |
910 | 0 | } |
911 | 0 | } |
912 | | |
913 | 0 | if (conn && conn->err_code) { |
914 | 0 | if (unclean_errno(errno)) |
915 | 0 | chunk_printf(&trash, "%s (%s)%s", conn_err_code_str(conn), strerror(errno), |
916 | 0 | chk->area); |
917 | 0 | else |
918 | 0 | chunk_printf(&trash, "%s%s", conn_err_code_str(conn), |
919 | 0 | chk->area); |
920 | 0 | err_msg = trash.area; |
921 | 0 | } |
922 | 0 | else { |
923 | 0 | if (unclean_errno(errno)) { |
924 | 0 | chunk_printf(&trash, "%s%s", strerror(errno), |
925 | 0 | chk->area); |
926 | 0 | err_msg = trash.area; |
927 | 0 | } |
928 | 0 | else { |
929 | 0 | err_msg = chk->area; |
930 | 0 | } |
931 | 0 | } |
932 | |
|
933 | 0 | if (check->state & CHK_ST_PORT_MISS) { |
934 | | /* NOTE: this is reported after <fall> tries */ |
935 | 0 | set_server_check_status(check, HCHK_STATUS_SOCKERR, err_msg); |
936 | 0 | } |
937 | |
|
938 | 0 | if (!conn || !conn->ctrl) { |
939 | | /* error before any connection attempt (connection allocation error or no control layer) */ |
940 | 0 | set_server_check_status(check, HCHK_STATUS_SOCKERR, err_msg); |
941 | 0 | } |
942 | 0 | else if (conn->flags & CO_FL_WAIT_L4_CONN) { |
943 | | /* L4 not established (yet) */ |
944 | 0 | if (conn->flags & CO_FL_ERROR || sc_ep_test(sc, SE_FL_ERROR)) |
945 | 0 | set_server_check_status(check, HCHK_STATUS_L4CON, err_msg); |
946 | 0 | else if (expired) |
947 | 0 | set_server_check_status(check, HCHK_STATUS_L4TOUT, err_msg); |
948 | | |
949 | | /* |
950 | | * might be due to a server IP change. |
951 | | * Let's trigger a DNS resolution if none are currently running. |
952 | | */ |
953 | 0 | if (check->server) |
954 | 0 | resolv_trigger_resolution(check->server->resolv_requester); |
955 | |
|
956 | 0 | } |
957 | 0 | else if (conn->flags & CO_FL_WAIT_L6_CONN) { |
958 | | /* L6 not established (yet) */ |
959 | 0 | if (conn->flags & CO_FL_ERROR || sc_ep_test(sc, SE_FL_ERROR)) |
960 | 0 | set_server_check_status(check, HCHK_STATUS_L6RSP, err_msg); |
961 | 0 | else if (expired) |
962 | 0 | set_server_check_status(check, HCHK_STATUS_L6TOUT, err_msg); |
963 | 0 | } |
964 | 0 | else if (conn->flags & CO_FL_ERROR || sc_ep_test(sc, SE_FL_ERROR)) { |
965 | | /* I/O error after connection was established and before we could diagnose */ |
966 | 0 | set_server_check_status(check, HCHK_STATUS_SOCKERR, err_msg); |
967 | 0 | } |
968 | 0 | else if (expired) { |
969 | 0 | enum healthcheck_status tout = HCHK_STATUS_L7TOUT; |
970 | | |
971 | | /* connection established but expired check */ |
972 | 0 | if (check->current_step && check->current_step->action == TCPCHK_ACT_EXPECT && |
973 | 0 | check->current_step->expect.tout_status != HCHK_STATUS_UNKNOWN) |
974 | 0 | tout = check->current_step->expect.tout_status; |
975 | 0 | set_server_check_status(check, tout, err_msg); |
976 | 0 | } |
977 | |
|
978 | 0 | if (check->result == CHK_RES_UNKNOWN) { |
979 | | /* No other reason found, report a socket error (may be an internal or a ressournce error) */ |
980 | 0 | set_server_check_status(check, HCHK_STATUS_SOCKERR, err_msg); |
981 | 0 | } |
982 | |
|
983 | 0 | TRACE_LEAVE(CHK_EV_HCHK_END|CHK_EV_HCHK_ERR, check); |
984 | 0 | return; |
985 | 0 | } |
986 | | |
987 | | |
988 | | /* Builds the server state header used by HTTP health-checks */ |
989 | | int httpchk_build_status_header(struct server *s, struct buffer *buf) |
990 | 0 | { |
991 | 0 | int sv_state; |
992 | 0 | int ratio; |
993 | 0 | char addr[46]; |
994 | 0 | char port[6]; |
995 | 0 | const char *srv_hlt_st[7] = { "DOWN", "DOWN %d/%d", |
996 | 0 | "UP %d/%d", "UP", |
997 | 0 | "NOLB %d/%d", "NOLB", |
998 | 0 | "no check" }; |
999 | 0 | unsigned long last_change = s->last_change; |
1000 | |
|
1001 | 0 | if (!(s->check.state & CHK_ST_ENABLED)) |
1002 | 0 | sv_state = 6; |
1003 | 0 | else if (s->cur_state != SRV_ST_STOPPED) { |
1004 | 0 | if (s->check.health == s->check.rise + s->check.fall - 1) |
1005 | 0 | sv_state = 3; /* UP */ |
1006 | 0 | else |
1007 | 0 | sv_state = 2; /* going down */ |
1008 | |
|
1009 | 0 | if (s->cur_state == SRV_ST_STOPPING) |
1010 | 0 | sv_state += 2; |
1011 | 0 | } else { |
1012 | 0 | if (s->check.health) |
1013 | 0 | sv_state = 1; /* going up */ |
1014 | 0 | else |
1015 | 0 | sv_state = 0; /* DOWN */ |
1016 | 0 | } |
1017 | |
|
1018 | 0 | chunk_appendf(buf, srv_hlt_st[sv_state], |
1019 | 0 | (s->cur_state != SRV_ST_STOPPED) ? (s->check.health - s->check.rise + 1) : (s->check.health), |
1020 | 0 | (s->cur_state != SRV_ST_STOPPED) ? (s->check.fall) : (s->check.rise)); |
1021 | |
|
1022 | 0 | addr_to_str(&s->addr, addr, sizeof(addr)); |
1023 | 0 | if (s->addr.ss_family == AF_INET || s->addr.ss_family == AF_INET6) |
1024 | 0 | snprintf(port, sizeof(port), "%u", s->svc_port); |
1025 | 0 | else |
1026 | 0 | *port = 0; |
1027 | |
|
1028 | 0 | chunk_appendf(buf, "; address=%s; port=%s; name=%s/%s; node=%s; weight=%d/%d; scur=%d/%d; qcur=%d", |
1029 | 0 | addr, port, s->proxy->id, s->id, |
1030 | 0 | global.node, |
1031 | 0 | (s->cur_eweight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv, |
1032 | 0 | (s->proxy->lbprm.tot_weight * s->proxy->lbprm.wmult + s->proxy->lbprm.wdiv - 1) / s->proxy->lbprm.wdiv, |
1033 | 0 | s->cur_sess, s->proxy->beconn - s->proxy->queueslength, |
1034 | 0 | s->queueslength); |
1035 | |
|
1036 | 0 | if ((s->cur_state == SRV_ST_STARTING) && |
1037 | 0 | ns_to_sec(now_ns) < last_change + s->slowstart && |
1038 | 0 | ns_to_sec(now_ns) >= last_change) { |
1039 | 0 | ratio = MAX(1, 100 * (ns_to_sec(now_ns) - last_change) / s->slowstart); |
1040 | 0 | chunk_appendf(buf, "; throttle=%d%%", ratio); |
1041 | 0 | } |
1042 | |
|
1043 | 0 | return b_data(buf); |
1044 | 0 | } |
1045 | | |
1046 | | /**************************************************************************/ |
1047 | | /***************** Health-checks based on connections *********************/ |
1048 | | /**************************************************************************/ |
1049 | | /* This function is used only for server health-checks. It handles connection |
1050 | | * status updates including errors. If necessary, it wakes the check task up. |
1051 | | * It returns 0 on normal cases, <0 if at least one close() has happened on the |
1052 | | * connection (eg: reconnect). It relies on tcpcheck_main(). |
1053 | | */ |
1054 | | int wake_srv_chk(struct stconn *sc) |
1055 | 0 | { |
1056 | 0 | struct connection *conn; |
1057 | 0 | struct check *check = __sc_check(sc); |
1058 | 0 | int ret = 0; |
1059 | |
|
1060 | 0 | BUG_ON(!check->server); |
1061 | |
|
1062 | 0 | TRACE_ENTER(CHK_EV_HCHK_WAKE, check); |
1063 | 0 | if (check->result != CHK_RES_UNKNOWN) |
1064 | 0 | goto end; |
1065 | | |
1066 | 0 | HA_SPIN_LOCK(SERVER_LOCK, &check->server->lock); |
1067 | | |
1068 | | /* we may have to make progress on the TCP checks */ |
1069 | 0 | ret = tcpcheck_main(check); |
1070 | |
|
1071 | 0 | sc = check->sc; |
1072 | 0 | conn = sc_conn(sc); |
1073 | |
|
1074 | 0 | if (unlikely(!conn || conn->flags & CO_FL_ERROR || sc_ep_test(sc, SE_FL_ERROR))) { |
1075 | | /* We may get error reports bypassing the I/O handlers, typically |
1076 | | * the case when sending a pure TCP check which fails, then the I/O |
1077 | | * handlers above are not called. This is completely handled by the |
1078 | | * main processing task so let's simply wake it up. If we get here, |
1079 | | * we expect errno to still be valid. |
1080 | | */ |
1081 | 0 | TRACE_ERROR("report connection error", CHK_EV_HCHK_WAKE|CHK_EV_HCHK_END|CHK_EV_HCHK_ERR, check); |
1082 | 0 | chk_report_conn_err(check, errno, 0); |
1083 | 0 | task_wakeup(check->task, TASK_WOKEN_IO); |
1084 | 0 | } |
1085 | |
|
1086 | 0 | if (check->result != CHK_RES_UNKNOWN || ret == -1) { |
1087 | | /* Check complete or aborted. Wake the check task up to be sure |
1088 | | * the result is handled ASAP. */ |
1089 | 0 | ret = -1; |
1090 | 0 | task_wakeup(check->task, TASK_WOKEN_IO); |
1091 | 0 | } |
1092 | 0 | else { |
1093 | | /* Check in progress. Queue it to eventually handle timeout |
1094 | | * update */ |
1095 | 0 | task_queue(check->task); |
1096 | 0 | } |
1097 | |
|
1098 | 0 | HA_SPIN_UNLOCK(SERVER_LOCK, &check->server->lock); |
1099 | |
|
1100 | 0 | end: |
1101 | 0 | TRACE_LEAVE(CHK_EV_HCHK_WAKE, check); |
1102 | 0 | return ret; |
1103 | 0 | } |
1104 | | |
1105 | | /* This function checks if any I/O is wanted, and if so, attempts to do so */ |
1106 | | struct task *srv_chk_io_cb(struct task *t, void *ctx, unsigned int state) |
1107 | 0 | { |
1108 | 0 | struct stconn *sc = ctx; |
1109 | |
|
1110 | 0 | wake_srv_chk(sc); |
1111 | 0 | return t; |
1112 | 0 | } |
1113 | | |
1114 | | /* returns <0, 0, >0 if check thread 1 is respectively less loaded than, |
1115 | | * equally as, or more loaded than thread 2. This is made to decide on |
1116 | | * migrations so a margin is applied in either direction. For ease of |
1117 | | * remembering the direction, consider this returns load1 - load2. |
1118 | | */ |
1119 | | static inline int check_thread_cmp_load(int thr1, int thr2) |
1120 | 0 | { |
1121 | 0 | uint t1_load = _HA_ATOMIC_LOAD(&ha_thread_ctx[thr1].rq_total); |
1122 | 0 | uint t1_act = _HA_ATOMIC_LOAD(&ha_thread_ctx[thr1].active_checks); |
1123 | 0 | uint t2_load = _HA_ATOMIC_LOAD(&ha_thread_ctx[thr2].rq_total); |
1124 | 0 | uint t2_act = _HA_ATOMIC_LOAD(&ha_thread_ctx[thr2].active_checks); |
1125 | | |
1126 | | /* twice as more active checks is a significant difference */ |
1127 | 0 | if (t1_act * 2 < t2_act) |
1128 | 0 | return -1; |
1129 | | |
1130 | 0 | if (t2_act * 2 < t1_act) |
1131 | 0 | return 1; |
1132 | | |
1133 | | /* twice as more rqload with more checks is also a significant |
1134 | | * difference. |
1135 | | */ |
1136 | 0 | if (t1_act <= t2_act && t1_load * 2 < t2_load) |
1137 | 0 | return -1; |
1138 | | |
1139 | 0 | if (t2_act <= t1_act && t2_load * 2 < t1_load) |
1140 | 0 | return 1; |
1141 | | |
1142 | | /* otherwise they're roughly equal */ |
1143 | 0 | return 0; |
1144 | 0 | } |
1145 | | |
1146 | | /* returns <0, 0, >0 if check thread 1's active checks count is respectively |
1147 | | * higher than, equal, or lower than thread 2's. This is made to decide on |
1148 | | * forced migrations upon overload, so only a very little margin is applied |
1149 | | * here (~1%). For ease of remembering the direction, consider this returns |
1150 | | * active1 - active2. |
1151 | | */ |
1152 | | static inline int check_thread_cmp_active(int thr1, int thr2) |
1153 | 0 | { |
1154 | 0 | uint t1_act = _HA_ATOMIC_LOAD(&ha_thread_ctx[thr1].active_checks); |
1155 | 0 | uint t2_act = _HA_ATOMIC_LOAD(&ha_thread_ctx[thr2].active_checks); |
1156 | |
|
1157 | 0 | if (t1_act * 128 >= t2_act * 129) |
1158 | 0 | return 1; |
1159 | 0 | if (t2_act * 128 >= t1_act * 129) |
1160 | 0 | return -1; |
1161 | 0 | return 0; |
1162 | 0 | } |
1163 | | |
1164 | | |
1165 | | /* manages a server health-check that uses a connection. Returns |
1166 | | * the time the task accepts to wait, or TIME_ETERNITY for infinity. |
1167 | | * |
1168 | | * Please do NOT place any return statement in this function and only leave |
1169 | | * via the out_unlock label. |
1170 | | */ |
1171 | | struct task *process_chk_conn(struct task *t, void *context, unsigned int state) |
1172 | 0 | { |
1173 | 0 | struct check *check = context; |
1174 | 0 | struct proxy *proxy = check->proxy; |
1175 | 0 | struct stconn *sc; |
1176 | 0 | struct connection *conn; |
1177 | 0 | int rv; |
1178 | 0 | int expired = tick_is_expired(t->expire, now_ms); |
1179 | |
|
1180 | 0 | TRACE_ENTER(CHK_EV_TASK_WAKE, check); |
1181 | |
|
1182 | 0 | if (check->state & CHK_ST_SLEEPING) { |
1183 | | /* This check just restarted. It's still time to verify if |
1184 | | * we're on an overloaded thread or if a more suitable one is |
1185 | | * available. This helps spread the load over the available |
1186 | | * threads, without migrating too often. For this we'll check |
1187 | | * our load, and pick a random thread, check if it has less |
1188 | | * than half of the current thread's load, and if so we'll |
1189 | | * bounce the task there. It's possible because it's not yet |
1190 | | * tied to the current thread. The other thread will not bounce |
1191 | | * the task again because we're setting CHK_ST_READY indicating |
1192 | | * a migration. |
1193 | | */ |
1194 | 0 | uint run_checks = _HA_ATOMIC_LOAD(&th_ctx->running_checks); |
1195 | 0 | uint my_load = HA_ATOMIC_LOAD(&th_ctx->rq_total); |
1196 | 0 | uint attempts = MIN(global.nbthread, 3); |
1197 | |
|
1198 | 0 | if (check->state & CHK_ST_READY) { |
1199 | | /* check was migrated, active already counted */ |
1200 | 0 | activity[tid].check_adopted++; |
1201 | 0 | } |
1202 | 0 | else { |
1203 | | /* first wakeup, let's check if another thread is less loaded |
1204 | | * than this one in order to smooth the load. If the current |
1205 | | * thread is not yet overloaded, we attempt an opportunistic |
1206 | | * migration to another thread that is not full and that is |
1207 | | * significantly less loaded. And if the current thread is |
1208 | | * already overloaded, we attempt a forced migration to a |
1209 | | * thread with less active checks. We try at most 3 random |
1210 | | * other thread. |
1211 | | */ |
1212 | 0 | while (attempts-- > 0 && |
1213 | 0 | (!LIST_ISEMPTY(&th_ctx->queued_checks) || my_load >= 3) && |
1214 | 0 | _HA_ATOMIC_LOAD(&th_ctx->active_checks) >= 3) { |
1215 | 0 | uint new_tid = statistical_prng_range(global.nbthread); |
1216 | |
|
1217 | 0 | if (new_tid == tid) |
1218 | 0 | continue; |
1219 | | |
1220 | 0 | ALREADY_CHECKED(new_tid); |
1221 | |
|
1222 | 0 | if (check_thread_cmp_active(tid, new_tid) > 0 && |
1223 | 0 | (run_checks >= global.tune.max_checks_per_thread || |
1224 | 0 | check_thread_cmp_load(tid, new_tid) > 0)) { |
1225 | | /* Found one. Let's migrate the task over there. We have to |
1226 | | * remove it from the WQ first and kill its expire time |
1227 | | * otherwise the scheduler will reinsert it and trigger a |
1228 | | * BUG_ON() as we're not allowed to call task_queue() for a |
1229 | | * foreign thread. The recipient will restore the expiration. |
1230 | | */ |
1231 | 0 | check->state |= CHK_ST_READY; |
1232 | 0 | HA_ATOMIC_INC(&ha_thread_ctx[new_tid].active_checks); |
1233 | 0 | task_unlink_wq(t); |
1234 | 0 | t->expire = TICK_ETERNITY; |
1235 | 0 | task_set_thread(t, new_tid); |
1236 | 0 | task_wakeup(t, TASK_WOKEN_MSG); |
1237 | 0 | TRACE_LEAVE(CHK_EV_TASK_WAKE, check); |
1238 | 0 | return t; |
1239 | 0 | } |
1240 | 0 | } |
1241 | | /* check just woke up, count it as active */ |
1242 | 0 | _HA_ATOMIC_INC(&th_ctx->active_checks); |
1243 | 0 | } |
1244 | | |
1245 | | /* OK we're keeping it so this check is ours now */ |
1246 | 0 | task_set_thread(t, tid); |
1247 | 0 | check->state &= ~CHK_ST_SLEEPING; |
1248 | | |
1249 | | /* if we just woke up and the thread is full of running, or |
1250 | | * already has others waiting, we might have to wait in queue |
1251 | | * (for health checks only). This means !SLEEPING && !READY. |
1252 | | */ |
1253 | 0 | if (check->server && |
1254 | 0 | (!LIST_ISEMPTY(&th_ctx->queued_checks) || |
1255 | 0 | (global.tune.max_checks_per_thread && |
1256 | 0 | _HA_ATOMIC_LOAD(&th_ctx->running_checks) >= global.tune.max_checks_per_thread))) { |
1257 | 0 | TRACE_DEVEL("health-check queued", CHK_EV_TASK_WAKE, check); |
1258 | 0 | t->expire = TICK_ETERNITY; |
1259 | 0 | LIST_APPEND(&th_ctx->queued_checks, &check->check_queue); |
1260 | | |
1261 | | /* reset fastinter flag (if set) so that srv_getinter() |
1262 | | * only returns fastinter if server health is degraded |
1263 | | */ |
1264 | 0 | check->state &= ~CHK_ST_FASTINTER; |
1265 | 0 | goto out_leave; |
1266 | 0 | } |
1267 | | |
1268 | | /* OK let's run, now we cannot roll back anymore */ |
1269 | 0 | check->state |= CHK_ST_READY; |
1270 | 0 | activity[tid].check_started++; |
1271 | 0 | _HA_ATOMIC_INC(&th_ctx->running_checks); |
1272 | 0 | } |
1273 | | |
1274 | | /* at this point, CHK_ST_SLEEPING = 0 and CHK_ST_READY = 1*/ |
1275 | | |
1276 | 0 | if (check->server) |
1277 | 0 | HA_SPIN_LOCK(SERVER_LOCK, &check->server->lock); |
1278 | |
|
1279 | 0 | if (!(check->state & (CHK_ST_INPROGRESS|CHK_ST_IN_ALLOC|CHK_ST_OUT_ALLOC))) { |
1280 | | /* This task might have bounced from another overloaded thread, it |
1281 | | * needs an expiration timer that was supposed to be now, but that |
1282 | | * was erased during the bounce. |
1283 | | */ |
1284 | 0 | if (!tick_isset(t->expire)) { |
1285 | 0 | t->expire = tick_add(now_ms, 0); |
1286 | 0 | expired = 0; |
1287 | 0 | } |
1288 | 0 | } |
1289 | |
|
1290 | 0 | if (unlikely(check->state & CHK_ST_PURGE)) { |
1291 | 0 | TRACE_STATE("health-check state to purge", CHK_EV_TASK_WAKE, check); |
1292 | 0 | } |
1293 | 0 | else if (!(check->state & (CHK_ST_INPROGRESS))) { |
1294 | | /* no check currently running, but we might have been woken up |
1295 | | * before the timer's expiration to update it according to a |
1296 | | * new state (e.g. fastinter), in which case we'll reprogram |
1297 | | * the new timer. |
1298 | | */ |
1299 | 0 | if (!tick_is_expired(t->expire, now_ms)) { /* woke up too early */ |
1300 | 0 | if (check->server) { |
1301 | 0 | int new_exp = tick_add(now_ms, MS_TO_TICKS(srv_getinter(check))); |
1302 | |
|
1303 | 0 | if (tick_is_expired(new_exp, t->expire)) { |
1304 | 0 | TRACE_STATE("health-check was advanced", CHK_EV_TASK_WAKE, check); |
1305 | 0 | goto update_timer; |
1306 | 0 | } |
1307 | 0 | } |
1308 | | |
1309 | 0 | TRACE_STATE("health-check wake up too early", CHK_EV_TASK_WAKE, check); |
1310 | 0 | goto out_unlock; |
1311 | 0 | } |
1312 | | |
1313 | | /* we don't send any health-checks when the proxy is |
1314 | | * stopped, the server should not be checked or the check |
1315 | | * is disabled. |
1316 | | */ |
1317 | 0 | if (((check->state & (CHK_ST_ENABLED | CHK_ST_PAUSED)) != CHK_ST_ENABLED) || |
1318 | 0 | (proxy->flags & (PR_FL_DISABLED|PR_FL_STOPPED))) { |
1319 | 0 | TRACE_STATE("health-check paused or disabled", CHK_EV_TASK_WAKE, check); |
1320 | 0 | goto reschedule; |
1321 | 0 | } |
1322 | | |
1323 | | /* we'll initiate a new check */ |
1324 | 0 | set_server_check_status(check, HCHK_STATUS_START, NULL); |
1325 | |
|
1326 | 0 | check->state |= CHK_ST_INPROGRESS; |
1327 | 0 | TRACE_STATE("init new health-check", CHK_EV_TASK_WAKE|CHK_EV_HCHK_START, check); |
1328 | |
|
1329 | 0 | check->current_step = NULL; |
1330 | |
|
1331 | 0 | check->sc = sc_new_from_check(check, SC_FL_NONE); |
1332 | 0 | if (!check->sc) { |
1333 | 0 | set_server_check_status(check, HCHK_STATUS_SOCKERR, NULL); |
1334 | 0 | goto end; |
1335 | 0 | } |
1336 | 0 | tcpcheck_main(check); |
1337 | 0 | expired = 0; |
1338 | 0 | } |
1339 | | |
1340 | | /* there was a test running. |
1341 | | * First, let's check whether there was an uncaught error, |
1342 | | * which can happen on connect timeout or error. |
1343 | | */ |
1344 | 0 | if (check->result == CHK_RES_UNKNOWN && likely(!(check->state & CHK_ST_PURGE))) { |
1345 | 0 | sc = check->sc; |
1346 | 0 | conn = sc_conn(sc); |
1347 | | |
1348 | | /* Here the connection must be defined. Otherwise the |
1349 | | * error would have already been detected |
1350 | | */ |
1351 | 0 | if ((conn && ((conn->flags & CO_FL_ERROR) || sc_ep_test(sc, SE_FL_ERROR))) || expired) { |
1352 | 0 | TRACE_ERROR("report connection error", CHK_EV_TASK_WAKE|CHK_EV_HCHK_END|CHK_EV_HCHK_ERR, check); |
1353 | 0 | chk_report_conn_err(check, 0, expired); |
1354 | 0 | } |
1355 | 0 | else { |
1356 | 0 | if (check->state & CHK_ST_CLOSE_CONN) { |
1357 | 0 | TRACE_DEVEL("closing current connection", CHK_EV_TASK_WAKE|CHK_EV_HCHK_RUN, check); |
1358 | 0 | check->state &= ~CHK_ST_CLOSE_CONN; |
1359 | 0 | if (!sc_reset_endp(check->sc)) { |
1360 | | /* error will be handled by tcpcheck_main(). |
1361 | | * On success, remove all flags except SE_FL_DETACHED |
1362 | | */ |
1363 | 0 | sc_ep_clr(check->sc, ~SE_FL_DETACHED); |
1364 | 0 | } |
1365 | 0 | tcpcheck_main(check); |
1366 | 0 | } |
1367 | 0 | if (check->result == CHK_RES_UNKNOWN) { |
1368 | 0 | TRACE_DEVEL("health-check not expired", CHK_EV_TASK_WAKE|CHK_EV_HCHK_RUN, check); |
1369 | 0 | goto out_unlock; /* timeout not reached, wait again */ |
1370 | 0 | } |
1371 | 0 | } |
1372 | 0 | } |
1373 | | |
1374 | | /* check complete or aborted */ |
1375 | 0 | TRACE_STATE("health-check complete or aborted", CHK_EV_TASK_WAKE|CHK_EV_HCHK_END, check); |
1376 | | |
1377 | | /* check->sc may be NULL when the healthcheck is purged */ |
1378 | 0 | check->current_step = NULL; |
1379 | 0 | sc = check->sc; |
1380 | 0 | conn = (sc ? sc_conn(sc) : NULL); |
1381 | |
|
1382 | 0 | if (conn && conn->xprt) { |
1383 | | /* The check was aborted and the connection was not yet closed. |
1384 | | * This can happen upon timeout, or when an external event such |
1385 | | * as a failed response coupled with "observe layer7" caused the |
1386 | | * server state to be suddenly changed. |
1387 | | */ |
1388 | 0 | se_shutdown(sc->sedesc, SE_SHR_DRAIN|SE_SHW_SILENT); |
1389 | 0 | } |
1390 | |
|
1391 | 0 | if (sc) { |
1392 | 0 | sc_destroy(sc); |
1393 | 0 | check->sc = NULL; |
1394 | 0 | } |
1395 | |
|
1396 | 0 | if (check->sess != NULL) { |
1397 | 0 | vars_prune(&check->vars, check->sess, NULL); |
1398 | 0 | session_free(check->sess); |
1399 | 0 | check->sess = NULL; |
1400 | 0 | } |
1401 | |
|
1402 | 0 | end: |
1403 | 0 | if (check->server && likely(!(check->state & CHK_ST_PURGE))) { |
1404 | 0 | if (check->result == CHK_RES_FAILED) { |
1405 | | /* a failure or timeout detected */ |
1406 | 0 | TRACE_DEVEL("report failure", CHK_EV_TASK_WAKE|CHK_EV_HCHK_END|CHK_EV_HCHK_ERR, check); |
1407 | 0 | check_notify_failure(check); |
1408 | 0 | } |
1409 | 0 | else if (check->result == CHK_RES_CONDPASS) { |
1410 | | /* check is OK but asks for stopping mode */ |
1411 | 0 | TRACE_DEVEL("report conditional success", CHK_EV_TASK_WAKE|CHK_EV_HCHK_END|CHK_EV_HCHK_SUCC, check); |
1412 | 0 | check_notify_stopping(check); |
1413 | 0 | } |
1414 | 0 | else if (check->result == CHK_RES_PASSED) { |
1415 | | /* a success was detected */ |
1416 | 0 | TRACE_DEVEL("report success", CHK_EV_TASK_WAKE|CHK_EV_HCHK_END|CHK_EV_HCHK_SUCC, check); |
1417 | 0 | check_notify_success(check); |
1418 | 0 | } |
1419 | 0 | } |
1420 | |
|
1421 | 0 | b_dequeue(&check->buf_wait); |
1422 | |
|
1423 | 0 | check_release_buf(check, &check->bi); |
1424 | 0 | check_release_buf(check, &check->bo); |
1425 | 0 | _HA_ATOMIC_DEC(&th_ctx->running_checks); |
1426 | 0 | _HA_ATOMIC_DEC(&th_ctx->active_checks); |
1427 | 0 | check->state &= ~(CHK_ST_INPROGRESS|CHK_ST_IN_ALLOC|CHK_ST_OUT_ALLOC); |
1428 | 0 | check->state &= ~CHK_ST_READY; |
1429 | 0 | check->state |= CHK_ST_SLEEPING; |
1430 | |
|
1431 | 0 | update_timer: |
1432 | | /* when going to sleep, we need to check if other checks are waiting |
1433 | | * for a slot. If so we pick them out of the queue and wake them up. |
1434 | | */ |
1435 | 0 | if (check->server && (check->state & CHK_ST_SLEEPING)) { |
1436 | 0 | if (!LIST_ISEMPTY(&th_ctx->queued_checks) && |
1437 | 0 | _HA_ATOMIC_LOAD(&th_ctx->running_checks) < global.tune.max_checks_per_thread) { |
1438 | 0 | struct check *next_chk = LIST_ELEM(th_ctx->queued_checks.n, struct check *, check_queue); |
1439 | | |
1440 | | /* wake up pending task */ |
1441 | 0 | LIST_DEL_INIT(&next_chk->check_queue); |
1442 | |
|
1443 | 0 | activity[tid].check_started++; |
1444 | 0 | _HA_ATOMIC_INC(&th_ctx->running_checks); |
1445 | 0 | next_chk->state |= CHK_ST_READY; |
1446 | | /* now running */ |
1447 | 0 | task_wakeup(next_chk->task, TASK_WOKEN_RES); |
1448 | 0 | } |
1449 | 0 | } |
1450 | |
|
1451 | 0 | if (check->server) { |
1452 | 0 | rv = 0; |
1453 | 0 | if (global.spread_checks > 0) { |
1454 | 0 | rv = srv_getinter(check) * global.spread_checks / 100; |
1455 | 0 | rv -= (int) (2 * rv * (statistical_prng() / 4294967295.0)); |
1456 | 0 | } |
1457 | 0 | t->expire = tick_add(now_ms, MS_TO_TICKS(srv_getinter(check) + rv)); |
1458 | | /* reset fastinter flag (if set) so that srv_getinter() |
1459 | | * only returns fastinter if server health is degraded |
1460 | | */ |
1461 | 0 | check->state &= ~CHK_ST_FASTINTER; |
1462 | 0 | } |
1463 | |
|
1464 | 0 | reschedule: |
1465 | 0 | if (proxy->flags & (PR_FL_DISABLED|PR_FL_STOPPED)) |
1466 | 0 | t->expire = TICK_ETERNITY; |
1467 | 0 | else { |
1468 | 0 | while (tick_is_expired(t->expire, now_ms)) |
1469 | 0 | t->expire = tick_add(t->expire, MS_TO_TICKS(check->inter)); |
1470 | 0 | } |
1471 | |
|
1472 | 0 | out_unlock: |
1473 | 0 | if (check->server) |
1474 | 0 | HA_SPIN_UNLOCK(SERVER_LOCK, &check->server->lock); |
1475 | |
|
1476 | 0 | out_leave: |
1477 | 0 | TRACE_LEAVE(CHK_EV_TASK_WAKE, check); |
1478 | | |
1479 | | /* Free the check if set to PURGE. After this, the check instance may be |
1480 | | * freed via the srv_drop invocation, so it must not be accessed after |
1481 | | * this point. |
1482 | | */ |
1483 | 0 | if (unlikely(check->state & CHK_ST_PURGE)) { |
1484 | 0 | free_check(check); |
1485 | 0 | if (check->server) |
1486 | 0 | srv_drop(check->server); |
1487 | |
|
1488 | 0 | t = NULL; |
1489 | 0 | } |
1490 | |
|
1491 | 0 | return t; |
1492 | 0 | } |
1493 | | |
1494 | | |
1495 | | /**************************************************************************/ |
1496 | | /************************** Init/deinit checks ****************************/ |
1497 | | /**************************************************************************/ |
1498 | | /* |
1499 | | * Tries to grab a buffer and to re-enables processing on check <target>. The |
1500 | | * check flags are used to figure what buffer was requested. It returns 1 if the |
1501 | | * allocation succeeds, in which case the I/O tasklet is woken up, or 0 if it's |
1502 | | * impossible to wake up and we prefer to be woken up later. |
1503 | | */ |
1504 | | int check_buf_available(void *target) |
1505 | 0 | { |
1506 | 0 | struct check *check = target; |
1507 | |
|
1508 | 0 | BUG_ON(!check->sc); |
1509 | |
|
1510 | 0 | if ((check->state & CHK_ST_IN_ALLOC) && b_alloc(&check->bi, DB_CHANNEL)) { |
1511 | 0 | TRACE_STATE("unblocking check, input buffer allocated", CHK_EV_TCPCHK_EXP|CHK_EV_RX_BLK, check); |
1512 | 0 | check->state &= ~CHK_ST_IN_ALLOC; |
1513 | 0 | tasklet_wakeup(check->sc->wait_event.tasklet); |
1514 | 0 | return 1; |
1515 | 0 | } |
1516 | 0 | if ((check->state & CHK_ST_OUT_ALLOC) && b_alloc(&check->bo, DB_CHANNEL)) { |
1517 | 0 | TRACE_STATE("unblocking check, output buffer allocated", CHK_EV_TCPCHK_SND|CHK_EV_TX_BLK, check); |
1518 | 0 | check->state &= ~CHK_ST_OUT_ALLOC; |
1519 | 0 | tasklet_wakeup(check->sc->wait_event.tasklet); |
1520 | 0 | return 1; |
1521 | 0 | } |
1522 | | |
1523 | 0 | return 0; |
1524 | 0 | } |
1525 | | |
1526 | | /* |
1527 | | * Allocate a buffer. If it fails, it adds the check in buffer wait queue. |
1528 | | */ |
1529 | | struct buffer *check_get_buf(struct check *check, struct buffer *bptr) |
1530 | 0 | { |
1531 | 0 | struct buffer *buf = NULL; |
1532 | |
|
1533 | 0 | if (likely(!LIST_INLIST(&check->buf_wait.list)) && |
1534 | 0 | unlikely((buf = b_alloc(bptr, DB_CHANNEL)) == NULL)) { |
1535 | 0 | b_queue(DB_CHANNEL, &check->buf_wait, check, check_buf_available); |
1536 | 0 | } |
1537 | 0 | return buf; |
1538 | 0 | } |
1539 | | |
1540 | | /* |
1541 | | * Release a buffer, if any, and try to wake up entities waiting in the buffer |
1542 | | * wait queue. |
1543 | | */ |
1544 | | void check_release_buf(struct check *check, struct buffer *bptr) |
1545 | 0 | { |
1546 | 0 | if (bptr->size) { |
1547 | 0 | b_free(bptr); |
1548 | 0 | offer_buffers(check->buf_wait.target, 1); |
1549 | 0 | } |
1550 | 0 | } |
1551 | | |
1552 | | const char *init_check(struct check *check, int type) |
1553 | 0 | { |
1554 | 0 | check->type = type; |
1555 | |
|
1556 | 0 | check->bi = BUF_NULL; |
1557 | 0 | check->bo = BUF_NULL; |
1558 | 0 | LIST_INIT(&check->buf_wait.list); |
1559 | 0 | LIST_INIT(&check->check_queue); |
1560 | 0 | return NULL; |
1561 | 0 | } |
1562 | | |
1563 | | /* Liberates the resources allocated for a check. |
1564 | | * |
1565 | | * This function must only be run by the thread owning the check. |
1566 | | */ |
1567 | | void free_check(struct check *check) |
1568 | 0 | { |
1569 | | /* For agent-check, free the rules / vars from the server. This is not |
1570 | | * done for health-check : the proxy is the owner of the rules / vars |
1571 | | * in this case. |
1572 | | */ |
1573 | 0 | if (check->state & CHK_ST_AGENT) { |
1574 | 0 | free_tcpcheck_vars(&check->tcpcheck_rules->preset_vars); |
1575 | 0 | ha_free(&check->tcpcheck_rules); |
1576 | 0 | } |
1577 | |
|
1578 | 0 | ha_free(&check->pool_conn_name); |
1579 | 0 | ha_free(&check->sni); |
1580 | 0 | ha_free(&check->alpn_str); |
1581 | 0 | task_destroy(check->task); |
1582 | |
|
1583 | 0 | check_release_buf(check, &check->bi); |
1584 | 0 | check_release_buf(check, &check->bo); |
1585 | 0 | if (check->sc) { |
1586 | 0 | sc_destroy(check->sc); |
1587 | 0 | check->sc = NULL; |
1588 | 0 | } |
1589 | 0 | } |
1590 | | |
1591 | | /* This function must be used in order to free a started check. The check will |
1592 | | * be scheduled for a next execution in order to properly close and free all |
1593 | | * check elements. |
1594 | | * |
1595 | | * Non thread-safe. |
1596 | | */ |
1597 | | void check_purge(struct check *check) |
1598 | 0 | { |
1599 | 0 | check->state |= CHK_ST_PURGE; |
1600 | 0 | task_wakeup(check->task, TASK_WOKEN_OTHER); |
1601 | 0 | } |
1602 | | |
1603 | | /* manages a server health-check. Returns the time the task accepts to wait, or |
1604 | | * TIME_ETERNITY for infinity. |
1605 | | */ |
1606 | | struct task *process_chk(struct task *t, void *context, unsigned int state) |
1607 | 0 | { |
1608 | 0 | struct check *check = context; |
1609 | |
|
1610 | 0 | if (check->type == PR_O2_EXT_CHK) |
1611 | 0 | return process_chk_proc(t, context, state); |
1612 | 0 | return process_chk_conn(t, context, state); |
1613 | |
|
1614 | 0 | } |
1615 | | |
1616 | | |
1617 | | int start_check_task(struct check *check, int mininter, |
1618 | | int nbcheck, int srvpos) |
1619 | 0 | { |
1620 | 0 | struct task *t; |
1621 | | |
1622 | | /* task for the check. Process-based checks exclusively run on thread 1. */ |
1623 | 0 | if (check->type == PR_O2_EXT_CHK) |
1624 | 0 | t = task_new_on(0); |
1625 | 0 | else |
1626 | 0 | t = task_new_anywhere(); |
1627 | |
|
1628 | 0 | if (!t) |
1629 | 0 | goto fail_alloc_task; |
1630 | | |
1631 | 0 | check->task = t; |
1632 | 0 | t->process = process_chk; |
1633 | 0 | t->context = check; |
1634 | |
|
1635 | 0 | if (mininter < srv_getinter(check)) |
1636 | 0 | mininter = srv_getinter(check); |
1637 | |
|
1638 | 0 | if (global.spread_checks > 0) { |
1639 | 0 | int rnd; |
1640 | |
|
1641 | 0 | rnd = srv_getinter(check) * global.spread_checks / 100; |
1642 | 0 | rnd -= (int) (2 * rnd * (ha_random32() / 4294967295.0)); |
1643 | 0 | mininter += rnd; |
1644 | 0 | } |
1645 | |
|
1646 | 0 | if (global.max_spread_checks && mininter > global.max_spread_checks) |
1647 | 0 | mininter = global.max_spread_checks; |
1648 | | |
1649 | | /* check this every ms */ |
1650 | 0 | t->expire = tick_add(now_ms, MS_TO_TICKS(mininter * srvpos / nbcheck)); |
1651 | 0 | check->start = now_ns; |
1652 | 0 | task_queue(t); |
1653 | |
|
1654 | 0 | return 1; |
1655 | | |
1656 | 0 | fail_alloc_task: |
1657 | 0 | ha_alert("Starting [%s:%s] check: out of memory.\n", |
1658 | 0 | check->server->proxy->id, check->server->id); |
1659 | 0 | return 0; |
1660 | 0 | } |
1661 | | |
1662 | | /* |
1663 | | * Start health-check. |
1664 | | * Returns 0 if OK, ERR_FATAL on error, and prints the error in this case. |
1665 | | */ |
1666 | | static int start_checks() |
1667 | 0 | { |
1668 | |
|
1669 | 0 | struct proxy *px; |
1670 | 0 | struct server *s; |
1671 | 0 | char *errmsg = NULL; |
1672 | 0 | int nbcheck=0, mininter=0, srvpos=0; |
1673 | | |
1674 | | /* 0- init the dummy frontend used to create all checks sessions */ |
1675 | 0 | if (!setup_new_proxy(&checks_fe, "CHECKS-FE", PR_CAP_FE | PR_CAP_BE | PR_CAP_INT, &errmsg)) { |
1676 | 0 | ha_alert("error during checks frontend creation: %s\n", errmsg); |
1677 | 0 | ha_free(&errmsg); |
1678 | 0 | return ERR_ALERT | ERR_FATAL; |
1679 | 0 | } |
1680 | 0 | checks_fe.mode = PR_MODE_TCP; |
1681 | 0 | checks_fe.maxconn = 0; |
1682 | 0 | checks_fe.conn_retries = CONN_RETRIES; |
1683 | 0 | checks_fe.options2 |= PR_O2_INDEPSTR | PR_O2_SMARTCON | PR_O2_SMARTACC; |
1684 | 0 | checks_fe.timeout.client = TICK_ETERNITY; |
1685 | | |
1686 | | /* 1- count the checkers to run simultaneously. |
1687 | | * We also determine the minimum interval among all of those which |
1688 | | * have an interval larger than SRV_CHK_INTER_THRES. This interval |
1689 | | * will be used to spread their start-up date. Those which have |
1690 | | * a shorter interval will start independently and will not dictate |
1691 | | * too short an interval for all others. |
1692 | | */ |
1693 | 0 | for (px = proxies_list; px; px = px->next) { |
1694 | 0 | for (s = px->srv; s; s = s->next) { |
1695 | 0 | if (s->check.state & CHK_ST_CONFIGURED) { |
1696 | 0 | nbcheck++; |
1697 | 0 | if ((srv_getinter(&s->check) >= SRV_CHK_INTER_THRES) && |
1698 | 0 | (!mininter || mininter > srv_getinter(&s->check))) |
1699 | 0 | mininter = srv_getinter(&s->check); |
1700 | 0 | } |
1701 | |
|
1702 | 0 | if (s->agent.state & CHK_ST_CONFIGURED) { |
1703 | 0 | nbcheck++; |
1704 | 0 | if ((srv_getinter(&s->agent) >= SRV_CHK_INTER_THRES) && |
1705 | 0 | (!mininter || mininter > srv_getinter(&s->agent))) |
1706 | 0 | mininter = srv_getinter(&s->agent); |
1707 | 0 | } |
1708 | 0 | } |
1709 | 0 | } |
1710 | |
|
1711 | 0 | if (!nbcheck) |
1712 | 0 | return ERR_NONE; |
1713 | | |
1714 | 0 | srand((unsigned)time(NULL)); |
1715 | | |
1716 | | /* 2- start them as far as possible from each other. For this, we will |
1717 | | * start them after their interval is set to the min interval divided |
1718 | | * by the number of servers, weighted by the server's position in the |
1719 | | * list. |
1720 | | */ |
1721 | 0 | for (px = proxies_list; px; px = px->next) { |
1722 | 0 | if ((px->options2 & PR_O2_CHK_ANY) == PR_O2_EXT_CHK) { |
1723 | 0 | if (init_pid_list()) { |
1724 | 0 | ha_alert("Starting [%s] check: out of memory.\n", px->id); |
1725 | 0 | return ERR_ALERT | ERR_FATAL; |
1726 | 0 | } |
1727 | 0 | } |
1728 | | |
1729 | 0 | for (s = px->srv; s; s = s->next) { |
1730 | | /* A task for the main check */ |
1731 | 0 | if (s->check.state & CHK_ST_CONFIGURED) { |
1732 | 0 | if (s->check.type == PR_O2_EXT_CHK) { |
1733 | 0 | if (!prepare_external_check(&s->check)) |
1734 | 0 | return ERR_ALERT | ERR_FATAL; |
1735 | 0 | } |
1736 | 0 | if (!start_check_task(&s->check, mininter, nbcheck, srvpos)) |
1737 | 0 | return ERR_ALERT | ERR_FATAL; |
1738 | 0 | srvpos++; |
1739 | 0 | } |
1740 | | |
1741 | | /* A task for a auxiliary agent check */ |
1742 | 0 | if (s->agent.state & CHK_ST_CONFIGURED) { |
1743 | 0 | if (!start_check_task(&s->agent, mininter, nbcheck, srvpos)) { |
1744 | 0 | return ERR_ALERT | ERR_FATAL; |
1745 | 0 | } |
1746 | 0 | srvpos++; |
1747 | 0 | } |
1748 | 0 | } |
1749 | 0 | } |
1750 | 0 | return ERR_NONE; |
1751 | 0 | } |
1752 | | |
1753 | | /* called during deinit */ |
1754 | | static void clear_checks() |
1755 | 0 | { |
1756 | 0 | if (checks_fe.id) |
1757 | 0 | deinit_proxy(&checks_fe); |
1758 | 0 | } |
1759 | | |
1760 | | /* |
1761 | | * Return value: |
1762 | | * the port to be used for the health check |
1763 | | * 0 in case no port could be found for the check |
1764 | | */ |
1765 | | static int srv_check_healthcheck_port(struct check *chk) |
1766 | 0 | { |
1767 | 0 | int i = 0; |
1768 | 0 | struct server *srv = NULL; |
1769 | |
|
1770 | 0 | srv = chk->server; |
1771 | | |
1772 | | /* by default, we use the health check port configured */ |
1773 | 0 | if (chk->port > 0) |
1774 | 0 | return chk->port; |
1775 | | |
1776 | | /* try to get the port from check_core.addr if check.port not set */ |
1777 | 0 | i = get_host_port(&chk->addr); |
1778 | 0 | if (i > 0) |
1779 | 0 | return i; |
1780 | | |
1781 | | /* try to get the port from server address */ |
1782 | | /* prevent MAPPORTS from working at this point, since checks could |
1783 | | * not be performed in such case (MAPPORTS impose a relative ports |
1784 | | * based on live traffic) |
1785 | | */ |
1786 | 0 | if (srv->flags & SRV_F_MAPPORTS) |
1787 | 0 | return 0; |
1788 | | |
1789 | 0 | i = srv->svc_port; /* by default */ |
1790 | 0 | if (i > 0) |
1791 | 0 | return i; |
1792 | | |
1793 | 0 | return 0; |
1794 | 0 | } |
1795 | | |
1796 | | /* Initializes an health-check attached to the server <srv>. Non-zero is returned |
1797 | | * if an error occurred. |
1798 | | */ |
1799 | | int init_srv_check(struct server *srv) |
1800 | 0 | { |
1801 | 0 | const char *err; |
1802 | 0 | struct tcpcheck_rule *r; |
1803 | 0 | int ret = ERR_NONE; |
1804 | 0 | int check_type; |
1805 | |
|
1806 | 0 | if (!srv->do_check || !(srv->proxy->cap & PR_CAP_BE)) |
1807 | 0 | goto out; |
1808 | | |
1809 | 0 | check_type = srv->check.tcpcheck_rules->flags & TCPCHK_RULES_PROTO_CHK; |
1810 | |
|
1811 | 0 | if (!(srv->flags & SRV_F_DYNAMIC)) { |
1812 | | /* If neither a port nor an addr was specified and no check |
1813 | | * transport layer is forced, then the transport layer used by |
1814 | | * the checks is the same as for the production traffic. |
1815 | | * Otherwise we use raw_sock by default, unless one is |
1816 | | * specified. |
1817 | | */ |
1818 | 0 | if (!srv->check.port && !is_addr(&srv->check.addr)) { |
1819 | 0 | if (!srv->check.use_ssl && srv->use_ssl != -1) |
1820 | 0 | srv->check.xprt = srv->xprt; |
1821 | 0 | else if (srv->check.use_ssl == 1) |
1822 | 0 | srv->check.xprt = xprt_get(XPRT_SSL); |
1823 | 0 | srv->check.send_proxy |= (srv->pp_opts); |
1824 | 0 | } |
1825 | 0 | else if (srv->check.use_ssl == 1) |
1826 | 0 | srv->check.xprt = xprt_get(XPRT_SSL); |
1827 | 0 | } |
1828 | 0 | else { |
1829 | | /* For dynamic servers, check-ssl and check-send-proxy must be |
1830 | | * explicitly defined even if the check port was not |
1831 | | * overridden. |
1832 | | */ |
1833 | 0 | if (srv->check.use_ssl == 1) |
1834 | 0 | srv->check.xprt = xprt_get(XPRT_SSL); |
1835 | 0 | } |
1836 | | |
1837 | | /* Inherit the mux protocol from the server if not already defined for |
1838 | | * the check |
1839 | | */ |
1840 | 0 | if (srv->mux_proto && !srv->check.mux_proto && |
1841 | 0 | ((srv->mux_proto->mode == PROTO_MODE_HTTP && check_type == TCPCHK_RULES_HTTP_CHK) || |
1842 | 0 | (srv->mux_proto->mode == PROTO_MODE_SPOP && check_type == TCPCHK_RULES_SPOP_CHK) || |
1843 | 0 | (srv->mux_proto->mode == PROTO_MODE_TCP && check_type != TCPCHK_RULES_HTTP_CHK))) { |
1844 | 0 | srv->check.mux_proto = srv->mux_proto; |
1845 | 0 | } |
1846 | | /* test that check proto is valid if explicitly defined */ |
1847 | 0 | else if (srv->check.mux_proto && |
1848 | 0 | ((srv->check.mux_proto->mode == PROTO_MODE_HTTP && check_type != TCPCHK_RULES_HTTP_CHK) || |
1849 | 0 | (srv->check.mux_proto->mode == PROTO_MODE_SPOP && check_type != TCPCHK_RULES_SPOP_CHK) || |
1850 | 0 | (srv->check.mux_proto->mode == PROTO_MODE_TCP && check_type == TCPCHK_RULES_HTTP_CHK))) { |
1851 | 0 | ha_alert("config: %s '%s': server '%s' uses an incompatible MUX protocol for the selected check type\n", |
1852 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id); |
1853 | 0 | ret |= ERR_ALERT | ERR_FATAL; |
1854 | 0 | goto out; |
1855 | 0 | } |
1856 | | |
1857 | | /* validate <srv> server health-check settings */ |
1858 | | |
1859 | 0 | if (srv_is_quic(srv)) { |
1860 | 0 | if (srv->check.mux_proto && srv->check.mux_proto != get_mux_proto(ist("quic"))) { |
1861 | 0 | ha_alert("config: %s '%s': QUIC server '%s' uses an incompatible MUX protocol for checks.\n", |
1862 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id); |
1863 | 0 | ret |= ERR_ALERT | ERR_FATAL; |
1864 | 0 | goto out; |
1865 | 0 | } |
1866 | | |
1867 | 0 | if (srv->check.use_ssl < 0) { |
1868 | 0 | ha_alert("config: %s '%s': SSL is mandatory for checks on QUIC server '%s'.\n", |
1869 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id); |
1870 | 0 | ret |= ERR_ALERT | ERR_FATAL; |
1871 | 0 | } |
1872 | |
|
1873 | 0 | if (srv->check.send_proxy) { |
1874 | 0 | ha_alert("config: %s '%s': cannot use PROXY protocol for checks on QUIC server '%s'.\n", |
1875 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id); |
1876 | 0 | ret |= ERR_ALERT | ERR_FATAL; |
1877 | 0 | } |
1878 | 0 | } |
1879 | 0 | else { |
1880 | 0 | if (srv->check.mux_proto && srv->check.mux_proto == get_mux_proto(ist("quic"))) { |
1881 | 0 | ha_alert("config: %s '%s': QUIC checks on non-QUIC server '%s' is not yet supported.\n", |
1882 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id); |
1883 | 0 | ret |= ERR_ALERT | ERR_FATAL; |
1884 | 0 | goto out; |
1885 | 0 | } |
1886 | 0 | } |
1887 | | |
1888 | | /* We need at least a service port, a check port or the first tcp-check |
1889 | | * rule must be a 'connect' one when checking an IPv4/IPv6 server. |
1890 | | */ |
1891 | 0 | if ((srv_check_healthcheck_port(&srv->check) != 0) || |
1892 | 0 | (!is_inet_addr(&srv->check.addr) && (is_addr(&srv->check.addr) || !is_inet_addr(&srv->addr)))) |
1893 | 0 | goto init; |
1894 | | |
1895 | 0 | if (!srv->proxy->tcpcheck_rules.list || LIST_ISEMPTY(srv->proxy->tcpcheck_rules.list)) { |
1896 | 0 | ha_alert("config: %s '%s': server '%s' has neither service port nor check port.\n", |
1897 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id); |
1898 | 0 | ret |= ERR_ALERT | ERR_ABORT; |
1899 | 0 | goto out; |
1900 | 0 | } |
1901 | | |
1902 | | /* search the first action (connect / send / expect) in the list */ |
1903 | 0 | r = get_first_tcpcheck_rule(&srv->proxy->tcpcheck_rules); |
1904 | 0 | if (!r || (r->action != TCPCHK_ACT_CONNECT) || (!r->connect.port && !get_host_port(&r->connect.addr))) { |
1905 | 0 | ha_alert("config: %s '%s': server '%s' has neither service port nor check port " |
1906 | 0 | "nor tcp_check rule 'connect' with port information.\n", |
1907 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id); |
1908 | 0 | ret |= ERR_ALERT | ERR_ABORT; |
1909 | 0 | goto out; |
1910 | 0 | } |
1911 | | |
1912 | | /* scan the tcp-check ruleset to ensure a port has been configured */ |
1913 | 0 | list_for_each_entry(r, srv->proxy->tcpcheck_rules.list, list) { |
1914 | 0 | if ((r->action == TCPCHK_ACT_CONNECT) && (!r->connect.port && !get_host_port(&r->connect.addr))) { |
1915 | 0 | ha_alert("config: %s '%s': server '%s' has neither service port nor check port, " |
1916 | 0 | "and a tcp_check rule 'connect' with no port information.\n", |
1917 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id); |
1918 | 0 | ret |= ERR_ALERT | ERR_ABORT; |
1919 | 0 | goto out; |
1920 | 0 | } |
1921 | 0 | } |
1922 | | |
1923 | 0 | init: |
1924 | 0 | err = init_check(&srv->check, srv->proxy->options2 & PR_O2_CHK_ANY); |
1925 | 0 | if (err) { |
1926 | 0 | ha_alert("config: %s '%s': unable to init check for server '%s' (%s).\n", |
1927 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id, err); |
1928 | 0 | ret |= ERR_ALERT | ERR_ABORT; |
1929 | 0 | goto out; |
1930 | 0 | } |
1931 | 0 | srv->check.state |= CHK_ST_CONFIGURED | CHK_ST_ENABLED | CHK_ST_SLEEPING; |
1932 | 0 | srv_take(srv); |
1933 | | |
1934 | | /* Only increment maxsock for servers from the configuration. Dynamic |
1935 | | * servers at the moment are not taken into account for the estimation |
1936 | | * of the resources limits. |
1937 | | */ |
1938 | 0 | if (global.mode & MODE_STARTING) |
1939 | 0 | global.maxsock++; |
1940 | |
|
1941 | 0 | out: |
1942 | 0 | return ret; |
1943 | 0 | } |
1944 | | |
1945 | | /* Initializes an agent-check attached to the server <srv>. Non-zero is returned |
1946 | | * if an error occurred. |
1947 | | */ |
1948 | | int init_srv_agent_check(struct server *srv) |
1949 | 0 | { |
1950 | 0 | struct tcpcheck_rule *chk; |
1951 | 0 | const char *err; |
1952 | 0 | int ret = ERR_NONE; |
1953 | |
|
1954 | 0 | if (!srv->do_agent || !(srv->proxy->cap & PR_CAP_BE)) |
1955 | 0 | goto out; |
1956 | | |
1957 | | /* If there is no connect rule preceding all send / expect rules, an |
1958 | | * implicit one is inserted before all others. |
1959 | | */ |
1960 | 0 | chk = get_first_tcpcheck_rule(srv->agent.tcpcheck_rules); |
1961 | 0 | if (!chk || chk->action != TCPCHK_ACT_CONNECT) { |
1962 | 0 | chk = calloc(1, sizeof(*chk)); |
1963 | 0 | if (!chk) { |
1964 | 0 | ha_alert("%s '%s': unable to add implicit tcp-check connect rule" |
1965 | 0 | " to agent-check for server '%s' (out of memory).\n", |
1966 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id); |
1967 | 0 | ret |= ERR_ALERT | ERR_FATAL; |
1968 | 0 | goto out; |
1969 | 0 | } |
1970 | 0 | chk->action = TCPCHK_ACT_CONNECT; |
1971 | 0 | chk->connect.options = (TCPCHK_OPT_DEFAULT_CONNECT|TCPCHK_OPT_IMPLICIT); |
1972 | 0 | LIST_INSERT(srv->agent.tcpcheck_rules->list, &chk->list); |
1973 | 0 | } |
1974 | | |
1975 | | /* <chk> is always defined here and it is a CONNECT action. If there is |
1976 | | * a preset variable, it means there is an agent string defined and data |
1977 | | * will be sent after the connect. |
1978 | | */ |
1979 | 0 | if (!LIST_ISEMPTY(&srv->agent.tcpcheck_rules->preset_vars)) |
1980 | 0 | chk->connect.options |= TCPCHK_OPT_HAS_DATA; |
1981 | | |
1982 | |
|
1983 | 0 | err = init_check(&srv->agent, PR_O2_TCPCHK_CHK); |
1984 | 0 | if (err) { |
1985 | 0 | ha_alert("config: %s '%s': unable to init agent-check for server '%s' (%s).\n", |
1986 | 0 | proxy_type_str(srv->proxy), srv->proxy->id, srv->id, err); |
1987 | 0 | ret |= ERR_ALERT | ERR_ABORT; |
1988 | 0 | goto out; |
1989 | 0 | } |
1990 | | |
1991 | 0 | if (!srv->agent.inter) |
1992 | 0 | srv->agent.inter = srv->check.inter; |
1993 | |
|
1994 | 0 | srv->agent.state |= CHK_ST_CONFIGURED | CHK_ST_ENABLED | CHK_ST_SLEEPING | CHK_ST_AGENT; |
1995 | 0 | srv_take(srv); |
1996 | | |
1997 | | /* Only increment maxsock for servers from the configuration. Dynamic |
1998 | | * servers at the moment are not taken into account for the estimation |
1999 | | * of the resources limits. |
2000 | | */ |
2001 | 0 | if (global.mode & MODE_STARTING) |
2002 | 0 | global.maxsock++; |
2003 | |
|
2004 | 0 | out: |
2005 | 0 | return ret; |
2006 | 0 | } |
2007 | | |
2008 | | static void deinit_srv_check(struct server *srv) |
2009 | 0 | { |
2010 | 0 | if (srv->check.state & CHK_ST_CONFIGURED) { |
2011 | 0 | free_check(&srv->check); |
2012 | | /* it is safe to drop now since the main server reference is still held by the proxy */ |
2013 | 0 | srv_drop(srv); |
2014 | 0 | } |
2015 | 0 | srv->check.state &= ~CHK_ST_CONFIGURED & ~CHK_ST_ENABLED; |
2016 | 0 | srv->do_check = 0; |
2017 | 0 | } |
2018 | | |
2019 | | |
2020 | | static void deinit_srv_agent_check(struct server *srv) |
2021 | 0 | { |
2022 | 0 | if (srv->agent.state & CHK_ST_CONFIGURED) { |
2023 | 0 | free_check(&srv->agent); |
2024 | | /* it is safe to drop now since the main server reference is still held by the proxy */ |
2025 | 0 | srv_drop(srv); |
2026 | 0 | } |
2027 | |
|
2028 | 0 | srv->agent.state &= ~CHK_ST_CONFIGURED & ~CHK_ST_ENABLED & ~CHK_ST_AGENT; |
2029 | 0 | srv->do_agent = 0; |
2030 | 0 | } |
2031 | | |
2032 | | REGISTER_POST_SERVER_CHECK(init_srv_check); |
2033 | | REGISTER_POST_SERVER_CHECK(init_srv_agent_check); |
2034 | | REGISTER_POST_CHECK(start_checks); |
2035 | | |
2036 | | REGISTER_SERVER_DEINIT(deinit_srv_check); |
2037 | | REGISTER_SERVER_DEINIT(deinit_srv_agent_check); |
2038 | | REGISTER_POST_DEINIT(clear_checks); |
2039 | | |
2040 | | /* perform minimal initializations */ |
2041 | | static void init_checks() |
2042 | 0 | { |
2043 | 0 | int i; |
2044 | |
|
2045 | 0 | for (i = 0; i < MAX_THREADS; i++) |
2046 | 0 | LIST_INIT(&ha_thread_ctx[i].queued_checks); |
2047 | 0 | } |
2048 | | |
2049 | | INITCALL0(STG_PREPARE, init_checks); |
2050 | | |
2051 | | /**************************************************************************/ |
2052 | | /************************** Check sample fetches **************************/ |
2053 | | /**************************************************************************/ |
2054 | | |
2055 | | static struct sample_fetch_kw_list smp_kws = {ILH, { |
2056 | | { /* END */ }, |
2057 | | }}; |
2058 | | |
2059 | | INITCALL1(STG_REGISTER, sample_register_fetches, &smp_kws); |
2060 | | |
2061 | | |
2062 | | /**************************************************************************/ |
2063 | | /************************ Check's parsing functions ***********************/ |
2064 | | /**************************************************************************/ |
2065 | | /* Parse the "addr" server keyword */ |
2066 | | static int srv_parse_addr(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2067 | | char **errmsg) |
2068 | 0 | { |
2069 | 0 | struct sockaddr_storage *sk; |
2070 | 0 | int port1, port2, err_code = 0; |
2071 | | |
2072 | |
|
2073 | 0 | if (!*args[*cur_arg+1]) { |
2074 | 0 | memprintf(errmsg, "'%s' expects <ipv4|ipv6> as argument.", args[*cur_arg]); |
2075 | 0 | goto error; |
2076 | 0 | } |
2077 | | |
2078 | 0 | sk = str2sa_range(args[*cur_arg+1], NULL, &port1, &port2, NULL, NULL, NULL, errmsg, NULL, NULL, NULL, |
2079 | 0 | PA_O_RESOLVE | PA_O_PORT_OK | PA_O_STREAM | PA_O_CONNECT); |
2080 | 0 | if (!sk) { |
2081 | 0 | memprintf(errmsg, "'%s' : %s", args[*cur_arg], *errmsg); |
2082 | 0 | goto error; |
2083 | 0 | } |
2084 | | |
2085 | 0 | srv->check.addr = *sk; |
2086 | | /* if agentaddr was never set, we can use addr */ |
2087 | 0 | if (!(srv->flags & SRV_F_AGENTADDR)) |
2088 | 0 | srv->agent.addr = *sk; |
2089 | |
|
2090 | 0 | out: |
2091 | 0 | return err_code; |
2092 | | |
2093 | 0 | error: |
2094 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2095 | 0 | goto out; |
2096 | 0 | } |
2097 | | |
2098 | | /* Parse the "agent-addr" server keyword */ |
2099 | | static int srv_parse_agent_addr(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2100 | | char **errmsg) |
2101 | 0 | { |
2102 | 0 | struct sockaddr_storage sk; |
2103 | 0 | int err_code = 0; |
2104 | |
|
2105 | 0 | if (!*(args[*cur_arg+1])) { |
2106 | 0 | memprintf(errmsg, "'%s' expects an address as argument.", args[*cur_arg]); |
2107 | 0 | goto error; |
2108 | 0 | } |
2109 | 0 | memset(&sk, 0, sizeof(sk)); |
2110 | 0 | if (str2ip(args[*cur_arg + 1], &sk) == NULL) { |
2111 | 0 | memprintf(errmsg, "parsing agent-addr failed. Check if '%s' is correct address.", args[*cur_arg+1]); |
2112 | 0 | goto error; |
2113 | 0 | } |
2114 | 0 | set_srv_agent_addr(srv, &sk); |
2115 | |
|
2116 | 0 | out: |
2117 | 0 | return err_code; |
2118 | | |
2119 | 0 | error: |
2120 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2121 | 0 | goto out; |
2122 | 0 | } |
2123 | | |
2124 | | /* Parse the "agent-check" server keyword */ |
2125 | | static int srv_parse_agent_check(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2126 | | char **errmsg) |
2127 | 0 | { |
2128 | 0 | struct tcpcheck_ruleset *rs = NULL; |
2129 | 0 | struct tcpcheck_rules *rules = srv->agent.tcpcheck_rules; |
2130 | 0 | struct tcpcheck_rule *chk; |
2131 | 0 | int err_code = 0; |
2132 | |
|
2133 | 0 | if (srv->do_agent) |
2134 | 0 | goto out; |
2135 | | |
2136 | 0 | if (!(curpx->cap & PR_CAP_BE)) { |
2137 | 0 | memprintf(errmsg, "'%s' ignored because %s '%s' has no backend capability", |
2138 | 0 | args[*cur_arg], proxy_type_str(curpx), curpx->id); |
2139 | 0 | return ERR_WARN; |
2140 | 0 | } |
2141 | | |
2142 | 0 | if (!rules) { |
2143 | 0 | rules = calloc(1, sizeof(*rules)); |
2144 | 0 | if (!rules) { |
2145 | 0 | memprintf(errmsg, "out of memory."); |
2146 | 0 | goto error; |
2147 | 0 | } |
2148 | 0 | LIST_INIT(&rules->preset_vars); |
2149 | 0 | srv->agent.tcpcheck_rules = rules; |
2150 | 0 | } |
2151 | 0 | rules->list = NULL; |
2152 | 0 | rules->flags = 0; |
2153 | |
|
2154 | 0 | rs = find_tcpcheck_ruleset("*agent-check"); |
2155 | 0 | if (rs) |
2156 | 0 | goto ruleset_found; |
2157 | | |
2158 | 0 | rs = create_tcpcheck_ruleset("*agent-check"); |
2159 | 0 | if (rs == NULL) { |
2160 | 0 | memprintf(errmsg, "out of memory."); |
2161 | 0 | goto error; |
2162 | 0 | } |
2163 | | |
2164 | 0 | chk = parse_tcpcheck_send((char *[]){"tcp-check", "send-lf", "%[var(check.agent_string)]", ""}, |
2165 | 0 | 1, curpx, &rs->rules, srv->conf.file, srv->conf.line, errmsg); |
2166 | 0 | if (!chk) { |
2167 | 0 | memprintf(errmsg, "'%s': %s", args[*cur_arg], *errmsg); |
2168 | 0 | goto error; |
2169 | 0 | } |
2170 | 0 | chk->index = 0; |
2171 | 0 | LIST_APPEND(&rs->rules, &chk->list); |
2172 | |
|
2173 | 0 | chk = parse_tcpcheck_expect((char *[]){"tcp-check", "expect", "custom", ""}, |
2174 | 0 | 1, curpx, &rs->rules, TCPCHK_RULES_AGENT_CHK, |
2175 | 0 | srv->conf.file, srv->conf.line, errmsg); |
2176 | 0 | if (!chk) { |
2177 | 0 | memprintf(errmsg, "'%s': %s", args[*cur_arg], *errmsg); |
2178 | 0 | goto error; |
2179 | 0 | } |
2180 | 0 | chk->expect.custom = tcpcheck_agent_expect_reply; |
2181 | 0 | chk->index = 1; |
2182 | 0 | LIST_APPEND(&rs->rules, &chk->list); |
2183 | |
|
2184 | 0 | ruleset_found: |
2185 | 0 | rules->list = &rs->rules; |
2186 | 0 | rules->flags &= ~(TCPCHK_RULES_PROTO_CHK|TCPCHK_RULES_UNUSED_RS); |
2187 | 0 | rules->flags |= TCPCHK_RULES_AGENT_CHK; |
2188 | 0 | srv->do_agent = 1; |
2189 | |
|
2190 | 0 | out: |
2191 | 0 | return err_code; |
2192 | | |
2193 | 0 | error: |
2194 | 0 | deinit_srv_agent_check(srv); |
2195 | 0 | free_tcpcheck_ruleset(rs); |
2196 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2197 | 0 | goto out; |
2198 | 0 | } |
2199 | | |
2200 | | /* Parse the "agent-inter" server keyword */ |
2201 | | static int srv_parse_agent_inter(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2202 | | char **errmsg) |
2203 | 0 | { |
2204 | 0 | const char *err = NULL; |
2205 | 0 | unsigned int delay; |
2206 | 0 | int err_code = 0; |
2207 | |
|
2208 | 0 | if (!*(args[*cur_arg+1])) { |
2209 | 0 | memprintf(errmsg, "'%s' expects a delay as argument.", args[*cur_arg]); |
2210 | 0 | goto error; |
2211 | 0 | } |
2212 | | |
2213 | 0 | err = parse_time_err(args[*cur_arg+1], &delay, TIME_UNIT_MS); |
2214 | 0 | if (err == PARSE_TIME_OVER) { |
2215 | 0 | memprintf(errmsg, "timer overflow in argument <%s> to <%s> of server %s, maximum value is 2147483647 ms (~24.8 days).", |
2216 | 0 | args[*cur_arg+1], args[*cur_arg], srv->id); |
2217 | 0 | goto error; |
2218 | 0 | } |
2219 | 0 | else if (err == PARSE_TIME_UNDER) { |
2220 | 0 | memprintf(errmsg, "timer underflow in argument <%s> to <%s> of server %s, minimum non-null value is 1 ms.", |
2221 | 0 | args[*cur_arg+1], args[*cur_arg], srv->id); |
2222 | 0 | goto error; |
2223 | 0 | } |
2224 | 0 | else if (err) { |
2225 | 0 | memprintf(errmsg, "unexpected character '%c' in 'agent-inter' argument of server %s.", |
2226 | 0 | *err, srv->id); |
2227 | 0 | goto error; |
2228 | 0 | } |
2229 | 0 | if (delay <= 0) { |
2230 | 0 | memprintf(errmsg, "invalid value %d for argument '%s' of server %s.", |
2231 | 0 | delay, args[*cur_arg], srv->id); |
2232 | 0 | goto error; |
2233 | 0 | } |
2234 | 0 | srv->agent.inter = delay; |
2235 | |
|
2236 | 0 | if (warn_if_lower(args[*cur_arg+1], 100)) { |
2237 | 0 | memprintf(errmsg, "'%s %u' in server '%s' is suspiciously small for a value in milliseconds. Please use an explicit unit ('%ums') if that was the intent", |
2238 | 0 | args[*cur_arg], delay, srv->id, delay); |
2239 | 0 | err_code |= ERR_WARN; |
2240 | 0 | } |
2241 | |
|
2242 | 0 | out: |
2243 | 0 | return err_code; |
2244 | | |
2245 | 0 | error: |
2246 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2247 | 0 | goto out; |
2248 | 0 | } |
2249 | | |
2250 | | /* Parse the "agent-port" server keyword */ |
2251 | | static int srv_parse_agent_port(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2252 | | char **errmsg) |
2253 | 0 | { |
2254 | 0 | int err_code = 0; |
2255 | |
|
2256 | 0 | if (!*(args[*cur_arg+1])) { |
2257 | 0 | memprintf(errmsg, "'%s' expects a port number as argument.", args[*cur_arg]); |
2258 | 0 | goto error; |
2259 | 0 | } |
2260 | | |
2261 | | /* Only increment maxsock for servers from the configuration. Dynamic |
2262 | | * servers at the moment are not taken into account for the estimation |
2263 | | * of the resources limits. |
2264 | | */ |
2265 | 0 | if (global.mode & MODE_STARTING) |
2266 | 0 | global.maxsock++; |
2267 | |
|
2268 | 0 | set_srv_agent_port(srv, atol(args[*cur_arg + 1])); |
2269 | |
|
2270 | 0 | out: |
2271 | 0 | return err_code; |
2272 | | |
2273 | 0 | error: |
2274 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2275 | 0 | goto out; |
2276 | 0 | } |
2277 | | |
2278 | | int set_srv_agent_send(struct server *srv, const char *send) |
2279 | 0 | { |
2280 | 0 | struct tcpcheck_rules *rules = srv->agent.tcpcheck_rules; |
2281 | 0 | struct tcpcheck_var *var = NULL; |
2282 | 0 | char *str; |
2283 | |
|
2284 | 0 | str = strdup(send); |
2285 | 0 | var = create_tcpcheck_var(ist("check.agent_string")); |
2286 | 0 | if (str == NULL || var == NULL) |
2287 | 0 | goto error; |
2288 | | |
2289 | 0 | free_tcpcheck_vars(&rules->preset_vars); |
2290 | |
|
2291 | 0 | var->data.type = SMP_T_STR; |
2292 | 0 | var->data.u.str.area = str; |
2293 | 0 | var->data.u.str.data = strlen(str); |
2294 | 0 | LIST_INIT(&var->list); |
2295 | 0 | LIST_APPEND(&rules->preset_vars, &var->list); |
2296 | |
|
2297 | 0 | return 1; |
2298 | | |
2299 | 0 | error: |
2300 | 0 | free(str); |
2301 | 0 | free(var); |
2302 | 0 | return 0; |
2303 | 0 | } |
2304 | | |
2305 | | /* Parse the "agent-send" server keyword */ |
2306 | | static int srv_parse_agent_send(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2307 | | char **errmsg) |
2308 | 0 | { |
2309 | 0 | struct tcpcheck_rules *rules = srv->agent.tcpcheck_rules; |
2310 | 0 | int err_code = 0; |
2311 | |
|
2312 | 0 | if (!*(args[*cur_arg+1])) { |
2313 | 0 | memprintf(errmsg, "'%s' expects a string as argument.", args[*cur_arg]); |
2314 | 0 | goto error; |
2315 | 0 | } |
2316 | | |
2317 | 0 | if (!rules) { |
2318 | 0 | rules = calloc(1, sizeof(*rules)); |
2319 | 0 | if (!rules) { |
2320 | 0 | memprintf(errmsg, "out of memory."); |
2321 | 0 | goto error; |
2322 | 0 | } |
2323 | 0 | LIST_INIT(&rules->preset_vars); |
2324 | 0 | srv->agent.tcpcheck_rules = rules; |
2325 | 0 | } |
2326 | | |
2327 | 0 | if (!set_srv_agent_send(srv, args[*cur_arg+1])) { |
2328 | 0 | memprintf(errmsg, "out of memory."); |
2329 | 0 | goto error; |
2330 | 0 | } |
2331 | | |
2332 | 0 | out: |
2333 | 0 | return err_code; |
2334 | | |
2335 | 0 | error: |
2336 | 0 | deinit_srv_agent_check(srv); |
2337 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2338 | 0 | goto out; |
2339 | 0 | } |
2340 | | |
2341 | | /* Parse the "no-agent-send" server keyword */ |
2342 | | static int srv_parse_no_agent_check(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2343 | | char **errmsg) |
2344 | 0 | { |
2345 | 0 | deinit_srv_agent_check(srv); |
2346 | 0 | return 0; |
2347 | 0 | } |
2348 | | |
2349 | | /* Parse the "check" server keyword */ |
2350 | | static int srv_parse_check(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2351 | | char **errmsg) |
2352 | 0 | { |
2353 | 0 | if (!(curpx->cap & PR_CAP_BE)) { |
2354 | 0 | memprintf(errmsg, "'%s' ignored because %s '%s' has no backend capability", |
2355 | 0 | args[*cur_arg], proxy_type_str(curpx), curpx->id); |
2356 | 0 | return ERR_WARN; |
2357 | 0 | } |
2358 | | |
2359 | 0 | srv->do_check = 1; |
2360 | 0 | return 0; |
2361 | 0 | } |
2362 | | |
2363 | | /* Parse the "check-send-proxy" server keyword */ |
2364 | | static int srv_parse_check_send_proxy(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2365 | | char **errmsg) |
2366 | 0 | { |
2367 | 0 | srv->check.send_proxy = 1; |
2368 | 0 | return 0; |
2369 | 0 | } |
2370 | | |
2371 | | /* Parse the "check-via-socks4" server keyword */ |
2372 | | static int srv_parse_check_via_socks4(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2373 | | char **errmsg) |
2374 | 0 | { |
2375 | 0 | srv->check.via_socks4 = 1; |
2376 | 0 | return 0; |
2377 | 0 | } |
2378 | | |
2379 | | /* Parse the "no-check" server keyword */ |
2380 | | static int srv_parse_no_check(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2381 | | char **errmsg) |
2382 | 0 | { |
2383 | 0 | deinit_srv_check(srv); |
2384 | 0 | return 0; |
2385 | 0 | } |
2386 | | |
2387 | | /* Parse the "no-check-reuse-pool" server keyword */ |
2388 | | static int srv_parse_no_check_reuse_pool(char **args, int *cur_arg, |
2389 | | struct proxy *curpx, struct server *srv, |
2390 | | char **errmsg) |
2391 | 0 | { |
2392 | 0 | srv->check.reuse_pool = 0; |
2393 | 0 | return 0; |
2394 | 0 | } |
2395 | | |
2396 | | /* Parse the "no-check-send-proxy" server keyword */ |
2397 | | static int srv_parse_no_check_send_proxy(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2398 | | char **errmsg) |
2399 | 0 | { |
2400 | 0 | srv->check.send_proxy = 0; |
2401 | 0 | return 0; |
2402 | 0 | } |
2403 | | |
2404 | | /* parse the "check-pool-conn-name" server keyword */ |
2405 | | static int srv_parse_check_pool_conn_name(char **args, int *cur_arg, |
2406 | | struct proxy *px, |
2407 | | struct server *newsrv, char **err) |
2408 | 0 | { |
2409 | 0 | int err_code = 0; |
2410 | |
|
2411 | 0 | if (!*args[*cur_arg + 1]) { |
2412 | 0 | memprintf(err, "'%s' : missing value", args[*cur_arg]); |
2413 | 0 | goto error; |
2414 | 0 | } |
2415 | | |
2416 | 0 | ha_free(&newsrv->check.pool_conn_name); |
2417 | 0 | newsrv->check.pool_conn_name = strdup(args[*cur_arg + 1]); |
2418 | 0 | if (!newsrv->check.pool_conn_name) { |
2419 | 0 | memprintf(err, "'%s' : out of memory", args[*cur_arg]); |
2420 | 0 | return ERR_ALERT | ERR_FATAL; |
2421 | 0 | } |
2422 | | |
2423 | 0 | out: |
2424 | 0 | return err_code; |
2425 | | |
2426 | 0 | error: |
2427 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2428 | 0 | goto out; |
2429 | 0 | } |
2430 | | |
2431 | | |
2432 | | /* parse the "check-proto" server keyword */ |
2433 | | static int srv_parse_check_proto(char **args, int *cur_arg, |
2434 | | struct proxy *px, struct server *newsrv, char **err) |
2435 | 0 | { |
2436 | 0 | int err_code = 0; |
2437 | |
|
2438 | 0 | if (!*args[*cur_arg + 1]) { |
2439 | 0 | memprintf(err, "'%s' : missing value", args[*cur_arg]); |
2440 | 0 | goto error; |
2441 | 0 | } |
2442 | 0 | newsrv->check.mux_proto = get_mux_proto(ist(args[*cur_arg + 1])); |
2443 | 0 | if (!newsrv->check.mux_proto) { |
2444 | 0 | memprintf(err, "'%s' : unknown MUX protocol '%s'", args[*cur_arg], args[*cur_arg+1]); |
2445 | 0 | goto error; |
2446 | 0 | } |
2447 | | |
2448 | 0 | out: |
2449 | 0 | return err_code; |
2450 | | |
2451 | 0 | error: |
2452 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2453 | 0 | goto out; |
2454 | 0 | } |
2455 | | |
2456 | | /* Parse the "check-reuse-pool" server keyword */ |
2457 | | static int srv_parse_check_reuse_pool(char **args, int *cur_arg, |
2458 | | struct proxy *curpx, struct server *srv, |
2459 | | char **errmsg) |
2460 | 0 | { |
2461 | 0 | srv->check.reuse_pool = 1; |
2462 | 0 | return 0; |
2463 | 0 | } |
2464 | | |
2465 | | |
2466 | | /* Parse the "rise" server keyword */ |
2467 | | static int srv_parse_check_rise(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2468 | | char **errmsg) |
2469 | 0 | { |
2470 | 0 | int err_code = 0; |
2471 | |
|
2472 | 0 | if (!*args[*cur_arg + 1]) { |
2473 | 0 | memprintf(errmsg, "'%s' expects an integer argument.", args[*cur_arg]); |
2474 | 0 | goto error; |
2475 | 0 | } |
2476 | | |
2477 | 0 | srv->check.rise = atol(args[*cur_arg+1]); |
2478 | 0 | if (srv->check.rise <= 0) { |
2479 | 0 | memprintf(errmsg, "'%s' has to be > 0.", args[*cur_arg]); |
2480 | 0 | goto error; |
2481 | 0 | } |
2482 | | |
2483 | 0 | if (srv->check.health) |
2484 | 0 | srv->check.health = srv->check.rise; |
2485 | |
|
2486 | 0 | out: |
2487 | 0 | return err_code; |
2488 | | |
2489 | 0 | error: |
2490 | 0 | deinit_srv_agent_check(srv); |
2491 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2492 | 0 | goto out; |
2493 | 0 | } |
2494 | | |
2495 | | /* Parse the "fall" server keyword */ |
2496 | | static int srv_parse_check_fall(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2497 | | char **errmsg) |
2498 | 0 | { |
2499 | 0 | int err_code = 0; |
2500 | |
|
2501 | 0 | if (!*args[*cur_arg + 1]) { |
2502 | 0 | memprintf(errmsg, "'%s' expects an integer argument.", args[*cur_arg]); |
2503 | 0 | goto error; |
2504 | 0 | } |
2505 | | |
2506 | 0 | srv->check.fall = atol(args[*cur_arg+1]); |
2507 | 0 | if (srv->check.fall <= 0) { |
2508 | 0 | memprintf(errmsg, "'%s' has to be > 0.", args[*cur_arg]); |
2509 | 0 | goto error; |
2510 | 0 | } |
2511 | | |
2512 | 0 | out: |
2513 | 0 | return err_code; |
2514 | | |
2515 | 0 | error: |
2516 | 0 | deinit_srv_agent_check(srv); |
2517 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2518 | 0 | goto out; |
2519 | 0 | } |
2520 | | |
2521 | | /* Parse the "inter" server keyword */ |
2522 | | static int srv_parse_check_inter(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2523 | | char **errmsg) |
2524 | 0 | { |
2525 | 0 | const char *err = NULL; |
2526 | 0 | unsigned int delay; |
2527 | 0 | int err_code = 0; |
2528 | |
|
2529 | 0 | if (!*(args[*cur_arg+1])) { |
2530 | 0 | memprintf(errmsg, "'%s' expects a delay as argument.", args[*cur_arg]); |
2531 | 0 | goto error; |
2532 | 0 | } |
2533 | | |
2534 | 0 | err = parse_time_err(args[*cur_arg+1], &delay, TIME_UNIT_MS); |
2535 | 0 | if (err == PARSE_TIME_OVER) { |
2536 | 0 | memprintf(errmsg, "timer overflow in argument <%s> to <%s> of server %s, maximum value is 2147483647 ms (~24.8 days).", |
2537 | 0 | args[*cur_arg+1], args[*cur_arg], srv->id); |
2538 | 0 | goto error; |
2539 | 0 | } |
2540 | 0 | else if (err == PARSE_TIME_UNDER) { |
2541 | 0 | memprintf(errmsg, "timer underflow in argument <%s> to <%s> of server %s, minimum non-null value is 1 ms.", |
2542 | 0 | args[*cur_arg+1], args[*cur_arg], srv->id); |
2543 | 0 | goto error; |
2544 | 0 | } |
2545 | 0 | else if (err) { |
2546 | 0 | memprintf(errmsg, "unexpected character '%c' in 'agent-inter' argument of server %s.", |
2547 | 0 | *err, srv->id); |
2548 | 0 | goto error; |
2549 | 0 | } |
2550 | 0 | if (delay <= 0) { |
2551 | 0 | memprintf(errmsg, "invalid value %d for argument '%s' of server %s.", |
2552 | 0 | delay, args[*cur_arg], srv->id); |
2553 | 0 | goto error; |
2554 | 0 | } |
2555 | 0 | srv->check.inter = delay; |
2556 | |
|
2557 | 0 | if (warn_if_lower(args[*cur_arg+1], 100)) { |
2558 | 0 | memprintf(errmsg, "'%s %u' in server '%s' is suspiciously small for a value in milliseconds. Please use an explicit unit ('%ums') if that was the intent", |
2559 | 0 | args[*cur_arg], delay, srv->id, delay); |
2560 | 0 | err_code |= ERR_WARN; |
2561 | 0 | } |
2562 | |
|
2563 | 0 | out: |
2564 | 0 | return err_code; |
2565 | | |
2566 | 0 | error: |
2567 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2568 | 0 | goto out; |
2569 | 0 | } |
2570 | | |
2571 | | |
2572 | | /* Parse the "fastinter" server keyword */ |
2573 | | static int srv_parse_check_fastinter(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2574 | | char **errmsg) |
2575 | 0 | { |
2576 | 0 | const char *err = NULL; |
2577 | 0 | unsigned int delay; |
2578 | 0 | int err_code = 0; |
2579 | |
|
2580 | 0 | if (!*(args[*cur_arg+1])) { |
2581 | 0 | memprintf(errmsg, "'%s' expects a delay as argument.", args[*cur_arg]); |
2582 | 0 | goto error; |
2583 | 0 | } |
2584 | | |
2585 | 0 | err = parse_time_err(args[*cur_arg+1], &delay, TIME_UNIT_MS); |
2586 | 0 | if (err == PARSE_TIME_OVER) { |
2587 | 0 | memprintf(errmsg, "timer overflow in argument <%s> to <%s> of server %s, maximum value is 2147483647 ms (~24.8 days).", |
2588 | 0 | args[*cur_arg+1], args[*cur_arg], srv->id); |
2589 | 0 | goto error; |
2590 | 0 | } |
2591 | 0 | else if (err == PARSE_TIME_UNDER) { |
2592 | 0 | memprintf(errmsg, "timer underflow in argument <%s> to <%s> of server %s, minimum non-null value is 1 ms.", |
2593 | 0 | args[*cur_arg+1], args[*cur_arg], srv->id); |
2594 | 0 | goto error; |
2595 | 0 | } |
2596 | 0 | else if (err) { |
2597 | 0 | memprintf(errmsg, "unexpected character '%c' in 'agent-inter' argument of server %s.", |
2598 | 0 | *err, srv->id); |
2599 | 0 | goto error; |
2600 | 0 | } |
2601 | 0 | if (delay <= 0) { |
2602 | 0 | memprintf(errmsg, "invalid value %d for argument '%s' of server %s.", |
2603 | 0 | delay, args[*cur_arg], srv->id); |
2604 | 0 | goto error; |
2605 | 0 | } |
2606 | 0 | srv->check.fastinter = delay; |
2607 | |
|
2608 | 0 | if (warn_if_lower(args[*cur_arg+1], 100)) { |
2609 | 0 | memprintf(errmsg, "'%s %u' in server '%s' is suspiciously small for a value in milliseconds. Please use an explicit unit ('%ums') if that was the intent", |
2610 | 0 | args[*cur_arg], delay, srv->id, delay); |
2611 | 0 | err_code |= ERR_WARN; |
2612 | 0 | } |
2613 | |
|
2614 | 0 | out: |
2615 | 0 | return err_code; |
2616 | | |
2617 | 0 | error: |
2618 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2619 | 0 | goto out; |
2620 | 0 | } |
2621 | | |
2622 | | |
2623 | | /* Parse the "downinter" server keyword */ |
2624 | | static int srv_parse_check_downinter(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2625 | | char **errmsg) |
2626 | 0 | { |
2627 | 0 | const char *err = NULL; |
2628 | 0 | unsigned int delay; |
2629 | 0 | int err_code = 0; |
2630 | |
|
2631 | 0 | if (!*(args[*cur_arg+1])) { |
2632 | 0 | memprintf(errmsg, "'%s' expects a delay as argument.", args[*cur_arg]); |
2633 | 0 | goto error; |
2634 | 0 | } |
2635 | | |
2636 | 0 | err = parse_time_err(args[*cur_arg+1], &delay, TIME_UNIT_MS); |
2637 | 0 | if (err == PARSE_TIME_OVER) { |
2638 | 0 | memprintf(errmsg, "timer overflow in argument <%s> to <%s> of server %s, maximum value is 2147483647 ms (~24.8 days).", |
2639 | 0 | args[*cur_arg+1], args[*cur_arg], srv->id); |
2640 | 0 | goto error; |
2641 | 0 | } |
2642 | 0 | else if (err == PARSE_TIME_UNDER) { |
2643 | 0 | memprintf(errmsg, "timer underflow in argument <%s> to <%s> of server %s, minimum non-null value is 1 ms.", |
2644 | 0 | args[*cur_arg+1], args[*cur_arg], srv->id); |
2645 | 0 | goto error; |
2646 | 0 | } |
2647 | 0 | else if (err) { |
2648 | 0 | memprintf(errmsg, "unexpected character '%c' in 'agent-inter' argument of server %s.", |
2649 | 0 | *err, srv->id); |
2650 | 0 | goto error; |
2651 | 0 | } |
2652 | 0 | if (delay <= 0) { |
2653 | 0 | memprintf(errmsg, "invalid value %d for argument '%s' of server %s.", |
2654 | 0 | delay, args[*cur_arg], srv->id); |
2655 | 0 | goto error; |
2656 | 0 | } |
2657 | 0 | srv->check.downinter = delay; |
2658 | |
|
2659 | 0 | if (warn_if_lower(args[*cur_arg+1], 100)) { |
2660 | 0 | memprintf(errmsg, "'%s %u' in server '%s' is suspiciously small for a value in milliseconds. Please use an explicit unit ('%ums') if that was the intent", |
2661 | 0 | args[*cur_arg], delay, srv->id, delay); |
2662 | 0 | err_code |= ERR_WARN; |
2663 | 0 | } |
2664 | |
|
2665 | 0 | out: |
2666 | 0 | return err_code; |
2667 | | |
2668 | 0 | error: |
2669 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2670 | 0 | goto out; |
2671 | 0 | } |
2672 | | |
2673 | | /* Parse the "port" server keyword */ |
2674 | | static int srv_parse_check_port(char **args, int *cur_arg, struct proxy *curpx, struct server *srv, |
2675 | | char **errmsg) |
2676 | 0 | { |
2677 | 0 | int err_code = 0; |
2678 | |
|
2679 | 0 | if (!*(args[*cur_arg+1])) { |
2680 | 0 | memprintf(errmsg, "'%s' expects a port number as argument.", args[*cur_arg]); |
2681 | 0 | goto error; |
2682 | 0 | } |
2683 | | |
2684 | | /* Only increment maxsock for servers from the configuration. Dynamic |
2685 | | * servers at the moment are not taken into account for the estimation |
2686 | | * of the resources limits. |
2687 | | */ |
2688 | 0 | if (global.mode & MODE_STARTING) |
2689 | 0 | global.maxsock++; |
2690 | |
|
2691 | 0 | srv->check.port = atol(args[*cur_arg+1]); |
2692 | | /* if agentport was never set, we can use port */ |
2693 | 0 | if (!(srv->flags & SRV_F_AGENTPORT)) |
2694 | 0 | srv->agent.port = srv->check.port; |
2695 | |
|
2696 | 0 | out: |
2697 | 0 | return err_code; |
2698 | | |
2699 | 0 | error: |
2700 | 0 | err_code |= ERR_ALERT | ERR_FATAL; |
2701 | 0 | goto out; |
2702 | 0 | } |
2703 | | |
2704 | | /* config parser for global "tune.max-checks-per-thread" */ |
2705 | | static int check_parse_global_max_checks(char **args, int section_type, struct proxy *curpx, |
2706 | | const struct proxy *defpx, const char *file, int line, |
2707 | | char **err) |
2708 | 0 | { |
2709 | 0 | if (too_many_args(1, args, err, NULL)) |
2710 | 0 | return -1; |
2711 | 0 | global.tune.max_checks_per_thread = atoi(args[1]); |
2712 | 0 | return 0; |
2713 | 0 | } |
2714 | | |
2715 | | /* register "global" section keywords */ |
2716 | | static struct cfg_kw_list chk_cfg_kws = {ILH, { |
2717 | | { CFG_GLOBAL, "tune.max-checks-per-thread", check_parse_global_max_checks }, |
2718 | | { 0, NULL, NULL } |
2719 | | }}; |
2720 | | |
2721 | | INITCALL1(STG_REGISTER, cfg_register_keywords, &chk_cfg_kws); |
2722 | | |
2723 | | /* register "server" line keywords */ |
2724 | | static struct srv_kw_list srv_kws = { "CHK", { }, { |
2725 | | { "addr", srv_parse_addr, 1, 1, 1 }, /* IP address to send health to or to probe from agent-check */ |
2726 | | { "agent-addr", srv_parse_agent_addr, 1, 1, 1 }, /* Enable an auxiliary agent check */ |
2727 | | { "agent-check", srv_parse_agent_check, 0, 1, 1 }, /* Enable agent checks */ |
2728 | | { "agent-inter", srv_parse_agent_inter, 1, 1, 1 }, /* Set the interval between two agent checks */ |
2729 | | { "agent-port", srv_parse_agent_port, 1, 1, 1 }, /* Set the TCP port used for agent checks. */ |
2730 | | { "agent-send", srv_parse_agent_send, 1, 1, 1 }, /* Set string to send to agent. */ |
2731 | | { "check", srv_parse_check, 0, 1, 1 }, /* Enable health checks */ |
2732 | | { "check-pool-conn-name", srv_parse_check_pool_conn_name, 1, 1, 1 }, /* */ |
2733 | | { "check-proto", srv_parse_check_proto, 1, 1, 1 }, /* Set the mux protocol for health checks */ |
2734 | | { "check-reuse-pool", srv_parse_check_reuse_pool, 0, 1, 1 }, /* Allows to reuse idle connections for checks */ |
2735 | | { "check-send-proxy", srv_parse_check_send_proxy, 0, 1, 1 }, /* Enable PROXY protocol for health checks */ |
2736 | | { "check-via-socks4", srv_parse_check_via_socks4, 0, 1, 1 }, /* Enable socks4 proxy for health checks */ |
2737 | | { "no-agent-check", srv_parse_no_agent_check, 0, 1, 0 }, /* Do not enable any auxiliary agent check */ |
2738 | | { "no-check", srv_parse_no_check, 0, 1, 0 }, /* Disable health checks */ |
2739 | | { "no-check-reuse-pool", srv_parse_no_check_reuse_pool, 0, 1, 0 }, /* Disable PROXY protocol for health checks */ |
2740 | | { "no-check-send-proxy", srv_parse_no_check_send_proxy, 0, 1, 0 }, /* Disable PROXY protocol for health checks */ |
2741 | | { "rise", srv_parse_check_rise, 1, 1, 1 }, /* Set rise value for health checks */ |
2742 | | { "fall", srv_parse_check_fall, 1, 1, 1 }, /* Set fall value for health checks */ |
2743 | | { "inter", srv_parse_check_inter, 1, 1, 1 }, /* Set inter value for health checks */ |
2744 | | { "fastinter", srv_parse_check_fastinter, 1, 1, 1 }, /* Set fastinter value for health checks */ |
2745 | | { "downinter", srv_parse_check_downinter, 1, 1, 1 }, /* Set downinter value for health checks */ |
2746 | | { "port", srv_parse_check_port, 1, 1, 1 }, /* Set the TCP port used for health checks. */ |
2747 | | { NULL, NULL, 0 }, |
2748 | | }}; |
2749 | | |
2750 | | INITCALL1(STG_REGISTER, srv_register_keywords, &srv_kws); |
2751 | | |
2752 | | /* |
2753 | | * Local variables: |
2754 | | * c-indent-level: 8 |
2755 | | * c-basic-offset: 8 |
2756 | | * End: |
2757 | | */ |