/src/trafficserver/include/mgmt/config/ConfigReloadTrace.h
Line | Count | Source |
1 | | /** @file |
2 | | * |
3 | | * ConfigReloadTrace - Reload task tracking and progress reporting |
4 | | * |
5 | | * @section license License |
6 | | * |
7 | | * Licensed to the Apache Software Foundation (ASF) under one |
8 | | * or more contributor license agreements. See the NOTICE file |
9 | | * distributed with this work for additional information |
10 | | * regarding copyright ownership. The ASF licenses this file |
11 | | * to you under the Apache License, Version 2.0 (the |
12 | | * "License"); you may not use this file except in compliance |
13 | | * with the License. You may obtain a copy of the License at |
14 | | * |
15 | | * http://www.apache.org/licenses/LICENSE-2.0 |
16 | | * |
17 | | * Unless required by applicable law or agreed to in writing, software |
18 | | * distributed under the License is distributed on an "AS IS" BASIS, |
19 | | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
20 | | * See the License for the specific language governing permissions and |
21 | | * limitations under the License. |
22 | | */ |
23 | | |
24 | | #pragma once |
25 | | |
26 | | #include <atomic> |
27 | | #include <chrono> |
28 | | #include <iostream> |
29 | | #include <string_view> |
30 | | #include <string> |
31 | | #include <mutex> |
32 | | #include <shared_mutex> |
33 | | #include <utility> |
34 | | |
35 | | #include <swoc/Errata.h> |
36 | | #include <tscore/ink_platform.h> |
37 | | #include "iocore/eventsystem/Continuation.h" |
38 | | #include "iocore/eventsystem/EventSystem.h" |
39 | | #include "iocore/eventsystem/Event.h" |
40 | | #include "iocore/eventsystem/Tasks.h" |
41 | | #include "tscore/Diags.h" |
42 | | |
43 | | class ConfigReloadTask; |
44 | | class ConfigContext; |
45 | | |
46 | | namespace YAML |
47 | | { |
48 | | template <typename T> struct convert; |
49 | | } // namespace YAML |
50 | | |
51 | | using ConfigReloadTaskPtr = std::shared_ptr<ConfigReloadTask>; |
52 | | |
53 | | /// |
54 | | /// @brief Progress checker for reload tasks - detects stuck/hanging tasks. |
55 | | /// |
56 | | /// Created per-reload by ConfigReloadTask::start_progress_checker(), which is called |
57 | | /// only for main tasks in the IN_PROGRESS state. Each instance is bound to a single |
58 | | /// ConfigReloadTask and holds a shared_ptr to it. |
59 | | /// |
60 | | /// Lifecycle: |
61 | | /// - Scheduled on ET_TASK when a reload starts. |
62 | | /// - check_progress() runs periodically (every check_interval). |
63 | | /// - Deletes itself and returns EVENT_DONE when: |
64 | | /// * The task reaches a terminal state (SUCCESS, FAIL, TIMEOUT). |
65 | | /// * The task exceeds the configured timeout (marked as TIMEOUT, then stops). |
66 | | /// * The _reload pointer is null (defensive). |
67 | | /// - Reschedules itself (EVENT_CONT) only while the task is still non-terminal. |
68 | | /// - No idle polling - when no reload is in progress, no checker exists. |
69 | | /// |
70 | | /// Configurable via records: |
71 | | /// - proxy.config.admin.reload.timeout: Duration string (default: "1h") |
72 | | /// Supports: "30s", "5min", "1h", "1 hour 30min", "0" (disabled) |
73 | | /// - proxy.config.admin.reload.check_interval: Duration string (default: "2s") |
74 | | /// Minimum: 1s (enforced). How often to check task progress. |
75 | | /// |
76 | | /// If timeout is 0 or empty, timeout is disabled. Tasks can hang forever (BAD). |
77 | | /// Use --force (traffic_ctl / RPC API) flag to bypass the in-progress guard and start a new reload. |
78 | | /// Note: --force only marks the existing tracking task as stale; it does not cancel the actual work. |
79 | | /// |
80 | | struct ConfigReloadProgress : public Continuation { |
81 | | /// Record names for configuration |
82 | | static constexpr std::string_view RECORD_TIMEOUT = "proxy.config.admin.reload.timeout"; |
83 | | static constexpr std::string_view RECORD_CHECK_INTERVAL = "proxy.config.admin.reload.check_interval"; |
84 | | |
85 | | /// Defaults |
86 | | static constexpr std::string_view DEFAULT_TIMEOUT = "1h"; ///< 1 hour |
87 | | static constexpr std::string_view DEFAULT_CHECK_INTERVAL = "2s"; ///< 2 seconds |
88 | | static constexpr int64_t MIN_CHECK_INTERVAL_MS = 1000; ///< 1 second minimum |
89 | | |
90 | | ConfigReloadProgress(ConfigReloadTaskPtr reload); |
91 | | int check_progress(int /* etype */, void * /* data */); |
92 | | |
93 | | /// Read timeout value from records, returns 0ms if disabled or invalid |
94 | | [[nodiscard]] static std::chrono::milliseconds get_configured_timeout(); |
95 | | |
96 | | /// Read check interval from records, enforces minimum of 1s |
97 | | [[nodiscard]] static std::chrono::milliseconds get_configured_check_interval(); |
98 | | |
99 | | /// Get the check interval for this instance |
100 | | [[nodiscard]] std::chrono::milliseconds |
101 | | get_check_interval() const |
102 | 0 | { |
103 | 0 | return _every; |
104 | 0 | } |
105 | | |
106 | | private: |
107 | | /// Grace period before accepting a terminal state as final. A late reserve_subtask() |
108 | | /// can pull the parent back from SUCCESS to IN_PROGRESS; this delay lets the checker |
109 | | /// detect that transition instead of stopping prematurely. |
110 | | static constexpr std::chrono::milliseconds TERMINAL_CONFIRMATION_DELAY{5000}; |
111 | | ConfigReloadTaskPtr _reload{nullptr}; |
112 | | std::chrono::milliseconds _every{std::chrono::seconds{2}}; ///< Set from config in constructor |
113 | | bool _awaiting_terminal_confirmation{false}; ///< true after first terminal observation |
114 | | }; |
115 | | |
116 | | /// |
117 | | /// @brief Tracks the status and progress of a single config reload operation. |
118 | | /// |
119 | | /// Represents either a top-level (main) reload task or a sub-task for an individual |
120 | | /// config module. Tasks form a tree: the main task has sub-tasks for each config, |
121 | | /// and sub-tasks can themselves have children (e.g., SSLClientCoordinator → SSLConfig, SNIConfig). |
122 | | /// |
123 | | /// State flows: CREATED → IN_PROGRESS → SUCCESS / FAIL / TIMEOUT |
124 | | /// Parent tasks aggregate state from their children automatically. |
125 | | /// |
126 | | /// Serialized to YAML via YAML::convert<ConfigReloadTask::Info> for RPC responses. |
127 | | /// |
128 | | class ConfigReloadTask : public std::enable_shared_from_this<ConfigReloadTask> |
129 | | { |
130 | | public: |
131 | | enum class State { |
132 | | INVALID = -1, |
133 | | CREATED, ///< Initial state - task exists but not started |
134 | | IN_PROGRESS, ///< Work is actively happening |
135 | | SUCCESS, ///< Terminal: completed successfully |
136 | | FAIL, ///< Terminal: error occurred |
137 | | TIMEOUT ///< Terminal: task exceeded time limit |
138 | | }; |
139 | | |
140 | | /// Check if a state represents a terminal (final) state |
141 | | [[nodiscard]] static constexpr bool |
142 | | is_terminal(State s) noexcept |
143 | 0 | { |
144 | 0 | return s == State::SUCCESS || s == State::FAIL || s == State::TIMEOUT; |
145 | 0 | } |
146 | | |
147 | | /// Convert State enum to string |
148 | | [[nodiscard]] static constexpr std::string_view |
149 | | state_to_string(State s) noexcept |
150 | 0 | { |
151 | 0 | switch (s) { |
152 | 0 | case State::INVALID: |
153 | 0 | return "invalid"; |
154 | 0 | case State::CREATED: |
155 | 0 | return "created"; |
156 | 0 | case State::IN_PROGRESS: |
157 | 0 | return "in_progress"; |
158 | 0 | case State::SUCCESS: |
159 | 0 | return "success"; |
160 | 0 | case State::FAIL: |
161 | 0 | return "fail"; |
162 | 0 | case State::TIMEOUT: |
163 | 0 | return "timeout"; |
164 | 0 | } |
165 | 0 | return "unknown"; |
166 | 0 | } |
167 | | |
168 | | /// Helper to get current time in milliseconds since epoch |
169 | | [[nodiscard]] static int64_t |
170 | | now_ms() |
171 | 0 | { |
172 | 0 | return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count(); |
173 | 0 | } |
174 | | |
175 | | /// A single log entry with optional severity. |
176 | | /// Entries from log(level, text) carry the supplied DiagsLevel. State-change |
177 | | /// convenience methods attach an implicit level: in_progress() and complete() |
178 | | /// log at DL_Note, fail() logs at DL_Error. The one-argument log(text) form |
179 | | /// stores DL_Undefined and is always displayed (never filtered by --min-level). |
180 | | struct LogEntry { |
181 | | DiagsLevel level{DL_Undefined}; ///< DL_Undefined = always shown (filter bypass) |
182 | | std::string text; |
183 | | }; |
184 | | |
185 | | struct Info { |
186 | | friend class ConfigReloadTask; |
187 | | /// Grant friendship to the specific YAML::convert specialization. |
188 | | friend struct YAML::convert<ConfigReloadTask::Info>; |
189 | 0 | Info() = default; |
190 | | Info(State p_state, std::string_view p_token, std::string_view p_description, bool p_main_task) |
191 | 0 | : state(p_state), token(p_token), description(p_description), main_task(p_main_task) |
192 | 0 | { |
193 | 0 | } |
194 | | |
195 | | protected: |
196 | | int64_t created_time_ms{now_ms()}; ///< milliseconds since epoch |
197 | | int64_t last_updated_time_ms{now_ms()}; ///< last time this task was updated (ms) |
198 | | std::vector<LogEntry> logs; ///< log messages from handler |
199 | | State state{State::CREATED}; |
200 | | std::string token; |
201 | | std::string description; |
202 | | std::string config_key; ///< registry key for dedup (empty for main/child tasks) |
203 | | std::string filename; ///< source file, if applicable |
204 | | std::vector<ConfigReloadTaskPtr> sub_tasks; ///< child tasks (if any) |
205 | | bool main_task{false}; ///< true for the top-level reload task |
206 | | std::string plugin_name; ///< Registering plugin (empty for core). |
207 | | }; |
208 | | |
209 | | using self_type = ConfigReloadTask; |
210 | | ConfigReloadTask() = default; |
211 | | ConfigReloadTask(std::string_view token, std::string_view description, bool main_task, const ConfigReloadTaskPtr &parent) |
212 | 0 | : _info(State::CREATED, token, description, main_task), _parent{parent} |
213 | 0 | { |
214 | 0 | if (_info.main_task) { |
215 | 0 | _info.state = State::IN_PROGRESS; |
216 | 0 | } |
217 | 0 | } |
218 | | |
219 | | /// Start the periodic progress checker (ConfigReloadProgress). |
220 | | /// Only runs once, and only for main tasks. The checker detects stuck tasks |
221 | | /// and marks them as TIMEOUT if they exceed the configured time limit. |
222 | | void start_progress_checker(); |
223 | | |
224 | | /// Create a child sub-task and return a ConfigContext wrapping it. |
225 | | /// The child inherits the parent's token and if passed, the supplied YAML content. |
226 | | [[nodiscard]] ConfigContext add_child(std::string_view description = "", std::string_view filename = ""); |
227 | | |
228 | | self_type &log(std::string const &text); |
229 | | self_type &log(DiagsLevel level, std::string const &text); |
230 | | void set_completed(); |
231 | | void set_failed(); |
232 | | void set_in_progress(); |
233 | | |
234 | | void |
235 | | set_description(std::string_view description) |
236 | 0 | { |
237 | 0 | std::unique_lock<std::shared_mutex> lock(_mutex); |
238 | 0 | _info.description = description; |
239 | 0 | } |
240 | | |
241 | | [[nodiscard]] std::string |
242 | | get_description() const |
243 | 0 | { |
244 | 0 | std::shared_lock<std::shared_mutex> lock(_mutex); |
245 | 0 | return _info.description; |
246 | 0 | } |
247 | | |
248 | | void |
249 | | set_filename(std::string_view filename) |
250 | 0 | { |
251 | 0 | std::unique_lock<std::shared_mutex> lock(_mutex); |
252 | 0 | _info.filename = filename; |
253 | 0 | } |
254 | | |
255 | | [[nodiscard]] std::string |
256 | | get_filename() const |
257 | 0 | { |
258 | 0 | std::shared_lock<std::shared_mutex> lock(_mutex); |
259 | 0 | return _info.filename; |
260 | 0 | } |
261 | | |
262 | | void |
263 | | set_config_key(std::string_view key) |
264 | 0 | { |
265 | 0 | std::unique_lock<std::shared_mutex> lock(_mutex); |
266 | 0 | _info.config_key = key; |
267 | 0 | } |
268 | | |
269 | | /// Set the registering plugin's name (empty for core). |
270 | | void |
271 | | set_plugin_name(std::string_view name) |
272 | 0 | { |
273 | 0 | std::unique_lock<std::shared_mutex> lock(_mutex); |
274 | 0 | _info.plugin_name = name; |
275 | 0 | } |
276 | | |
277 | | /// Check if any immediate subtask has the given config_key. |
278 | | /// Used by ReloadCoordinator to prevent duplicate subtasks within a single reload cycle. |
279 | | [[nodiscard]] bool has_subtask_for_key(std::string_view key) const; |
280 | | |
281 | | /// Find a subtask by its config_key. Returns nullptr if not found. |
282 | | [[nodiscard]] ConfigReloadTaskPtr find_subtask_by_key(std::string_view key) const; |
283 | | |
284 | | /// Debug utility: dump task tree to an output stream. |
285 | | /// Recursively prints this task and all sub-tasks with indentation. |
286 | | |
287 | | [[nodiscard]] bool |
288 | | contains_dependents() const |
289 | 0 | { |
290 | 0 | std::shared_lock<std::shared_mutex> lock(_mutex); |
291 | 0 | return !_info.sub_tasks.empty(); |
292 | 0 | } |
293 | | |
294 | | /// Get created time in seconds (for Date formatting and metrics). |
295 | | /// No lock needed - created_time_ms is immutable after construction. |
296 | | [[nodiscard]] std::time_t |
297 | | get_created_time() const |
298 | 0 | { |
299 | 0 | return static_cast<std::time_t>( |
300 | 0 | std::chrono::duration_cast<std::chrono::seconds>(std::chrono::milliseconds{_info.created_time_ms}).count()); |
301 | 0 | } |
302 | | |
303 | | /// Get created time in milliseconds since epoch. |
304 | | /// No lock needed - created_time_ms is immutable after construction. |
305 | | [[nodiscard]] int64_t |
306 | | get_created_time_ms() const |
307 | 0 | { |
308 | 0 | return _info.created_time_ms; |
309 | 0 | } |
310 | | |
311 | | [[nodiscard]] State |
312 | | get_state() const |
313 | 0 | { |
314 | 0 | std::shared_lock<std::shared_mutex> lock(_mutex); |
315 | 0 | return _info.state; |
316 | 0 | } |
317 | | |
318 | | /// Mark task as TIMEOUT with an optional reason logged |
319 | | void mark_as_bad_state(std::string_view reason = ""); |
320 | | |
321 | | [[nodiscard]] std::vector<LogEntry> |
322 | | get_logs() const |
323 | 0 | { |
324 | 0 | std::shared_lock<std::shared_mutex> lock(_mutex); |
325 | 0 | return _info.logs; |
326 | 0 | } |
327 | | |
328 | | [[nodiscard]] std::string |
329 | | get_token() const |
330 | 0 | { |
331 | 0 | std::shared_lock<std::shared_mutex> lock(_mutex); |
332 | 0 | return _info.token; |
333 | 0 | } |
334 | | |
335 | | [[nodiscard]] bool |
336 | | is_main_task() const |
337 | 0 | { |
338 | 0 | std::shared_lock<std::shared_mutex> lock(_mutex); |
339 | 0 | return _info.main_task; |
340 | 0 | } |
341 | | |
342 | | /// Create a copy of the current task info |
343 | | [[nodiscard]] Info |
344 | | get_info() const |
345 | 0 | { |
346 | 0 | std::shared_lock<std::shared_mutex> lock(_mutex); |
347 | 0 | Info copy = _info; |
348 | 0 | copy.last_updated_time_ms = _atomic_last_updated_ms.load(std::memory_order_acquire); |
349 | 0 | return copy; |
350 | 0 | } |
351 | | |
352 | | /// Get last updated time in seconds (considers subtasks) |
353 | | [[nodiscard]] std::time_t get_last_updated_time() const; |
354 | | |
355 | | /// Get last updated time in milliseconds (considers subtasks) |
356 | | [[nodiscard]] int64_t get_last_updated_time_ms() const; |
357 | | |
358 | | void |
359 | | update_last_updated_time() |
360 | 0 | { |
361 | 0 | _atomic_last_updated_ms.store(now_ms(), std::memory_order_release); |
362 | 0 | } |
363 | | |
364 | | /// Read the last updated time for this task only (no subtask traversal, lock-free) |
365 | | [[nodiscard]] int64_t |
366 | | get_own_last_updated_time_ms() const |
367 | 0 | { |
368 | 0 | return _atomic_last_updated_ms.load(std::memory_order_acquire); |
369 | 0 | } |
370 | | |
371 | | private: |
372 | | /// Add a pre-created sub-task to this task's children list. |
373 | | /// Called by ReloadCoordinator::create_config_context(). |
374 | | void add_sub_task(ConfigReloadTaskPtr sub_task); |
375 | | void on_sub_task_update(State state); |
376 | | void aggregate_status(); |
377 | | void notify_parent(); |
378 | | void set_state_and_notify(State state); |
379 | | |
380 | | friend struct ConfigReloadProgress; |
381 | | void log_reload_summary(State final_state); |
382 | | static void dump_subtask_tree(const std::vector<ConfigReloadTaskPtr> &tasks, int indent); |
383 | | |
384 | | mutable std::shared_mutex _mutex; |
385 | | bool _reload_progress_checker_started{false}; |
386 | | bool _summary_logged{false}; |
387 | | Info _info; |
388 | | std::weak_ptr<ConfigReloadTask> _parent; ///< parent task, if any. Weak because the parent owns this task via sub_tasks. |
389 | | |
390 | | std::atomic<int64_t> _atomic_last_updated_ms{now_ms()}; |
391 | | |
392 | | friend class ReloadCoordinator; |
393 | | }; |