Coverage Report

Created: 2026-10-28 07:07

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/gbmcweb/tlbmc/scheduler/scheduler.cc
Line
Count
Source
1
#include "tlbmc/scheduler/scheduler.h"
2
3
#include <algorithm>
4
#include <atomic>
5
#include <cstddef>
6
#include <memory>
7
#include <optional>
8
#include <thread>  //NOLINT: <thread> is commonly used in BMC
9
#include <utility>
10
#include <vector>
11
12
#include "absl/container/flat_hash_map.h"
13
#include "absl/functional/any_invocable.h"
14
#include "absl/log/log.h"
15
#include "absl/synchronization/mutex.h"
16
#include "absl/time/time.h"
17
#include "boost/asio/executor.hpp"  //NOLINT: boost is commonly used in BMC
18
#include "boost/asio/executor_work_guard.hpp"  //NOLINT: boost is commonly used in BMC
19
#include "boost/asio/steady_timer.hpp"  //NOLINT: boost is commonly used in BMC
20
#include "time/clock.h"
21
#include <nlohmann/json.hpp>
22
23
namespace milotic_tlbmc {
24
25
8.37k
nlohmann::json TimeInfo::ToJson() const {
26
8.37k
  nlohmann::json json;
27
8.37k
  json["last_scheduled_time"] = absl::FormatTime(last_scheduled_time);
28
8.37k
  json["last_run_time"] = absl::FormatTime(last_run_time);
29
  // Get the last 10 wait times.
30
8.37k
  auto it = wait_times.rbegin();
31
8.37k
  auto& wait_times_recent_to_oldest = json["wait_times_recent_to_oldest"];
32
8.37k
  wait_times_recent_to_oldest = nlohmann::json::array();
33
8.37k
  for (int i = 0; i < 10 && it != wait_times.rend(); ++i, ++it) {
34
0
    wait_times_recent_to_oldest.push_back(absl::FormatDuration(*it));
35
0
  }
36
8.37k
  return json;
37
8.37k
}
38
39
8.37k
nlohmann::json Task::ToJson() const {
40
8.37k
  nlohmann::json json;
41
8.37k
  json["task_id"] = task_id_;
42
8.37k
  json["period"] = absl::FormatDuration(period_);
43
8.37k
  json["execution_count"] = execution_count_.load();
44
8.37k
  {
45
8.37k
    absl::MutexLock lock(&time_info_mutex_);
46
8.37k
    json["time_info"] = time_info_.ToJson();
47
8.37k
  }
48
8.37k
  json["average_wait_time"] = absl::FormatDuration(GetAverageWaitTime());
49
8.37k
  json["cancelled"] = cancelled_.load();
50
8.37k
  json["task_mode"] = task_mode_ == TaskMode::kPeriodic ? "periodic" : "once";
51
8.37k
  return json;
52
8.37k
}
53
54
// Overload constructor for tasks that need to be invoked with an ack callback.
55
Task::Task(int task_id,
56
           absl::AnyInvocable<void(absl::AnyInvocable<void()>)> task_fn,
57
           absl::Duration period, boost::asio::io_context* io,
58
           absl::Duration task_timeout, TaskMode task_mode,
59
           ecclesia::Clock* clock)
60
4.18k
    : task_id_(task_id),
61
4.18k
      task_mode_(task_mode),
62
4.18k
      clock_(clock),
63
4.18k
      period_(period),
64
4.18k
      cancelled_(false),
65
4.18k
      task_fn_with_ack_(std::move(task_fn)),
66
4.18k
      timer_(std::make_unique<boost::asio::steady_timer>(*io)),
67
4.18k
      task_timeout_(task_timeout) {}
68
69
0
absl::Duration Task::GetPeriod() const { return period_.load(); }
70
71
16.7k
absl::Duration Task::GetAverageWaitTime() const {
72
16.7k
  absl::MutexLock lock(&time_info_mutex_);
73
16.7k
  if (time_info_.wait_times.empty()) {
74
    // No wait times have been recorded yet. This happens when the task is yet
75
    // to be scheduled after it's initial run.
76
16.7k
    return absl::InfiniteDuration();
77
16.7k
  }
78
0
  absl::Duration total_wait_time = absl::ZeroDuration();
79
0
  for (const auto& wait_time : time_info_.wait_times) {
80
0
    total_wait_time += wait_time;
81
0
  }
82
0
  return total_wait_time / time_info_.wait_times.size();
83
16.7k
}
84
85
8.37k
absl::Time Task::GetLastRunTime() const {
86
8.37k
  absl::MutexLock lock(&time_info_mutex_);
87
8.37k
  return time_info_.last_run_time;
88
8.37k
}
89
90
4.18k
void Task::ScheduleNextRun() {
91
4.18k
  {
92
4.18k
    absl::MutexLock lock(&time_info_mutex_);
93
4.18k
    if (clock_ != nullptr) {
94
4.18k
      time_info_.last_scheduled_time = clock_->Now();
95
4.18k
    }
96
4.18k
  }
97
98
4.18k
  {
99
4.18k
    absl::MutexLock lock(&timer_mutex_);
100
4.18k
    if (cancelled_.load()) {
101
0
      return;
102
0
    }
103
104
4.18k
    last_execution_duration_ =
105
4.18k
        GetLastRunTime() == absl::InfiniteFuture()
106
4.18k
            ? absl::ZeroDuration()
107
4.18k
            : std::max(absl::ZeroDuration(), clock_->Now() - GetLastRunTime());
108
4.18k
    absl::Duration wait_time = period_ - last_execution_duration_;
109
4.18k
    if (wait_time < absl::ZeroDuration()) {
110
0
      wait_time = absl::ZeroDuration();
111
0
    }
112
4.18k
    timer_->expires_after(absl::ToChronoNanoseconds(wait_time));
113
114
4.18k
    std::weak_ptr<Task> weak_task = weak_from_this();
115
4.18k
    timer_->async_wait([weak_task](boost::system::error_code error) {
116
12
      if (error == boost::asio::error::operation_aborted) {
117
36
        DLOG(ERROR) << "Failed to wait for task to complete with error: "
118
36
                    << error.message();
119
12
        return;
120
12
      }
121
0
      auto task = weak_task.lock();
122
0
      if (task != nullptr) {
123
0
        task->Run();
124
0
      }
125
0
    });
126
4.18k
  }
127
4.18k
}
128
129
2.09k
void Task::OnDone() {
130
2.09k
  ++execution_count_;
131
2.09k
  if (task_mode_ == TaskMode::kOnce) {
132
0
    return;
133
0
  }
134
2.09k
  ScheduleNextRun();
135
2.09k
}
136
137
// Runs the task and updates the next run time.
138
2.09k
void Task::Run() {
139
2.09k
  if (cancelled_.load()) {
140
0
    DLOG(WARNING) << "Task is cancelled before running!";
141
0
    return;
142
0
  }
143
144
2.09k
  {
145
2.09k
    absl::MutexLock lock(&time_info_mutex_);
146
2.09k
    absl::Time now = clock_->Now();
147
2.09k
    if (time_info_.last_scheduled_time != absl::InfiniteFuture()) {
148
0
      time_info_.wait_times.push_back(now - time_info_.last_scheduled_time);
149
0
    }
150
2.09k
    time_info_.last_run_time = now;
151
2.09k
  }
152
153
2.09k
  {
154
2.09k
    absl::MutexLock lock(&task_mutex_);
155
2.09k
    if (task_fn_with_ack_ != nullptr) {
156
2.09k
      std::weak_ptr<Task> weak_task = weak_from_this();
157
2.09k
      task_fn_with_ack_([weak_task]() {
158
2.09k
        auto task = weak_task.lock();
159
2.09k
        if (task != nullptr) {
160
2.09k
          task->OnDone();
161
2.09k
        }
162
2.09k
      });
163
2.09k
      return;
164
2.09k
    }
165
2.09k
  }
166
2.09k
}
167
168
// Cancels the task.
169
4.18k
void Task::Cancel() {
170
4.18k
  cancelled_.store(true);
171
4.18k
  absl::MutexLock lock(&timer_mutex_);
172
4.18k
  timer_->cancel();
173
4.18k
}
174
175
0
void Task::SetPeriod(absl::Duration period) {
176
0
  Cancel();
177
0
  period_ = period;
178
0
  cancelled_.store(false);
179
0
  ScheduleNextRun();
180
0
}
181
182
TaskScheduler::TaskScheduler(ecclesia::Clock* clock, const Options& options)
183
2.09k
    : alive_token_(std::make_shared<AliveToken>()),
184
2.09k
      next_task_id_(1),
185
2.09k
      clock_(clock),
186
2.09k
      io_(std::make_unique<boost::asio::io_context>()),
187
2.09k
      work_guard_(io_->get_executor()),
188
2.09k
      async_executor_thread_([this]() { io_->run(); }),
189
2.09k
      default_task_timeout_(options.default_task_timeout) {
190
2.09k
  cleanup_task_id_ = RunAndScheduleAsync(
191
2.09k
      [this](absl::AnyInvocable<void()> on_done) {
192
2.09k
        CleanupInactiveTasks();
193
2.09k
        on_done();
194
2.09k
      },
195
2.09k
      options.cleanup_tasks_period);
196
2.09k
}
197
198
2.09k
TaskScheduler::~TaskScheduler() { Stop(); }
199
200
int TaskScheduler::ScheduleAsyncInternal(
201
    absl::AnyInvocable<void(absl::AnyInvocable<void()>)> task,
202
    absl::Duration period, std::optional<absl::Duration> timeout,
203
4.18k
    ExecutionMode execution_mode) {
204
4.18k
  if (task == nullptr) {
205
0
    LOG(ERROR) << "Task is null";
206
0
    return -1;
207
0
  }
208
209
4.18k
  if (stop_.load()) {
210
0
    LOG(ERROR) << "TaskScheduler is stopped, cannot schedule async task";
211
0
    return -1;
212
0
  }
213
214
4.18k
  if (period == absl::ZeroDuration()) {
215
0
    LOG(ERROR) << "Task period is zero, cannot schedule async recurring task";
216
0
    return -1;
217
0
  }
218
219
4.18k
  int task_id = next_task_id_++;
220
4.18k
  absl::Duration task_timeout = timeout.value_or(default_task_timeout_);
221
4.18k
  auto task_ptr =
222
4.18k
      std::make_shared<Task>(task_id, std::move(task), period, io_.get(),
223
4.18k
                             task_timeout, TaskMode::kPeriodic, clock_);
224
4.18k
  {
225
4.18k
    absl::MutexLock lock(&async_tasks_mutex_);
226
4.18k
    id_to_async_tasks_.emplace(task_id, task_ptr);
227
4.18k
  }
228
4.18k
  if (execution_mode == ExecutionMode::kRunImmediately) {
229
2.09k
    task_ptr->Run();
230
2.09k
  } else if (execution_mode == ExecutionMode::kRunAfterScheduling) {
231
2.09k
    task_ptr->ScheduleNextRun();
232
2.09k
  }
233
4.18k
  return task_id;
234
4.18k
}
235
236
int TaskScheduler::ScheduleAsync(
237
    absl::AnyInvocable<void(absl::AnyInvocable<void()>)> task,
238
2.09k
    absl::Duration period, std::optional<absl::Duration> timeout) {
239
2.09k
  return ScheduleAsyncInternal(std::move(task), period, timeout,
240
2.09k
                               ExecutionMode::kRunAfterScheduling);
241
2.09k
}
242
243
int TaskScheduler::RunAndScheduleAsync(
244
    absl::AnyInvocable<void(absl::AnyInvocable<void()>)> task,
245
2.09k
    absl::Duration period, std::optional<absl::Duration> timeout) {
246
2.09k
  return ScheduleAsyncInternal(std::move(task), period, timeout,
247
2.09k
                               ExecutionMode::kRunImmediately);
248
2.09k
}
249
250
int TaskScheduler::ScheduleOneShotAsync(
251
    absl::AnyInvocable<void(absl::AnyInvocable<void()>)> task,
252
0
    absl::Duration period, std::optional<absl::Duration> timeout) {
253
0
  if (task == nullptr) {
254
0
    LOG(ERROR) << "Task is null";
255
0
    return -1;
256
0
  }
257
258
0
  if (stop_.load()) {
259
0
    LOG(ERROR) << "TaskScheduler is stopped, cannot run and schedule task";
260
0
    return -1;
261
0
  }
262
263
0
  if (period == absl::ZeroDuration()) {
264
0
    LOG(ERROR) << "Task period is zero, cannot schedule task";
265
0
    return -1;
266
0
  }
267
268
0
  int task_id = next_task_id_++;
269
0
  absl::Duration task_timeout = timeout.value_or(default_task_timeout_);
270
271
0
  auto oneshot_on_done = [this, weak_token = std::weak_ptr(alive_token_),
272
0
                          task_id]() {
273
0
    if (auto token = weak_token.lock()) {
274
0
      absl::MutexLock lock(&token->mu);
275
0
      if (token->alive) {
276
0
        Cancel(task_id);
277
0
      }
278
0
    }
279
0
  };
280
0
  auto task_ptr = std::make_shared<Task>(
281
0
      task_id,
282
0
      [task = std::move(task), oneshot_on_done = std::move(oneshot_on_done)](
283
0
          [[maybe_unused]] absl::AnyInvocable<void()> on_done) mutable {
284
0
        task(std::move(oneshot_on_done));
285
0
      },
286
0
      period, io_.get(), task_timeout, TaskMode::kOnce, clock_);
287
0
  {
288
0
    absl::MutexLock lock(&async_tasks_mutex_);
289
0
    id_to_async_tasks_.emplace(task_id, task_ptr);
290
0
  }
291
0
  task_ptr->ScheduleNextRun();
292
293
0
  return task_id;
294
0
}
295
296
absl::flat_hash_map<int, std::shared_ptr<Task>> TaskScheduler::GetSnapshot()
297
0
    const {
298
0
  absl::flat_hash_map<int, std::shared_ptr<Task>> snapshot;
299
0
  if (stop_.load()) {
300
0
    return snapshot;
301
0
  }
302
0
  {
303
0
    absl::MutexLock lock(&async_tasks_mutex_);
304
0
    for (const auto& [task_id, task] : id_to_async_tasks_) {
305
0
      snapshot.emplace(task_id, task);
306
0
    }
307
0
  }
308
0
  return snapshot;
309
0
}
310
311
0
size_t TaskScheduler::GetAllUserTaskCount() const {
312
0
  if (stop_.load()) {
313
0
    return 0;
314
0
  }
315
0
  absl::MutexLock lock(&async_tasks_mutex_);
316
0
  size_t count = 0;
317
0
  for (const auto& [task_id, task] : id_to_async_tasks_) {
318
0
    if (task_id != cleanup_task_id_) {
319
0
      ++count;
320
0
    }
321
0
  }
322
0
  return count;
323
0
}
324
325
0
size_t TaskScheduler::GetAllTaskCount() const {
326
0
  if (stop_.load()) {
327
0
    return 0;
328
0
  }
329
0
  absl::MutexLock lock(&async_tasks_mutex_);
330
0
  return id_to_async_tasks_.size();
331
0
}
332
333
0
void TaskScheduler::UpdateTaskPeriod(int task_id, absl::Duration period) {
334
0
  if (stop_.load()) {
335
0
    LOG(WARNING) << "TaskScheduler is stopped, cannot update task period: "
336
0
                 << task_id;
337
0
    return;
338
0
  }
339
340
0
  if (period == absl::ZeroDuration()) {
341
0
    LOG(ERROR) << "Cannot set async task period to zero";
342
0
    return;
343
0
  }
344
345
0
  absl::MutexLock lock(&async_tasks_mutex_);
346
0
  auto it = id_to_async_tasks_.find(task_id);
347
0
  if (it != id_to_async_tasks_.end()) {
348
0
    if (it->second->GetPeriod() == period) {
349
0
      return;
350
0
    }
351
0
    LOG(WARNING) << "Updating task period for task: " << task_id << " from "
352
0
                 << it->second->GetPeriod() << " to " << period;
353
0
    it->second->SetPeriod(period);
354
0
  }
355
0
}
356
357
0
void TaskScheduler::Cancel(int task_id) {
358
0
  if (stop_.load()) {
359
0
    LOG(WARNING) << "TaskScheduler is stopped, cannot cancel task: " << task_id;
360
0
    return;
361
0
  }
362
363
  // Cancel the async task.
364
0
  {
365
0
    absl::MutexLock lock(&async_tasks_mutex_);
366
0
    auto it = id_to_async_tasks_.find(task_id);
367
0
    if (it != id_to_async_tasks_.end()) {
368
0
      it->second->Cancel();
369
      // Remove the task from the tasks map.
370
0
      id_to_async_tasks_.erase(it);
371
0
    }
372
0
  }
373
0
}
374
375
0
void TaskScheduler::CancelAll() {
376
0
  if (stop_.load()) {
377
0
    LOG(ERROR) << "TaskScheduler is stopped, cannot cancel all tasks";
378
0
    return;
379
0
  }
380
381
0
  {
382
0
    absl::MutexLock lock(&async_tasks_mutex_);
383
0
    for (const auto& [task_id, task] : id_to_async_tasks_) {
384
0
      task->Cancel();
385
0
    }
386
0
    id_to_async_tasks_.clear();
387
0
  }
388
0
}
389
390
2.09k
void TaskScheduler::CleanupInactiveTasks() {
391
2.09k
  if (stop_.load()) {
392
0
    return;
393
0
  }
394
395
  // Copy id_to_async_tasks_ to a vector to avoid holding the lock while
396
  // cancelling tasks.
397
2.09k
  std::vector<std::pair<int, std::shared_ptr<Task>>> id_to_async_tasks_copy;
398
  // Task ids of Tasks to cancel.
399
2.09k
  std::vector<std::pair<int, absl::Duration>> tasks_to_cancel;
400
2.09k
  {
401
2.09k
    absl::MutexLock lock(&async_tasks_mutex_);
402
2.09k
    id_to_async_tasks_copy = std::vector<std::pair<int, std::shared_ptr<Task>>>(
403
2.09k
        id_to_async_tasks_.begin(), id_to_async_tasks_.end());
404
2.09k
  }
405
2.09k
  for (const auto& [task_id, task] : id_to_async_tasks_copy) {
406
2.09k
    absl::Time last_run_time = task->GetLastRunTime();
407
2.09k
    if (last_run_time == absl::InfiniteFuture()) {
408
0
      continue;
409
0
    }
410
2.09k
    absl::Duration time_since_last_run = clock_->Now() - last_run_time;
411
2.09k
    if (time_since_last_run > task->GetTaskTimeout()) {
412
0
      tasks_to_cancel.push_back({task_id, task->GetTaskTimeout()});
413
0
    }
414
2.09k
  }
415
2.09k
  for (const auto& [task_id, task_timeout] : tasks_to_cancel) {
416
0
    LOG(ERROR) << "Cleanup inactive task: " << task_id
417
0
               << " as it has not run for the last " << task_timeout;
418
0
    Cancel(task_id);
419
0
  }
420
2.09k
}
421
422
4.18k
void TaskScheduler::Stop() {
423
4.18k
  if (!stop_.exchange(true)) {
424
2.09k
    if (alive_token_ != nullptr) {
425
2.09k
      absl::MutexLock lock(&alive_token_->mu);
426
2.09k
      alive_token_->alive = false;
427
2.09k
    }
428
2.09k
    {
429
2.09k
      absl::MutexLock lock(&async_tasks_mutex_);
430
4.18k
      for (const auto& [task_id, task] : id_to_async_tasks_) {
431
4.18k
        task->Cancel();
432
4.18k
      }
433
2.09k
      id_to_async_tasks_.clear();
434
2.09k
    }
435
2.09k
    io_->stop();
436
2.09k
  }
437
438
4.18k
  {
439
4.18k
    absl::MutexLock lock(&thread_join_mutex_);
440
4.18k
    if (async_executor_thread_.joinable() &&
441
2.09k
        std::this_thread::get_id() != async_executor_thread_.get_id()) {
442
2.09k
      async_executor_thread_.join();
443
2.09k
    }
444
4.18k
  }
445
4.18k
}
446
447
4.18k
size_t TaskScheduler::GetSchedulingAccuracyPercentage() const {
448
  // Copy id_to_async_tasks_ to avoid holding the lock while calculating
449
  // accuracy.
450
4.18k
  absl::flat_hash_map<int, std::shared_ptr<Task>> id_to_async_tasks_copy;
451
4.18k
  {
452
4.18k
    absl::MutexLock lock(&async_tasks_mutex_);
453
4.18k
    id_to_async_tasks_copy = id_to_async_tasks_;
454
4.18k
  }
455
456
4.18k
  if (id_to_async_tasks_copy.empty()) {
457
    // If there are no tasks, we consider the scheduler to be accurate.
458
0
    return 100;
459
0
  }
460
461
4.18k
  double total_inaccuracy = 0.0;
462
463
8.37k
  for (const auto& [task_id, task] : id_to_async_tasks_copy) {
464
8.37k
    absl::Duration average_wait_time = task->GetAverageWaitTime();
465
8.37k
    double task_inaccuracy;
466
8.37k
    if (average_wait_time == absl::InfiniteDuration()) {
467
      // No wait times have been recorded yet. This happens when the task is yet
468
      // to be scheduled after it's initial run.
469
      // We don't want to penalize the scheduler for this case. So we consider
470
      // the inaccuracy to be 0.
471
8.37k
      continue;
472
8.37k
    }
473
0
    absl::Duration period = task->GetPeriod();
474
0
    if (average_wait_time <= period) {
475
0
      task_inaccuracy = 0;
476
0
    } else {
477
      // Inaccuracy based on how much *later* than the period it waited
478
0
      absl::Duration lateness = average_wait_time - period;
479
0
      DLOG(INFO) << "Task " << task_id << " lateness: " << lateness;
480
      // Define how inaccuracy scales with lateness.
481
0
      task_inaccuracy = (absl::ToDoubleNanoseconds(lateness) /
482
0
                         absl::ToDoubleNanoseconds(period)) *
483
0
                        100.0;
484
      // Ensure inaccuracy is not negative (shouldn't happen here but for
485
      // safety)
486
0
      task_inaccuracy = std::max(0.0, std::min(100.0, task_inaccuracy));
487
0
    }
488
0
    total_inaccuracy += task_inaccuracy;
489
0
    DLOG(INFO) << "Task " << task_id << " has period: " << period
490
0
               << " average wait time: " << average_wait_time
491
0
               << " with inaccuracy: " << task_inaccuracy
492
0
               << "% (based on being late)";
493
0
  }
494
495
4.18k
  size_t overall_inaccuracy_percentage = 0;
496
4.18k
  overall_inaccuracy_percentage = static_cast<size_t>(
497
4.18k
      total_inaccuracy / static_cast<double>(id_to_async_tasks_copy.size()));
498
4.18k
  size_t overall_accuracy_percentage = 100 - overall_inaccuracy_percentage;
499
12.5k
  DLOG(INFO) << "Overall accuracy percentage: " << overall_accuracy_percentage;
500
4.18k
  return overall_accuracy_percentage;
501
4.18k
}
502
503
4.18k
nlohmann::json TaskScheduler::ToJson() const {
504
4.18k
  nlohmann::json json;
505
4.18k
  json["scheduling_accuracy_percentage"] = GetSchedulingAccuracyPercentage();
506
4.18k
  json["scheduler_tasks"] = nlohmann::json::array();
507
4.18k
  json["user_tasks"] = nlohmann::json::array();
508
4.18k
  absl::MutexLock lock(&async_tasks_mutex_);
509
8.37k
  for (const auto& [task_id, task] : id_to_async_tasks_) {
510
8.37k
    if (task_id == cleanup_task_id_) {
511
4.18k
      json["scheduler_tasks"].push_back(task->ToJson());
512
4.18k
      continue;
513
4.18k
    }
514
4.18k
    json["user_tasks"].push_back(task->ToJson());
515
4.18k
  }
516
4.18k
  json["user_async_task_count"] = json["user_tasks"].size();
517
4.18k
  json["total_async_task_count"] = id_to_async_tasks_.size();
518
4.18k
  return json;
519
4.18k
}
520
521
}  // namespace milotic_tlbmc