Coverage Report

Created: 2026-07-26 07:11

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/gbmcweb/tlbmc/scheduler/scheduler.cc
Line
Count
Source
1
#include "tlbmc/scheduler/scheduler.h"
2
3
#include <algorithm>
4
#include <cstddef>
5
#include <memory>
6
#include <optional>
7
#include <thread>
8
#include <utility>
9
#include <vector>
10
11
#include "absl/container/flat_hash_map.h"
12
#include "absl/functional/any_invocable.h"
13
#include "absl/log/log.h"
14
#include "absl/synchronization/mutex.h"
15
#include "absl/time/time.h"
16
#include "boost/asio/executor.hpp"
17
#include "boost/asio/executor_work_guard.hpp"
18
#include "boost/asio/steady_timer.hpp"  // NOLINT: boost is commonly used in BMC
19
#include "time/clock.h"
20
#include "nlohmann/json.hpp"
21
22
namespace milotic_tlbmc {
23
24
8.16k
nlohmann::json TimeInfo::ToJson() const {
25
8.16k
  nlohmann::json json;
26
8.16k
  json["last_scheduled_time"] = absl::FormatTime(last_scheduled_time);
27
8.16k
  json["last_run_time"] = absl::FormatTime(last_run_time);
28
  // Get the last 10 wait times.
29
8.16k
  auto it = wait_times.rbegin();
30
8.16k
  auto& wait_times_recent_to_oldest = json["wait_times_recent_to_oldest"];
31
8.16k
  wait_times_recent_to_oldest = nlohmann::json::array();
32
8.16k
  for (int i = 0; i < 10 && it != wait_times.rend(); ++i, ++it) {
33
0
    wait_times_recent_to_oldest.push_back(absl::FormatDuration(*it));
34
0
  }
35
8.16k
  return json;
36
8.16k
}
37
38
8.16k
nlohmann::json Task::ToJson() const {
39
8.16k
  nlohmann::json json;
40
8.16k
  json["task_id"] = task_id_;
41
8.16k
  json["period"] = absl::FormatDuration(period_);
42
8.16k
  json["execution_count"] = execution_count_.load();
43
8.16k
  {
44
8.16k
    absl::MutexLock lock(&time_info_mutex_);
45
8.16k
    json["time_info"] = time_info_.ToJson();
46
8.16k
  }
47
8.16k
  json["average_wait_time"] = absl::FormatDuration(GetAverageWaitTime());
48
8.16k
  json["cancelled"] = cancelled_.load();
49
8.16k
  json["task_mode"] = task_mode_ == TaskMode::kPeriodic ? "periodic" : "once";
50
8.16k
  return json;
51
8.16k
}
52
53
// Overload constructor for tasks that need to be invoked with an ack callback.
54
Task::Task(int task_id,
55
           absl::AnyInvocable<void(absl::AnyInvocable<void()>)> task_fn,
56
           absl::Duration period, boost::asio::io_context* io,
57
           absl::Duration task_timeout, TaskMode task_mode,
58
           ecclesia::Clock* clock)
59
4.08k
    : task_id_(task_id),
60
4.08k
      task_mode_(task_mode),
61
4.08k
      clock_(clock),
62
4.08k
      period_(period),
63
4.08k
      cancelled_(false),
64
4.08k
      task_fn_with_ack_(std::move(task_fn)),
65
4.08k
      timer_(std::make_unique<boost::asio::steady_timer>(*io)),
66
4.08k
      task_timeout_(task_timeout) {}
67
68
0
absl::Duration Task::GetPeriod() const { return period_.load(); }
69
70
16.3k
absl::Duration Task::GetAverageWaitTime() const {
71
16.3k
  absl::MutexLock lock(&time_info_mutex_);
72
16.3k
  if (time_info_.wait_times.empty()) {
73
    // No wait times have been recorded yet. This happens when the task is yet
74
    // to be scheduled after it's initial run.
75
16.3k
    return absl::InfiniteDuration();
76
16.3k
  }
77
0
  absl::Duration total_wait_time = absl::ZeroDuration();
78
0
  for (const auto& wait_time : time_info_.wait_times) {
79
0
    total_wait_time += wait_time;
80
0
  }
81
0
  return total_wait_time / time_info_.wait_times.size();
82
16.3k
}
83
84
6.12k
absl::Time Task::GetLastRunTime() const {
85
6.12k
  absl::MutexLock lock(&time_info_mutex_);
86
6.12k
  return time_info_.last_run_time;
87
6.12k
}
88
89
4.08k
void Task::ScheduleNextRun() {
90
4.08k
  {
91
4.08k
    absl::MutexLock lock(&time_info_mutex_);
92
4.08k
    if (clock_ != nullptr) {
93
4.08k
      time_info_.last_scheduled_time = clock_->Now();
94
4.08k
    }
95
4.08k
  }
96
97
4.08k
  {
98
4.08k
    absl::MutexLock lock(&timer_mutex_);
99
4.08k
    if (cancelled_.load()) {
100
0
      return;
101
0
    }
102
103
4.08k
    last_execution_duration_ = GetLastRunTime() == absl::InfiniteFuture()
104
4.08k
                                   ? absl::ZeroDuration()
105
4.08k
                                   : clock_->Now() - GetLastRunTime();
106
4.08k
    absl::Duration wait_time = period_ - last_execution_duration_;
107
4.08k
    if (wait_time < absl::ZeroDuration()) {
108
0
      wait_time = absl::ZeroDuration();
109
0
    }
110
4.08k
    timer_->expires_after(absl::ToChronoNanoseconds(wait_time));
111
4.08k
    timer_->async_wait([weak_self = std::weak_ptr<Task>(shared_from_this())](
112
4.08k
                           boost::system::error_code error) {
113
18
      if (error == boost::asio::error::operation_aborted) {
114
54
        DLOG(ERROR) << "Failed to wait for task to complete with error: "
115
54
                    << error.message();
116
18
        return;
117
18
      }
118
0
      std::shared_ptr<Task> task = weak_self.lock();
119
0
      if (task == nullptr) {
120
0
        DLOG(WARNING) << "Task is cancelled";
121
0
        return;
122
0
      }
123
0
      task->Run();
124
0
    });
125
4.08k
  }
126
4.08k
}
127
128
2.04k
void Task::OnDone() {
129
2.04k
  ++execution_count_;
130
2.04k
  ScheduleNextRun();
131
2.04k
}
132
133
// Runs the task and updates the next run time.
134
2.04k
void Task::Run() {
135
2.04k
  if (cancelled_.load()) {
136
0
    DLOG(WARNING) << "Task is cancelled before running!";
137
0
    return;
138
0
  }
139
140
2.04k
  {
141
2.04k
    absl::MutexLock lock(&time_info_mutex_);
142
2.04k
    absl::Time now = clock_->Now();
143
2.04k
    if (time_info_.last_scheduled_time != absl::InfiniteFuture()) {
144
0
      time_info_.wait_times.push_back(now - time_info_.last_scheduled_time);
145
0
    }
146
2.04k
    time_info_.last_run_time = now;
147
2.04k
  }
148
149
2.04k
  {
150
2.04k
    absl::MutexLock lock(&task_mutex_);
151
2.04k
    if (task_fn_with_ack_ != nullptr) {
152
2.04k
      task_fn_with_ack_(
153
2.04k
          [weak_self = std::weak_ptr<Task>(shared_from_this())]() {
154
2.04k
            std::shared_ptr<Task> task = weak_self.lock();
155
2.04k
            if (task == nullptr) {
156
0
              DLOG(WARNING) << "Task is destroyed while waiting for ack!";
157
0
              return;
158
0
            }
159
2.04k
            task->OnDone();
160
2.04k
          });
161
2.04k
      return;
162
2.04k
    }
163
2.04k
  }
164
2.04k
}
165
166
// Cancels the task.
167
4.08k
void Task::Cancel() {
168
4.08k
  cancelled_.store(true);
169
4.08k
  absl::MutexLock lock(&timer_mutex_);
170
4.08k
  timer_->cancel();
171
4.08k
}
172
173
0
void Task::SetPeriod(absl::Duration period) {
174
0
  Cancel();
175
0
  period_ = period;
176
0
  cancelled_.store(false);
177
0
  ScheduleNextRun();
178
0
}
179
180
TaskScheduler::TaskScheduler(ecclesia::Clock* clock, const Options& options)
181
2.04k
    : next_task_id_(1),
182
2.04k
      clock_(clock),
183
2.04k
      io_(std::make_unique<boost::asio::io_context>()),
184
2.04k
      work_guard_(io_->get_executor()),
185
2.04k
      async_executor_thread_([this]() { io_->run(); }),
186
2.04k
      default_task_timeout_(options.default_task_timeout) {
187
2.04k
  cleanup_task_id_ = RunAndScheduleAsync(
188
2.04k
      [this](absl::AnyInvocable<void()> on_done) {
189
2.04k
        CleanupInactiveTasks();
190
2.04k
        on_done();
191
2.04k
      },
192
2.04k
      options.cleanup_tasks_period);
193
2.04k
}
194
195
2.04k
TaskScheduler::~TaskScheduler() { Stop(); }
196
197
int TaskScheduler::ScheduleAsyncInternal(
198
    absl::AnyInvocable<void(absl::AnyInvocable<void()>)> task,
199
    absl::Duration period, std::optional<absl::Duration> timeout,
200
4.08k
    ExecutionMode execution_mode) {
201
4.08k
  if (task == nullptr) {
202
0
    LOG(ERROR) << "Task is null";
203
0
    return -1;
204
0
  }
205
206
4.08k
  if (stop_.load()) {
207
0
    LOG(ERROR) << "TaskScheduler is stopped, cannot schedule async task";
208
0
    return -1;
209
0
  }
210
211
4.08k
  if (period == absl::ZeroDuration()) {
212
0
    LOG(ERROR) << "Task period is zero, cannot schedule async recurring task";
213
0
    return -1;
214
0
  }
215
216
4.08k
  int task_id = next_task_id_++;
217
4.08k
  absl::Duration task_timeout = timeout.value_or(default_task_timeout_);
218
4.08k
  auto task_ptr =
219
4.08k
      std::make_shared<Task>(task_id, std::move(task), period, io_.get(),
220
4.08k
                             task_timeout, TaskMode::kPeriodic, clock_);
221
4.08k
  if (execution_mode == ExecutionMode::kRunImmediately) {
222
2.04k
    task_ptr->Run();
223
2.04k
  } else if (execution_mode == ExecutionMode::kRunAfterScheduling) {
224
2.04k
    task_ptr->ScheduleNextRun();
225
2.04k
  }
226
4.08k
  absl::MutexLock lock(&async_tasks_mutex_);
227
4.08k
  id_to_async_tasks_.emplace(task_id, task_ptr);
228
4.08k
  return task_id;
229
4.08k
}
230
231
int TaskScheduler::ScheduleAsync(
232
    absl::AnyInvocable<void(absl::AnyInvocable<void()>)> task,
233
2.04k
    absl::Duration period, std::optional<absl::Duration> timeout) {
234
2.04k
  return ScheduleAsyncInternal(std::move(task), period, timeout,
235
2.04k
                               ExecutionMode::kRunAfterScheduling);
236
2.04k
}
237
238
int TaskScheduler::RunAndScheduleAsync(
239
    absl::AnyInvocable<void(absl::AnyInvocable<void()>)> task,
240
2.04k
    absl::Duration period, std::optional<absl::Duration> timeout) {
241
2.04k
  return ScheduleAsyncInternal(std::move(task), period, timeout,
242
2.04k
                               ExecutionMode::kRunImmediately);
243
2.04k
}
244
245
int TaskScheduler::ScheduleOneShotAsync(
246
    absl::AnyInvocable<void(absl::AnyInvocable<void()>)> task,
247
0
    absl::Duration period, std::optional<absl::Duration> timeout) {
248
0
  if (task == nullptr) {
249
0
    LOG(ERROR) << "Task is null";
250
0
    return -1;
251
0
  }
252
253
0
  if (stop_.load()) {
254
0
    LOG(ERROR) << "TaskScheduler is stopped, cannot run and schedule task";
255
0
    return -1;
256
0
  }
257
258
0
  if (period == absl::ZeroDuration()) {
259
0
    LOG(ERROR) << "Task period is zero, cannot schedule task";
260
0
    return -1;
261
0
  }
262
263
0
  int task_id = next_task_id_++;
264
0
  absl::Duration task_timeout = timeout.value_or(default_task_timeout_);
265
266
0
  auto oneshot_on_done = [this, task_id]() { Cancel(task_id); };
267
0
  auto task_ptr = std::make_shared<Task>(
268
0
      task_id,
269
0
      [task = std::move(task), oneshot_on_done = std::move(oneshot_on_done)](
270
0
          absl::AnyInvocable<void()> on_done) mutable {
271
0
        task(std::move(oneshot_on_done));
272
0
      },
273
0
      period, io_.get(), task_timeout, TaskMode::kOnce, clock_);
274
0
  {
275
0
    absl::MutexLock lock(&async_tasks_mutex_);
276
0
    id_to_async_tasks_.emplace(task_id, task_ptr);
277
0
  }
278
0
  task_ptr->ScheduleNextRun();
279
280
0
  return task_id;
281
0
}
282
283
absl::flat_hash_map<int, std::shared_ptr<Task>> TaskScheduler::GetSnapshot()
284
0
    const {
285
0
  absl::flat_hash_map<int, std::shared_ptr<Task>> snapshot;
286
0
  {
287
0
    absl::MutexLock lock(&async_tasks_mutex_);
288
0
    for (const auto& [task_id, task] : id_to_async_tasks_) {
289
0
      snapshot.emplace(task_id, task);
290
0
    }
291
0
  }
292
0
  return snapshot;
293
0
}
294
295
0
size_t TaskScheduler::GetAllUserTaskCount() const {
296
0
  absl::MutexLock lock(&async_tasks_mutex_);
297
  // We always have one internal task scheduled for cleanup.
298
0
  return id_to_async_tasks_.size() - 1;
299
0
}
300
301
0
size_t TaskScheduler::GetAllTaskCount() const {
302
0
  absl::MutexLock lock(&async_tasks_mutex_);
303
0
  return id_to_async_tasks_.size();
304
0
}
305
306
0
void TaskScheduler::UpdateTaskPeriod(int task_id, absl::Duration period) {
307
0
  if (stop_.load()) {
308
0
    LOG(WARNING) << "TaskScheduler is stopped, cannot update task period: "
309
0
                 << task_id;
310
0
    return;
311
0
  }
312
313
0
  if (period == absl::ZeroDuration()) {
314
0
    LOG(ERROR) << "Cannot set async task period to zero";
315
0
    return;
316
0
  }
317
318
0
  absl::MutexLock lock(&async_tasks_mutex_);
319
0
  auto it = id_to_async_tasks_.find(task_id);
320
0
  if (it != id_to_async_tasks_.end()) {
321
0
    if (it->second->GetPeriod() == period) {
322
0
      return;
323
0
    }
324
0
    LOG(WARNING) << "Updating task period for task: " << task_id << " from "
325
0
                 << it->second->GetPeriod() << " to " << period;
326
0
    it->second->SetPeriod(period);
327
0
  }
328
0
}
329
330
0
void TaskScheduler::Cancel(int task_id) {
331
0
  if (stop_.load()) {
332
0
    LOG(WARNING) << "TaskScheduler is stopped, cannot cancel task: " << task_id;
333
0
    return;
334
0
  }
335
336
  // Cancel the async task.
337
0
  {
338
0
    absl::MutexLock lock(&async_tasks_mutex_);
339
0
    auto it = id_to_async_tasks_.find(task_id);
340
0
    if (it != id_to_async_tasks_.end()) {
341
0
      it->second->Cancel();
342
      // Remove the task from the tasks map.
343
0
      id_to_async_tasks_.erase(it);
344
0
    }
345
0
  }
346
0
}
347
348
2.04k
void TaskScheduler::CancelAll() {
349
2.04k
  if (stop_.load()) {
350
0
    LOG(ERROR) << "TaskScheduler is stopped, cannot cancel all tasks";
351
0
    return;
352
0
  }
353
354
2.04k
  {
355
2.04k
    absl::MutexLock lock(&async_tasks_mutex_);
356
4.08k
    for (const auto& [task_id, task] : id_to_async_tasks_) {
357
4.08k
      task->Cancel();
358
4.08k
    }
359
2.04k
    id_to_async_tasks_.clear();
360
2.04k
  }
361
2.04k
}
362
363
2.04k
void TaskScheduler::CleanupInactiveTasks() {
364
2.04k
  if (stop_.load()) {
365
0
    return;
366
0
  }
367
368
  // Copy id_to_async_tasks_ to a vector to avoid holding the lock while
369
  // cancelling tasks.
370
2.04k
  std::vector<std::pair<int, std::shared_ptr<Task>>> id_to_async_tasks_copy;
371
  // Task ids of Tasks to cancel.
372
2.04k
  std::vector<int> tasks_to_cancel;
373
2.04k
  {
374
2.04k
    absl::MutexLock lock(&async_tasks_mutex_);
375
2.04k
    id_to_async_tasks_copy = std::vector<std::pair<int, std::shared_ptr<Task>>>(
376
2.04k
        id_to_async_tasks_.begin(), id_to_async_tasks_.end());
377
2.04k
  }
378
2.04k
  for (const auto& [task_id, task] : id_to_async_tasks_copy) {
379
0
    absl::Time last_run_time = task->GetLastRunTime();
380
0
    if (last_run_time == absl::InfiniteFuture()) {
381
0
      continue;
382
0
    }
383
0
    absl::Duration time_since_last_run = clock_->Now() - last_run_time;
384
0
    if (time_since_last_run > task->GetTaskTimeout()) {
385
0
      tasks_to_cancel.push_back(task_id);
386
0
    }
387
0
  }
388
2.04k
  for (int task_id : tasks_to_cancel) {
389
0
    LOG(ERROR) << "Cleanup inactive task: " << task_id
390
0
               << " as it has not run for the last " << default_task_timeout_;
391
0
    Cancel(task_id);
392
0
  }
393
2.04k
}
394
395
4.08k
void TaskScheduler::Stop() {
396
4.08k
  if (stop_.load()) {
397
    // Already stopped.
398
2.04k
    return;
399
2.04k
  }
400
401
2.04k
  CancelAll();
402
2.04k
  stop_.store(true);
403
404
2.04k
  if (async_executor_thread_.joinable()) {
405
2.04k
    io_->stop();
406
2.04k
    async_executor_thread_.join();
407
2.04k
  }
408
2.04k
}
409
410
4.08k
size_t TaskScheduler::GetSchedulingAccuracyPercentage() const {
411
  // Copy id_to_async_tasks_ to avoid holding the lock while calculating
412
  // accuracy.
413
4.08k
  absl::flat_hash_map<int, std::shared_ptr<Task>> id_to_async_tasks_copy;
414
4.08k
  {
415
4.08k
    absl::MutexLock lock(&async_tasks_mutex_);
416
4.08k
    id_to_async_tasks_copy = id_to_async_tasks_;
417
4.08k
  }
418
419
4.08k
  if (id_to_async_tasks_copy.empty()) {
420
    // If there are no tasks, we consider the scheduler to be accurate.
421
0
    return 100;
422
0
  }
423
424
4.08k
  double total_inaccuracy = 0.0;
425
426
8.16k
  for (const auto& [task_id, task] : id_to_async_tasks_copy) {
427
8.16k
    absl::Duration average_wait_time = task->GetAverageWaitTime();
428
8.16k
    double task_inaccuracy;
429
8.16k
    if (average_wait_time == absl::InfiniteDuration()) {
430
      // No wait times have been recorded yet. This happens when the task is yet
431
      // to be scheduled after it's initial run.
432
      // We don't want to penalize the scheduler for this case. So we consider
433
      // the inaccuracy to be 0.
434
8.16k
      continue;
435
8.16k
    }
436
0
    absl::Duration period = task->GetPeriod();
437
0
    if (average_wait_time <= period) {
438
0
      task_inaccuracy = 0;
439
0
    } else {
440
      // Inaccuracy based on how much *later* than the period it waited
441
0
      absl::Duration lateness = average_wait_time - period;
442
0
      DLOG(INFO) << "Task " << task_id << " lateness: " << lateness;
443
      // Define how inaccuracy scales with lateness.
444
0
      task_inaccuracy = (absl::ToDoubleNanoseconds(lateness) /
445
0
                         absl::ToDoubleNanoseconds(period)) *
446
0
                        100.0;
447
      // Ensure inaccuracy is not negative (shouldn't happen here but for
448
      // safety)
449
0
      task_inaccuracy = std::max(0.0, std::min(100.0, task_inaccuracy));
450
0
    }
451
0
    total_inaccuracy += task_inaccuracy;
452
0
    DLOG(INFO) << "Task " << task_id << " has period: " << period
453
0
               << " average wait time: " << average_wait_time
454
0
               << " with inaccuracy: " << task_inaccuracy
455
0
               << "% (based on being late)";
456
0
  }
457
458
4.08k
  size_t overall_inaccuracy_percentage = 0;
459
4.08k
  overall_inaccuracy_percentage = static_cast<size_t>(
460
4.08k
      total_inaccuracy / static_cast<double>(id_to_async_tasks_copy.size()));
461
4.08k
  size_t overall_accuracy_percentage = 100 - overall_inaccuracy_percentage;
462
12.2k
  DLOG(INFO) << "Overall accuracy percentage: " << overall_accuracy_percentage;
463
4.08k
  return overall_accuracy_percentage;
464
4.08k
}
465
466
4.08k
nlohmann::json TaskScheduler::ToJson() const {
467
4.08k
  nlohmann::json json;
468
4.08k
  json["scheduling_accuracy_percentage"] = GetSchedulingAccuracyPercentage();
469
4.08k
  absl::MutexLock lock(&async_tasks_mutex_);
470
8.16k
  for (const auto& [task_id, task] : id_to_async_tasks_) {
471
8.16k
    if (task_id == cleanup_task_id_) {
472
4.08k
      json["scheduler_tasks"].push_back(task->ToJson());
473
4.08k
      continue;
474
4.08k
    }
475
4.08k
    json["user_tasks"].push_back(task->ToJson());
476
4.08k
  }
477
4.08k
  json["user_async_task_count"] = id_to_async_tasks_.size() - 1;
478
4.08k
  json["total_async_task_count"] = id_to_async_tasks_.size();
479
4.08k
  return json;
480
4.08k
}
481
482
}  // namespace milotic_tlbmc