/src/ruby/vm_sync.c

Source
#include "internal/gc.h"
#include "internal/thread.h"
#include "vm_core.h"
#include "vm_sync.h"
#include "ractor_core.h"
#include "vm_debug.h"

void rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr);
void rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr);
void rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr);

static bool
vm_locked(rb_vm_t *vm)
{
    return vm_locked_by_ractor_p(vm, GET_RACTOR());
}

#if RUBY_DEBUG > 0
void
RUBY_ASSERT_vm_locking(void)
{
    if (rb_multi_ractor_p()) {
        rb_vm_t *vm = GET_VM();
        VM_ASSERT(vm_locked(vm));
    }
}

void
RUBY_ASSERT_vm_locking_with_barrier(void)
{
    if (rb_multi_ractor_p()) {
        rb_vm_t *vm = GET_VM();
        VM_ASSERT(vm_locked(vm));

        if (vm->ractor.cnt > 1) {
            /* Written to only when holding both ractor.sync and ractor.sched lock */
            VM_ASSERT(vm->ractor.sched.barrier_waiting);
        }
    }
}

void
RUBY_ASSERT_vm_unlocking(void)
{
    if (rb_multi_ractor_p()) {
        rb_vm_t *vm = GET_VM();
        VM_ASSERT(!vm_locked(vm));
    }
}
#endif

bool
rb_vm_locked_p(void)
{
    return vm_locked(GET_VM());
}

static bool
vm_need_barrier_waiting(const rb_vm_t *vm)
{
#ifdef RUBY_THREAD_PTHREAD_H
    return vm->ractor.sched.barrier_waiting;
#else
    return vm->ractor.sync.barrier_waiting;
#endif
}

static bool
vm_need_barrier(bool no_barrier, const rb_ractor_t *cr, const rb_vm_t *vm)
{
#ifdef RUBY_THREAD_PTHREAD_H
    return !no_barrier && cr->threads.sched.running != NULL && vm_need_barrier_waiting(vm); // ractor has running threads.
#else
    return !no_barrier && vm_need_barrier_waiting(vm);
#endif
}

static void
vm_lock_enter(rb_ractor_t *cr, rb_vm_t *vm, bool locked, bool no_barrier, unsigned int *lev APPEND_LOCATION_ARGS)
{
    RUBY_DEBUG_LOG2(file, line, "start locked:%d", locked);

    if (locked) {
        ASSERT_vm_locking();
    }
    else {
#if RACTOR_CHECK_MODE
        // locking ractor and acquire VM lock will cause deadlock
        VM_ASSERT(cr->sync.locked_by != rb_ractor_self(cr));
#endif
        // lock
        rb_native_mutex_lock(&vm->ractor.sync.lock);
        VM_ASSERT(vm->ractor.sync.lock_owner == NULL);
        VM_ASSERT(vm->ractor.sync.lock_rec == 0);

        // barrier
        if (vm_need_barrier(no_barrier, cr, vm)) {
            rb_execution_context_t *ec = GET_EC();
            RB_VM_SAVE_MACHINE_CONTEXT(rb_ec_thread_ptr(ec));

            do {
                VM_ASSERT(vm_need_barrier_waiting(vm));
                RUBY_DEBUG_LOG("barrier serial:%u", vm->ractor.sched.barrier_serial);
                rb_ractor_sched_barrier_join(vm, cr);
            } while (vm_need_barrier_waiting(vm));
        }

        VM_ASSERT(vm->ractor.sync.lock_rec == 0);
        VM_ASSERT(vm->ractor.sync.lock_owner == NULL);
        vm->ractor.sync.lock_owner = cr;
    }

    vm->ractor.sync.lock_rec++;
    *lev = vm->ractor.sync.lock_rec;

    RUBY_DEBUG_LOG2(file, line, "rec:%u owner:%u", vm->ractor.sync.lock_rec,
                    (unsigned int)rb_ractor_id(vm->ractor.sync.lock_owner));
}

static void
vm_lock_leave(rb_vm_t *vm, bool no_barrier, unsigned int *lev APPEND_LOCATION_ARGS)
{
    MAYBE_UNUSED(rb_ractor_t *cr = vm->ractor.sync.lock_owner);

    RUBY_DEBUG_LOG2(file, line, "rec:%u owner:%u%s", vm->ractor.sync.lock_rec,
                    (unsigned int)rb_ractor_id(cr),
                    vm->ractor.sync.lock_rec == 1 ? " (leave)" : "");

    ASSERT_vm_locking();
    VM_ASSERT(vm->ractor.sync.lock_rec > 0);
    VM_ASSERT(vm->ractor.sync.lock_rec == *lev);
    VM_ASSERT(cr == GET_RACTOR());

#ifdef RUBY_THREAD_PTHREAD_H
    if (vm->ractor.sched.barrier_ractor == cr &&
        vm->ractor.sched.barrier_lock_rec == vm->ractor.sync.lock_rec) {
        VM_ASSERT(!no_barrier);
        rb_ractor_sched_barrier_end(vm, cr);
    }
#endif

    vm->ractor.sync.lock_rec--;
    *lev = vm->ractor.sync.lock_rec;

    if (vm->ractor.sync.lock_rec == 0) {
        vm->ractor.sync.lock_owner = NULL;
        rb_native_mutex_unlock(&vm->ractor.sync.lock);
    }
}

void
rb_vm_lock_enter_body(unsigned int *lev APPEND_LOCATION_ARGS)
{
    rb_vm_t *vm = GET_VM();
    if (vm_locked(vm)) {
        vm_lock_enter(NULL, vm, true, false, lev APPEND_LOCATION_PARAMS);
    }
    else {
        vm_lock_enter(GET_RACTOR(), vm, false, false, lev APPEND_LOCATION_PARAMS);
    }
}

void
rb_vm_lock_enter_body_nb(unsigned int *lev APPEND_LOCATION_ARGS)
{
    rb_vm_t *vm = GET_VM();
    if (vm_locked(vm)) {
        vm_lock_enter(NULL, vm, true, true, lev APPEND_LOCATION_PARAMS);
    }
    else {
        vm_lock_enter(GET_RACTOR(), vm, false, true, lev APPEND_LOCATION_PARAMS);
    }
}

void
rb_vm_lock_enter_body_cr(rb_ractor_t *cr, unsigned int *lev APPEND_LOCATION_ARGS)
{
    rb_vm_t *vm = GET_VM();
    vm_lock_enter(cr, vm, vm_locked(vm), false, lev APPEND_LOCATION_PARAMS);
}

void
rb_vm_lock_leave_body_nb(unsigned int *lev APPEND_LOCATION_ARGS)
{
    vm_lock_leave(GET_VM(), true, lev APPEND_LOCATION_PARAMS);
}

void
rb_vm_lock_leave_body(unsigned int *lev APPEND_LOCATION_ARGS)
{
    vm_lock_leave(GET_VM(),  false, lev APPEND_LOCATION_PARAMS);
}

void
rb_vm_lock_body(LOCATION_ARGS)
{
    rb_vm_t *vm = GET_VM();
    ASSERT_vm_unlocking();

    vm_lock_enter(GET_RACTOR(), vm, false, false, &vm->ractor.sync.lock_rec APPEND_LOCATION_PARAMS);
}

void
rb_vm_unlock_body(LOCATION_ARGS)
{
    rb_vm_t *vm = GET_VM();
    ASSERT_vm_locking();
    VM_ASSERT(vm->ractor.sync.lock_rec == 1);
    vm_lock_leave(vm, false, &vm->ractor.sync.lock_rec APPEND_LOCATION_PARAMS);
}

static void
vm_cond_wait(rb_vm_t *vm, rb_nativethread_cond_t *cond, unsigned long msec)
{
    ASSERT_vm_locking();
    unsigned int lock_rec = vm->ractor.sync.lock_rec;
    rb_ractor_t *cr = vm->ractor.sync.lock_owner;

    vm->ractor.sync.lock_rec = 0;
    vm->ractor.sync.lock_owner = NULL;
    if (msec > 0) {
        rb_native_cond_timedwait(cond, &vm->ractor.sync.lock, msec);
    }
    else {
        rb_native_cond_wait(cond, &vm->ractor.sync.lock);
    }
    vm->ractor.sync.lock_rec = lock_rec;
    vm->ractor.sync.lock_owner = cr;
}

void
rb_vm_cond_wait(rb_vm_t *vm, rb_nativethread_cond_t *cond)
{
    vm_cond_wait(vm, cond, 0);
}

void
rb_vm_cond_timedwait(rb_vm_t *vm, rb_nativethread_cond_t *cond, unsigned long msec)
{
    vm_cond_wait(vm, cond, msec);
}

static bool
vm_barrier_acquired_p(const rb_vm_t *vm, const rb_ractor_t *cr)
{
#ifdef RUBY_THREAD_PTHREAD_H
    return vm->ractor.sched.barrier_ractor == cr;
#else
    return false;
#endif
}

void
rb_vm_barrier(void)
{
    RB_DEBUG_COUNTER_INC(vm_sync_barrier);

    if (!rb_multi_ractor_p()) {
        // no other ractors
        return;
    }
    else {
        rb_vm_t *vm = GET_VM();
        rb_ractor_t *cr = vm->ractor.sync.lock_owner;

        ASSERT_vm_locking();
        VM_ASSERT(cr == GET_RACTOR());
        VM_ASSERT(rb_ractor_status_p(cr, ractor_running));

        if (vm_barrier_acquired_p(vm, cr)) {
            // already in barrier synchronization
            return;
        }
        else {
            VM_ASSERT(!vm->ractor.sched.barrier_waiting);
            rb_ractor_sched_barrier_start(vm, cr);
        }
    }
}

void
rb_ec_vm_lock_rec_release(const rb_execution_context_t *ec,
                          unsigned int recorded_lock_rec,
                          unsigned int current_lock_rec)
{
    VM_ASSERT(recorded_lock_rec != current_lock_rec);

    if (UNLIKELY(recorded_lock_rec > current_lock_rec)) {
        rb_bug("unexpected situation - recordd:%u current:%u",
               recorded_lock_rec, current_lock_rec);
    }
    else {
        while (recorded_lock_rec < current_lock_rec) {
            RB_VM_LOCK_LEAVE_LEV(&current_lock_rec);
        }
    }

    VM_ASSERT(recorded_lock_rec == rb_ec_vm_lock_rec(ec));
}

VALUE
rb_vm_lock_with_barrier(VALUE (*func)(void *args), void *args)
{
    VALUE result = 0;
    RB_VM_LOCKING() {
        rb_vm_barrier();
        result = func(args);
    }
    return result;
}

Coverage Report

Created: 2026-03-31 07:30

Line	Count	Source
1		#include "internal/gc.h"
2		#include "internal/thread.h"
3		#include "vm_core.h"
4		#include "vm_sync.h"
5		#include "ractor_core.h"
6		#include "vm_debug.h"
7
8		void rb_ractor_sched_barrier_start(rb_vm_t vm, rb_ractor_t cr);
9		void rb_ractor_sched_barrier_join(rb_vm_t vm, rb_ractor_t cr);
10		void rb_ractor_sched_barrier_end(rb_vm_t vm, rb_ractor_t cr);
11
12		static bool
13		vm_locked(rb_vm_t *vm)
14	278k	{
15	278k	return vm_locked_by_ractor_p(vm, GET_RACTOR());
16	278k	}
17
18		#if RUBY_DEBUG > 0
19		void
20		RUBY_ASSERT_vm_locking(void)
21		{
22		if (rb_multi_ractor_p()) {
23		rb_vm_t *vm = GET_VM();
24		VM_ASSERT(vm_locked(vm));
25		}
26		}
27
28		void
29		RUBY_ASSERT_vm_locking_with_barrier(void)
30		{
31		if (rb_multi_ractor_p()) {
32		rb_vm_t *vm = GET_VM();
33		VM_ASSERT(vm_locked(vm));
34
35		if (vm->ractor.cnt > 1) {
36		/* Written to only when holding both ractor.sync and ractor.sched lock */
37		VM_ASSERT(vm->ractor.sched.barrier_waiting);
38		}
39		}
40		}
41
42		void
43		RUBY_ASSERT_vm_unlocking(void)
44		{
45		if (rb_multi_ractor_p()) {
46		rb_vm_t *vm = GET_VM();
47		VM_ASSERT(!vm_locked(vm));
48		}
49		}
50		#endif
51
52		bool
53		rb_vm_locked_p(void)
54	0	{
55	0	return vm_locked(GET_VM());
56	0	}
57
58		static bool
59		vm_need_barrier_waiting(const rb_vm_t *vm)
60	278k	{
61	278k	#ifdef RUBY_THREAD_PTHREAD_H
62	278k	return vm->ractor.sched.barrier_waiting;
63		#else
64		return vm->ractor.sync.barrier_waiting;
65		#endif
66	278k	}
67
68		static bool
69		vm_need_barrier(bool no_barrier, const rb_ractor_t cr, const rb_vm_t vm)
70	278k	{
71	278k	#ifdef RUBY_THREAD_PTHREAD_H
72	278k	return !no_barrier && cr->threads.sched.running != NULL && vm_need_barrier_waiting(vm); // ractor has running threads.
73		#else
74		return !no_barrier && vm_need_barrier_waiting(vm);
75		#endif
76	278k	}
77
78		static void
79		vm_lock_enter(rb_ractor_t cr, rb_vm_t vm, bool locked, bool no_barrier, unsigned int *lev APPEND_LOCATION_ARGS)
80	278k	{
81	278k	RUBY_DEBUG_LOG2(file, line, "start locked:%d", locked);
82
83	278k	if (locked) {
84	0	ASSERT_vm_locking();
85	0	}
86	278k	else {
87		#if RACTOR_CHECK_MODE
88		// locking ractor and acquire VM lock will cause deadlock
89		VM_ASSERT(cr->sync.locked_by != rb_ractor_self(cr));
90		#endif
91		// lock
92	278k	rb_native_mutex_lock(&vm->ractor.sync.lock);
93	278k	VM_ASSERT(vm->ractor.sync.lock_owner == NULL);
94	278k	VM_ASSERT(vm->ractor.sync.lock_rec == 0);
95
96		// barrier
97	278k	if (vm_need_barrier(no_barrier, cr, vm)) {
98	0	rb_execution_context_t *ec = GET_EC();
99	0	RB_VM_SAVE_MACHINE_CONTEXT(rb_ec_thread_ptr(ec));
100
101	0	do {
102	0	VM_ASSERT(vm_need_barrier_waiting(vm));
103	0	RUBY_DEBUG_LOG("barrier serial:%u", vm->ractor.sched.barrier_serial);
104	0	rb_ractor_sched_barrier_join(vm, cr);
105	0	} while (vm_need_barrier_waiting(vm));
106	0	}
107
108	278k	VM_ASSERT(vm->ractor.sync.lock_rec == 0);
109	278k	VM_ASSERT(vm->ractor.sync.lock_owner == NULL);
110	278k	vm->ractor.sync.lock_owner = cr;
111	278k	}
112
113	278k	vm->ractor.sync.lock_rec++;
114	278k	*lev = vm->ractor.sync.lock_rec;
115
116	278k	RUBY_DEBUG_LOG2(file, line, "rec:%u owner:%u", vm->ractor.sync.lock_rec,
117	278k	(unsigned int)rb_ractor_id(vm->ractor.sync.lock_owner));
118	278k	}
119
120		static void
121		vm_lock_leave(rb_vm_t vm, bool no_barrier, unsigned int lev APPEND_LOCATION_ARGS)
122	278k	{
123	278k	MAYBE_UNUSED(rb_ractor_t *cr = vm->ractor.sync.lock_owner);
124
125	278k	RUBY_DEBUG_LOG2(file, line, "rec:%u owner:%u%s", vm->ractor.sync.lock_rec,
126	278k	(unsigned int)rb_ractor_id(cr),
127	278k	vm->ractor.sync.lock_rec == 1 ? " (leave)" : "");
128
129	278k	ASSERT_vm_locking();
130	278k	VM_ASSERT(vm->ractor.sync.lock_rec > 0);
131	278k	VM_ASSERT(vm->ractor.sync.lock_rec == *lev);
132	278k	VM_ASSERT(cr == GET_RACTOR());
133
134	278k	#ifdef RUBY_THREAD_PTHREAD_H
135	278k	if (vm->ractor.sched.barrier_ractor == cr &&
136	0	vm->ractor.sched.barrier_lock_rec == vm->ractor.sync.lock_rec) {
137	0	VM_ASSERT(!no_barrier);
138	0	rb_ractor_sched_barrier_end(vm, cr);
139	0	}
140	278k	#endif
141
142	278k	vm->ractor.sync.lock_rec--;
143	278k	*lev = vm->ractor.sync.lock_rec;
144
145	278k	if (vm->ractor.sync.lock_rec == 0) {
146	278k	vm->ractor.sync.lock_owner = NULL;
147	278k	rb_native_mutex_unlock(&vm->ractor.sync.lock);
148	278k	}
149	278k	}
150
151		void
152		rb_vm_lock_enter_body(unsigned int *lev APPEND_LOCATION_ARGS)
153	0	{
154	0	rb_vm_t *vm = GET_VM();
155	0	if (vm_locked(vm)) {
156	0	vm_lock_enter(NULL, vm, true, false, lev APPEND_LOCATION_PARAMS);
157	0	}
158	0	else {
159	0	vm_lock_enter(GET_RACTOR(), vm, false, false, lev APPEND_LOCATION_PARAMS);
160	0	}
161	0	}
162
163		void
164		rb_vm_lock_enter_body_nb(unsigned int *lev APPEND_LOCATION_ARGS)
165	0	{
166	0	rb_vm_t *vm = GET_VM();
167	0	if (vm_locked(vm)) {
168	0	vm_lock_enter(NULL, vm, true, true, lev APPEND_LOCATION_PARAMS);
169	0	}
170	0	else {
171	0	vm_lock_enter(GET_RACTOR(), vm, false, true, lev APPEND_LOCATION_PARAMS);
172	0	}
173	0	}
174
175		void
176		rb_vm_lock_enter_body_cr(rb_ractor_t cr, unsigned int lev APPEND_LOCATION_ARGS)
177	278k	{
178	278k	rb_vm_t *vm = GET_VM();
179	278k	vm_lock_enter(cr, vm, vm_locked(vm), false, lev APPEND_LOCATION_PARAMS);
180	278k	}
181
182		void
183		rb_vm_lock_leave_body_nb(unsigned int *lev APPEND_LOCATION_ARGS)
184	0	{
185	0	vm_lock_leave(GET_VM(), true, lev APPEND_LOCATION_PARAMS);
186	0	}
187
188		void
189		rb_vm_lock_leave_body(unsigned int *lev APPEND_LOCATION_ARGS)
190	278k	{
191	278k	vm_lock_leave(GET_VM(), false, lev APPEND_LOCATION_PARAMS);
192	278k	}
193
194		void
195		rb_vm_lock_body(LOCATION_ARGS)
196	0	{
197	0	rb_vm_t *vm = GET_VM();
198	0	ASSERT_vm_unlocking();
199
200	0	vm_lock_enter(GET_RACTOR(), vm, false, false, &vm->ractor.sync.lock_rec APPEND_LOCATION_PARAMS);
201	0	}
202
203		void
204		rb_vm_unlock_body(LOCATION_ARGS)
205	0	{
206	0	rb_vm_t *vm = GET_VM();
207	0	ASSERT_vm_locking();
208	0	VM_ASSERT(vm->ractor.sync.lock_rec == 1);
209	0	vm_lock_leave(vm, false, &vm->ractor.sync.lock_rec APPEND_LOCATION_PARAMS);
210	0	}
211
212		static void
213		vm_cond_wait(rb_vm_t vm, rb_nativethread_cond_t cond, unsigned long msec)
214	0	{
215	0	ASSERT_vm_locking();
216	0	unsigned int lock_rec = vm->ractor.sync.lock_rec;
217	0	rb_ractor_t *cr = vm->ractor.sync.lock_owner;
218
219	0	vm->ractor.sync.lock_rec = 0;
220	0	vm->ractor.sync.lock_owner = NULL;
221	0	if (msec > 0) {
222	0	rb_native_cond_timedwait(cond, &vm->ractor.sync.lock, msec);
223	0	}
224	0	else {
225	0	rb_native_cond_wait(cond, &vm->ractor.sync.lock);
226	0	}
227	0	vm->ractor.sync.lock_rec = lock_rec;
228	0	vm->ractor.sync.lock_owner = cr;
229	0	}
230
231		void
232		rb_vm_cond_wait(rb_vm_t vm, rb_nativethread_cond_t cond)
233	0	{
234	0	vm_cond_wait(vm, cond, 0);
235	0	}
236
237		void
238		rb_vm_cond_timedwait(rb_vm_t vm, rb_nativethread_cond_t cond, unsigned long msec)
239	0	{
240	0	vm_cond_wait(vm, cond, msec);
241	0	}
242
243		static bool
244		vm_barrier_acquired_p(const rb_vm_t vm, const rb_ractor_t cr)
245	0	{
246	0	#ifdef RUBY_THREAD_PTHREAD_H
247	0	return vm->ractor.sched.barrier_ractor == cr;
248		#else
249		return false;
250		#endif
251	0	}
252
253		void
254		rb_vm_barrier(void)
255	72.9k	{
256	72.9k	RB_DEBUG_COUNTER_INC(vm_sync_barrier);
257
258	72.9k	if (!rb_multi_ractor_p()) {
259		// no other ractors
260	72.9k	return;
261	72.9k	}
262	0	else {
263	0	rb_vm_t *vm = GET_VM();
264	0	rb_ractor_t *cr = vm->ractor.sync.lock_owner;
265
266	0	ASSERT_vm_locking();
267	0	VM_ASSERT(cr == GET_RACTOR());
268	0	VM_ASSERT(rb_ractor_status_p(cr, ractor_running));
269
270	0	if (vm_barrier_acquired_p(vm, cr)) {
271		// already in barrier synchronization
272	0	return;
273	0	}
274	0	else {
275	0	VM_ASSERT(!vm->ractor.sched.barrier_waiting);
276	0	rb_ractor_sched_barrier_start(vm, cr);
277	0	}
278	0	}
279	72.9k	}
280
281		void
282		rb_ec_vm_lock_rec_release(const rb_execution_context_t *ec,
283		unsigned int recorded_lock_rec,
284		unsigned int current_lock_rec)
285	0	{
286	0	VM_ASSERT(recorded_lock_rec != current_lock_rec);
287
288	0	if (UNLIKELY(recorded_lock_rec > current_lock_rec)) {
289	0	rb_bug("unexpected situation - recordd:%u current:%u",
290	0	recorded_lock_rec, current_lock_rec);
291	0	}
292	0	else {
293	0	while (recorded_lock_rec < current_lock_rec) {
294	0	RB_VM_LOCK_LEAVE_LEV(&current_lock_rec);
295	0	}
296	0	}
297
298	0	VM_ASSERT(recorded_lock_rec == rb_ec_vm_lock_rec(ec));
299	0	}
300
301		VALUE
302		rb_vm_lock_with_barrier(VALUE (func)(void args), void *args)
303	0	{
304	0	VALUE result = 0;
305	0	RB_VM_LOCKING() {
306	0	rb_vm_barrier();
307	0	result = func(args);
308	0	}
309	0	return result;
310	0	}