/src/swift-nio/Sources/NIOPosix/SelectorEpoll.swift
Line | Count | Source |
1 | | //===----------------------------------------------------------------------===// |
2 | | // |
3 | | // This source file is part of the SwiftNIO open source project |
4 | | // |
5 | | // Copyright (c) 2017-2024 Apple Inc. and the SwiftNIO project authors |
6 | | // Licensed under Apache License v2.0 |
7 | | // |
8 | | // See LICENSE.txt for license information |
9 | | // See CONTRIBUTORS.txt for the list of SwiftNIO project authors |
10 | | // |
11 | | // SPDX-License-Identifier: Apache-2.0 |
12 | | // |
13 | | //===----------------------------------------------------------------------===// |
14 | | |
15 | | #if !os(WASI) |
16 | | |
17 | | import NIOConcurrencyHelpers |
18 | | import NIOCore |
19 | | |
20 | | #if !SWIFTNIO_USE_IO_URING |
21 | | |
22 | | #if os(Linux) || os(Android) |
23 | | import CNIOLinux |
24 | | |
25 | | /// Represents the `epoll` filters/events we might use: |
26 | | /// |
27 | | /// - `hangup` corresponds to `EPOLLHUP` |
28 | | /// - `readHangup` corresponds to `EPOLLRDHUP` |
29 | | /// - `input` corresponds to `EPOLLIN` |
30 | | /// - `output` corresponds to `EPOLLOUT` |
31 | | /// - `error` corresponds to `EPOLLERR` |
32 | | private struct EpollFilterSet: OptionSet, Equatable { |
33 | | typealias RawValue = UInt8 |
34 | | |
35 | | let rawValue: RawValue |
36 | | |
37 | | static let _none = EpollFilterSet([]) |
38 | | static let hangup = EpollFilterSet(rawValue: 1 << 0) |
39 | | static let readHangup = EpollFilterSet(rawValue: 1 << 1) |
40 | | static let input = EpollFilterSet(rawValue: 1 << 2) |
41 | | static let output = EpollFilterSet(rawValue: 1 << 3) |
42 | | static let error = EpollFilterSet(rawValue: 1 << 4) |
43 | | |
44 | 0 | init(rawValue: RawValue) { |
45 | 0 | self.rawValue = rawValue |
46 | 0 | } |
47 | | } |
48 | | |
49 | | extension EpollFilterSet { |
50 | | /// Convert NIO's `SelectorEventSet` set to a `EpollFilterSet` |
51 | 0 | init(selectorEventSet: SelectorEventSet) { |
52 | 0 | var thing: EpollFilterSet = [.error, .hangup] |
53 | 0 | if selectorEventSet.contains(.read) { |
54 | 0 | thing.formUnion(.input) |
55 | 0 | } |
56 | 0 | if selectorEventSet.contains(.write) { |
57 | 0 | thing.formUnion(.output) |
58 | 0 | } |
59 | 0 | if selectorEventSet.contains(.readEOF) { |
60 | 0 | thing.formUnion(.readHangup) |
61 | 0 | } |
62 | 0 | self = thing |
63 | 0 | } |
64 | | } |
65 | | |
66 | | extension SelectorEventSet { |
67 | 0 | var epollEventSet: UInt32 { |
68 | 0 | assert(self != ._none) |
69 | 0 | // EPOLLERR | EPOLLHUP is always set unconditionally anyway but it's easier to understand if we explicitly ask. |
70 | 0 | var filter: UInt32 = Epoll.EPOLLERR | Epoll.EPOLLHUP |
71 | 0 | let epollFilters = EpollFilterSet(selectorEventSet: self) |
72 | 0 | if epollFilters.contains(.input) { |
73 | 0 | filter |= Epoll.EPOLLIN |
74 | 0 | } |
75 | 0 | if epollFilters.contains(.output) { |
76 | 0 | filter |= Epoll.EPOLLOUT |
77 | 0 | } |
78 | 0 | if epollFilters.contains(.readHangup) { |
79 | 0 | filter |= Epoll.EPOLLRDHUP |
80 | 0 | } |
81 | 0 | assert(filter & Epoll.EPOLLHUP != 0) // both of these are reported |
82 | 0 | assert(filter & Epoll.EPOLLERR != 0) // always and can't be masked. |
83 | 0 | return filter |
84 | 0 | } |
85 | | |
86 | | @inlinable |
87 | 0 | internal init(epollEvent: Epoll.epoll_event) { |
88 | 0 | var selectorEventSet: SelectorEventSet = ._none |
89 | 0 | if epollEvent.events & Epoll.EPOLLIN != 0 { |
90 | 0 | selectorEventSet.formUnion(.read) |
91 | 0 | } |
92 | 0 | if epollEvent.events & Epoll.EPOLLOUT != 0 { |
93 | 0 | selectorEventSet.formUnion(.write) |
94 | 0 | } |
95 | 0 | if epollEvent.events & Epoll.EPOLLRDHUP != 0 { |
96 | 0 | selectorEventSet.formUnion(.readEOF) |
97 | 0 | } |
98 | 0 | if epollEvent.events & Epoll.EPOLLERR != 0 { |
99 | 0 | selectorEventSet.formUnion(.error) |
100 | 0 | } |
101 | 0 | if epollEvent.events & Epoll.EPOLLHUP != 0 { |
102 | 0 | selectorEventSet.formUnion(.reset) |
103 | 0 | } |
104 | 0 | self = selectorEventSet |
105 | 0 | } |
106 | | } |
107 | | |
108 | | // EPollUserData supports (un)packing into an `UInt64` because epoll has a user info field that we can attach which is |
109 | | // up to 64 bits wide. We're using all of those 64 bits, 32 for a "registration ID" and 32 for the file descriptor. |
110 | | @usableFromInline struct EPollUserData { |
111 | | @usableFromInline var registrationID: SelectorRegistrationID |
112 | | @usableFromInline var fileDescriptor: CInt |
113 | | |
114 | 0 | @inlinable init(registrationID: SelectorRegistrationID, fileDescriptor: CInt) { |
115 | 0 | assert(MemoryLayout<UInt64>.size == MemoryLayout<EPollUserData>.size) |
116 | 0 | self.registrationID = registrationID |
117 | 0 | self.fileDescriptor = fileDescriptor |
118 | 0 | } |
119 | | |
120 | 0 | @inlinable init(rawValue: UInt64) { |
121 | 0 | let unpacked = IntegerBitPacking.unpackUInt32CInt(rawValue) |
122 | 0 | self = .init(registrationID: SelectorRegistrationID(rawValue: unpacked.0), fileDescriptor: unpacked.1) |
123 | 0 | } |
124 | | } |
125 | | |
126 | | extension UInt64 { |
127 | | @inlinable |
128 | 0 | init(_ epollUserData: EPollUserData) { |
129 | 0 | let fd = epollUserData.fileDescriptor |
130 | 0 | assert(fd >= 0, "\(fd) is not a valid file descriptor") |
131 | 0 | self = IntegerBitPacking.packUInt32CInt(epollUserData.registrationID.rawValue, fd) |
132 | 0 | } |
133 | | } |
134 | | |
135 | | extension Selector: _SelectorBackendProtocol { |
136 | 0 | func initialiseState0() throws { |
137 | 0 | self.selectorFD = try Epoll.epoll_create(size: 128) |
138 | 0 | self.eventFD = try EventFd.eventfd(initval: 0, flags: Int32(EventFd.EFD_CLOEXEC | EventFd.EFD_NONBLOCK)) |
139 | 0 | self.timerFD = try TimerFd.timerfd_create( |
140 | 0 | clockId: CLOCK_MONOTONIC, |
141 | 0 | flags: Int32(TimerFd.TFD_CLOEXEC | TimerFd.TFD_NONBLOCK) |
142 | 0 | ) |
143 | 0 |
|
144 | 0 | self.lifecycleState = .open |
145 | 0 |
|
146 | 0 | var ev = Epoll.epoll_event() |
147 | 0 | ev.events = SelectorEventSet.read.epollEventSet |
148 | 0 | ev.data.u64 = UInt64( |
149 | 0 | EPollUserData( |
150 | 0 | registrationID: .initialRegistrationID, |
151 | 0 | fileDescriptor: self.eventFD |
152 | 0 | ) |
153 | 0 | ) |
154 | 0 |
|
155 | 0 | try Epoll.epoll_ctl(epfd: self.selectorFD, op: Epoll.EPOLL_CTL_ADD, fd: self.eventFD, event: &ev) |
156 | 0 |
|
157 | 0 | var timerev = Epoll.epoll_event() |
158 | 0 | timerev.events = Epoll.EPOLLIN | Epoll.EPOLLERR | Epoll.EPOLLRDHUP |
159 | 0 | timerev.data.u64 = UInt64( |
160 | 0 | EPollUserData( |
161 | 0 | registrationID: .initialRegistrationID, |
162 | 0 | fileDescriptor: self.timerFD |
163 | 0 | ) |
164 | 0 | ) |
165 | 0 | try Epoll.epoll_ctl(epfd: self.selectorFD, op: Epoll.EPOLL_CTL_ADD, fd: self.timerFD, event: &timerev) |
166 | 0 | } |
167 | | |
168 | 0 | func deinitAssertions0() { |
169 | 0 | assert(self.eventFD == -1, "self.eventFD == \(self.eventFD) in deinitAssertions0, forgot close?") |
170 | 0 | assert(self.timerFD == -1, "self.timerFD == \(self.timerFD) in deinitAssertions0, forgot close?") |
171 | 0 | } |
172 | | |
173 | | func register0( |
174 | | selectableFD: CInt, |
175 | | fileDescriptor: CInt, |
176 | | interested: SelectorEventSet, |
177 | | registrationID: SelectorRegistrationID |
178 | 0 | ) throws { |
179 | 0 | var ev = Epoll.epoll_event() |
180 | 0 | ev.events = interested.epollEventSet |
181 | 0 | ev.data.u64 = UInt64(EPollUserData(registrationID: registrationID, fileDescriptor: fileDescriptor)) |
182 | 0 |
|
183 | 0 | try Epoll.epoll_ctl(epfd: self.selectorFD, op: Epoll.EPOLL_CTL_ADD, fd: fileDescriptor, event: &ev) |
184 | 0 | } |
185 | | |
186 | | func reregister0( |
187 | | selectableFD: CInt, |
188 | | fileDescriptor: CInt, |
189 | | oldInterested: SelectorEventSet, |
190 | | newInterested: SelectorEventSet, |
191 | | registrationID: SelectorRegistrationID |
192 | 0 | ) throws { |
193 | 0 | var ev = Epoll.epoll_event() |
194 | 0 | ev.events = newInterested.epollEventSet |
195 | 0 | ev.data.u64 = UInt64(EPollUserData(registrationID: registrationID, fileDescriptor: fileDescriptor)) |
196 | 0 |
|
197 | 0 | _ = try Epoll.epoll_ctl(epfd: self.selectorFD, op: Epoll.EPOLL_CTL_MOD, fd: fileDescriptor, event: &ev) |
198 | 0 | } |
199 | | |
200 | | func deregister0( |
201 | | selectableFD: CInt, |
202 | | fileDescriptor: CInt, |
203 | | oldInterested: SelectorEventSet, |
204 | | registrationID: SelectorRegistrationID |
205 | 0 | ) throws { |
206 | 0 | var ev = Epoll.epoll_event() |
207 | 0 | _ = try Epoll.epoll_ctl(epfd: self.selectorFD, op: Epoll.EPOLL_CTL_DEL, fd: fileDescriptor, event: &ev) |
208 | 0 | } |
209 | | |
210 | | /// Apply the given `SelectorStrategy` and execute `body` once it's complete (which may produce `SelectorEvent`s to handle). |
211 | | /// |
212 | | /// - Parameters: |
213 | | /// - strategy: The `SelectorStrategy` to apply |
214 | | /// - body: The function to execute for each `SelectorEvent` that was produced. |
215 | | @inlinable |
216 | | func whenReady0( |
217 | | strategy: SelectorStrategy, |
218 | | onLoopBegin loopStart: () -> Void, |
219 | | _ body: (SelectorEvent<R>) throws -> Void |
220 | 0 | ) throws { |
221 | 0 | assert(self.myThread.isCurrentSlow) |
222 | 0 | guard self.lifecycleState == .open else { |
223 | 0 | throw IOError(errnoCode: EBADF, reason: "can't call whenReady for selector as it's \(self.lifecycleState).") |
224 | 0 | } |
225 | 0 | let ready: Int |
226 | 0 |
|
227 | 0 | switch strategy { |
228 | 0 | case .now: |
229 | 0 | ready = Int( |
230 | 0 | try Epoll.epoll_wait( |
231 | 0 | epfd: self.selectorFD, |
232 | 0 | events: events, |
233 | 0 | maxevents: Int32(eventsCapacity), |
234 | 0 | timeout: 0 |
235 | 0 | ) |
236 | 0 | ) |
237 | 0 | case .blockUntilTimeout(let timeAmount): |
238 | 0 | // Only call timerfd_settime if we're not already scheduled one that will cover it. |
239 | 0 | // This guards against calling timerfd_settime if not needed as this is generally speaking |
240 | 0 | // expensive. |
241 | 0 | let next = NIODeadline.now() + timeAmount |
242 | 0 | if next < self.earliestTimer { |
243 | 0 | self.earliestTimer = next |
244 | 0 |
|
245 | 0 | var ts = itimerspec() |
246 | 0 | ts.it_value = timespec(timeAmount: timeAmount) |
247 | 0 | try TimerFd.timerfd_settime(fd: self.timerFD, flags: 0, newValue: &ts, oldValue: nil) |
248 | 0 | } |
249 | 0 | fallthrough |
250 | 0 | case .block: |
251 | 0 | ready = Int( |
252 | 0 | try Epoll.epoll_wait( |
253 | 0 | epfd: self.selectorFD, |
254 | 0 | events: events, |
255 | 0 | maxevents: Int32(eventsCapacity), |
256 | 0 | timeout: -1 |
257 | 0 | ) |
258 | 0 | ) |
259 | 0 | } |
260 | 0 |
|
261 | 0 | loopStart() |
262 | 0 |
|
263 | 0 | for i in 0..<ready { |
264 | 0 | let ev = events[i] |
265 | 0 | let epollUserData = EPollUserData(rawValue: ev.data.u64) |
266 | 0 | let fd = epollUserData.fileDescriptor |
267 | 0 | let eventRegistrationID = epollUserData.registrationID |
268 | 0 | switch fd { |
269 | 0 | case self.eventFD: |
270 | 0 | var val = EventFd.eventfd_t() |
271 | 0 | // Consume event |
272 | 0 | _ = try EventFd.eventfd_read(fd: self.eventFD, value: &val) |
273 | 0 | case self.timerFD: |
274 | 0 | // Consume event |
275 | 0 | var val: UInt64 = 0 |
276 | 0 | // We are not interested in the result |
277 | 0 | _ = try! Posix.read(descriptor: self.timerFD, pointer: &val, size: MemoryLayout.size(ofValue: val)) |
278 | 0 |
|
279 | 0 | // Processed the earliest set timer so reset it. |
280 | 0 | self.earliestTimer = .distantFuture |
281 | 0 | default: |
282 | 0 | // If the registration is not in the Map anymore we deregistered it during the processing of whenReady(...). In this case just skip it. |
283 | 0 | if let registration = registrations[Int(fd)] { |
284 | 0 | guard eventRegistrationID == registration.registrationID else { |
285 | 0 | continue |
286 | 0 | } |
287 | 0 |
|
288 | 0 | var selectorEvent = SelectorEventSet(epollEvent: ev) |
289 | 0 | // we can only verify the events for i == 0 as for i > 0 the user might have changed the registrations since then. |
290 | 0 | assert( |
291 | 0 | i != 0 || selectorEvent.isSubset(of: registration.interested), |
292 | 0 | "selectorEvent: \(selectorEvent), registration: \(registration)" |
293 | 0 | ) |
294 | 0 |
|
295 | 0 | // in any case we only want what the user is currently registered for & what we got |
296 | 0 | selectorEvent = selectorEvent.intersection(registration.interested) |
297 | 0 |
|
298 | 0 | guard selectorEvent != ._none else { |
299 | 0 | continue |
300 | 0 | } |
301 | 0 |
|
302 | 0 | try body((SelectorEvent(io: selectorEvent, registration: registration))) |
303 | 0 | } |
304 | 0 | } |
305 | 0 | } |
306 | 0 | growEventArrayIfNeeded(ready: ready) |
307 | 0 | } |
308 | | |
309 | | /// Close the `Selector`. |
310 | | /// |
311 | | /// After closing the `Selector` it's no longer possible to use it. |
312 | 0 | public func close0() throws { |
313 | 0 | self.externalSelectorFDLock.withLock { |
314 | 0 | // We try! all of the closes because close can only fail in the following ways: |
315 | 0 | // - EINTR, which we eat in Posix.close |
316 | 0 | // - EIO, which can only happen for on-disk files |
317 | 0 | // - EBADF, which can't happen here because we would crash as EBADF is marked unacceptable |
318 | 0 | // Therefore, we assert here that close will always succeed and if not, that's a NIO bug we need to know |
319 | 0 | // about. |
320 | 0 |
|
321 | 0 | try! Posix.close(descriptor: self.timerFD) |
322 | 0 | self.timerFD = -1 |
323 | 0 |
|
324 | 0 | try! Posix.close(descriptor: self.eventFD) |
325 | 0 | self.eventFD = -1 |
326 | 0 |
|
327 | 0 | try! Posix.close(descriptor: self.selectorFD) |
328 | 0 | self.selectorFD = -1 |
329 | 0 | } |
330 | 0 | } |
331 | | |
332 | | // attention, this may (will!) be called from outside the event loop, ie. can't access mutable shared state (such as `self.open`) |
333 | 0 | func wakeup0() throws { |
334 | 0 | assert(!self.myThread.isCurrentSlow) |
335 | 0 | try self.externalSelectorFDLock.withLock { |
336 | 0 | guard self.eventFD >= 0 else { |
337 | 0 | throw EventLoopError.shutdown |
338 | 0 | } |
339 | 0 | _ = try EventFd.eventfd_write(fd: self.eventFD, value: 1) |
340 | 0 | } |
341 | 0 | } |
342 | | } |
343 | | |
344 | | #endif |
345 | | |
346 | | #endif |
347 | | #endif // !os(WASI) |