/src/swift-nio/Sources/NIOCore/NIOSplitLinesMessageDecoder.swift
Line | Count | Source |
1 | | //===----------------------------------------------------------------------===// |
2 | | // |
3 | | // This source file is part of the SwiftNIO open source project |
4 | | // |
5 | | // Copyright (c) 2025 Apple Inc. and the SwiftNIO project authors |
6 | | // Licensed under Apache License v2.0 |
7 | | // |
8 | | // See LICENSE.txt for license information |
9 | | // See CONTRIBUTORS.txt for the list of SwiftNIO project authors |
10 | | // |
11 | | // SPDX-License-Identifier: Apache-2.0 |
12 | | // |
13 | | //===----------------------------------------------------------------------===// |
14 | | |
15 | | @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) |
16 | | extension AsyncSequence where Element == ByteBuffer { |
17 | | /// Returns the longest possible subsequences of the sequence, in order, |
18 | | /// that are separated by line breaks. |
19 | | /// |
20 | | /// The following Characters are considered line breaks, similar to |
21 | | /// standard library's `String.split(whereSeparator: \.isNewline)`: |
22 | | /// - "\n" (U+000A): LINE FEED (LF) |
23 | | /// - U+000B: LINE TABULATION (VT) |
24 | | /// - U+000C: FORM FEED (FF) |
25 | | /// - "\r" (U+000D): CARRIAGE RETURN (CR) |
26 | | /// - "\r\n" (U+000D U+000A): CR-LF |
27 | | /// |
28 | | /// The following Characters are NOT considered line breaks, unlike in |
29 | | /// standard library's `String.split(whereSeparator: \.isNewline)`: |
30 | | /// - U+0085: NEXT LINE (NEL) |
31 | | /// - U+2028: LINE SEPARATOR |
32 | | /// - U+2029: PARAGRAPH SEPARATOR |
33 | | /// |
34 | | /// This is because these characters would require unicode and data-encoding awareness, which |
35 | | /// are outside swift-nio's scope. |
36 | | /// |
37 | | /// Usage: |
38 | | /// ```swift |
39 | | /// let baseSequence = MyAsyncSequence<ByteBuffer>(...) |
40 | | /// let splitLinesSequence = baseSequence.splitLines() |
41 | | /// |
42 | | /// for try await buffer in splitLinesSequence { |
43 | | /// print("Split by line breaks!\n", buffer.hexDump(format: .detailed)) |
44 | | /// } |
45 | | /// ``` |
46 | | /// |
47 | | /// - Parameters: |
48 | | /// - omittingEmptySubsequences: If `false`, an empty subsequence is |
49 | | /// returned in the result for each consecutive line break in the sequence. |
50 | | /// If `true`, only nonempty subsequences are returned. The default value is `true`. |
51 | | /// - maximumBufferSize: The maximum number of bytes to aggregate in-memory. |
52 | | /// An error will be thrown if after decoding an element there is more aggregated data than this amount. |
53 | | /// - Returns: An `AsyncSequence` of ``ByteBuffer``s, split from the this async sequence's bytes. |
54 | | /// |
55 | | /// - Complexity: O(*n*), where *n* is the length of the file. |
56 | | @inlinable |
57 | | public func splitLines( |
58 | | omittingEmptySubsequences: Bool = true, |
59 | | maximumBufferSize: Int? = nil |
60 | 0 | ) -> NIODecodedAsyncSequence<Self, NIOSplitLinesMessageDecoder> { |
61 | 0 | self.decode( |
62 | 0 | using: NIOSplitLinesMessageDecoder( |
63 | 0 | omittingEmptySubsequences: omittingEmptySubsequences |
64 | 0 | ), |
65 | 0 | maximumBufferSize: maximumBufferSize |
66 | 0 | ) |
67 | 0 | } |
68 | | |
69 | | /// Returns the longest possible `String`s of the sequence, in order, |
70 | | /// that are separated by line breaks. |
71 | | /// |
72 | | /// The following Characters are considered line breaks, similar to |
73 | | /// standard library's `String.split(whereSeparator: \.isNewline)`: |
74 | | /// - "\n" (U+000A): LINE FEED (LF) |
75 | | /// - U+000B: LINE TABULATION (VT) |
76 | | /// - U+000C: FORM FEED (FF) |
77 | | /// - "\r" (U+000D): CARRIAGE RETURN (CR) |
78 | | /// - "\r\n" (U+000D U+000A): CR-LF |
79 | | /// |
80 | | /// The following Characters are NOT considered line breaks, unlike in |
81 | | /// standard library's `String.split(whereSeparator: \.isNewline)`: |
82 | | /// - U+0085: NEXT LINE (NEL) |
83 | | /// - U+2028: LINE SEPARATOR |
84 | | /// - U+2029: PARAGRAPH SEPARATOR |
85 | | /// |
86 | | /// This is because these characters would require unicode and data-encoding awareness, which |
87 | | /// are outside swift-nio's scope. |
88 | | /// |
89 | | /// Usage: |
90 | | /// ```swift |
91 | | /// let baseSequence = MyAsyncSequence<ByteBuffer>(...) |
92 | | /// let splitLinesSequence = baseSequence.splitUTF8Lines() |
93 | | /// |
94 | | /// for try await string in splitLinesSequence { |
95 | | /// print("Split by line breaks!\n", string) |
96 | | /// } |
97 | | /// ``` |
98 | | /// |
99 | | /// - Parameters: |
100 | | /// - omittingEmptySubsequences: If `false`, an empty subsequence is |
101 | | /// returned in the result for each consecutive line break in the sequence. |
102 | | /// If `true`, only nonempty subsequences are returned. The default value is `true`. |
103 | | /// - maximumBufferSize: The maximum number of bytes to aggregate in-memory. |
104 | | /// An error will be thrown if after decoding an element there is more aggregated data than this amount. |
105 | | /// - Returns: An `AsyncSequence` of `String`s, split from the this async sequence's bytes. |
106 | | /// |
107 | | /// - Complexity: O(*n*), where *n* is the length of the file. |
108 | | @inlinable |
109 | | public func splitUTF8Lines( |
110 | | omittingEmptySubsequences: Bool = true, |
111 | | maximumBufferSize: Int? = nil |
112 | 0 | ) -> NIODecodedAsyncSequence<Self, NIOSplitUTF8LinesMessageDecoder> { |
113 | 0 | self.decode( |
114 | 0 | using: NIOSplitUTF8LinesMessageDecoder( |
115 | 0 | omittingEmptySubsequences: omittingEmptySubsequences |
116 | 0 | ), |
117 | 0 | maximumBufferSize: maximumBufferSize |
118 | 0 | ) |
119 | 0 | } |
120 | | } |
121 | | |
122 | | // MARK: - SplitMessageDecoder |
123 | | |
124 | | /// A decoder which splits the data into subsequences that are separated by a given separator. |
125 | | /// Similar to standard library's `String.split(separator:maxSplits:omittingEmptySubsequences:)`. |
126 | | /// |
127 | | /// This decoder can be used to introduce a `AsyncSequence/split(omittingEmptySubsequences:maximumBufferSize:whereSeparator:)` |
128 | | /// function. We could not come up with valid use-cases for such a function so we held off on introducing it. |
129 | | /// See https://github.com/apple/swift-nio/pull/3411 for more info if you need such a function. |
130 | | @usableFromInline |
131 | | struct SplitMessageDecoder: NIOSingleStepByteToMessageDecoder { |
132 | | @usableFromInline |
133 | | typealias InboundOut = ByteBuffer |
134 | | |
135 | | @usableFromInline |
136 | | let omittingEmptySubsequences: Bool |
137 | | @usableFromInline |
138 | | let isSeparator: (UInt8) -> Bool |
139 | | @usableFromInline |
140 | | var ended: Bool |
141 | | @usableFromInline |
142 | | var bytesWithNoSeparatorsCount: Int |
143 | | |
144 | | @inlinable |
145 | | init( |
146 | | omittingEmptySubsequences: Bool = true, |
147 | | whereSeparator isSeparator: @escaping (UInt8) -> Bool |
148 | 0 | ) { |
149 | 0 | self.omittingEmptySubsequences = omittingEmptySubsequences |
150 | 0 | self.isSeparator = isSeparator |
151 | 0 | self.ended = false |
152 | 0 | self.bytesWithNoSeparatorsCount = 0 |
153 | 0 | } |
154 | | |
155 | | /// Decode the next message from the given buffer. |
156 | | @inlinable |
157 | | mutating func decode( |
158 | | buffer: inout ByteBuffer, |
159 | | hasReceivedLastChunk: Bool |
160 | 0 | ) throws -> (buffer: InboundOut, separator: UInt8?)? { |
161 | 0 | if self.ended { return nil } |
162 | 0 |
|
163 | 0 | while true { |
164 | 0 | let startIndex = buffer.readerIndex + self.bytesWithNoSeparatorsCount |
165 | 0 | if let separatorIndex = buffer.readableBytesView[startIndex...].firstIndex(where: self.isSeparator) { |
166 | 0 | // Safe to force unwrap. We just found a separator somewhere in the buffer. |
167 | 0 | let slice = buffer.readSlice(length: separatorIndex - buffer.readerIndex)! |
168 | 0 | // Reset for the next search since we found a separator. |
169 | 0 | self.bytesWithNoSeparatorsCount = 0 |
170 | 0 |
|
171 | 0 | if self.omittingEmptySubsequences, |
172 | 0 | slice.readableBytes == 0 |
173 | 0 | { |
174 | 0 | // Mark the separator itself as read |
175 | 0 | buffer._moveReaderIndex(forwardBy: 1) |
176 | 0 | continue |
177 | 0 | } |
178 | 0 |
|
179 | 0 | // Read the separator itself |
180 | 0 | // Safe to force unwrap. We just found a separator somewhere in the buffer. |
181 | 0 | let separator = buffer.readInteger(as: UInt8.self)! |
182 | 0 |
|
183 | 0 | return (slice, separator) |
184 | 0 | } else { |
185 | 0 | guard hasReceivedLastChunk else { |
186 | 0 | // Make sure we don't double-check these no-separator bytes again. |
187 | 0 | self.bytesWithNoSeparatorsCount = buffer.readableBytes |
188 | 0 | // Need more data |
189 | 0 | return nil |
190 | 0 | } |
191 | 0 |
|
192 | 0 | // At this point, we're ending the decoding process. |
193 | 0 | self.ended = true |
194 | 0 |
|
195 | 0 | if self.omittingEmptySubsequences, |
196 | 0 | buffer.readableBytes == 0 |
197 | 0 | { |
198 | 0 | return nil |
199 | 0 | } |
200 | 0 |
|
201 | 0 | // Just send the whole buffer if we're at the last chunk but we can find no separators |
202 | 0 | // Safe to force unwrap. `buffer.readableBytes` is `0` in the worst case. |
203 | 0 | let slice = buffer.readSlice(length: buffer.readableBytes)! |
204 | 0 |
|
205 | 0 | return (slice, nil) |
206 | 0 | } |
207 | 0 | } |
208 | 0 | } |
209 | | |
210 | | /// Decode the next message separated by the provided separator. |
211 | | /// To be used when we're still receiving data. |
212 | | @inlinable |
213 | 0 | mutating func decode(buffer: inout ByteBuffer) throws -> InboundOut? { |
214 | 0 | try self.decode(buffer: &buffer, hasReceivedLastChunk: false)?.buffer |
215 | 0 | } |
216 | | |
217 | | /// Decode the next message separated by the provided separator. |
218 | | /// To be used when the last chunk of data has been received. |
219 | | @inlinable |
220 | 0 | mutating func decodeLast(buffer: inout ByteBuffer, seenEOF: Bool) throws -> InboundOut? { |
221 | 0 | try self.decode(buffer: &buffer, hasReceivedLastChunk: true)?.buffer |
222 | 0 | } |
223 | | } |
224 | | |
225 | | @available(*, unavailable) |
226 | | extension SplitMessageDecoder: Sendable {} |
227 | | |
228 | | // MARK: - NIOSplitLinesMessageDecoder |
229 | | |
230 | | /// A decoder which splits the data into subsequences that are separated by line breaks. |
231 | | /// |
232 | | /// You can initialize this type directly, or use |
233 | | /// `AsyncSequence/splitLines(omittingEmptySubsequences:maximumBufferSize:)` to create a |
234 | | /// `NIODecodedAsyncSequence` that uses this decoder. |
235 | | /// |
236 | | /// The following Characters are considered line breaks, similar to |
237 | | /// standard library's `String.split(whereSeparator: \.isNewline)`: |
238 | | /// - "\n" (U+000A): LINE FEED (LF) |
239 | | /// - U+000B: LINE TABULATION (VT) |
240 | | /// - U+000C: FORM FEED (FF) |
241 | | /// - "\r" (U+000D): CARRIAGE RETURN (CR) |
242 | | /// - "\r\n" (U+000D U+000A): CR-LF |
243 | | /// |
244 | | /// The following Characters are NOT considered line breaks, unlike in |
245 | | /// standard library's `String.split(whereSeparator: \.isNewline)`: |
246 | | /// - U+0085: NEXT LINE (NEL) |
247 | | /// - U+2028: LINE SEPARATOR |
248 | | /// - U+2029: PARAGRAPH SEPARATOR |
249 | | /// |
250 | | /// This is because these characters would require unicode and data-encoding awareness, which |
251 | | /// are outside swift-nio's scope. |
252 | | /// |
253 | | /// Usage: |
254 | | /// ```swift |
255 | | /// let baseSequence = MyAsyncSequence<ByteBuffer>(...) |
256 | | /// let splitLinesSequence = baseSequence.splitLines() |
257 | | /// |
258 | | /// for try await buffer in splitLinesSequence { |
259 | | /// print("Split by line breaks!\n", buffer.hexDump(format: .detailed)) |
260 | | /// } |
261 | | /// ``` |
262 | | public struct NIOSplitLinesMessageDecoder: NIOSingleStepByteToMessageDecoder { |
263 | | public typealias InboundOut = ByteBuffer |
264 | | |
265 | | @usableFromInline |
266 | | var splitDecoder: SplitMessageDecoder |
267 | | @usableFromInline |
268 | | var previousSeparatorWasCR: Bool |
269 | | |
270 | | @inlinable |
271 | 0 | public init(omittingEmptySubsequences: Bool) { |
272 | 0 | self.splitDecoder = SplitMessageDecoder( |
273 | 0 | omittingEmptySubsequences: omittingEmptySubsequences, |
274 | 0 | whereSeparator: Self.isLineBreak |
275 | 0 | ) |
276 | 0 | self.previousSeparatorWasCR = false |
277 | 0 | } |
278 | | |
279 | | /// - ASCII 10 - "\n" (U+000A): LINE FEED (LF) |
280 | | /// - ASCII 11 - U+000B: LINE TABULATION (VT) |
281 | | /// - ASCII 12 - U+000C: FORM FEED (FF) |
282 | | /// - ASCII 13 - "\r" (U+000D): CARRIAGE RETURN (CR) |
283 | | /// |
284 | | /// "\r\n" is manually accounted for during the decoding. |
285 | | @inlinable |
286 | 0 | static func isLineBreak(_ byte: UInt8) -> Bool { |
287 | 0 | /// All the 4 ASCII bytes are in range of \n to \r. |
288 | 0 | (UInt8(ascii: "\n")...UInt8(ascii: "\r")).contains(byte) |
289 | 0 | } |
290 | | |
291 | | /// Decode the next message from the given buffer. |
292 | | @inlinable |
293 | 0 | mutating func decode(buffer: inout ByteBuffer, hasReceivedLastChunk: Bool) throws -> InboundOut? { |
294 | 0 | while true { |
295 | 0 | guard |
296 | 0 | let (slice, separator) = try self.splitDecoder.decode( |
297 | 0 | buffer: &buffer, |
298 | 0 | hasReceivedLastChunk: hasReceivedLastChunk |
299 | 0 | ) |
300 | 0 | else { |
301 | 0 | return nil |
302 | 0 | } |
303 | 0 |
|
304 | 0 | // If we are getting rid of empty subsequences then it doesn't matter if we detect |
305 | 0 | // \r\n as a CR-LF, or as a CR + a LF. The backing decoder gets rid of the empty subsequence |
306 | 0 | // anyway. Therefore, we can return early right here and skip the rest of the logic. |
307 | 0 | if self.splitDecoder.omittingEmptySubsequences { |
308 | 0 | return slice |
309 | 0 | } |
310 | 0 |
|
311 | 0 | // "\r\n" is 2 bytes long, so we need to manually account for it. |
312 | 0 | switch separator { |
313 | 0 | case UInt8(ascii: "\n") where slice.readableBytes == 0: |
314 | 0 | let isCRLF = self.previousSeparatorWasCR |
315 | 0 | self.previousSeparatorWasCR = false |
316 | 0 | if isCRLF { |
317 | 0 | continue |
318 | 0 | } |
319 | 0 | case UInt8(ascii: "\r"): |
320 | 0 | self.previousSeparatorWasCR = true |
321 | 0 | default: |
322 | 0 | self.previousSeparatorWasCR = false |
323 | 0 | } |
324 | 0 |
|
325 | 0 | return slice |
326 | 0 | } |
327 | 0 | } |
328 | | |
329 | | /// Decode the next message separated by one of the ASCII line breaks. |
330 | | /// To be used when we're still receiving data. |
331 | | @inlinable |
332 | 0 | public mutating func decode(buffer: inout ByteBuffer) throws -> InboundOut? { |
333 | 0 | try self.decode(buffer: &buffer, hasReceivedLastChunk: false) |
334 | 0 | } |
335 | | |
336 | | /// Decode the next message separated by one of the ASCII line breaks. |
337 | | /// To be used when the last chunk of data has been received. |
338 | | @inlinable |
339 | 0 | public mutating func decodeLast(buffer: inout ByteBuffer, seenEOF: Bool) throws -> InboundOut? { |
340 | 0 | try self.decode(buffer: &buffer, hasReceivedLastChunk: true) |
341 | 0 | } |
342 | | } |
343 | | |
344 | | @available(*, unavailable) |
345 | | extension NIOSplitLinesMessageDecoder: Sendable {} |
346 | | |
347 | | // MARK: - NIOSplitUTF8LinesMessageDecoder |
348 | | |
349 | | /// A decoder which splits the data into subsequences that are separated by line breaks. |
350 | | /// |
351 | | /// You can initialize this type directly, or use |
352 | | /// `AsyncSequence/splitUTF8Lines(omittingEmptySubsequences:maximumBufferSize:)` to create a |
353 | | /// `NIODecodedAsyncSequence` that uses this decoder. |
354 | | /// |
355 | | /// The following Characters are considered line breaks, similar to |
356 | | /// standard library's `String.split(whereSeparator: \.isNewline)`: |
357 | | /// - "\n" (U+000A): LINE FEED (LF) |
358 | | /// - U+000B: LINE TABULATION (VT) |
359 | | /// - U+000C: FORM FEED (FF) |
360 | | /// - "\r" (U+000D): CARRIAGE RETURN (CR) |
361 | | /// - "\r\n" (U+000D U+000A): CR-LF |
362 | | /// |
363 | | /// The following Characters are NOT considered line breaks, unlike in |
364 | | /// standard library's `String.split(whereSeparator: \.isNewline)`: |
365 | | /// - U+0085: NEXT LINE (NEL) |
366 | | /// - U+2028: LINE SEPARATOR |
367 | | /// - U+2029: PARAGRAPH SEPARATOR |
368 | | /// |
369 | | /// This is because these characters would require unicode and data-encoding awareness, which |
370 | | /// are outside swift-nio's scope. |
371 | | /// |
372 | | /// Usage: |
373 | | /// ```swift |
374 | | /// let baseSequence = MyAsyncSequence<ByteBuffer>(...) |
375 | | /// let splitLinesSequence = baseSequence.splitUTF8Lines() |
376 | | /// |
377 | | /// for try await string in splitLinesSequence { |
378 | | /// print("Split by line breaks!\n", string) |
379 | | /// } |
380 | | /// ``` |
381 | | public struct NIOSplitUTF8LinesMessageDecoder: NIOSingleStepByteToMessageDecoder { |
382 | | public typealias InboundOut = String |
383 | | |
384 | | @usableFromInline |
385 | | var splitLinesDecoder: NIOSplitLinesMessageDecoder |
386 | | |
387 | | @inlinable |
388 | 0 | public init(omittingEmptySubsequences: Bool) { |
389 | 0 | self.splitLinesDecoder = NIOSplitLinesMessageDecoder( |
390 | 0 | omittingEmptySubsequences: omittingEmptySubsequences |
391 | 0 | ) |
392 | 0 | } |
393 | | |
394 | | /// Decode the next message separated by one of the ASCII line breaks. |
395 | | /// To be used when we're still receiving data. |
396 | | @inlinable |
397 | 0 | public mutating func decode(buffer: inout ByteBuffer) throws -> InboundOut? { |
398 | 0 | try self.splitLinesDecoder.decode(buffer: &buffer, hasReceivedLastChunk: false).map { |
399 | 0 | String(buffer: $0) |
400 | 0 | } |
401 | 0 | } |
402 | | |
403 | | /// Decode the next message separated by one of the ASCII line breaks. |
404 | | /// To be used when the last chunk of data has been received. |
405 | | @inlinable |
406 | 0 | public mutating func decodeLast(buffer: inout ByteBuffer, seenEOF: Bool) throws -> InboundOut? { |
407 | 0 | try self.splitLinesDecoder.decode(buffer: &buffer, hasReceivedLastChunk: true).map { |
408 | 0 | String(buffer: $0) |
409 | 0 | } |
410 | 0 | } |
411 | | } |
412 | | |
413 | | @available(*, unavailable) |
414 | | extension NIOSplitUTF8LinesMessageDecoder: Sendable {} |