/src/php-src/main/io/php_io_copy_linux.c
Line | Count | Source |
1 | | /* |
2 | | +----------------------------------------------------------------------+ |
3 | | | Copyright © The PHP Group and Contributors. | |
4 | | +----------------------------------------------------------------------+ |
5 | | | This source file is subject to the Modified BSD License that is | |
6 | | | bundled with this package in the file LICENSE, and is available | |
7 | | | through the World Wide Web at <https://www.php.net/license/>. | |
8 | | | | |
9 | | | SPDX-License-Identifier: BSD-3-Clause | |
10 | | +----------------------------------------------------------------------+ |
11 | | | Authors: Jakub Zelenka <bukka@php.net> | |
12 | | +----------------------------------------------------------------------+ |
13 | | */ |
14 | | |
15 | | #ifdef __linux__ |
16 | | |
17 | | #include "php_io_internal.h" |
18 | | #include <unistd.h> |
19 | | #include <errno.h> |
20 | | #include <sys/syscall.h> |
21 | | |
22 | | #if !defined(HAVE_COPY_FILE_RANGE) && defined(__NR_copy_file_range) |
23 | | #define HAVE_COPY_FILE_RANGE 1 |
24 | | static inline ssize_t copy_file_range( |
25 | | int fd_in, off_t *off_in, int fd_out, off_t *off_out, size_t len, unsigned int flags) |
26 | | { |
27 | | return syscall(__NR_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags); |
28 | | } |
29 | | #endif |
30 | | |
31 | | #ifdef HAVE_SENDFILE |
32 | | #include <sys/sendfile.h> |
33 | | #endif |
34 | | |
35 | | #ifdef HAVE_SPLICE |
36 | | #include <fcntl.h> |
37 | | #include <sys/socket.h> |
38 | | #include <netinet/in.h> |
39 | | #include <netinet/tcp.h> |
40 | | #endif |
41 | | |
42 | | static inline int php_io_linux_wait_for_data(php_io_fd *fd) |
43 | 0 | { |
44 | 0 | if (fd->fd_type != PHP_IO_FD_SOCKET || !fd->is_blocked) { |
45 | 0 | return 1; |
46 | 0 | } |
47 | | |
48 | 0 | struct timeval *ptimeout = (fd->timeout.tv_sec == -1) ? NULL : &fd->timeout; |
49 | 0 | int timeout_ms; |
50 | |
|
51 | 0 | if (ptimeout == NULL) { |
52 | 0 | timeout_ms = -1; |
53 | 0 | } else { |
54 | 0 | timeout_ms = ptimeout->tv_sec * 1000 + ptimeout->tv_usec / 1000; |
55 | 0 | } |
56 | |
|
57 | 0 | int ret; |
58 | 0 | do { |
59 | 0 | ret = php_pollfd_for_ms(fd->fd, POLLIN, timeout_ms); |
60 | 0 | } while (ret == -1 && errno == EINTR); |
61 | |
|
62 | 0 | return ret; |
63 | 0 | } |
64 | | |
65 | | static zend_result php_io_linux_copy_file_to_file(int src_fd, int dest_fd, size_t maxlen, size_t *copied) |
66 | 0 | { |
67 | 0 | #ifdef HAVE_COPY_FILE_RANGE |
68 | 0 | size_t total_copied = 0; |
69 | 0 | size_t remaining = (maxlen == PHP_IO_COPY_ALL) ? SIZE_MAX : maxlen; |
70 | |
|
71 | 0 | while (remaining > 0) { |
72 | 0 | size_t to_copy = (remaining < SSIZE_MAX) ? remaining : SSIZE_MAX; |
73 | 0 | ssize_t result = copy_file_range(src_fd, NULL, dest_fd, NULL, to_copy, 0); |
74 | |
|
75 | 0 | if (result > 0) { |
76 | 0 | total_copied += result; |
77 | 0 | if (maxlen != PHP_IO_COPY_ALL) { |
78 | 0 | remaining -= result; |
79 | 0 | } |
80 | 0 | } else if (result == 0) { |
81 | 0 | break; |
82 | 0 | } else { |
83 | 0 | switch (errno) { |
84 | 0 | case EINVAL: |
85 | 0 | case EXDEV: |
86 | 0 | case ENOSYS: |
87 | 0 | case EIO: |
88 | 0 | if (total_copied == 0) { |
89 | 0 | return php_io_generic_copy_fallback(src_fd, dest_fd, maxlen, copied); |
90 | 0 | } |
91 | 0 | break; |
92 | 0 | default: |
93 | 0 | *copied = total_copied; |
94 | 0 | return FAILURE; |
95 | 0 | } |
96 | 0 | break; |
97 | 0 | } |
98 | 0 | } |
99 | | |
100 | 0 | if (total_copied > 0) { |
101 | 0 | *copied = total_copied; |
102 | 0 | return SUCCESS; |
103 | 0 | } |
104 | 0 | #endif |
105 | | |
106 | 0 | return php_io_generic_copy_fallback(src_fd, dest_fd, maxlen, copied); |
107 | 0 | } |
108 | | |
109 | | static zend_result php_io_linux_sendfile(int src_fd, int dest_fd, size_t maxlen, size_t *copied) |
110 | 0 | { |
111 | 0 | #ifdef HAVE_SENDFILE |
112 | 0 | size_t total_copied = 0; |
113 | 0 | size_t remaining = (maxlen == PHP_IO_COPY_ALL) ? SIZE_MAX : maxlen; |
114 | |
|
115 | 0 | while (remaining > 0) { |
116 | 0 | size_t to_send = (remaining < SSIZE_MAX) ? remaining : SSIZE_MAX; |
117 | 0 | ssize_t result = sendfile(dest_fd, src_fd, NULL, to_send); |
118 | |
|
119 | 0 | if (result > 0) { |
120 | 0 | total_copied += result; |
121 | 0 | if (maxlen != PHP_IO_COPY_ALL) { |
122 | 0 | remaining -= result; |
123 | 0 | } |
124 | 0 | } else if (result == 0) { |
125 | 0 | break; |
126 | 0 | } else { |
127 | 0 | switch (errno) { |
128 | 0 | case EINTR: |
129 | 0 | continue; |
130 | 0 | case EINVAL: |
131 | 0 | case ENOSYS: |
132 | 0 | if (total_copied == 0) { |
133 | 0 | return php_io_generic_copy_fallback(src_fd, dest_fd, maxlen, copied); |
134 | 0 | } |
135 | 0 | break; |
136 | 0 | case EAGAIN: |
137 | 0 | break; |
138 | 0 | default: |
139 | 0 | *copied = total_copied; |
140 | 0 | return FAILURE; |
141 | 0 | } |
142 | 0 | break; |
143 | 0 | } |
144 | 0 | } |
145 | | |
146 | 0 | if (total_copied > 0) { |
147 | 0 | *copied = total_copied; |
148 | 0 | return SUCCESS; |
149 | 0 | } |
150 | 0 | #endif |
151 | | |
152 | 0 | return php_io_generic_copy_fallback(src_fd, dest_fd, maxlen, copied); |
153 | 0 | } |
154 | | |
155 | | #ifdef HAVE_SPLICE |
156 | | /* Enlarge the intermediate pipe so socket transfers move more data per splice |
157 | | * round-trip. Capped by /proc/sys/fs/pipe-max-size (1 MiB by default); a failed |
158 | | * fcntl() simply leaves the kernel default (64 KiB) in place. */ |
159 | 0 | #define PHP_IO_PIPE_SIZE (1 << 20) |
160 | | |
161 | | /* Kernel MAX_RW_COUNT; a larger len fails the pos + len overflow check with |
162 | | * EINVAL once a file destination sits at a non-zero offset. */ |
163 | 0 | #define PHP_IO_SPLICE_MAX ((size_t) 0x7ffff000) |
164 | | |
165 | | /* SPLICE_F_MORE corks the socket the same way MSG_MORE does, letting the kernel |
166 | | * coalesce splices into full segments. The final partial segment is not flushed |
167 | | * by the kernel until the cork timer (<= 200 ms) expires or the socket is next |
168 | | * written/closed, so the copy loops below clear the cork once they are done. */ |
169 | | static inline unsigned int php_io_linux_out_flags(const php_io_fd *dest) |
170 | 0 | { |
171 | 0 | return (dest->fd_type == PHP_IO_FD_SOCKET) ? SPLICE_F_MORE : 0; |
172 | 0 | } |
173 | | |
174 | | /* Clearing TCP_CORK pushes out any segment still held by SPLICE_F_MORE; |
175 | | * on non-TCP sockets the setsockopt() harmlessly fails. */ |
176 | | static inline void php_io_linux_socket_uncork(const php_io_fd *dest, size_t total_copied) |
177 | 0 | { |
178 | 0 | if (dest->fd_type == PHP_IO_FD_SOCKET && total_copied > 0) { |
179 | 0 | int off = 0; |
180 | 0 | setsockopt(dest->fd, IPPROTO_TCP, TCP_CORK, &off, sizeof(off)); |
181 | 0 | } |
182 | 0 | } |
183 | | |
184 | | static zend_result php_io_linux_splice_from_pipe(php_io_fd *src, php_io_fd *dest, size_t maxlen, size_t *copied) |
185 | 0 | { |
186 | 0 | int dest_fd = dest->fd; |
187 | 0 | unsigned int out_flags = php_io_linux_out_flags(dest); |
188 | 0 | size_t total_copied = 0; |
189 | 0 | size_t remaining = (maxlen == PHP_IO_COPY_ALL) ? SIZE_MAX : maxlen; |
190 | 0 | zend_result result = SUCCESS; |
191 | |
|
192 | 0 | while (remaining > 0) { |
193 | 0 | int ready = php_io_linux_wait_for_data(src); |
194 | 0 | if (ready == 0) { |
195 | 0 | break; |
196 | 0 | } else if (ready < 0) { |
197 | 0 | result = FAILURE; |
198 | 0 | break; |
199 | 0 | } |
200 | | |
201 | 0 | size_t to_copy = (remaining < PHP_IO_SPLICE_MAX) ? remaining : PHP_IO_SPLICE_MAX; |
202 | 0 | ssize_t spliced = splice(src->fd, NULL, dest_fd, NULL, to_copy, out_flags); |
203 | |
|
204 | 0 | if (spliced > 0) { |
205 | 0 | total_copied += spliced; |
206 | 0 | if (maxlen != PHP_IO_COPY_ALL) { |
207 | 0 | remaining -= spliced; |
208 | 0 | } |
209 | 0 | } else if (spliced == 0) { |
210 | 0 | break; |
211 | 0 | } else { |
212 | 0 | if (total_copied == 0) { |
213 | 0 | return php_io_generic_copy_fallback(src->fd, dest_fd, maxlen, copied); |
214 | 0 | } |
215 | 0 | result = FAILURE; |
216 | 0 | break; |
217 | 0 | } |
218 | 0 | } |
219 | | |
220 | 0 | php_io_linux_socket_uncork(dest, total_copied); |
221 | 0 | *copied = total_copied; |
222 | 0 | return result; |
223 | 0 | } |
224 | | |
225 | | static zend_result php_io_linux_splice_via_pipe(php_io_fd *src, php_io_fd *dest, size_t maxlen, size_t *copied) |
226 | 0 | { |
227 | 0 | int dest_fd = dest->fd; |
228 | 0 | unsigned int out_flags = php_io_linux_out_flags(dest); |
229 | 0 | int pipefd[2]; |
230 | 0 | if (pipe(pipefd) == -1) { |
231 | 0 | return php_io_generic_copy_fallback(src->fd, dest_fd, maxlen, copied); |
232 | 0 | } |
233 | | |
234 | 0 | #ifdef F_SETPIPE_SZ |
235 | 0 | fcntl(pipefd[1], F_SETPIPE_SZ, PHP_IO_PIPE_SIZE); |
236 | 0 | #endif |
237 | |
|
238 | 0 | size_t total_copied = 0; |
239 | 0 | size_t remaining = (maxlen == PHP_IO_COPY_ALL) ? SIZE_MAX : maxlen; |
240 | 0 | zend_result result = SUCCESS; |
241 | |
|
242 | 0 | while (remaining > 0) { |
243 | 0 | int ready = php_io_linux_wait_for_data(src); |
244 | 0 | if (ready == 0) { |
245 | | /* timeout */ |
246 | 0 | break; |
247 | 0 | } else if (ready < 0) { |
248 | 0 | result = FAILURE; |
249 | 0 | break; |
250 | 0 | } |
251 | | |
252 | 0 | size_t to_copy = (remaining < SSIZE_MAX) ? remaining : SSIZE_MAX; |
253 | |
|
254 | 0 | ssize_t in_pipe = splice(src->fd, NULL, pipefd[1], NULL, to_copy, 0); |
255 | 0 | if (in_pipe < 0) { |
256 | 0 | if (total_copied == 0) { |
257 | 0 | close(pipefd[0]); |
258 | 0 | close(pipefd[1]); |
259 | 0 | return php_io_generic_copy_fallback(src->fd, dest_fd, maxlen, copied); |
260 | 0 | } |
261 | 0 | result = FAILURE; |
262 | 0 | break; |
263 | 0 | } |
264 | 0 | if (in_pipe == 0) { |
265 | 0 | break; |
266 | 0 | } |
267 | | |
268 | 0 | size_t pipe_remaining = in_pipe; |
269 | 0 | while (pipe_remaining > 0) { |
270 | 0 | ssize_t out = splice(pipefd[0], NULL, dest_fd, NULL, pipe_remaining, out_flags); |
271 | 0 | if (out <= 0) { |
272 | | /* the dest refused the splice; salvage what already sits in |
273 | | * the pipe with a plain read/write loop before failing */ |
274 | 0 | char drain_buf[1024]; |
275 | 0 | while (pipe_remaining > 0) { |
276 | 0 | size_t to_drain = (pipe_remaining < sizeof(drain_buf)) |
277 | 0 | ? pipe_remaining : sizeof(drain_buf); |
278 | 0 | ssize_t drained; |
279 | 0 | do { |
280 | 0 | drained = read(pipefd[0], drain_buf, to_drain); |
281 | 0 | } while (drained < 0 && errno == EINTR); |
282 | 0 | if (drained <= 0) { |
283 | 0 | break; |
284 | 0 | } |
285 | 0 | ssize_t drain_written = 0; |
286 | 0 | while (drain_written < drained) { |
287 | 0 | ssize_t written; |
288 | 0 | do { |
289 | 0 | written = write(dest_fd, drain_buf + drain_written, drained - drain_written); |
290 | 0 | } while (written < 0 && errno == EINTR); |
291 | 0 | if (written <= 0) { |
292 | 0 | total_copied += drain_written; |
293 | 0 | result = FAILURE; |
294 | 0 | goto out; |
295 | 0 | } |
296 | 0 | drain_written += written; |
297 | 0 | } |
298 | 0 | pipe_remaining -= drain_written; |
299 | 0 | total_copied += drain_written; |
300 | 0 | } |
301 | 0 | result = FAILURE; |
302 | 0 | goto out; |
303 | 0 | } |
304 | 0 | pipe_remaining -= out; |
305 | 0 | total_copied += out; |
306 | 0 | } |
307 | | |
308 | 0 | if (maxlen != PHP_IO_COPY_ALL) { |
309 | 0 | remaining -= in_pipe; |
310 | 0 | } |
311 | 0 | } |
312 | | |
313 | 0 | out: |
314 | 0 | close(pipefd[0]); |
315 | 0 | close(pipefd[1]); |
316 | 0 | php_io_linux_socket_uncork(dest, total_copied); |
317 | 0 | *copied = total_copied; |
318 | 0 | return result; |
319 | 0 | } |
320 | | #endif /* HAVE_SPLICE */ |
321 | | |
322 | | zend_result php_io_linux_copy(php_io_fd *src, php_io_fd *dest, size_t maxlen, size_t *copied) |
323 | 0 | { |
324 | 0 | if (src->fd_type == PHP_IO_FD_FILE && dest->fd_type == PHP_IO_FD_FILE) { |
325 | 0 | return php_io_linux_copy_file_to_file(src->fd, dest->fd, maxlen, copied); |
326 | 0 | } |
327 | | |
328 | 0 | if (src->fd_type == PHP_IO_FD_FILE && dest->fd_type == PHP_IO_FD_SOCKET) { |
329 | 0 | return php_io_linux_sendfile(src->fd, dest->fd, maxlen, copied); |
330 | 0 | } |
331 | | |
332 | 0 | if (src->fd_type == PHP_IO_FD_FILE && dest->fd_type == PHP_IO_FD_PIPE) { |
333 | 0 | return php_io_linux_sendfile(src->fd, dest->fd, maxlen, copied); |
334 | 0 | } |
335 | | |
336 | 0 | #ifdef HAVE_SPLICE |
337 | 0 | if (src->fd_type == PHP_IO_FD_PIPE) { |
338 | 0 | return php_io_linux_splice_from_pipe(src, dest, maxlen, copied); |
339 | 0 | } |
340 | | |
341 | 0 | if (src->fd_type == PHP_IO_FD_SOCKET) { |
342 | 0 | return php_io_linux_splice_via_pipe(src, dest, maxlen, copied); |
343 | 0 | } |
344 | 0 | #endif |
345 | | |
346 | | /* php_io_generic_copy honours the stream timeout for socket sources */ |
347 | 0 | return php_io_generic_copy(src, dest, maxlen, copied); |
348 | 0 | } |
349 | | |
350 | | #endif /* __linux__ */ |