/src/php-src/main/io/php_io_copy_linux.c
Line | Count | Source |
1 | | /* |
2 | | +----------------------------------------------------------------------+ |
3 | | | Copyright © The PHP Group and Contributors. | |
4 | | +----------------------------------------------------------------------+ |
5 | | | This source file is subject to the Modified BSD License that is | |
6 | | | bundled with this package in the file LICENSE, and is available | |
7 | | | through the World Wide Web at <https://www.php.net/license/>. | |
8 | | | | |
9 | | | SPDX-License-Identifier: BSD-3-Clause | |
10 | | +----------------------------------------------------------------------+ |
11 | | | Authors: Jakub Zelenka <bukka@php.net> | |
12 | | +----------------------------------------------------------------------+ |
13 | | */ |
14 | | |
15 | | #ifdef __linux__ |
16 | | |
17 | | #include "php_io_internal.h" |
18 | | #include <unistd.h> |
19 | | #include <errno.h> |
20 | | #include <sys/syscall.h> |
21 | | |
22 | | #if !defined(HAVE_COPY_FILE_RANGE) && defined(__NR_copy_file_range) |
23 | | #define HAVE_COPY_FILE_RANGE 1 |
24 | | static inline ssize_t php_copy_file_range( |
25 | | int fd_in, off_t *off_in, int fd_out, off_t *off_out, size_t len, unsigned int flags) |
26 | | { |
27 | | return syscall(__NR_copy_file_range, fd_in, off_in, fd_out, off_out, len, flags); |
28 | | } |
29 | | #define copy_file_range php_copy_file_range |
30 | | #endif |
31 | | |
32 | | #ifdef HAVE_SENDFILE |
33 | | #include <sys/sendfile.h> |
34 | | #endif |
35 | | |
36 | | #ifdef HAVE_SPLICE |
37 | | #include <fcntl.h> |
38 | | #include <sys/socket.h> |
39 | | #include <netinet/in.h> |
40 | | #include <netinet/tcp.h> |
41 | | #endif |
42 | | |
43 | | static inline int php_io_linux_wait_for_data(php_io_fd *fd) |
44 | 0 | { |
45 | 0 | if (fd->fd_type != PHP_IO_FD_SOCKET || !fd->is_blocked) { |
46 | 0 | return 1; |
47 | 0 | } |
48 | | |
49 | 0 | struct timeval *ptimeout = (fd->timeout.tv_sec == -1) ? NULL : &fd->timeout; |
50 | 0 | int timeout_ms; |
51 | |
|
52 | 0 | if (ptimeout == NULL) { |
53 | 0 | timeout_ms = -1; |
54 | 0 | } else { |
55 | 0 | timeout_ms = ptimeout->tv_sec * 1000 + ptimeout->tv_usec / 1000; |
56 | 0 | } |
57 | |
|
58 | 0 | int ret; |
59 | 0 | do { |
60 | 0 | ret = php_pollfd_for_ms(fd->fd, POLLIN, timeout_ms); |
61 | 0 | } while (ret == -1 && errno == EINTR); |
62 | |
|
63 | 0 | return ret; |
64 | 0 | } |
65 | | |
66 | | static zend_result php_io_linux_copy_file_to_file(int src_fd, int dest_fd, size_t maxlen, size_t *copied) |
67 | 0 | { |
68 | 0 | #ifdef HAVE_COPY_FILE_RANGE |
69 | 0 | size_t total_copied = 0; |
70 | 0 | size_t remaining = (maxlen == PHP_IO_COPY_ALL) ? SIZE_MAX : maxlen; |
71 | |
|
72 | 0 | while (remaining > 0) { |
73 | 0 | size_t to_copy = (remaining < SSIZE_MAX) ? remaining : SSIZE_MAX; |
74 | 0 | ssize_t result = copy_file_range(src_fd, NULL, dest_fd, NULL, to_copy, 0); |
75 | |
|
76 | 0 | if (result > 0) { |
77 | 0 | total_copied += result; |
78 | 0 | if (maxlen != PHP_IO_COPY_ALL) { |
79 | 0 | remaining -= result; |
80 | 0 | } |
81 | 0 | } else if (result == 0) { |
82 | 0 | break; |
83 | 0 | } else { |
84 | 0 | switch (errno) { |
85 | 0 | case EINVAL: |
86 | 0 | case EXDEV: |
87 | 0 | case ENOSYS: |
88 | 0 | case EIO: |
89 | 0 | if (total_copied == 0) { |
90 | 0 | return php_io_generic_copy_fallback(src_fd, dest_fd, maxlen, copied); |
91 | 0 | } |
92 | 0 | break; |
93 | 0 | default: |
94 | 0 | *copied = total_copied; |
95 | 0 | return FAILURE; |
96 | 0 | } |
97 | 0 | break; |
98 | 0 | } |
99 | 0 | } |
100 | | |
101 | 0 | if (total_copied > 0) { |
102 | 0 | *copied = total_copied; |
103 | 0 | return SUCCESS; |
104 | 0 | } |
105 | 0 | #endif |
106 | | |
107 | 0 | return php_io_generic_copy_fallback(src_fd, dest_fd, maxlen, copied); |
108 | 0 | } |
109 | | |
110 | | static zend_result php_io_linux_sendfile(int src_fd, int dest_fd, size_t maxlen, size_t *copied) |
111 | 0 | { |
112 | 0 | #ifdef HAVE_SENDFILE |
113 | 0 | size_t total_copied = 0; |
114 | 0 | size_t remaining = (maxlen == PHP_IO_COPY_ALL) ? SIZE_MAX : maxlen; |
115 | |
|
116 | 0 | while (remaining > 0) { |
117 | 0 | size_t to_send = (remaining < SSIZE_MAX) ? remaining : SSIZE_MAX; |
118 | 0 | ssize_t result = sendfile(dest_fd, src_fd, NULL, to_send); |
119 | |
|
120 | 0 | if (result > 0) { |
121 | 0 | total_copied += result; |
122 | 0 | if (maxlen != PHP_IO_COPY_ALL) { |
123 | 0 | remaining -= result; |
124 | 0 | } |
125 | 0 | } else if (result == 0) { |
126 | 0 | break; |
127 | 0 | } else { |
128 | 0 | switch (errno) { |
129 | 0 | case EINTR: |
130 | 0 | continue; |
131 | 0 | case EINVAL: |
132 | 0 | case ENOSYS: |
133 | 0 | if (total_copied == 0) { |
134 | 0 | return php_io_generic_copy_fallback(src_fd, dest_fd, maxlen, copied); |
135 | 0 | } |
136 | 0 | break; |
137 | 0 | case EAGAIN: |
138 | 0 | break; |
139 | 0 | default: |
140 | 0 | *copied = total_copied; |
141 | 0 | return FAILURE; |
142 | 0 | } |
143 | 0 | break; |
144 | 0 | } |
145 | 0 | } |
146 | | |
147 | 0 | if (total_copied > 0) { |
148 | 0 | *copied = total_copied; |
149 | 0 | return SUCCESS; |
150 | 0 | } |
151 | 0 | #endif |
152 | | |
153 | 0 | return php_io_generic_copy_fallback(src_fd, dest_fd, maxlen, copied); |
154 | 0 | } |
155 | | |
156 | | #ifdef HAVE_SPLICE |
157 | | /* Enlarge the intermediate pipe so socket transfers move more data per splice |
158 | | * round-trip. Capped by /proc/sys/fs/pipe-max-size (1 MiB by default); a failed |
159 | | * fcntl() simply leaves the kernel default (64 KiB) in place. */ |
160 | 0 | #define PHP_IO_PIPE_SIZE (1 << 20) |
161 | | |
162 | | /* Kernel MAX_RW_COUNT; a larger len fails the pos + len overflow check with |
163 | | * EINVAL once a file destination sits at a non-zero offset. */ |
164 | 0 | #define PHP_IO_SPLICE_MAX ((size_t) 0x7ffff000) |
165 | | |
166 | | /* SPLICE_F_MORE corks the socket the same way MSG_MORE does, letting the kernel |
167 | | * coalesce splices into full segments. The final partial segment is not flushed |
168 | | * by the kernel until the cork timer (<= 200 ms) expires or the socket is next |
169 | | * written/closed, so the copy loops below clear the cork once they are done. */ |
170 | | static inline unsigned int php_io_linux_out_flags(const php_io_fd *dest) |
171 | 0 | { |
172 | 0 | return (dest->fd_type == PHP_IO_FD_SOCKET) ? SPLICE_F_MORE : 0; |
173 | 0 | } |
174 | | |
175 | | /* Clearing TCP_CORK pushes out any segment still held by SPLICE_F_MORE; |
176 | | * on non-TCP sockets the setsockopt() harmlessly fails. */ |
177 | | static inline void php_io_linux_socket_uncork(const php_io_fd *dest, size_t total_copied) |
178 | 0 | { |
179 | 0 | if (dest->fd_type == PHP_IO_FD_SOCKET && total_copied > 0) { |
180 | 0 | int off = 0; |
181 | 0 | setsockopt(dest->fd, IPPROTO_TCP, TCP_CORK, &off, sizeof(off)); |
182 | 0 | } |
183 | 0 | } |
184 | | |
185 | | static zend_result php_io_linux_splice_from_pipe(php_io_fd *src, php_io_fd *dest, size_t maxlen, size_t *copied) |
186 | 0 | { |
187 | 0 | int dest_fd = dest->fd; |
188 | 0 | unsigned int out_flags = php_io_linux_out_flags(dest); |
189 | 0 | size_t total_copied = 0; |
190 | 0 | size_t remaining = (maxlen == PHP_IO_COPY_ALL) ? SIZE_MAX : maxlen; |
191 | 0 | zend_result result = SUCCESS; |
192 | |
|
193 | 0 | while (remaining > 0) { |
194 | 0 | int ready = php_io_linux_wait_for_data(src); |
195 | 0 | if (ready == 0) { |
196 | 0 | break; |
197 | 0 | } else if (ready < 0) { |
198 | 0 | result = FAILURE; |
199 | 0 | break; |
200 | 0 | } |
201 | | |
202 | 0 | size_t to_copy = (remaining < PHP_IO_SPLICE_MAX) ? remaining : PHP_IO_SPLICE_MAX; |
203 | 0 | ssize_t spliced = splice(src->fd, NULL, dest_fd, NULL, to_copy, out_flags); |
204 | |
|
205 | 0 | if (spliced > 0) { |
206 | 0 | total_copied += spliced; |
207 | 0 | if (maxlen != PHP_IO_COPY_ALL) { |
208 | 0 | remaining -= spliced; |
209 | 0 | } |
210 | 0 | } else if (spliced == 0) { |
211 | 0 | break; |
212 | 0 | } else { |
213 | 0 | if (total_copied == 0) { |
214 | 0 | return php_io_generic_copy_fallback(src->fd, dest_fd, maxlen, copied); |
215 | 0 | } |
216 | 0 | result = FAILURE; |
217 | 0 | break; |
218 | 0 | } |
219 | 0 | } |
220 | | |
221 | 0 | php_io_linux_socket_uncork(dest, total_copied); |
222 | 0 | *copied = total_copied; |
223 | 0 | return result; |
224 | 0 | } |
225 | | |
226 | | static zend_result php_io_linux_splice_via_pipe(php_io_fd *src, php_io_fd *dest, size_t maxlen, size_t *copied) |
227 | 0 | { |
228 | 0 | int dest_fd = dest->fd; |
229 | 0 | unsigned int out_flags = php_io_linux_out_flags(dest); |
230 | 0 | int pipefd[2]; |
231 | 0 | if (pipe(pipefd) == -1) { |
232 | 0 | return php_io_generic_copy_fallback(src->fd, dest_fd, maxlen, copied); |
233 | 0 | } |
234 | | |
235 | 0 | #ifdef F_SETPIPE_SZ |
236 | 0 | fcntl(pipefd[1], F_SETPIPE_SZ, PHP_IO_PIPE_SIZE); |
237 | 0 | #endif |
238 | |
|
239 | 0 | size_t total_copied = 0; |
240 | 0 | size_t remaining = (maxlen == PHP_IO_COPY_ALL) ? SIZE_MAX : maxlen; |
241 | 0 | zend_result result = SUCCESS; |
242 | |
|
243 | 0 | while (remaining > 0) { |
244 | 0 | int ready = php_io_linux_wait_for_data(src); |
245 | 0 | if (ready == 0) { |
246 | | /* timeout */ |
247 | 0 | break; |
248 | 0 | } else if (ready < 0) { |
249 | 0 | result = FAILURE; |
250 | 0 | break; |
251 | 0 | } |
252 | | |
253 | 0 | size_t to_copy = (remaining < SSIZE_MAX) ? remaining : SSIZE_MAX; |
254 | |
|
255 | 0 | ssize_t in_pipe = splice(src->fd, NULL, pipefd[1], NULL, to_copy, 0); |
256 | 0 | if (in_pipe < 0) { |
257 | 0 | if (total_copied == 0) { |
258 | 0 | close(pipefd[0]); |
259 | 0 | close(pipefd[1]); |
260 | 0 | return php_io_generic_copy_fallback(src->fd, dest_fd, maxlen, copied); |
261 | 0 | } |
262 | 0 | result = FAILURE; |
263 | 0 | break; |
264 | 0 | } |
265 | 0 | if (in_pipe == 0) { |
266 | 0 | break; |
267 | 0 | } |
268 | | |
269 | 0 | size_t pipe_remaining = in_pipe; |
270 | 0 | while (pipe_remaining > 0) { |
271 | 0 | ssize_t out = splice(pipefd[0], NULL, dest_fd, NULL, pipe_remaining, out_flags); |
272 | 0 | if (out <= 0) { |
273 | | /* the dest refused the splice; salvage what already sits in |
274 | | * the pipe with a plain read/write loop before failing */ |
275 | 0 | char drain_buf[1024]; |
276 | 0 | while (pipe_remaining > 0) { |
277 | 0 | size_t to_drain = (pipe_remaining < sizeof(drain_buf)) |
278 | 0 | ? pipe_remaining : sizeof(drain_buf); |
279 | 0 | ssize_t drained; |
280 | 0 | do { |
281 | 0 | drained = read(pipefd[0], drain_buf, to_drain); |
282 | 0 | } while (drained < 0 && errno == EINTR); |
283 | 0 | if (drained <= 0) { |
284 | 0 | break; |
285 | 0 | } |
286 | 0 | ssize_t drain_written = 0; |
287 | 0 | while (drain_written < drained) { |
288 | 0 | ssize_t written; |
289 | 0 | do { |
290 | 0 | written = write(dest_fd, drain_buf + drain_written, drained - drain_written); |
291 | 0 | } while (written < 0 && errno == EINTR); |
292 | 0 | if (written <= 0) { |
293 | 0 | total_copied += drain_written; |
294 | 0 | result = FAILURE; |
295 | 0 | goto out; |
296 | 0 | } |
297 | 0 | drain_written += written; |
298 | 0 | } |
299 | 0 | pipe_remaining -= drain_written; |
300 | 0 | total_copied += drain_written; |
301 | 0 | } |
302 | 0 | result = FAILURE; |
303 | 0 | goto out; |
304 | 0 | } |
305 | 0 | pipe_remaining -= out; |
306 | 0 | total_copied += out; |
307 | 0 | } |
308 | | |
309 | 0 | if (maxlen != PHP_IO_COPY_ALL) { |
310 | 0 | remaining -= in_pipe; |
311 | 0 | } |
312 | 0 | } |
313 | | |
314 | 0 | out: |
315 | 0 | close(pipefd[0]); |
316 | 0 | close(pipefd[1]); |
317 | 0 | php_io_linux_socket_uncork(dest, total_copied); |
318 | 0 | *copied = total_copied; |
319 | 0 | return result; |
320 | 0 | } |
321 | | #endif /* HAVE_SPLICE */ |
322 | | |
323 | | zend_result php_io_linux_copy(php_io_fd *src, php_io_fd *dest, size_t maxlen, size_t *copied) |
324 | 0 | { |
325 | 0 | if (src->fd_type == PHP_IO_FD_FILE && dest->fd_type == PHP_IO_FD_FILE) { |
326 | 0 | return php_io_linux_copy_file_to_file(src->fd, dest->fd, maxlen, copied); |
327 | 0 | } |
328 | | |
329 | 0 | if (src->fd_type == PHP_IO_FD_FILE && dest->fd_type == PHP_IO_FD_SOCKET) { |
330 | 0 | return php_io_linux_sendfile(src->fd, dest->fd, maxlen, copied); |
331 | 0 | } |
332 | | |
333 | 0 | if (src->fd_type == PHP_IO_FD_FILE && dest->fd_type == PHP_IO_FD_PIPE) { |
334 | 0 | return php_io_linux_sendfile(src->fd, dest->fd, maxlen, copied); |
335 | 0 | } |
336 | | |
337 | 0 | #ifdef HAVE_SPLICE |
338 | 0 | if (src->fd_type == PHP_IO_FD_PIPE) { |
339 | 0 | return php_io_linux_splice_from_pipe(src, dest, maxlen, copied); |
340 | 0 | } |
341 | | |
342 | 0 | if (src->fd_type == PHP_IO_FD_SOCKET) { |
343 | 0 | return php_io_linux_splice_via_pipe(src, dest, maxlen, copied); |
344 | 0 | } |
345 | 0 | #endif |
346 | | |
347 | | /* php_io_generic_copy honours the stream timeout for socket sources */ |
348 | 0 | return php_io_generic_copy(src, dest, maxlen, copied); |
349 | 0 | } |
350 | | |
351 | | #endif /* __linux__ */ |