Line data Source code
1 : // Copyright 2018 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef V8_MEMCOPY_H_
6 : #define V8_MEMCOPY_H_
7 :
8 : #include <stdint.h>
9 : #include <stdlib.h>
10 : #include <string.h>
11 :
12 : #include "src/base/logging.h"
13 : #include "src/base/macros.h"
14 :
15 : namespace v8 {
16 : namespace internal {
17 :
18 : typedef uintptr_t Address;
19 :
20 : // ----------------------------------------------------------------------------
21 : // Generated memcpy/memmove for ia32, arm, and mips.
22 :
23 : void init_memcopy_functions();
24 :
25 : #if defined(V8_TARGET_ARCH_IA32)
26 : // Limit below which the extra overhead of the MemCopy function is likely
27 : // to outweigh the benefits of faster copying.
28 : const int kMinComplexMemCopy = 64;
29 :
30 : // Copy memory area. No restrictions.
31 : V8_EXPORT_PRIVATE void MemMove(void* dest, const void* src, size_t size);
32 : typedef void (*MemMoveFunction)(void* dest, const void* src, size_t size);
33 :
34 : // Keep the distinction of "move" vs. "copy" for the benefit of other
35 : // architectures.
36 : V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
37 : MemMove(dest, src, size);
38 : }
39 : #elif defined(V8_HOST_ARCH_ARM)
40 : typedef void (*MemCopyUint8Function)(uint8_t* dest, const uint8_t* src,
41 : size_t size);
42 : V8_EXPORT_PRIVATE extern MemCopyUint8Function memcopy_uint8_function;
43 : V8_INLINE void MemCopyUint8Wrapper(uint8_t* dest, const uint8_t* src,
44 : size_t chars) {
45 : memcpy(dest, src, chars);
46 : }
47 : // For values < 16, the assembler function is slower than the inlined C code.
48 : const int kMinComplexMemCopy = 16;
49 : V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
50 : (*memcopy_uint8_function)(reinterpret_cast<uint8_t*>(dest),
51 : reinterpret_cast<const uint8_t*>(src), size);
52 : }
53 : V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
54 : size_t size) {
55 : memmove(dest, src, size);
56 : }
57 :
58 : typedef void (*MemCopyUint16Uint8Function)(uint16_t* dest, const uint8_t* src,
59 : size_t size);
60 : extern MemCopyUint16Uint8Function memcopy_uint16_uint8_function;
61 : void MemCopyUint16Uint8Wrapper(uint16_t* dest, const uint8_t* src,
62 : size_t chars);
63 : // For values < 12, the assembler function is slower than the inlined C code.
64 : const int kMinComplexConvertMemCopy = 12;
65 : V8_INLINE void MemCopyUint16Uint8(uint16_t* dest, const uint8_t* src,
66 : size_t size) {
67 : (*memcopy_uint16_uint8_function)(dest, src, size);
68 : }
69 : #elif defined(V8_HOST_ARCH_MIPS)
70 : typedef void (*MemCopyUint8Function)(uint8_t* dest, const uint8_t* src,
71 : size_t size);
72 : V8_EXPORT_PRIVATE extern MemCopyUint8Function memcopy_uint8_function;
73 : V8_INLINE void MemCopyUint8Wrapper(uint8_t* dest, const uint8_t* src,
74 : size_t chars) {
75 : memcpy(dest, src, chars);
76 : }
77 : // For values < 16, the assembler function is slower than the inlined C code.
78 : const int kMinComplexMemCopy = 16;
79 : V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
80 : (*memcopy_uint8_function)(reinterpret_cast<uint8_t*>(dest),
81 : reinterpret_cast<const uint8_t*>(src), size);
82 : }
83 : V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
84 : size_t size) {
85 : memmove(dest, src, size);
86 : }
87 : #else
88 : // Copy memory area to disjoint memory area.
89 : V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
90 : memcpy(dest, src, size);
91 : }
92 : V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
93 : size_t size) {
94 : memmove(dest, src, size);
95 : }
96 : const int kMinComplexMemCopy = 8;
97 : #endif // V8_TARGET_ARCH_IA32
98 :
99 : // Copies words from |src| to |dst|. The data spans must not overlap.
100 : // |src| and |dst| must be kSystemPointerSize-aligned.
101 186841442 : inline void CopyWords(Address dst, const Address src, size_t num_words) {
102 : constexpr int kSystemPointerSize = sizeof(void*); // to avoid src/globals.h
103 : DCHECK(IsAligned(dst, kSystemPointerSize));
104 : DCHECK(IsAligned(src, kSystemPointerSize));
105 : DCHECK(((src <= dst) && ((src + num_words * kSystemPointerSize) <= dst)) ||
106 : ((dst <= src) && ((dst + num_words * kSystemPointerSize) <= src)));
107 :
108 : // Use block copying MemCopy if the segment we're copying is
109 : // enough to justify the extra call/setup overhead.
110 : static const size_t kBlockCopyLimit = 16;
111 :
112 186841442 : Address* dst_ptr = reinterpret_cast<Address*>(dst);
113 186841442 : Address* src_ptr = reinterpret_cast<Address*>(src);
114 186841442 : if (num_words < kBlockCopyLimit) {
115 694766859 : do {
116 694766859 : num_words--;
117 694766859 : *dst_ptr++ = *src_ptr++;
118 : } while (num_words > 0);
119 : } else {
120 5194308 : MemCopy(dst_ptr, src_ptr, num_words * kSystemPointerSize);
121 : }
122 186841442 : }
123 :
124 : // Copies data from |src| to |dst|. The data spans must not overlap.
125 : template <typename T>
126 6520165 : inline void CopyBytes(T* dst, const T* src, size_t num_bytes) {
127 : STATIC_ASSERT(sizeof(T) == 1);
128 : DCHECK(((src <= dst) && ((src + num_bytes) <= dst)) ||
129 : ((dst <= src) && ((dst + num_bytes) <= src)));
130 13040330 : if (num_bytes == 0) return;
131 :
132 : // Use block copying MemCopy if the segment we're copying is
133 : // enough to justify the extra call/setup overhead.
134 : static const int kBlockCopyLimit = kMinComplexMemCopy;
135 :
136 6146672 : if (num_bytes < static_cast<size_t>(kBlockCopyLimit)) {
137 4727663 : do {
138 4727663 : num_bytes--;
139 4727663 : *dst++ = *src++;
140 : } while (num_bytes > 0);
141 : } else {
142 : MemCopy(dst, src, num_bytes);
143 : }
144 : }
145 :
146 : inline void MemsetPointer(Address* dest, Address value, size_t counter) {
147 : #if V8_HOST_ARCH_IA32
148 : #define STOS "stosl"
149 : #elif V8_HOST_ARCH_X64
150 : #define STOS "stosq"
151 : #endif
152 :
153 : #if defined(MEMORY_SANITIZER)
154 : // MemorySanitizer does not understand inline assembly.
155 : #undef STOS
156 : #endif
157 :
158 : #if defined(__GNUC__) && defined(STOS)
159 : asm volatile(
160 : "cld;"
161 : "rep ; " STOS
162 : : "+&c"(counter), "+&D"(dest)
163 : : "a"(value)
164 54524471 : : "memory", "cc");
165 : #else
166 : for (size_t i = 0; i < counter; i++) {
167 : dest[i] = value;
168 : }
169 : #endif
170 :
171 : #undef STOS
172 : }
173 :
174 : template <typename T, typename U>
175 : inline void MemsetPointer(T** dest, U* value, size_t counter) {
176 : #ifdef DEBUG
177 : T* a = nullptr;
178 : U* b = nullptr;
179 : a = b; // Fake assignment to check assignability.
180 : USE(a);
181 : #endif // DEBUG
182 : MemsetPointer(reinterpret_cast<Address*>(dest),
183 609977 : reinterpret_cast<Address>(value), counter);
184 : }
185 :
186 : template <typename sourcechar, typename sinkchar>
187 : V8_INLINE static void CopyCharsUnsigned(sinkchar* dest, const sourcechar* src,
188 : size_t chars);
189 : #if defined(V8_HOST_ARCH_ARM)
190 : V8_INLINE void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src,
191 : size_t chars);
192 : V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint8_t* src,
193 : size_t chars);
194 : V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src,
195 : size_t chars);
196 : #elif defined(V8_HOST_ARCH_MIPS)
197 : V8_INLINE void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src,
198 : size_t chars);
199 : V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src,
200 : size_t chars);
201 : #elif defined(V8_HOST_ARCH_PPC) || defined(V8_HOST_ARCH_S390)
202 : V8_INLINE void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src,
203 : size_t chars);
204 : V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src,
205 : size_t chars);
206 : #endif
207 :
208 : // Copy from 8bit/16bit chars to 8bit/16bit chars.
209 : template <typename sourcechar, typename sinkchar>
210 : V8_INLINE void CopyChars(sinkchar* dest, const sourcechar* src, size_t chars);
211 :
212 : template <typename sourcechar, typename sinkchar>
213 : void CopyChars(sinkchar* dest, const sourcechar* src, size_t chars) {
214 : DCHECK_LE(sizeof(sourcechar), 2);
215 : DCHECK_LE(sizeof(sinkchar), 2);
216 : if (sizeof(sinkchar) == 1) {
217 : if (sizeof(sourcechar) == 1) {
218 : CopyCharsUnsigned(reinterpret_cast<uint8_t*>(dest),
219 : reinterpret_cast<const uint8_t*>(src), chars);
220 : } else {
221 : CopyCharsUnsigned(reinterpret_cast<uint8_t*>(dest),
222 : reinterpret_cast<const uint16_t*>(src), chars);
223 : }
224 : } else {
225 : if (sizeof(sourcechar) == 1) {
226 : CopyCharsUnsigned(reinterpret_cast<uint16_t*>(dest),
227 : reinterpret_cast<const uint8_t*>(src), chars);
228 : } else {
229 : CopyCharsUnsigned(reinterpret_cast<uint16_t*>(dest),
230 : reinterpret_cast<const uint16_t*>(src), chars);
231 : }
232 : }
233 : }
234 :
235 : template <typename sourcechar, typename sinkchar>
236 : void CopyCharsUnsigned(sinkchar* dest, const sourcechar* src, size_t chars) {
237 238218213 : sinkchar* limit = dest + chars;
238 211379325 : if ((sizeof(*dest) == sizeof(*src)) &&
239 : (chars >= static_cast<int>(kMinComplexMemCopy / sizeof(*dest)))) {
240 : MemCopy(dest, src, chars * sizeof(*dest));
241 : } else {
242 5231973332 : while (dest < limit) *dest++ = static_cast<sinkchar>(*src++);
243 : }
244 : }
245 :
246 : #if defined(V8_HOST_ARCH_ARM)
247 : void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src, size_t chars) {
248 : switch (static_cast<unsigned>(chars)) {
249 : case 0:
250 : break;
251 : case 1:
252 : *dest = *src;
253 : break;
254 : case 2:
255 : memcpy(dest, src, 2);
256 : break;
257 : case 3:
258 : memcpy(dest, src, 3);
259 : break;
260 : case 4:
261 : memcpy(dest, src, 4);
262 : break;
263 : case 5:
264 : memcpy(dest, src, 5);
265 : break;
266 : case 6:
267 : memcpy(dest, src, 6);
268 : break;
269 : case 7:
270 : memcpy(dest, src, 7);
271 : break;
272 : case 8:
273 : memcpy(dest, src, 8);
274 : break;
275 : case 9:
276 : memcpy(dest, src, 9);
277 : break;
278 : case 10:
279 : memcpy(dest, src, 10);
280 : break;
281 : case 11:
282 : memcpy(dest, src, 11);
283 : break;
284 : case 12:
285 : memcpy(dest, src, 12);
286 : break;
287 : case 13:
288 : memcpy(dest, src, 13);
289 : break;
290 : case 14:
291 : memcpy(dest, src, 14);
292 : break;
293 : case 15:
294 : memcpy(dest, src, 15);
295 : break;
296 : default:
297 : MemCopy(dest, src, chars);
298 : break;
299 : }
300 : }
301 :
302 : void CopyCharsUnsigned(uint16_t* dest, const uint8_t* src, size_t chars) {
303 : if (chars >= static_cast<size_t>(kMinComplexConvertMemCopy)) {
304 : MemCopyUint16Uint8(dest, src, chars);
305 : } else {
306 : MemCopyUint16Uint8Wrapper(dest, src, chars);
307 : }
308 : }
309 :
310 : void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src, size_t chars) {
311 : switch (static_cast<unsigned>(chars)) {
312 : case 0:
313 : break;
314 : case 1:
315 : *dest = *src;
316 : break;
317 : case 2:
318 : memcpy(dest, src, 4);
319 : break;
320 : case 3:
321 : memcpy(dest, src, 6);
322 : break;
323 : case 4:
324 : memcpy(dest, src, 8);
325 : break;
326 : case 5:
327 : memcpy(dest, src, 10);
328 : break;
329 : case 6:
330 : memcpy(dest, src, 12);
331 : break;
332 : case 7:
333 : memcpy(dest, src, 14);
334 : break;
335 : default:
336 : MemCopy(dest, src, chars * sizeof(*dest));
337 : break;
338 : }
339 : }
340 :
341 : #elif defined(V8_HOST_ARCH_MIPS)
342 : void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src, size_t chars) {
343 : if (chars < kMinComplexMemCopy) {
344 : memcpy(dest, src, chars);
345 : } else {
346 : MemCopy(dest, src, chars);
347 : }
348 : }
349 :
350 : void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src, size_t chars) {
351 : if (chars < kMinComplexMemCopy) {
352 : memcpy(dest, src, chars * sizeof(*dest));
353 : } else {
354 : MemCopy(dest, src, chars * sizeof(*dest));
355 : }
356 : }
357 : #elif defined(V8_HOST_ARCH_PPC) || defined(V8_HOST_ARCH_S390)
358 : #define CASE(n) \
359 : case n: \
360 : memcpy(dest, src, n); \
361 : break
362 : void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src, size_t chars) {
363 : switch (static_cast<unsigned>(chars)) {
364 : case 0:
365 : break;
366 : case 1:
367 : *dest = *src;
368 : break;
369 : CASE(2);
370 : CASE(3);
371 : CASE(4);
372 : CASE(5);
373 : CASE(6);
374 : CASE(7);
375 : CASE(8);
376 : CASE(9);
377 : CASE(10);
378 : CASE(11);
379 : CASE(12);
380 : CASE(13);
381 : CASE(14);
382 : CASE(15);
383 : CASE(16);
384 : CASE(17);
385 : CASE(18);
386 : CASE(19);
387 : CASE(20);
388 : CASE(21);
389 : CASE(22);
390 : CASE(23);
391 : CASE(24);
392 : CASE(25);
393 : CASE(26);
394 : CASE(27);
395 : CASE(28);
396 : CASE(29);
397 : CASE(30);
398 : CASE(31);
399 : CASE(32);
400 : CASE(33);
401 : CASE(34);
402 : CASE(35);
403 : CASE(36);
404 : CASE(37);
405 : CASE(38);
406 : CASE(39);
407 : CASE(40);
408 : CASE(41);
409 : CASE(42);
410 : CASE(43);
411 : CASE(44);
412 : CASE(45);
413 : CASE(46);
414 : CASE(47);
415 : CASE(48);
416 : CASE(49);
417 : CASE(50);
418 : CASE(51);
419 : CASE(52);
420 : CASE(53);
421 : CASE(54);
422 : CASE(55);
423 : CASE(56);
424 : CASE(57);
425 : CASE(58);
426 : CASE(59);
427 : CASE(60);
428 : CASE(61);
429 : CASE(62);
430 : CASE(63);
431 : CASE(64);
432 : default:
433 : memcpy(dest, src, chars);
434 : break;
435 : }
436 : }
437 : #undef CASE
438 :
439 : #define CASE(n) \
440 : case n: \
441 : memcpy(dest, src, n * 2); \
442 : break
443 : void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src, size_t chars) {
444 : switch (static_cast<unsigned>(chars)) {
445 : case 0:
446 : break;
447 : case 1:
448 : *dest = *src;
449 : break;
450 : CASE(2);
451 : CASE(3);
452 : CASE(4);
453 : CASE(5);
454 : CASE(6);
455 : CASE(7);
456 : CASE(8);
457 : CASE(9);
458 : CASE(10);
459 : CASE(11);
460 : CASE(12);
461 : CASE(13);
462 : CASE(14);
463 : CASE(15);
464 : CASE(16);
465 : CASE(17);
466 : CASE(18);
467 : CASE(19);
468 : CASE(20);
469 : CASE(21);
470 : CASE(22);
471 : CASE(23);
472 : CASE(24);
473 : CASE(25);
474 : CASE(26);
475 : CASE(27);
476 : CASE(28);
477 : CASE(29);
478 : CASE(30);
479 : CASE(31);
480 : CASE(32);
481 : default:
482 : memcpy(dest, src, chars * 2);
483 : break;
484 : }
485 : }
486 : #undef CASE
487 : #endif
488 :
489 : } // namespace internal
490 : } // namespace v8
491 :
492 : #endif // V8_MEMCOPY_H_
|
