Line data Source code
1 : // Copyright 2018 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef V8_MEMCOPY_H_
6 : #define V8_MEMCOPY_H_
7 :
8 : #include <stdint.h>
9 : #include <stdlib.h>
10 : #include <string.h>
11 :
12 : #include "src/base/logging.h"
13 : #include "src/base/macros.h"
14 :
15 : namespace v8 {
16 : namespace internal {
17 :
18 : typedef uintptr_t Address;
19 :
20 : // ----------------------------------------------------------------------------
21 : // Generated memcpy/memmove for ia32, arm, and mips.
22 :
23 : void init_memcopy_functions();
24 :
25 : #if defined(V8_TARGET_ARCH_IA32)
26 : // Limit below which the extra overhead of the MemCopy function is likely
27 : // to outweigh the benefits of faster copying.
28 : const size_t kMinComplexMemCopy = 64;
29 :
30 : // Copy memory area. No restrictions.
31 : V8_EXPORT_PRIVATE void MemMove(void* dest, const void* src, size_t size);
32 : typedef void (*MemMoveFunction)(void* dest, const void* src, size_t size);
33 :
34 : // Keep the distinction of "move" vs. "copy" for the benefit of other
35 : // architectures.
36 : V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
37 : MemMove(dest, src, size);
38 : }
39 : #elif defined(V8_HOST_ARCH_ARM)
40 : typedef void (*MemCopyUint8Function)(uint8_t* dest, const uint8_t* src,
41 : size_t size);
42 : V8_EXPORT_PRIVATE extern MemCopyUint8Function memcopy_uint8_function;
43 : V8_INLINE void MemCopyUint8Wrapper(uint8_t* dest, const uint8_t* src,
44 : size_t chars) {
45 : memcpy(dest, src, chars);
46 : }
47 : // For values < 16, the assembler function is slower than the inlined C code.
48 : const size_t kMinComplexMemCopy = 16;
49 : V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
50 : (*memcopy_uint8_function)(reinterpret_cast<uint8_t*>(dest),
51 : reinterpret_cast<const uint8_t*>(src), size);
52 : }
53 : V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
54 : size_t size) {
55 : memmove(dest, src, size);
56 : }
57 :
58 : typedef void (*MemCopyUint16Uint8Function)(uint16_t* dest, const uint8_t* src,
59 : size_t size);
60 : extern MemCopyUint16Uint8Function memcopy_uint16_uint8_function;
61 : void MemCopyUint16Uint8Wrapper(uint16_t* dest, const uint8_t* src,
62 : size_t chars);
63 : // For values < 12, the assembler function is slower than the inlined C code.
64 : const int kMinComplexConvertMemCopy = 12;
65 : V8_INLINE void MemCopyUint16Uint8(uint16_t* dest, const uint8_t* src,
66 : size_t size) {
67 : (*memcopy_uint16_uint8_function)(dest, src, size);
68 : }
69 : #elif defined(V8_HOST_ARCH_MIPS)
70 : typedef void (*MemCopyUint8Function)(uint8_t* dest, const uint8_t* src,
71 : size_t size);
72 : V8_EXPORT_PRIVATE extern MemCopyUint8Function memcopy_uint8_function;
73 : V8_INLINE void MemCopyUint8Wrapper(uint8_t* dest, const uint8_t* src,
74 : size_t chars) {
75 : memcpy(dest, src, chars);
76 : }
77 : // For values < 16, the assembler function is slower than the inlined C code.
78 : const size_t kMinComplexMemCopy = 16;
79 : V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
80 : (*memcopy_uint8_function)(reinterpret_cast<uint8_t*>(dest),
81 : reinterpret_cast<const uint8_t*>(src), size);
82 : }
83 : V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
84 : size_t size) {
85 : memmove(dest, src, size);
86 : }
87 : #else
88 : // Copy memory area to disjoint memory area.
89 : V8_INLINE void MemCopy(void* dest, const void* src, size_t size) {
90 : memcpy(dest, src, size);
91 : }
92 : V8_EXPORT_PRIVATE V8_INLINE void MemMove(void* dest, const void* src,
93 : size_t size) {
94 : memmove(dest, src, size);
95 : }
96 : const size_t kMinComplexMemCopy = 8;
97 : #endif // V8_TARGET_ARCH_IA32
98 :
99 : // Copies words from |src| to |dst|. The data spans must not overlap.
100 : // |src| and |dst| must be TWord-size aligned.
101 : template <size_t kBlockCopyLimit, typename T>
102 181246743 : inline void CopyImpl(T* dst_ptr, const T* src_ptr, size_t count) {
103 : constexpr int kTWordSize = sizeof(T);
104 : #ifdef DEBUG
105 : Address dst = reinterpret_cast<Address>(dst_ptr);
106 : Address src = reinterpret_cast<Address>(src_ptr);
107 : DCHECK(IsAligned(dst, kTWordSize));
108 : DCHECK(IsAligned(src, kTWordSize));
109 : DCHECK(((src <= dst) && ((src + count * kTWordSize) <= dst)) ||
110 : ((dst <= src) && ((dst + count * kTWordSize) <= src)));
111 : #endif
112 :
113 : // Use block copying MemCopy if the segment we're copying is
114 : // enough to justify the extra call/setup overhead.
115 181246743 : if (count < kBlockCopyLimit) {
116 634068690 : do {
117 634068690 : count--;
118 634068690 : *dst_ptr++ = *src_ptr++;
119 : } while (count > 0);
120 : } else {
121 5170106 : MemCopy(dst_ptr, src_ptr, count * kTWordSize);
122 : }
123 181246743 : }
124 :
125 : // Copies kSystemPointerSize-sized words from |src| to |dst|. The data spans
126 : // must not overlap. |src| and |dst| must be kSystemPointerSize-aligned.
127 : inline void CopyWords(Address dst, const Address src, size_t num_words) {
128 : static const size_t kBlockCopyLimit = 16;
129 1133 : CopyImpl<kBlockCopyLimit>(reinterpret_cast<Address*>(dst),
130 1133 : reinterpret_cast<const Address*>(src), num_words);
131 : }
132 :
133 : // Copies data from |src| to |dst|. The data spans must not overlap.
134 : template <typename T>
135 : inline void CopyBytes(T* dst, const T* src, size_t num_bytes) {
136 : STATIC_ASSERT(sizeof(T) == 1);
137 5938291 : if (num_bytes == 0) return;
138 5614121 : CopyImpl<kMinComplexMemCopy>(dst, src, num_bytes);
139 : }
140 :
141 : inline void MemsetInt32(int32_t* dest, int32_t value, size_t counter) {
142 : #if V8_HOST_ARCH_IA32 || V8_HOST_ARCH_X64
143 : #define STOS "stosl"
144 : #endif
145 :
146 : #if defined(MEMORY_SANITIZER)
147 : // MemorySanitizer does not understand inline assembly.
148 : #undef STOS
149 : #endif
150 :
151 : #if defined(__GNUC__) && defined(STOS)
152 : asm volatile(
153 : "cld;"
154 : "rep ; " STOS
155 : : "+&c"(counter), "+&D"(dest)
156 : : "a"(value)
157 : : "memory", "cc");
158 : #else
159 : for (size_t i = 0; i < counter; i++) {
160 : dest[i] = value;
161 : }
162 : #endif
163 :
164 : #undef STOS
165 : }
166 :
167 : inline void MemsetPointer(Address* dest, Address value, size_t counter) {
168 : #if V8_HOST_ARCH_IA32
169 : #define STOS "stosl"
170 : #elif V8_HOST_ARCH_X64
171 : #define STOS "stosq"
172 : #endif
173 :
174 : #if defined(MEMORY_SANITIZER)
175 : // MemorySanitizer does not understand inline assembly.
176 : #undef STOS
177 : #endif
178 :
179 : #if defined(__GNUC__) && defined(STOS)
180 : asm volatile(
181 : "cld;"
182 : "rep ; " STOS
183 : : "+&c"(counter), "+&D"(dest)
184 : : "a"(value)
185 62415235 : : "memory", "cc");
186 : #else
187 : for (size_t i = 0; i < counter; i++) {
188 : dest[i] = value;
189 : }
190 : #endif
191 :
192 : #undef STOS
193 : }
194 :
195 : template <typename T, typename U>
196 : inline void MemsetPointer(T** dest, U* value, size_t counter) {
197 : #ifdef DEBUG
198 : T* a = nullptr;
199 : U* b = nullptr;
200 : a = b; // Fake assignment to check assignability.
201 : USE(a);
202 : #endif // DEBUG
203 605298 : MemsetPointer(reinterpret_cast<Address*>(dest),
204 : reinterpret_cast<Address>(value), counter);
205 : }
206 :
207 : template <typename sourcechar, typename sinkchar>
208 : V8_INLINE static void CopyCharsUnsigned(sinkchar* dest, const sourcechar* src,
209 : size_t chars);
210 : #if defined(V8_HOST_ARCH_ARM)
211 : V8_INLINE void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src,
212 : size_t chars);
213 : V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint8_t* src,
214 : size_t chars);
215 : V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src,
216 : size_t chars);
217 : #elif defined(V8_HOST_ARCH_MIPS)
218 : V8_INLINE void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src,
219 : size_t chars);
220 : V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src,
221 : size_t chars);
222 : #elif defined(V8_HOST_ARCH_PPC) || defined(V8_HOST_ARCH_S390)
223 : V8_INLINE void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src,
224 : size_t chars);
225 : V8_INLINE void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src,
226 : size_t chars);
227 : #endif
228 :
229 : // Copy from 8bit/16bit chars to 8bit/16bit chars.
230 : template <typename sourcechar, typename sinkchar>
231 : V8_INLINE void CopyChars(sinkchar* dest, const sourcechar* src, size_t chars);
232 :
233 : template <typename sourcechar, typename sinkchar>
234 : void CopyChars(sinkchar* dest, const sourcechar* src, size_t chars) {
235 : DCHECK_LE(sizeof(sourcechar), 2);
236 : DCHECK_LE(sizeof(sinkchar), 2);
237 : if (sizeof(sinkchar) == 1) {
238 : if (sizeof(sourcechar) == 1) {
239 : CopyCharsUnsigned(reinterpret_cast<uint8_t*>(dest),
240 : reinterpret_cast<const uint8_t*>(src), chars);
241 : } else {
242 : CopyCharsUnsigned(reinterpret_cast<uint8_t*>(dest),
243 : reinterpret_cast<const uint16_t*>(src), chars);
244 : }
245 : } else {
246 : if (sizeof(sourcechar) == 1) {
247 : CopyCharsUnsigned(reinterpret_cast<uint16_t*>(dest),
248 : reinterpret_cast<const uint8_t*>(src), chars);
249 : } else {
250 : CopyCharsUnsigned(reinterpret_cast<uint16_t*>(dest),
251 : reinterpret_cast<const uint16_t*>(src), chars);
252 : }
253 : }
254 : }
255 :
256 : template <typename sourcechar, typename sinkchar>
257 : void CopyCharsUnsigned(sinkchar* dest, const sourcechar* src, size_t chars) {
258 248458740 : sinkchar* limit = dest + chars;
259 221308071 : if ((sizeof(*dest) == sizeof(*src)) &&
260 : (chars >= kMinComplexMemCopy / sizeof(*dest))) {
261 : MemCopy(dest, src, chars * sizeof(*dest));
262 : } else {
263 5562021704 : while (dest < limit) *dest++ = static_cast<sinkchar>(*src++);
264 : }
265 : }
266 :
267 : #if defined(V8_HOST_ARCH_ARM)
268 : void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src, size_t chars) {
269 : switch (static_cast<unsigned>(chars)) {
270 : case 0:
271 : break;
272 : case 1:
273 : *dest = *src;
274 : break;
275 : case 2:
276 : memcpy(dest, src, 2);
277 : break;
278 : case 3:
279 : memcpy(dest, src, 3);
280 : break;
281 : case 4:
282 : memcpy(dest, src, 4);
283 : break;
284 : case 5:
285 : memcpy(dest, src, 5);
286 : break;
287 : case 6:
288 : memcpy(dest, src, 6);
289 : break;
290 : case 7:
291 : memcpy(dest, src, 7);
292 : break;
293 : case 8:
294 : memcpy(dest, src, 8);
295 : break;
296 : case 9:
297 : memcpy(dest, src, 9);
298 : break;
299 : case 10:
300 : memcpy(dest, src, 10);
301 : break;
302 : case 11:
303 : memcpy(dest, src, 11);
304 : break;
305 : case 12:
306 : memcpy(dest, src, 12);
307 : break;
308 : case 13:
309 : memcpy(dest, src, 13);
310 : break;
311 : case 14:
312 : memcpy(dest, src, 14);
313 : break;
314 : case 15:
315 : memcpy(dest, src, 15);
316 : break;
317 : default:
318 : MemCopy(dest, src, chars);
319 : break;
320 : }
321 : }
322 :
323 : void CopyCharsUnsigned(uint16_t* dest, const uint8_t* src, size_t chars) {
324 : if (chars >= static_cast<size_t>(kMinComplexConvertMemCopy)) {
325 : MemCopyUint16Uint8(dest, src, chars);
326 : } else {
327 : MemCopyUint16Uint8Wrapper(dest, src, chars);
328 : }
329 : }
330 :
331 : void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src, size_t chars) {
332 : switch (static_cast<unsigned>(chars)) {
333 : case 0:
334 : break;
335 : case 1:
336 : *dest = *src;
337 : break;
338 : case 2:
339 : memcpy(dest, src, 4);
340 : break;
341 : case 3:
342 : memcpy(dest, src, 6);
343 : break;
344 : case 4:
345 : memcpy(dest, src, 8);
346 : break;
347 : case 5:
348 : memcpy(dest, src, 10);
349 : break;
350 : case 6:
351 : memcpy(dest, src, 12);
352 : break;
353 : case 7:
354 : memcpy(dest, src, 14);
355 : break;
356 : default:
357 : MemCopy(dest, src, chars * sizeof(*dest));
358 : break;
359 : }
360 : }
361 :
362 : #elif defined(V8_HOST_ARCH_MIPS)
363 : void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src, size_t chars) {
364 : if (chars < kMinComplexMemCopy) {
365 : memcpy(dest, src, chars);
366 : } else {
367 : MemCopy(dest, src, chars);
368 : }
369 : }
370 :
371 : void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src, size_t chars) {
372 : if (chars < kMinComplexMemCopy) {
373 : memcpy(dest, src, chars * sizeof(*dest));
374 : } else {
375 : MemCopy(dest, src, chars * sizeof(*dest));
376 : }
377 : }
378 : #elif defined(V8_HOST_ARCH_PPC) || defined(V8_HOST_ARCH_S390)
379 : #define CASE(n) \
380 : case n: \
381 : memcpy(dest, src, n); \
382 : break
383 : void CopyCharsUnsigned(uint8_t* dest, const uint8_t* src, size_t chars) {
384 : switch (static_cast<unsigned>(chars)) {
385 : case 0:
386 : break;
387 : case 1:
388 : *dest = *src;
389 : break;
390 : CASE(2);
391 : CASE(3);
392 : CASE(4);
393 : CASE(5);
394 : CASE(6);
395 : CASE(7);
396 : CASE(8);
397 : CASE(9);
398 : CASE(10);
399 : CASE(11);
400 : CASE(12);
401 : CASE(13);
402 : CASE(14);
403 : CASE(15);
404 : CASE(16);
405 : CASE(17);
406 : CASE(18);
407 : CASE(19);
408 : CASE(20);
409 : CASE(21);
410 : CASE(22);
411 : CASE(23);
412 : CASE(24);
413 : CASE(25);
414 : CASE(26);
415 : CASE(27);
416 : CASE(28);
417 : CASE(29);
418 : CASE(30);
419 : CASE(31);
420 : CASE(32);
421 : CASE(33);
422 : CASE(34);
423 : CASE(35);
424 : CASE(36);
425 : CASE(37);
426 : CASE(38);
427 : CASE(39);
428 : CASE(40);
429 : CASE(41);
430 : CASE(42);
431 : CASE(43);
432 : CASE(44);
433 : CASE(45);
434 : CASE(46);
435 : CASE(47);
436 : CASE(48);
437 : CASE(49);
438 : CASE(50);
439 : CASE(51);
440 : CASE(52);
441 : CASE(53);
442 : CASE(54);
443 : CASE(55);
444 : CASE(56);
445 : CASE(57);
446 : CASE(58);
447 : CASE(59);
448 : CASE(60);
449 : CASE(61);
450 : CASE(62);
451 : CASE(63);
452 : CASE(64);
453 : default:
454 : memcpy(dest, src, chars);
455 : break;
456 : }
457 : }
458 : #undef CASE
459 :
460 : #define CASE(n) \
461 : case n: \
462 : memcpy(dest, src, n * 2); \
463 : break
464 : void CopyCharsUnsigned(uint16_t* dest, const uint16_t* src, size_t chars) {
465 : switch (static_cast<unsigned>(chars)) {
466 : case 0:
467 : break;
468 : case 1:
469 : *dest = *src;
470 : break;
471 : CASE(2);
472 : CASE(3);
473 : CASE(4);
474 : CASE(5);
475 : CASE(6);
476 : CASE(7);
477 : CASE(8);
478 : CASE(9);
479 : CASE(10);
480 : CASE(11);
481 : CASE(12);
482 : CASE(13);
483 : CASE(14);
484 : CASE(15);
485 : CASE(16);
486 : CASE(17);
487 : CASE(18);
488 : CASE(19);
489 : CASE(20);
490 : CASE(21);
491 : CASE(22);
492 : CASE(23);
493 : CASE(24);
494 : CASE(25);
495 : CASE(26);
496 : CASE(27);
497 : CASE(28);
498 : CASE(29);
499 : CASE(30);
500 : CASE(31);
501 : CASE(32);
502 : default:
503 : memcpy(dest, src, chars * 2);
504 : break;
505 : }
506 : }
507 : #undef CASE
508 : #endif
509 :
510 : } // namespace internal
511 : } // namespace v8
512 :
513 : #endif // V8_MEMCOPY_H_
|
