Coverage Report

Created: 2025-07-12 07:07

/src/irssi/subprojects/glib-2.74.3/glib/valgrind.h
Line
Count
Source (jump to first uncovered line)
1
/* -*- c -*-
2
   ----------------------------------------------------------------
3
4
   Notice that the following BSD-style license applies to this one
5
   file (valgrind.h) only.  The rest of Valgrind is licensed under the
6
   terms of the GNU General Public License, version 2, unless
7
   otherwise indicated.  See the COPYING file in the source
8
   distribution for details.
9
10
   ----------------------------------------------------------------
11
12
   This file is part of Valgrind, a dynamic binary instrumentation
13
   framework.
14
15
   Copyright (C) 2000-2017 Julian Seward.  All rights reserved.
16
17
   Redistribution and use in source and binary forms, with or without
18
   modification, are permitted provided that the following conditions
19
   are met:
20
21
   1. Redistributions of source code must retain the above copyright
22
      notice, this list of conditions and the following disclaimer.
23
24
   2. The origin of this software must not be misrepresented; you must 
25
      not claim that you wrote the original software.  If you use this 
26
      software in a product, an acknowledgment in the product 
27
      documentation would be appreciated but is not required.
28
29
   3. Altered source versions must be plainly marked as such, and must
30
      not be misrepresented as being the original software.
31
32
   4. The name of the author may not be used to endorse or promote 
33
      products derived from this software without specific prior written 
34
      permission.
35
36
   THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
37
   OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
38
   WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
39
   ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
40
   DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
41
   DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
42
   GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
43
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
44
   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
45
   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
46
   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
47
48
   ----------------------------------------------------------------
49
50
   Notice that the above BSD-style license applies to this one file
51
   (valgrind.h) only.  The entire rest of Valgrind is licensed under
52
   the terms of the GNU General Public License, version 2.  See the
53
   COPYING file in the source distribution for details.
54
55
   ---------------------------------------------------------------- 
56
*/
57
58
59
/* This file is for inclusion into client (your!) code.
60
61
   You can use these macros to manipulate and query Valgrind's 
62
   execution inside your own programs.
63
64
   The resulting executables will still run without Valgrind, just a
65
   little bit more slowly than they otherwise would, but otherwise
66
   unchanged.  When not running on valgrind, each client request
67
   consumes very few (eg. 7) instructions, so the resulting performance
68
   loss is negligible unless you plan to execute client requests
69
   millions of times per second.  Nevertheless, if that is still a
70
   problem, you can compile with the NVALGRIND symbol defined (gcc
71
   -DNVALGRIND) so that client requests are not even compiled in.  */
72
73
#ifndef __VALGRIND_H
74
#define __VALGRIND_H
75
76
77
/* ------------------------------------------------------------------ */
78
/* VERSION NUMBER OF VALGRIND                                         */
79
/* ------------------------------------------------------------------ */
80
81
/* Specify Valgrind's version number, so that user code can
82
   conditionally compile based on our version number.  Note that these
83
   were introduced at version 3.6 and so do not exist in version 3.5
84
   or earlier.  The recommended way to use them to check for "version
85
   X.Y or later" is (eg)
86
87
#if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__)   \
88
    && (__VALGRIND_MAJOR__ > 3                                   \
89
        || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6))
90
*/
91
#define __VALGRIND_MAJOR__    3
92
#define __VALGRIND_MINOR__    15
93
94
95
#include <stdarg.h>
96
97
/* Nb: this file might be included in a file compiled with -ansi.  So
98
   we can't use C++ style "//" comments nor the "asm" keyword (instead
99
   use "__asm__"). */
100
101
/* Derive some tags indicating what the target platform is.  Note
102
   that in this file we're using the compiler's CPP symbols for
103
   identifying architectures, which are different to the ones we use
104
   within the rest of Valgrind.  Note, __powerpc__ is active for both
105
   32 and 64-bit PPC, whereas __powerpc64__ is only active for the
106
   latter (on Linux, that is).
107
108
   Misc note: how to find out what's predefined in gcc by default:
109
   gcc -Wp,-dM somefile.c
110
*/
111
#undef PLAT_x86_darwin
112
#undef PLAT_amd64_darwin
113
#undef PLAT_x86_win32
114
#undef PLAT_amd64_win64
115
#undef PLAT_x86_linux
116
#undef PLAT_amd64_linux
117
#undef PLAT_ppc32_linux
118
#undef PLAT_ppc64be_linux
119
#undef PLAT_ppc64le_linux
120
#undef PLAT_arm_linux
121
#undef PLAT_arm64_linux
122
#undef PLAT_s390x_linux
123
#undef PLAT_mips32_linux
124
#undef PLAT_mips64_linux
125
#undef PLAT_x86_solaris
126
#undef PLAT_amd64_solaris
127
128
129
#if defined(__APPLE__) && defined(__i386__)
130
#  define PLAT_x86_darwin 1
131
#elif defined(__APPLE__) && defined(__x86_64__)
132
#  define PLAT_amd64_darwin 1
133
#elif (defined(__MINGW32__) && defined(__i386__)) \
134
      || defined(__CYGWIN32__) \
135
      || (defined(_WIN32) && defined(_M_IX86))
136
#  define PLAT_x86_win32 1
137
#elif (defined(__MINGW32__) && defined(__x86_64__)) \
138
      || (defined(_WIN32) && defined(_M_X64))
139
/* __MINGW32__ and _WIN32 are defined in 64 bit mode as well. */
140
#  define PLAT_amd64_win64 1
141
#elif defined(__linux__) && defined(__i386__)
142
#  define PLAT_x86_linux 1
143
#elif defined(__linux__) && defined(__x86_64__) && !defined(__ILP32__)
144
#  define PLAT_amd64_linux 1
145
#elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__)
146
#  define PLAT_ppc32_linux 1
147
#elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF != 2
148
/* Big Endian uses ELF version 1 */
149
#  define PLAT_ppc64be_linux 1
150
#elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF == 2
151
/* Little Endian uses ELF version 2 */
152
#  define PLAT_ppc64le_linux 1
153
#elif defined(__linux__) && defined(__arm__) && !defined(__aarch64__)
154
#  define PLAT_arm_linux 1
155
#elif defined(__linux__) && defined(__aarch64__) && !defined(__arm__)
156
#  define PLAT_arm64_linux 1
157
#elif defined(__linux__) && defined(__s390__) && defined(__s390x__)
158
#  define PLAT_s390x_linux 1
159
#elif defined(__linux__) && defined(__mips__) && (__mips==64)
160
#  define PLAT_mips64_linux 1
161
#elif defined(__linux__) && defined(__mips__) && (__mips!=64)
162
#  define PLAT_mips32_linux 1
163
#elif defined(__sun) && defined(__i386__)
164
#  define PLAT_x86_solaris 1
165
#elif defined(__sun) && defined(__x86_64__)
166
#  define PLAT_amd64_solaris 1
167
#else
168
/* If we're not compiling for our target platform, don't generate
169
   any inline asms.  */
170
#  if !defined(NVALGRIND)
171
#    define NVALGRIND 1
172
#  endif
173
#endif
174
175
176
/* ------------------------------------------------------------------ */
177
/* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS.  There is nothing */
178
/* in here of use to end-users -- skip to the next section.           */
179
/* ------------------------------------------------------------------ */
180
181
/*
182
 * VALGRIND_DO_CLIENT_REQUEST(): a statement that invokes a Valgrind client
183
 * request. Accepts both pointers and integers as arguments.
184
 *
185
 * VALGRIND_DO_CLIENT_REQUEST_STMT(): a statement that invokes a Valgrind
186
 * client request that does not return a value.
187
188
 * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind
189
 * client request and whose value equals the client request result.  Accepts
190
 * both pointers and integers as arguments.  Note that such calls are not
191
 * necessarily pure functions -- they may have side effects.
192
 */
193
194
#define VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default,            \
195
                                   _zzq_request, _zzq_arg1, _zzq_arg2,  \
196
                                   _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
197
  do { (_zzq_rlval) = VALGRIND_DO_CLIENT_REQUEST_EXPR((_zzq_default),   \
198
                        (_zzq_request), (_zzq_arg1), (_zzq_arg2),       \
199
                        (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
200
201
#define VALGRIND_DO_CLIENT_REQUEST_STMT(_zzq_request, _zzq_arg1,        \
202
                           _zzq_arg2,  _zzq_arg3, _zzq_arg4, _zzq_arg5) \
203
0
  do { (void) VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                        \
204
0
                    (_zzq_request), (_zzq_arg1), (_zzq_arg2),           \
205
0
                    (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0)
206
207
#if defined(NVALGRIND)
208
209
/* Define NVALGRIND to completely remove the Valgrind magic sequence
210
   from the compiled code (analogous to NDEBUG's effects on
211
   assert()) */
212
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
213
        _zzq_default, _zzq_request,                               \
214
        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
215
      (_zzq_default)
216
217
#else  /* ! NVALGRIND */
218
219
/* The following defines the magic code sequences which the JITter
220
   spots and handles magically.  Don't look too closely at them as
221
   they will rot your brain.
222
223
   The assembly code sequences for all architectures is in this one
224
   file.  This is because this file must be stand-alone, and we don't
225
   want to have multiple files.
226
227
   For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default
228
   value gets put in the return slot, so that everything works when
229
   this is executed not under Valgrind.  Args are passed in a memory
230
   block, and so there's no intrinsic limit to the number that could
231
   be passed, but it's currently five.
232
   
233
   The macro args are: 
234
      _zzq_rlval    result lvalue
235
      _zzq_default  default value (result returned when running on real CPU)
236
      _zzq_request  request code
237
      _zzq_arg1..5  request params
238
239
   The other two macros are used to support function wrapping, and are
240
   a lot simpler.  VALGRIND_GET_NR_CONTEXT returns the value of the
241
   guest's NRADDR pseudo-register and whatever other information is
242
   needed to safely run the call original from the wrapper: on
243
   ppc64-linux, the R2 value at the divert point is also needed.  This
244
   information is abstracted into a user-visible type, OrigFn.
245
246
   VALGRIND_CALL_NOREDIR_* behaves the same as the following on the
247
   guest, but guarantees that the branch instruction will not be
248
   redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64:
249
   branch-and-link-to-r11.  VALGRIND_CALL_NOREDIR is just text, not a
250
   complete inline asm, since it needs to be combined with more magic
251
   inline asm stuff to be useful.
252
*/
253
254
/* ----------------- x86-{linux,darwin,solaris} ---------------- */
255
256
#if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin)  \
257
    ||  (defined(PLAT_x86_win32) && defined(__GNUC__)) \
258
    ||  defined(PLAT_x86_solaris)
259
260
typedef
261
   struct { 
262
      unsigned int nraddr; /* where's the code? */
263
   }
264
   OrigFn;
265
266
#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
267
                     "roll $3,  %%edi ; roll $13, %%edi\n\t"      \
268
                     "roll $29, %%edi ; roll $19, %%edi\n\t"
269
270
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
271
        _zzq_default, _zzq_request,                               \
272
        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
273
  __extension__                                                   \
274
  ({volatile unsigned int _zzq_args[6];                           \
275
    volatile unsigned int _zzq_result;                            \
276
    _zzq_args[0] = (unsigned int)(_zzq_request);                  \
277
    _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
278
    _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
279
    _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
280
    _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
281
    _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
282
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
283
                     /* %EDX = client_request ( %EAX ) */         \
284
                     "xchgl %%ebx,%%ebx"                          \
285
                     : "=d" (_zzq_result)                         \
286
                     : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
287
                     : "cc", "memory"                             \
288
                    );                                            \
289
    _zzq_result;                                                  \
290
  })
291
292
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
293
  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
294
    volatile unsigned int __addr;                                 \
295
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
296
                     /* %EAX = guest_NRADDR */                    \
297
                     "xchgl %%ecx,%%ecx"                          \
298
                     : "=a" (__addr)                              \
299
                     :                                            \
300
                     : "cc", "memory"                             \
301
                    );                                            \
302
    _zzq_orig->nraddr = __addr;                                   \
303
  }
304
305
#define VALGRIND_CALL_NOREDIR_EAX                                 \
306
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
307
                     /* call-noredir *%EAX */                     \
308
                     "xchgl %%edx,%%edx\n\t"
309
310
#define VALGRIND_VEX_INJECT_IR()                                 \
311
 do {                                                            \
312
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
313
                     "xchgl %%edi,%%edi\n\t"                     \
314
                     : : : "cc", "memory"                        \
315
                    );                                           \
316
 } while (0)
317
318
#endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__)
319
          || PLAT_x86_solaris */
320
321
/* ------------------------- x86-Win32 ------------------------- */
322
323
#if defined(PLAT_x86_win32) && !defined(__GNUC__)
324
325
typedef
326
   struct { 
327
      unsigned int nraddr; /* where's the code? */
328
   }
329
   OrigFn;
330
331
#if defined(_MSC_VER)
332
333
#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
334
                     __asm rol edi, 3  __asm rol edi, 13          \
335
                     __asm rol edi, 29 __asm rol edi, 19
336
337
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
338
        _zzq_default, _zzq_request,                               \
339
        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
340
    valgrind_do_client_request_expr((uintptr_t)(_zzq_default),    \
341
        (uintptr_t)(_zzq_request), (uintptr_t)(_zzq_arg1),        \
342
        (uintptr_t)(_zzq_arg2), (uintptr_t)(_zzq_arg3),           \
343
        (uintptr_t)(_zzq_arg4), (uintptr_t)(_zzq_arg5))
344
345
static __inline uintptr_t
346
valgrind_do_client_request_expr(uintptr_t _zzq_default, uintptr_t _zzq_request,
347
                                uintptr_t _zzq_arg1, uintptr_t _zzq_arg2,
348
                                uintptr_t _zzq_arg3, uintptr_t _zzq_arg4,
349
                                uintptr_t _zzq_arg5)
350
{
351
    volatile uintptr_t _zzq_args[6];
352
    volatile unsigned int _zzq_result;
353
    _zzq_args[0] = (uintptr_t)(_zzq_request);
354
    _zzq_args[1] = (uintptr_t)(_zzq_arg1);
355
    _zzq_args[2] = (uintptr_t)(_zzq_arg2);
356
    _zzq_args[3] = (uintptr_t)(_zzq_arg3);
357
    _zzq_args[4] = (uintptr_t)(_zzq_arg4);
358
    _zzq_args[5] = (uintptr_t)(_zzq_arg5);
359
    __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default
360
            __SPECIAL_INSTRUCTION_PREAMBLE
361
            /* %EDX = client_request ( %EAX ) */
362
            __asm xchg ebx,ebx
363
            __asm mov _zzq_result, edx
364
    }
365
    return _zzq_result;
366
}
367
368
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
369
  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
370
    volatile unsigned int __addr;                                 \
371
    __asm { __SPECIAL_INSTRUCTION_PREAMBLE                        \
372
            /* %EAX = guest_NRADDR */                             \
373
            __asm xchg ecx,ecx                                    \
374
            __asm mov __addr, eax                                 \
375
    }                                                             \
376
    _zzq_orig->nraddr = __addr;                                   \
377
  }
378
379
#define VALGRIND_CALL_NOREDIR_EAX ERROR
380
381
#define VALGRIND_VEX_INJECT_IR()                                 \
382
 do {                                                            \
383
    __asm { __SPECIAL_INSTRUCTION_PREAMBLE                       \
384
            __asm xchg edi,edi                                   \
385
    }                                                            \
386
 } while (0)
387
388
#else
389
#error Unsupported compiler.
390
#endif
391
392
#endif /* PLAT_x86_win32 */
393
394
/* ----------------- amd64-{linux,darwin,solaris} --------------- */
395
396
#if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin) \
397
    ||  defined(PLAT_amd64_solaris) \
398
    ||  (defined(PLAT_amd64_win64) && defined(__GNUC__))
399
400
typedef
401
   struct { 
402
      unsigned long int nraddr; /* where's the code? */
403
   }
404
   OrigFn;
405
406
#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
407
                     "rolq $3,  %%rdi ; rolq $13, %%rdi\n\t"      \
408
                     "rolq $61, %%rdi ; rolq $51, %%rdi\n\t"
409
410
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
411
        _zzq_default, _zzq_request,                               \
412
        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
413
928k
    __extension__                                                 \
414
928k
    ({ volatile unsigned long int _zzq_args[6];                   \
415
928k
    volatile unsigned long int _zzq_result;                       \
416
928k
    _zzq_args[0] = (unsigned long int)(_zzq_request);             \
417
928k
    _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
418
928k
    _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
419
928k
    _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
420
928k
    _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
421
928k
    _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
422
928k
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
423
928k
                     /* %RDX = client_request ( %RAX ) */         \
424
928k
                     "xchgq %%rbx,%%rbx"                          \
425
928k
                     : "=d" (_zzq_result)                         \
426
928k
                     : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
427
928k
                     : "cc", "memory"                             \
428
928k
                    );                                            \
429
928k
    _zzq_result;                                                  \
430
928k
    })
431
432
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
433
  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
434
    volatile unsigned long int __addr;                            \
435
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
436
                     /* %RAX = guest_NRADDR */                    \
437
                     "xchgq %%rcx,%%rcx"                          \
438
                     : "=a" (__addr)                              \
439
                     :                                            \
440
                     : "cc", "memory"                             \
441
                    );                                            \
442
    _zzq_orig->nraddr = __addr;                                   \
443
  }
444
445
#define VALGRIND_CALL_NOREDIR_RAX                                 \
446
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
447
                     /* call-noredir *%RAX */                     \
448
                     "xchgq %%rdx,%%rdx\n\t"
449
450
#define VALGRIND_VEX_INJECT_IR()                                 \
451
 do {                                                            \
452
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
453
                     "xchgq %%rdi,%%rdi\n\t"                     \
454
                     : : : "cc", "memory"                        \
455
                    );                                           \
456
 } while (0)
457
458
#endif /* PLAT_amd64_linux || PLAT_amd64_darwin || PLAT_amd64_solaris */
459
460
/* ------------------------- amd64-Win64 ------------------------- */
461
462
#if defined(PLAT_amd64_win64) && !defined(__GNUC__)
463
464
#error Unsupported compiler.
465
466
#endif /* PLAT_amd64_win64 */
467
468
/* ------------------------ ppc32-linux ------------------------ */
469
470
#if defined(PLAT_ppc32_linux)
471
472
typedef
473
   struct { 
474
      unsigned int nraddr; /* where's the code? */
475
   }
476
   OrigFn;
477
478
#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
479
                    "rlwinm 0,0,3,0,31  ; rlwinm 0,0,13,0,31\n\t" \
480
                    "rlwinm 0,0,29,0,31 ; rlwinm 0,0,19,0,31\n\t"
481
482
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
483
        _zzq_default, _zzq_request,                               \
484
        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
485
                                                                  \
486
    __extension__                                                 \
487
  ({         unsigned int  _zzq_args[6];                          \
488
             unsigned int  _zzq_result;                           \
489
             unsigned int* _zzq_ptr;                              \
490
    _zzq_args[0] = (unsigned int)(_zzq_request);                  \
491
    _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
492
    _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
493
    _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
494
    _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
495
    _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
496
    _zzq_ptr = _zzq_args;                                         \
497
    __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
498
                     "mr 4,%2\n\t" /*ptr*/                        \
499
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
500
                     /* %R3 = client_request ( %R4 ) */           \
501
                     "or 1,1,1\n\t"                               \
502
                     "mr %0,3"     /*result*/                     \
503
                     : "=b" (_zzq_result)                         \
504
                     : "b" (_zzq_default), "b" (_zzq_ptr)         \
505
                     : "cc", "memory", "r3", "r4");               \
506
    _zzq_result;                                                  \
507
    })
508
509
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
510
  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
511
    unsigned int __addr;                                          \
512
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
513
                     /* %R3 = guest_NRADDR */                     \
514
                     "or 2,2,2\n\t"                               \
515
                     "mr %0,3"                                    \
516
                     : "=b" (__addr)                              \
517
                     :                                            \
518
                     : "cc", "memory", "r3"                       \
519
                    );                                            \
520
    _zzq_orig->nraddr = __addr;                                   \
521
  }
522
523
#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
524
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
525
                     /* branch-and-link-to-noredir *%R11 */       \
526
                     "or 3,3,3\n\t"
527
528
#define VALGRIND_VEX_INJECT_IR()                                 \
529
 do {                                                            \
530
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
531
                     "or 5,5,5\n\t"                              \
532
                    );                                           \
533
 } while (0)
534
535
#endif /* PLAT_ppc32_linux */
536
537
/* ------------------------ ppc64-linux ------------------------ */
538
539
#if defined(PLAT_ppc64be_linux)
540
541
typedef
542
   struct { 
543
      unsigned long int nraddr; /* where's the code? */
544
      unsigned long int r2;  /* what tocptr do we need? */
545
   }
546
   OrigFn;
547
548
#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
549
                     "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
550
                     "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
551
552
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
553
        _zzq_default, _zzq_request,                               \
554
        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
555
                                                                  \
556
  __extension__                                                   \
557
  ({         unsigned long int  _zzq_args[6];                     \
558
             unsigned long int  _zzq_result;                      \
559
             unsigned long int* _zzq_ptr;                         \
560
    _zzq_args[0] = (unsigned long int)(_zzq_request);             \
561
    _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
562
    _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
563
    _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
564
    _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
565
    _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
566
    _zzq_ptr = _zzq_args;                                         \
567
    __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
568
                     "mr 4,%2\n\t" /*ptr*/                        \
569
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
570
                     /* %R3 = client_request ( %R4 ) */           \
571
                     "or 1,1,1\n\t"                               \
572
                     "mr %0,3"     /*result*/                     \
573
                     : "=b" (_zzq_result)                         \
574
                     : "b" (_zzq_default), "b" (_zzq_ptr)         \
575
                     : "cc", "memory", "r3", "r4");               \
576
    _zzq_result;                                                  \
577
  })
578
579
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
580
  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
581
    unsigned long int __addr;                                     \
582
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
583
                     /* %R3 = guest_NRADDR */                     \
584
                     "or 2,2,2\n\t"                               \
585
                     "mr %0,3"                                    \
586
                     : "=b" (__addr)                              \
587
                     :                                            \
588
                     : "cc", "memory", "r3"                       \
589
                    );                                            \
590
    _zzq_orig->nraddr = __addr;                                   \
591
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
592
                     /* %R3 = guest_NRADDR_GPR2 */                \
593
                     "or 4,4,4\n\t"                               \
594
                     "mr %0,3"                                    \
595
                     : "=b" (__addr)                              \
596
                     :                                            \
597
                     : "cc", "memory", "r3"                       \
598
                    );                                            \
599
    _zzq_orig->r2 = __addr;                                       \
600
  }
601
602
#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                   \
603
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
604
                     /* branch-and-link-to-noredir *%R11 */       \
605
                     "or 3,3,3\n\t"
606
607
#define VALGRIND_VEX_INJECT_IR()                                 \
608
 do {                                                            \
609
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
610
                     "or 5,5,5\n\t"                              \
611
                    );                                           \
612
 } while (0)
613
614
#endif /* PLAT_ppc64be_linux */
615
616
#if defined(PLAT_ppc64le_linux)
617
618
typedef
619
   struct {
620
      unsigned long int nraddr; /* where's the code? */
621
      unsigned long int r2;     /* what tocptr do we need? */
622
   }
623
   OrigFn;
624
625
#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
626
                     "rotldi 0,0,3  ; rotldi 0,0,13\n\t"          \
627
                     "rotldi 0,0,61 ; rotldi 0,0,51\n\t"
628
629
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
630
        _zzq_default, _zzq_request,                               \
631
        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
632
                                                                  \
633
  __extension__                                                   \
634
  ({         unsigned long int  _zzq_args[6];                     \
635
             unsigned long int  _zzq_result;                      \
636
             unsigned long int* _zzq_ptr;                         \
637
    _zzq_args[0] = (unsigned long int)(_zzq_request);             \
638
    _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
639
    _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
640
    _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
641
    _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
642
    _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
643
    _zzq_ptr = _zzq_args;                                         \
644
    __asm__ volatile("mr 3,%1\n\t" /*default*/                    \
645
                     "mr 4,%2\n\t" /*ptr*/                        \
646
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
647
                     /* %R3 = client_request ( %R4 ) */           \
648
                     "or 1,1,1\n\t"                               \
649
                     "mr %0,3"     /*result*/                     \
650
                     : "=b" (_zzq_result)                         \
651
                     : "b" (_zzq_default), "b" (_zzq_ptr)         \
652
                     : "cc", "memory", "r3", "r4");               \
653
    _zzq_result;                                                  \
654
  })
655
656
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
657
  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
658
    unsigned long int __addr;                                     \
659
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
660
                     /* %R3 = guest_NRADDR */                     \
661
                     "or 2,2,2\n\t"                               \
662
                     "mr %0,3"                                    \
663
                     : "=b" (__addr)                              \
664
                     :                                            \
665
                     : "cc", "memory", "r3"                       \
666
                    );                                            \
667
    _zzq_orig->nraddr = __addr;                                   \
668
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
669
                     /* %R3 = guest_NRADDR_GPR2 */                \
670
                     "or 4,4,4\n\t"                               \
671
                     "mr %0,3"                                    \
672
                     : "=b" (__addr)                              \
673
                     :                                            \
674
                     : "cc", "memory", "r3"                       \
675
                    );                                            \
676
    _zzq_orig->r2 = __addr;                                       \
677
  }
678
679
#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                   \
680
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
681
                     /* branch-and-link-to-noredir *%R12 */       \
682
                     "or 3,3,3\n\t"
683
684
#define VALGRIND_VEX_INJECT_IR()                                 \
685
 do {                                                            \
686
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
687
                     "or 5,5,5\n\t"                              \
688
                    );                                           \
689
 } while (0)
690
691
#endif /* PLAT_ppc64le_linux */
692
693
/* ------------------------- arm-linux ------------------------- */
694
695
#if defined(PLAT_arm_linux)
696
697
typedef
698
   struct { 
699
      unsigned int nraddr; /* where's the code? */
700
   }
701
   OrigFn;
702
703
#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
704
            "mov r12, r12, ror #3  ; mov r12, r12, ror #13 \n\t"  \
705
            "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t"
706
707
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
708
        _zzq_default, _zzq_request,                               \
709
        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
710
                                                                  \
711
  __extension__                                                   \
712
  ({volatile unsigned int  _zzq_args[6];                          \
713
    volatile unsigned int  _zzq_result;                           \
714
    _zzq_args[0] = (unsigned int)(_zzq_request);                  \
715
    _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
716
    _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
717
    _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
718
    _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
719
    _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
720
    __asm__ volatile("mov r3, %1\n\t" /*default*/                 \
721
                     "mov r4, %2\n\t" /*ptr*/                     \
722
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
723
                     /* R3 = client_request ( R4 ) */             \
724
                     "orr r10, r10, r10\n\t"                      \
725
                     "mov %0, r3"     /*result*/                  \
726
                     : "=r" (_zzq_result)                         \
727
                     : "r" (_zzq_default), "r" (&_zzq_args[0])    \
728
                     : "cc","memory", "r3", "r4");                \
729
    _zzq_result;                                                  \
730
  })
731
732
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
733
  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
734
    unsigned int __addr;                                          \
735
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
736
                     /* R3 = guest_NRADDR */                      \
737
                     "orr r11, r11, r11\n\t"                      \
738
                     "mov %0, r3"                                 \
739
                     : "=r" (__addr)                              \
740
                     :                                            \
741
                     : "cc", "memory", "r3"                       \
742
                    );                                            \
743
    _zzq_orig->nraddr = __addr;                                   \
744
  }
745
746
#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                    \
747
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
748
                     /* branch-and-link-to-noredir *%R4 */        \
749
                     "orr r12, r12, r12\n\t"
750
751
#define VALGRIND_VEX_INJECT_IR()                                 \
752
 do {                                                            \
753
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
754
                     "orr r9, r9, r9\n\t"                        \
755
                     : : : "cc", "memory"                        \
756
                    );                                           \
757
 } while (0)
758
759
#endif /* PLAT_arm_linux */
760
761
/* ------------------------ arm64-linux ------------------------- */
762
763
#if defined(PLAT_arm64_linux)
764
765
typedef
766
   struct { 
767
      unsigned long int nraddr; /* where's the code? */
768
   }
769
   OrigFn;
770
771
#define __SPECIAL_INSTRUCTION_PREAMBLE                            \
772
            "ror x12, x12, #3  ;  ror x12, x12, #13 \n\t"         \
773
            "ror x12, x12, #51 ;  ror x12, x12, #61 \n\t"
774
775
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
776
        _zzq_default, _zzq_request,                               \
777
        _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
778
                                                                  \
779
  __extension__                                                   \
780
  ({volatile unsigned long int  _zzq_args[6];                     \
781
    volatile unsigned long int  _zzq_result;                      \
782
    _zzq_args[0] = (unsigned long int)(_zzq_request);             \
783
    _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
784
    _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
785
    _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
786
    _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
787
    _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
788
    __asm__ volatile("mov x3, %1\n\t" /*default*/                 \
789
                     "mov x4, %2\n\t" /*ptr*/                     \
790
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
791
                     /* X3 = client_request ( X4 ) */             \
792
                     "orr x10, x10, x10\n\t"                      \
793
                     "mov %0, x3"     /*result*/                  \
794
                     : "=r" (_zzq_result)                         \
795
                     : "r" ((unsigned long int)(_zzq_default)),   \
796
                       "r" (&_zzq_args[0])                        \
797
                     : "cc","memory", "x3", "x4");                \
798
    _zzq_result;                                                  \
799
  })
800
801
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
802
  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
803
    unsigned long int __addr;                                     \
804
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
805
                     /* X3 = guest_NRADDR */                      \
806
                     "orr x11, x11, x11\n\t"                      \
807
                     "mov %0, x3"                                 \
808
                     : "=r" (__addr)                              \
809
                     :                                            \
810
                     : "cc", "memory", "x3"                       \
811
                    );                                            \
812
    _zzq_orig->nraddr = __addr;                                   \
813
  }
814
815
#define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                    \
816
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
817
                     /* branch-and-link-to-noredir X8 */          \
818
                     "orr x12, x12, x12\n\t"
819
820
#define VALGRIND_VEX_INJECT_IR()                                 \
821
 do {                                                            \
822
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
823
                     "orr x9, x9, x9\n\t"                        \
824
                     : : : "cc", "memory"                        \
825
                    );                                           \
826
 } while (0)
827
828
#endif /* PLAT_arm64_linux */
829
830
/* ------------------------ s390x-linux ------------------------ */
831
832
#if defined(PLAT_s390x_linux)
833
834
typedef
835
  struct {
836
     unsigned long int nraddr; /* where's the code? */
837
  }
838
  OrigFn;
839
840
/* __SPECIAL_INSTRUCTION_PREAMBLE will be used to identify Valgrind specific
841
 * code. This detection is implemented in platform specific toIR.c
842
 * (e.g. VEX/priv/guest_s390_decoder.c).
843
 */
844
#define __SPECIAL_INSTRUCTION_PREAMBLE                           \
845
                     "lr 15,15\n\t"                              \
846
                     "lr 1,1\n\t"                                \
847
                     "lr 2,2\n\t"                                \
848
                     "lr 3,3\n\t"
849
850
#define __CLIENT_REQUEST_CODE "lr 2,2\n\t"
851
#define __GET_NR_CONTEXT_CODE "lr 3,3\n\t"
852
#define __CALL_NO_REDIR_CODE  "lr 4,4\n\t"
853
#define __VEX_INJECT_IR_CODE  "lr 5,5\n\t"
854
855
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                         \
856
       _zzq_default, _zzq_request,                               \
857
       _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)    \
858
  __extension__                                                  \
859
 ({volatile unsigned long int _zzq_args[6];                      \
860
   volatile unsigned long int _zzq_result;                       \
861
   _zzq_args[0] = (unsigned long int)(_zzq_request);             \
862
   _zzq_args[1] = (unsigned long int)(_zzq_arg1);                \
863
   _zzq_args[2] = (unsigned long int)(_zzq_arg2);                \
864
   _zzq_args[3] = (unsigned long int)(_zzq_arg3);                \
865
   _zzq_args[4] = (unsigned long int)(_zzq_arg4);                \
866
   _zzq_args[5] = (unsigned long int)(_zzq_arg5);                \
867
   __asm__ volatile(/* r2 = args */                              \
868
                    "lgr 2,%1\n\t"                               \
869
                    /* r3 = default */                           \
870
                    "lgr 3,%2\n\t"                               \
871
                    __SPECIAL_INSTRUCTION_PREAMBLE               \
872
                    __CLIENT_REQUEST_CODE                        \
873
                    /* results = r3 */                           \
874
                    "lgr %0, 3\n\t"                              \
875
                    : "=d" (_zzq_result)                         \
876
                    : "a" (&_zzq_args[0]), "0" (_zzq_default)    \
877
                    : "cc", "2", "3", "memory"                   \
878
                   );                                            \
879
   _zzq_result;                                                  \
880
 })
881
882
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                      \
883
 { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
884
   volatile unsigned long int __addr;                            \
885
   __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
886
                    __GET_NR_CONTEXT_CODE                        \
887
                    "lgr %0, 3\n\t"                              \
888
                    : "=a" (__addr)                              \
889
                    :                                            \
890
                    : "cc", "3", "memory"                        \
891
                   );                                            \
892
   _zzq_orig->nraddr = __addr;                                   \
893
 }
894
895
#define VALGRIND_CALL_NOREDIR_R1                                 \
896
                    __SPECIAL_INSTRUCTION_PREAMBLE               \
897
                    __CALL_NO_REDIR_CODE
898
899
#define VALGRIND_VEX_INJECT_IR()                                 \
900
 do {                                                            \
901
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
902
                     __VEX_INJECT_IR_CODE);                      \
903
 } while (0)
904
905
#endif /* PLAT_s390x_linux */
906
907
/* ------------------------- mips32-linux ---------------- */
908
909
#if defined(PLAT_mips32_linux)
910
911
typedef
912
   struct { 
913
      unsigned int nraddr; /* where's the code? */
914
   }
915
   OrigFn;
916
917
/* .word  0x342
918
 * .word  0x742
919
 * .word  0xC2
920
 * .word  0x4C2*/
921
#define __SPECIAL_INSTRUCTION_PREAMBLE          \
922
                     "srl $0, $0, 13\n\t"       \
923
                     "srl $0, $0, 29\n\t"       \
924
                     "srl $0, $0, 3\n\t"        \
925
                     "srl $0, $0, 19\n\t"
926
                    
927
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                          \
928
       _zzq_default, _zzq_request,                                \
929
       _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)     \
930
  __extension__                                                   \
931
  ({ volatile unsigned int _zzq_args[6];                          \
932
    volatile unsigned int _zzq_result;                            \
933
    _zzq_args[0] = (unsigned int)(_zzq_request);                  \
934
    _zzq_args[1] = (unsigned int)(_zzq_arg1);                     \
935
    _zzq_args[2] = (unsigned int)(_zzq_arg2);                     \
936
    _zzq_args[3] = (unsigned int)(_zzq_arg3);                     \
937
    _zzq_args[4] = (unsigned int)(_zzq_arg4);                     \
938
    _zzq_args[5] = (unsigned int)(_zzq_arg5);                     \
939
        __asm__ volatile("move $11, %1\n\t" /*default*/           \
940
                     "move $12, %2\n\t" /*ptr*/                   \
941
                     __SPECIAL_INSTRUCTION_PREAMBLE               \
942
                     /* T3 = client_request ( T4 ) */             \
943
                     "or $13, $13, $13\n\t"                       \
944
                     "move %0, $11\n\t"     /*result*/            \
945
                     : "=r" (_zzq_result)                         \
946
                     : "r" (_zzq_default), "r" (&_zzq_args[0])    \
947
                     : "$11", "$12", "memory");                   \
948
    _zzq_result;                                                  \
949
  })
950
951
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                       \
952
  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                   \
953
    volatile unsigned int __addr;                                 \
954
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE               \
955
                     /* %t9 = guest_NRADDR */                     \
956
                     "or $14, $14, $14\n\t"                       \
957
                     "move %0, $11"     /*result*/                \
958
                     : "=r" (__addr)                              \
959
                     :                                            \
960
                     : "$11"                                      \
961
                    );                                            \
962
    _zzq_orig->nraddr = __addr;                                   \
963
  }
964
965
#define VALGRIND_CALL_NOREDIR_T9                                 \
966
                     __SPECIAL_INSTRUCTION_PREAMBLE              \
967
                     /* call-noredir *%t9 */                     \
968
                     "or $15, $15, $15\n\t"
969
970
#define VALGRIND_VEX_INJECT_IR()                                 \
971
 do {                                                            \
972
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE              \
973
                     "or $11, $11, $11\n\t"                      \
974
                    );                                           \
975
 } while (0)
976
977
978
#endif /* PLAT_mips32_linux */
979
980
/* ------------------------- mips64-linux ---------------- */
981
982
#if defined(PLAT_mips64_linux)
983
984
typedef
985
   struct {
986
      unsigned long nraddr; /* where's the code? */
987
   }
988
   OrigFn;
989
990
/* dsll $0,$0, 3
991
 * dsll $0,$0, 13
992
 * dsll $0,$0, 29
993
 * dsll $0,$0, 19*/
994
#define __SPECIAL_INSTRUCTION_PREAMBLE                              \
995
                     "dsll $0,$0, 3 ; dsll $0,$0,13\n\t"            \
996
                     "dsll $0,$0,29 ; dsll $0,$0,19\n\t"
997
998
#define VALGRIND_DO_CLIENT_REQUEST_EXPR(                            \
999
       _zzq_default, _zzq_request,                                  \
1000
       _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5)       \
1001
  __extension__                                                     \
1002
  ({ volatile unsigned long int _zzq_args[6];                       \
1003
    volatile unsigned long int _zzq_result;                         \
1004
    _zzq_args[0] = (unsigned long int)(_zzq_request);               \
1005
    _zzq_args[1] = (unsigned long int)(_zzq_arg1);                  \
1006
    _zzq_args[2] = (unsigned long int)(_zzq_arg2);                  \
1007
    _zzq_args[3] = (unsigned long int)(_zzq_arg3);                  \
1008
    _zzq_args[4] = (unsigned long int)(_zzq_arg4);                  \
1009
    _zzq_args[5] = (unsigned long int)(_zzq_arg5);                  \
1010
        __asm__ volatile("move $11, %1\n\t" /*default*/             \
1011
                         "move $12, %2\n\t" /*ptr*/                 \
1012
                         __SPECIAL_INSTRUCTION_PREAMBLE             \
1013
                         /* $11 = client_request ( $12 ) */         \
1014
                         "or $13, $13, $13\n\t"                     \
1015
                         "move %0, $11\n\t"     /*result*/          \
1016
                         : "=r" (_zzq_result)                       \
1017
                         : "r" (_zzq_default), "r" (&_zzq_args[0])  \
1018
                         : "$11", "$12", "memory");                 \
1019
    _zzq_result;                                                    \
1020
  })
1021
1022
#define VALGRIND_GET_NR_CONTEXT(_zzq_rlval)                         \
1023
  { volatile OrigFn* _zzq_orig = &(_zzq_rlval);                     \
1024
    volatile unsigned long int __addr;                              \
1025
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
1026
                     /* $11 = guest_NRADDR */                       \
1027
                     "or $14, $14, $14\n\t"                         \
1028
                     "move %0, $11"     /*result*/                  \
1029
                     : "=r" (__addr)                                \
1030
                     :                                              \
1031
                     : "$11");                                      \
1032
    _zzq_orig->nraddr = __addr;                                     \
1033
  }
1034
1035
#define VALGRIND_CALL_NOREDIR_T9                                    \
1036
                     __SPECIAL_INSTRUCTION_PREAMBLE                 \
1037
                     /* call-noredir $25 */                         \
1038
                     "or $15, $15, $15\n\t"
1039
1040
#define VALGRIND_VEX_INJECT_IR()                                    \
1041
 do {                                                               \
1042
    __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE                 \
1043
                     "or $11, $11, $11\n\t"                         \
1044
                    );                                              \
1045
 } while (0)
1046
1047
#endif /* PLAT_mips64_linux */
1048
1049
/* Insert assembly code for other platforms here... */
1050
1051
#endif /* NVALGRIND */
1052
1053
1054
/* ------------------------------------------------------------------ */
1055
/* PLATFORM SPECIFICS for FUNCTION WRAPPING.  This is all very        */
1056
/* ugly.  It's the least-worst tradeoff I can think of.               */
1057
/* ------------------------------------------------------------------ */
1058
1059
/* This section defines magic (a.k.a appalling-hack) macros for doing
1060
   guaranteed-no-redirection macros, so as to get from function
1061
   wrappers to the functions they are wrapping.  The whole point is to
1062
   construct standard call sequences, but to do the call itself with a
1063
   special no-redirect call pseudo-instruction that the JIT
1064
   understands and handles specially.  This section is long and
1065
   repetitious, and I can't see a way to make it shorter.
1066
1067
   The naming scheme is as follows:
1068
1069
      CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc}
1070
1071
   'W' stands for "word" and 'v' for "void".  Hence there are
1072
   different macros for calling arity 0, 1, 2, 3, 4, etc, functions,
1073
   and for each, the possibility of returning a word-typed result, or
1074
   no result.
1075
*/
1076
1077
/* Use these to write the name of your wrapper.  NOTE: duplicates
1078
   VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h.  NOTE also: inserts
1079
   the default behaviour equivalance class tag "0000" into the name.
1080
   See pub_tool_redir.h for details -- normally you don't need to
1081
   think about this, though. */
1082
1083
/* Use an extra level of macroisation so as to ensure the soname/fnname
1084
   args are fully macro-expanded before pasting them together. */
1085
#define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd
1086
1087
#define I_WRAP_SONAME_FNNAME_ZU(soname,fnname)                    \
1088
   VG_CONCAT4(_vgw00000ZU_,soname,_,fnname)
1089
1090
#define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname)                    \
1091
   VG_CONCAT4(_vgw00000ZZ_,soname,_,fnname)
1092
1093
/* Use this macro from within a wrapper function to collect the
1094
   context (address and possibly other info) of the original function.
1095
   Once you have that you can then use it in one of the CALL_FN_
1096
   macros.  The type of the argument _lval is OrigFn. */
1097
#define VALGRIND_GET_ORIG_FN(_lval)  VALGRIND_GET_NR_CONTEXT(_lval)
1098
1099
/* Also provide end-user facilities for function replacement, rather
1100
   than wrapping.  A replacement function differs from a wrapper in
1101
   that it has no way to get hold of the original function being
1102
   called, and hence no way to call onwards to it.  In a replacement
1103
   function, VALGRIND_GET_ORIG_FN always returns zero. */
1104
1105
#define I_REPLACE_SONAME_FNNAME_ZU(soname,fnname)                 \
1106
   VG_CONCAT4(_vgr00000ZU_,soname,_,fnname)
1107
1108
#define I_REPLACE_SONAME_FNNAME_ZZ(soname,fnname)                 \
1109
   VG_CONCAT4(_vgr00000ZZ_,soname,_,fnname)
1110
1111
/* Derivatives of the main macros below, for calling functions
1112
   returning void. */
1113
1114
#define CALL_FN_v_v(fnptr)                                        \
1115
   do { volatile unsigned long _junk;                             \
1116
        CALL_FN_W_v(_junk,fnptr); } while (0)
1117
1118
#define CALL_FN_v_W(fnptr, arg1)                                  \
1119
   do { volatile unsigned long _junk;                             \
1120
        CALL_FN_W_W(_junk,fnptr,arg1); } while (0)
1121
1122
#define CALL_FN_v_WW(fnptr, arg1,arg2)                            \
1123
   do { volatile unsigned long _junk;                             \
1124
        CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0)
1125
1126
#define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3)                      \
1127
   do { volatile unsigned long _junk;                             \
1128
        CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0)
1129
1130
#define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4)                \
1131
   do { volatile unsigned long _junk;                             \
1132
        CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0)
1133
1134
#define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5)             \
1135
   do { volatile unsigned long _junk;                             \
1136
        CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0)
1137
1138
#define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6)        \
1139
   do { volatile unsigned long _junk;                             \
1140
        CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0)
1141
1142
#define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7)   \
1143
   do { volatile unsigned long _junk;                             \
1144
        CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0)
1145
1146
/* ----------------- x86-{linux,darwin,solaris} ---------------- */
1147
1148
#if defined(PLAT_x86_linux)  ||  defined(PLAT_x86_darwin) \
1149
    ||  defined(PLAT_x86_solaris)
1150
1151
/* These regs are trashed by the hidden call.  No need to mention eax
1152
   as gcc can already see that, plus causes gcc to bomb. */
1153
#define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx"
1154
1155
/* Macros to save and align the stack before making a function
1156
   call and restore it afterwards as gcc may not keep the stack
1157
   pointer aligned if it doesn't realise calls are being made
1158
   to other functions. */
1159
1160
#define VALGRIND_ALIGN_STACK               \
1161
      "movl %%esp,%%edi\n\t"               \
1162
      "andl $0xfffffff0,%%esp\n\t"
1163
#define VALGRIND_RESTORE_STACK             \
1164
      "movl %%edi,%%esp\n\t"
1165
1166
/* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned
1167
   long) == 4. */
1168
1169
#define CALL_FN_W_v(lval, orig)                                   \
1170
   do {                                                           \
1171
      volatile OrigFn        _orig = (orig);                      \
1172
      volatile unsigned long _argvec[1];                          \
1173
      volatile unsigned long _res;                                \
1174
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1175
      __asm__ volatile(                                           \
1176
         VALGRIND_ALIGN_STACK                                     \
1177
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1178
         VALGRIND_CALL_NOREDIR_EAX                                \
1179
         VALGRIND_RESTORE_STACK                                   \
1180
         : /*out*/   "=a" (_res)                                  \
1181
         : /*in*/    "a" (&_argvec[0])                            \
1182
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1183
      );                                                          \
1184
      lval = (__typeof__(lval)) _res;                             \
1185
   } while (0)
1186
1187
#define CALL_FN_W_W(lval, orig, arg1)                             \
1188
   do {                                                           \
1189
      volatile OrigFn        _orig = (orig);                      \
1190
      volatile unsigned long _argvec[2];                          \
1191
      volatile unsigned long _res;                                \
1192
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1193
      _argvec[1] = (unsigned long)(arg1);                         \
1194
      __asm__ volatile(                                           \
1195
         VALGRIND_ALIGN_STACK                                     \
1196
         "subl $12, %%esp\n\t"                                    \
1197
         "pushl 4(%%eax)\n\t"                                     \
1198
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1199
         VALGRIND_CALL_NOREDIR_EAX                                \
1200
         VALGRIND_RESTORE_STACK                                   \
1201
         : /*out*/   "=a" (_res)                                  \
1202
         : /*in*/    "a" (&_argvec[0])                            \
1203
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1204
      );                                                          \
1205
      lval = (__typeof__(lval)) _res;                             \
1206
   } while (0)
1207
1208
#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
1209
   do {                                                           \
1210
      volatile OrigFn        _orig = (orig);                      \
1211
      volatile unsigned long _argvec[3];                          \
1212
      volatile unsigned long _res;                                \
1213
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1214
      _argvec[1] = (unsigned long)(arg1);                         \
1215
      _argvec[2] = (unsigned long)(arg2);                         \
1216
      __asm__ volatile(                                           \
1217
         VALGRIND_ALIGN_STACK                                     \
1218
         "subl $8, %%esp\n\t"                                     \
1219
         "pushl 8(%%eax)\n\t"                                     \
1220
         "pushl 4(%%eax)\n\t"                                     \
1221
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1222
         VALGRIND_CALL_NOREDIR_EAX                                \
1223
         VALGRIND_RESTORE_STACK                                   \
1224
         : /*out*/   "=a" (_res)                                  \
1225
         : /*in*/    "a" (&_argvec[0])                            \
1226
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1227
      );                                                          \
1228
      lval = (__typeof__(lval)) _res;                             \
1229
   } while (0)
1230
1231
#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
1232
   do {                                                           \
1233
      volatile OrigFn        _orig = (orig);                      \
1234
      volatile unsigned long _argvec[4];                          \
1235
      volatile unsigned long _res;                                \
1236
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1237
      _argvec[1] = (unsigned long)(arg1);                         \
1238
      _argvec[2] = (unsigned long)(arg2);                         \
1239
      _argvec[3] = (unsigned long)(arg3);                         \
1240
      __asm__ volatile(                                           \
1241
         VALGRIND_ALIGN_STACK                                     \
1242
         "subl $4, %%esp\n\t"                                     \
1243
         "pushl 12(%%eax)\n\t"                                    \
1244
         "pushl 8(%%eax)\n\t"                                     \
1245
         "pushl 4(%%eax)\n\t"                                     \
1246
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1247
         VALGRIND_CALL_NOREDIR_EAX                                \
1248
         VALGRIND_RESTORE_STACK                                   \
1249
         : /*out*/   "=a" (_res)                                  \
1250
         : /*in*/    "a" (&_argvec[0])                            \
1251
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1252
      );                                                          \
1253
      lval = (__typeof__(lval)) _res;                             \
1254
   } while (0)
1255
1256
#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
1257
   do {                                                           \
1258
      volatile OrigFn        _orig = (orig);                      \
1259
      volatile unsigned long _argvec[5];                          \
1260
      volatile unsigned long _res;                                \
1261
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1262
      _argvec[1] = (unsigned long)(arg1);                         \
1263
      _argvec[2] = (unsigned long)(arg2);                         \
1264
      _argvec[3] = (unsigned long)(arg3);                         \
1265
      _argvec[4] = (unsigned long)(arg4);                         \
1266
      __asm__ volatile(                                           \
1267
         VALGRIND_ALIGN_STACK                                     \
1268
         "pushl 16(%%eax)\n\t"                                    \
1269
         "pushl 12(%%eax)\n\t"                                    \
1270
         "pushl 8(%%eax)\n\t"                                     \
1271
         "pushl 4(%%eax)\n\t"                                     \
1272
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1273
         VALGRIND_CALL_NOREDIR_EAX                                \
1274
         VALGRIND_RESTORE_STACK                                   \
1275
         : /*out*/   "=a" (_res)                                  \
1276
         : /*in*/    "a" (&_argvec[0])                            \
1277
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1278
      );                                                          \
1279
      lval = (__typeof__(lval)) _res;                             \
1280
   } while (0)
1281
1282
#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
1283
   do {                                                           \
1284
      volatile OrigFn        _orig = (orig);                      \
1285
      volatile unsigned long _argvec[6];                          \
1286
      volatile unsigned long _res;                                \
1287
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1288
      _argvec[1] = (unsigned long)(arg1);                         \
1289
      _argvec[2] = (unsigned long)(arg2);                         \
1290
      _argvec[3] = (unsigned long)(arg3);                         \
1291
      _argvec[4] = (unsigned long)(arg4);                         \
1292
      _argvec[5] = (unsigned long)(arg5);                         \
1293
      __asm__ volatile(                                           \
1294
         VALGRIND_ALIGN_STACK                                     \
1295
         "subl $12, %%esp\n\t"                                    \
1296
         "pushl 20(%%eax)\n\t"                                    \
1297
         "pushl 16(%%eax)\n\t"                                    \
1298
         "pushl 12(%%eax)\n\t"                                    \
1299
         "pushl 8(%%eax)\n\t"                                     \
1300
         "pushl 4(%%eax)\n\t"                                     \
1301
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1302
         VALGRIND_CALL_NOREDIR_EAX                                \
1303
         VALGRIND_RESTORE_STACK                                   \
1304
         : /*out*/   "=a" (_res)                                  \
1305
         : /*in*/    "a" (&_argvec[0])                            \
1306
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1307
      );                                                          \
1308
      lval = (__typeof__(lval)) _res;                             \
1309
   } while (0)
1310
1311
#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
1312
   do {                                                           \
1313
      volatile OrigFn        _orig = (orig);                      \
1314
      volatile unsigned long _argvec[7];                          \
1315
      volatile unsigned long _res;                                \
1316
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1317
      _argvec[1] = (unsigned long)(arg1);                         \
1318
      _argvec[2] = (unsigned long)(arg2);                         \
1319
      _argvec[3] = (unsigned long)(arg3);                         \
1320
      _argvec[4] = (unsigned long)(arg4);                         \
1321
      _argvec[5] = (unsigned long)(arg5);                         \
1322
      _argvec[6] = (unsigned long)(arg6);                         \
1323
      __asm__ volatile(                                           \
1324
         VALGRIND_ALIGN_STACK                                     \
1325
         "subl $8, %%esp\n\t"                                     \
1326
         "pushl 24(%%eax)\n\t"                                    \
1327
         "pushl 20(%%eax)\n\t"                                    \
1328
         "pushl 16(%%eax)\n\t"                                    \
1329
         "pushl 12(%%eax)\n\t"                                    \
1330
         "pushl 8(%%eax)\n\t"                                     \
1331
         "pushl 4(%%eax)\n\t"                                     \
1332
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1333
         VALGRIND_CALL_NOREDIR_EAX                                \
1334
         VALGRIND_RESTORE_STACK                                   \
1335
         : /*out*/   "=a" (_res)                                  \
1336
         : /*in*/    "a" (&_argvec[0])                            \
1337
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1338
      );                                                          \
1339
      lval = (__typeof__(lval)) _res;                             \
1340
   } while (0)
1341
1342
#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1343
                                 arg7)                            \
1344
   do {                                                           \
1345
      volatile OrigFn        _orig = (orig);                      \
1346
      volatile unsigned long _argvec[8];                          \
1347
      volatile unsigned long _res;                                \
1348
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1349
      _argvec[1] = (unsigned long)(arg1);                         \
1350
      _argvec[2] = (unsigned long)(arg2);                         \
1351
      _argvec[3] = (unsigned long)(arg3);                         \
1352
      _argvec[4] = (unsigned long)(arg4);                         \
1353
      _argvec[5] = (unsigned long)(arg5);                         \
1354
      _argvec[6] = (unsigned long)(arg6);                         \
1355
      _argvec[7] = (unsigned long)(arg7);                         \
1356
      __asm__ volatile(                                           \
1357
         VALGRIND_ALIGN_STACK                                     \
1358
         "subl $4, %%esp\n\t"                                     \
1359
         "pushl 28(%%eax)\n\t"                                    \
1360
         "pushl 24(%%eax)\n\t"                                    \
1361
         "pushl 20(%%eax)\n\t"                                    \
1362
         "pushl 16(%%eax)\n\t"                                    \
1363
         "pushl 12(%%eax)\n\t"                                    \
1364
         "pushl 8(%%eax)\n\t"                                     \
1365
         "pushl 4(%%eax)\n\t"                                     \
1366
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1367
         VALGRIND_CALL_NOREDIR_EAX                                \
1368
         VALGRIND_RESTORE_STACK                                   \
1369
         : /*out*/   "=a" (_res)                                  \
1370
         : /*in*/    "a" (&_argvec[0])                            \
1371
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1372
      );                                                          \
1373
      lval = (__typeof__(lval)) _res;                             \
1374
   } while (0)
1375
1376
#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1377
                                 arg7,arg8)                       \
1378
   do {                                                           \
1379
      volatile OrigFn        _orig = (orig);                      \
1380
      volatile unsigned long _argvec[9];                          \
1381
      volatile unsigned long _res;                                \
1382
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1383
      _argvec[1] = (unsigned long)(arg1);                         \
1384
      _argvec[2] = (unsigned long)(arg2);                         \
1385
      _argvec[3] = (unsigned long)(arg3);                         \
1386
      _argvec[4] = (unsigned long)(arg4);                         \
1387
      _argvec[5] = (unsigned long)(arg5);                         \
1388
      _argvec[6] = (unsigned long)(arg6);                         \
1389
      _argvec[7] = (unsigned long)(arg7);                         \
1390
      _argvec[8] = (unsigned long)(arg8);                         \
1391
      __asm__ volatile(                                           \
1392
         VALGRIND_ALIGN_STACK                                     \
1393
         "pushl 32(%%eax)\n\t"                                    \
1394
         "pushl 28(%%eax)\n\t"                                    \
1395
         "pushl 24(%%eax)\n\t"                                    \
1396
         "pushl 20(%%eax)\n\t"                                    \
1397
         "pushl 16(%%eax)\n\t"                                    \
1398
         "pushl 12(%%eax)\n\t"                                    \
1399
         "pushl 8(%%eax)\n\t"                                     \
1400
         "pushl 4(%%eax)\n\t"                                     \
1401
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1402
         VALGRIND_CALL_NOREDIR_EAX                                \
1403
         VALGRIND_RESTORE_STACK                                   \
1404
         : /*out*/   "=a" (_res)                                  \
1405
         : /*in*/    "a" (&_argvec[0])                            \
1406
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1407
      );                                                          \
1408
      lval = (__typeof__(lval)) _res;                             \
1409
   } while (0)
1410
1411
#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
1412
                                 arg7,arg8,arg9)                  \
1413
   do {                                                           \
1414
      volatile OrigFn        _orig = (orig);                      \
1415
      volatile unsigned long _argvec[10];                         \
1416
      volatile unsigned long _res;                                \
1417
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1418
      _argvec[1] = (unsigned long)(arg1);                         \
1419
      _argvec[2] = (unsigned long)(arg2);                         \
1420
      _argvec[3] = (unsigned long)(arg3);                         \
1421
      _argvec[4] = (unsigned long)(arg4);                         \
1422
      _argvec[5] = (unsigned long)(arg5);                         \
1423
      _argvec[6] = (unsigned long)(arg6);                         \
1424
      _argvec[7] = (unsigned long)(arg7);                         \
1425
      _argvec[8] = (unsigned long)(arg8);                         \
1426
      _argvec[9] = (unsigned long)(arg9);                         \
1427
      __asm__ volatile(                                           \
1428
         VALGRIND_ALIGN_STACK                                     \
1429
         "subl $12, %%esp\n\t"                                    \
1430
         "pushl 36(%%eax)\n\t"                                    \
1431
         "pushl 32(%%eax)\n\t"                                    \
1432
         "pushl 28(%%eax)\n\t"                                    \
1433
         "pushl 24(%%eax)\n\t"                                    \
1434
         "pushl 20(%%eax)\n\t"                                    \
1435
         "pushl 16(%%eax)\n\t"                                    \
1436
         "pushl 12(%%eax)\n\t"                                    \
1437
         "pushl 8(%%eax)\n\t"                                     \
1438
         "pushl 4(%%eax)\n\t"                                     \
1439
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1440
         VALGRIND_CALL_NOREDIR_EAX                                \
1441
         VALGRIND_RESTORE_STACK                                   \
1442
         : /*out*/   "=a" (_res)                                  \
1443
         : /*in*/    "a" (&_argvec[0])                            \
1444
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1445
      );                                                          \
1446
      lval = (__typeof__(lval)) _res;                             \
1447
   } while (0)
1448
1449
#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
1450
                                  arg7,arg8,arg9,arg10)           \
1451
   do {                                                           \
1452
      volatile OrigFn        _orig = (orig);                      \
1453
      volatile unsigned long _argvec[11];                         \
1454
      volatile unsigned long _res;                                \
1455
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1456
      _argvec[1] = (unsigned long)(arg1);                         \
1457
      _argvec[2] = (unsigned long)(arg2);                         \
1458
      _argvec[3] = (unsigned long)(arg3);                         \
1459
      _argvec[4] = (unsigned long)(arg4);                         \
1460
      _argvec[5] = (unsigned long)(arg5);                         \
1461
      _argvec[6] = (unsigned long)(arg6);                         \
1462
      _argvec[7] = (unsigned long)(arg7);                         \
1463
      _argvec[8] = (unsigned long)(arg8);                         \
1464
      _argvec[9] = (unsigned long)(arg9);                         \
1465
      _argvec[10] = (unsigned long)(arg10);                       \
1466
      __asm__ volatile(                                           \
1467
         VALGRIND_ALIGN_STACK                                     \
1468
         "subl $8, %%esp\n\t"                                     \
1469
         "pushl 40(%%eax)\n\t"                                    \
1470
         "pushl 36(%%eax)\n\t"                                    \
1471
         "pushl 32(%%eax)\n\t"                                    \
1472
         "pushl 28(%%eax)\n\t"                                    \
1473
         "pushl 24(%%eax)\n\t"                                    \
1474
         "pushl 20(%%eax)\n\t"                                    \
1475
         "pushl 16(%%eax)\n\t"                                    \
1476
         "pushl 12(%%eax)\n\t"                                    \
1477
         "pushl 8(%%eax)\n\t"                                     \
1478
         "pushl 4(%%eax)\n\t"                                     \
1479
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1480
         VALGRIND_CALL_NOREDIR_EAX                                \
1481
         VALGRIND_RESTORE_STACK                                   \
1482
         : /*out*/   "=a" (_res)                                  \
1483
         : /*in*/    "a" (&_argvec[0])                            \
1484
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1485
      );                                                          \
1486
      lval = (__typeof__(lval)) _res;                             \
1487
   } while (0)
1488
1489
#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1490
                                  arg6,arg7,arg8,arg9,arg10,      \
1491
                                  arg11)                          \
1492
   do {                                                           \
1493
      volatile OrigFn        _orig = (orig);                      \
1494
      volatile unsigned long _argvec[12];                         \
1495
      volatile unsigned long _res;                                \
1496
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1497
      _argvec[1] = (unsigned long)(arg1);                         \
1498
      _argvec[2] = (unsigned long)(arg2);                         \
1499
      _argvec[3] = (unsigned long)(arg3);                         \
1500
      _argvec[4] = (unsigned long)(arg4);                         \
1501
      _argvec[5] = (unsigned long)(arg5);                         \
1502
      _argvec[6] = (unsigned long)(arg6);                         \
1503
      _argvec[7] = (unsigned long)(arg7);                         \
1504
      _argvec[8] = (unsigned long)(arg8);                         \
1505
      _argvec[9] = (unsigned long)(arg9);                         \
1506
      _argvec[10] = (unsigned long)(arg10);                       \
1507
      _argvec[11] = (unsigned long)(arg11);                       \
1508
      __asm__ volatile(                                           \
1509
         VALGRIND_ALIGN_STACK                                     \
1510
         "subl $4, %%esp\n\t"                                     \
1511
         "pushl 44(%%eax)\n\t"                                    \
1512
         "pushl 40(%%eax)\n\t"                                    \
1513
         "pushl 36(%%eax)\n\t"                                    \
1514
         "pushl 32(%%eax)\n\t"                                    \
1515
         "pushl 28(%%eax)\n\t"                                    \
1516
         "pushl 24(%%eax)\n\t"                                    \
1517
         "pushl 20(%%eax)\n\t"                                    \
1518
         "pushl 16(%%eax)\n\t"                                    \
1519
         "pushl 12(%%eax)\n\t"                                    \
1520
         "pushl 8(%%eax)\n\t"                                     \
1521
         "pushl 4(%%eax)\n\t"                                     \
1522
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1523
         VALGRIND_CALL_NOREDIR_EAX                                \
1524
         VALGRIND_RESTORE_STACK                                   \
1525
         : /*out*/   "=a" (_res)                                  \
1526
         : /*in*/    "a" (&_argvec[0])                            \
1527
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1528
      );                                                          \
1529
      lval = (__typeof__(lval)) _res;                             \
1530
   } while (0)
1531
1532
#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
1533
                                  arg6,arg7,arg8,arg9,arg10,      \
1534
                                  arg11,arg12)                    \
1535
   do {                                                           \
1536
      volatile OrigFn        _orig = (orig);                      \
1537
      volatile unsigned long _argvec[13];                         \
1538
      volatile unsigned long _res;                                \
1539
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
1540
      _argvec[1] = (unsigned long)(arg1);                         \
1541
      _argvec[2] = (unsigned long)(arg2);                         \
1542
      _argvec[3] = (unsigned long)(arg3);                         \
1543
      _argvec[4] = (unsigned long)(arg4);                         \
1544
      _argvec[5] = (unsigned long)(arg5);                         \
1545
      _argvec[6] = (unsigned long)(arg6);                         \
1546
      _argvec[7] = (unsigned long)(arg7);                         \
1547
      _argvec[8] = (unsigned long)(arg8);                         \
1548
      _argvec[9] = (unsigned long)(arg9);                         \
1549
      _argvec[10] = (unsigned long)(arg10);                       \
1550
      _argvec[11] = (unsigned long)(arg11);                       \
1551
      _argvec[12] = (unsigned long)(arg12);                       \
1552
      __asm__ volatile(                                           \
1553
         VALGRIND_ALIGN_STACK                                     \
1554
         "pushl 48(%%eax)\n\t"                                    \
1555
         "pushl 44(%%eax)\n\t"                                    \
1556
         "pushl 40(%%eax)\n\t"                                    \
1557
         "pushl 36(%%eax)\n\t"                                    \
1558
         "pushl 32(%%eax)\n\t"                                    \
1559
         "pushl 28(%%eax)\n\t"                                    \
1560
         "pushl 24(%%eax)\n\t"                                    \
1561
         "pushl 20(%%eax)\n\t"                                    \
1562
         "pushl 16(%%eax)\n\t"                                    \
1563
         "pushl 12(%%eax)\n\t"                                    \
1564
         "pushl 8(%%eax)\n\t"                                     \
1565
         "pushl 4(%%eax)\n\t"                                     \
1566
         "movl (%%eax), %%eax\n\t"  /* target->%eax */            \
1567
         VALGRIND_CALL_NOREDIR_EAX                                \
1568
         VALGRIND_RESTORE_STACK                                   \
1569
         : /*out*/   "=a" (_res)                                  \
1570
         : /*in*/    "a" (&_argvec[0])                            \
1571
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi"   \
1572
      );                                                          \
1573
      lval = (__typeof__(lval)) _res;                             \
1574
   } while (0)
1575
1576
#endif /* PLAT_x86_linux || PLAT_x86_darwin || PLAT_x86_solaris */
1577
1578
/* ---------------- amd64-{linux,darwin,solaris} --------------- */
1579
1580
#if defined(PLAT_amd64_linux)  ||  defined(PLAT_amd64_darwin) \
1581
    ||  defined(PLAT_amd64_solaris)
1582
1583
/* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */
1584
1585
/* These regs are trashed by the hidden call. */
1586
#define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi",       \
1587
                            "rdi", "r8", "r9", "r10", "r11"
1588
1589
/* This is all pretty complex.  It's so as to make stack unwinding
1590
   work reliably.  See bug 243270.  The basic problem is the sub and
1591
   add of 128 of %rsp in all of the following macros.  If gcc believes
1592
   the CFA is in %rsp, then unwinding may fail, because what's at the
1593
   CFA is not what gcc "expected" when it constructs the CFIs for the
1594
   places where the macros are instantiated.
1595
1596
   But we can't just add a CFI annotation to increase the CFA offset
1597
   by 128, to match the sub of 128 from %rsp, because we don't know
1598
   whether gcc has chosen %rsp as the CFA at that point, or whether it
1599
   has chosen some other register (eg, %rbp).  In the latter case,
1600
   adding a CFI annotation to change the CFA offset is simply wrong.
1601
1602
   So the solution is to get hold of the CFA using
1603
   __builtin_dwarf_cfa(), put it in a known register, and add a
1604
   CFI annotation to say what the register is.  We choose %rbp for
1605
   this (perhaps perversely), because:
1606
1607
   (1) %rbp is already subject to unwinding.  If a new register was
1608
       chosen then the unwinder would have to unwind it in all stack
1609
       traces, which is expensive, and
1610
1611
   (2) %rbp is already subject to precise exception updates in the
1612
       JIT.  If a new register was chosen, we'd have to have precise
1613
       exceptions for it too, which reduces performance of the
1614
       generated code.
1615
1616
   However .. one extra complication.  We can't just whack the result
1617
   of __builtin_dwarf_cfa() into %rbp and then add %rbp to the
1618
   list of trashed registers at the end of the inline assembly
1619
   fragments; gcc won't allow %rbp to appear in that list.  Hence
1620
   instead we need to stash %rbp in %r15 for the duration of the asm,
1621
   and say that %r15 is trashed instead.  gcc seems happy to go with
1622
   that.
1623
1624
   Oh .. and this all needs to be conditionalised so that it is
1625
   unchanged from before this commit, when compiled with older gccs
1626
   that don't support __builtin_dwarf_cfa.  Furthermore, since
1627
   this header file is freestanding, it has to be independent of
1628
   config.h, and so the following conditionalisation cannot depend on
1629
   configure time checks.
1630
1631
   Although it's not clear from
1632
   'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)',
1633
   this expression excludes Darwin.
1634
   .cfi directives in Darwin assembly appear to be completely
1635
   different and I haven't investigated how they work.
1636
1637
   For even more entertainment value, note we have to use the
1638
   completely undocumented __builtin_dwarf_cfa(), which appears to
1639
   really compute the CFA, whereas __builtin_frame_address(0) claims
1640
   to but actually doesn't.  See
1641
   https://bugs.kde.org/show_bug.cgi?id=243270#c47
1642
*/
1643
#if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
1644
#  define __FRAME_POINTER                                         \
1645
      ,"r"(__builtin_dwarf_cfa())
1646
#  define VALGRIND_CFI_PROLOGUE                                   \
1647
      "movq %%rbp, %%r15\n\t"                                     \
1648
      "movq %2, %%rbp\n\t"                                        \
1649
      ".cfi_remember_state\n\t"                                   \
1650
      ".cfi_def_cfa rbp, 0\n\t"
1651
#  define VALGRIND_CFI_EPILOGUE                                   \
1652
      "movq %%r15, %%rbp\n\t"                                     \
1653
      ".cfi_restore_state\n\t"
1654
#else
1655
#  define __FRAME_POINTER
1656
#  define VALGRIND_CFI_PROLOGUE
1657
#  define VALGRIND_CFI_EPILOGUE
1658
#endif
1659
1660
/* Macros to save and align the stack before making a function
1661
   call and restore it afterwards as gcc may not keep the stack
1662
   pointer aligned if it doesn't realise calls are being made
1663
   to other functions. */
1664
1665
#define VALGRIND_ALIGN_STACK               \
1666
      "movq %%rsp,%%r14\n\t"               \
1667
      "andq $0xfffffffffffffff0,%%rsp\n\t"
1668
#define VALGRIND_RESTORE_STACK             \
1669
      "movq %%r14,%%rsp\n\t"
1670
1671
/* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned
1672
   long) == 8. */
1673
1674
/* NB 9 Sept 07.  There is a nasty kludge here in all these CALL_FN_
1675
   macros.  In order not to trash the stack redzone, we need to drop
1676
   %rsp by 128 before the hidden call, and restore afterwards.  The
1677
   nastyness is that it is only by luck that the stack still appears
1678
   to be unwindable during the hidden call - since then the behaviour
1679
   of any routine using this macro does not match what the CFI data
1680
   says.  Sigh.
1681
1682
   Why is this important?  Imagine that a wrapper has a stack
1683
   allocated local, and passes to the hidden call, a pointer to it.
1684
   Because gcc does not know about the hidden call, it may allocate
1685
   that local in the redzone.  Unfortunately the hidden call may then
1686
   trash it before it comes to use it.  So we must step clear of the
1687
   redzone, for the duration of the hidden call, to make it safe.
1688
1689
   Probably the same problem afflicts the other redzone-style ABIs too
1690
   (ppc64-linux); but for those, the stack is
1691
   self describing (none of this CFI nonsense) so at least messing
1692
   with the stack pointer doesn't give a danger of non-unwindable
1693
   stack. */
1694
1695
#define CALL_FN_W_v(lval, orig)                                        \
1696
   do {                                                                \
1697
      volatile OrigFn        _orig = (orig);                           \
1698
      volatile unsigned long _argvec[1];                               \
1699
      volatile unsigned long _res;                                     \
1700
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
1701
      __asm__ volatile(                                                \
1702
         VALGRIND_CFI_PROLOGUE                                         \
1703
         VALGRIND_ALIGN_STACK                                          \
1704
         "subq $128,%%rsp\n\t"                                         \
1705
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1706
         VALGRIND_CALL_NOREDIR_RAX                                     \
1707
         VALGRIND_RESTORE_STACK                                        \
1708
         VALGRIND_CFI_EPILOGUE                                         \
1709
         : /*out*/   "=a" (_res)                                       \
1710
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1711
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1712
      );                                                               \
1713
      lval = (__typeof__(lval)) _res;                                  \
1714
   } while (0)
1715
1716
#define CALL_FN_W_W(lval, orig, arg1)                                  \
1717
   do {                                                                \
1718
      volatile OrigFn        _orig = (orig);                           \
1719
      volatile unsigned long _argvec[2];                               \
1720
      volatile unsigned long _res;                                     \
1721
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
1722
      _argvec[1] = (unsigned long)(arg1);                              \
1723
      __asm__ volatile(                                                \
1724
         VALGRIND_CFI_PROLOGUE                                         \
1725
         VALGRIND_ALIGN_STACK                                          \
1726
         "subq $128,%%rsp\n\t"                                         \
1727
         "movq 8(%%rax), %%rdi\n\t"                                    \
1728
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1729
         VALGRIND_CALL_NOREDIR_RAX                                     \
1730
         VALGRIND_RESTORE_STACK                                        \
1731
         VALGRIND_CFI_EPILOGUE                                         \
1732
         : /*out*/   "=a" (_res)                                       \
1733
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1734
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1735
      );                                                               \
1736
      lval = (__typeof__(lval)) _res;                                  \
1737
   } while (0)
1738
1739
#define CALL_FN_W_WW(lval, orig, arg1,arg2)                            \
1740
   do {                                                                \
1741
      volatile OrigFn        _orig = (orig);                           \
1742
      volatile unsigned long _argvec[3];                               \
1743
      volatile unsigned long _res;                                     \
1744
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
1745
      _argvec[1] = (unsigned long)(arg1);                              \
1746
      _argvec[2] = (unsigned long)(arg2);                              \
1747
      __asm__ volatile(                                                \
1748
         VALGRIND_CFI_PROLOGUE                                         \
1749
         VALGRIND_ALIGN_STACK                                          \
1750
         "subq $128,%%rsp\n\t"                                         \
1751
         "movq 16(%%rax), %%rsi\n\t"                                   \
1752
         "movq 8(%%rax), %%rdi\n\t"                                    \
1753
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1754
         VALGRIND_CALL_NOREDIR_RAX                                     \
1755
         VALGRIND_RESTORE_STACK                                        \
1756
         VALGRIND_CFI_EPILOGUE                                         \
1757
         : /*out*/   "=a" (_res)                                       \
1758
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1759
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1760
      );                                                               \
1761
      lval = (__typeof__(lval)) _res;                                  \
1762
   } while (0)
1763
1764
#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                      \
1765
   do {                                                                \
1766
      volatile OrigFn        _orig = (orig);                           \
1767
      volatile unsigned long _argvec[4];                               \
1768
      volatile unsigned long _res;                                     \
1769
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
1770
      _argvec[1] = (unsigned long)(arg1);                              \
1771
      _argvec[2] = (unsigned long)(arg2);                              \
1772
      _argvec[3] = (unsigned long)(arg3);                              \
1773
      __asm__ volatile(                                                \
1774
         VALGRIND_CFI_PROLOGUE                                         \
1775
         VALGRIND_ALIGN_STACK                                          \
1776
         "subq $128,%%rsp\n\t"                                         \
1777
         "movq 24(%%rax), %%rdx\n\t"                                   \
1778
         "movq 16(%%rax), %%rsi\n\t"                                   \
1779
         "movq 8(%%rax), %%rdi\n\t"                                    \
1780
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1781
         VALGRIND_CALL_NOREDIR_RAX                                     \
1782
         VALGRIND_RESTORE_STACK                                        \
1783
         VALGRIND_CFI_EPILOGUE                                         \
1784
         : /*out*/   "=a" (_res)                                       \
1785
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1786
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1787
      );                                                               \
1788
      lval = (__typeof__(lval)) _res;                                  \
1789
   } while (0)
1790
1791
#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)                \
1792
   do {                                                                \
1793
      volatile OrigFn        _orig = (orig);                           \
1794
      volatile unsigned long _argvec[5];                               \
1795
      volatile unsigned long _res;                                     \
1796
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
1797
      _argvec[1] = (unsigned long)(arg1);                              \
1798
      _argvec[2] = (unsigned long)(arg2);                              \
1799
      _argvec[3] = (unsigned long)(arg3);                              \
1800
      _argvec[4] = (unsigned long)(arg4);                              \
1801
      __asm__ volatile(                                                \
1802
         VALGRIND_CFI_PROLOGUE                                         \
1803
         VALGRIND_ALIGN_STACK                                          \
1804
         "subq $128,%%rsp\n\t"                                         \
1805
         "movq 32(%%rax), %%rcx\n\t"                                   \
1806
         "movq 24(%%rax), %%rdx\n\t"                                   \
1807
         "movq 16(%%rax), %%rsi\n\t"                                   \
1808
         "movq 8(%%rax), %%rdi\n\t"                                    \
1809
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1810
         VALGRIND_CALL_NOREDIR_RAX                                     \
1811
         VALGRIND_RESTORE_STACK                                        \
1812
         VALGRIND_CFI_EPILOGUE                                         \
1813
         : /*out*/   "=a" (_res)                                       \
1814
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1815
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1816
      );                                                               \
1817
      lval = (__typeof__(lval)) _res;                                  \
1818
   } while (0)
1819
1820
#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)             \
1821
   do {                                                                \
1822
      volatile OrigFn        _orig = (orig);                           \
1823
      volatile unsigned long _argvec[6];                               \
1824
      volatile unsigned long _res;                                     \
1825
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
1826
      _argvec[1] = (unsigned long)(arg1);                              \
1827
      _argvec[2] = (unsigned long)(arg2);                              \
1828
      _argvec[3] = (unsigned long)(arg3);                              \
1829
      _argvec[4] = (unsigned long)(arg4);                              \
1830
      _argvec[5] = (unsigned long)(arg5);                              \
1831
      __asm__ volatile(                                                \
1832
         VALGRIND_CFI_PROLOGUE                                         \
1833
         VALGRIND_ALIGN_STACK                                          \
1834
         "subq $128,%%rsp\n\t"                                         \
1835
         "movq 40(%%rax), %%r8\n\t"                                    \
1836
         "movq 32(%%rax), %%rcx\n\t"                                   \
1837
         "movq 24(%%rax), %%rdx\n\t"                                   \
1838
         "movq 16(%%rax), %%rsi\n\t"                                   \
1839
         "movq 8(%%rax), %%rdi\n\t"                                    \
1840
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1841
         VALGRIND_CALL_NOREDIR_RAX                                     \
1842
         VALGRIND_RESTORE_STACK                                        \
1843
         VALGRIND_CFI_EPILOGUE                                         \
1844
         : /*out*/   "=a" (_res)                                       \
1845
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1846
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1847
      );                                                               \
1848
      lval = (__typeof__(lval)) _res;                                  \
1849
   } while (0)
1850
1851
#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)        \
1852
   do {                                                                \
1853
      volatile OrigFn        _orig = (orig);                           \
1854
      volatile unsigned long _argvec[7];                               \
1855
      volatile unsigned long _res;                                     \
1856
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
1857
      _argvec[1] = (unsigned long)(arg1);                              \
1858
      _argvec[2] = (unsigned long)(arg2);                              \
1859
      _argvec[3] = (unsigned long)(arg3);                              \
1860
      _argvec[4] = (unsigned long)(arg4);                              \
1861
      _argvec[5] = (unsigned long)(arg5);                              \
1862
      _argvec[6] = (unsigned long)(arg6);                              \
1863
      __asm__ volatile(                                                \
1864
         VALGRIND_CFI_PROLOGUE                                         \
1865
         VALGRIND_ALIGN_STACK                                          \
1866
         "subq $128,%%rsp\n\t"                                         \
1867
         "movq 48(%%rax), %%r9\n\t"                                    \
1868
         "movq 40(%%rax), %%r8\n\t"                                    \
1869
         "movq 32(%%rax), %%rcx\n\t"                                   \
1870
         "movq 24(%%rax), %%rdx\n\t"                                   \
1871
         "movq 16(%%rax), %%rsi\n\t"                                   \
1872
         "movq 8(%%rax), %%rdi\n\t"                                    \
1873
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1874
         VALGRIND_CALL_NOREDIR_RAX                                     \
1875
         VALGRIND_RESTORE_STACK                                        \
1876
         VALGRIND_CFI_EPILOGUE                                         \
1877
         : /*out*/   "=a" (_res)                                       \
1878
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1879
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1880
      );                                                               \
1881
      lval = (__typeof__(lval)) _res;                                  \
1882
   } while (0)
1883
1884
#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
1885
                                 arg7)                                 \
1886
   do {                                                                \
1887
      volatile OrigFn        _orig = (orig);                           \
1888
      volatile unsigned long _argvec[8];                               \
1889
      volatile unsigned long _res;                                     \
1890
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
1891
      _argvec[1] = (unsigned long)(arg1);                              \
1892
      _argvec[2] = (unsigned long)(arg2);                              \
1893
      _argvec[3] = (unsigned long)(arg3);                              \
1894
      _argvec[4] = (unsigned long)(arg4);                              \
1895
      _argvec[5] = (unsigned long)(arg5);                              \
1896
      _argvec[6] = (unsigned long)(arg6);                              \
1897
      _argvec[7] = (unsigned long)(arg7);                              \
1898
      __asm__ volatile(                                                \
1899
         VALGRIND_CFI_PROLOGUE                                         \
1900
         VALGRIND_ALIGN_STACK                                          \
1901
         "subq $136,%%rsp\n\t"                                         \
1902
         "pushq 56(%%rax)\n\t"                                         \
1903
         "movq 48(%%rax), %%r9\n\t"                                    \
1904
         "movq 40(%%rax), %%r8\n\t"                                    \
1905
         "movq 32(%%rax), %%rcx\n\t"                                   \
1906
         "movq 24(%%rax), %%rdx\n\t"                                   \
1907
         "movq 16(%%rax), %%rsi\n\t"                                   \
1908
         "movq 8(%%rax), %%rdi\n\t"                                    \
1909
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1910
         VALGRIND_CALL_NOREDIR_RAX                                     \
1911
         VALGRIND_RESTORE_STACK                                        \
1912
         VALGRIND_CFI_EPILOGUE                                         \
1913
         : /*out*/   "=a" (_res)                                       \
1914
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1915
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1916
      );                                                               \
1917
      lval = (__typeof__(lval)) _res;                                  \
1918
   } while (0)
1919
1920
#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
1921
                                 arg7,arg8)                            \
1922
   do {                                                                \
1923
      volatile OrigFn        _orig = (orig);                           \
1924
      volatile unsigned long _argvec[9];                               \
1925
      volatile unsigned long _res;                                     \
1926
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
1927
      _argvec[1] = (unsigned long)(arg1);                              \
1928
      _argvec[2] = (unsigned long)(arg2);                              \
1929
      _argvec[3] = (unsigned long)(arg3);                              \
1930
      _argvec[4] = (unsigned long)(arg4);                              \
1931
      _argvec[5] = (unsigned long)(arg5);                              \
1932
      _argvec[6] = (unsigned long)(arg6);                              \
1933
      _argvec[7] = (unsigned long)(arg7);                              \
1934
      _argvec[8] = (unsigned long)(arg8);                              \
1935
      __asm__ volatile(                                                \
1936
         VALGRIND_CFI_PROLOGUE                                         \
1937
         VALGRIND_ALIGN_STACK                                          \
1938
         "subq $128,%%rsp\n\t"                                         \
1939
         "pushq 64(%%rax)\n\t"                                         \
1940
         "pushq 56(%%rax)\n\t"                                         \
1941
         "movq 48(%%rax), %%r9\n\t"                                    \
1942
         "movq 40(%%rax), %%r8\n\t"                                    \
1943
         "movq 32(%%rax), %%rcx\n\t"                                   \
1944
         "movq 24(%%rax), %%rdx\n\t"                                   \
1945
         "movq 16(%%rax), %%rsi\n\t"                                   \
1946
         "movq 8(%%rax), %%rdi\n\t"                                    \
1947
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1948
         VALGRIND_CALL_NOREDIR_RAX                                     \
1949
         VALGRIND_RESTORE_STACK                                        \
1950
         VALGRIND_CFI_EPILOGUE                                         \
1951
         : /*out*/   "=a" (_res)                                       \
1952
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1953
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1954
      );                                                               \
1955
      lval = (__typeof__(lval)) _res;                                  \
1956
   } while (0)
1957
1958
#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,        \
1959
                                 arg7,arg8,arg9)                       \
1960
   do {                                                                \
1961
      volatile OrigFn        _orig = (orig);                           \
1962
      volatile unsigned long _argvec[10];                              \
1963
      volatile unsigned long _res;                                     \
1964
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
1965
      _argvec[1] = (unsigned long)(arg1);                              \
1966
      _argvec[2] = (unsigned long)(arg2);                              \
1967
      _argvec[3] = (unsigned long)(arg3);                              \
1968
      _argvec[4] = (unsigned long)(arg4);                              \
1969
      _argvec[5] = (unsigned long)(arg5);                              \
1970
      _argvec[6] = (unsigned long)(arg6);                              \
1971
      _argvec[7] = (unsigned long)(arg7);                              \
1972
      _argvec[8] = (unsigned long)(arg8);                              \
1973
      _argvec[9] = (unsigned long)(arg9);                              \
1974
      __asm__ volatile(                                                \
1975
         VALGRIND_CFI_PROLOGUE                                         \
1976
         VALGRIND_ALIGN_STACK                                          \
1977
         "subq $136,%%rsp\n\t"                                         \
1978
         "pushq 72(%%rax)\n\t"                                         \
1979
         "pushq 64(%%rax)\n\t"                                         \
1980
         "pushq 56(%%rax)\n\t"                                         \
1981
         "movq 48(%%rax), %%r9\n\t"                                    \
1982
         "movq 40(%%rax), %%r8\n\t"                                    \
1983
         "movq 32(%%rax), %%rcx\n\t"                                   \
1984
         "movq 24(%%rax), %%rdx\n\t"                                   \
1985
         "movq 16(%%rax), %%rsi\n\t"                                   \
1986
         "movq 8(%%rax), %%rdi\n\t"                                    \
1987
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
1988
         VALGRIND_CALL_NOREDIR_RAX                                     \
1989
         VALGRIND_RESTORE_STACK                                        \
1990
         VALGRIND_CFI_EPILOGUE                                         \
1991
         : /*out*/   "=a" (_res)                                       \
1992
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
1993
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
1994
      );                                                               \
1995
      lval = (__typeof__(lval)) _res;                                  \
1996
   } while (0)
1997
1998
#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
1999
                                  arg7,arg8,arg9,arg10)                \
2000
   do {                                                                \
2001
      volatile OrigFn        _orig = (orig);                           \
2002
      volatile unsigned long _argvec[11];                              \
2003
      volatile unsigned long _res;                                     \
2004
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
2005
      _argvec[1] = (unsigned long)(arg1);                              \
2006
      _argvec[2] = (unsigned long)(arg2);                              \
2007
      _argvec[3] = (unsigned long)(arg3);                              \
2008
      _argvec[4] = (unsigned long)(arg4);                              \
2009
      _argvec[5] = (unsigned long)(arg5);                              \
2010
      _argvec[6] = (unsigned long)(arg6);                              \
2011
      _argvec[7] = (unsigned long)(arg7);                              \
2012
      _argvec[8] = (unsigned long)(arg8);                              \
2013
      _argvec[9] = (unsigned long)(arg9);                              \
2014
      _argvec[10] = (unsigned long)(arg10);                            \
2015
      __asm__ volatile(                                                \
2016
         VALGRIND_CFI_PROLOGUE                                         \
2017
         VALGRIND_ALIGN_STACK                                          \
2018
         "subq $128,%%rsp\n\t"                                         \
2019
         "pushq 80(%%rax)\n\t"                                         \
2020
         "pushq 72(%%rax)\n\t"                                         \
2021
         "pushq 64(%%rax)\n\t"                                         \
2022
         "pushq 56(%%rax)\n\t"                                         \
2023
         "movq 48(%%rax), %%r9\n\t"                                    \
2024
         "movq 40(%%rax), %%r8\n\t"                                    \
2025
         "movq 32(%%rax), %%rcx\n\t"                                   \
2026
         "movq 24(%%rax), %%rdx\n\t"                                   \
2027
         "movq 16(%%rax), %%rsi\n\t"                                   \
2028
         "movq 8(%%rax), %%rdi\n\t"                                    \
2029
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2030
         VALGRIND_CALL_NOREDIR_RAX                                     \
2031
         VALGRIND_RESTORE_STACK                                        \
2032
         VALGRIND_CFI_EPILOGUE                                         \
2033
         : /*out*/   "=a" (_res)                                       \
2034
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2035
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2036
      );                                                               \
2037
      lval = (__typeof__(lval)) _res;                                  \
2038
   } while (0)
2039
2040
#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
2041
                                  arg7,arg8,arg9,arg10,arg11)          \
2042
   do {                                                                \
2043
      volatile OrigFn        _orig = (orig);                           \
2044
      volatile unsigned long _argvec[12];                              \
2045
      volatile unsigned long _res;                                     \
2046
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
2047
      _argvec[1] = (unsigned long)(arg1);                              \
2048
      _argvec[2] = (unsigned long)(arg2);                              \
2049
      _argvec[3] = (unsigned long)(arg3);                              \
2050
      _argvec[4] = (unsigned long)(arg4);                              \
2051
      _argvec[5] = (unsigned long)(arg5);                              \
2052
      _argvec[6] = (unsigned long)(arg6);                              \
2053
      _argvec[7] = (unsigned long)(arg7);                              \
2054
      _argvec[8] = (unsigned long)(arg8);                              \
2055
      _argvec[9] = (unsigned long)(arg9);                              \
2056
      _argvec[10] = (unsigned long)(arg10);                            \
2057
      _argvec[11] = (unsigned long)(arg11);                            \
2058
      __asm__ volatile(                                                \
2059
         VALGRIND_CFI_PROLOGUE                                         \
2060
         VALGRIND_ALIGN_STACK                                          \
2061
         "subq $136,%%rsp\n\t"                                         \
2062
         "pushq 88(%%rax)\n\t"                                         \
2063
         "pushq 80(%%rax)\n\t"                                         \
2064
         "pushq 72(%%rax)\n\t"                                         \
2065
         "pushq 64(%%rax)\n\t"                                         \
2066
         "pushq 56(%%rax)\n\t"                                         \
2067
         "movq 48(%%rax), %%r9\n\t"                                    \
2068
         "movq 40(%%rax), %%r8\n\t"                                    \
2069
         "movq 32(%%rax), %%rcx\n\t"                                   \
2070
         "movq 24(%%rax), %%rdx\n\t"                                   \
2071
         "movq 16(%%rax), %%rsi\n\t"                                   \
2072
         "movq 8(%%rax), %%rdi\n\t"                                    \
2073
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2074
         VALGRIND_CALL_NOREDIR_RAX                                     \
2075
         VALGRIND_RESTORE_STACK                                        \
2076
         VALGRIND_CFI_EPILOGUE                                         \
2077
         : /*out*/   "=a" (_res)                                       \
2078
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2079
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2080
      );                                                               \
2081
      lval = (__typeof__(lval)) _res;                                  \
2082
   } while (0)
2083
2084
#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,       \
2085
                                arg7,arg8,arg9,arg10,arg11,arg12)      \
2086
   do {                                                                \
2087
      volatile OrigFn        _orig = (orig);                           \
2088
      volatile unsigned long _argvec[13];                              \
2089
      volatile unsigned long _res;                                     \
2090
      _argvec[0] = (unsigned long)_orig.nraddr;                        \
2091
      _argvec[1] = (unsigned long)(arg1);                              \
2092
      _argvec[2] = (unsigned long)(arg2);                              \
2093
      _argvec[3] = (unsigned long)(arg3);                              \
2094
      _argvec[4] = (unsigned long)(arg4);                              \
2095
      _argvec[5] = (unsigned long)(arg5);                              \
2096
      _argvec[6] = (unsigned long)(arg6);                              \
2097
      _argvec[7] = (unsigned long)(arg7);                              \
2098
      _argvec[8] = (unsigned long)(arg8);                              \
2099
      _argvec[9] = (unsigned long)(arg9);                              \
2100
      _argvec[10] = (unsigned long)(arg10);                            \
2101
      _argvec[11] = (unsigned long)(arg11);                            \
2102
      _argvec[12] = (unsigned long)(arg12);                            \
2103
      __asm__ volatile(                                                \
2104
         VALGRIND_CFI_PROLOGUE                                         \
2105
         VALGRIND_ALIGN_STACK                                          \
2106
         "subq $128,%%rsp\n\t"                                         \
2107
         "pushq 96(%%rax)\n\t"                                         \
2108
         "pushq 88(%%rax)\n\t"                                         \
2109
         "pushq 80(%%rax)\n\t"                                         \
2110
         "pushq 72(%%rax)\n\t"                                         \
2111
         "pushq 64(%%rax)\n\t"                                         \
2112
         "pushq 56(%%rax)\n\t"                                         \
2113
         "movq 48(%%rax), %%r9\n\t"                                    \
2114
         "movq 40(%%rax), %%r8\n\t"                                    \
2115
         "movq 32(%%rax), %%rcx\n\t"                                   \
2116
         "movq 24(%%rax), %%rdx\n\t"                                   \
2117
         "movq 16(%%rax), %%rsi\n\t"                                   \
2118
         "movq 8(%%rax), %%rdi\n\t"                                    \
2119
         "movq (%%rax), %%rax\n\t"  /* target->%rax */                 \
2120
         VALGRIND_CALL_NOREDIR_RAX                                     \
2121
         VALGRIND_RESTORE_STACK                                        \
2122
         VALGRIND_CFI_EPILOGUE                                         \
2123
         : /*out*/   "=a" (_res)                                       \
2124
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER                 \
2125
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \
2126
      );                                                               \
2127
      lval = (__typeof__(lval)) _res;                                  \
2128
   } while (0)
2129
2130
#endif /* PLAT_amd64_linux || PLAT_amd64_darwin || PLAT_amd64_solaris */
2131
2132
/* ------------------------ ppc32-linux ------------------------ */
2133
2134
#if defined(PLAT_ppc32_linux)
2135
2136
/* This is useful for finding out about the on-stack stuff:
2137
2138
   extern int f9  ( int,int,int,int,int,int,int,int,int );
2139
   extern int f10 ( int,int,int,int,int,int,int,int,int,int );
2140
   extern int f11 ( int,int,int,int,int,int,int,int,int,int,int );
2141
   extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int );
2142
2143
   int g9 ( void ) {
2144
      return f9(11,22,33,44,55,66,77,88,99);
2145
   }
2146
   int g10 ( void ) {
2147
      return f10(11,22,33,44,55,66,77,88,99,110);
2148
   }
2149
   int g11 ( void ) {
2150
      return f11(11,22,33,44,55,66,77,88,99,110,121);
2151
   }
2152
   int g12 ( void ) {
2153
      return f12(11,22,33,44,55,66,77,88,99,110,121,132);
2154
   }
2155
*/
2156
2157
/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2158
2159
/* These regs are trashed by the hidden call. */
2160
#define __CALLER_SAVED_REGS                                       \
2161
   "lr", "ctr", "xer",                                            \
2162
   "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2163
   "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",   \
2164
   "r11", "r12", "r13"
2165
2166
/* Macros to save and align the stack before making a function
2167
   call and restore it afterwards as gcc may not keep the stack
2168
   pointer aligned if it doesn't realise calls are being made
2169
   to other functions. */
2170
2171
#define VALGRIND_ALIGN_STACK               \
2172
      "mr 28,1\n\t"                        \
2173
      "rlwinm 1,1,0,0,27\n\t"
2174
#define VALGRIND_RESTORE_STACK             \
2175
      "mr 1,28\n\t"
2176
2177
/* These CALL_FN_ macros assume that on ppc32-linux, 
2178
   sizeof(unsigned long) == 4. */
2179
2180
#define CALL_FN_W_v(lval, orig)                                   \
2181
   do {                                                           \
2182
      volatile OrigFn        _orig = (orig);                      \
2183
      volatile unsigned long _argvec[1];                          \
2184
      volatile unsigned long _res;                                \
2185
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2186
      __asm__ volatile(                                           \
2187
         VALGRIND_ALIGN_STACK                                     \
2188
         "mr 11,%1\n\t"                                           \
2189
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2190
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2191
         VALGRIND_RESTORE_STACK                                   \
2192
         "mr %0,3"                                                \
2193
         : /*out*/   "=r" (_res)                                  \
2194
         : /*in*/    "r" (&_argvec[0])                            \
2195
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2196
      );                                                          \
2197
      lval = (__typeof__(lval)) _res;                             \
2198
   } while (0)
2199
2200
#define CALL_FN_W_W(lval, orig, arg1)                             \
2201
   do {                                                           \
2202
      volatile OrigFn        _orig = (orig);                      \
2203
      volatile unsigned long _argvec[2];                          \
2204
      volatile unsigned long _res;                                \
2205
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2206
      _argvec[1] = (unsigned long)arg1;                           \
2207
      __asm__ volatile(                                           \
2208
         VALGRIND_ALIGN_STACK                                     \
2209
         "mr 11,%1\n\t"                                           \
2210
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2211
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2212
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2213
         VALGRIND_RESTORE_STACK                                   \
2214
         "mr %0,3"                                                \
2215
         : /*out*/   "=r" (_res)                                  \
2216
         : /*in*/    "r" (&_argvec[0])                            \
2217
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2218
      );                                                          \
2219
      lval = (__typeof__(lval)) _res;                             \
2220
   } while (0)
2221
2222
#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2223
   do {                                                           \
2224
      volatile OrigFn        _orig = (orig);                      \
2225
      volatile unsigned long _argvec[3];                          \
2226
      volatile unsigned long _res;                                \
2227
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2228
      _argvec[1] = (unsigned long)arg1;                           \
2229
      _argvec[2] = (unsigned long)arg2;                           \
2230
      __asm__ volatile(                                           \
2231
         VALGRIND_ALIGN_STACK                                     \
2232
         "mr 11,%1\n\t"                                           \
2233
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2234
         "lwz 4,8(11)\n\t"                                        \
2235
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2236
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2237
         VALGRIND_RESTORE_STACK                                   \
2238
         "mr %0,3"                                                \
2239
         : /*out*/   "=r" (_res)                                  \
2240
         : /*in*/    "r" (&_argvec[0])                            \
2241
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2242
      );                                                          \
2243
      lval = (__typeof__(lval)) _res;                             \
2244
   } while (0)
2245
2246
#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2247
   do {                                                           \
2248
      volatile OrigFn        _orig = (orig);                      \
2249
      volatile unsigned long _argvec[4];                          \
2250
      volatile unsigned long _res;                                \
2251
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2252
      _argvec[1] = (unsigned long)arg1;                           \
2253
      _argvec[2] = (unsigned long)arg2;                           \
2254
      _argvec[3] = (unsigned long)arg3;                           \
2255
      __asm__ volatile(                                           \
2256
         VALGRIND_ALIGN_STACK                                     \
2257
         "mr 11,%1\n\t"                                           \
2258
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2259
         "lwz 4,8(11)\n\t"                                        \
2260
         "lwz 5,12(11)\n\t"                                       \
2261
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2262
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2263
         VALGRIND_RESTORE_STACK                                   \
2264
         "mr %0,3"                                                \
2265
         : /*out*/   "=r" (_res)                                  \
2266
         : /*in*/    "r" (&_argvec[0])                            \
2267
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2268
      );                                                          \
2269
      lval = (__typeof__(lval)) _res;                             \
2270
   } while (0)
2271
2272
#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2273
   do {                                                           \
2274
      volatile OrigFn        _orig = (orig);                      \
2275
      volatile unsigned long _argvec[5];                          \
2276
      volatile unsigned long _res;                                \
2277
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2278
      _argvec[1] = (unsigned long)arg1;                           \
2279
      _argvec[2] = (unsigned long)arg2;                           \
2280
      _argvec[3] = (unsigned long)arg3;                           \
2281
      _argvec[4] = (unsigned long)arg4;                           \
2282
      __asm__ volatile(                                           \
2283
         VALGRIND_ALIGN_STACK                                     \
2284
         "mr 11,%1\n\t"                                           \
2285
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2286
         "lwz 4,8(11)\n\t"                                        \
2287
         "lwz 5,12(11)\n\t"                                       \
2288
         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2289
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2290
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2291
         VALGRIND_RESTORE_STACK                                   \
2292
         "mr %0,3"                                                \
2293
         : /*out*/   "=r" (_res)                                  \
2294
         : /*in*/    "r" (&_argvec[0])                            \
2295
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2296
      );                                                          \
2297
      lval = (__typeof__(lval)) _res;                             \
2298
   } while (0)
2299
2300
#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2301
   do {                                                           \
2302
      volatile OrigFn        _orig = (orig);                      \
2303
      volatile unsigned long _argvec[6];                          \
2304
      volatile unsigned long _res;                                \
2305
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2306
      _argvec[1] = (unsigned long)arg1;                           \
2307
      _argvec[2] = (unsigned long)arg2;                           \
2308
      _argvec[3] = (unsigned long)arg3;                           \
2309
      _argvec[4] = (unsigned long)arg4;                           \
2310
      _argvec[5] = (unsigned long)arg5;                           \
2311
      __asm__ volatile(                                           \
2312
         VALGRIND_ALIGN_STACK                                     \
2313
         "mr 11,%1\n\t"                                           \
2314
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2315
         "lwz 4,8(11)\n\t"                                        \
2316
         "lwz 5,12(11)\n\t"                                       \
2317
         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2318
         "lwz 7,20(11)\n\t"                                       \
2319
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2320
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2321
         VALGRIND_RESTORE_STACK                                   \
2322
         "mr %0,3"                                                \
2323
         : /*out*/   "=r" (_res)                                  \
2324
         : /*in*/    "r" (&_argvec[0])                            \
2325
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2326
      );                                                          \
2327
      lval = (__typeof__(lval)) _res;                             \
2328
   } while (0)
2329
2330
#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2331
   do {                                                           \
2332
      volatile OrigFn        _orig = (orig);                      \
2333
      volatile unsigned long _argvec[7];                          \
2334
      volatile unsigned long _res;                                \
2335
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2336
      _argvec[1] = (unsigned long)arg1;                           \
2337
      _argvec[2] = (unsigned long)arg2;                           \
2338
      _argvec[3] = (unsigned long)arg3;                           \
2339
      _argvec[4] = (unsigned long)arg4;                           \
2340
      _argvec[5] = (unsigned long)arg5;                           \
2341
      _argvec[6] = (unsigned long)arg6;                           \
2342
      __asm__ volatile(                                           \
2343
         VALGRIND_ALIGN_STACK                                     \
2344
         "mr 11,%1\n\t"                                           \
2345
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2346
         "lwz 4,8(11)\n\t"                                        \
2347
         "lwz 5,12(11)\n\t"                                       \
2348
         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2349
         "lwz 7,20(11)\n\t"                                       \
2350
         "lwz 8,24(11)\n\t"                                       \
2351
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2352
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2353
         VALGRIND_RESTORE_STACK                                   \
2354
         "mr %0,3"                                                \
2355
         : /*out*/   "=r" (_res)                                  \
2356
         : /*in*/    "r" (&_argvec[0])                            \
2357
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2358
      );                                                          \
2359
      lval = (__typeof__(lval)) _res;                             \
2360
   } while (0)
2361
2362
#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2363
                                 arg7)                            \
2364
   do {                                                           \
2365
      volatile OrigFn        _orig = (orig);                      \
2366
      volatile unsigned long _argvec[8];                          \
2367
      volatile unsigned long _res;                                \
2368
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2369
      _argvec[1] = (unsigned long)arg1;                           \
2370
      _argvec[2] = (unsigned long)arg2;                           \
2371
      _argvec[3] = (unsigned long)arg3;                           \
2372
      _argvec[4] = (unsigned long)arg4;                           \
2373
      _argvec[5] = (unsigned long)arg5;                           \
2374
      _argvec[6] = (unsigned long)arg6;                           \
2375
      _argvec[7] = (unsigned long)arg7;                           \
2376
      __asm__ volatile(                                           \
2377
         VALGRIND_ALIGN_STACK                                     \
2378
         "mr 11,%1\n\t"                                           \
2379
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2380
         "lwz 4,8(11)\n\t"                                        \
2381
         "lwz 5,12(11)\n\t"                                       \
2382
         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2383
         "lwz 7,20(11)\n\t"                                       \
2384
         "lwz 8,24(11)\n\t"                                       \
2385
         "lwz 9,28(11)\n\t"                                       \
2386
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2387
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2388
         VALGRIND_RESTORE_STACK                                   \
2389
         "mr %0,3"                                                \
2390
         : /*out*/   "=r" (_res)                                  \
2391
         : /*in*/    "r" (&_argvec[0])                            \
2392
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2393
      );                                                          \
2394
      lval = (__typeof__(lval)) _res;                             \
2395
   } while (0)
2396
2397
#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2398
                                 arg7,arg8)                       \
2399
   do {                                                           \
2400
      volatile OrigFn        _orig = (orig);                      \
2401
      volatile unsigned long _argvec[9];                          \
2402
      volatile unsigned long _res;                                \
2403
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2404
      _argvec[1] = (unsigned long)arg1;                           \
2405
      _argvec[2] = (unsigned long)arg2;                           \
2406
      _argvec[3] = (unsigned long)arg3;                           \
2407
      _argvec[4] = (unsigned long)arg4;                           \
2408
      _argvec[5] = (unsigned long)arg5;                           \
2409
      _argvec[6] = (unsigned long)arg6;                           \
2410
      _argvec[7] = (unsigned long)arg7;                           \
2411
      _argvec[8] = (unsigned long)arg8;                           \
2412
      __asm__ volatile(                                           \
2413
         VALGRIND_ALIGN_STACK                                     \
2414
         "mr 11,%1\n\t"                                           \
2415
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2416
         "lwz 4,8(11)\n\t"                                        \
2417
         "lwz 5,12(11)\n\t"                                       \
2418
         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2419
         "lwz 7,20(11)\n\t"                                       \
2420
         "lwz 8,24(11)\n\t"                                       \
2421
         "lwz 9,28(11)\n\t"                                       \
2422
         "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2423
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2424
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2425
         VALGRIND_RESTORE_STACK                                   \
2426
         "mr %0,3"                                                \
2427
         : /*out*/   "=r" (_res)                                  \
2428
         : /*in*/    "r" (&_argvec[0])                            \
2429
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2430
      );                                                          \
2431
      lval = (__typeof__(lval)) _res;                             \
2432
   } while (0)
2433
2434
#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2435
                                 arg7,arg8,arg9)                  \
2436
   do {                                                           \
2437
      volatile OrigFn        _orig = (orig);                      \
2438
      volatile unsigned long _argvec[10];                         \
2439
      volatile unsigned long _res;                                \
2440
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2441
      _argvec[1] = (unsigned long)arg1;                           \
2442
      _argvec[2] = (unsigned long)arg2;                           \
2443
      _argvec[3] = (unsigned long)arg3;                           \
2444
      _argvec[4] = (unsigned long)arg4;                           \
2445
      _argvec[5] = (unsigned long)arg5;                           \
2446
      _argvec[6] = (unsigned long)arg6;                           \
2447
      _argvec[7] = (unsigned long)arg7;                           \
2448
      _argvec[8] = (unsigned long)arg8;                           \
2449
      _argvec[9] = (unsigned long)arg9;                           \
2450
      __asm__ volatile(                                           \
2451
         VALGRIND_ALIGN_STACK                                     \
2452
         "mr 11,%1\n\t"                                           \
2453
         "addi 1,1,-16\n\t"                                       \
2454
         /* arg9 */                                               \
2455
         "lwz 3,36(11)\n\t"                                       \
2456
         "stw 3,8(1)\n\t"                                         \
2457
         /* args1-8 */                                            \
2458
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2459
         "lwz 4,8(11)\n\t"                                        \
2460
         "lwz 5,12(11)\n\t"                                       \
2461
         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2462
         "lwz 7,20(11)\n\t"                                       \
2463
         "lwz 8,24(11)\n\t"                                       \
2464
         "lwz 9,28(11)\n\t"                                       \
2465
         "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2466
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2467
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2468
         VALGRIND_RESTORE_STACK                                   \
2469
         "mr %0,3"                                                \
2470
         : /*out*/   "=r" (_res)                                  \
2471
         : /*in*/    "r" (&_argvec[0])                            \
2472
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2473
      );                                                          \
2474
      lval = (__typeof__(lval)) _res;                             \
2475
   } while (0)
2476
2477
#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2478
                                  arg7,arg8,arg9,arg10)           \
2479
   do {                                                           \
2480
      volatile OrigFn        _orig = (orig);                      \
2481
      volatile unsigned long _argvec[11];                         \
2482
      volatile unsigned long _res;                                \
2483
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2484
      _argvec[1] = (unsigned long)arg1;                           \
2485
      _argvec[2] = (unsigned long)arg2;                           \
2486
      _argvec[3] = (unsigned long)arg3;                           \
2487
      _argvec[4] = (unsigned long)arg4;                           \
2488
      _argvec[5] = (unsigned long)arg5;                           \
2489
      _argvec[6] = (unsigned long)arg6;                           \
2490
      _argvec[7] = (unsigned long)arg7;                           \
2491
      _argvec[8] = (unsigned long)arg8;                           \
2492
      _argvec[9] = (unsigned long)arg9;                           \
2493
      _argvec[10] = (unsigned long)arg10;                         \
2494
      __asm__ volatile(                                           \
2495
         VALGRIND_ALIGN_STACK                                     \
2496
         "mr 11,%1\n\t"                                           \
2497
         "addi 1,1,-16\n\t"                                       \
2498
         /* arg10 */                                              \
2499
         "lwz 3,40(11)\n\t"                                       \
2500
         "stw 3,12(1)\n\t"                                        \
2501
         /* arg9 */                                               \
2502
         "lwz 3,36(11)\n\t"                                       \
2503
         "stw 3,8(1)\n\t"                                         \
2504
         /* args1-8 */                                            \
2505
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2506
         "lwz 4,8(11)\n\t"                                        \
2507
         "lwz 5,12(11)\n\t"                                       \
2508
         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2509
         "lwz 7,20(11)\n\t"                                       \
2510
         "lwz 8,24(11)\n\t"                                       \
2511
         "lwz 9,28(11)\n\t"                                       \
2512
         "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2513
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2514
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2515
         VALGRIND_RESTORE_STACK                                   \
2516
         "mr %0,3"                                                \
2517
         : /*out*/   "=r" (_res)                                  \
2518
         : /*in*/    "r" (&_argvec[0])                            \
2519
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2520
      );                                                          \
2521
      lval = (__typeof__(lval)) _res;                             \
2522
   } while (0)
2523
2524
#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2525
                                  arg7,arg8,arg9,arg10,arg11)     \
2526
   do {                                                           \
2527
      volatile OrigFn        _orig = (orig);                      \
2528
      volatile unsigned long _argvec[12];                         \
2529
      volatile unsigned long _res;                                \
2530
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2531
      _argvec[1] = (unsigned long)arg1;                           \
2532
      _argvec[2] = (unsigned long)arg2;                           \
2533
      _argvec[3] = (unsigned long)arg3;                           \
2534
      _argvec[4] = (unsigned long)arg4;                           \
2535
      _argvec[5] = (unsigned long)arg5;                           \
2536
      _argvec[6] = (unsigned long)arg6;                           \
2537
      _argvec[7] = (unsigned long)arg7;                           \
2538
      _argvec[8] = (unsigned long)arg8;                           \
2539
      _argvec[9] = (unsigned long)arg9;                           \
2540
      _argvec[10] = (unsigned long)arg10;                         \
2541
      _argvec[11] = (unsigned long)arg11;                         \
2542
      __asm__ volatile(                                           \
2543
         VALGRIND_ALIGN_STACK                                     \
2544
         "mr 11,%1\n\t"                                           \
2545
         "addi 1,1,-32\n\t"                                       \
2546
         /* arg11 */                                              \
2547
         "lwz 3,44(11)\n\t"                                       \
2548
         "stw 3,16(1)\n\t"                                        \
2549
         /* arg10 */                                              \
2550
         "lwz 3,40(11)\n\t"                                       \
2551
         "stw 3,12(1)\n\t"                                        \
2552
         /* arg9 */                                               \
2553
         "lwz 3,36(11)\n\t"                                       \
2554
         "stw 3,8(1)\n\t"                                         \
2555
         /* args1-8 */                                            \
2556
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2557
         "lwz 4,8(11)\n\t"                                        \
2558
         "lwz 5,12(11)\n\t"                                       \
2559
         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2560
         "lwz 7,20(11)\n\t"                                       \
2561
         "lwz 8,24(11)\n\t"                                       \
2562
         "lwz 9,28(11)\n\t"                                       \
2563
         "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2564
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2565
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2566
         VALGRIND_RESTORE_STACK                                   \
2567
         "mr %0,3"                                                \
2568
         : /*out*/   "=r" (_res)                                  \
2569
         : /*in*/    "r" (&_argvec[0])                            \
2570
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2571
      );                                                          \
2572
      lval = (__typeof__(lval)) _res;                             \
2573
   } while (0)
2574
2575
#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
2576
                                arg7,arg8,arg9,arg10,arg11,arg12) \
2577
   do {                                                           \
2578
      volatile OrigFn        _orig = (orig);                      \
2579
      volatile unsigned long _argvec[13];                         \
2580
      volatile unsigned long _res;                                \
2581
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
2582
      _argvec[1] = (unsigned long)arg1;                           \
2583
      _argvec[2] = (unsigned long)arg2;                           \
2584
      _argvec[3] = (unsigned long)arg3;                           \
2585
      _argvec[4] = (unsigned long)arg4;                           \
2586
      _argvec[5] = (unsigned long)arg5;                           \
2587
      _argvec[6] = (unsigned long)arg6;                           \
2588
      _argvec[7] = (unsigned long)arg7;                           \
2589
      _argvec[8] = (unsigned long)arg8;                           \
2590
      _argvec[9] = (unsigned long)arg9;                           \
2591
      _argvec[10] = (unsigned long)arg10;                         \
2592
      _argvec[11] = (unsigned long)arg11;                         \
2593
      _argvec[12] = (unsigned long)arg12;                         \
2594
      __asm__ volatile(                                           \
2595
         VALGRIND_ALIGN_STACK                                     \
2596
         "mr 11,%1\n\t"                                           \
2597
         "addi 1,1,-32\n\t"                                       \
2598
         /* arg12 */                                              \
2599
         "lwz 3,48(11)\n\t"                                       \
2600
         "stw 3,20(1)\n\t"                                        \
2601
         /* arg11 */                                              \
2602
         "lwz 3,44(11)\n\t"                                       \
2603
         "stw 3,16(1)\n\t"                                        \
2604
         /* arg10 */                                              \
2605
         "lwz 3,40(11)\n\t"                                       \
2606
         "stw 3,12(1)\n\t"                                        \
2607
         /* arg9 */                                               \
2608
         "lwz 3,36(11)\n\t"                                       \
2609
         "stw 3,8(1)\n\t"                                         \
2610
         /* args1-8 */                                            \
2611
         "lwz 3,4(11)\n\t"   /* arg1->r3 */                       \
2612
         "lwz 4,8(11)\n\t"                                        \
2613
         "lwz 5,12(11)\n\t"                                       \
2614
         "lwz 6,16(11)\n\t"  /* arg4->r6 */                       \
2615
         "lwz 7,20(11)\n\t"                                       \
2616
         "lwz 8,24(11)\n\t"                                       \
2617
         "lwz 9,28(11)\n\t"                                       \
2618
         "lwz 10,32(11)\n\t" /* arg8->r10 */                      \
2619
         "lwz 11,0(11)\n\t"  /* target->r11 */                    \
2620
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2621
         VALGRIND_RESTORE_STACK                                   \
2622
         "mr %0,3"                                                \
2623
         : /*out*/   "=r" (_res)                                  \
2624
         : /*in*/    "r" (&_argvec[0])                            \
2625
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2626
      );                                                          \
2627
      lval = (__typeof__(lval)) _res;                             \
2628
   } while (0)
2629
2630
#endif /* PLAT_ppc32_linux */
2631
2632
/* ------------------------ ppc64-linux ------------------------ */
2633
2634
#if defined(PLAT_ppc64be_linux)
2635
2636
/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
2637
2638
/* These regs are trashed by the hidden call. */
2639
#define __CALLER_SAVED_REGS                                       \
2640
   "lr", "ctr", "xer",                                            \
2641
   "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
2642
   "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",         \
2643
   "r11", "r12", "r13"
2644
2645
/* Macros to save and align the stack before making a function
2646
   call and restore it afterwards as gcc may not keep the stack
2647
   pointer aligned if it doesn't realise calls are being made
2648
   to other functions. */
2649
2650
#define VALGRIND_ALIGN_STACK               \
2651
      "mr 28,1\n\t"                        \
2652
      "rldicr 1,1,0,59\n\t"
2653
#define VALGRIND_RESTORE_STACK             \
2654
      "mr 1,28\n\t"
2655
2656
/* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
2657
   long) == 8. */
2658
2659
#define CALL_FN_W_v(lval, orig)                                   \
2660
   do {                                                           \
2661
      volatile OrigFn        _orig = (orig);                      \
2662
      volatile unsigned long _argvec[3+0];                        \
2663
      volatile unsigned long _res;                                \
2664
      /* _argvec[0] holds current r2 across the call */           \
2665
      _argvec[1] = (unsigned long)_orig.r2;                       \
2666
      _argvec[2] = (unsigned long)_orig.nraddr;                   \
2667
      __asm__ volatile(                                           \
2668
         VALGRIND_ALIGN_STACK                                     \
2669
         "mr 11,%1\n\t"                                           \
2670
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
2671
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2672
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2673
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2674
         "mr 11,%1\n\t"                                           \
2675
         "mr %0,3\n\t"                                            \
2676
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2677
         VALGRIND_RESTORE_STACK                                   \
2678
         : /*out*/   "=r" (_res)                                  \
2679
         : /*in*/    "r" (&_argvec[2])                            \
2680
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2681
      );                                                          \
2682
      lval = (__typeof__(lval)) _res;                             \
2683
   } while (0)
2684
2685
#define CALL_FN_W_W(lval, orig, arg1)                             \
2686
   do {                                                           \
2687
      volatile OrigFn        _orig = (orig);                      \
2688
      volatile unsigned long _argvec[3+1];                        \
2689
      volatile unsigned long _res;                                \
2690
      /* _argvec[0] holds current r2 across the call */           \
2691
      _argvec[1]   = (unsigned long)_orig.r2;                     \
2692
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2693
      _argvec[2+1] = (unsigned long)arg1;                         \
2694
      __asm__ volatile(                                           \
2695
         VALGRIND_ALIGN_STACK                                     \
2696
         "mr 11,%1\n\t"                                           \
2697
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
2698
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2699
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2700
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2701
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2702
         "mr 11,%1\n\t"                                           \
2703
         "mr %0,3\n\t"                                            \
2704
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2705
         VALGRIND_RESTORE_STACK                                   \
2706
         : /*out*/   "=r" (_res)                                  \
2707
         : /*in*/    "r" (&_argvec[2])                            \
2708
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2709
      );                                                          \
2710
      lval = (__typeof__(lval)) _res;                             \
2711
   } while (0)
2712
2713
#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
2714
   do {                                                           \
2715
      volatile OrigFn        _orig = (orig);                      \
2716
      volatile unsigned long _argvec[3+2];                        \
2717
      volatile unsigned long _res;                                \
2718
      /* _argvec[0] holds current r2 across the call */           \
2719
      _argvec[1]   = (unsigned long)_orig.r2;                     \
2720
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2721
      _argvec[2+1] = (unsigned long)arg1;                         \
2722
      _argvec[2+2] = (unsigned long)arg2;                         \
2723
      __asm__ volatile(                                           \
2724
         VALGRIND_ALIGN_STACK                                     \
2725
         "mr 11,%1\n\t"                                           \
2726
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
2727
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2728
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2729
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2730
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2731
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2732
         "mr 11,%1\n\t"                                           \
2733
         "mr %0,3\n\t"                                            \
2734
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2735
         VALGRIND_RESTORE_STACK                                   \
2736
         : /*out*/   "=r" (_res)                                  \
2737
         : /*in*/    "r" (&_argvec[2])                            \
2738
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2739
      );                                                          \
2740
      lval = (__typeof__(lval)) _res;                             \
2741
   } while (0)
2742
2743
#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
2744
   do {                                                           \
2745
      volatile OrigFn        _orig = (orig);                      \
2746
      volatile unsigned long _argvec[3+3];                        \
2747
      volatile unsigned long _res;                                \
2748
      /* _argvec[0] holds current r2 across the call */           \
2749
      _argvec[1]   = (unsigned long)_orig.r2;                     \
2750
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2751
      _argvec[2+1] = (unsigned long)arg1;                         \
2752
      _argvec[2+2] = (unsigned long)arg2;                         \
2753
      _argvec[2+3] = (unsigned long)arg3;                         \
2754
      __asm__ volatile(                                           \
2755
         VALGRIND_ALIGN_STACK                                     \
2756
         "mr 11,%1\n\t"                                           \
2757
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
2758
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2759
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2760
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2761
         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2762
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2763
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2764
         "mr 11,%1\n\t"                                           \
2765
         "mr %0,3\n\t"                                            \
2766
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2767
         VALGRIND_RESTORE_STACK                                   \
2768
         : /*out*/   "=r" (_res)                                  \
2769
         : /*in*/    "r" (&_argvec[2])                            \
2770
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2771
      );                                                          \
2772
      lval = (__typeof__(lval)) _res;                             \
2773
   } while (0)
2774
2775
#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
2776
   do {                                                           \
2777
      volatile OrigFn        _orig = (orig);                      \
2778
      volatile unsigned long _argvec[3+4];                        \
2779
      volatile unsigned long _res;                                \
2780
      /* _argvec[0] holds current r2 across the call */           \
2781
      _argvec[1]   = (unsigned long)_orig.r2;                     \
2782
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2783
      _argvec[2+1] = (unsigned long)arg1;                         \
2784
      _argvec[2+2] = (unsigned long)arg2;                         \
2785
      _argvec[2+3] = (unsigned long)arg3;                         \
2786
      _argvec[2+4] = (unsigned long)arg4;                         \
2787
      __asm__ volatile(                                           \
2788
         VALGRIND_ALIGN_STACK                                     \
2789
         "mr 11,%1\n\t"                                           \
2790
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
2791
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2792
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2793
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2794
         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2795
         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2796
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2797
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2798
         "mr 11,%1\n\t"                                           \
2799
         "mr %0,3\n\t"                                            \
2800
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2801
         VALGRIND_RESTORE_STACK                                   \
2802
         : /*out*/   "=r" (_res)                                  \
2803
         : /*in*/    "r" (&_argvec[2])                            \
2804
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2805
      );                                                          \
2806
      lval = (__typeof__(lval)) _res;                             \
2807
   } while (0)
2808
2809
#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
2810
   do {                                                           \
2811
      volatile OrigFn        _orig = (orig);                      \
2812
      volatile unsigned long _argvec[3+5];                        \
2813
      volatile unsigned long _res;                                \
2814
      /* _argvec[0] holds current r2 across the call */           \
2815
      _argvec[1]   = (unsigned long)_orig.r2;                     \
2816
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2817
      _argvec[2+1] = (unsigned long)arg1;                         \
2818
      _argvec[2+2] = (unsigned long)arg2;                         \
2819
      _argvec[2+3] = (unsigned long)arg3;                         \
2820
      _argvec[2+4] = (unsigned long)arg4;                         \
2821
      _argvec[2+5] = (unsigned long)arg5;                         \
2822
      __asm__ volatile(                                           \
2823
         VALGRIND_ALIGN_STACK                                     \
2824
         "mr 11,%1\n\t"                                           \
2825
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
2826
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2827
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2828
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2829
         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2830
         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2831
         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2832
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2833
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2834
         "mr 11,%1\n\t"                                           \
2835
         "mr %0,3\n\t"                                            \
2836
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2837
         VALGRIND_RESTORE_STACK                                   \
2838
         : /*out*/   "=r" (_res)                                  \
2839
         : /*in*/    "r" (&_argvec[2])                            \
2840
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2841
      );                                                          \
2842
      lval = (__typeof__(lval)) _res;                             \
2843
   } while (0)
2844
2845
#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
2846
   do {                                                           \
2847
      volatile OrigFn        _orig = (orig);                      \
2848
      volatile unsigned long _argvec[3+6];                        \
2849
      volatile unsigned long _res;                                \
2850
      /* _argvec[0] holds current r2 across the call */           \
2851
      _argvec[1]   = (unsigned long)_orig.r2;                     \
2852
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2853
      _argvec[2+1] = (unsigned long)arg1;                         \
2854
      _argvec[2+2] = (unsigned long)arg2;                         \
2855
      _argvec[2+3] = (unsigned long)arg3;                         \
2856
      _argvec[2+4] = (unsigned long)arg4;                         \
2857
      _argvec[2+5] = (unsigned long)arg5;                         \
2858
      _argvec[2+6] = (unsigned long)arg6;                         \
2859
      __asm__ volatile(                                           \
2860
         VALGRIND_ALIGN_STACK                                     \
2861
         "mr 11,%1\n\t"                                           \
2862
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
2863
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2864
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2865
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2866
         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2867
         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2868
         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2869
         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2870
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2871
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2872
         "mr 11,%1\n\t"                                           \
2873
         "mr %0,3\n\t"                                            \
2874
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2875
         VALGRIND_RESTORE_STACK                                   \
2876
         : /*out*/   "=r" (_res)                                  \
2877
         : /*in*/    "r" (&_argvec[2])                            \
2878
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2879
      );                                                          \
2880
      lval = (__typeof__(lval)) _res;                             \
2881
   } while (0)
2882
2883
#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2884
                                 arg7)                            \
2885
   do {                                                           \
2886
      volatile OrigFn        _orig = (orig);                      \
2887
      volatile unsigned long _argvec[3+7];                        \
2888
      volatile unsigned long _res;                                \
2889
      /* _argvec[0] holds current r2 across the call */           \
2890
      _argvec[1]   = (unsigned long)_orig.r2;                     \
2891
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2892
      _argvec[2+1] = (unsigned long)arg1;                         \
2893
      _argvec[2+2] = (unsigned long)arg2;                         \
2894
      _argvec[2+3] = (unsigned long)arg3;                         \
2895
      _argvec[2+4] = (unsigned long)arg4;                         \
2896
      _argvec[2+5] = (unsigned long)arg5;                         \
2897
      _argvec[2+6] = (unsigned long)arg6;                         \
2898
      _argvec[2+7] = (unsigned long)arg7;                         \
2899
      __asm__ volatile(                                           \
2900
         VALGRIND_ALIGN_STACK                                     \
2901
         "mr 11,%1\n\t"                                           \
2902
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
2903
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2904
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2905
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2906
         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2907
         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2908
         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2909
         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2910
         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2911
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2912
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2913
         "mr 11,%1\n\t"                                           \
2914
         "mr %0,3\n\t"                                            \
2915
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2916
         VALGRIND_RESTORE_STACK                                   \
2917
         : /*out*/   "=r" (_res)                                  \
2918
         : /*in*/    "r" (&_argvec[2])                            \
2919
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2920
      );                                                          \
2921
      lval = (__typeof__(lval)) _res;                             \
2922
   } while (0)
2923
2924
#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2925
                                 arg7,arg8)                       \
2926
   do {                                                           \
2927
      volatile OrigFn        _orig = (orig);                      \
2928
      volatile unsigned long _argvec[3+8];                        \
2929
      volatile unsigned long _res;                                \
2930
      /* _argvec[0] holds current r2 across the call */           \
2931
      _argvec[1]   = (unsigned long)_orig.r2;                     \
2932
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2933
      _argvec[2+1] = (unsigned long)arg1;                         \
2934
      _argvec[2+2] = (unsigned long)arg2;                         \
2935
      _argvec[2+3] = (unsigned long)arg3;                         \
2936
      _argvec[2+4] = (unsigned long)arg4;                         \
2937
      _argvec[2+5] = (unsigned long)arg5;                         \
2938
      _argvec[2+6] = (unsigned long)arg6;                         \
2939
      _argvec[2+7] = (unsigned long)arg7;                         \
2940
      _argvec[2+8] = (unsigned long)arg8;                         \
2941
      __asm__ volatile(                                           \
2942
         VALGRIND_ALIGN_STACK                                     \
2943
         "mr 11,%1\n\t"                                           \
2944
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
2945
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2946
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2947
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2948
         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2949
         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2950
         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
2951
         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
2952
         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
2953
         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
2954
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
2955
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
2956
         "mr 11,%1\n\t"                                           \
2957
         "mr %0,3\n\t"                                            \
2958
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
2959
         VALGRIND_RESTORE_STACK                                   \
2960
         : /*out*/   "=r" (_res)                                  \
2961
         : /*in*/    "r" (&_argvec[2])                            \
2962
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
2963
      );                                                          \
2964
      lval = (__typeof__(lval)) _res;                             \
2965
   } while (0)
2966
2967
#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
2968
                                 arg7,arg8,arg9)                  \
2969
   do {                                                           \
2970
      volatile OrigFn        _orig = (orig);                      \
2971
      volatile unsigned long _argvec[3+9];                        \
2972
      volatile unsigned long _res;                                \
2973
      /* _argvec[0] holds current r2 across the call */           \
2974
      _argvec[1]   = (unsigned long)_orig.r2;                     \
2975
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
2976
      _argvec[2+1] = (unsigned long)arg1;                         \
2977
      _argvec[2+2] = (unsigned long)arg2;                         \
2978
      _argvec[2+3] = (unsigned long)arg3;                         \
2979
      _argvec[2+4] = (unsigned long)arg4;                         \
2980
      _argvec[2+5] = (unsigned long)arg5;                         \
2981
      _argvec[2+6] = (unsigned long)arg6;                         \
2982
      _argvec[2+7] = (unsigned long)arg7;                         \
2983
      _argvec[2+8] = (unsigned long)arg8;                         \
2984
      _argvec[2+9] = (unsigned long)arg9;                         \
2985
      __asm__ volatile(                                           \
2986
         VALGRIND_ALIGN_STACK                                     \
2987
         "mr 11,%1\n\t"                                           \
2988
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
2989
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
2990
         "addi 1,1,-128\n\t"  /* expand stack frame */            \
2991
         /* arg9 */                                               \
2992
         "ld  3,72(11)\n\t"                                       \
2993
         "std 3,112(1)\n\t"                                       \
2994
         /* args1-8 */                                            \
2995
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
2996
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
2997
         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
2998
         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
2999
         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3000
         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3001
         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3002
         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3003
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3004
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3005
         "mr 11,%1\n\t"                                           \
3006
         "mr %0,3\n\t"                                            \
3007
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3008
         VALGRIND_RESTORE_STACK                                   \
3009
         : /*out*/   "=r" (_res)                                  \
3010
         : /*in*/    "r" (&_argvec[2])                            \
3011
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3012
      );                                                          \
3013
      lval = (__typeof__(lval)) _res;                             \
3014
   } while (0)
3015
3016
#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3017
                                  arg7,arg8,arg9,arg10)           \
3018
   do {                                                           \
3019
      volatile OrigFn        _orig = (orig);                      \
3020
      volatile unsigned long _argvec[3+10];                       \
3021
      volatile unsigned long _res;                                \
3022
      /* _argvec[0] holds current r2 across the call */           \
3023
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3024
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3025
      _argvec[2+1] = (unsigned long)arg1;                         \
3026
      _argvec[2+2] = (unsigned long)arg2;                         \
3027
      _argvec[2+3] = (unsigned long)arg3;                         \
3028
      _argvec[2+4] = (unsigned long)arg4;                         \
3029
      _argvec[2+5] = (unsigned long)arg5;                         \
3030
      _argvec[2+6] = (unsigned long)arg6;                         \
3031
      _argvec[2+7] = (unsigned long)arg7;                         \
3032
      _argvec[2+8] = (unsigned long)arg8;                         \
3033
      _argvec[2+9] = (unsigned long)arg9;                         \
3034
      _argvec[2+10] = (unsigned long)arg10;                       \
3035
      __asm__ volatile(                                           \
3036
         VALGRIND_ALIGN_STACK                                     \
3037
         "mr 11,%1\n\t"                                           \
3038
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
3039
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3040
         "addi 1,1,-128\n\t"  /* expand stack frame */            \
3041
         /* arg10 */                                              \
3042
         "ld  3,80(11)\n\t"                                       \
3043
         "std 3,120(1)\n\t"                                       \
3044
         /* arg9 */                                               \
3045
         "ld  3,72(11)\n\t"                                       \
3046
         "std 3,112(1)\n\t"                                       \
3047
         /* args1-8 */                                            \
3048
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3049
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3050
         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3051
         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3052
         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3053
         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3054
         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3055
         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3056
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3057
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3058
         "mr 11,%1\n\t"                                           \
3059
         "mr %0,3\n\t"                                            \
3060
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3061
         VALGRIND_RESTORE_STACK                                   \
3062
         : /*out*/   "=r" (_res)                                  \
3063
         : /*in*/    "r" (&_argvec[2])                            \
3064
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3065
      );                                                          \
3066
      lval = (__typeof__(lval)) _res;                             \
3067
   } while (0)
3068
3069
#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3070
                                  arg7,arg8,arg9,arg10,arg11)     \
3071
   do {                                                           \
3072
      volatile OrigFn        _orig = (orig);                      \
3073
      volatile unsigned long _argvec[3+11];                       \
3074
      volatile unsigned long _res;                                \
3075
      /* _argvec[0] holds current r2 across the call */           \
3076
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3077
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3078
      _argvec[2+1] = (unsigned long)arg1;                         \
3079
      _argvec[2+2] = (unsigned long)arg2;                         \
3080
      _argvec[2+3] = (unsigned long)arg3;                         \
3081
      _argvec[2+4] = (unsigned long)arg4;                         \
3082
      _argvec[2+5] = (unsigned long)arg5;                         \
3083
      _argvec[2+6] = (unsigned long)arg6;                         \
3084
      _argvec[2+7] = (unsigned long)arg7;                         \
3085
      _argvec[2+8] = (unsigned long)arg8;                         \
3086
      _argvec[2+9] = (unsigned long)arg9;                         \
3087
      _argvec[2+10] = (unsigned long)arg10;                       \
3088
      _argvec[2+11] = (unsigned long)arg11;                       \
3089
      __asm__ volatile(                                           \
3090
         VALGRIND_ALIGN_STACK                                     \
3091
         "mr 11,%1\n\t"                                           \
3092
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
3093
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3094
         "addi 1,1,-144\n\t"  /* expand stack frame */            \
3095
         /* arg11 */                                              \
3096
         "ld  3,88(11)\n\t"                                       \
3097
         "std 3,128(1)\n\t"                                       \
3098
         /* arg10 */                                              \
3099
         "ld  3,80(11)\n\t"                                       \
3100
         "std 3,120(1)\n\t"                                       \
3101
         /* arg9 */                                               \
3102
         "ld  3,72(11)\n\t"                                       \
3103
         "std 3,112(1)\n\t"                                       \
3104
         /* args1-8 */                                            \
3105
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3106
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3107
         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3108
         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3109
         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3110
         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3111
         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3112
         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3113
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3114
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3115
         "mr 11,%1\n\t"                                           \
3116
         "mr %0,3\n\t"                                            \
3117
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3118
         VALGRIND_RESTORE_STACK                                   \
3119
         : /*out*/   "=r" (_res)                                  \
3120
         : /*in*/    "r" (&_argvec[2])                            \
3121
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3122
      );                                                          \
3123
      lval = (__typeof__(lval)) _res;                             \
3124
   } while (0)
3125
3126
#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3127
                                arg7,arg8,arg9,arg10,arg11,arg12) \
3128
   do {                                                           \
3129
      volatile OrigFn        _orig = (orig);                      \
3130
      volatile unsigned long _argvec[3+12];                       \
3131
      volatile unsigned long _res;                                \
3132
      /* _argvec[0] holds current r2 across the call */           \
3133
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3134
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3135
      _argvec[2+1] = (unsigned long)arg1;                         \
3136
      _argvec[2+2] = (unsigned long)arg2;                         \
3137
      _argvec[2+3] = (unsigned long)arg3;                         \
3138
      _argvec[2+4] = (unsigned long)arg4;                         \
3139
      _argvec[2+5] = (unsigned long)arg5;                         \
3140
      _argvec[2+6] = (unsigned long)arg6;                         \
3141
      _argvec[2+7] = (unsigned long)arg7;                         \
3142
      _argvec[2+8] = (unsigned long)arg8;                         \
3143
      _argvec[2+9] = (unsigned long)arg9;                         \
3144
      _argvec[2+10] = (unsigned long)arg10;                       \
3145
      _argvec[2+11] = (unsigned long)arg11;                       \
3146
      _argvec[2+12] = (unsigned long)arg12;                       \
3147
      __asm__ volatile(                                           \
3148
         VALGRIND_ALIGN_STACK                                     \
3149
         "mr 11,%1\n\t"                                           \
3150
         "std 2,-16(11)\n\t"  /* save tocptr */                   \
3151
         "ld   2,-8(11)\n\t"  /* use nraddr's tocptr */           \
3152
         "addi 1,1,-144\n\t"  /* expand stack frame */            \
3153
         /* arg12 */                                              \
3154
         "ld  3,96(11)\n\t"                                       \
3155
         "std 3,136(1)\n\t"                                       \
3156
         /* arg11 */                                              \
3157
         "ld  3,88(11)\n\t"                                       \
3158
         "std 3,128(1)\n\t"                                       \
3159
         /* arg10 */                                              \
3160
         "ld  3,80(11)\n\t"                                       \
3161
         "std 3,120(1)\n\t"                                       \
3162
         /* arg9 */                                               \
3163
         "ld  3,72(11)\n\t"                                       \
3164
         "std 3,112(1)\n\t"                                       \
3165
         /* args1-8 */                                            \
3166
         "ld   3, 8(11)\n\t"  /* arg1->r3 */                      \
3167
         "ld   4, 16(11)\n\t" /* arg2->r4 */                      \
3168
         "ld   5, 24(11)\n\t" /* arg3->r5 */                      \
3169
         "ld   6, 32(11)\n\t" /* arg4->r6 */                      \
3170
         "ld   7, 40(11)\n\t" /* arg5->r7 */                      \
3171
         "ld   8, 48(11)\n\t" /* arg6->r8 */                      \
3172
         "ld   9, 56(11)\n\t" /* arg7->r9 */                      \
3173
         "ld  10, 64(11)\n\t" /* arg8->r10 */                     \
3174
         "ld  11, 0(11)\n\t"  /* target->r11 */                   \
3175
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11                  \
3176
         "mr 11,%1\n\t"                                           \
3177
         "mr %0,3\n\t"                                            \
3178
         "ld 2,-16(11)\n\t" /* restore tocptr */                  \
3179
         VALGRIND_RESTORE_STACK                                   \
3180
         : /*out*/   "=r" (_res)                                  \
3181
         : /*in*/    "r" (&_argvec[2])                            \
3182
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3183
      );                                                          \
3184
      lval = (__typeof__(lval)) _res;                             \
3185
   } while (0)
3186
3187
#endif /* PLAT_ppc64be_linux */
3188
3189
/* ------------------------- ppc64le-linux ----------------------- */
3190
#if defined(PLAT_ppc64le_linux)
3191
3192
/* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */
3193
3194
/* These regs are trashed by the hidden call. */
3195
#define __CALLER_SAVED_REGS                                       \
3196
   "lr", "ctr", "xer",                                            \
3197
   "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7",        \
3198
   "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10",         \
3199
   "r11", "r12", "r13"
3200
3201
/* Macros to save and align the stack before making a function
3202
   call and restore it afterwards as gcc may not keep the stack
3203
   pointer aligned if it doesn't realise calls are being made
3204
   to other functions. */
3205
3206
#define VALGRIND_ALIGN_STACK               \
3207
      "mr 28,1\n\t"                        \
3208
      "rldicr 1,1,0,59\n\t"
3209
#define VALGRIND_RESTORE_STACK             \
3210
      "mr 1,28\n\t"
3211
3212
/* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned
3213
   long) == 8. */
3214
3215
#define CALL_FN_W_v(lval, orig)                                   \
3216
   do {                                                           \
3217
      volatile OrigFn        _orig = (orig);                      \
3218
      volatile unsigned long _argvec[3+0];                        \
3219
      volatile unsigned long _res;                                \
3220
      /* _argvec[0] holds current r2 across the call */           \
3221
      _argvec[1] = (unsigned long)_orig.r2;                       \
3222
      _argvec[2] = (unsigned long)_orig.nraddr;                   \
3223
      __asm__ volatile(                                           \
3224
         VALGRIND_ALIGN_STACK                                     \
3225
         "mr 12,%1\n\t"                                           \
3226
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3227
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3228
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3229
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3230
         "mr 12,%1\n\t"                                           \
3231
         "mr %0,3\n\t"                                            \
3232
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3233
         VALGRIND_RESTORE_STACK                                   \
3234
         : /*out*/   "=r" (_res)                                  \
3235
         : /*in*/    "r" (&_argvec[2])                            \
3236
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3237
      );                                                          \
3238
      lval = (__typeof__(lval)) _res;                             \
3239
   } while (0)
3240
3241
#define CALL_FN_W_W(lval, orig, arg1)                             \
3242
   do {                                                           \
3243
      volatile OrigFn        _orig = (orig);                      \
3244
      volatile unsigned long _argvec[3+1];                        \
3245
      volatile unsigned long _res;                                \
3246
      /* _argvec[0] holds current r2 across the call */           \
3247
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3248
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3249
      _argvec[2+1] = (unsigned long)arg1;                         \
3250
      __asm__ volatile(                                           \
3251
         VALGRIND_ALIGN_STACK                                     \
3252
         "mr 12,%1\n\t"                                           \
3253
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3254
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3255
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3256
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3257
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3258
         "mr 12,%1\n\t"                                           \
3259
         "mr %0,3\n\t"                                            \
3260
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3261
         VALGRIND_RESTORE_STACK                                   \
3262
         : /*out*/   "=r" (_res)                                  \
3263
         : /*in*/    "r" (&_argvec[2])                            \
3264
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3265
      );                                                          \
3266
      lval = (__typeof__(lval)) _res;                             \
3267
   } while (0)
3268
3269
#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3270
   do {                                                           \
3271
      volatile OrigFn        _orig = (orig);                      \
3272
      volatile unsigned long _argvec[3+2];                        \
3273
      volatile unsigned long _res;                                \
3274
      /* _argvec[0] holds current r2 across the call */           \
3275
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3276
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3277
      _argvec[2+1] = (unsigned long)arg1;                         \
3278
      _argvec[2+2] = (unsigned long)arg2;                         \
3279
      __asm__ volatile(                                           \
3280
         VALGRIND_ALIGN_STACK                                     \
3281
         "mr 12,%1\n\t"                                           \
3282
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3283
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3284
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3285
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3286
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3287
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3288
         "mr 12,%1\n\t"                                           \
3289
         "mr %0,3\n\t"                                            \
3290
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3291
         VALGRIND_RESTORE_STACK                                   \
3292
         : /*out*/   "=r" (_res)                                  \
3293
         : /*in*/    "r" (&_argvec[2])                            \
3294
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3295
      );                                                          \
3296
      lval = (__typeof__(lval)) _res;                             \
3297
   } while (0)
3298
3299
#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3300
   do {                                                           \
3301
      volatile OrigFn        _orig = (orig);                      \
3302
      volatile unsigned long _argvec[3+3];                        \
3303
      volatile unsigned long _res;                                \
3304
      /* _argvec[0] holds current r2 across the call */           \
3305
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3306
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3307
      _argvec[2+1] = (unsigned long)arg1;                         \
3308
      _argvec[2+2] = (unsigned long)arg2;                         \
3309
      _argvec[2+3] = (unsigned long)arg3;                         \
3310
      __asm__ volatile(                                           \
3311
         VALGRIND_ALIGN_STACK                                     \
3312
         "mr 12,%1\n\t"                                           \
3313
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3314
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3315
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3316
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3317
         "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3318
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3319
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3320
         "mr 12,%1\n\t"                                           \
3321
         "mr %0,3\n\t"                                            \
3322
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3323
         VALGRIND_RESTORE_STACK                                   \
3324
         : /*out*/   "=r" (_res)                                  \
3325
         : /*in*/    "r" (&_argvec[2])                            \
3326
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3327
      );                                                          \
3328
      lval = (__typeof__(lval)) _res;                             \
3329
   } while (0)
3330
3331
#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3332
   do {                                                           \
3333
      volatile OrigFn        _orig = (orig);                      \
3334
      volatile unsigned long _argvec[3+4];                        \
3335
      volatile unsigned long _res;                                \
3336
      /* _argvec[0] holds current r2 across the call */           \
3337
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3338
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3339
      _argvec[2+1] = (unsigned long)arg1;                         \
3340
      _argvec[2+2] = (unsigned long)arg2;                         \
3341
      _argvec[2+3] = (unsigned long)arg3;                         \
3342
      _argvec[2+4] = (unsigned long)arg4;                         \
3343
      __asm__ volatile(                                           \
3344
         VALGRIND_ALIGN_STACK                                     \
3345
         "mr 12,%1\n\t"                                           \
3346
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3347
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3348
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3349
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3350
         "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3351
         "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3352
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3353
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3354
         "mr 12,%1\n\t"                                           \
3355
         "mr %0,3\n\t"                                            \
3356
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3357
         VALGRIND_RESTORE_STACK                                   \
3358
         : /*out*/   "=r" (_res)                                  \
3359
         : /*in*/    "r" (&_argvec[2])                            \
3360
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3361
      );                                                          \
3362
      lval = (__typeof__(lval)) _res;                             \
3363
   } while (0)
3364
3365
#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3366
   do {                                                           \
3367
      volatile OrigFn        _orig = (orig);                      \
3368
      volatile unsigned long _argvec[3+5];                        \
3369
      volatile unsigned long _res;                                \
3370
      /* _argvec[0] holds current r2 across the call */           \
3371
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3372
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3373
      _argvec[2+1] = (unsigned long)arg1;                         \
3374
      _argvec[2+2] = (unsigned long)arg2;                         \
3375
      _argvec[2+3] = (unsigned long)arg3;                         \
3376
      _argvec[2+4] = (unsigned long)arg4;                         \
3377
      _argvec[2+5] = (unsigned long)arg5;                         \
3378
      __asm__ volatile(                                           \
3379
         VALGRIND_ALIGN_STACK                                     \
3380
         "mr 12,%1\n\t"                                           \
3381
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3382
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3383
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3384
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3385
         "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3386
         "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3387
         "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3388
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3389
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3390
         "mr 12,%1\n\t"                                           \
3391
         "mr %0,3\n\t"                                            \
3392
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3393
         VALGRIND_RESTORE_STACK                                   \
3394
         : /*out*/   "=r" (_res)                                  \
3395
         : /*in*/    "r" (&_argvec[2])                            \
3396
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3397
      );                                                          \
3398
      lval = (__typeof__(lval)) _res;                             \
3399
   } while (0)
3400
3401
#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3402
   do {                                                           \
3403
      volatile OrigFn        _orig = (orig);                      \
3404
      volatile unsigned long _argvec[3+6];                        \
3405
      volatile unsigned long _res;                                \
3406
      /* _argvec[0] holds current r2 across the call */           \
3407
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3408
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3409
      _argvec[2+1] = (unsigned long)arg1;                         \
3410
      _argvec[2+2] = (unsigned long)arg2;                         \
3411
      _argvec[2+3] = (unsigned long)arg3;                         \
3412
      _argvec[2+4] = (unsigned long)arg4;                         \
3413
      _argvec[2+5] = (unsigned long)arg5;                         \
3414
      _argvec[2+6] = (unsigned long)arg6;                         \
3415
      __asm__ volatile(                                           \
3416
         VALGRIND_ALIGN_STACK                                     \
3417
         "mr 12,%1\n\t"                                           \
3418
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3419
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3420
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3421
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3422
         "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3423
         "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3424
         "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3425
         "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3426
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3427
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3428
         "mr 12,%1\n\t"                                           \
3429
         "mr %0,3\n\t"                                            \
3430
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3431
         VALGRIND_RESTORE_STACK                                   \
3432
         : /*out*/   "=r" (_res)                                  \
3433
         : /*in*/    "r" (&_argvec[2])                            \
3434
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3435
      );                                                          \
3436
      lval = (__typeof__(lval)) _res;                             \
3437
   } while (0)
3438
3439
#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3440
                                 arg7)                            \
3441
   do {                                                           \
3442
      volatile OrigFn        _orig = (orig);                      \
3443
      volatile unsigned long _argvec[3+7];                        \
3444
      volatile unsigned long _res;                                \
3445
      /* _argvec[0] holds current r2 across the call */           \
3446
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3447
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3448
      _argvec[2+1] = (unsigned long)arg1;                         \
3449
      _argvec[2+2] = (unsigned long)arg2;                         \
3450
      _argvec[2+3] = (unsigned long)arg3;                         \
3451
      _argvec[2+4] = (unsigned long)arg4;                         \
3452
      _argvec[2+5] = (unsigned long)arg5;                         \
3453
      _argvec[2+6] = (unsigned long)arg6;                         \
3454
      _argvec[2+7] = (unsigned long)arg7;                         \
3455
      __asm__ volatile(                                           \
3456
         VALGRIND_ALIGN_STACK                                     \
3457
         "mr 12,%1\n\t"                                           \
3458
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3459
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3460
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3461
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3462
         "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3463
         "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3464
         "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3465
         "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3466
         "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3467
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3468
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3469
         "mr 12,%1\n\t"                                           \
3470
         "mr %0,3\n\t"                                            \
3471
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3472
         VALGRIND_RESTORE_STACK                                   \
3473
         : /*out*/   "=r" (_res)                                  \
3474
         : /*in*/    "r" (&_argvec[2])                            \
3475
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3476
      );                                                          \
3477
      lval = (__typeof__(lval)) _res;                             \
3478
   } while (0)
3479
3480
#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3481
                                 arg7,arg8)                       \
3482
   do {                                                           \
3483
      volatile OrigFn        _orig = (orig);                      \
3484
      volatile unsigned long _argvec[3+8];                        \
3485
      volatile unsigned long _res;                                \
3486
      /* _argvec[0] holds current r2 across the call */           \
3487
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3488
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3489
      _argvec[2+1] = (unsigned long)arg1;                         \
3490
      _argvec[2+2] = (unsigned long)arg2;                         \
3491
      _argvec[2+3] = (unsigned long)arg3;                         \
3492
      _argvec[2+4] = (unsigned long)arg4;                         \
3493
      _argvec[2+5] = (unsigned long)arg5;                         \
3494
      _argvec[2+6] = (unsigned long)arg6;                         \
3495
      _argvec[2+7] = (unsigned long)arg7;                         \
3496
      _argvec[2+8] = (unsigned long)arg8;                         \
3497
      __asm__ volatile(                                           \
3498
         VALGRIND_ALIGN_STACK                                     \
3499
         "mr 12,%1\n\t"                                           \
3500
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3501
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3502
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3503
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3504
         "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3505
         "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3506
         "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3507
         "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3508
         "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3509
         "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
3510
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3511
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3512
         "mr 12,%1\n\t"                                           \
3513
         "mr %0,3\n\t"                                            \
3514
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3515
         VALGRIND_RESTORE_STACK                                   \
3516
         : /*out*/   "=r" (_res)                                  \
3517
         : /*in*/    "r" (&_argvec[2])                            \
3518
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3519
      );                                                          \
3520
      lval = (__typeof__(lval)) _res;                             \
3521
   } while (0)
3522
3523
#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3524
                                 arg7,arg8,arg9)                  \
3525
   do {                                                           \
3526
      volatile OrigFn        _orig = (orig);                      \
3527
      volatile unsigned long _argvec[3+9];                        \
3528
      volatile unsigned long _res;                                \
3529
      /* _argvec[0] holds current r2 across the call */           \
3530
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3531
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3532
      _argvec[2+1] = (unsigned long)arg1;                         \
3533
      _argvec[2+2] = (unsigned long)arg2;                         \
3534
      _argvec[2+3] = (unsigned long)arg3;                         \
3535
      _argvec[2+4] = (unsigned long)arg4;                         \
3536
      _argvec[2+5] = (unsigned long)arg5;                         \
3537
      _argvec[2+6] = (unsigned long)arg6;                         \
3538
      _argvec[2+7] = (unsigned long)arg7;                         \
3539
      _argvec[2+8] = (unsigned long)arg8;                         \
3540
      _argvec[2+9] = (unsigned long)arg9;                         \
3541
      __asm__ volatile(                                           \
3542
         VALGRIND_ALIGN_STACK                                     \
3543
         "mr 12,%1\n\t"                                           \
3544
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3545
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3546
         "addi 1,1,-128\n\t"  /* expand stack frame */            \
3547
         /* arg9 */                                               \
3548
         "ld  3,72(12)\n\t"                                       \
3549
         "std 3,96(1)\n\t"                                        \
3550
         /* args1-8 */                                            \
3551
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3552
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3553
         "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3554
         "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3555
         "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3556
         "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3557
         "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3558
         "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
3559
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3560
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3561
         "mr 12,%1\n\t"                                           \
3562
         "mr %0,3\n\t"                                            \
3563
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3564
         VALGRIND_RESTORE_STACK                                   \
3565
         : /*out*/   "=r" (_res)                                  \
3566
         : /*in*/    "r" (&_argvec[2])                            \
3567
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3568
      );                                                          \
3569
      lval = (__typeof__(lval)) _res;                             \
3570
   } while (0)
3571
3572
#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3573
                                  arg7,arg8,arg9,arg10)           \
3574
   do {                                                           \
3575
      volatile OrigFn        _orig = (orig);                      \
3576
      volatile unsigned long _argvec[3+10];                       \
3577
      volatile unsigned long _res;                                \
3578
      /* _argvec[0] holds current r2 across the call */           \
3579
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3580
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3581
      _argvec[2+1] = (unsigned long)arg1;                         \
3582
      _argvec[2+2] = (unsigned long)arg2;                         \
3583
      _argvec[2+3] = (unsigned long)arg3;                         \
3584
      _argvec[2+4] = (unsigned long)arg4;                         \
3585
      _argvec[2+5] = (unsigned long)arg5;                         \
3586
      _argvec[2+6] = (unsigned long)arg6;                         \
3587
      _argvec[2+7] = (unsigned long)arg7;                         \
3588
      _argvec[2+8] = (unsigned long)arg8;                         \
3589
      _argvec[2+9] = (unsigned long)arg9;                         \
3590
      _argvec[2+10] = (unsigned long)arg10;                       \
3591
      __asm__ volatile(                                           \
3592
         VALGRIND_ALIGN_STACK                                     \
3593
         "mr 12,%1\n\t"                                           \
3594
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3595
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3596
         "addi 1,1,-128\n\t"  /* expand stack frame */            \
3597
         /* arg10 */                                              \
3598
         "ld  3,80(12)\n\t"                                       \
3599
         "std 3,104(1)\n\t"                                       \
3600
         /* arg9 */                                               \
3601
         "ld  3,72(12)\n\t"                                       \
3602
         "std 3,96(1)\n\t"                                        \
3603
         /* args1-8 */                                            \
3604
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3605
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3606
         "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3607
         "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3608
         "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3609
         "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3610
         "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3611
         "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
3612
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3613
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3614
         "mr 12,%1\n\t"                                           \
3615
         "mr %0,3\n\t"                                            \
3616
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3617
         VALGRIND_RESTORE_STACK                                   \
3618
         : /*out*/   "=r" (_res)                                  \
3619
         : /*in*/    "r" (&_argvec[2])                            \
3620
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3621
      );                                                          \
3622
      lval = (__typeof__(lval)) _res;                             \
3623
   } while (0)
3624
3625
#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3626
                                  arg7,arg8,arg9,arg10,arg11)     \
3627
   do {                                                           \
3628
      volatile OrigFn        _orig = (orig);                      \
3629
      volatile unsigned long _argvec[3+11];                       \
3630
      volatile unsigned long _res;                                \
3631
      /* _argvec[0] holds current r2 across the call */           \
3632
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3633
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3634
      _argvec[2+1] = (unsigned long)arg1;                         \
3635
      _argvec[2+2] = (unsigned long)arg2;                         \
3636
      _argvec[2+3] = (unsigned long)arg3;                         \
3637
      _argvec[2+4] = (unsigned long)arg4;                         \
3638
      _argvec[2+5] = (unsigned long)arg5;                         \
3639
      _argvec[2+6] = (unsigned long)arg6;                         \
3640
      _argvec[2+7] = (unsigned long)arg7;                         \
3641
      _argvec[2+8] = (unsigned long)arg8;                         \
3642
      _argvec[2+9] = (unsigned long)arg9;                         \
3643
      _argvec[2+10] = (unsigned long)arg10;                       \
3644
      _argvec[2+11] = (unsigned long)arg11;                       \
3645
      __asm__ volatile(                                           \
3646
         VALGRIND_ALIGN_STACK                                     \
3647
         "mr 12,%1\n\t"                                           \
3648
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3649
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3650
         "addi 1,1,-144\n\t"  /* expand stack frame */            \
3651
         /* arg11 */                                              \
3652
         "ld  3,88(12)\n\t"                                       \
3653
         "std 3,112(1)\n\t"                                       \
3654
         /* arg10 */                                              \
3655
         "ld  3,80(12)\n\t"                                       \
3656
         "std 3,104(1)\n\t"                                       \
3657
         /* arg9 */                                               \
3658
         "ld  3,72(12)\n\t"                                       \
3659
         "std 3,96(1)\n\t"                                        \
3660
         /* args1-8 */                                            \
3661
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3662
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3663
         "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3664
         "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3665
         "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3666
         "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3667
         "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3668
         "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
3669
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3670
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3671
         "mr 12,%1\n\t"                                           \
3672
         "mr %0,3\n\t"                                            \
3673
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3674
         VALGRIND_RESTORE_STACK                                   \
3675
         : /*out*/   "=r" (_res)                                  \
3676
         : /*in*/    "r" (&_argvec[2])                            \
3677
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3678
      );                                                          \
3679
      lval = (__typeof__(lval)) _res;                             \
3680
   } while (0)
3681
3682
#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
3683
                                arg7,arg8,arg9,arg10,arg11,arg12) \
3684
   do {                                                           \
3685
      volatile OrigFn        _orig = (orig);                      \
3686
      volatile unsigned long _argvec[3+12];                       \
3687
      volatile unsigned long _res;                                \
3688
      /* _argvec[0] holds current r2 across the call */           \
3689
      _argvec[1]   = (unsigned long)_orig.r2;                     \
3690
      _argvec[2]   = (unsigned long)_orig.nraddr;                 \
3691
      _argvec[2+1] = (unsigned long)arg1;                         \
3692
      _argvec[2+2] = (unsigned long)arg2;                         \
3693
      _argvec[2+3] = (unsigned long)arg3;                         \
3694
      _argvec[2+4] = (unsigned long)arg4;                         \
3695
      _argvec[2+5] = (unsigned long)arg5;                         \
3696
      _argvec[2+6] = (unsigned long)arg6;                         \
3697
      _argvec[2+7] = (unsigned long)arg7;                         \
3698
      _argvec[2+8] = (unsigned long)arg8;                         \
3699
      _argvec[2+9] = (unsigned long)arg9;                         \
3700
      _argvec[2+10] = (unsigned long)arg10;                       \
3701
      _argvec[2+11] = (unsigned long)arg11;                       \
3702
      _argvec[2+12] = (unsigned long)arg12;                       \
3703
      __asm__ volatile(                                           \
3704
         VALGRIND_ALIGN_STACK                                     \
3705
         "mr 12,%1\n\t"                                           \
3706
         "std 2,-16(12)\n\t"  /* save tocptr */                   \
3707
         "ld   2,-8(12)\n\t"  /* use nraddr's tocptr */           \
3708
         "addi 1,1,-144\n\t"  /* expand stack frame */            \
3709
         /* arg12 */                                              \
3710
         "ld  3,96(12)\n\t"                                       \
3711
         "std 3,120(1)\n\t"                                       \
3712
         /* arg11 */                                              \
3713
         "ld  3,88(12)\n\t"                                       \
3714
         "std 3,112(1)\n\t"                                       \
3715
         /* arg10 */                                              \
3716
         "ld  3,80(12)\n\t"                                       \
3717
         "std 3,104(1)\n\t"                                       \
3718
         /* arg9 */                                               \
3719
         "ld  3,72(12)\n\t"                                       \
3720
         "std 3,96(1)\n\t"                                        \
3721
         /* args1-8 */                                            \
3722
         "ld   3, 8(12)\n\t"  /* arg1->r3 */                      \
3723
         "ld   4, 16(12)\n\t" /* arg2->r4 */                      \
3724
         "ld   5, 24(12)\n\t" /* arg3->r5 */                      \
3725
         "ld   6, 32(12)\n\t" /* arg4->r6 */                      \
3726
         "ld   7, 40(12)\n\t" /* arg5->r7 */                      \
3727
         "ld   8, 48(12)\n\t" /* arg6->r8 */                      \
3728
         "ld   9, 56(12)\n\t" /* arg7->r9 */                      \
3729
         "ld  10, 64(12)\n\t" /* arg8->r10 */                     \
3730
         "ld  12, 0(12)\n\t"  /* target->r12 */                   \
3731
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12                  \
3732
         "mr 12,%1\n\t"                                           \
3733
         "mr %0,3\n\t"                                            \
3734
         "ld 2,-16(12)\n\t" /* restore tocptr */                  \
3735
         VALGRIND_RESTORE_STACK                                   \
3736
         : /*out*/   "=r" (_res)                                  \
3737
         : /*in*/    "r" (&_argvec[2])                            \
3738
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28"   \
3739
      );                                                          \
3740
      lval = (__typeof__(lval)) _res;                             \
3741
   } while (0)
3742
3743
#endif /* PLAT_ppc64le_linux */
3744
3745
/* ------------------------- arm-linux ------------------------- */
3746
3747
#if defined(PLAT_arm_linux)
3748
3749
/* These regs are trashed by the hidden call. */
3750
#define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4", "r12", "r14"
3751
3752
/* Macros to save and align the stack before making a function
3753
   call and restore it afterwards as gcc may not keep the stack
3754
   pointer aligned if it doesn't realise calls are being made
3755
   to other functions. */
3756
3757
/* This is a bit tricky.  We store the original stack pointer in r10
3758
   as it is callee-saves.  gcc doesn't allow the use of r11 for some
3759
   reason.  Also, we can't directly "bic" the stack pointer in thumb
3760
   mode since r13 isn't an allowed register number in that context.
3761
   So use r4 as a temporary, since that is about to get trashed
3762
   anyway, just after each use of this macro.  Side effect is we need
3763
   to be very careful about any future changes, since
3764
   VALGRIND_ALIGN_STACK simply assumes r4 is usable. */
3765
#define VALGRIND_ALIGN_STACK               \
3766
      "mov r10, sp\n\t"                    \
3767
      "mov r4,  sp\n\t"                    \
3768
      "bic r4,  r4, #7\n\t"                \
3769
      "mov sp,  r4\n\t"
3770
#define VALGRIND_RESTORE_STACK             \
3771
      "mov sp,  r10\n\t"
3772
3773
/* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned
3774
   long) == 4. */
3775
3776
#define CALL_FN_W_v(lval, orig)                                   \
3777
   do {                                                           \
3778
      volatile OrigFn        _orig = (orig);                      \
3779
      volatile unsigned long _argvec[1];                          \
3780
      volatile unsigned long _res;                                \
3781
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
3782
      __asm__ volatile(                                           \
3783
         VALGRIND_ALIGN_STACK                                     \
3784
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3785
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3786
         VALGRIND_RESTORE_STACK                                   \
3787
         "mov %0, r0\n"                                           \
3788
         : /*out*/   "=r" (_res)                                  \
3789
         : /*in*/    "0" (&_argvec[0])                            \
3790
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3791
      );                                                          \
3792
      lval = (__typeof__(lval)) _res;                             \
3793
   } while (0)
3794
3795
#define CALL_FN_W_W(lval, orig, arg1)                             \
3796
   do {                                                           \
3797
      volatile OrigFn        _orig = (orig);                      \
3798
      volatile unsigned long _argvec[2];                          \
3799
      volatile unsigned long _res;                                \
3800
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
3801
      _argvec[1] = (unsigned long)(arg1);                         \
3802
      __asm__ volatile(                                           \
3803
         VALGRIND_ALIGN_STACK                                     \
3804
         "ldr r0, [%1, #4] \n\t"                                  \
3805
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3806
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3807
         VALGRIND_RESTORE_STACK                                   \
3808
         "mov %0, r0\n"                                           \
3809
         : /*out*/   "=r" (_res)                                  \
3810
         : /*in*/    "0" (&_argvec[0])                            \
3811
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3812
      );                                                          \
3813
      lval = (__typeof__(lval)) _res;                             \
3814
   } while (0)
3815
3816
#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
3817
   do {                                                           \
3818
      volatile OrigFn        _orig = (orig);                      \
3819
      volatile unsigned long _argvec[3];                          \
3820
      volatile unsigned long _res;                                \
3821
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
3822
      _argvec[1] = (unsigned long)(arg1);                         \
3823
      _argvec[2] = (unsigned long)(arg2);                         \
3824
      __asm__ volatile(                                           \
3825
         VALGRIND_ALIGN_STACK                                     \
3826
         "ldr r0, [%1, #4] \n\t"                                  \
3827
         "ldr r1, [%1, #8] \n\t"                                  \
3828
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3829
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3830
         VALGRIND_RESTORE_STACK                                   \
3831
         "mov %0, r0\n"                                           \
3832
         : /*out*/   "=r" (_res)                                  \
3833
         : /*in*/    "0" (&_argvec[0])                            \
3834
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3835
      );                                                          \
3836
      lval = (__typeof__(lval)) _res;                             \
3837
   } while (0)
3838
3839
#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
3840
   do {                                                           \
3841
      volatile OrigFn        _orig = (orig);                      \
3842
      volatile unsigned long _argvec[4];                          \
3843
      volatile unsigned long _res;                                \
3844
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
3845
      _argvec[1] = (unsigned long)(arg1);                         \
3846
      _argvec[2] = (unsigned long)(arg2);                         \
3847
      _argvec[3] = (unsigned long)(arg3);                         \
3848
      __asm__ volatile(                                           \
3849
         VALGRIND_ALIGN_STACK                                     \
3850
         "ldr r0, [%1, #4] \n\t"                                  \
3851
         "ldr r1, [%1, #8] \n\t"                                  \
3852
         "ldr r2, [%1, #12] \n\t"                                 \
3853
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3854
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3855
         VALGRIND_RESTORE_STACK                                   \
3856
         "mov %0, r0\n"                                           \
3857
         : /*out*/   "=r" (_res)                                  \
3858
         : /*in*/    "0" (&_argvec[0])                            \
3859
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3860
      );                                                          \
3861
      lval = (__typeof__(lval)) _res;                             \
3862
   } while (0)
3863
3864
#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
3865
   do {                                                           \
3866
      volatile OrigFn        _orig = (orig);                      \
3867
      volatile unsigned long _argvec[5];                          \
3868
      volatile unsigned long _res;                                \
3869
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
3870
      _argvec[1] = (unsigned long)(arg1);                         \
3871
      _argvec[2] = (unsigned long)(arg2);                         \
3872
      _argvec[3] = (unsigned long)(arg3);                         \
3873
      _argvec[4] = (unsigned long)(arg4);                         \
3874
      __asm__ volatile(                                           \
3875
         VALGRIND_ALIGN_STACK                                     \
3876
         "ldr r0, [%1, #4] \n\t"                                  \
3877
         "ldr r1, [%1, #8] \n\t"                                  \
3878
         "ldr r2, [%1, #12] \n\t"                                 \
3879
         "ldr r3, [%1, #16] \n\t"                                 \
3880
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3881
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3882
         VALGRIND_RESTORE_STACK                                   \
3883
         "mov %0, r0"                                             \
3884
         : /*out*/   "=r" (_res)                                  \
3885
         : /*in*/    "0" (&_argvec[0])                            \
3886
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3887
      );                                                          \
3888
      lval = (__typeof__(lval)) _res;                             \
3889
   } while (0)
3890
3891
#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
3892
   do {                                                           \
3893
      volatile OrigFn        _orig = (orig);                      \
3894
      volatile unsigned long _argvec[6];                          \
3895
      volatile unsigned long _res;                                \
3896
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
3897
      _argvec[1] = (unsigned long)(arg1);                         \
3898
      _argvec[2] = (unsigned long)(arg2);                         \
3899
      _argvec[3] = (unsigned long)(arg3);                         \
3900
      _argvec[4] = (unsigned long)(arg4);                         \
3901
      _argvec[5] = (unsigned long)(arg5);                         \
3902
      __asm__ volatile(                                           \
3903
         VALGRIND_ALIGN_STACK                                     \
3904
         "sub sp, sp, #4 \n\t"                                    \
3905
         "ldr r0, [%1, #20] \n\t"                                 \
3906
         "push {r0} \n\t"                                         \
3907
         "ldr r0, [%1, #4] \n\t"                                  \
3908
         "ldr r1, [%1, #8] \n\t"                                  \
3909
         "ldr r2, [%1, #12] \n\t"                                 \
3910
         "ldr r3, [%1, #16] \n\t"                                 \
3911
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3912
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3913
         VALGRIND_RESTORE_STACK                                   \
3914
         "mov %0, r0"                                             \
3915
         : /*out*/   "=r" (_res)                                  \
3916
         : /*in*/    "0" (&_argvec[0])                            \
3917
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3918
      );                                                          \
3919
      lval = (__typeof__(lval)) _res;                             \
3920
   } while (0)
3921
3922
#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
3923
   do {                                                           \
3924
      volatile OrigFn        _orig = (orig);                      \
3925
      volatile unsigned long _argvec[7];                          \
3926
      volatile unsigned long _res;                                \
3927
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
3928
      _argvec[1] = (unsigned long)(arg1);                         \
3929
      _argvec[2] = (unsigned long)(arg2);                         \
3930
      _argvec[3] = (unsigned long)(arg3);                         \
3931
      _argvec[4] = (unsigned long)(arg4);                         \
3932
      _argvec[5] = (unsigned long)(arg5);                         \
3933
      _argvec[6] = (unsigned long)(arg6);                         \
3934
      __asm__ volatile(                                           \
3935
         VALGRIND_ALIGN_STACK                                     \
3936
         "ldr r0, [%1, #20] \n\t"                                 \
3937
         "ldr r1, [%1, #24] \n\t"                                 \
3938
         "push {r0, r1} \n\t"                                     \
3939
         "ldr r0, [%1, #4] \n\t"                                  \
3940
         "ldr r1, [%1, #8] \n\t"                                  \
3941
         "ldr r2, [%1, #12] \n\t"                                 \
3942
         "ldr r3, [%1, #16] \n\t"                                 \
3943
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3944
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3945
         VALGRIND_RESTORE_STACK                                   \
3946
         "mov %0, r0"                                             \
3947
         : /*out*/   "=r" (_res)                                  \
3948
         : /*in*/    "0" (&_argvec[0])                            \
3949
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3950
      );                                                          \
3951
      lval = (__typeof__(lval)) _res;                             \
3952
   } while (0)
3953
3954
#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3955
                                 arg7)                            \
3956
   do {                                                           \
3957
      volatile OrigFn        _orig = (orig);                      \
3958
      volatile unsigned long _argvec[8];                          \
3959
      volatile unsigned long _res;                                \
3960
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
3961
      _argvec[1] = (unsigned long)(arg1);                         \
3962
      _argvec[2] = (unsigned long)(arg2);                         \
3963
      _argvec[3] = (unsigned long)(arg3);                         \
3964
      _argvec[4] = (unsigned long)(arg4);                         \
3965
      _argvec[5] = (unsigned long)(arg5);                         \
3966
      _argvec[6] = (unsigned long)(arg6);                         \
3967
      _argvec[7] = (unsigned long)(arg7);                         \
3968
      __asm__ volatile(                                           \
3969
         VALGRIND_ALIGN_STACK                                     \
3970
         "sub sp, sp, #4 \n\t"                                    \
3971
         "ldr r0, [%1, #20] \n\t"                                 \
3972
         "ldr r1, [%1, #24] \n\t"                                 \
3973
         "ldr r2, [%1, #28] \n\t"                                 \
3974
         "push {r0, r1, r2} \n\t"                                 \
3975
         "ldr r0, [%1, #4] \n\t"                                  \
3976
         "ldr r1, [%1, #8] \n\t"                                  \
3977
         "ldr r2, [%1, #12] \n\t"                                 \
3978
         "ldr r3, [%1, #16] \n\t"                                 \
3979
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
3980
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
3981
         VALGRIND_RESTORE_STACK                                   \
3982
         "mov %0, r0"                                             \
3983
         : /*out*/   "=r" (_res)                                  \
3984
         : /*in*/    "0" (&_argvec[0])                            \
3985
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
3986
      );                                                          \
3987
      lval = (__typeof__(lval)) _res;                             \
3988
   } while (0)
3989
3990
#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
3991
                                 arg7,arg8)                       \
3992
   do {                                                           \
3993
      volatile OrigFn        _orig = (orig);                      \
3994
      volatile unsigned long _argvec[9];                          \
3995
      volatile unsigned long _res;                                \
3996
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
3997
      _argvec[1] = (unsigned long)(arg1);                         \
3998
      _argvec[2] = (unsigned long)(arg2);                         \
3999
      _argvec[3] = (unsigned long)(arg3);                         \
4000
      _argvec[4] = (unsigned long)(arg4);                         \
4001
      _argvec[5] = (unsigned long)(arg5);                         \
4002
      _argvec[6] = (unsigned long)(arg6);                         \
4003
      _argvec[7] = (unsigned long)(arg7);                         \
4004
      _argvec[8] = (unsigned long)(arg8);                         \
4005
      __asm__ volatile(                                           \
4006
         VALGRIND_ALIGN_STACK                                     \
4007
         "ldr r0, [%1, #20] \n\t"                                 \
4008
         "ldr r1, [%1, #24] \n\t"                                 \
4009
         "ldr r2, [%1, #28] \n\t"                                 \
4010
         "ldr r3, [%1, #32] \n\t"                                 \
4011
         "push {r0, r1, r2, r3} \n\t"                             \
4012
         "ldr r0, [%1, #4] \n\t"                                  \
4013
         "ldr r1, [%1, #8] \n\t"                                  \
4014
         "ldr r2, [%1, #12] \n\t"                                 \
4015
         "ldr r3, [%1, #16] \n\t"                                 \
4016
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4017
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4018
         VALGRIND_RESTORE_STACK                                   \
4019
         "mov %0, r0"                                             \
4020
         : /*out*/   "=r" (_res)                                  \
4021
         : /*in*/    "0" (&_argvec[0])                            \
4022
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4023
      );                                                          \
4024
      lval = (__typeof__(lval)) _res;                             \
4025
   } while (0)
4026
4027
#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4028
                                 arg7,arg8,arg9)                  \
4029
   do {                                                           \
4030
      volatile OrigFn        _orig = (orig);                      \
4031
      volatile unsigned long _argvec[10];                         \
4032
      volatile unsigned long _res;                                \
4033
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4034
      _argvec[1] = (unsigned long)(arg1);                         \
4035
      _argvec[2] = (unsigned long)(arg2);                         \
4036
      _argvec[3] = (unsigned long)(arg3);                         \
4037
      _argvec[4] = (unsigned long)(arg4);                         \
4038
      _argvec[5] = (unsigned long)(arg5);                         \
4039
      _argvec[6] = (unsigned long)(arg6);                         \
4040
      _argvec[7] = (unsigned long)(arg7);                         \
4041
      _argvec[8] = (unsigned long)(arg8);                         \
4042
      _argvec[9] = (unsigned long)(arg9);                         \
4043
      __asm__ volatile(                                           \
4044
         VALGRIND_ALIGN_STACK                                     \
4045
         "sub sp, sp, #4 \n\t"                                    \
4046
         "ldr r0, [%1, #20] \n\t"                                 \
4047
         "ldr r1, [%1, #24] \n\t"                                 \
4048
         "ldr r2, [%1, #28] \n\t"                                 \
4049
         "ldr r3, [%1, #32] \n\t"                                 \
4050
         "ldr r4, [%1, #36] \n\t"                                 \
4051
         "push {r0, r1, r2, r3, r4} \n\t"                         \
4052
         "ldr r0, [%1, #4] \n\t"                                  \
4053
         "ldr r1, [%1, #8] \n\t"                                  \
4054
         "ldr r2, [%1, #12] \n\t"                                 \
4055
         "ldr r3, [%1, #16] \n\t"                                 \
4056
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4057
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4058
         VALGRIND_RESTORE_STACK                                   \
4059
         "mov %0, r0"                                             \
4060
         : /*out*/   "=r" (_res)                                  \
4061
         : /*in*/    "0" (&_argvec[0])                            \
4062
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4063
      );                                                          \
4064
      lval = (__typeof__(lval)) _res;                             \
4065
   } while (0)
4066
4067
#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4068
                                  arg7,arg8,arg9,arg10)           \
4069
   do {                                                           \
4070
      volatile OrigFn        _orig = (orig);                      \
4071
      volatile unsigned long _argvec[11];                         \
4072
      volatile unsigned long _res;                                \
4073
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4074
      _argvec[1] = (unsigned long)(arg1);                         \
4075
      _argvec[2] = (unsigned long)(arg2);                         \
4076
      _argvec[3] = (unsigned long)(arg3);                         \
4077
      _argvec[4] = (unsigned long)(arg4);                         \
4078
      _argvec[5] = (unsigned long)(arg5);                         \
4079
      _argvec[6] = (unsigned long)(arg6);                         \
4080
      _argvec[7] = (unsigned long)(arg7);                         \
4081
      _argvec[8] = (unsigned long)(arg8);                         \
4082
      _argvec[9] = (unsigned long)(arg9);                         \
4083
      _argvec[10] = (unsigned long)(arg10);                       \
4084
      __asm__ volatile(                                           \
4085
         VALGRIND_ALIGN_STACK                                     \
4086
         "ldr r0, [%1, #40] \n\t"                                 \
4087
         "push {r0} \n\t"                                         \
4088
         "ldr r0, [%1, #20] \n\t"                                 \
4089
         "ldr r1, [%1, #24] \n\t"                                 \
4090
         "ldr r2, [%1, #28] \n\t"                                 \
4091
         "ldr r3, [%1, #32] \n\t"                                 \
4092
         "ldr r4, [%1, #36] \n\t"                                 \
4093
         "push {r0, r1, r2, r3, r4} \n\t"                         \
4094
         "ldr r0, [%1, #4] \n\t"                                  \
4095
         "ldr r1, [%1, #8] \n\t"                                  \
4096
         "ldr r2, [%1, #12] \n\t"                                 \
4097
         "ldr r3, [%1, #16] \n\t"                                 \
4098
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4099
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4100
         VALGRIND_RESTORE_STACK                                   \
4101
         "mov %0, r0"                                             \
4102
         : /*out*/   "=r" (_res)                                  \
4103
         : /*in*/    "0" (&_argvec[0])                            \
4104
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4105
      );                                                          \
4106
      lval = (__typeof__(lval)) _res;                             \
4107
   } while (0)
4108
4109
#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
4110
                                  arg6,arg7,arg8,arg9,arg10,      \
4111
                                  arg11)                          \
4112
   do {                                                           \
4113
      volatile OrigFn        _orig = (orig);                      \
4114
      volatile unsigned long _argvec[12];                         \
4115
      volatile unsigned long _res;                                \
4116
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4117
      _argvec[1] = (unsigned long)(arg1);                         \
4118
      _argvec[2] = (unsigned long)(arg2);                         \
4119
      _argvec[3] = (unsigned long)(arg3);                         \
4120
      _argvec[4] = (unsigned long)(arg4);                         \
4121
      _argvec[5] = (unsigned long)(arg5);                         \
4122
      _argvec[6] = (unsigned long)(arg6);                         \
4123
      _argvec[7] = (unsigned long)(arg7);                         \
4124
      _argvec[8] = (unsigned long)(arg8);                         \
4125
      _argvec[9] = (unsigned long)(arg9);                         \
4126
      _argvec[10] = (unsigned long)(arg10);                       \
4127
      _argvec[11] = (unsigned long)(arg11);                       \
4128
      __asm__ volatile(                                           \
4129
         VALGRIND_ALIGN_STACK                                     \
4130
         "sub sp, sp, #4 \n\t"                                    \
4131
         "ldr r0, [%1, #40] \n\t"                                 \
4132
         "ldr r1, [%1, #44] \n\t"                                 \
4133
         "push {r0, r1} \n\t"                                     \
4134
         "ldr r0, [%1, #20] \n\t"                                 \
4135
         "ldr r1, [%1, #24] \n\t"                                 \
4136
         "ldr r2, [%1, #28] \n\t"                                 \
4137
         "ldr r3, [%1, #32] \n\t"                                 \
4138
         "ldr r4, [%1, #36] \n\t"                                 \
4139
         "push {r0, r1, r2, r3, r4} \n\t"                         \
4140
         "ldr r0, [%1, #4] \n\t"                                  \
4141
         "ldr r1, [%1, #8] \n\t"                                  \
4142
         "ldr r2, [%1, #12] \n\t"                                 \
4143
         "ldr r3, [%1, #16] \n\t"                                 \
4144
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4145
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4146
         VALGRIND_RESTORE_STACK                                   \
4147
         "mov %0, r0"                                             \
4148
         : /*out*/   "=r" (_res)                                  \
4149
         : /*in*/    "0" (&_argvec[0])                            \
4150
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4151
      );                                                          \
4152
      lval = (__typeof__(lval)) _res;                             \
4153
   } while (0)
4154
4155
#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
4156
                                  arg6,arg7,arg8,arg9,arg10,      \
4157
                                  arg11,arg12)                    \
4158
   do {                                                           \
4159
      volatile OrigFn        _orig = (orig);                      \
4160
      volatile unsigned long _argvec[13];                         \
4161
      volatile unsigned long _res;                                \
4162
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4163
      _argvec[1] = (unsigned long)(arg1);                         \
4164
      _argvec[2] = (unsigned long)(arg2);                         \
4165
      _argvec[3] = (unsigned long)(arg3);                         \
4166
      _argvec[4] = (unsigned long)(arg4);                         \
4167
      _argvec[5] = (unsigned long)(arg5);                         \
4168
      _argvec[6] = (unsigned long)(arg6);                         \
4169
      _argvec[7] = (unsigned long)(arg7);                         \
4170
      _argvec[8] = (unsigned long)(arg8);                         \
4171
      _argvec[9] = (unsigned long)(arg9);                         \
4172
      _argvec[10] = (unsigned long)(arg10);                       \
4173
      _argvec[11] = (unsigned long)(arg11);                       \
4174
      _argvec[12] = (unsigned long)(arg12);                       \
4175
      __asm__ volatile(                                           \
4176
         VALGRIND_ALIGN_STACK                                     \
4177
         "ldr r0, [%1, #40] \n\t"                                 \
4178
         "ldr r1, [%1, #44] \n\t"                                 \
4179
         "ldr r2, [%1, #48] \n\t"                                 \
4180
         "push {r0, r1, r2} \n\t"                                 \
4181
         "ldr r0, [%1, #20] \n\t"                                 \
4182
         "ldr r1, [%1, #24] \n\t"                                 \
4183
         "ldr r2, [%1, #28] \n\t"                                 \
4184
         "ldr r3, [%1, #32] \n\t"                                 \
4185
         "ldr r4, [%1, #36] \n\t"                                 \
4186
         "push {r0, r1, r2, r3, r4} \n\t"                         \
4187
         "ldr r0, [%1, #4] \n\t"                                  \
4188
         "ldr r1, [%1, #8] \n\t"                                  \
4189
         "ldr r2, [%1, #12] \n\t"                                 \
4190
         "ldr r3, [%1, #16] \n\t"                                 \
4191
         "ldr r4, [%1] \n\t"  /* target->r4 */                    \
4192
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4                   \
4193
         VALGRIND_RESTORE_STACK                                   \
4194
         "mov %0, r0"                                             \
4195
         : /*out*/   "=r" (_res)                                  \
4196
         : /*in*/    "0" (&_argvec[0])                            \
4197
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10"   \
4198
      );                                                          \
4199
      lval = (__typeof__(lval)) _res;                             \
4200
   } while (0)
4201
4202
#endif /* PLAT_arm_linux */
4203
4204
/* ------------------------ arm64-linux ------------------------ */
4205
4206
#if defined(PLAT_arm64_linux)
4207
4208
/* These regs are trashed by the hidden call. */
4209
#define __CALLER_SAVED_REGS \
4210
     "x0", "x1", "x2", "x3","x4", "x5", "x6", "x7", "x8", "x9",   \
4211
     "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17",      \
4212
     "x18", "x19", "x20", "x30",                                  \
4213
     "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",  \
4214
     "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17",      \
4215
     "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25",      \
4216
     "v26", "v27", "v28", "v29", "v30", "v31"
4217
4218
/* x21 is callee-saved, so we can use it to save and restore SP around
4219
   the hidden call. */
4220
#define VALGRIND_ALIGN_STACK               \
4221
      "mov x21, sp\n\t"                    \
4222
      "bic sp, x21, #15\n\t"
4223
#define VALGRIND_RESTORE_STACK             \
4224
      "mov sp,  x21\n\t"
4225
4226
/* These CALL_FN_ macros assume that on arm64-linux,
4227
   sizeof(unsigned long) == 8. */
4228
4229
#define CALL_FN_W_v(lval, orig)                                   \
4230
   do {                                                           \
4231
      volatile OrigFn        _orig = (orig);                      \
4232
      volatile unsigned long _argvec[1];                          \
4233
      volatile unsigned long _res;                                \
4234
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4235
      __asm__ volatile(                                           \
4236
         VALGRIND_ALIGN_STACK                                     \
4237
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4238
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4239
         VALGRIND_RESTORE_STACK                                   \
4240
         "mov %0, x0\n"                                           \
4241
         : /*out*/   "=r" (_res)                                  \
4242
         : /*in*/    "0" (&_argvec[0])                            \
4243
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4244
      );                                                          \
4245
      lval = (__typeof__(lval)) _res;                             \
4246
   } while (0)
4247
4248
#define CALL_FN_W_W(lval, orig, arg1)                             \
4249
   do {                                                           \
4250
      volatile OrigFn        _orig = (orig);                      \
4251
      volatile unsigned long _argvec[2];                          \
4252
      volatile unsigned long _res;                                \
4253
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4254
      _argvec[1] = (unsigned long)(arg1);                         \
4255
      __asm__ volatile(                                           \
4256
         VALGRIND_ALIGN_STACK                                     \
4257
         "ldr x0, [%1, #8] \n\t"                                  \
4258
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4259
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4260
         VALGRIND_RESTORE_STACK                                   \
4261
         "mov %0, x0\n"                                           \
4262
         : /*out*/   "=r" (_res)                                  \
4263
         : /*in*/    "0" (&_argvec[0])                            \
4264
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4265
      );                                                          \
4266
      lval = (__typeof__(lval)) _res;                             \
4267
   } while (0)
4268
4269
#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
4270
   do {                                                           \
4271
      volatile OrigFn        _orig = (orig);                      \
4272
      volatile unsigned long _argvec[3];                          \
4273
      volatile unsigned long _res;                                \
4274
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4275
      _argvec[1] = (unsigned long)(arg1);                         \
4276
      _argvec[2] = (unsigned long)(arg2);                         \
4277
      __asm__ volatile(                                           \
4278
         VALGRIND_ALIGN_STACK                                     \
4279
         "ldr x0, [%1, #8] \n\t"                                  \
4280
         "ldr x1, [%1, #16] \n\t"                                 \
4281
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4282
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4283
         VALGRIND_RESTORE_STACK                                   \
4284
         "mov %0, x0\n"                                           \
4285
         : /*out*/   "=r" (_res)                                  \
4286
         : /*in*/    "0" (&_argvec[0])                            \
4287
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4288
      );                                                          \
4289
      lval = (__typeof__(lval)) _res;                             \
4290
   } while (0)
4291
4292
#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
4293
   do {                                                           \
4294
      volatile OrigFn        _orig = (orig);                      \
4295
      volatile unsigned long _argvec[4];                          \
4296
      volatile unsigned long _res;                                \
4297
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4298
      _argvec[1] = (unsigned long)(arg1);                         \
4299
      _argvec[2] = (unsigned long)(arg2);                         \
4300
      _argvec[3] = (unsigned long)(arg3);                         \
4301
      __asm__ volatile(                                           \
4302
         VALGRIND_ALIGN_STACK                                     \
4303
         "ldr x0, [%1, #8] \n\t"                                  \
4304
         "ldr x1, [%1, #16] \n\t"                                 \
4305
         "ldr x2, [%1, #24] \n\t"                                 \
4306
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4307
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4308
         VALGRIND_RESTORE_STACK                                   \
4309
         "mov %0, x0\n"                                           \
4310
         : /*out*/   "=r" (_res)                                  \
4311
         : /*in*/    "0" (&_argvec[0])                            \
4312
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4313
      );                                                          \
4314
      lval = (__typeof__(lval)) _res;                             \
4315
   } while (0)
4316
4317
#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
4318
   do {                                                           \
4319
      volatile OrigFn        _orig = (orig);                      \
4320
      volatile unsigned long _argvec[5];                          \
4321
      volatile unsigned long _res;                                \
4322
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4323
      _argvec[1] = (unsigned long)(arg1);                         \
4324
      _argvec[2] = (unsigned long)(arg2);                         \
4325
      _argvec[3] = (unsigned long)(arg3);                         \
4326
      _argvec[4] = (unsigned long)(arg4);                         \
4327
      __asm__ volatile(                                           \
4328
         VALGRIND_ALIGN_STACK                                     \
4329
         "ldr x0, [%1, #8] \n\t"                                  \
4330
         "ldr x1, [%1, #16] \n\t"                                 \
4331
         "ldr x2, [%1, #24] \n\t"                                 \
4332
         "ldr x3, [%1, #32] \n\t"                                 \
4333
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4334
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4335
         VALGRIND_RESTORE_STACK                                   \
4336
         "mov %0, x0"                                             \
4337
         : /*out*/   "=r" (_res)                                  \
4338
         : /*in*/    "0" (&_argvec[0])                            \
4339
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4340
      );                                                          \
4341
      lval = (__typeof__(lval)) _res;                             \
4342
   } while (0)
4343
4344
#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
4345
   do {                                                           \
4346
      volatile OrigFn        _orig = (orig);                      \
4347
      volatile unsigned long _argvec[6];                          \
4348
      volatile unsigned long _res;                                \
4349
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4350
      _argvec[1] = (unsigned long)(arg1);                         \
4351
      _argvec[2] = (unsigned long)(arg2);                         \
4352
      _argvec[3] = (unsigned long)(arg3);                         \
4353
      _argvec[4] = (unsigned long)(arg4);                         \
4354
      _argvec[5] = (unsigned long)(arg5);                         \
4355
      __asm__ volatile(                                           \
4356
         VALGRIND_ALIGN_STACK                                     \
4357
         "ldr x0, [%1, #8] \n\t"                                  \
4358
         "ldr x1, [%1, #16] \n\t"                                 \
4359
         "ldr x2, [%1, #24] \n\t"                                 \
4360
         "ldr x3, [%1, #32] \n\t"                                 \
4361
         "ldr x4, [%1, #40] \n\t"                                 \
4362
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4363
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4364
         VALGRIND_RESTORE_STACK                                   \
4365
         "mov %0, x0"                                             \
4366
         : /*out*/   "=r" (_res)                                  \
4367
         : /*in*/    "0" (&_argvec[0])                            \
4368
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4369
      );                                                          \
4370
      lval = (__typeof__(lval)) _res;                             \
4371
   } while (0)
4372
4373
#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
4374
   do {                                                           \
4375
      volatile OrigFn        _orig = (orig);                      \
4376
      volatile unsigned long _argvec[7];                          \
4377
      volatile unsigned long _res;                                \
4378
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4379
      _argvec[1] = (unsigned long)(arg1);                         \
4380
      _argvec[2] = (unsigned long)(arg2);                         \
4381
      _argvec[3] = (unsigned long)(arg3);                         \
4382
      _argvec[4] = (unsigned long)(arg4);                         \
4383
      _argvec[5] = (unsigned long)(arg5);                         \
4384
      _argvec[6] = (unsigned long)(arg6);                         \
4385
      __asm__ volatile(                                           \
4386
         VALGRIND_ALIGN_STACK                                     \
4387
         "ldr x0, [%1, #8] \n\t"                                  \
4388
         "ldr x1, [%1, #16] \n\t"                                 \
4389
         "ldr x2, [%1, #24] \n\t"                                 \
4390
         "ldr x3, [%1, #32] \n\t"                                 \
4391
         "ldr x4, [%1, #40] \n\t"                                 \
4392
         "ldr x5, [%1, #48] \n\t"                                 \
4393
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4394
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4395
         VALGRIND_RESTORE_STACK                                   \
4396
         "mov %0, x0"                                             \
4397
         : /*out*/   "=r" (_res)                                  \
4398
         : /*in*/    "0" (&_argvec[0])                            \
4399
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4400
      );                                                          \
4401
      lval = (__typeof__(lval)) _res;                             \
4402
   } while (0)
4403
4404
#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4405
                                 arg7)                            \
4406
   do {                                                           \
4407
      volatile OrigFn        _orig = (orig);                      \
4408
      volatile unsigned long _argvec[8];                          \
4409
      volatile unsigned long _res;                                \
4410
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4411
      _argvec[1] = (unsigned long)(arg1);                         \
4412
      _argvec[2] = (unsigned long)(arg2);                         \
4413
      _argvec[3] = (unsigned long)(arg3);                         \
4414
      _argvec[4] = (unsigned long)(arg4);                         \
4415
      _argvec[5] = (unsigned long)(arg5);                         \
4416
      _argvec[6] = (unsigned long)(arg6);                         \
4417
      _argvec[7] = (unsigned long)(arg7);                         \
4418
      __asm__ volatile(                                           \
4419
         VALGRIND_ALIGN_STACK                                     \
4420
         "ldr x0, [%1, #8] \n\t"                                  \
4421
         "ldr x1, [%1, #16] \n\t"                                 \
4422
         "ldr x2, [%1, #24] \n\t"                                 \
4423
         "ldr x3, [%1, #32] \n\t"                                 \
4424
         "ldr x4, [%1, #40] \n\t"                                 \
4425
         "ldr x5, [%1, #48] \n\t"                                 \
4426
         "ldr x6, [%1, #56] \n\t"                                 \
4427
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4428
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4429
         VALGRIND_RESTORE_STACK                                   \
4430
         "mov %0, x0"                                             \
4431
         : /*out*/   "=r" (_res)                                  \
4432
         : /*in*/    "0" (&_argvec[0])                            \
4433
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4434
      );                                                          \
4435
      lval = (__typeof__(lval)) _res;                             \
4436
   } while (0)
4437
4438
#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4439
                                 arg7,arg8)                       \
4440
   do {                                                           \
4441
      volatile OrigFn        _orig = (orig);                      \
4442
      volatile unsigned long _argvec[9];                          \
4443
      volatile unsigned long _res;                                \
4444
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4445
      _argvec[1] = (unsigned long)(arg1);                         \
4446
      _argvec[2] = (unsigned long)(arg2);                         \
4447
      _argvec[3] = (unsigned long)(arg3);                         \
4448
      _argvec[4] = (unsigned long)(arg4);                         \
4449
      _argvec[5] = (unsigned long)(arg5);                         \
4450
      _argvec[6] = (unsigned long)(arg6);                         \
4451
      _argvec[7] = (unsigned long)(arg7);                         \
4452
      _argvec[8] = (unsigned long)(arg8);                         \
4453
      __asm__ volatile(                                           \
4454
         VALGRIND_ALIGN_STACK                                     \
4455
         "ldr x0, [%1, #8] \n\t"                                  \
4456
         "ldr x1, [%1, #16] \n\t"                                 \
4457
         "ldr x2, [%1, #24] \n\t"                                 \
4458
         "ldr x3, [%1, #32] \n\t"                                 \
4459
         "ldr x4, [%1, #40] \n\t"                                 \
4460
         "ldr x5, [%1, #48] \n\t"                                 \
4461
         "ldr x6, [%1, #56] \n\t"                                 \
4462
         "ldr x7, [%1, #64] \n\t"                                 \
4463
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4464
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4465
         VALGRIND_RESTORE_STACK                                   \
4466
         "mov %0, x0"                                             \
4467
         : /*out*/   "=r" (_res)                                  \
4468
         : /*in*/    "0" (&_argvec[0])                            \
4469
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4470
      );                                                          \
4471
      lval = (__typeof__(lval)) _res;                             \
4472
   } while (0)
4473
4474
#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
4475
                                 arg7,arg8,arg9)                  \
4476
   do {                                                           \
4477
      volatile OrigFn        _orig = (orig);                      \
4478
      volatile unsigned long _argvec[10];                         \
4479
      volatile unsigned long _res;                                \
4480
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4481
      _argvec[1] = (unsigned long)(arg1);                         \
4482
      _argvec[2] = (unsigned long)(arg2);                         \
4483
      _argvec[3] = (unsigned long)(arg3);                         \
4484
      _argvec[4] = (unsigned long)(arg4);                         \
4485
      _argvec[5] = (unsigned long)(arg5);                         \
4486
      _argvec[6] = (unsigned long)(arg6);                         \
4487
      _argvec[7] = (unsigned long)(arg7);                         \
4488
      _argvec[8] = (unsigned long)(arg8);                         \
4489
      _argvec[9] = (unsigned long)(arg9);                         \
4490
      __asm__ volatile(                                           \
4491
         VALGRIND_ALIGN_STACK                                     \
4492
         "sub sp, sp, #0x20 \n\t"                                 \
4493
         "ldr x0, [%1, #8] \n\t"                                  \
4494
         "ldr x1, [%1, #16] \n\t"                                 \
4495
         "ldr x2, [%1, #24] \n\t"                                 \
4496
         "ldr x3, [%1, #32] \n\t"                                 \
4497
         "ldr x4, [%1, #40] \n\t"                                 \
4498
         "ldr x5, [%1, #48] \n\t"                                 \
4499
         "ldr x6, [%1, #56] \n\t"                                 \
4500
         "ldr x7, [%1, #64] \n\t"                                 \
4501
         "ldr x8, [%1, #72] \n\t"                                 \
4502
         "str x8, [sp, #0]  \n\t"                                 \
4503
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4504
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4505
         VALGRIND_RESTORE_STACK                                   \
4506
         "mov %0, x0"                                             \
4507
         : /*out*/   "=r" (_res)                                  \
4508
         : /*in*/    "0" (&_argvec[0])                            \
4509
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4510
      );                                                          \
4511
      lval = (__typeof__(lval)) _res;                             \
4512
   } while (0)
4513
4514
#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4515
                                  arg7,arg8,arg9,arg10)           \
4516
   do {                                                           \
4517
      volatile OrigFn        _orig = (orig);                      \
4518
      volatile unsigned long _argvec[11];                         \
4519
      volatile unsigned long _res;                                \
4520
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4521
      _argvec[1] = (unsigned long)(arg1);                         \
4522
      _argvec[2] = (unsigned long)(arg2);                         \
4523
      _argvec[3] = (unsigned long)(arg3);                         \
4524
      _argvec[4] = (unsigned long)(arg4);                         \
4525
      _argvec[5] = (unsigned long)(arg5);                         \
4526
      _argvec[6] = (unsigned long)(arg6);                         \
4527
      _argvec[7] = (unsigned long)(arg7);                         \
4528
      _argvec[8] = (unsigned long)(arg8);                         \
4529
      _argvec[9] = (unsigned long)(arg9);                         \
4530
      _argvec[10] = (unsigned long)(arg10);                       \
4531
      __asm__ volatile(                                           \
4532
         VALGRIND_ALIGN_STACK                                     \
4533
         "sub sp, sp, #0x20 \n\t"                                 \
4534
         "ldr x0, [%1, #8] \n\t"                                  \
4535
         "ldr x1, [%1, #16] \n\t"                                 \
4536
         "ldr x2, [%1, #24] \n\t"                                 \
4537
         "ldr x3, [%1, #32] \n\t"                                 \
4538
         "ldr x4, [%1, #40] \n\t"                                 \
4539
         "ldr x5, [%1, #48] \n\t"                                 \
4540
         "ldr x6, [%1, #56] \n\t"                                 \
4541
         "ldr x7, [%1, #64] \n\t"                                 \
4542
         "ldr x8, [%1, #72] \n\t"                                 \
4543
         "str x8, [sp, #0]  \n\t"                                 \
4544
         "ldr x8, [%1, #80] \n\t"                                 \
4545
         "str x8, [sp, #8]  \n\t"                                 \
4546
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4547
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4548
         VALGRIND_RESTORE_STACK                                   \
4549
         "mov %0, x0"                                             \
4550
         : /*out*/   "=r" (_res)                                  \
4551
         : /*in*/    "0" (&_argvec[0])                            \
4552
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4553
      );                                                          \
4554
      lval = (__typeof__(lval)) _res;                             \
4555
   } while (0)
4556
4557
#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4558
                                  arg7,arg8,arg9,arg10,arg11)     \
4559
   do {                                                           \
4560
      volatile OrigFn        _orig = (orig);                      \
4561
      volatile unsigned long _argvec[12];                         \
4562
      volatile unsigned long _res;                                \
4563
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4564
      _argvec[1] = (unsigned long)(arg1);                         \
4565
      _argvec[2] = (unsigned long)(arg2);                         \
4566
      _argvec[3] = (unsigned long)(arg3);                         \
4567
      _argvec[4] = (unsigned long)(arg4);                         \
4568
      _argvec[5] = (unsigned long)(arg5);                         \
4569
      _argvec[6] = (unsigned long)(arg6);                         \
4570
      _argvec[7] = (unsigned long)(arg7);                         \
4571
      _argvec[8] = (unsigned long)(arg8);                         \
4572
      _argvec[9] = (unsigned long)(arg9);                         \
4573
      _argvec[10] = (unsigned long)(arg10);                       \
4574
      _argvec[11] = (unsigned long)(arg11);                       \
4575
      __asm__ volatile(                                           \
4576
         VALGRIND_ALIGN_STACK                                     \
4577
         "sub sp, sp, #0x30 \n\t"                                 \
4578
         "ldr x0, [%1, #8] \n\t"                                  \
4579
         "ldr x1, [%1, #16] \n\t"                                 \
4580
         "ldr x2, [%1, #24] \n\t"                                 \
4581
         "ldr x3, [%1, #32] \n\t"                                 \
4582
         "ldr x4, [%1, #40] \n\t"                                 \
4583
         "ldr x5, [%1, #48] \n\t"                                 \
4584
         "ldr x6, [%1, #56] \n\t"                                 \
4585
         "ldr x7, [%1, #64] \n\t"                                 \
4586
         "ldr x8, [%1, #72] \n\t"                                 \
4587
         "str x8, [sp, #0]  \n\t"                                 \
4588
         "ldr x8, [%1, #80] \n\t"                                 \
4589
         "str x8, [sp, #8]  \n\t"                                 \
4590
         "ldr x8, [%1, #88] \n\t"                                 \
4591
         "str x8, [sp, #16] \n\t"                                 \
4592
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4593
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4594
         VALGRIND_RESTORE_STACK                                   \
4595
         "mov %0, x0"                                             \
4596
         : /*out*/   "=r" (_res)                                  \
4597
         : /*in*/    "0" (&_argvec[0])                            \
4598
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4599
      );                                                          \
4600
      lval = (__typeof__(lval)) _res;                             \
4601
   } while (0)
4602
4603
#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
4604
                                  arg7,arg8,arg9,arg10,arg11,     \
4605
                                  arg12)                          \
4606
   do {                                                           \
4607
      volatile OrigFn        _orig = (orig);                      \
4608
      volatile unsigned long _argvec[13];                         \
4609
      volatile unsigned long _res;                                \
4610
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
4611
      _argvec[1] = (unsigned long)(arg1);                         \
4612
      _argvec[2] = (unsigned long)(arg2);                         \
4613
      _argvec[3] = (unsigned long)(arg3);                         \
4614
      _argvec[4] = (unsigned long)(arg4);                         \
4615
      _argvec[5] = (unsigned long)(arg5);                         \
4616
      _argvec[6] = (unsigned long)(arg6);                         \
4617
      _argvec[7] = (unsigned long)(arg7);                         \
4618
      _argvec[8] = (unsigned long)(arg8);                         \
4619
      _argvec[9] = (unsigned long)(arg9);                         \
4620
      _argvec[10] = (unsigned long)(arg10);                       \
4621
      _argvec[11] = (unsigned long)(arg11);                       \
4622
      _argvec[12] = (unsigned long)(arg12);                       \
4623
      __asm__ volatile(                                           \
4624
         VALGRIND_ALIGN_STACK                                     \
4625
         "sub sp, sp, #0x30 \n\t"                                 \
4626
         "ldr x0, [%1, #8] \n\t"                                  \
4627
         "ldr x1, [%1, #16] \n\t"                                 \
4628
         "ldr x2, [%1, #24] \n\t"                                 \
4629
         "ldr x3, [%1, #32] \n\t"                                 \
4630
         "ldr x4, [%1, #40] \n\t"                                 \
4631
         "ldr x5, [%1, #48] \n\t"                                 \
4632
         "ldr x6, [%1, #56] \n\t"                                 \
4633
         "ldr x7, [%1, #64] \n\t"                                 \
4634
         "ldr x8, [%1, #72] \n\t"                                 \
4635
         "str x8, [sp, #0]  \n\t"                                 \
4636
         "ldr x8, [%1, #80] \n\t"                                 \
4637
         "str x8, [sp, #8]  \n\t"                                 \
4638
         "ldr x8, [%1, #88] \n\t"                                 \
4639
         "str x8, [sp, #16] \n\t"                                 \
4640
         "ldr x8, [%1, #96] \n\t"                                 \
4641
         "str x8, [sp, #24] \n\t"                                 \
4642
         "ldr x8, [%1] \n\t"  /* target->x8 */                    \
4643
         VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8                   \
4644
         VALGRIND_RESTORE_STACK                                   \
4645
         "mov %0, x0"                                             \
4646
         : /*out*/   "=r" (_res)                                  \
4647
         : /*in*/    "0" (&_argvec[0])                            \
4648
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21"   \
4649
      );                                                          \
4650
      lval = (__typeof__(lval)) _res;                             \
4651
   } while (0)
4652
4653
#endif /* PLAT_arm64_linux */
4654
4655
/* ------------------------- s390x-linux ------------------------- */
4656
4657
#if defined(PLAT_s390x_linux)
4658
4659
/* Similar workaround as amd64 (see above), but we use r11 as frame
4660
   pointer and save the old r11 in r7. r11 might be used for
4661
   argvec, therefore we copy argvec in r1 since r1 is clobbered
4662
   after the call anyway.  */
4663
#if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)
4664
#  define __FRAME_POINTER                                         \
4665
      ,"d"(__builtin_dwarf_cfa())
4666
#  define VALGRIND_CFI_PROLOGUE                                   \
4667
      ".cfi_remember_state\n\t"                                   \
4668
      "lgr 1,%1\n\t" /* copy the argvec pointer in r1 */          \
4669
      "lgr 7,11\n\t"                                              \
4670
      "lgr 11,%2\n\t"                                             \
4671
      ".cfi_def_cfa r11, 0\n\t"
4672
#  define VALGRIND_CFI_EPILOGUE                                   \
4673
      "lgr 11, 7\n\t"                                             \
4674
      ".cfi_restore_state\n\t"
4675
#else
4676
#  define __FRAME_POINTER
4677
#  define VALGRIND_CFI_PROLOGUE                                   \
4678
      "lgr 1,%1\n\t"
4679
#  define VALGRIND_CFI_EPILOGUE
4680
#endif
4681
4682
/* Nb: On s390 the stack pointer is properly aligned *at all times*
4683
   according to the s390 GCC maintainer. (The ABI specification is not
4684
   precise in this regard.) Therefore, VALGRIND_ALIGN_STACK and
4685
   VALGRIND_RESTORE_STACK are not defined here. */
4686
4687
/* These regs are trashed by the hidden call. Note that we overwrite
4688
   r14 in s390_irgen_noredir (VEX/priv/guest_s390_irgen.c) to give the
4689
   function a proper return address. All others are ABI defined call
4690
   clobbers. */
4691
#define __CALLER_SAVED_REGS "0","1","2","3","4","5","14", \
4692
                           "f0","f1","f2","f3","f4","f5","f6","f7"
4693
4694
/* Nb: Although r11 is modified in the asm snippets below (inside 
4695
   VALGRIND_CFI_PROLOGUE) it is not listed in the clobber section, for
4696
   two reasons:
4697
   (1) r11 is restored in VALGRIND_CFI_EPILOGUE, so effectively it is not
4698
       modified
4699
   (2) GCC will complain that r11 cannot appear inside a clobber section,
4700
       when compiled with -O -fno-omit-frame-pointer
4701
 */
4702
4703
#define CALL_FN_W_v(lval, orig)                                  \
4704
   do {                                                          \
4705
      volatile OrigFn        _orig = (orig);                     \
4706
      volatile unsigned long  _argvec[1];                        \
4707
      volatile unsigned long _res;                               \
4708
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
4709
      __asm__ volatile(                                          \
4710
         VALGRIND_CFI_PROLOGUE                                   \
4711
         "aghi 15,-160\n\t"                                      \
4712
         "lg 1, 0(1)\n\t"  /* target->r1 */                      \
4713
         VALGRIND_CALL_NOREDIR_R1                                \
4714
         "lgr %0, 2\n\t"                                         \
4715
         "aghi 15,160\n\t"                                       \
4716
         VALGRIND_CFI_EPILOGUE                                   \
4717
         : /*out*/   "=d" (_res)                                 \
4718
         : /*in*/    "d" (&_argvec[0]) __FRAME_POINTER           \
4719
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4720
      );                                                         \
4721
      lval = (__typeof__(lval)) _res;                            \
4722
   } while (0)
4723
4724
/* The call abi has the arguments in r2-r6 and stack */
4725
#define CALL_FN_W_W(lval, orig, arg1)                            \
4726
   do {                                                          \
4727
      volatile OrigFn        _orig = (orig);                     \
4728
      volatile unsigned long _argvec[2];                         \
4729
      volatile unsigned long _res;                               \
4730
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
4731
      _argvec[1] = (unsigned long)arg1;                          \
4732
      __asm__ volatile(                                          \
4733
         VALGRIND_CFI_PROLOGUE                                   \
4734
         "aghi 15,-160\n\t"                                      \
4735
         "lg 2, 8(1)\n\t"                                        \
4736
         "lg 1, 0(1)\n\t"                                        \
4737
         VALGRIND_CALL_NOREDIR_R1                                \
4738
         "lgr %0, 2\n\t"                                         \
4739
         "aghi 15,160\n\t"                                       \
4740
         VALGRIND_CFI_EPILOGUE                                   \
4741
         : /*out*/   "=d" (_res)                                 \
4742
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4743
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4744
      );                                                         \
4745
      lval = (__typeof__(lval)) _res;                            \
4746
   } while (0)
4747
4748
#define CALL_FN_W_WW(lval, orig, arg1, arg2)                     \
4749
   do {                                                          \
4750
      volatile OrigFn        _orig = (orig);                     \
4751
      volatile unsigned long _argvec[3];                         \
4752
      volatile unsigned long _res;                               \
4753
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
4754
      _argvec[1] = (unsigned long)arg1;                          \
4755
      _argvec[2] = (unsigned long)arg2;                          \
4756
      __asm__ volatile(                                          \
4757
         VALGRIND_CFI_PROLOGUE                                   \
4758
         "aghi 15,-160\n\t"                                      \
4759
         "lg 2, 8(1)\n\t"                                        \
4760
         "lg 3,16(1)\n\t"                                        \
4761
         "lg 1, 0(1)\n\t"                                        \
4762
         VALGRIND_CALL_NOREDIR_R1                                \
4763
         "lgr %0, 2\n\t"                                         \
4764
         "aghi 15,160\n\t"                                       \
4765
         VALGRIND_CFI_EPILOGUE                                   \
4766
         : /*out*/   "=d" (_res)                                 \
4767
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4768
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4769
      );                                                         \
4770
      lval = (__typeof__(lval)) _res;                            \
4771
   } while (0)
4772
4773
#define CALL_FN_W_WWW(lval, orig, arg1, arg2, arg3)              \
4774
   do {                                                          \
4775
      volatile OrigFn        _orig = (orig);                     \
4776
      volatile unsigned long _argvec[4];                         \
4777
      volatile unsigned long _res;                               \
4778
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
4779
      _argvec[1] = (unsigned long)arg1;                          \
4780
      _argvec[2] = (unsigned long)arg2;                          \
4781
      _argvec[3] = (unsigned long)arg3;                          \
4782
      __asm__ volatile(                                          \
4783
         VALGRIND_CFI_PROLOGUE                                   \
4784
         "aghi 15,-160\n\t"                                      \
4785
         "lg 2, 8(1)\n\t"                                        \
4786
         "lg 3,16(1)\n\t"                                        \
4787
         "lg 4,24(1)\n\t"                                        \
4788
         "lg 1, 0(1)\n\t"                                        \
4789
         VALGRIND_CALL_NOREDIR_R1                                \
4790
         "lgr %0, 2\n\t"                                         \
4791
         "aghi 15,160\n\t"                                       \
4792
         VALGRIND_CFI_EPILOGUE                                   \
4793
         : /*out*/   "=d" (_res)                                 \
4794
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4795
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4796
      );                                                         \
4797
      lval = (__typeof__(lval)) _res;                            \
4798
   } while (0)
4799
4800
#define CALL_FN_W_WWWW(lval, orig, arg1, arg2, arg3, arg4)       \
4801
   do {                                                          \
4802
      volatile OrigFn        _orig = (orig);                     \
4803
      volatile unsigned long _argvec[5];                         \
4804
      volatile unsigned long _res;                               \
4805
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
4806
      _argvec[1] = (unsigned long)arg1;                          \
4807
      _argvec[2] = (unsigned long)arg2;                          \
4808
      _argvec[3] = (unsigned long)arg3;                          \
4809
      _argvec[4] = (unsigned long)arg4;                          \
4810
      __asm__ volatile(                                          \
4811
         VALGRIND_CFI_PROLOGUE                                   \
4812
         "aghi 15,-160\n\t"                                      \
4813
         "lg 2, 8(1)\n\t"                                        \
4814
         "lg 3,16(1)\n\t"                                        \
4815
         "lg 4,24(1)\n\t"                                        \
4816
         "lg 5,32(1)\n\t"                                        \
4817
         "lg 1, 0(1)\n\t"                                        \
4818
         VALGRIND_CALL_NOREDIR_R1                                \
4819
         "lgr %0, 2\n\t"                                         \
4820
         "aghi 15,160\n\t"                                       \
4821
         VALGRIND_CFI_EPILOGUE                                   \
4822
         : /*out*/   "=d" (_res)                                 \
4823
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4824
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7"     \
4825
      );                                                         \
4826
      lval = (__typeof__(lval)) _res;                            \
4827
   } while (0)
4828
4829
#define CALL_FN_W_5W(lval, orig, arg1, arg2, arg3, arg4, arg5)   \
4830
   do {                                                          \
4831
      volatile OrigFn        _orig = (orig);                     \
4832
      volatile unsigned long _argvec[6];                         \
4833
      volatile unsigned long _res;                               \
4834
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
4835
      _argvec[1] = (unsigned long)arg1;                          \
4836
      _argvec[2] = (unsigned long)arg2;                          \
4837
      _argvec[3] = (unsigned long)arg3;                          \
4838
      _argvec[4] = (unsigned long)arg4;                          \
4839
      _argvec[5] = (unsigned long)arg5;                          \
4840
      __asm__ volatile(                                          \
4841
         VALGRIND_CFI_PROLOGUE                                   \
4842
         "aghi 15,-160\n\t"                                      \
4843
         "lg 2, 8(1)\n\t"                                        \
4844
         "lg 3,16(1)\n\t"                                        \
4845
         "lg 4,24(1)\n\t"                                        \
4846
         "lg 5,32(1)\n\t"                                        \
4847
         "lg 6,40(1)\n\t"                                        \
4848
         "lg 1, 0(1)\n\t"                                        \
4849
         VALGRIND_CALL_NOREDIR_R1                                \
4850
         "lgr %0, 2\n\t"                                         \
4851
         "aghi 15,160\n\t"                                       \
4852
         VALGRIND_CFI_EPILOGUE                                   \
4853
         : /*out*/   "=d" (_res)                                 \
4854
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4855
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4856
      );                                                         \
4857
      lval = (__typeof__(lval)) _res;                            \
4858
   } while (0)
4859
4860
#define CALL_FN_W_6W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4861
                     arg6)                                       \
4862
   do {                                                          \
4863
      volatile OrigFn        _orig = (orig);                     \
4864
      volatile unsigned long _argvec[7];                         \
4865
      volatile unsigned long _res;                               \
4866
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
4867
      _argvec[1] = (unsigned long)arg1;                          \
4868
      _argvec[2] = (unsigned long)arg2;                          \
4869
      _argvec[3] = (unsigned long)arg3;                          \
4870
      _argvec[4] = (unsigned long)arg4;                          \
4871
      _argvec[5] = (unsigned long)arg5;                          \
4872
      _argvec[6] = (unsigned long)arg6;                          \
4873
      __asm__ volatile(                                          \
4874
         VALGRIND_CFI_PROLOGUE                                   \
4875
         "aghi 15,-168\n\t"                                      \
4876
         "lg 2, 8(1)\n\t"                                        \
4877
         "lg 3,16(1)\n\t"                                        \
4878
         "lg 4,24(1)\n\t"                                        \
4879
         "lg 5,32(1)\n\t"                                        \
4880
         "lg 6,40(1)\n\t"                                        \
4881
         "mvc 160(8,15), 48(1)\n\t"                              \
4882
         "lg 1, 0(1)\n\t"                                        \
4883
         VALGRIND_CALL_NOREDIR_R1                                \
4884
         "lgr %0, 2\n\t"                                         \
4885
         "aghi 15,168\n\t"                                       \
4886
         VALGRIND_CFI_EPILOGUE                                   \
4887
         : /*out*/   "=d" (_res)                                 \
4888
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4889
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4890
      );                                                         \
4891
      lval = (__typeof__(lval)) _res;                            \
4892
   } while (0)
4893
4894
#define CALL_FN_W_7W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4895
                     arg6, arg7)                                 \
4896
   do {                                                          \
4897
      volatile OrigFn        _orig = (orig);                     \
4898
      volatile unsigned long _argvec[8];                         \
4899
      volatile unsigned long _res;                               \
4900
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
4901
      _argvec[1] = (unsigned long)arg1;                          \
4902
      _argvec[2] = (unsigned long)arg2;                          \
4903
      _argvec[3] = (unsigned long)arg3;                          \
4904
      _argvec[4] = (unsigned long)arg4;                          \
4905
      _argvec[5] = (unsigned long)arg5;                          \
4906
      _argvec[6] = (unsigned long)arg6;                          \
4907
      _argvec[7] = (unsigned long)arg7;                          \
4908
      __asm__ volatile(                                          \
4909
         VALGRIND_CFI_PROLOGUE                                   \
4910
         "aghi 15,-176\n\t"                                      \
4911
         "lg 2, 8(1)\n\t"                                        \
4912
         "lg 3,16(1)\n\t"                                        \
4913
         "lg 4,24(1)\n\t"                                        \
4914
         "lg 5,32(1)\n\t"                                        \
4915
         "lg 6,40(1)\n\t"                                        \
4916
         "mvc 160(8,15), 48(1)\n\t"                              \
4917
         "mvc 168(8,15), 56(1)\n\t"                              \
4918
         "lg 1, 0(1)\n\t"                                        \
4919
         VALGRIND_CALL_NOREDIR_R1                                \
4920
         "lgr %0, 2\n\t"                                         \
4921
         "aghi 15,176\n\t"                                       \
4922
         VALGRIND_CFI_EPILOGUE                                   \
4923
         : /*out*/   "=d" (_res)                                 \
4924
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4925
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4926
      );                                                         \
4927
      lval = (__typeof__(lval)) _res;                            \
4928
   } while (0)
4929
4930
#define CALL_FN_W_8W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4931
                     arg6, arg7 ,arg8)                           \
4932
   do {                                                          \
4933
      volatile OrigFn        _orig = (orig);                     \
4934
      volatile unsigned long _argvec[9];                         \
4935
      volatile unsigned long _res;                               \
4936
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
4937
      _argvec[1] = (unsigned long)arg1;                          \
4938
      _argvec[2] = (unsigned long)arg2;                          \
4939
      _argvec[3] = (unsigned long)arg3;                          \
4940
      _argvec[4] = (unsigned long)arg4;                          \
4941
      _argvec[5] = (unsigned long)arg5;                          \
4942
      _argvec[6] = (unsigned long)arg6;                          \
4943
      _argvec[7] = (unsigned long)arg7;                          \
4944
      _argvec[8] = (unsigned long)arg8;                          \
4945
      __asm__ volatile(                                          \
4946
         VALGRIND_CFI_PROLOGUE                                   \
4947
         "aghi 15,-184\n\t"                                      \
4948
         "lg 2, 8(1)\n\t"                                        \
4949
         "lg 3,16(1)\n\t"                                        \
4950
         "lg 4,24(1)\n\t"                                        \
4951
         "lg 5,32(1)\n\t"                                        \
4952
         "lg 6,40(1)\n\t"                                        \
4953
         "mvc 160(8,15), 48(1)\n\t"                              \
4954
         "mvc 168(8,15), 56(1)\n\t"                              \
4955
         "mvc 176(8,15), 64(1)\n\t"                              \
4956
         "lg 1, 0(1)\n\t"                                        \
4957
         VALGRIND_CALL_NOREDIR_R1                                \
4958
         "lgr %0, 2\n\t"                                         \
4959
         "aghi 15,184\n\t"                                       \
4960
         VALGRIND_CFI_EPILOGUE                                   \
4961
         : /*out*/   "=d" (_res)                                 \
4962
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
4963
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
4964
      );                                                         \
4965
      lval = (__typeof__(lval)) _res;                            \
4966
   } while (0)
4967
4968
#define CALL_FN_W_9W(lval, orig, arg1, arg2, arg3, arg4, arg5,   \
4969
                     arg6, arg7 ,arg8, arg9)                     \
4970
   do {                                                          \
4971
      volatile OrigFn        _orig = (orig);                     \
4972
      volatile unsigned long _argvec[10];                        \
4973
      volatile unsigned long _res;                               \
4974
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
4975
      _argvec[1] = (unsigned long)arg1;                          \
4976
      _argvec[2] = (unsigned long)arg2;                          \
4977
      _argvec[3] = (unsigned long)arg3;                          \
4978
      _argvec[4] = (unsigned long)arg4;                          \
4979
      _argvec[5] = (unsigned long)arg5;                          \
4980
      _argvec[6] = (unsigned long)arg6;                          \
4981
      _argvec[7] = (unsigned long)arg7;                          \
4982
      _argvec[8] = (unsigned long)arg8;                          \
4983
      _argvec[9] = (unsigned long)arg9;                          \
4984
      __asm__ volatile(                                          \
4985
         VALGRIND_CFI_PROLOGUE                                   \
4986
         "aghi 15,-192\n\t"                                      \
4987
         "lg 2, 8(1)\n\t"                                        \
4988
         "lg 3,16(1)\n\t"                                        \
4989
         "lg 4,24(1)\n\t"                                        \
4990
         "lg 5,32(1)\n\t"                                        \
4991
         "lg 6,40(1)\n\t"                                        \
4992
         "mvc 160(8,15), 48(1)\n\t"                              \
4993
         "mvc 168(8,15), 56(1)\n\t"                              \
4994
         "mvc 176(8,15), 64(1)\n\t"                              \
4995
         "mvc 184(8,15), 72(1)\n\t"                              \
4996
         "lg 1, 0(1)\n\t"                                        \
4997
         VALGRIND_CALL_NOREDIR_R1                                \
4998
         "lgr %0, 2\n\t"                                         \
4999
         "aghi 15,192\n\t"                                       \
5000
         VALGRIND_CFI_EPILOGUE                                   \
5001
         : /*out*/   "=d" (_res)                                 \
5002
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5003
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5004
      );                                                         \
5005
      lval = (__typeof__(lval)) _res;                            \
5006
   } while (0)
5007
5008
#define CALL_FN_W_10W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
5009
                     arg6, arg7 ,arg8, arg9, arg10)              \
5010
   do {                                                          \
5011
      volatile OrigFn        _orig = (orig);                     \
5012
      volatile unsigned long _argvec[11];                        \
5013
      volatile unsigned long _res;                               \
5014
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
5015
      _argvec[1] = (unsigned long)arg1;                          \
5016
      _argvec[2] = (unsigned long)arg2;                          \
5017
      _argvec[3] = (unsigned long)arg3;                          \
5018
      _argvec[4] = (unsigned long)arg4;                          \
5019
      _argvec[5] = (unsigned long)arg5;                          \
5020
      _argvec[6] = (unsigned long)arg6;                          \
5021
      _argvec[7] = (unsigned long)arg7;                          \
5022
      _argvec[8] = (unsigned long)arg8;                          \
5023
      _argvec[9] = (unsigned long)arg9;                          \
5024
      _argvec[10] = (unsigned long)arg10;                        \
5025
      __asm__ volatile(                                          \
5026
         VALGRIND_CFI_PROLOGUE                                   \
5027
         "aghi 15,-200\n\t"                                      \
5028
         "lg 2, 8(1)\n\t"                                        \
5029
         "lg 3,16(1)\n\t"                                        \
5030
         "lg 4,24(1)\n\t"                                        \
5031
         "lg 5,32(1)\n\t"                                        \
5032
         "lg 6,40(1)\n\t"                                        \
5033
         "mvc 160(8,15), 48(1)\n\t"                              \
5034
         "mvc 168(8,15), 56(1)\n\t"                              \
5035
         "mvc 176(8,15), 64(1)\n\t"                              \
5036
         "mvc 184(8,15), 72(1)\n\t"                              \
5037
         "mvc 192(8,15), 80(1)\n\t"                              \
5038
         "lg 1, 0(1)\n\t"                                        \
5039
         VALGRIND_CALL_NOREDIR_R1                                \
5040
         "lgr %0, 2\n\t"                                         \
5041
         "aghi 15,200\n\t"                                       \
5042
         VALGRIND_CFI_EPILOGUE                                   \
5043
         : /*out*/   "=d" (_res)                                 \
5044
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5045
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5046
      );                                                         \
5047
      lval = (__typeof__(lval)) _res;                            \
5048
   } while (0)
5049
5050
#define CALL_FN_W_11W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
5051
                     arg6, arg7 ,arg8, arg9, arg10, arg11)       \
5052
   do {                                                          \
5053
      volatile OrigFn        _orig = (orig);                     \
5054
      volatile unsigned long _argvec[12];                        \
5055
      volatile unsigned long _res;                               \
5056
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
5057
      _argvec[1] = (unsigned long)arg1;                          \
5058
      _argvec[2] = (unsigned long)arg2;                          \
5059
      _argvec[3] = (unsigned long)arg3;                          \
5060
      _argvec[4] = (unsigned long)arg4;                          \
5061
      _argvec[5] = (unsigned long)arg5;                          \
5062
      _argvec[6] = (unsigned long)arg6;                          \
5063
      _argvec[7] = (unsigned long)arg7;                          \
5064
      _argvec[8] = (unsigned long)arg8;                          \
5065
      _argvec[9] = (unsigned long)arg9;                          \
5066
      _argvec[10] = (unsigned long)arg10;                        \
5067
      _argvec[11] = (unsigned long)arg11;                        \
5068
      __asm__ volatile(                                          \
5069
         VALGRIND_CFI_PROLOGUE                                   \
5070
         "aghi 15,-208\n\t"                                      \
5071
         "lg 2, 8(1)\n\t"                                        \
5072
         "lg 3,16(1)\n\t"                                        \
5073
         "lg 4,24(1)\n\t"                                        \
5074
         "lg 5,32(1)\n\t"                                        \
5075
         "lg 6,40(1)\n\t"                                        \
5076
         "mvc 160(8,15), 48(1)\n\t"                              \
5077
         "mvc 168(8,15), 56(1)\n\t"                              \
5078
         "mvc 176(8,15), 64(1)\n\t"                              \
5079
         "mvc 184(8,15), 72(1)\n\t"                              \
5080
         "mvc 192(8,15), 80(1)\n\t"                              \
5081
         "mvc 200(8,15), 88(1)\n\t"                              \
5082
         "lg 1, 0(1)\n\t"                                        \
5083
         VALGRIND_CALL_NOREDIR_R1                                \
5084
         "lgr %0, 2\n\t"                                         \
5085
         "aghi 15,208\n\t"                                       \
5086
         VALGRIND_CFI_EPILOGUE                                   \
5087
         : /*out*/   "=d" (_res)                                 \
5088
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5089
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5090
      );                                                         \
5091
      lval = (__typeof__(lval)) _res;                            \
5092
   } while (0)
5093
5094
#define CALL_FN_W_12W(lval, orig, arg1, arg2, arg3, arg4, arg5,  \
5095
                     arg6, arg7 ,arg8, arg9, arg10, arg11, arg12)\
5096
   do {                                                          \
5097
      volatile OrigFn        _orig = (orig);                     \
5098
      volatile unsigned long _argvec[13];                        \
5099
      volatile unsigned long _res;                               \
5100
      _argvec[0] = (unsigned long)_orig.nraddr;                  \
5101
      _argvec[1] = (unsigned long)arg1;                          \
5102
      _argvec[2] = (unsigned long)arg2;                          \
5103
      _argvec[3] = (unsigned long)arg3;                          \
5104
      _argvec[4] = (unsigned long)arg4;                          \
5105
      _argvec[5] = (unsigned long)arg5;                          \
5106
      _argvec[6] = (unsigned long)arg6;                          \
5107
      _argvec[7] = (unsigned long)arg7;                          \
5108
      _argvec[8] = (unsigned long)arg8;                          \
5109
      _argvec[9] = (unsigned long)arg9;                          \
5110
      _argvec[10] = (unsigned long)arg10;                        \
5111
      _argvec[11] = (unsigned long)arg11;                        \
5112
      _argvec[12] = (unsigned long)arg12;                        \
5113
      __asm__ volatile(                                          \
5114
         VALGRIND_CFI_PROLOGUE                                   \
5115
         "aghi 15,-216\n\t"                                      \
5116
         "lg 2, 8(1)\n\t"                                        \
5117
         "lg 3,16(1)\n\t"                                        \
5118
         "lg 4,24(1)\n\t"                                        \
5119
         "lg 5,32(1)\n\t"                                        \
5120
         "lg 6,40(1)\n\t"                                        \
5121
         "mvc 160(8,15), 48(1)\n\t"                              \
5122
         "mvc 168(8,15), 56(1)\n\t"                              \
5123
         "mvc 176(8,15), 64(1)\n\t"                              \
5124
         "mvc 184(8,15), 72(1)\n\t"                              \
5125
         "mvc 192(8,15), 80(1)\n\t"                              \
5126
         "mvc 200(8,15), 88(1)\n\t"                              \
5127
         "mvc 208(8,15), 96(1)\n\t"                              \
5128
         "lg 1, 0(1)\n\t"                                        \
5129
         VALGRIND_CALL_NOREDIR_R1                                \
5130
         "lgr %0, 2\n\t"                                         \
5131
         "aghi 15,216\n\t"                                       \
5132
         VALGRIND_CFI_EPILOGUE                                   \
5133
         : /*out*/   "=d" (_res)                                 \
5134
         : /*in*/    "a" (&_argvec[0]) __FRAME_POINTER           \
5135
         : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \
5136
      );                                                         \
5137
      lval = (__typeof__(lval)) _res;                            \
5138
   } while (0)
5139
5140
5141
#endif /* PLAT_s390x_linux */
5142
5143
/* ------------------------- mips32-linux ----------------------- */
5144
 
5145
#if defined(PLAT_mips32_linux)
5146
5147
/* These regs are trashed by the hidden call. */
5148
#define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6",       \
5149
"$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \
5150
"$25", "$31"
5151
5152
/* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned
5153
   long) == 4. */
5154
5155
#define CALL_FN_W_v(lval, orig)                                   \
5156
   do {                                                           \
5157
      volatile OrigFn        _orig = (orig);                      \
5158
      volatile unsigned long _argvec[1];                          \
5159
      volatile unsigned long _res;                                \
5160
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5161
      __asm__ volatile(                                           \
5162
         "subu $29, $29, 8 \n\t"                                  \
5163
         "sw $28, 0($29) \n\t"                                    \
5164
         "sw $31, 4($29) \n\t"                                    \
5165
         "subu $29, $29, 16 \n\t"                                 \
5166
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5167
         VALGRIND_CALL_NOREDIR_T9                                 \
5168
         "addu $29, $29, 16\n\t"                                  \
5169
         "lw $28, 0($29) \n\t"                                    \
5170
         "lw $31, 4($29) \n\t"                                    \
5171
         "addu $29, $29, 8 \n\t"                                  \
5172
         "move %0, $2\n"                                          \
5173
         : /*out*/   "=r" (_res)                                  \
5174
         : /*in*/    "0" (&_argvec[0])                            \
5175
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5176
      );                                                          \
5177
      lval = (__typeof__(lval)) _res;                             \
5178
   } while (0)
5179
5180
#define CALL_FN_W_W(lval, orig, arg1)                             \
5181
   do {                                                           \
5182
      volatile OrigFn        _orig = (orig);                      \
5183
     volatile unsigned long _argvec[2];                           \
5184
      volatile unsigned long _res;                                \
5185
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5186
      _argvec[1] = (unsigned long)(arg1);                         \
5187
      __asm__ volatile(                                           \
5188
         "subu $29, $29, 8 \n\t"                                  \
5189
         "sw $28, 0($29) \n\t"                                    \
5190
         "sw $31, 4($29) \n\t"                                    \
5191
         "subu $29, $29, 16 \n\t"                                 \
5192
         "lw $4, 4(%1) \n\t"   /* arg1*/                          \
5193
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5194
         VALGRIND_CALL_NOREDIR_T9                                 \
5195
         "addu $29, $29, 16 \n\t"                                 \
5196
         "lw $28, 0($29) \n\t"                                    \
5197
         "lw $31, 4($29) \n\t"                                    \
5198
         "addu $29, $29, 8 \n\t"                                  \
5199
         "move %0, $2\n"                                          \
5200
         : /*out*/   "=r" (_res)                                  \
5201
         : /*in*/    "0" (&_argvec[0])                            \
5202
         : /*trash*/ "memory",  __CALLER_SAVED_REGS               \
5203
      );                                                          \
5204
      lval = (__typeof__(lval)) _res;                             \
5205
   } while (0)
5206
5207
#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
5208
   do {                                                           \
5209
      volatile OrigFn        _orig = (orig);                      \
5210
      volatile unsigned long _argvec[3];                          \
5211
      volatile unsigned long _res;                                \
5212
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5213
      _argvec[1] = (unsigned long)(arg1);                         \
5214
      _argvec[2] = (unsigned long)(arg2);                         \
5215
      __asm__ volatile(                                           \
5216
         "subu $29, $29, 8 \n\t"                                  \
5217
         "sw $28, 0($29) \n\t"                                    \
5218
         "sw $31, 4($29) \n\t"                                    \
5219
         "subu $29, $29, 16 \n\t"                                 \
5220
         "lw $4, 4(%1) \n\t"                                      \
5221
         "lw $5, 8(%1) \n\t"                                      \
5222
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5223
         VALGRIND_CALL_NOREDIR_T9                                 \
5224
         "addu $29, $29, 16 \n\t"                                 \
5225
         "lw $28, 0($29) \n\t"                                    \
5226
         "lw $31, 4($29) \n\t"                                    \
5227
         "addu $29, $29, 8 \n\t"                                  \
5228
         "move %0, $2\n"                                          \
5229
         : /*out*/   "=r" (_res)                                  \
5230
         : /*in*/    "0" (&_argvec[0])                            \
5231
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5232
      );                                                          \
5233
      lval = (__typeof__(lval)) _res;                             \
5234
   } while (0)
5235
5236
#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
5237
   do {                                                           \
5238
      volatile OrigFn        _orig = (orig);                      \
5239
      volatile unsigned long _argvec[4];                          \
5240
      volatile unsigned long _res;                                \
5241
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5242
      _argvec[1] = (unsigned long)(arg1);                         \
5243
      _argvec[2] = (unsigned long)(arg2);                         \
5244
      _argvec[3] = (unsigned long)(arg3);                         \
5245
      __asm__ volatile(                                           \
5246
         "subu $29, $29, 8 \n\t"                                  \
5247
         "sw $28, 0($29) \n\t"                                    \
5248
         "sw $31, 4($29) \n\t"                                    \
5249
         "subu $29, $29, 16 \n\t"                                 \
5250
         "lw $4, 4(%1) \n\t"                                      \
5251
         "lw $5, 8(%1) \n\t"                                      \
5252
         "lw $6, 12(%1) \n\t"                                     \
5253
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5254
         VALGRIND_CALL_NOREDIR_T9                                 \
5255
         "addu $29, $29, 16 \n\t"                                 \
5256
         "lw $28, 0($29) \n\t"                                    \
5257
         "lw $31, 4($29) \n\t"                                    \
5258
         "addu $29, $29, 8 \n\t"                                  \
5259
         "move %0, $2\n"                                          \
5260
         : /*out*/   "=r" (_res)                                  \
5261
         : /*in*/    "0" (&_argvec[0])                            \
5262
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5263
      );                                                          \
5264
      lval = (__typeof__(lval)) _res;                             \
5265
   } while (0)
5266
5267
#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
5268
   do {                                                           \
5269
      volatile OrigFn        _orig = (orig);                      \
5270
      volatile unsigned long _argvec[5];                          \
5271
      volatile unsigned long _res;                                \
5272
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5273
      _argvec[1] = (unsigned long)(arg1);                         \
5274
      _argvec[2] = (unsigned long)(arg2);                         \
5275
      _argvec[3] = (unsigned long)(arg3);                         \
5276
      _argvec[4] = (unsigned long)(arg4);                         \
5277
      __asm__ volatile(                                           \
5278
         "subu $29, $29, 8 \n\t"                                  \
5279
         "sw $28, 0($29) \n\t"                                    \
5280
         "sw $31, 4($29) \n\t"                                    \
5281
         "subu $29, $29, 16 \n\t"                                 \
5282
         "lw $4, 4(%1) \n\t"                                      \
5283
         "lw $5, 8(%1) \n\t"                                      \
5284
         "lw $6, 12(%1) \n\t"                                     \
5285
         "lw $7, 16(%1) \n\t"                                     \
5286
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5287
         VALGRIND_CALL_NOREDIR_T9                                 \
5288
         "addu $29, $29, 16 \n\t"                                 \
5289
         "lw $28, 0($29) \n\t"                                    \
5290
         "lw $31, 4($29) \n\t"                                    \
5291
         "addu $29, $29, 8 \n\t"                                  \
5292
         "move %0, $2\n"                                          \
5293
         : /*out*/   "=r" (_res)                                  \
5294
         : /*in*/    "0" (&_argvec[0])                            \
5295
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5296
      );                                                          \
5297
      lval = (__typeof__(lval)) _res;                             \
5298
   } while (0)
5299
5300
#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
5301
   do {                                                           \
5302
      volatile OrigFn        _orig = (orig);                      \
5303
      volatile unsigned long _argvec[6];                          \
5304
      volatile unsigned long _res;                                \
5305
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5306
      _argvec[1] = (unsigned long)(arg1);                         \
5307
      _argvec[2] = (unsigned long)(arg2);                         \
5308
      _argvec[3] = (unsigned long)(arg3);                         \
5309
      _argvec[4] = (unsigned long)(arg4);                         \
5310
      _argvec[5] = (unsigned long)(arg5);                         \
5311
      __asm__ volatile(                                           \
5312
         "subu $29, $29, 8 \n\t"                                  \
5313
         "sw $28, 0($29) \n\t"                                    \
5314
         "sw $31, 4($29) \n\t"                                    \
5315
         "lw $4, 20(%1) \n\t"                                     \
5316
         "subu $29, $29, 24\n\t"                                  \
5317
         "sw $4, 16($29) \n\t"                                    \
5318
         "lw $4, 4(%1) \n\t"                                      \
5319
         "lw $5, 8(%1) \n\t"                                      \
5320
         "lw $6, 12(%1) \n\t"                                     \
5321
         "lw $7, 16(%1) \n\t"                                     \
5322
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5323
         VALGRIND_CALL_NOREDIR_T9                                 \
5324
         "addu $29, $29, 24 \n\t"                                 \
5325
         "lw $28, 0($29) \n\t"                                    \
5326
         "lw $31, 4($29) \n\t"                                    \
5327
         "addu $29, $29, 8 \n\t"                                  \
5328
         "move %0, $2\n"                                          \
5329
         : /*out*/   "=r" (_res)                                  \
5330
         : /*in*/    "0" (&_argvec[0])                            \
5331
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5332
      );                                                          \
5333
      lval = (__typeof__(lval)) _res;                             \
5334
   } while (0)
5335
#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
5336
   do {                                                           \
5337
      volatile OrigFn        _orig = (orig);                      \
5338
      volatile unsigned long _argvec[7];                          \
5339
      volatile unsigned long _res;                                \
5340
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5341
      _argvec[1] = (unsigned long)(arg1);                         \
5342
      _argvec[2] = (unsigned long)(arg2);                         \
5343
      _argvec[3] = (unsigned long)(arg3);                         \
5344
      _argvec[4] = (unsigned long)(arg4);                         \
5345
      _argvec[5] = (unsigned long)(arg5);                         \
5346
      _argvec[6] = (unsigned long)(arg6);                         \
5347
      __asm__ volatile(                                           \
5348
         "subu $29, $29, 8 \n\t"                                  \
5349
         "sw $28, 0($29) \n\t"                                    \
5350
         "sw $31, 4($29) \n\t"                                    \
5351
         "lw $4, 20(%1) \n\t"                                     \
5352
         "subu $29, $29, 32\n\t"                                  \
5353
         "sw $4, 16($29) \n\t"                                    \
5354
         "lw $4, 24(%1) \n\t"                                     \
5355
         "nop\n\t"                                                \
5356
         "sw $4, 20($29) \n\t"                                    \
5357
         "lw $4, 4(%1) \n\t"                                      \
5358
         "lw $5, 8(%1) \n\t"                                      \
5359
         "lw $6, 12(%1) \n\t"                                     \
5360
         "lw $7, 16(%1) \n\t"                                     \
5361
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5362
         VALGRIND_CALL_NOREDIR_T9                                 \
5363
         "addu $29, $29, 32 \n\t"                                 \
5364
         "lw $28, 0($29) \n\t"                                    \
5365
         "lw $31, 4($29) \n\t"                                    \
5366
         "addu $29, $29, 8 \n\t"                                  \
5367
         "move %0, $2\n"                                          \
5368
         : /*out*/   "=r" (_res)                                  \
5369
         : /*in*/    "0" (&_argvec[0])                            \
5370
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5371
      );                                                          \
5372
      lval = (__typeof__(lval)) _res;                             \
5373
   } while (0)
5374
5375
#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5376
                                 arg7)                            \
5377
   do {                                                           \
5378
      volatile OrigFn        _orig = (orig);                      \
5379
      volatile unsigned long _argvec[8];                          \
5380
      volatile unsigned long _res;                                \
5381
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5382
      _argvec[1] = (unsigned long)(arg1);                         \
5383
      _argvec[2] = (unsigned long)(arg2);                         \
5384
      _argvec[3] = (unsigned long)(arg3);                         \
5385
      _argvec[4] = (unsigned long)(arg4);                         \
5386
      _argvec[5] = (unsigned long)(arg5);                         \
5387
      _argvec[6] = (unsigned long)(arg6);                         \
5388
      _argvec[7] = (unsigned long)(arg7);                         \
5389
      __asm__ volatile(                                           \
5390
         "subu $29, $29, 8 \n\t"                                  \
5391
         "sw $28, 0($29) \n\t"                                    \
5392
         "sw $31, 4($29) \n\t"                                    \
5393
         "lw $4, 20(%1) \n\t"                                     \
5394
         "subu $29, $29, 32\n\t"                                  \
5395
         "sw $4, 16($29) \n\t"                                    \
5396
         "lw $4, 24(%1) \n\t"                                     \
5397
         "sw $4, 20($29) \n\t"                                    \
5398
         "lw $4, 28(%1) \n\t"                                     \
5399
         "sw $4, 24($29) \n\t"                                    \
5400
         "lw $4, 4(%1) \n\t"                                      \
5401
         "lw $5, 8(%1) \n\t"                                      \
5402
         "lw $6, 12(%1) \n\t"                                     \
5403
         "lw $7, 16(%1) \n\t"                                     \
5404
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5405
         VALGRIND_CALL_NOREDIR_T9                                 \
5406
         "addu $29, $29, 32 \n\t"                                 \
5407
         "lw $28, 0($29) \n\t"                                    \
5408
         "lw $31, 4($29) \n\t"                                    \
5409
         "addu $29, $29, 8 \n\t"                                  \
5410
         "move %0, $2\n"                                          \
5411
         : /*out*/   "=r" (_res)                                  \
5412
         : /*in*/    "0" (&_argvec[0])                            \
5413
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5414
      );                                                          \
5415
      lval = (__typeof__(lval)) _res;                             \
5416
   } while (0)
5417
5418
#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5419
                                 arg7,arg8)                       \
5420
   do {                                                           \
5421
      volatile OrigFn        _orig = (orig);                      \
5422
      volatile unsigned long _argvec[9];                          \
5423
      volatile unsigned long _res;                                \
5424
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5425
      _argvec[1] = (unsigned long)(arg1);                         \
5426
      _argvec[2] = (unsigned long)(arg2);                         \
5427
      _argvec[3] = (unsigned long)(arg3);                         \
5428
      _argvec[4] = (unsigned long)(arg4);                         \
5429
      _argvec[5] = (unsigned long)(arg5);                         \
5430
      _argvec[6] = (unsigned long)(arg6);                         \
5431
      _argvec[7] = (unsigned long)(arg7);                         \
5432
      _argvec[8] = (unsigned long)(arg8);                         \
5433
      __asm__ volatile(                                           \
5434
         "subu $29, $29, 8 \n\t"                                  \
5435
         "sw $28, 0($29) \n\t"                                    \
5436
         "sw $31, 4($29) \n\t"                                    \
5437
         "lw $4, 20(%1) \n\t"                                     \
5438
         "subu $29, $29, 40\n\t"                                  \
5439
         "sw $4, 16($29) \n\t"                                    \
5440
         "lw $4, 24(%1) \n\t"                                     \
5441
         "sw $4, 20($29) \n\t"                                    \
5442
         "lw $4, 28(%1) \n\t"                                     \
5443
         "sw $4, 24($29) \n\t"                                    \
5444
         "lw $4, 32(%1) \n\t"                                     \
5445
         "sw $4, 28($29) \n\t"                                    \
5446
         "lw $4, 4(%1) \n\t"                                      \
5447
         "lw $5, 8(%1) \n\t"                                      \
5448
         "lw $6, 12(%1) \n\t"                                     \
5449
         "lw $7, 16(%1) \n\t"                                     \
5450
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5451
         VALGRIND_CALL_NOREDIR_T9                                 \
5452
         "addu $29, $29, 40 \n\t"                                 \
5453
         "lw $28, 0($29) \n\t"                                    \
5454
         "lw $31, 4($29) \n\t"                                    \
5455
         "addu $29, $29, 8 \n\t"                                  \
5456
         "move %0, $2\n"                                          \
5457
         : /*out*/   "=r" (_res)                                  \
5458
         : /*in*/    "0" (&_argvec[0])                            \
5459
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5460
      );                                                          \
5461
      lval = (__typeof__(lval)) _res;                             \
5462
   } while (0)
5463
5464
#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5465
                                 arg7,arg8,arg9)                  \
5466
   do {                                                           \
5467
      volatile OrigFn        _orig = (orig);                      \
5468
      volatile unsigned long _argvec[10];                         \
5469
      volatile unsigned long _res;                                \
5470
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5471
      _argvec[1] = (unsigned long)(arg1);                         \
5472
      _argvec[2] = (unsigned long)(arg2);                         \
5473
      _argvec[3] = (unsigned long)(arg3);                         \
5474
      _argvec[4] = (unsigned long)(arg4);                         \
5475
      _argvec[5] = (unsigned long)(arg5);                         \
5476
      _argvec[6] = (unsigned long)(arg6);                         \
5477
      _argvec[7] = (unsigned long)(arg7);                         \
5478
      _argvec[8] = (unsigned long)(arg8);                         \
5479
      _argvec[9] = (unsigned long)(arg9);                         \
5480
      __asm__ volatile(                                           \
5481
         "subu $29, $29, 8 \n\t"                                  \
5482
         "sw $28, 0($29) \n\t"                                    \
5483
         "sw $31, 4($29) \n\t"                                    \
5484
         "lw $4, 20(%1) \n\t"                                     \
5485
         "subu $29, $29, 40\n\t"                                  \
5486
         "sw $4, 16($29) \n\t"                                    \
5487
         "lw $4, 24(%1) \n\t"                                     \
5488
         "sw $4, 20($29) \n\t"                                    \
5489
         "lw $4, 28(%1) \n\t"                                     \
5490
         "sw $4, 24($29) \n\t"                                    \
5491
         "lw $4, 32(%1) \n\t"                                     \
5492
         "sw $4, 28($29) \n\t"                                    \
5493
         "lw $4, 36(%1) \n\t"                                     \
5494
         "sw $4, 32($29) \n\t"                                    \
5495
         "lw $4, 4(%1) \n\t"                                      \
5496
         "lw $5, 8(%1) \n\t"                                      \
5497
         "lw $6, 12(%1) \n\t"                                     \
5498
         "lw $7, 16(%1) \n\t"                                     \
5499
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5500
         VALGRIND_CALL_NOREDIR_T9                                 \
5501
         "addu $29, $29, 40 \n\t"                                 \
5502
         "lw $28, 0($29) \n\t"                                    \
5503
         "lw $31, 4($29) \n\t"                                    \
5504
         "addu $29, $29, 8 \n\t"                                  \
5505
         "move %0, $2\n"                                          \
5506
         : /*out*/   "=r" (_res)                                  \
5507
         : /*in*/    "0" (&_argvec[0])                            \
5508
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5509
      );                                                          \
5510
      lval = (__typeof__(lval)) _res;                             \
5511
   } while (0)
5512
5513
#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
5514
                                  arg7,arg8,arg9,arg10)           \
5515
   do {                                                           \
5516
      volatile OrigFn        _orig = (orig);                      \
5517
      volatile unsigned long _argvec[11];                         \
5518
      volatile unsigned long _res;                                \
5519
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5520
      _argvec[1] = (unsigned long)(arg1);                         \
5521
      _argvec[2] = (unsigned long)(arg2);                         \
5522
      _argvec[3] = (unsigned long)(arg3);                         \
5523
      _argvec[4] = (unsigned long)(arg4);                         \
5524
      _argvec[5] = (unsigned long)(arg5);                         \
5525
      _argvec[6] = (unsigned long)(arg6);                         \
5526
      _argvec[7] = (unsigned long)(arg7);                         \
5527
      _argvec[8] = (unsigned long)(arg8);                         \
5528
      _argvec[9] = (unsigned long)(arg9);                         \
5529
      _argvec[10] = (unsigned long)(arg10);                       \
5530
      __asm__ volatile(                                           \
5531
         "subu $29, $29, 8 \n\t"                                  \
5532
         "sw $28, 0($29) \n\t"                                    \
5533
         "sw $31, 4($29) \n\t"                                    \
5534
         "lw $4, 20(%1) \n\t"                                     \
5535
         "subu $29, $29, 48\n\t"                                  \
5536
         "sw $4, 16($29) \n\t"                                    \
5537
         "lw $4, 24(%1) \n\t"                                     \
5538
         "sw $4, 20($29) \n\t"                                    \
5539
         "lw $4, 28(%1) \n\t"                                     \
5540
         "sw $4, 24($29) \n\t"                                    \
5541
         "lw $4, 32(%1) \n\t"                                     \
5542
         "sw $4, 28($29) \n\t"                                    \
5543
         "lw $4, 36(%1) \n\t"                                     \
5544
         "sw $4, 32($29) \n\t"                                    \
5545
         "lw $4, 40(%1) \n\t"                                     \
5546
         "sw $4, 36($29) \n\t"                                    \
5547
         "lw $4, 4(%1) \n\t"                                      \
5548
         "lw $5, 8(%1) \n\t"                                      \
5549
         "lw $6, 12(%1) \n\t"                                     \
5550
         "lw $7, 16(%1) \n\t"                                     \
5551
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5552
         VALGRIND_CALL_NOREDIR_T9                                 \
5553
         "addu $29, $29, 48 \n\t"                                 \
5554
         "lw $28, 0($29) \n\t"                                    \
5555
         "lw $31, 4($29) \n\t"                                    \
5556
         "addu $29, $29, 8 \n\t"                                  \
5557
         "move %0, $2\n"                                          \
5558
         : /*out*/   "=r" (_res)                                  \
5559
         : /*in*/    "0" (&_argvec[0])                            \
5560
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5561
      );                                                          \
5562
      lval = (__typeof__(lval)) _res;                             \
5563
   } while (0)
5564
5565
#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
5566
                                  arg6,arg7,arg8,arg9,arg10,      \
5567
                                  arg11)                          \
5568
   do {                                                           \
5569
      volatile OrigFn        _orig = (orig);                      \
5570
      volatile unsigned long _argvec[12];                         \
5571
      volatile unsigned long _res;                                \
5572
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5573
      _argvec[1] = (unsigned long)(arg1);                         \
5574
      _argvec[2] = (unsigned long)(arg2);                         \
5575
      _argvec[3] = (unsigned long)(arg3);                         \
5576
      _argvec[4] = (unsigned long)(arg4);                         \
5577
      _argvec[5] = (unsigned long)(arg5);                         \
5578
      _argvec[6] = (unsigned long)(arg6);                         \
5579
      _argvec[7] = (unsigned long)(arg7);                         \
5580
      _argvec[8] = (unsigned long)(arg8);                         \
5581
      _argvec[9] = (unsigned long)(arg9);                         \
5582
      _argvec[10] = (unsigned long)(arg10);                       \
5583
      _argvec[11] = (unsigned long)(arg11);                       \
5584
      __asm__ volatile(                                           \
5585
         "subu $29, $29, 8 \n\t"                                  \
5586
         "sw $28, 0($29) \n\t"                                    \
5587
         "sw $31, 4($29) \n\t"                                    \
5588
         "lw $4, 20(%1) \n\t"                                     \
5589
         "subu $29, $29, 48\n\t"                                  \
5590
         "sw $4, 16($29) \n\t"                                    \
5591
         "lw $4, 24(%1) \n\t"                                     \
5592
         "sw $4, 20($29) \n\t"                                    \
5593
         "lw $4, 28(%1) \n\t"                                     \
5594
         "sw $4, 24($29) \n\t"                                    \
5595
         "lw $4, 32(%1) \n\t"                                     \
5596
         "sw $4, 28($29) \n\t"                                    \
5597
         "lw $4, 36(%1) \n\t"                                     \
5598
         "sw $4, 32($29) \n\t"                                    \
5599
         "lw $4, 40(%1) \n\t"                                     \
5600
         "sw $4, 36($29) \n\t"                                    \
5601
         "lw $4, 44(%1) \n\t"                                     \
5602
         "sw $4, 40($29) \n\t"                                    \
5603
         "lw $4, 4(%1) \n\t"                                      \
5604
         "lw $5, 8(%1) \n\t"                                      \
5605
         "lw $6, 12(%1) \n\t"                                     \
5606
         "lw $7, 16(%1) \n\t"                                     \
5607
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5608
         VALGRIND_CALL_NOREDIR_T9                                 \
5609
         "addu $29, $29, 48 \n\t"                                 \
5610
         "lw $28, 0($29) \n\t"                                    \
5611
         "lw $31, 4($29) \n\t"                                    \
5612
         "addu $29, $29, 8 \n\t"                                  \
5613
         "move %0, $2\n"                                          \
5614
         : /*out*/   "=r" (_res)                                  \
5615
         : /*in*/    "0" (&_argvec[0])                            \
5616
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5617
      );                                                          \
5618
      lval = (__typeof__(lval)) _res;                             \
5619
   } while (0)
5620
5621
#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
5622
                                  arg6,arg7,arg8,arg9,arg10,      \
5623
                                  arg11,arg12)                    \
5624
   do {                                                           \
5625
      volatile OrigFn        _orig = (orig);                      \
5626
      volatile unsigned long _argvec[13];                         \
5627
      volatile unsigned long _res;                                \
5628
      _argvec[0] = (unsigned long)_orig.nraddr;                   \
5629
      _argvec[1] = (unsigned long)(arg1);                         \
5630
      _argvec[2] = (unsigned long)(arg2);                         \
5631
      _argvec[3] = (unsigned long)(arg3);                         \
5632
      _argvec[4] = (unsigned long)(arg4);                         \
5633
      _argvec[5] = (unsigned long)(arg5);                         \
5634
      _argvec[6] = (unsigned long)(arg6);                         \
5635
      _argvec[7] = (unsigned long)(arg7);                         \
5636
      _argvec[8] = (unsigned long)(arg8);                         \
5637
      _argvec[9] = (unsigned long)(arg9);                         \
5638
      _argvec[10] = (unsigned long)(arg10);                       \
5639
      _argvec[11] = (unsigned long)(arg11);                       \
5640
      _argvec[12] = (unsigned long)(arg12);                       \
5641
      __asm__ volatile(                                           \
5642
         "subu $29, $29, 8 \n\t"                                  \
5643
         "sw $28, 0($29) \n\t"                                    \
5644
         "sw $31, 4($29) \n\t"                                    \
5645
         "lw $4, 20(%1) \n\t"                                     \
5646
         "subu $29, $29, 56\n\t"                                  \
5647
         "sw $4, 16($29) \n\t"                                    \
5648
         "lw $4, 24(%1) \n\t"                                     \
5649
         "sw $4, 20($29) \n\t"                                    \
5650
         "lw $4, 28(%1) \n\t"                                     \
5651
         "sw $4, 24($29) \n\t"                                    \
5652
         "lw $4, 32(%1) \n\t"                                     \
5653
         "sw $4, 28($29) \n\t"                                    \
5654
         "lw $4, 36(%1) \n\t"                                     \
5655
         "sw $4, 32($29) \n\t"                                    \
5656
         "lw $4, 40(%1) \n\t"                                     \
5657
         "sw $4, 36($29) \n\t"                                    \
5658
         "lw $4, 44(%1) \n\t"                                     \
5659
         "sw $4, 40($29) \n\t"                                    \
5660
         "lw $4, 48(%1) \n\t"                                     \
5661
         "sw $4, 44($29) \n\t"                                    \
5662
         "lw $4, 4(%1) \n\t"                                      \
5663
         "lw $5, 8(%1) \n\t"                                      \
5664
         "lw $6, 12(%1) \n\t"                                     \
5665
         "lw $7, 16(%1) \n\t"                                     \
5666
         "lw $25, 0(%1) \n\t"  /* target->t9 */                   \
5667
         VALGRIND_CALL_NOREDIR_T9                                 \
5668
         "addu $29, $29, 56 \n\t"                                 \
5669
         "lw $28, 0($29) \n\t"                                    \
5670
         "lw $31, 4($29) \n\t"                                    \
5671
         "addu $29, $29, 8 \n\t"                                  \
5672
         "move %0, $2\n"                                          \
5673
         : /*out*/   "=r" (_res)                                  \
5674
         : /*in*/    "r" (&_argvec[0])                            \
5675
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5676
      );                                                          \
5677
      lval = (__typeof__(lval)) _res;                             \
5678
   } while (0)
5679
5680
#endif /* PLAT_mips32_linux */
5681
5682
/* ------------------------- mips64-linux ------------------------- */
5683
5684
#if defined(PLAT_mips64_linux)
5685
5686
/* These regs are trashed by the hidden call. */
5687
#define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6",       \
5688
"$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \
5689
"$25", "$31"
5690
5691
/* These CALL_FN_ macros assume that on mips64-linux,
5692
   sizeof(long long) == 8. */
5693
5694
#define MIPS64_LONG2REG_CAST(x) ((long long)(long)x)
5695
5696
#define CALL_FN_W_v(lval, orig)                                   \
5697
   do {                                                           \
5698
      volatile OrigFn        _orig = (orig);                      \
5699
      volatile unsigned long long _argvec[1];                     \
5700
      volatile unsigned long long _res;                           \
5701
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
5702
      __asm__ volatile(                                           \
5703
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5704
         VALGRIND_CALL_NOREDIR_T9                                 \
5705
         "move %0, $2\n"                                          \
5706
         : /*out*/   "=r" (_res)                                  \
5707
         : /*in*/    "0" (&_argvec[0])                            \
5708
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5709
      );                                                          \
5710
      lval = (__typeof__(lval)) (long)_res;                       \
5711
   } while (0)
5712
5713
#define CALL_FN_W_W(lval, orig, arg1)                             \
5714
   do {                                                           \
5715
      volatile OrigFn        _orig = (orig);                      \
5716
      volatile unsigned long long _argvec[2];                     \
5717
      volatile unsigned long long  _res;                          \
5718
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
5719
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
5720
      __asm__ volatile(                                           \
5721
         "ld $4, 8(%1)\n\t"   /* arg1*/                           \
5722
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5723
         VALGRIND_CALL_NOREDIR_T9                                 \
5724
         "move %0, $2\n"                                          \
5725
         : /*out*/   "=r" (_res)                                  \
5726
         : /*in*/    "r" (&_argvec[0])                            \
5727
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5728
      );                                                          \
5729
      lval = (__typeof__(lval)) (long)_res;                       \
5730
   } while (0)
5731
5732
#define CALL_FN_W_WW(lval, orig, arg1,arg2)                       \
5733
   do {                                                           \
5734
      volatile OrigFn        _orig = (orig);                      \
5735
      volatile unsigned long long _argvec[3];                     \
5736
      volatile unsigned long long _res;                           \
5737
      _argvec[0] = _orig.nraddr;                                  \
5738
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
5739
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
5740
      __asm__ volatile(                                           \
5741
         "ld $4, 8(%1)\n\t"                                       \
5742
         "ld $5, 16(%1)\n\t"                                      \
5743
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5744
         VALGRIND_CALL_NOREDIR_T9                                 \
5745
         "move %0, $2\n"                                          \
5746
         : /*out*/   "=r" (_res)                                  \
5747
         : /*in*/    "r" (&_argvec[0])                            \
5748
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5749
      );                                                          \
5750
      lval = (__typeof__(lval)) (long)_res;                       \
5751
   } while (0)
5752
5753
5754
#define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3)                 \
5755
   do {                                                           \
5756
      volatile OrigFn        _orig = (orig);                      \
5757
      volatile unsigned long long _argvec[4];                     \
5758
      volatile unsigned long long _res;                           \
5759
      _argvec[0] = _orig.nraddr;                                  \
5760
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
5761
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
5762
      _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
5763
      __asm__ volatile(                                           \
5764
         "ld $4, 8(%1)\n\t"                                       \
5765
         "ld $5, 16(%1)\n\t"                                      \
5766
         "ld $6, 24(%1)\n\t"                                      \
5767
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5768
         VALGRIND_CALL_NOREDIR_T9                                 \
5769
         "move %0, $2\n"                                          \
5770
         : /*out*/   "=r" (_res)                                  \
5771
         : /*in*/    "r" (&_argvec[0])                            \
5772
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5773
      );                                                          \
5774
      lval = (__typeof__(lval)) (long)_res;                       \
5775
   } while (0)
5776
5777
#define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4)           \
5778
   do {                                                           \
5779
      volatile OrigFn        _orig = (orig);                      \
5780
      volatile unsigned long long _argvec[5];                     \
5781
      volatile unsigned long long _res;                           \
5782
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
5783
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
5784
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
5785
      _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
5786
      _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
5787
      __asm__ volatile(                                           \
5788
         "ld $4, 8(%1)\n\t"                                       \
5789
         "ld $5, 16(%1)\n\t"                                      \
5790
         "ld $6, 24(%1)\n\t"                                      \
5791
         "ld $7, 32(%1)\n\t"                                      \
5792
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5793
         VALGRIND_CALL_NOREDIR_T9                                 \
5794
         "move %0, $2\n"                                          \
5795
         : /*out*/   "=r" (_res)                                  \
5796
         : /*in*/    "r" (&_argvec[0])                            \
5797
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5798
      );                                                          \
5799
      lval = (__typeof__(lval)) (long)_res;                       \
5800
   } while (0)
5801
5802
#define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5)        \
5803
   do {                                                           \
5804
      volatile OrigFn        _orig = (orig);                      \
5805
      volatile unsigned long long _argvec[6];                     \
5806
      volatile unsigned long long _res;                           \
5807
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
5808
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
5809
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
5810
      _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
5811
      _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
5812
      _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
5813
      __asm__ volatile(                                           \
5814
         "ld $4, 8(%1)\n\t"                                       \
5815
         "ld $5, 16(%1)\n\t"                                      \
5816
         "ld $6, 24(%1)\n\t"                                      \
5817
         "ld $7, 32(%1)\n\t"                                      \
5818
         "ld $8, 40(%1)\n\t"                                      \
5819
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5820
         VALGRIND_CALL_NOREDIR_T9                                 \
5821
         "move %0, $2\n"                                          \
5822
         : /*out*/   "=r" (_res)                                  \
5823
         : /*in*/    "r" (&_argvec[0])                            \
5824
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5825
      );                                                          \
5826
      lval = (__typeof__(lval)) (long)_res;                       \
5827
   } while (0)
5828
5829
#define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6)   \
5830
   do {                                                           \
5831
      volatile OrigFn        _orig = (orig);                      \
5832
      volatile unsigned long long _argvec[7];                     \
5833
      volatile unsigned long long _res;                           \
5834
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
5835
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
5836
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
5837
      _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
5838
      _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
5839
      _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
5840
      _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
5841
      __asm__ volatile(                                           \
5842
         "ld $4, 8(%1)\n\t"                                       \
5843
         "ld $5, 16(%1)\n\t"                                      \
5844
         "ld $6, 24(%1)\n\t"                                      \
5845
         "ld $7, 32(%1)\n\t"                                      \
5846
         "ld $8, 40(%1)\n\t"                                      \
5847
         "ld $9, 48(%1)\n\t"                                      \
5848
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5849
         VALGRIND_CALL_NOREDIR_T9                                 \
5850
         "move %0, $2\n"                                          \
5851
         : /*out*/   "=r" (_res)                                  \
5852
         : /*in*/    "r" (&_argvec[0])                            \
5853
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5854
      );                                                          \
5855
      lval = (__typeof__(lval)) (long)_res;                       \
5856
   } while (0)
5857
5858
#define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5859
                                 arg7)                            \
5860
   do {                                                           \
5861
      volatile OrigFn        _orig = (orig);                      \
5862
      volatile unsigned long long _argvec[8];                     \
5863
      volatile unsigned long long _res;                           \
5864
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
5865
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
5866
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
5867
      _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
5868
      _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
5869
      _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
5870
      _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
5871
      _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
5872
      __asm__ volatile(                                           \
5873
         "ld $4, 8(%1)\n\t"                                       \
5874
         "ld $5, 16(%1)\n\t"                                      \
5875
         "ld $6, 24(%1)\n\t"                                      \
5876
         "ld $7, 32(%1)\n\t"                                      \
5877
         "ld $8, 40(%1)\n\t"                                      \
5878
         "ld $9, 48(%1)\n\t"                                      \
5879
         "ld $10, 56(%1)\n\t"                                     \
5880
         "ld $25, 0(%1) \n\t"  /* target->t9 */                   \
5881
         VALGRIND_CALL_NOREDIR_T9                                 \
5882
         "move %0, $2\n"                                          \
5883
         : /*out*/   "=r" (_res)                                  \
5884
         : /*in*/    "r" (&_argvec[0])                            \
5885
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5886
      );                                                          \
5887
      lval = (__typeof__(lval)) (long)_res;                       \
5888
   } while (0)
5889
5890
#define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5891
                                 arg7,arg8)                       \
5892
   do {                                                           \
5893
      volatile OrigFn        _orig = (orig);                      \
5894
      volatile unsigned long long _argvec[9];                     \
5895
      volatile unsigned long long _res;                           \
5896
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
5897
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
5898
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
5899
      _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
5900
      _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
5901
      _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
5902
      _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
5903
      _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
5904
      _argvec[8] = MIPS64_LONG2REG_CAST(arg8);                    \
5905
      __asm__ volatile(                                           \
5906
         "ld $4, 8(%1)\n\t"                                       \
5907
         "ld $5, 16(%1)\n\t"                                      \
5908
         "ld $6, 24(%1)\n\t"                                      \
5909
         "ld $7, 32(%1)\n\t"                                      \
5910
         "ld $8, 40(%1)\n\t"                                      \
5911
         "ld $9, 48(%1)\n\t"                                      \
5912
         "ld $10, 56(%1)\n\t"                                     \
5913
         "ld $11, 64(%1)\n\t"                                     \
5914
         "ld $25, 0(%1) \n\t"  /* target->t9 */                   \
5915
         VALGRIND_CALL_NOREDIR_T9                                 \
5916
         "move %0, $2\n"                                          \
5917
         : /*out*/   "=r" (_res)                                  \
5918
         : /*in*/    "r" (&_argvec[0])                            \
5919
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5920
      );                                                          \
5921
      lval = (__typeof__(lval)) (long)_res;                       \
5922
   } while (0)
5923
5924
#define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,   \
5925
                                 arg7,arg8,arg9)                  \
5926
   do {                                                           \
5927
      volatile OrigFn        _orig = (orig);                      \
5928
      volatile unsigned long long _argvec[10];                    \
5929
      volatile unsigned long long _res;                           \
5930
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
5931
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
5932
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
5933
      _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
5934
      _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
5935
      _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
5936
      _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
5937
      _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
5938
      _argvec[8] = MIPS64_LONG2REG_CAST(arg8);                    \
5939
      _argvec[9] = MIPS64_LONG2REG_CAST(arg9);                    \
5940
      __asm__ volatile(                                           \
5941
         "dsubu $29, $29, 8\n\t"                                  \
5942
         "ld $4, 72(%1)\n\t"                                      \
5943
         "sd $4, 0($29)\n\t"                                      \
5944
         "ld $4, 8(%1)\n\t"                                       \
5945
         "ld $5, 16(%1)\n\t"                                      \
5946
         "ld $6, 24(%1)\n\t"                                      \
5947
         "ld $7, 32(%1)\n\t"                                      \
5948
         "ld $8, 40(%1)\n\t"                                      \
5949
         "ld $9, 48(%1)\n\t"                                      \
5950
         "ld $10, 56(%1)\n\t"                                     \
5951
         "ld $11, 64(%1)\n\t"                                     \
5952
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5953
         VALGRIND_CALL_NOREDIR_T9                                 \
5954
         "daddu $29, $29, 8\n\t"                                  \
5955
         "move %0, $2\n"                                          \
5956
         : /*out*/   "=r" (_res)                                  \
5957
         : /*in*/    "r" (&_argvec[0])                            \
5958
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
5959
      );                                                          \
5960
      lval = (__typeof__(lval)) (long)_res;                       \
5961
   } while (0)
5962
5963
#define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6,  \
5964
                                  arg7,arg8,arg9,arg10)           \
5965
   do {                                                           \
5966
      volatile OrigFn        _orig = (orig);                      \
5967
      volatile unsigned long long _argvec[11];                    \
5968
      volatile unsigned long long _res;                           \
5969
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
5970
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
5971
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
5972
      _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
5973
      _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
5974
      _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
5975
      _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
5976
      _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
5977
      _argvec[8] = MIPS64_LONG2REG_CAST(arg8);                    \
5978
      _argvec[9] = MIPS64_LONG2REG_CAST(arg9);                    \
5979
      _argvec[10] = MIPS64_LONG2REG_CAST(arg10);                  \
5980
      __asm__ volatile(                                           \
5981
         "dsubu $29, $29, 16\n\t"                                 \
5982
         "ld $4, 72(%1)\n\t"                                      \
5983
         "sd $4, 0($29)\n\t"                                      \
5984
         "ld $4, 80(%1)\n\t"                                      \
5985
         "sd $4, 8($29)\n\t"                                      \
5986
         "ld $4, 8(%1)\n\t"                                       \
5987
         "ld $5, 16(%1)\n\t"                                      \
5988
         "ld $6, 24(%1)\n\t"                                      \
5989
         "ld $7, 32(%1)\n\t"                                      \
5990
         "ld $8, 40(%1)\n\t"                                      \
5991
         "ld $9, 48(%1)\n\t"                                      \
5992
         "ld $10, 56(%1)\n\t"                                     \
5993
         "ld $11, 64(%1)\n\t"                                     \
5994
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
5995
         VALGRIND_CALL_NOREDIR_T9                                 \
5996
         "daddu $29, $29, 16\n\t"                                 \
5997
         "move %0, $2\n"                                          \
5998
         : /*out*/   "=r" (_res)                                  \
5999
         : /*in*/    "r" (&_argvec[0])                            \
6000
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6001
      );                                                          \
6002
      lval = (__typeof__(lval)) (long)_res;                       \
6003
   } while (0)
6004
6005
#define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
6006
                                  arg6,arg7,arg8,arg9,arg10,      \
6007
                                  arg11)                          \
6008
   do {                                                           \
6009
      volatile OrigFn        _orig = (orig);                      \
6010
      volatile unsigned long long _argvec[12];                    \
6011
      volatile unsigned long long _res;                           \
6012
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6013
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6014
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6015
      _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6016
      _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6017
      _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
6018
      _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
6019
      _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
6020
      _argvec[8] = MIPS64_LONG2REG_CAST(arg8);                    \
6021
      _argvec[9] = MIPS64_LONG2REG_CAST(arg9);                    \
6022
      _argvec[10] = MIPS64_LONG2REG_CAST(arg10);                  \
6023
      _argvec[11] = MIPS64_LONG2REG_CAST(arg11);                  \
6024
      __asm__ volatile(                                           \
6025
         "dsubu $29, $29, 24\n\t"                                 \
6026
         "ld $4, 72(%1)\n\t"                                      \
6027
         "sd $4, 0($29)\n\t"                                      \
6028
         "ld $4, 80(%1)\n\t"                                      \
6029
         "sd $4, 8($29)\n\t"                                      \
6030
         "ld $4, 88(%1)\n\t"                                      \
6031
         "sd $4, 16($29)\n\t"                                     \
6032
         "ld $4, 8(%1)\n\t"                                       \
6033
         "ld $5, 16(%1)\n\t"                                      \
6034
         "ld $6, 24(%1)\n\t"                                      \
6035
         "ld $7, 32(%1)\n\t"                                      \
6036
         "ld $8, 40(%1)\n\t"                                      \
6037
         "ld $9, 48(%1)\n\t"                                      \
6038
         "ld $10, 56(%1)\n\t"                                     \
6039
         "ld $11, 64(%1)\n\t"                                     \
6040
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6041
         VALGRIND_CALL_NOREDIR_T9                                 \
6042
         "daddu $29, $29, 24\n\t"                                 \
6043
         "move %0, $2\n"                                          \
6044
         : /*out*/   "=r" (_res)                                  \
6045
         : /*in*/    "r" (&_argvec[0])                            \
6046
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6047
      );                                                          \
6048
      lval = (__typeof__(lval)) (long)_res;                       \
6049
   } while (0)
6050
6051
#define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,       \
6052
                                  arg6,arg7,arg8,arg9,arg10,      \
6053
                                  arg11,arg12)                    \
6054
   do {                                                           \
6055
      volatile OrigFn        _orig = (orig);                      \
6056
      volatile unsigned long long _argvec[13];                    \
6057
      volatile unsigned long long _res;                           \
6058
      _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr);            \
6059
      _argvec[1] = MIPS64_LONG2REG_CAST(arg1);                    \
6060
      _argvec[2] = MIPS64_LONG2REG_CAST(arg2);                    \
6061
      _argvec[3] = MIPS64_LONG2REG_CAST(arg3);                    \
6062
      _argvec[4] = MIPS64_LONG2REG_CAST(arg4);                    \
6063
      _argvec[5] = MIPS64_LONG2REG_CAST(arg5);                    \
6064
      _argvec[6] = MIPS64_LONG2REG_CAST(arg6);                    \
6065
      _argvec[7] = MIPS64_LONG2REG_CAST(arg7);                    \
6066
      _argvec[8] = MIPS64_LONG2REG_CAST(arg8);                    \
6067
      _argvec[9] = MIPS64_LONG2REG_CAST(arg9);                    \
6068
      _argvec[10] = MIPS64_LONG2REG_CAST(arg10);                  \
6069
      _argvec[11] = MIPS64_LONG2REG_CAST(arg11);                  \
6070
      _argvec[12] = MIPS64_LONG2REG_CAST(arg12);                  \
6071
      __asm__ volatile(                                           \
6072
         "dsubu $29, $29, 32\n\t"                                 \
6073
         "ld $4, 72(%1)\n\t"                                      \
6074
         "sd $4, 0($29)\n\t"                                      \
6075
         "ld $4, 80(%1)\n\t"                                      \
6076
         "sd $4, 8($29)\n\t"                                      \
6077
         "ld $4, 88(%1)\n\t"                                      \
6078
         "sd $4, 16($29)\n\t"                                     \
6079
         "ld $4, 96(%1)\n\t"                                      \
6080
         "sd $4, 24($29)\n\t"                                     \
6081
         "ld $4, 8(%1)\n\t"                                       \
6082
         "ld $5, 16(%1)\n\t"                                      \
6083
         "ld $6, 24(%1)\n\t"                                      \
6084
         "ld $7, 32(%1)\n\t"                                      \
6085
         "ld $8, 40(%1)\n\t"                                      \
6086
         "ld $9, 48(%1)\n\t"                                      \
6087
         "ld $10, 56(%1)\n\t"                                     \
6088
         "ld $11, 64(%1)\n\t"                                     \
6089
         "ld $25, 0(%1)\n\t"  /* target->t9 */                    \
6090
         VALGRIND_CALL_NOREDIR_T9                                 \
6091
         "daddu $29, $29, 32\n\t"                                 \
6092
         "move %0, $2\n"                                          \
6093
         : /*out*/   "=r" (_res)                                  \
6094
         : /*in*/    "r" (&_argvec[0])                            \
6095
         : /*trash*/ "memory", __CALLER_SAVED_REGS                \
6096
      );                                                          \
6097
      lval = (__typeof__(lval)) (long)_res;                       \
6098
   } while (0)
6099
6100
#endif /* PLAT_mips64_linux */
6101
6102
/* ------------------------------------------------------------------ */
6103
/* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS.               */
6104
/*                                                                    */
6105
/* ------------------------------------------------------------------ */
6106
6107
/* Some request codes.  There are many more of these, but most are not
6108
   exposed to end-user view.  These are the public ones, all of the
6109
   form 0x1000 + small_number.
6110
6111
   Core ones are in the range 0x00000000--0x0000ffff.  The non-public
6112
   ones start at 0x2000.
6113
*/
6114
6115
/* These macros are used by tools -- they must be public, but don't
6116
   embed them into other programs. */
6117
#define VG_USERREQ_TOOL_BASE(a,b) \
6118
   ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16))
6119
#define VG_IS_TOOL_USERREQ(a, b, v) \
6120
   (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000))
6121
6122
/* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !! 
6123
   This enum comprises an ABI exported by Valgrind to programs
6124
   which use client requests.  DO NOT CHANGE THE NUMERIC VALUES OF THESE
6125
   ENTRIES, NOR DELETE ANY -- add new ones at the end of the most
6126
   relevant group. */
6127
typedef
6128
   enum { VG_USERREQ__RUNNING_ON_VALGRIND  = 0x1001,
6129
          VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002,
6130
6131
          /* These allow any function to be called from the simulated
6132
             CPU but run on the real CPU.  Nb: the first arg passed to
6133
             the function is always the ThreadId of the running
6134
             thread!  So CLIENT_CALL0 actually requires a 1 arg
6135
             function, etc. */
6136
          VG_USERREQ__CLIENT_CALL0 = 0x1101,
6137
          VG_USERREQ__CLIENT_CALL1 = 0x1102,
6138
          VG_USERREQ__CLIENT_CALL2 = 0x1103,
6139
          VG_USERREQ__CLIENT_CALL3 = 0x1104,
6140
6141
          /* Can be useful in regression testing suites -- eg. can
6142
             send Valgrind's output to /dev/null and still count
6143
             errors. */
6144
          VG_USERREQ__COUNT_ERRORS = 0x1201,
6145
6146
          /* Allows the client program and/or gdbserver to execute a monitor
6147
             command. */
6148
          VG_USERREQ__GDB_MONITOR_COMMAND = 0x1202,
6149
6150
          /* These are useful and can be interpreted by any tool that
6151
             tracks malloc() et al, by using vg_replace_malloc.c. */
6152
          VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301,
6153
          VG_USERREQ__RESIZEINPLACE_BLOCK = 0x130b,
6154
          VG_USERREQ__FREELIKE_BLOCK   = 0x1302,
6155
          /* Memory pool support. */
6156
          VG_USERREQ__CREATE_MEMPOOL   = 0x1303,
6157
          VG_USERREQ__DESTROY_MEMPOOL  = 0x1304,
6158
          VG_USERREQ__MEMPOOL_ALLOC    = 0x1305,
6159
          VG_USERREQ__MEMPOOL_FREE     = 0x1306,
6160
          VG_USERREQ__MEMPOOL_TRIM     = 0x1307,
6161
          VG_USERREQ__MOVE_MEMPOOL     = 0x1308,
6162
          VG_USERREQ__MEMPOOL_CHANGE   = 0x1309,
6163
          VG_USERREQ__MEMPOOL_EXISTS   = 0x130a,
6164
6165
          /* Allow printfs to valgrind log. */
6166
          /* The first two pass the va_list argument by value, which
6167
             assumes it is the same size as or smaller than a UWord,
6168
             which generally isn't the case.  Hence are deprecated.
6169
             The second two pass the vargs by reference and so are
6170
             immune to this problem. */
6171
          /* both :: char* fmt, va_list vargs (DEPRECATED) */
6172
          VG_USERREQ__PRINTF           = 0x1401,
6173
          VG_USERREQ__PRINTF_BACKTRACE = 0x1402,
6174
          /* both :: char* fmt, va_list* vargs */
6175
          VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403,
6176
          VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404,
6177
6178
          /* Stack support. */
6179
          VG_USERREQ__STACK_REGISTER   = 0x1501,
6180
          VG_USERREQ__STACK_DEREGISTER = 0x1502,
6181
          VG_USERREQ__STACK_CHANGE     = 0x1503,
6182
6183
          /* Wine support */
6184
          VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601,
6185
6186
          /* Querying of debug info. */
6187
          VG_USERREQ__MAP_IP_TO_SRCLOC = 0x1701,
6188
6189
          /* Disable/enable error reporting level.  Takes a single
6190
             Word arg which is the delta to this thread's error
6191
             disablement indicator.  Hence 1 disables or further
6192
             disables errors, and -1 moves back towards enablement.
6193
             Other values are not allowed. */
6194
          VG_USERREQ__CHANGE_ERR_DISABLEMENT = 0x1801,
6195
6196
          /* Some requests used for Valgrind internal, such as
6197
             self-test or self-hosting. */
6198
          /* Initialise IR injection */
6199
          VG_USERREQ__VEX_INIT_FOR_IRI = 0x1901,
6200
          /* Used by Inner Valgrind to inform Outer Valgrind where to
6201
             find the list of inner guest threads */
6202
          VG_USERREQ__INNER_THREADS    = 0x1902
6203
   } Vg_ClientRequest;
6204
6205
#if !defined(__GNUC__)
6206
#  define __extension__ /* */
6207
#endif
6208
6209
6210
/* Returns the number of Valgrinds this code is running under.  That
6211
   is, 0 if running natively, 1 if running under Valgrind, 2 if
6212
   running under Valgrind which is running under another Valgrind,
6213
   etc. */
6214
#define RUNNING_ON_VALGRIND                                           \
6215
928k
    (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */,         \
6216
928k
                                    VG_USERREQ__RUNNING_ON_VALGRIND,  \
6217
928k
                                    0, 0, 0, 0, 0)                    \
6218
6219
6220
/* Discard translation of code in the range [_qzz_addr .. _qzz_addr +
6221
   _qzz_len - 1].  Useful if you are debugging a JITter or some such,
6222
   since it provides a way to make sure valgrind will retranslate the
6223
   invalidated area.  Returns no value. */
6224
#define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len)              \
6225
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DISCARD_TRANSLATIONS,  \
6226
                                    _qzz_addr, _qzz_len, 0, 0, 0)
6227
6228
#define VALGRIND_INNER_THREADS(_qzz_addr)                               \
6229
   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__INNER_THREADS,           \
6230
                                   _qzz_addr, 0, 0, 0, 0)
6231
6232
6233
/* These requests are for getting Valgrind itself to print something.
6234
   Possibly with a backtrace.  This is a really ugly hack.  The return value
6235
   is the number of characters printed, excluding the "**<pid>** " part at the
6236
   start and the backtrace (if present). */
6237
6238
#if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER)
6239
/* Modern GCC will optimize the static routine out if unused,
6240
   and unused attribute will shut down warnings about it.  */
6241
static int VALGRIND_PRINTF(const char *format, ...)
6242
   __attribute__((format(__printf__, 1, 2), __unused__));
6243
#endif
6244
static int
6245
#if defined(_MSC_VER)
6246
__inline
6247
#endif
6248
VALGRIND_PRINTF(const char *format, ...)
6249
0
{
6250
0
#if defined(NVALGRIND)
6251
0
   (void)format;
6252
0
   return 0;
6253
0
#else /* NVALGRIND */
6254
0
#if defined(_MSC_VER) || defined(__MINGW64__)
6255
0
   uintptr_t _qzz_res;
6256
0
#else
6257
0
   unsigned long _qzz_res;
6258
0
#endif
6259
0
   va_list vargs;
6260
0
   va_start(vargs, format);
6261
0
#if defined(_MSC_VER) || defined(__MINGW64__)
6262
0
   _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
6263
0
                              VG_USERREQ__PRINTF_VALIST_BY_REF,
6264
0
                              (uintptr_t)format,
6265
0
                              (uintptr_t)&vargs,
6266
0
                              0, 0, 0);
6267
0
#else
6268
0
   _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
6269
0
                              VG_USERREQ__PRINTF_VALIST_BY_REF,
6270
0
                              (unsigned long)format,
6271
0
                              (unsigned long)&vargs, 
6272
0
                              0, 0, 0);
6273
0
#endif
6274
0
   va_end(vargs);
6275
0
   return (int)_qzz_res;
6276
0
#endif /* NVALGRIND */
6277
0
}
Unexecuted instantiation: gerror.c:VALGRIND_PRINTF
Unexecuted instantiation: ghash.c:VALGRIND_PRINTF
Unexecuted instantiation: gslice.c:VALGRIND_PRINTF
6278
6279
#if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER)
6280
static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
6281
   __attribute__((format(__printf__, 1, 2), __unused__));
6282
#endif
6283
static int
6284
#if defined(_MSC_VER)
6285
__inline
6286
#endif
6287
VALGRIND_PRINTF_BACKTRACE(const char *format, ...)
6288
0
{
6289
0
#if defined(NVALGRIND)
6290
0
   (void)format;
6291
0
   return 0;
6292
0
#else /* NVALGRIND */
6293
0
#if defined(_MSC_VER) || defined(__MINGW64__)
6294
0
   uintptr_t _qzz_res;
6295
0
#else
6296
0
   unsigned long _qzz_res;
6297
0
#endif
6298
0
   va_list vargs;
6299
0
   va_start(vargs, format);
6300
0
#if defined(_MSC_VER) || defined(__MINGW64__)
6301
0
   _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
6302
0
                              VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
6303
0
                              (uintptr_t)format,
6304
0
                              (uintptr_t)&vargs,
6305
0
                              0, 0, 0);
6306
0
#else
6307
0
   _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0,
6308
0
                              VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF,
6309
0
                              (unsigned long)format,
6310
0
                              (unsigned long)&vargs, 
6311
0
                              0, 0, 0);
6312
0
#endif
6313
0
   va_end(vargs);
6314
0
   return (int)_qzz_res;
6315
0
#endif /* NVALGRIND */
6316
0
}
Unexecuted instantiation: gerror.c:VALGRIND_PRINTF_BACKTRACE
Unexecuted instantiation: ghash.c:VALGRIND_PRINTF_BACKTRACE
Unexecuted instantiation: gslice.c:VALGRIND_PRINTF_BACKTRACE
6317
6318
6319
/* These requests allow control to move from the simulated CPU to the
6320
   real CPU, calling an arbitrary function.
6321
   
6322
   Note that the current ThreadId is inserted as the first argument.
6323
   So this call:
6324
6325
     VALGRIND_NON_SIMD_CALL2(f, arg1, arg2)
6326
6327
   requires f to have this signature:
6328
6329
     Word f(Word tid, Word arg1, Word arg2)
6330
6331
   where "Word" is a word-sized type.
6332
6333
   Note that these client requests are not entirely reliable.  For example,
6334
   if you call a function with them that subsequently calls printf(),
6335
   there's a high chance Valgrind will crash.  Generally, your prospects of
6336
   these working are made higher if the called function does not refer to
6337
   any global variables, and does not refer to any libc or other functions
6338
   (printf et al).  Any kind of entanglement with libc or dynamic linking is
6339
   likely to have a bad outcome, for tricky reasons which we've grappled
6340
   with a lot in the past.
6341
*/
6342
#define VALGRIND_NON_SIMD_CALL0(_qyy_fn)                          \
6343
    VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,       \
6344
                                    VG_USERREQ__CLIENT_CALL0,     \
6345
                                    _qyy_fn,                      \
6346
                                    0, 0, 0, 0)
6347
6348
#define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1)                    \
6349
    VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,            \
6350
                                    VG_USERREQ__CLIENT_CALL1,          \
6351
                                    _qyy_fn,                           \
6352
                                    _qyy_arg1, 0, 0, 0)
6353
6354
#define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2)         \
6355
    VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,            \
6356
                                    VG_USERREQ__CLIENT_CALL2,          \
6357
                                    _qyy_fn,                           \
6358
                                    _qyy_arg1, _qyy_arg2, 0, 0)
6359
6360
#define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \
6361
    VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */,             \
6362
                                    VG_USERREQ__CLIENT_CALL3,           \
6363
                                    _qyy_fn,                            \
6364
                                    _qyy_arg1, _qyy_arg2,               \
6365
                                    _qyy_arg3, 0)
6366
6367
6368
/* Counts the number of errors that have been recorded by a tool.  Nb:
6369
   the tool must record the errors with VG_(maybe_record_error)() or
6370
   VG_(unique_error)() for them to be counted. */
6371
#define VALGRIND_COUNT_ERRORS                                     \
6372
    (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(                    \
6373
                               0 /* default return */,            \
6374
                               VG_USERREQ__COUNT_ERRORS,          \
6375
                               0, 0, 0, 0, 0)
6376
6377
/* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing
6378
   when heap blocks are allocated in order to give accurate results.  This
6379
   happens automatically for the standard allocator functions such as
6380
   malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete,
6381
   delete[], etc.
6382
6383
   But if your program uses a custom allocator, this doesn't automatically
6384
   happen, and Valgrind will not do as well.  For example, if you allocate
6385
   superblocks with mmap() and then allocates chunks of the superblocks, all
6386
   Valgrind's observations will be at the mmap() level and it won't know that
6387
   the chunks should be considered separate entities.  In Memcheck's case,
6388
   that means you probably won't get heap block overrun detection (because
6389
   there won't be redzones marked as unaddressable) and you definitely won't
6390
   get any leak detection.
6391
6392
   The following client requests allow a custom allocator to be annotated so
6393
   that it can be handled accurately by Valgrind.
6394
6395
   VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated
6396
   by a malloc()-like function.  For Memcheck (an illustrative case), this
6397
   does two things:
6398
6399
   - It records that the block has been allocated.  This means any addresses
6400
     within the block mentioned in error messages will be
6401
     identified as belonging to the block.  It also means that if the block
6402
     isn't freed it will be detected by the leak checker.
6403
6404
   - It marks the block as being addressable and undefined (if 'is_zeroed' is
6405
     not set), or addressable and defined (if 'is_zeroed' is set).  This
6406
     controls how accesses to the block by the program are handled.
6407
   
6408
   'addr' is the start of the usable block (ie. after any
6409
   redzone), 'sizeB' is its size.  'rzB' is the redzone size if the allocator
6410
   can apply redzones -- these are blocks of padding at the start and end of
6411
   each block.  Adding redzones is recommended as it makes it much more likely
6412
   Valgrind will spot block overruns.  `is_zeroed' indicates if the memory is
6413
   zeroed (or filled with another predictable value), as is the case for
6414
   calloc().
6415
   
6416
   VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a
6417
   heap block -- that will be used by the client program -- is allocated.
6418
   It's best to put it at the outermost level of the allocator if possible;
6419
   for example, if you have a function my_alloc() which calls
6420
   internal_alloc(), and the client request is put inside internal_alloc(),
6421
   stack traces relating to the heap block will contain entries for both
6422
   my_alloc() and internal_alloc(), which is probably not what you want.
6423
6424
   For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out
6425
   custom blocks from within a heap block, B, that has been allocated with
6426
   malloc/calloc/new/etc, then block B will be *ignored* during leak-checking
6427
   -- the custom blocks will take precedence.
6428
6429
   VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK.  For
6430
   Memcheck, it does two things:
6431
6432
   - It records that the block has been deallocated.  This assumes that the
6433
     block was annotated as having been allocated via
6434
     VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
6435
6436
   - It marks the block as being unaddressable.
6437
6438
   VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a
6439
   heap block is deallocated.
6440
6441
   VALGRIND_RESIZEINPLACE_BLOCK informs a tool about reallocation. For
6442
   Memcheck, it does four things:
6443
6444
   - It records that the size of a block has been changed.  This assumes that
6445
     the block was annotated as having been allocated via
6446
     VALGRIND_MALLOCLIKE_BLOCK.  Otherwise, an error will be issued.
6447
6448
   - If the block shrunk, it marks the freed memory as being unaddressable.
6449
6450
   - If the block grew, it marks the new area as undefined and defines a red
6451
     zone past the end of the new block.
6452
6453
   - The V-bits of the overlap between the old and the new block are preserved.
6454
6455
   VALGRIND_RESIZEINPLACE_BLOCK should be put after allocation of the new block
6456
   and before deallocation of the old block.
6457
6458
   In many cases, these three client requests will not be enough to get your
6459
   allocator working well with Memcheck.  More specifically, if your allocator
6460
   writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call
6461
   will be necessary to mark the memory as addressable just before the zeroing
6462
   occurs, otherwise you'll get a lot of invalid write errors.  For example,
6463
   you'll need to do this if your allocator recycles freed blocks, but it
6464
   zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK).
6465
   Alternatively, if your allocator reuses freed blocks for allocator-internal
6466
   data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary.
6467
6468
   Really, what's happening is a blurring of the lines between the client
6469
   program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the
6470
   memory should be considered unaddressable to the client program, but the
6471
   allocator knows more than the rest of the client program and so may be able
6472
   to safely access it.  Extra client requests are necessary for Valgrind to
6473
   understand the distinction between the allocator and the rest of the
6474
   program.
6475
6476
   Ignored if addr == 0.
6477
*/
6478
#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)          \
6479
0
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MALLOCLIKE_BLOCK,       \
6480
0
                                    addr, sizeB, rzB, is_zeroed, 0)
6481
6482
/* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
6483
   Ignored if addr == 0.
6484
*/
6485
#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB)     \
6486
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__RESIZEINPLACE_BLOCK,    \
6487
                                    addr, oldSizeB, newSizeB, rzB, 0)
6488
6489
/* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details.
6490
   Ignored if addr == 0.
6491
*/
6492
#define VALGRIND_FREELIKE_BLOCK(addr, rzB)                              \
6493
0
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__FREELIKE_BLOCK,         \
6494
0
                                    addr, rzB, 0, 0, 0)
6495
6496
/* Create a memory pool. */
6497
#define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed)             \
6498
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL,   \
6499
                                    pool, rzB, is_zeroed, 0, 0)
6500
6501
/* Create a memory pool with some flags specifying extended behaviour.
6502
   When flags is zero, the behaviour is identical to VALGRIND_CREATE_MEMPOOL.
6503
   
6504
   The flag VALGRIND_MEMPOOL_METAPOOL specifies that the pieces of memory 
6505
   associated with the pool using VALGRIND_MEMPOOL_ALLOC  will be used
6506
   by the application as superblocks to dole out MALLOC_LIKE blocks using
6507
   VALGRIND_MALLOCLIKE_BLOCK. In other words, a meta pool is a "2 levels"
6508
   pool : first level is the blocks described by VALGRIND_MEMPOOL_ALLOC.
6509
   The second level blocks are described using VALGRIND_MALLOCLIKE_BLOCK.
6510
   Note that the association between the pool and the second level blocks
6511
   is implicit : second level blocks will be located inside first level
6512
   blocks. It is necessary to use the VALGRIND_MEMPOOL_METAPOOL flag
6513
   for such 2 levels pools, as otherwise valgrind will detect overlapping
6514
   memory blocks, and will abort execution (e.g. during leak search).
6515
6516
   Such a meta pool can also be marked as an 'auto free' pool using the flag
6517
   VALGRIND_MEMPOOL_AUTO_FREE, which must be OR-ed together with the
6518
   VALGRIND_MEMPOOL_METAPOOL. For an 'auto free' pool, VALGRIND_MEMPOOL_FREE
6519
   will automatically free the second level blocks that are contained
6520
   inside the first level block freed with VALGRIND_MEMPOOL_FREE.
6521
   In other words, calling VALGRIND_MEMPOOL_FREE will cause implicit calls
6522
   to VALGRIND_FREELIKE_BLOCK for all the second level blocks included
6523
   in the first level block.
6524
   Note: it is an error to use the VALGRIND_MEMPOOL_AUTO_FREE flag
6525
   without the VALGRIND_MEMPOOL_METAPOOL flag.
6526
*/
6527
#define VALGRIND_MEMPOOL_AUTO_FREE  1
6528
#define VALGRIND_MEMPOOL_METAPOOL   2
6529
#define VALGRIND_CREATE_MEMPOOL_EXT(pool, rzB, is_zeroed, flags)        \
6530
   VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL,          \
6531
                                   pool, rzB, is_zeroed, flags, 0)
6532
6533
/* Destroy a memory pool. */
6534
#define VALGRIND_DESTROY_MEMPOOL(pool)                            \
6535
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DESTROY_MEMPOOL,  \
6536
                                    pool, 0, 0, 0, 0)
6537
6538
/* Associate a piece of memory with a memory pool. */
6539
#define VALGRIND_MEMPOOL_ALLOC(pool, addr, size)                  \
6540
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_ALLOC,    \
6541
                                    pool, addr, size, 0, 0)
6542
6543
/* Disassociate a piece of memory from a memory pool. */
6544
#define VALGRIND_MEMPOOL_FREE(pool, addr)                         \
6545
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_FREE,     \
6546
                                    pool, addr, 0, 0, 0)
6547
6548
/* Disassociate any pieces outside a particular range. */
6549
#define VALGRIND_MEMPOOL_TRIM(pool, addr, size)                   \
6550
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_TRIM,     \
6551
                                    pool, addr, size, 0, 0)
6552
6553
/* Resize and/or move a piece associated with a memory pool. */
6554
#define VALGRIND_MOVE_MEMPOOL(poolA, poolB)                       \
6555
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MOVE_MEMPOOL,     \
6556
                                    poolA, poolB, 0, 0, 0)
6557
6558
/* Resize and/or move a piece associated with a memory pool. */
6559
#define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size)         \
6560
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_CHANGE,   \
6561
                                    pool, addrA, addrB, size, 0)
6562
6563
/* Return 1 if a mempool exists, else 0. */
6564
#define VALGRIND_MEMPOOL_EXISTS(pool)                             \
6565
    (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
6566
                               VG_USERREQ__MEMPOOL_EXISTS,        \
6567
                               pool, 0, 0, 0, 0)
6568
6569
/* Mark a piece of memory as being a stack. Returns a stack id.
6570
   start is the lowest addressable stack byte, end is the highest
6571
   addressable stack byte. */
6572
#define VALGRIND_STACK_REGISTER(start, end)                       \
6573
    (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
6574
                               VG_USERREQ__STACK_REGISTER,        \
6575
                               start, end, 0, 0, 0)
6576
6577
/* Unmark the piece of memory associated with a stack id as being a
6578
   stack. */
6579
#define VALGRIND_STACK_DEREGISTER(id)                             \
6580
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_DEREGISTER, \
6581
                                    id, 0, 0, 0, 0)
6582
6583
/* Change the start and end address of the stack id.
6584
   start is the new lowest addressable stack byte, end is the new highest
6585
   addressable stack byte. */
6586
#define VALGRIND_STACK_CHANGE(id, start, end)                     \
6587
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_CHANGE,     \
6588
                                    id, start, end, 0, 0)
6589
6590
/* Load PDB debug info for Wine PE image_map. */
6591
#define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta)     \
6592
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__LOAD_PDB_DEBUGINFO, \
6593
                                    fd, ptr, total_size, delta, 0)
6594
6595
/* Map a code address to a source file name and line number.  buf64
6596
   must point to a 64-byte buffer in the caller's address space.  The
6597
   result will be dumped in there and is guaranteed to be zero
6598
   terminated.  If no info is found, the first byte is set to zero. */
6599
#define VALGRIND_MAP_IP_TO_SRCLOC(addr, buf64)                    \
6600
    (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0,                  \
6601
                               VG_USERREQ__MAP_IP_TO_SRCLOC,      \
6602
                               addr, buf64, 0, 0, 0)
6603
6604
/* Disable error reporting for this thread.  Behaves in a stack like
6605
   way, so you can safely call this multiple times provided that
6606
   VALGRIND_ENABLE_ERROR_REPORTING is called the same number of times
6607
   to re-enable reporting.  The first call of this macro disables
6608
   reporting.  Subsequent calls have no effect except to increase the
6609
   number of VALGRIND_ENABLE_ERROR_REPORTING calls needed to re-enable
6610
   reporting.  Child threads do not inherit this setting from their
6611
   parents -- they are always created with reporting enabled. */
6612
#define VALGRIND_DISABLE_ERROR_REPORTING                                \
6613
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \
6614
                                    1, 0, 0, 0, 0)
6615
6616
/* Re-enable error reporting, as per comments on
6617
   VALGRIND_DISABLE_ERROR_REPORTING. */
6618
#define VALGRIND_ENABLE_ERROR_REPORTING                                 \
6619
    VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \
6620
                                    -1, 0, 0, 0, 0)
6621
6622
/* Execute a monitor command from the client program.
6623
   If a connection is opened with GDB, the output will be sent
6624
   according to the output mode set for vgdb.
6625
   If no connection is opened, output will go to the log output.
6626
   Returns 1 if command not recognised, 0 otherwise. */
6627
#define VALGRIND_MONITOR_COMMAND(command)                               \
6628
   VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__GDB_MONITOR_COMMAND, \
6629
                                   command, 0, 0, 0, 0)
6630
6631
6632
#undef PLAT_x86_darwin
6633
#undef PLAT_amd64_darwin
6634
#undef PLAT_x86_win32
6635
#undef PLAT_amd64_win64
6636
#undef PLAT_x86_linux
6637
#undef PLAT_amd64_linux
6638
#undef PLAT_ppc32_linux
6639
#undef PLAT_ppc64be_linux
6640
#undef PLAT_ppc64le_linux
6641
#undef PLAT_arm_linux
6642
#undef PLAT_s390x_linux
6643
#undef PLAT_mips32_linux
6644
#undef PLAT_mips64_linux
6645
#undef PLAT_x86_solaris
6646
#undef PLAT_amd64_solaris
6647
6648
#endif   /* __VALGRIND_H */