Coverage Report

Created: 2023-11-27 07:06

/src/libbpf/src/libbpf.c
Line
Count
Source (jump to first uncovered line)
1
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2
3
/*
4
 * Common eBPF ELF object loading operations.
5
 *
6
 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7
 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8
 * Copyright (C) 2015 Huawei Inc.
9
 * Copyright (C) 2017 Nicira, Inc.
10
 * Copyright (C) 2019 Isovalent, Inc.
11
 */
12
13
#ifndef _GNU_SOURCE
14
#define _GNU_SOURCE
15
#endif
16
#include <stdlib.h>
17
#include <stdio.h>
18
#include <stdarg.h>
19
#include <libgen.h>
20
#include <inttypes.h>
21
#include <limits.h>
22
#include <string.h>
23
#include <unistd.h>
24
#include <endian.h>
25
#include <fcntl.h>
26
#include <errno.h>
27
#include <ctype.h>
28
#include <asm/unistd.h>
29
#include <linux/err.h>
30
#include <linux/kernel.h>
31
#include <linux/bpf.h>
32
#include <linux/btf.h>
33
#include <linux/filter.h>
34
#include <linux/limits.h>
35
#include <linux/perf_event.h>
36
#include <linux/ring_buffer.h>
37
#include <sys/epoll.h>
38
#include <sys/ioctl.h>
39
#include <sys/mman.h>
40
#include <sys/stat.h>
41
#include <sys/types.h>
42
#include <sys/vfs.h>
43
#include <sys/utsname.h>
44
#include <sys/resource.h>
45
#include <libelf.h>
46
#include <gelf.h>
47
#include <zlib.h>
48
49
#include "libbpf.h"
50
#include "bpf.h"
51
#include "btf.h"
52
#include "str_error.h"
53
#include "libbpf_internal.h"
54
#include "hashmap.h"
55
#include "bpf_gen_internal.h"
56
#include "zip.h"
57
58
#ifndef BPF_FS_MAGIC
59
0
#define BPF_FS_MAGIC    0xcafe4a11
60
#endif
61
62
98.5k
#define BPF_INSN_SZ (sizeof(struct bpf_insn))
63
64
/* vsprintf() in __base_pr() uses nonliteral format string. It may break
65
 * compilation if user enables corresponding warning. Disable it explicitly.
66
 */
67
#pragma GCC diagnostic ignored "-Wformat-nonliteral"
68
69
#define __printf(a, b)  __attribute__((format(printf, a, b)))
70
71
static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
72
static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
73
74
static const char * const attach_type_name[] = {
75
  [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress",
76
  [BPF_CGROUP_INET_EGRESS]  = "cgroup_inet_egress",
77
  [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create",
78
  [BPF_CGROUP_INET_SOCK_RELEASE]  = "cgroup_inet_sock_release",
79
  [BPF_CGROUP_SOCK_OPS]   = "cgroup_sock_ops",
80
  [BPF_CGROUP_DEVICE]   = "cgroup_device",
81
  [BPF_CGROUP_INET4_BIND]   = "cgroup_inet4_bind",
82
  [BPF_CGROUP_INET6_BIND]   = "cgroup_inet6_bind",
83
  [BPF_CGROUP_INET4_CONNECT]  = "cgroup_inet4_connect",
84
  [BPF_CGROUP_INET6_CONNECT]  = "cgroup_inet6_connect",
85
  [BPF_CGROUP_UNIX_CONNECT]       = "cgroup_unix_connect",
86
  [BPF_CGROUP_INET4_POST_BIND]  = "cgroup_inet4_post_bind",
87
  [BPF_CGROUP_INET6_POST_BIND]  = "cgroup_inet6_post_bind",
88
  [BPF_CGROUP_INET4_GETPEERNAME]  = "cgroup_inet4_getpeername",
89
  [BPF_CGROUP_INET6_GETPEERNAME]  = "cgroup_inet6_getpeername",
90
  [BPF_CGROUP_UNIX_GETPEERNAME] = "cgroup_unix_getpeername",
91
  [BPF_CGROUP_INET4_GETSOCKNAME]  = "cgroup_inet4_getsockname",
92
  [BPF_CGROUP_INET6_GETSOCKNAME]  = "cgroup_inet6_getsockname",
93
  [BPF_CGROUP_UNIX_GETSOCKNAME] = "cgroup_unix_getsockname",
94
  [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg",
95
  [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg",
96
  [BPF_CGROUP_UNIX_SENDMSG] = "cgroup_unix_sendmsg",
97
  [BPF_CGROUP_SYSCTL]   = "cgroup_sysctl",
98
  [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg",
99
  [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg",
100
  [BPF_CGROUP_UNIX_RECVMSG] = "cgroup_unix_recvmsg",
101
  [BPF_CGROUP_GETSOCKOPT]   = "cgroup_getsockopt",
102
  [BPF_CGROUP_SETSOCKOPT]   = "cgroup_setsockopt",
103
  [BPF_SK_SKB_STREAM_PARSER]  = "sk_skb_stream_parser",
104
  [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
105
  [BPF_SK_SKB_VERDICT]    = "sk_skb_verdict",
106
  [BPF_SK_MSG_VERDICT]    = "sk_msg_verdict",
107
  [BPF_LIRC_MODE2]    = "lirc_mode2",
108
  [BPF_FLOW_DISSECTOR]    = "flow_dissector",
109
  [BPF_TRACE_RAW_TP]    = "trace_raw_tp",
110
  [BPF_TRACE_FENTRY]    = "trace_fentry",
111
  [BPF_TRACE_FEXIT]   = "trace_fexit",
112
  [BPF_MODIFY_RETURN]   = "modify_return",
113
  [BPF_LSM_MAC]     = "lsm_mac",
114
  [BPF_LSM_CGROUP]    = "lsm_cgroup",
115
  [BPF_SK_LOOKUP]     = "sk_lookup",
116
  [BPF_TRACE_ITER]    = "trace_iter",
117
  [BPF_XDP_DEVMAP]    = "xdp_devmap",
118
  [BPF_XDP_CPUMAP]    = "xdp_cpumap",
119
  [BPF_XDP]     = "xdp",
120
  [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select",
121
  [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE]  = "sk_reuseport_select_or_migrate",
122
  [BPF_PERF_EVENT]    = "perf_event",
123
  [BPF_TRACE_KPROBE_MULTI]  = "trace_kprobe_multi",
124
  [BPF_STRUCT_OPS]    = "struct_ops",
125
  [BPF_NETFILTER]     = "netfilter",
126
  [BPF_TCX_INGRESS]   = "tcx_ingress",
127
  [BPF_TCX_EGRESS]    = "tcx_egress",
128
  [BPF_TRACE_UPROBE_MULTI]  = "trace_uprobe_multi",
129
  [BPF_NETKIT_PRIMARY]    = "netkit_primary",
130
  [BPF_NETKIT_PEER]   = "netkit_peer",
131
};
132
133
static const char * const link_type_name[] = {
134
  [BPF_LINK_TYPE_UNSPEC]      = "unspec",
135
  [BPF_LINK_TYPE_RAW_TRACEPOINT]    = "raw_tracepoint",
136
  [BPF_LINK_TYPE_TRACING]     = "tracing",
137
  [BPF_LINK_TYPE_CGROUP]      = "cgroup",
138
  [BPF_LINK_TYPE_ITER]      = "iter",
139
  [BPF_LINK_TYPE_NETNS]     = "netns",
140
  [BPF_LINK_TYPE_XDP]     = "xdp",
141
  [BPF_LINK_TYPE_PERF_EVENT]    = "perf_event",
142
  [BPF_LINK_TYPE_KPROBE_MULTI]    = "kprobe_multi",
143
  [BPF_LINK_TYPE_STRUCT_OPS]    = "struct_ops",
144
  [BPF_LINK_TYPE_NETFILTER]   = "netfilter",
145
  [BPF_LINK_TYPE_TCX]     = "tcx",
146
  [BPF_LINK_TYPE_UPROBE_MULTI]    = "uprobe_multi",
147
  [BPF_LINK_TYPE_NETKIT]      = "netkit",
148
};
149
150
static const char * const map_type_name[] = {
151
  [BPF_MAP_TYPE_UNSPEC]     = "unspec",
152
  [BPF_MAP_TYPE_HASH]     = "hash",
153
  [BPF_MAP_TYPE_ARRAY]      = "array",
154
  [BPF_MAP_TYPE_PROG_ARRAY]   = "prog_array",
155
  [BPF_MAP_TYPE_PERF_EVENT_ARRAY]   = "perf_event_array",
156
  [BPF_MAP_TYPE_PERCPU_HASH]    = "percpu_hash",
157
  [BPF_MAP_TYPE_PERCPU_ARRAY]   = "percpu_array",
158
  [BPF_MAP_TYPE_STACK_TRACE]    = "stack_trace",
159
  [BPF_MAP_TYPE_CGROUP_ARRAY]   = "cgroup_array",
160
  [BPF_MAP_TYPE_LRU_HASH]     = "lru_hash",
161
  [BPF_MAP_TYPE_LRU_PERCPU_HASH]    = "lru_percpu_hash",
162
  [BPF_MAP_TYPE_LPM_TRIE]     = "lpm_trie",
163
  [BPF_MAP_TYPE_ARRAY_OF_MAPS]    = "array_of_maps",
164
  [BPF_MAP_TYPE_HASH_OF_MAPS]   = "hash_of_maps",
165
  [BPF_MAP_TYPE_DEVMAP]     = "devmap",
166
  [BPF_MAP_TYPE_DEVMAP_HASH]    = "devmap_hash",
167
  [BPF_MAP_TYPE_SOCKMAP]      = "sockmap",
168
  [BPF_MAP_TYPE_CPUMAP]     = "cpumap",
169
  [BPF_MAP_TYPE_XSKMAP]     = "xskmap",
170
  [BPF_MAP_TYPE_SOCKHASH]     = "sockhash",
171
  [BPF_MAP_TYPE_CGROUP_STORAGE]   = "cgroup_storage",
172
  [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY]  = "reuseport_sockarray",
173
  [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE]  = "percpu_cgroup_storage",
174
  [BPF_MAP_TYPE_QUEUE]      = "queue",
175
  [BPF_MAP_TYPE_STACK]      = "stack",
176
  [BPF_MAP_TYPE_SK_STORAGE]   = "sk_storage",
177
  [BPF_MAP_TYPE_STRUCT_OPS]   = "struct_ops",
178
  [BPF_MAP_TYPE_RINGBUF]      = "ringbuf",
179
  [BPF_MAP_TYPE_INODE_STORAGE]    = "inode_storage",
180
  [BPF_MAP_TYPE_TASK_STORAGE]   = "task_storage",
181
  [BPF_MAP_TYPE_BLOOM_FILTER]   = "bloom_filter",
182
  [BPF_MAP_TYPE_USER_RINGBUF]             = "user_ringbuf",
183
  [BPF_MAP_TYPE_CGRP_STORAGE]   = "cgrp_storage",
184
};
185
186
static const char * const prog_type_name[] = {
187
  [BPF_PROG_TYPE_UNSPEC]      = "unspec",
188
  [BPF_PROG_TYPE_SOCKET_FILTER]   = "socket_filter",
189
  [BPF_PROG_TYPE_KPROBE]      = "kprobe",
190
  [BPF_PROG_TYPE_SCHED_CLS]   = "sched_cls",
191
  [BPF_PROG_TYPE_SCHED_ACT]   = "sched_act",
192
  [BPF_PROG_TYPE_TRACEPOINT]    = "tracepoint",
193
  [BPF_PROG_TYPE_XDP]     = "xdp",
194
  [BPF_PROG_TYPE_PERF_EVENT]    = "perf_event",
195
  [BPF_PROG_TYPE_CGROUP_SKB]    = "cgroup_skb",
196
  [BPF_PROG_TYPE_CGROUP_SOCK]   = "cgroup_sock",
197
  [BPF_PROG_TYPE_LWT_IN]      = "lwt_in",
198
  [BPF_PROG_TYPE_LWT_OUT]     = "lwt_out",
199
  [BPF_PROG_TYPE_LWT_XMIT]    = "lwt_xmit",
200
  [BPF_PROG_TYPE_SOCK_OPS]    = "sock_ops",
201
  [BPF_PROG_TYPE_SK_SKB]      = "sk_skb",
202
  [BPF_PROG_TYPE_CGROUP_DEVICE]   = "cgroup_device",
203
  [BPF_PROG_TYPE_SK_MSG]      = "sk_msg",
204
  [BPF_PROG_TYPE_RAW_TRACEPOINT]    = "raw_tracepoint",
205
  [BPF_PROG_TYPE_CGROUP_SOCK_ADDR]  = "cgroup_sock_addr",
206
  [BPF_PROG_TYPE_LWT_SEG6LOCAL]   = "lwt_seg6local",
207
  [BPF_PROG_TYPE_LIRC_MODE2]    = "lirc_mode2",
208
  [BPF_PROG_TYPE_SK_REUSEPORT]    = "sk_reuseport",
209
  [BPF_PROG_TYPE_FLOW_DISSECTOR]    = "flow_dissector",
210
  [BPF_PROG_TYPE_CGROUP_SYSCTL]   = "cgroup_sysctl",
211
  [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
212
  [BPF_PROG_TYPE_CGROUP_SOCKOPT]    = "cgroup_sockopt",
213
  [BPF_PROG_TYPE_TRACING]     = "tracing",
214
  [BPF_PROG_TYPE_STRUCT_OPS]    = "struct_ops",
215
  [BPF_PROG_TYPE_EXT]     = "ext",
216
  [BPF_PROG_TYPE_LSM]     = "lsm",
217
  [BPF_PROG_TYPE_SK_LOOKUP]   = "sk_lookup",
218
  [BPF_PROG_TYPE_SYSCALL]     = "syscall",
219
  [BPF_PROG_TYPE_NETFILTER]   = "netfilter",
220
};
221
222
static int __base_pr(enum libbpf_print_level level, const char *format,
223
         va_list args)
224
0
{
225
0
  if (level == LIBBPF_DEBUG)
226
0
    return 0;
227
228
0
  return vfprintf(stderr, format, args);
229
0
}
230
231
static libbpf_print_fn_t __libbpf_pr = __base_pr;
232
233
libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
234
10.3k
{
235
10.3k
  libbpf_print_fn_t old_print_fn;
236
237
10.3k
  old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED);
238
239
10.3k
  return old_print_fn;
240
10.3k
}
241
242
__printf(2, 3)
243
void libbpf_print(enum libbpf_print_level level, const char *format, ...)
244
105k
{
245
105k
  va_list args;
246
105k
  int old_errno;
247
105k
  libbpf_print_fn_t print_fn;
248
249
105k
  print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED);
250
105k
  if (!print_fn)
251
0
    return;
252
253
105k
  old_errno = errno;
254
255
105k
  va_start(args, format);
256
105k
  __libbpf_pr(level, format, args);
257
105k
  va_end(args);
258
259
105k
  errno = old_errno;
260
105k
}
261
262
static void pr_perm_msg(int err)
263
0
{
264
0
  struct rlimit limit;
265
0
  char buf[100];
266
267
0
  if (err != -EPERM || geteuid() != 0)
268
0
    return;
269
270
0
  err = getrlimit(RLIMIT_MEMLOCK, &limit);
271
0
  if (err)
272
0
    return;
273
274
0
  if (limit.rlim_cur == RLIM_INFINITY)
275
0
    return;
276
277
0
  if (limit.rlim_cur < 1024)
278
0
    snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
279
0
  else if (limit.rlim_cur < 1024*1024)
280
0
    snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
281
0
  else
282
0
    snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
283
284
0
  pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
285
0
    buf);
286
0
}
287
288
#define STRERR_BUFSIZE  128
289
290
/* Copied from tools/perf/util/util.h */
291
#ifndef zfree
292
171k
# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
293
#endif
294
295
#ifndef zclose
296
34.5k
# define zclose(fd) ({      \
297
34.5k
  int ___err = 0;     \
298
34.5k
  if ((fd) >= 0)     \
299
34.5k
    ___err = close((fd)); \
300
34.5k
  fd = -1;      \
301
34.5k
  ___err; })
302
#endif
303
304
static inline __u64 ptr_to_u64(const void *ptr)
305
0
{
306
0
  return (__u64) (unsigned long) ptr;
307
0
}
308
309
int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
310
0
{
311
  /* as of v1.0 libbpf_set_strict_mode() is a no-op */
312
0
  return 0;
313
0
}
314
315
__u32 libbpf_major_version(void)
316
0
{
317
0
  return LIBBPF_MAJOR_VERSION;
318
0
}
319
320
__u32 libbpf_minor_version(void)
321
0
{
322
0
  return LIBBPF_MINOR_VERSION;
323
0
}
324
325
const char *libbpf_version_string(void)
326
0
{
327
0
#define __S(X) #X
328
0
#define _S(X) __S(X)
329
0
  return  "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);
330
0
#undef _S
331
0
#undef __S
332
0
}
333
334
enum reloc_type {
335
  RELO_LD64,
336
  RELO_CALL,
337
  RELO_DATA,
338
  RELO_EXTERN_LD64,
339
  RELO_EXTERN_CALL,
340
  RELO_SUBPROG_ADDR,
341
  RELO_CORE,
342
};
343
344
struct reloc_desc {
345
  enum reloc_type type;
346
  int insn_idx;
347
  union {
348
    const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */
349
    struct {
350
      int map_idx;
351
      int sym_off;
352
      int ext_idx;
353
    };
354
  };
355
};
356
357
/* stored as sec_def->cookie for all libbpf-supported SEC()s */
358
enum sec_def_flags {
359
  SEC_NONE = 0,
360
  /* expected_attach_type is optional, if kernel doesn't support that */
361
  SEC_EXP_ATTACH_OPT = 1,
362
  /* legacy, only used by libbpf_get_type_names() and
363
   * libbpf_attach_type_by_name(), not used by libbpf itself at all.
364
   * This used to be associated with cgroup (and few other) BPF programs
365
   * that were attachable through BPF_PROG_ATTACH command. Pretty
366
   * meaningless nowadays, though.
367
   */
368
  SEC_ATTACHABLE = 2,
369
  SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT,
370
  /* attachment target is specified through BTF ID in either kernel or
371
   * other BPF program's BTF object
372
   */
373
  SEC_ATTACH_BTF = 4,
374
  /* BPF program type allows sleeping/blocking in kernel */
375
  SEC_SLEEPABLE = 8,
376
  /* BPF program support non-linear XDP buffer */
377
  SEC_XDP_FRAGS = 16,
378
  /* Setup proper attach type for usdt probes. */
379
  SEC_USDT = 32,
380
};
381
382
struct bpf_sec_def {
383
  char *sec;
384
  enum bpf_prog_type prog_type;
385
  enum bpf_attach_type expected_attach_type;
386
  long cookie;
387
  int handler_id;
388
389
  libbpf_prog_setup_fn_t prog_setup_fn;
390
  libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;
391
  libbpf_prog_attach_fn_t prog_attach_fn;
392
};
393
394
/*
395
 * bpf_prog should be a better name but it has been used in
396
 * linux/filter.h.
397
 */
398
struct bpf_program {
399
  char *name;
400
  char *sec_name;
401
  size_t sec_idx;
402
  const struct bpf_sec_def *sec_def;
403
  /* this program's instruction offset (in number of instructions)
404
   * within its containing ELF section
405
   */
406
  size_t sec_insn_off;
407
  /* number of original instructions in ELF section belonging to this
408
   * program, not taking into account subprogram instructions possible
409
   * appended later during relocation
410
   */
411
  size_t sec_insn_cnt;
412
  /* Offset (in number of instructions) of the start of instruction
413
   * belonging to this BPF program  within its containing main BPF
414
   * program. For the entry-point (main) BPF program, this is always
415
   * zero. For a sub-program, this gets reset before each of main BPF
416
   * programs are processed and relocated and is used to determined
417
   * whether sub-program was already appended to the main program, and
418
   * if yes, at which instruction offset.
419
   */
420
  size_t sub_insn_off;
421
422
  /* instructions that belong to BPF program; insns[0] is located at
423
   * sec_insn_off instruction within its ELF section in ELF file, so
424
   * when mapping ELF file instruction index to the local instruction,
425
   * one needs to subtract sec_insn_off; and vice versa.
426
   */
427
  struct bpf_insn *insns;
428
  /* actual number of instruction in this BPF program's image; for
429
   * entry-point BPF programs this includes the size of main program
430
   * itself plus all the used sub-programs, appended at the end
431
   */
432
  size_t insns_cnt;
433
434
  struct reloc_desc *reloc_desc;
435
  int nr_reloc;
436
437
  /* BPF verifier log settings */
438
  char *log_buf;
439
  size_t log_size;
440
  __u32 log_level;
441
442
  struct bpf_object *obj;
443
444
  int fd;
445
  bool autoload;
446
  bool autoattach;
447
  bool sym_global;
448
  bool mark_btf_static;
449
  enum bpf_prog_type type;
450
  enum bpf_attach_type expected_attach_type;
451
  int exception_cb_idx;
452
453
  int prog_ifindex;
454
  __u32 attach_btf_obj_fd;
455
  __u32 attach_btf_id;
456
  __u32 attach_prog_fd;
457
458
  void *func_info;
459
  __u32 func_info_rec_size;
460
  __u32 func_info_cnt;
461
462
  void *line_info;
463
  __u32 line_info_rec_size;
464
  __u32 line_info_cnt;
465
  __u32 prog_flags;
466
};
467
468
struct bpf_struct_ops {
469
  const char *tname;
470
  const struct btf_type *type;
471
  struct bpf_program **progs;
472
  __u32 *kern_func_off;
473
  /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
474
  void *data;
475
  /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
476
   *      btf_vmlinux's format.
477
   * struct bpf_struct_ops_tcp_congestion_ops {
478
   *  [... some other kernel fields ...]
479
   *  struct tcp_congestion_ops data;
480
   * }
481
   * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
482
   * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
483
   * from "data".
484
   */
485
  void *kern_vdata;
486
  __u32 type_id;
487
};
488
489
1.45k
#define DATA_SEC ".data"
490
1.33k
#define BSS_SEC ".bss"
491
940
#define RODATA_SEC ".rodata"
492
5.04k
#define KCONFIG_SEC ".kconfig"
493
6.41k
#define KSYMS_SEC ".ksyms"
494
3.56k
#define STRUCT_OPS_SEC ".struct_ops"
495
3.37k
#define STRUCT_OPS_LINK_SEC ".struct_ops.link"
496
497
enum libbpf_map_type {
498
  LIBBPF_MAP_UNSPEC,
499
  LIBBPF_MAP_DATA,
500
  LIBBPF_MAP_BSS,
501
  LIBBPF_MAP_RODATA,
502
  LIBBPF_MAP_KCONFIG,
503
};
504
505
struct bpf_map_def {
506
  unsigned int type;
507
  unsigned int key_size;
508
  unsigned int value_size;
509
  unsigned int max_entries;
510
  unsigned int map_flags;
511
};
512
513
struct bpf_map {
514
  struct bpf_object *obj;
515
  char *name;
516
  /* real_name is defined for special internal maps (.rodata*,
517
   * .data*, .bss, .kconfig) and preserves their original ELF section
518
   * name. This is important to be able to find corresponding BTF
519
   * DATASEC information.
520
   */
521
  char *real_name;
522
  int fd;
523
  int sec_idx;
524
  size_t sec_offset;
525
  int map_ifindex;
526
  int inner_map_fd;
527
  struct bpf_map_def def;
528
  __u32 numa_node;
529
  __u32 btf_var_idx;
530
  __u32 btf_key_type_id;
531
  __u32 btf_value_type_id;
532
  __u32 btf_vmlinux_value_type_id;
533
  enum libbpf_map_type libbpf_type;
534
  void *mmaped;
535
  struct bpf_struct_ops *st_ops;
536
  struct bpf_map *inner_map;
537
  void **init_slots;
538
  int init_slots_sz;
539
  char *pin_path;
540
  bool pinned;
541
  bool reused;
542
  bool autocreate;
543
  __u64 map_extra;
544
};
545
546
enum extern_type {
547
  EXT_UNKNOWN,
548
  EXT_KCFG,
549
  EXT_KSYM,
550
};
551
552
enum kcfg_type {
553
  KCFG_UNKNOWN,
554
  KCFG_CHAR,
555
  KCFG_BOOL,
556
  KCFG_INT,
557
  KCFG_TRISTATE,
558
  KCFG_CHAR_ARR,
559
};
560
561
struct extern_desc {
562
  enum extern_type type;
563
  int sym_idx;
564
  int btf_id;
565
  int sec_btf_id;
566
  const char *name;
567
  char *essent_name;
568
  bool is_set;
569
  bool is_weak;
570
  union {
571
    struct {
572
      enum kcfg_type type;
573
      int sz;
574
      int align;
575
      int data_off;
576
      bool is_signed;
577
    } kcfg;
578
    struct {
579
      unsigned long long addr;
580
581
      /* target btf_id of the corresponding kernel var. */
582
      int kernel_btf_obj_fd;
583
      int kernel_btf_id;
584
585
      /* local btf_id of the ksym extern's type. */
586
      __u32 type_id;
587
      /* BTF fd index to be patched in for insn->off, this is
588
       * 0 for vmlinux BTF, index in obj->fd_array for module
589
       * BTF
590
       */
591
      __s16 btf_fd_idx;
592
    } ksym;
593
  };
594
};
595
596
struct module_btf {
597
  struct btf *btf;
598
  char *name;
599
  __u32 id;
600
  int fd;
601
  int fd_array_idx;
602
};
603
604
enum sec_type {
605
  SEC_UNUSED = 0,
606
  SEC_RELO,
607
  SEC_BSS,
608
  SEC_DATA,
609
  SEC_RODATA,
610
};
611
612
struct elf_sec_desc {
613
  enum sec_type sec_type;
614
  Elf64_Shdr *shdr;
615
  Elf_Data *data;
616
};
617
618
struct elf_state {
619
  int fd;
620
  const void *obj_buf;
621
  size_t obj_buf_sz;
622
  Elf *elf;
623
  Elf64_Ehdr *ehdr;
624
  Elf_Data *symbols;
625
  Elf_Data *st_ops_data;
626
  Elf_Data *st_ops_link_data;
627
  size_t shstrndx; /* section index for section name strings */
628
  size_t strtabidx;
629
  struct elf_sec_desc *secs;
630
  size_t sec_cnt;
631
  int btf_maps_shndx;
632
  __u32 btf_maps_sec_btf_id;
633
  int text_shndx;
634
  int symbols_shndx;
635
  int st_ops_shndx;
636
  int st_ops_link_shndx;
637
};
638
639
struct usdt_manager;
640
641
struct bpf_object {
642
  char name[BPF_OBJ_NAME_LEN];
643
  char license[64];
644
  __u32 kern_version;
645
646
  struct bpf_program *programs;
647
  size_t nr_programs;
648
  struct bpf_map *maps;
649
  size_t nr_maps;
650
  size_t maps_cap;
651
652
  char *kconfig;
653
  struct extern_desc *externs;
654
  int nr_extern;
655
  int kconfig_map_idx;
656
657
  bool loaded;
658
  bool has_subcalls;
659
  bool has_rodata;
660
661
  struct bpf_gen *gen_loader;
662
663
  /* Information when doing ELF related work. Only valid if efile.elf is not NULL */
664
  struct elf_state efile;
665
666
  struct btf *btf;
667
  struct btf_ext *btf_ext;
668
669
  /* Parse and load BTF vmlinux if any of the programs in the object need
670
   * it at load time.
671
   */
672
  struct btf *btf_vmlinux;
673
  /* Path to the custom BTF to be used for BPF CO-RE relocations as an
674
   * override for vmlinux BTF.
675
   */
676
  char *btf_custom_path;
677
  /* vmlinux BTF override for CO-RE relocations */
678
  struct btf *btf_vmlinux_override;
679
  /* Lazily initialized kernel module BTFs */
680
  struct module_btf *btf_modules;
681
  bool btf_modules_loaded;
682
  size_t btf_module_cnt;
683
  size_t btf_module_cap;
684
685
  /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */
686
  char *log_buf;
687
  size_t log_size;
688
  __u32 log_level;
689
690
  int *fd_array;
691
  size_t fd_array_cap;
692
  size_t fd_array_cnt;
693
694
  struct usdt_manager *usdt_man;
695
696
  char path[];
697
};
698
699
static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
700
static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
701
static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
702
static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
703
static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);
704
static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
705
static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
706
static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);
707
static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx);
708
709
void bpf_program__unload(struct bpf_program *prog)
710
22.4k
{
711
22.4k
  if (!prog)
712
0
    return;
713
714
22.4k
  zclose(prog->fd);
715
716
22.4k
  zfree(&prog->func_info);
717
22.4k
  zfree(&prog->line_info);
718
22.4k
}
719
720
static void bpf_program__exit(struct bpf_program *prog)
721
11.2k
{
722
11.2k
  if (!prog)
723
0
    return;
724
725
11.2k
  bpf_program__unload(prog);
726
11.2k
  zfree(&prog->name);
727
11.2k
  zfree(&prog->sec_name);
728
11.2k
  zfree(&prog->insns);
729
11.2k
  zfree(&prog->reloc_desc);
730
731
11.2k
  prog->nr_reloc = 0;
732
11.2k
  prog->insns_cnt = 0;
733
11.2k
  prog->sec_idx = -1;
734
11.2k
}
735
736
static bool insn_is_subprog_call(const struct bpf_insn *insn)
737
0
{
738
0
  return BPF_CLASS(insn->code) == BPF_JMP &&
739
0
         BPF_OP(insn->code) == BPF_CALL &&
740
0
         BPF_SRC(insn->code) == BPF_K &&
741
0
         insn->src_reg == BPF_PSEUDO_CALL &&
742
0
         insn->dst_reg == 0 &&
743
0
         insn->off == 0;
744
0
}
745
746
static bool is_call_insn(const struct bpf_insn *insn)
747
1.90k
{
748
1.90k
  return insn->code == (BPF_JMP | BPF_CALL);
749
1.90k
}
750
751
static bool insn_is_pseudo_func(struct bpf_insn *insn)
752
0
{
753
0
  return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
754
0
}
755
756
static int
757
bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
758
          const char *name, size_t sec_idx, const char *sec_name,
759
          size_t sec_off, void *insn_data, size_t insn_data_sz)
760
11.2k
{
761
11.2k
  if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
762
59
    pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
763
59
      sec_name, name, sec_off, insn_data_sz);
764
59
    return -EINVAL;
765
59
  }
766
767
11.2k
  memset(prog, 0, sizeof(*prog));
768
11.2k
  prog->obj = obj;
769
770
11.2k
  prog->sec_idx = sec_idx;
771
11.2k
  prog->sec_insn_off = sec_off / BPF_INSN_SZ;
772
11.2k
  prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
773
  /* insns_cnt can later be increased by appending used subprograms */
774
11.2k
  prog->insns_cnt = prog->sec_insn_cnt;
775
776
11.2k
  prog->type = BPF_PROG_TYPE_UNSPEC;
777
11.2k
  prog->fd = -1;
778
11.2k
  prog->exception_cb_idx = -1;
779
780
  /* libbpf's convention for SEC("?abc...") is that it's just like
781
   * SEC("abc...") but the corresponding bpf_program starts out with
782
   * autoload set to false.
783
   */
784
11.2k
  if (sec_name[0] == '?') {
785
222
    prog->autoload = false;
786
    /* from now on forget there was ? in section name */
787
222
    sec_name++;
788
11.0k
  } else {
789
11.0k
    prog->autoload = true;
790
11.0k
  }
791
792
11.2k
  prog->autoattach = true;
793
794
  /* inherit object's log_level */
795
11.2k
  prog->log_level = obj->log_level;
796
797
11.2k
  prog->sec_name = strdup(sec_name);
798
11.2k
  if (!prog->sec_name)
799
0
    goto errout;
800
801
11.2k
  prog->name = strdup(name);
802
11.2k
  if (!prog->name)
803
0
    goto errout;
804
805
11.2k
  prog->insns = malloc(insn_data_sz);
806
11.2k
  if (!prog->insns)
807
17
    goto errout;
808
11.2k
  memcpy(prog->insns, insn_data, insn_data_sz);
809
810
11.2k
  return 0;
811
17
errout:
812
17
  pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
813
17
  bpf_program__exit(prog);
814
17
  return -ENOMEM;
815
11.2k
}
816
817
static int
818
bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
819
       const char *sec_name, int sec_idx)
820
1.20k
{
821
1.20k
  Elf_Data *symbols = obj->efile.symbols;
822
1.20k
  struct bpf_program *prog, *progs;
823
1.20k
  void *data = sec_data->d_buf;
824
1.20k
  size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
825
1.20k
  int nr_progs, err, i;
826
1.20k
  const char *name;
827
1.20k
  Elf64_Sym *sym;
828
829
1.20k
  progs = obj->programs;
830
1.20k
  nr_progs = obj->nr_programs;
831
1.20k
  nr_syms = symbols->d_size / sizeof(Elf64_Sym);
832
833
257k
  for (i = 0; i < nr_syms; i++) {
834
256k
    sym = elf_sym_by_idx(obj, i);
835
836
256k
    if (sym->st_shndx != sec_idx)
837
243k
      continue;
838
13.2k
    if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
839
1.72k
      continue;
840
841
11.4k
    prog_sz = sym->st_size;
842
11.4k
    sec_off = sym->st_value;
843
844
11.4k
    name = elf_sym_str(obj, sym->st_name);
845
11.4k
    if (!name) {
846
68
      pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
847
68
        sec_name, sec_off);
848
68
      return -LIBBPF_ERRNO__FORMAT;
849
68
    }
850
851
11.4k
    if (sec_off + prog_sz > sec_sz) {
852
121
      pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
853
121
        sec_name, sec_off);
854
121
      return -LIBBPF_ERRNO__FORMAT;
855
121
    }
856
857
11.3k
    if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
858
6
      pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);
859
6
      return -ENOTSUP;
860
6
    }
861
862
11.2k
    pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
863
22.5k
       sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
864
865
11.2k
    progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
866
11.2k
    if (!progs) {
867
      /*
868
       * In this case the original obj->programs
869
       * is still valid, so don't need special treat for
870
       * bpf_close_object().
871
       */
872
0
      pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
873
0
        sec_name, name);
874
0
      return -ENOMEM;
875
0
    }
876
11.2k
    obj->programs = progs;
877
878
11.2k
    prog = &progs[nr_progs];
879
880
11.2k
    err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
881
11.2k
              sec_off, data + sec_off, prog_sz);
882
11.2k
    if (err)
883
76
      return err;
884
885
11.2k
    if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL)
886
10.9k
      prog->sym_global = true;
887
888
    /* if function is a global/weak symbol, but has restricted
889
     * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC
890
     * as static to enable more permissive BPF verification mode
891
     * with more outside context available to BPF verifier
892
     */
893
11.2k
    if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
894
10.9k
        || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
895
7.93k
      prog->mark_btf_static = true;
896
897
11.2k
    nr_progs++;
898
11.2k
    obj->nr_programs = nr_progs;
899
11.2k
  }
900
901
929
  return 0;
902
1.20k
}
903
904
static const struct btf_member *
905
find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
906
0
{
907
0
  struct btf_member *m;
908
0
  int i;
909
910
0
  for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
911
0
    if (btf_member_bit_offset(t, i) == bit_offset)
912
0
      return m;
913
0
  }
914
915
0
  return NULL;
916
0
}
917
918
static const struct btf_member *
919
find_member_by_name(const struct btf *btf, const struct btf_type *t,
920
        const char *name)
921
0
{
922
0
  struct btf_member *m;
923
0
  int i;
924
925
0
  for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
926
0
    if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
927
0
      return m;
928
0
  }
929
930
0
  return NULL;
931
0
}
932
933
0
#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
934
static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
935
           const char *name, __u32 kind);
936
937
static int
938
find_struct_ops_kern_types(const struct btf *btf, const char *tname,
939
         const struct btf_type **type, __u32 *type_id,
940
         const struct btf_type **vtype, __u32 *vtype_id,
941
         const struct btf_member **data_member)
942
0
{
943
0
  const struct btf_type *kern_type, *kern_vtype;
944
0
  const struct btf_member *kern_data_member;
945
0
  __s32 kern_vtype_id, kern_type_id;
946
0
  __u32 i;
947
948
0
  kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT);
949
0
  if (kern_type_id < 0) {
950
0
    pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
951
0
      tname);
952
0
    return kern_type_id;
953
0
  }
954
0
  kern_type = btf__type_by_id(btf, kern_type_id);
955
956
  /* Find the corresponding "map_value" type that will be used
957
   * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,
958
   * find "struct bpf_struct_ops_tcp_congestion_ops" from the
959
   * btf_vmlinux.
960
   */
961
0
  kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
962
0
            tname, BTF_KIND_STRUCT);
963
0
  if (kern_vtype_id < 0) {
964
0
    pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
965
0
      STRUCT_OPS_VALUE_PREFIX, tname);
966
0
    return kern_vtype_id;
967
0
  }
968
0
  kern_vtype = btf__type_by_id(btf, kern_vtype_id);
969
970
  /* Find "struct tcp_congestion_ops" from
971
   * struct bpf_struct_ops_tcp_congestion_ops {
972
   *  [ ... ]
973
   *  struct tcp_congestion_ops data;
974
   * }
975
   */
976
0
  kern_data_member = btf_members(kern_vtype);
977
0
  for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
978
0
    if (kern_data_member->type == kern_type_id)
979
0
      break;
980
0
  }
981
0
  if (i == btf_vlen(kern_vtype)) {
982
0
    pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
983
0
      tname, STRUCT_OPS_VALUE_PREFIX, tname);
984
0
    return -EINVAL;
985
0
  }
986
987
0
  *type = kern_type;
988
0
  *type_id = kern_type_id;
989
0
  *vtype = kern_vtype;
990
0
  *vtype_id = kern_vtype_id;
991
0
  *data_member = kern_data_member;
992
993
0
  return 0;
994
0
}
995
996
static bool bpf_map__is_struct_ops(const struct bpf_map *map)
997
377
{
998
377
  return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
999
377
}
1000
1001
/* Init the map's fields that depend on kern_btf */
1002
static int bpf_map__init_kern_struct_ops(struct bpf_map *map,
1003
           const struct btf *btf,
1004
           const struct btf *kern_btf)
1005
0
{
1006
0
  const struct btf_member *member, *kern_member, *kern_data_member;
1007
0
  const struct btf_type *type, *kern_type, *kern_vtype;
1008
0
  __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
1009
0
  struct bpf_struct_ops *st_ops;
1010
0
  void *data, *kern_data;
1011
0
  const char *tname;
1012
0
  int err;
1013
1014
0
  st_ops = map->st_ops;
1015
0
  type = st_ops->type;
1016
0
  tname = st_ops->tname;
1017
0
  err = find_struct_ops_kern_types(kern_btf, tname,
1018
0
           &kern_type, &kern_type_id,
1019
0
           &kern_vtype, &kern_vtype_id,
1020
0
           &kern_data_member);
1021
0
  if (err)
1022
0
    return err;
1023
1024
0
  pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
1025
0
     map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
1026
1027
0
  map->def.value_size = kern_vtype->size;
1028
0
  map->btf_vmlinux_value_type_id = kern_vtype_id;
1029
1030
0
  st_ops->kern_vdata = calloc(1, kern_vtype->size);
1031
0
  if (!st_ops->kern_vdata)
1032
0
    return -ENOMEM;
1033
1034
0
  data = st_ops->data;
1035
0
  kern_data_off = kern_data_member->offset / 8;
1036
0
  kern_data = st_ops->kern_vdata + kern_data_off;
1037
1038
0
  member = btf_members(type);
1039
0
  for (i = 0; i < btf_vlen(type); i++, member++) {
1040
0
    const struct btf_type *mtype, *kern_mtype;
1041
0
    __u32 mtype_id, kern_mtype_id;
1042
0
    void *mdata, *kern_mdata;
1043
0
    __s64 msize, kern_msize;
1044
0
    __u32 moff, kern_moff;
1045
0
    __u32 kern_member_idx;
1046
0
    const char *mname;
1047
1048
0
    mname = btf__name_by_offset(btf, member->name_off);
1049
0
    kern_member = find_member_by_name(kern_btf, kern_type, mname);
1050
0
    if (!kern_member) {
1051
0
      pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
1052
0
        map->name, mname);
1053
0
      return -ENOTSUP;
1054
0
    }
1055
1056
0
    kern_member_idx = kern_member - btf_members(kern_type);
1057
0
    if (btf_member_bitfield_size(type, i) ||
1058
0
        btf_member_bitfield_size(kern_type, kern_member_idx)) {
1059
0
      pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
1060
0
        map->name, mname);
1061
0
      return -ENOTSUP;
1062
0
    }
1063
1064
0
    moff = member->offset / 8;
1065
0
    kern_moff = kern_member->offset / 8;
1066
1067
0
    mdata = data + moff;
1068
0
    kern_mdata = kern_data + kern_moff;
1069
1070
0
    mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
1071
0
    kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
1072
0
                &kern_mtype_id);
1073
0
    if (BTF_INFO_KIND(mtype->info) !=
1074
0
        BTF_INFO_KIND(kern_mtype->info)) {
1075
0
      pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
1076
0
        map->name, mname, BTF_INFO_KIND(mtype->info),
1077
0
        BTF_INFO_KIND(kern_mtype->info));
1078
0
      return -ENOTSUP;
1079
0
    }
1080
1081
0
    if (btf_is_ptr(mtype)) {
1082
0
      struct bpf_program *prog;
1083
1084
0
      prog = st_ops->progs[i];
1085
0
      if (!prog)
1086
0
        continue;
1087
1088
0
      kern_mtype = skip_mods_and_typedefs(kern_btf,
1089
0
                  kern_mtype->type,
1090
0
                  &kern_mtype_id);
1091
1092
      /* mtype->type must be a func_proto which was
1093
       * guaranteed in bpf_object__collect_st_ops_relos(),
1094
       * so only check kern_mtype for func_proto here.
1095
       */
1096
0
      if (!btf_is_func_proto(kern_mtype)) {
1097
0
        pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
1098
0
          map->name, mname);
1099
0
        return -ENOTSUP;
1100
0
      }
1101
1102
0
      prog->attach_btf_id = kern_type_id;
1103
0
      prog->expected_attach_type = kern_member_idx;
1104
1105
0
      st_ops->kern_func_off[i] = kern_data_off + kern_moff;
1106
1107
0
      pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1108
0
         map->name, mname, prog->name, moff,
1109
0
         kern_moff);
1110
1111
0
      continue;
1112
0
    }
1113
1114
0
    msize = btf__resolve_size(btf, mtype_id);
1115
0
    kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
1116
0
    if (msize < 0 || kern_msize < 0 || msize != kern_msize) {
1117
0
      pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
1118
0
        map->name, mname, (ssize_t)msize,
1119
0
        (ssize_t)kern_msize);
1120
0
      return -ENOTSUP;
1121
0
    }
1122
1123
0
    pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
1124
0
       map->name, mname, (unsigned int)msize,
1125
0
       moff, kern_moff);
1126
0
    memcpy(kern_mdata, mdata, msize);
1127
0
  }
1128
1129
0
  return 0;
1130
0
}
1131
1132
static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
1133
0
{
1134
0
  struct bpf_map *map;
1135
0
  size_t i;
1136
0
  int err;
1137
1138
0
  for (i = 0; i < obj->nr_maps; i++) {
1139
0
    map = &obj->maps[i];
1140
1141
0
    if (!bpf_map__is_struct_ops(map))
1142
0
      continue;
1143
1144
0
    err = bpf_map__init_kern_struct_ops(map, obj->btf,
1145
0
                obj->btf_vmlinux);
1146
0
    if (err)
1147
0
      return err;
1148
0
  }
1149
1150
0
  return 0;
1151
0
}
1152
1153
static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name,
1154
        int shndx, Elf_Data *data, __u32 map_flags)
1155
4.04k
{
1156
4.04k
  const struct btf_type *type, *datasec;
1157
4.04k
  const struct btf_var_secinfo *vsi;
1158
4.04k
  struct bpf_struct_ops *st_ops;
1159
4.04k
  const char *tname, *var_name;
1160
4.04k
  __s32 type_id, datasec_id;
1161
4.04k
  const struct btf *btf;
1162
4.04k
  struct bpf_map *map;
1163
4.04k
  __u32 i;
1164
1165
4.04k
  if (shndx == -1)
1166
3.96k
    return 0;
1167
1168
80
  btf = obj->btf;
1169
80
  datasec_id = btf__find_by_name_kind(btf, sec_name,
1170
80
              BTF_KIND_DATASEC);
1171
80
  if (datasec_id < 0) {
1172
76
    pr_warn("struct_ops init: DATASEC %s not found\n",
1173
76
      sec_name);
1174
76
    return -EINVAL;
1175
76
  }
1176
1177
4
  datasec = btf__type_by_id(btf, datasec_id);
1178
4
  vsi = btf_var_secinfos(datasec);
1179
4
  for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
1180
0
    type = btf__type_by_id(obj->btf, vsi->type);
1181
0
    var_name = btf__name_by_offset(obj->btf, type->name_off);
1182
1183
0
    type_id = btf__resolve_type(obj->btf, vsi->type);
1184
0
    if (type_id < 0) {
1185
0
      pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
1186
0
        vsi->type, sec_name);
1187
0
      return -EINVAL;
1188
0
    }
1189
1190
0
    type = btf__type_by_id(obj->btf, type_id);
1191
0
    tname = btf__name_by_offset(obj->btf, type->name_off);
1192
0
    if (!tname[0]) {
1193
0
      pr_warn("struct_ops init: anonymous type is not supported\n");
1194
0
      return -ENOTSUP;
1195
0
    }
1196
0
    if (!btf_is_struct(type)) {
1197
0
      pr_warn("struct_ops init: %s is not a struct\n", tname);
1198
0
      return -EINVAL;
1199
0
    }
1200
1201
0
    map = bpf_object__add_map(obj);
1202
0
    if (IS_ERR(map))
1203
0
      return PTR_ERR(map);
1204
1205
0
    map->sec_idx = shndx;
1206
0
    map->sec_offset = vsi->offset;
1207
0
    map->name = strdup(var_name);
1208
0
    if (!map->name)
1209
0
      return -ENOMEM;
1210
1211
0
    map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1212
0
    map->def.key_size = sizeof(int);
1213
0
    map->def.value_size = type->size;
1214
0
    map->def.max_entries = 1;
1215
0
    map->def.map_flags = map_flags;
1216
1217
0
    map->st_ops = calloc(1, sizeof(*map->st_ops));
1218
0
    if (!map->st_ops)
1219
0
      return -ENOMEM;
1220
0
    st_ops = map->st_ops;
1221
0
    st_ops->data = malloc(type->size);
1222
0
    st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1223
0
    st_ops->kern_func_off = malloc(btf_vlen(type) *
1224
0
                 sizeof(*st_ops->kern_func_off));
1225
0
    if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1226
0
      return -ENOMEM;
1227
1228
0
    if (vsi->offset + type->size > data->d_size) {
1229
0
      pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1230
0
        var_name, sec_name);
1231
0
      return -EINVAL;
1232
0
    }
1233
1234
0
    memcpy(st_ops->data,
1235
0
           data->d_buf + vsi->offset,
1236
0
           type->size);
1237
0
    st_ops->tname = tname;
1238
0
    st_ops->type = type;
1239
0
    st_ops->type_id = type_id;
1240
1241
0
    pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1242
0
       tname, type_id, var_name, vsi->offset);
1243
0
  }
1244
1245
4
  return 0;
1246
4
}
1247
1248
static int bpf_object_init_struct_ops(struct bpf_object *obj)
1249
2.03k
{
1250
2.03k
  int err;
1251
1252
2.03k
  err = init_struct_ops_maps(obj, STRUCT_OPS_SEC, obj->efile.st_ops_shndx,
1253
2.03k
           obj->efile.st_ops_data, 0);
1254
2.03k
  err = err ?: init_struct_ops_maps(obj, STRUCT_OPS_LINK_SEC,
1255
2.01k
            obj->efile.st_ops_link_shndx,
1256
2.01k
            obj->efile.st_ops_link_data,
1257
2.01k
            BPF_F_LINK);
1258
2.03k
  return err;
1259
2.03k
}
1260
1261
static struct bpf_object *bpf_object__new(const char *path,
1262
            const void *obj_buf,
1263
            size_t obj_buf_sz,
1264
            const char *obj_name)
1265
10.3k
{
1266
10.3k
  struct bpf_object *obj;
1267
10.3k
  char *end;
1268
1269
10.3k
  obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1270
10.3k
  if (!obj) {
1271
0
    pr_warn("alloc memory failed for %s\n", path);
1272
0
    return ERR_PTR(-ENOMEM);
1273
0
  }
1274
1275
10.3k
  strcpy(obj->path, path);
1276
10.3k
  if (obj_name) {
1277
10.3k
    libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1278
10.3k
  } else {
1279
    /* Using basename() GNU version which doesn't modify arg. */
1280
0
    libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1281
0
    end = strchr(obj->name, '.');
1282
0
    if (end)
1283
0
      *end = 0;
1284
0
  }
1285
1286
10.3k
  obj->efile.fd = -1;
1287
  /*
1288
   * Caller of this function should also call
1289
   * bpf_object__elf_finish() after data collection to return
1290
   * obj_buf to user. If not, we should duplicate the buffer to
1291
   * avoid user freeing them before elf finish.
1292
   */
1293
10.3k
  obj->efile.obj_buf = obj_buf;
1294
10.3k
  obj->efile.obj_buf_sz = obj_buf_sz;
1295
10.3k
  obj->efile.btf_maps_shndx = -1;
1296
10.3k
  obj->efile.st_ops_shndx = -1;
1297
10.3k
  obj->efile.st_ops_link_shndx = -1;
1298
10.3k
  obj->kconfig_map_idx = -1;
1299
1300
10.3k
  obj->kern_version = get_kernel_version();
1301
10.3k
  obj->loaded = false;
1302
1303
10.3k
  return obj;
1304
10.3k
}
1305
1306
static void bpf_object__elf_finish(struct bpf_object *obj)
1307
14.0k
{
1308
14.0k
  if (!obj->efile.elf)
1309
3.86k
    return;
1310
1311
10.2k
  elf_end(obj->efile.elf);
1312
10.2k
  obj->efile.elf = NULL;
1313
10.2k
  obj->efile.symbols = NULL;
1314
10.2k
  obj->efile.st_ops_data = NULL;
1315
10.2k
  obj->efile.st_ops_link_data = NULL;
1316
1317
10.2k
  zfree(&obj->efile.secs);
1318
10.2k
  obj->efile.sec_cnt = 0;
1319
10.2k
  zclose(obj->efile.fd);
1320
10.2k
  obj->efile.obj_buf = NULL;
1321
10.2k
  obj->efile.obj_buf_sz = 0;
1322
10.2k
}
1323
1324
static int bpf_object__elf_init(struct bpf_object *obj)
1325
10.3k
{
1326
10.3k
  Elf64_Ehdr *ehdr;
1327
10.3k
  int err = 0;
1328
10.3k
  Elf *elf;
1329
1330
10.3k
  if (obj->efile.elf) {
1331
0
    pr_warn("elf: init internal error\n");
1332
0
    return -LIBBPF_ERRNO__LIBELF;
1333
0
  }
1334
1335
10.3k
  if (obj->efile.obj_buf_sz > 0) {
1336
    /* obj_buf should have been validated by bpf_object__open_mem(). */
1337
10.3k
    elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1338
10.3k
  } else {
1339
0
    obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1340
0
    if (obj->efile.fd < 0) {
1341
0
      char errmsg[STRERR_BUFSIZE], *cp;
1342
1343
0
      err = -errno;
1344
0
      cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1345
0
      pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1346
0
      return err;
1347
0
    }
1348
1349
0
    elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1350
0
  }
1351
1352
10.3k
  if (!elf) {
1353
87
    pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1354
87
    err = -LIBBPF_ERRNO__LIBELF;
1355
87
    goto errout;
1356
87
  }
1357
1358
10.2k
  obj->efile.elf = elf;
1359
1360
10.2k
  if (elf_kind(elf) != ELF_K_ELF) {
1361
121
    err = -LIBBPF_ERRNO__FORMAT;
1362
121
    pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1363
121
    goto errout;
1364
121
  }
1365
1366
10.0k
  if (gelf_getclass(elf) != ELFCLASS64) {
1367
505
    err = -LIBBPF_ERRNO__FORMAT;
1368
505
    pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1369
505
    goto errout;
1370
505
  }
1371
1372
9.58k
  obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1373
9.58k
  if (!obj->efile.ehdr) {
1374
0
    pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1375
0
    err = -LIBBPF_ERRNO__FORMAT;
1376
0
    goto errout;
1377
0
  }
1378
1379
9.58k
  if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1380
30
    pr_warn("elf: failed to get section names section index for %s: %s\n",
1381
30
      obj->path, elf_errmsg(-1));
1382
30
    err = -LIBBPF_ERRNO__FORMAT;
1383
30
    goto errout;
1384
30
  }
1385
1386
  /* ELF is corrupted/truncated, avoid calling elf_strptr. */
1387
9.55k
  if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1388
1.10k
    pr_warn("elf: failed to get section names strings from %s: %s\n",
1389
1.10k
      obj->path, elf_errmsg(-1));
1390
1.10k
    err = -LIBBPF_ERRNO__FORMAT;
1391
1.10k
    goto errout;
1392
1.10k
  }
1393
1394
  /* Old LLVM set e_machine to EM_NONE */
1395
8.45k
  if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1396
476
    pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1397
476
    err = -LIBBPF_ERRNO__FORMAT;
1398
476
    goto errout;
1399
476
  }
1400
1401
7.97k
  return 0;
1402
2.32k
errout:
1403
2.32k
  bpf_object__elf_finish(obj);
1404
2.32k
  return err;
1405
8.45k
}
1406
1407
static int bpf_object__check_endianness(struct bpf_object *obj)
1408
7.97k
{
1409
7.97k
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1410
7.97k
  if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1411
7.95k
    return 0;
1412
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1413
  if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1414
    return 0;
1415
#else
1416
# error "Unrecognized __BYTE_ORDER__"
1417
#endif
1418
17
  pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1419
17
  return -LIBBPF_ERRNO__ENDIAN;
1420
7.97k
}
1421
1422
static int
1423
bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1424
229
{
1425
229
  if (!data) {
1426
2
    pr_warn("invalid license section in %s\n", obj->path);
1427
2
    return -LIBBPF_ERRNO__FORMAT;
1428
2
  }
1429
  /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1430
   * go over allowed ELF data section buffer
1431
   */
1432
227
  libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1433
227
  pr_debug("license of %s is %s\n", obj->path, obj->license);
1434
227
  return 0;
1435
229
}
1436
1437
static int
1438
bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1439
12
{
1440
12
  __u32 kver;
1441
1442
12
  if (!data || size != sizeof(kver)) {
1443
2
    pr_warn("invalid kver section in %s\n", obj->path);
1444
2
    return -LIBBPF_ERRNO__FORMAT;
1445
2
  }
1446
10
  memcpy(&kver, data, sizeof(kver));
1447
10
  obj->kern_version = kver;
1448
10
  pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1449
10
  return 0;
1450
12
}
1451
1452
static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1453
1
{
1454
1
  if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1455
1
      type == BPF_MAP_TYPE_HASH_OF_MAPS)
1456
0
    return true;
1457
1
  return false;
1458
1
}
1459
1460
static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size)
1461
447
{
1462
447
  Elf_Data *data;
1463
447
  Elf_Scn *scn;
1464
1465
447
  if (!name)
1466
0
    return -EINVAL;
1467
1468
447
  scn = elf_sec_by_name(obj, name);
1469
447
  data = elf_sec_data(obj, scn);
1470
447
  if (data) {
1471
294
    *size = data->d_size;
1472
294
    return 0; /* found it */
1473
294
  }
1474
1475
153
  return -ENOENT;
1476
447
}
1477
1478
static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name)
1479
982
{
1480
982
  Elf_Data *symbols = obj->efile.symbols;
1481
982
  const char *sname;
1482
982
  size_t si;
1483
1484
11.4k
  for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1485
11.2k
    Elf64_Sym *sym = elf_sym_by_idx(obj, si);
1486
1487
11.2k
    if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1488
8.99k
      continue;
1489
1490
2.27k
    if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1491
2.27k
        ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1492
1.15k
      continue;
1493
1494
1.12k
    sname = elf_sym_str(obj, sym->st_name);
1495
1.12k
    if (!sname) {
1496
58
      pr_warn("failed to get sym name string for var %s\n", name);
1497
58
      return ERR_PTR(-EIO);
1498
58
    }
1499
1.06k
    if (strcmp(name, sname) == 0)
1500
708
      return sym;
1501
1.06k
  }
1502
1503
216
  return ERR_PTR(-ENOENT);
1504
982
}
1505
1506
static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1507
1.84k
{
1508
1.84k
  struct bpf_map *map;
1509
1.84k
  int err;
1510
1511
1.84k
  err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1512
1.84k
        sizeof(*obj->maps), obj->nr_maps + 1);
1513
1.84k
  if (err)
1514
0
    return ERR_PTR(err);
1515
1516
1.84k
  map = &obj->maps[obj->nr_maps++];
1517
1.84k
  map->obj = obj;
1518
1.84k
  map->fd = -1;
1519
1.84k
  map->inner_map_fd = -1;
1520
1.84k
  map->autocreate = true;
1521
1522
1.84k
  return map;
1523
1.84k
}
1524
1525
static size_t bpf_map_mmap_sz(unsigned int value_sz, unsigned int max_entries)
1526
2.72k
{
1527
2.72k
  const long page_sz = sysconf(_SC_PAGE_SIZE);
1528
2.72k
  size_t map_sz;
1529
1530
2.72k
  map_sz = (size_t)roundup(value_sz, 8) * max_entries;
1531
2.72k
  map_sz = roundup(map_sz, page_sz);
1532
2.72k
  return map_sz;
1533
2.72k
}
1534
1535
static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz)
1536
0
{
1537
0
  void *mmaped;
1538
1539
0
  if (!map->mmaped)
1540
0
    return -EINVAL;
1541
1542
0
  if (old_sz == new_sz)
1543
0
    return 0;
1544
1545
0
  mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1546
0
  if (mmaped == MAP_FAILED)
1547
0
    return -errno;
1548
1549
0
  memcpy(mmaped, map->mmaped, min(old_sz, new_sz));
1550
0
  munmap(map->mmaped, old_sz);
1551
0
  map->mmaped = mmaped;
1552
0
  return 0;
1553
0
}
1554
1555
static char *internal_map_name(struct bpf_object *obj, const char *real_name)
1556
1.42k
{
1557
1.42k
  char map_name[BPF_OBJ_NAME_LEN], *p;
1558
1.42k
  int pfx_len, sfx_len = max((size_t)7, strlen(real_name));
1559
1560
  /* This is one of the more confusing parts of libbpf for various
1561
   * reasons, some of which are historical. The original idea for naming
1562
   * internal names was to include as much of BPF object name prefix as
1563
   * possible, so that it can be distinguished from similar internal
1564
   * maps of a different BPF object.
1565
   * As an example, let's say we have bpf_object named 'my_object_name'
1566
   * and internal map corresponding to '.rodata' ELF section. The final
1567
   * map name advertised to user and to the kernel will be
1568
   * 'my_objec.rodata', taking first 8 characters of object name and
1569
   * entire 7 characters of '.rodata'.
1570
   * Somewhat confusingly, if internal map ELF section name is shorter
1571
   * than 7 characters, e.g., '.bss', we still reserve 7 characters
1572
   * for the suffix, even though we only have 4 actual characters, and
1573
   * resulting map will be called 'my_objec.bss', not even using all 15
1574
   * characters allowed by the kernel. Oh well, at least the truncated
1575
   * object name is somewhat consistent in this case. But if the map
1576
   * name is '.kconfig', we'll still have entirety of '.kconfig' added
1577
   * (8 chars) and thus will be left with only first 7 characters of the
1578
   * object name ('my_obje'). Happy guessing, user, that the final map
1579
   * name will be "my_obje.kconfig".
1580
   * Now, with libbpf starting to support arbitrarily named .rodata.*
1581
   * and .data.* data sections, it's possible that ELF section name is
1582
   * longer than allowed 15 chars, so we now need to be careful to take
1583
   * only up to 15 first characters of ELF name, taking no BPF object
1584
   * name characters at all. So '.rodata.abracadabra' will result in
1585
   * '.rodata.abracad' kernel and user-visible name.
1586
   * We need to keep this convoluted logic intact for .data, .bss and
1587
   * .rodata maps, but for new custom .data.custom and .rodata.custom
1588
   * maps we use their ELF names as is, not prepending bpf_object name
1589
   * in front. We still need to truncate them to 15 characters for the
1590
   * kernel. Full name can be recovered for such maps by using DATASEC
1591
   * BTF type associated with such map's value type, though.
1592
   */
1593
1.42k
  if (sfx_len >= BPF_OBJ_NAME_LEN)
1594
241
    sfx_len = BPF_OBJ_NAME_LEN - 1;
1595
1596
  /* if there are two or more dots in map name, it's a custom dot map */
1597
1.42k
  if (strchr(real_name + 1, '.') != NULL)
1598
983
    pfx_len = 0;
1599
443
  else
1600
443
    pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1601
1602
1.42k
  snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1603
1.42k
     sfx_len, real_name);
1604
1605
  /* sanitise map name to characters allowed by kernel */
1606
18.0k
  for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1607
16.6k
    if (!isalnum(*p) && *p != '_' && *p != '.')
1608
2.67k
      *p = '_';
1609
1610
1.42k
  return strdup(map_name);
1611
1.42k
}
1612
1613
static int
1614
map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map);
1615
1616
/* Internal BPF map is mmap()'able only if at least one of corresponding
1617
 * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL
1618
 * variable and it's not marked as __hidden (which turns it into, effectively,
1619
 * a STATIC variable).
1620
 */
1621
static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map)
1622
1.42k
{
1623
1.42k
  const struct btf_type *t, *vt;
1624
1.42k
  struct btf_var_secinfo *vsi;
1625
1.42k
  int i, n;
1626
1627
1.42k
  if (!map->btf_value_type_id)
1628
1.23k
    return false;
1629
1630
187
  t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1631
187
  if (!btf_is_datasec(t))
1632
28
    return false;
1633
1634
159
  vsi = btf_var_secinfos(t);
1635
296
  for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) {
1636
258
    vt = btf__type_by_id(obj->btf, vsi->type);
1637
258
    if (!btf_is_var(vt))
1638
127
      continue;
1639
1640
131
    if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1641
121
      return true;
1642
131
  }
1643
1644
38
  return false;
1645
159
}
1646
1647
static int
1648
bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1649
            const char *real_name, int sec_idx, void *data, size_t data_sz)
1650
1.42k
{
1651
1.42k
  struct bpf_map_def *def;
1652
1.42k
  struct bpf_map *map;
1653
1.42k
  size_t mmap_sz;
1654
1.42k
  int err;
1655
1656
1.42k
  map = bpf_object__add_map(obj);
1657
1.42k
  if (IS_ERR(map))
1658
0
    return PTR_ERR(map);
1659
1660
1.42k
  map->libbpf_type = type;
1661
1.42k
  map->sec_idx = sec_idx;
1662
1.42k
  map->sec_offset = 0;
1663
1.42k
  map->real_name = strdup(real_name);
1664
1.42k
  map->name = internal_map_name(obj, real_name);
1665
1.42k
  if (!map->real_name || !map->name) {
1666
0
    zfree(&map->real_name);
1667
0
    zfree(&map->name);
1668
0
    return -ENOMEM;
1669
0
  }
1670
1671
1.42k
  def = &map->def;
1672
1.42k
  def->type = BPF_MAP_TYPE_ARRAY;
1673
1.42k
  def->key_size = sizeof(int);
1674
1.42k
  def->value_size = data_sz;
1675
1.42k
  def->max_entries = 1;
1676
1.42k
  def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1677
1.42k
       ? BPF_F_RDONLY_PROG : 0;
1678
1679
  /* failures are fine because of maps like .rodata.str1.1 */
1680
1.42k
  (void) map_fill_btf_type_info(obj, map);
1681
1682
1.42k
  if (map_is_mmapable(obj, map))
1683
121
    def->map_flags |= BPF_F_MMAPABLE;
1684
1685
1.42k
  pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1686
1.42k
     map->name, map->sec_idx, map->sec_offset, def->map_flags);
1687
1688
1.42k
  mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries);
1689
1.42k
  map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1690
1.42k
         MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1691
1.42k
  if (map->mmaped == MAP_FAILED) {
1692
127
    err = -errno;
1693
127
    map->mmaped = NULL;
1694
127
    pr_warn("failed to alloc map '%s' content buffer: %d\n",
1695
127
      map->name, err);
1696
127
    zfree(&map->real_name);
1697
127
    zfree(&map->name);
1698
127
    return err;
1699
127
  }
1700
1701
1.29k
  if (data)
1702
721
    memcpy(map->mmaped, data, data_sz);
1703
1704
1.29k
  pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1705
1.29k
  return 0;
1706
1.42k
}
1707
1708
static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1709
2.15k
{
1710
2.15k
  struct elf_sec_desc *sec_desc;
1711
2.15k
  const char *sec_name;
1712
2.15k
  int err = 0, sec_idx;
1713
1714
  /*
1715
   * Populate obj->maps with libbpf internal maps.
1716
   */
1717
15.2k
  for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1718
13.1k
    sec_desc = &obj->efile.secs[sec_idx];
1719
1720
    /* Skip recognized sections with size 0. */
1721
13.1k
    if (!sec_desc->data || sec_desc->data->d_size == 0)
1722
10.9k
      continue;
1723
1724
2.23k
    switch (sec_desc->sec_type) {
1725
482
    case SEC_DATA:
1726
482
      sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1727
482
      err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1728
482
                  sec_name, sec_idx,
1729
482
                  sec_desc->data->d_buf,
1730
482
                  sec_desc->data->d_size);
1731
482
      break;
1732
239
    case SEC_RODATA:
1733
239
      obj->has_rodata = true;
1734
239
      sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1735
239
      err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1736
239
                  sec_name, sec_idx,
1737
239
                  sec_desc->data->d_buf,
1738
239
                  sec_desc->data->d_size);
1739
239
      break;
1740
576
    case SEC_BSS:
1741
576
      sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1742
576
      err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1743
576
                  sec_name, sec_idx,
1744
576
                  NULL,
1745
576
                  sec_desc->data->d_size);
1746
576
      break;
1747
940
    default:
1748
      /* skip */
1749
940
      break;
1750
2.23k
    }
1751
2.23k
    if (err)
1752
37
      return err;
1753
2.23k
  }
1754
2.12k
  return 0;
1755
2.15k
}
1756
1757
1758
static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1759
                 const void *name)
1760
629
{
1761
629
  int i;
1762
1763
1.39k
  for (i = 0; i < obj->nr_extern; i++) {
1764
1.06k
    if (strcmp(obj->externs[i].name, name) == 0)
1765
297
      return &obj->externs[i];
1766
1.06k
  }
1767
332
  return NULL;
1768
629
}
1769
1770
static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
1771
            char value)
1772
0
{
1773
0
  switch (ext->kcfg.type) {
1774
0
  case KCFG_BOOL:
1775
0
    if (value == 'm') {
1776
0
      pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",
1777
0
        ext->name, value);
1778
0
      return -EINVAL;
1779
0
    }
1780
0
    *(bool *)ext_val = value == 'y' ? true : false;
1781
0
    break;
1782
0
  case KCFG_TRISTATE:
1783
0
    if (value == 'y')
1784
0
      *(enum libbpf_tristate *)ext_val = TRI_YES;
1785
0
    else if (value == 'm')
1786
0
      *(enum libbpf_tristate *)ext_val = TRI_MODULE;
1787
0
    else /* value == 'n' */
1788
0
      *(enum libbpf_tristate *)ext_val = TRI_NO;
1789
0
    break;
1790
0
  case KCFG_CHAR:
1791
0
    *(char *)ext_val = value;
1792
0
    break;
1793
0
  case KCFG_UNKNOWN:
1794
0
  case KCFG_INT:
1795
0
  case KCFG_CHAR_ARR:
1796
0
  default:
1797
0
    pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",
1798
0
      ext->name, value);
1799
0
    return -EINVAL;
1800
0
  }
1801
0
  ext->is_set = true;
1802
0
  return 0;
1803
0
}
1804
1805
static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
1806
            const char *value)
1807
0
{
1808
0
  size_t len;
1809
1810
0
  if (ext->kcfg.type != KCFG_CHAR_ARR) {
1811
0
    pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",
1812
0
      ext->name, value);
1813
0
    return -EINVAL;
1814
0
  }
1815
1816
0
  len = strlen(value);
1817
0
  if (value[len - 1] != '"') {
1818
0
    pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
1819
0
      ext->name, value);
1820
0
    return -EINVAL;
1821
0
  }
1822
1823
  /* strip quotes */
1824
0
  len -= 2;
1825
0
  if (len >= ext->kcfg.sz) {
1826
0
    pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",
1827
0
      ext->name, value, len, ext->kcfg.sz - 1);
1828
0
    len = ext->kcfg.sz - 1;
1829
0
  }
1830
0
  memcpy(ext_val, value + 1, len);
1831
0
  ext_val[len] = '\0';
1832
0
  ext->is_set = true;
1833
0
  return 0;
1834
0
}
1835
1836
static int parse_u64(const char *value, __u64 *res)
1837
0
{
1838
0
  char *value_end;
1839
0
  int err;
1840
1841
0
  errno = 0;
1842
0
  *res = strtoull(value, &value_end, 0);
1843
0
  if (errno) {
1844
0
    err = -errno;
1845
0
    pr_warn("failed to parse '%s' as integer: %d\n", value, err);
1846
0
    return err;
1847
0
  }
1848
0
  if (*value_end) {
1849
0
    pr_warn("failed to parse '%s' as integer completely\n", value);
1850
0
    return -EINVAL;
1851
0
  }
1852
0
  return 0;
1853
0
}
1854
1855
static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
1856
0
{
1857
0
  int bit_sz = ext->kcfg.sz * 8;
1858
1859
0
  if (ext->kcfg.sz == 8)
1860
0
    return true;
1861
1862
  /* Validate that value stored in u64 fits in integer of `ext->sz`
1863
   * bytes size without any loss of information. If the target integer
1864
   * is signed, we rely on the following limits of integer type of
1865
   * Y bits and subsequent transformation:
1866
   *
1867
   *     -2^(Y-1) <= X           <= 2^(Y-1) - 1
1868
   *            0 <= X + 2^(Y-1) <= 2^Y - 1
1869
   *            0 <= X + 2^(Y-1) <  2^Y
1870
   *
1871
   *  For unsigned target integer, check that all the (64 - Y) bits are
1872
   *  zero.
1873
   */
1874
0
  if (ext->kcfg.is_signed)
1875
0
    return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
1876
0
  else
1877
0
    return (v >> bit_sz) == 0;
1878
0
}
1879
1880
static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
1881
            __u64 value)
1882
0
{
1883
0
  if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
1884
0
      ext->kcfg.type != KCFG_BOOL) {
1885
0
    pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",
1886
0
      ext->name, (unsigned long long)value);
1887
0
    return -EINVAL;
1888
0
  }
1889
0
  if (ext->kcfg.type == KCFG_BOOL && value > 1) {
1890
0
    pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",
1891
0
      ext->name, (unsigned long long)value);
1892
0
    return -EINVAL;
1893
1894
0
  }
1895
0
  if (!is_kcfg_value_in_range(ext, value)) {
1896
0
    pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",
1897
0
      ext->name, (unsigned long long)value, ext->kcfg.sz);
1898
0
    return -ERANGE;
1899
0
  }
1900
0
  switch (ext->kcfg.sz) {
1901
0
  case 1:
1902
0
    *(__u8 *)ext_val = value;
1903
0
    break;
1904
0
  case 2:
1905
0
    *(__u16 *)ext_val = value;
1906
0
    break;
1907
0
  case 4:
1908
0
    *(__u32 *)ext_val = value;
1909
0
    break;
1910
0
  case 8:
1911
0
    *(__u64 *)ext_val = value;
1912
0
    break;
1913
0
  default:
1914
0
    return -EINVAL;
1915
0
  }
1916
0
  ext->is_set = true;
1917
0
  return 0;
1918
0
}
1919
1920
static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1921
              char *buf, void *data)
1922
0
{
1923
0
  struct extern_desc *ext;
1924
0
  char *sep, *value;
1925
0
  int len, err = 0;
1926
0
  void *ext_val;
1927
0
  __u64 num;
1928
1929
0
  if (!str_has_pfx(buf, "CONFIG_"))
1930
0
    return 0;
1931
1932
0
  sep = strchr(buf, '=');
1933
0
  if (!sep) {
1934
0
    pr_warn("failed to parse '%s': no separator\n", buf);
1935
0
    return -EINVAL;
1936
0
  }
1937
1938
  /* Trim ending '\n' */
1939
0
  len = strlen(buf);
1940
0
  if (buf[len - 1] == '\n')
1941
0
    buf[len - 1] = '\0';
1942
  /* Split on '=' and ensure that a value is present. */
1943
0
  *sep = '\0';
1944
0
  if (!sep[1]) {
1945
0
    *sep = '=';
1946
0
    pr_warn("failed to parse '%s': no value\n", buf);
1947
0
    return -EINVAL;
1948
0
  }
1949
1950
0
  ext = find_extern_by_name(obj, buf);
1951
0
  if (!ext || ext->is_set)
1952
0
    return 0;
1953
1954
0
  ext_val = data + ext->kcfg.data_off;
1955
0
  value = sep + 1;
1956
1957
0
  switch (*value) {
1958
0
  case 'y': case 'n': case 'm':
1959
0
    err = set_kcfg_value_tri(ext, ext_val, *value);
1960
0
    break;
1961
0
  case '"':
1962
0
    err = set_kcfg_value_str(ext, ext_val, value);
1963
0
    break;
1964
0
  default:
1965
    /* assume integer */
1966
0
    err = parse_u64(value, &num);
1967
0
    if (err) {
1968
0
      pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
1969
0
      return err;
1970
0
    }
1971
0
    if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
1972
0
      pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
1973
0
      return -EINVAL;
1974
0
    }
1975
0
    err = set_kcfg_value_num(ext, ext_val, num);
1976
0
    break;
1977
0
  }
1978
0
  if (err)
1979
0
    return err;
1980
0
  pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
1981
0
  return 0;
1982
0
}
1983
1984
static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1985
0
{
1986
0
  char buf[PATH_MAX];
1987
0
  struct utsname uts;
1988
0
  int len, err = 0;
1989
0
  gzFile file;
1990
1991
0
  uname(&uts);
1992
0
  len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
1993
0
  if (len < 0)
1994
0
    return -EINVAL;
1995
0
  else if (len >= PATH_MAX)
1996
0
    return -ENAMETOOLONG;
1997
1998
  /* gzopen also accepts uncompressed files. */
1999
0
  file = gzopen(buf, "re");
2000
0
  if (!file)
2001
0
    file = gzopen("/proc/config.gz", "re");
2002
2003
0
  if (!file) {
2004
0
    pr_warn("failed to open system Kconfig\n");
2005
0
    return -ENOENT;
2006
0
  }
2007
2008
0
  while (gzgets(file, buf, sizeof(buf))) {
2009
0
    err = bpf_object__process_kconfig_line(obj, buf, data);
2010
0
    if (err) {
2011
0
      pr_warn("error parsing system Kconfig line '%s': %d\n",
2012
0
        buf, err);
2013
0
      goto out;
2014
0
    }
2015
0
  }
2016
2017
0
out:
2018
0
  gzclose(file);
2019
0
  return err;
2020
0
}
2021
2022
static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
2023
          const char *config, void *data)
2024
0
{
2025
0
  char buf[PATH_MAX];
2026
0
  int err = 0;
2027
0
  FILE *file;
2028
2029
0
  file = fmemopen((void *)config, strlen(config), "r");
2030
0
  if (!file) {
2031
0
    err = -errno;
2032
0
    pr_warn("failed to open in-memory Kconfig: %d\n", err);
2033
0
    return err;
2034
0
  }
2035
2036
0
  while (fgets(buf, sizeof(buf), file)) {
2037
0
    err = bpf_object__process_kconfig_line(obj, buf, data);
2038
0
    if (err) {
2039
0
      pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
2040
0
        buf, err);
2041
0
      break;
2042
0
    }
2043
0
  }
2044
2045
0
  fclose(file);
2046
0
  return err;
2047
0
}
2048
2049
static int bpf_object__init_kconfig_map(struct bpf_object *obj)
2050
2.12k
{
2051
2.12k
  struct extern_desc *last_ext = NULL, *ext;
2052
2.12k
  size_t map_sz;
2053
2.12k
  int i, err;
2054
2055
2.48k
  for (i = 0; i < obj->nr_extern; i++) {
2056
364
    ext = &obj->externs[i];
2057
364
    if (ext->type == EXT_KCFG)
2058
202
      last_ext = ext;
2059
364
  }
2060
2061
2.12k
  if (!last_ext)
2062
1.99k
    return 0;
2063
2064
129
  map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2065
129
  err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
2066
129
              ".kconfig", obj->efile.symbols_shndx,
2067
129
              NULL, map_sz);
2068
129
  if (err)
2069
90
    return err;
2070
2071
39
  obj->kconfig_map_idx = obj->nr_maps - 1;
2072
2073
39
  return 0;
2074
129
}
2075
2076
const struct btf_type *
2077
skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
2078
1.90k
{
2079
1.90k
  const struct btf_type *t = btf__type_by_id(btf, id);
2080
2081
1.90k
  if (res_id)
2082
634
    *res_id = id;
2083
2084
2.75k
  while (btf_is_mod(t) || btf_is_typedef(t)) {
2085
852
    if (res_id)
2086
433
      *res_id = t->type;
2087
852
    t = btf__type_by_id(btf, t->type);
2088
852
  }
2089
2090
1.90k
  return t;
2091
1.90k
}
2092
2093
static const struct btf_type *
2094
resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
2095
0
{
2096
0
  const struct btf_type *t;
2097
2098
0
  t = skip_mods_and_typedefs(btf, id, NULL);
2099
0
  if (!btf_is_ptr(t))
2100
0
    return NULL;
2101
2102
0
  t = skip_mods_and_typedefs(btf, t->type, res_id);
2103
2104
0
  return btf_is_func_proto(t) ? t : NULL;
2105
0
}
2106
2107
static const char *__btf_kind_str(__u16 kind)
2108
212
{
2109
212
  switch (kind) {
2110
28
  case BTF_KIND_UNKN: return "void";
2111
67
  case BTF_KIND_INT: return "int";
2112
0
  case BTF_KIND_PTR: return "ptr";
2113
0
  case BTF_KIND_ARRAY: return "array";
2114
3
  case BTF_KIND_STRUCT: return "struct";
2115
1
  case BTF_KIND_UNION: return "union";
2116
4
  case BTF_KIND_ENUM: return "enum";
2117
4
  case BTF_KIND_FWD: return "fwd";
2118
0
  case BTF_KIND_TYPEDEF: return "typedef";
2119
0
  case BTF_KIND_VOLATILE: return "volatile";
2120
0
  case BTF_KIND_CONST: return "const";
2121
3
  case BTF_KIND_RESTRICT: return "restrict";
2122
7
  case BTF_KIND_FUNC: return "func";
2123
42
  case BTF_KIND_FUNC_PROTO: return "func_proto";
2124
17
  case BTF_KIND_VAR: return "var";
2125
25
  case BTF_KIND_DATASEC: return "datasec";
2126
9
  case BTF_KIND_FLOAT: return "float";
2127
1
  case BTF_KIND_DECL_TAG: return "decl_tag";
2128
1
  case BTF_KIND_TYPE_TAG: return "type_tag";
2129
0
  case BTF_KIND_ENUM64: return "enum64";
2130
0
  default: return "unknown";
2131
212
  }
2132
212
}
2133
2134
const char *btf_kind_str(const struct btf_type *t)
2135
212
{
2136
212
  return __btf_kind_str(btf_kind(t));
2137
212
}
2138
2139
/*
2140
 * Fetch integer attribute of BTF map definition. Such attributes are
2141
 * represented using a pointer to an array, in which dimensionality of array
2142
 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
2143
 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
2144
 * type definition, while using only sizeof(void *) space in ELF data section.
2145
 */
2146
static bool get_map_field_int(const char *map_name, const struct btf *btf,
2147
            const struct btf_member *m, __u32 *res)
2148
16
{
2149
16
  const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2150
16
  const char *name = btf__name_by_offset(btf, m->name_off);
2151
16
  const struct btf_array *arr_info;
2152
16
  const struct btf_type *arr_t;
2153
2154
16
  if (!btf_is_ptr(t)) {
2155
13
    pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
2156
13
      map_name, name, btf_kind_str(t));
2157
13
    return false;
2158
13
  }
2159
2160
3
  arr_t = btf__type_by_id(btf, t->type);
2161
3
  if (!arr_t) {
2162
0
    pr_warn("map '%s': attr '%s': type [%u] not found.\n",
2163
0
      map_name, name, t->type);
2164
0
    return false;
2165
0
  }
2166
3
  if (!btf_is_array(arr_t)) {
2167
2
    pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
2168
2
      map_name, name, btf_kind_str(arr_t));
2169
2
    return false;
2170
2
  }
2171
1
  arr_info = btf_array(arr_t);
2172
1
  *res = arr_info->nelems;
2173
1
  return true;
2174
3
}
2175
2176
static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name)
2177
0
{
2178
0
  int len;
2179
2180
0
  len = snprintf(buf, buf_sz, "%s/%s", path, name);
2181
0
  if (len < 0)
2182
0
    return -EINVAL;
2183
0
  if (len >= buf_sz)
2184
0
    return -ENAMETOOLONG;
2185
2186
0
  return 0;
2187
0
}
2188
2189
static int build_map_pin_path(struct bpf_map *map, const char *path)
2190
0
{
2191
0
  char buf[PATH_MAX];
2192
0
  int err;
2193
2194
0
  if (!path)
2195
0
    path = "/sys/fs/bpf";
2196
2197
0
  err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
2198
0
  if (err)
2199
0
    return err;
2200
2201
0
  return bpf_map__set_pin_path(map, buf);
2202
0
}
2203
2204
/* should match definition in bpf_helpers.h */
2205
enum libbpf_pin_type {
2206
  LIBBPF_PIN_NONE,
2207
  /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
2208
  LIBBPF_PIN_BY_NAME,
2209
};
2210
2211
int parse_btf_map_def(const char *map_name, struct btf *btf,
2212
          const struct btf_type *def_t, bool strict,
2213
          struct btf_map_def *map_def, struct btf_map_def *inner_def)
2214
418
{
2215
418
  const struct btf_type *t;
2216
418
  const struct btf_member *m;
2217
418
  bool is_inner = inner_def == NULL;
2218
418
  int vlen, i;
2219
2220
418
  vlen = btf_vlen(def_t);
2221
418
  m = btf_members(def_t);
2222
419
  for (i = 0; i < vlen; i++, m++) {
2223
417
    const char *name = btf__name_by_offset(btf, m->name_off);
2224
2225
417
    if (!name) {
2226
0
      pr_warn("map '%s': invalid field #%d.\n", map_name, i);
2227
0
      return -EINVAL;
2228
0
    }
2229
417
    if (strcmp(name, "type") == 0) {
2230
1
      if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2231
1
        return -EINVAL;
2232
0
      map_def->parts |= MAP_DEF_MAP_TYPE;
2233
416
    } else if (strcmp(name, "max_entries") == 0) {
2234
1
      if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2235
1
        return -EINVAL;
2236
0
      map_def->parts |= MAP_DEF_MAX_ENTRIES;
2237
415
    } else if (strcmp(name, "map_flags") == 0) {
2238
7
      if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2239
6
        return -EINVAL;
2240
1
      map_def->parts |= MAP_DEF_MAP_FLAGS;
2241
408
    } else if (strcmp(name, "numa_node") == 0) {
2242
2
      if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2243
2
        return -EINVAL;
2244
0
      map_def->parts |= MAP_DEF_NUMA_NODE;
2245
406
    } else if (strcmp(name, "key_size") == 0) {
2246
1
      __u32 sz;
2247
2248
1
      if (!get_map_field_int(map_name, btf, m, &sz))
2249
1
        return -EINVAL;
2250
0
      if (map_def->key_size && map_def->key_size != sz) {
2251
0
        pr_warn("map '%s': conflicting key size %u != %u.\n",
2252
0
          map_name, map_def->key_size, sz);
2253
0
        return -EINVAL;
2254
0
      }
2255
0
      map_def->key_size = sz;
2256
0
      map_def->parts |= MAP_DEF_KEY_SIZE;
2257
405
    } else if (strcmp(name, "key") == 0) {
2258
2
      __s64 sz;
2259
2260
2
      t = btf__type_by_id(btf, m->type);
2261
2
      if (!t) {
2262
0
        pr_warn("map '%s': key type [%d] not found.\n",
2263
0
          map_name, m->type);
2264
0
        return -EINVAL;
2265
0
      }
2266
2
      if (!btf_is_ptr(t)) {
2267
2
        pr_warn("map '%s': key spec is not PTR: %s.\n",
2268
2
          map_name, btf_kind_str(t));
2269
2
        return -EINVAL;
2270
2
      }
2271
0
      sz = btf__resolve_size(btf, t->type);
2272
0
      if (sz < 0) {
2273
0
        pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2274
0
          map_name, t->type, (ssize_t)sz);
2275
0
        return sz;
2276
0
      }
2277
0
      if (map_def->key_size && map_def->key_size != sz) {
2278
0
        pr_warn("map '%s': conflicting key size %u != %zd.\n",
2279
0
          map_name, map_def->key_size, (ssize_t)sz);
2280
0
        return -EINVAL;
2281
0
      }
2282
0
      map_def->key_size = sz;
2283
0
      map_def->key_type_id = t->type;
2284
0
      map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2285
403
    } else if (strcmp(name, "value_size") == 0) {
2286
1
      __u32 sz;
2287
2288
1
      if (!get_map_field_int(map_name, btf, m, &sz))
2289
1
        return -EINVAL;
2290
0
      if (map_def->value_size && map_def->value_size != sz) {
2291
0
        pr_warn("map '%s': conflicting value size %u != %u.\n",
2292
0
          map_name, map_def->value_size, sz);
2293
0
        return -EINVAL;
2294
0
      }
2295
0
      map_def->value_size = sz;
2296
0
      map_def->parts |= MAP_DEF_VALUE_SIZE;
2297
402
    } else if (strcmp(name, "value") == 0) {
2298
2
      __s64 sz;
2299
2300
2
      t = btf__type_by_id(btf, m->type);
2301
2
      if (!t) {
2302
0
        pr_warn("map '%s': value type [%d] not found.\n",
2303
0
          map_name, m->type);
2304
0
        return -EINVAL;
2305
0
      }
2306
2
      if (!btf_is_ptr(t)) {
2307
2
        pr_warn("map '%s': value spec is not PTR: %s.\n",
2308
2
          map_name, btf_kind_str(t));
2309
2
        return -EINVAL;
2310
2
      }
2311
0
      sz = btf__resolve_size(btf, t->type);
2312
0
      if (sz < 0) {
2313
0
        pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2314
0
          map_name, t->type, (ssize_t)sz);
2315
0
        return sz;
2316
0
      }
2317
0
      if (map_def->value_size && map_def->value_size != sz) {
2318
0
        pr_warn("map '%s': conflicting value size %u != %zd.\n",
2319
0
          map_name, map_def->value_size, (ssize_t)sz);
2320
0
        return -EINVAL;
2321
0
      }
2322
0
      map_def->value_size = sz;
2323
0
      map_def->value_type_id = t->type;
2324
0
      map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2325
0
    }
2326
400
    else if (strcmp(name, "values") == 0) {
2327
1
      bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2328
1
      bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2329
1
      const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2330
1
      char inner_map_name[128];
2331
1
      int err;
2332
2333
1
      if (is_inner) {
2334
0
        pr_warn("map '%s': multi-level inner maps not supported.\n",
2335
0
          map_name);
2336
0
        return -ENOTSUP;
2337
0
      }
2338
1
      if (i != vlen - 1) {
2339
0
        pr_warn("map '%s': '%s' member should be last.\n",
2340
0
          map_name, name);
2341
0
        return -EINVAL;
2342
0
      }
2343
1
      if (!is_map_in_map && !is_prog_array) {
2344
1
        pr_warn("map '%s': should be map-in-map or prog-array.\n",
2345
1
          map_name);
2346
1
        return -ENOTSUP;
2347
1
      }
2348
0
      if (map_def->value_size && map_def->value_size != 4) {
2349
0
        pr_warn("map '%s': conflicting value size %u != 4.\n",
2350
0
          map_name, map_def->value_size);
2351
0
        return -EINVAL;
2352
0
      }
2353
0
      map_def->value_size = 4;
2354
0
      t = btf__type_by_id(btf, m->type);
2355
0
      if (!t) {
2356
0
        pr_warn("map '%s': %s type [%d] not found.\n",
2357
0
          map_name, desc, m->type);
2358
0
        return -EINVAL;
2359
0
      }
2360
0
      if (!btf_is_array(t) || btf_array(t)->nelems) {
2361
0
        pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2362
0
          map_name, desc);
2363
0
        return -EINVAL;
2364
0
      }
2365
0
      t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2366
0
      if (!btf_is_ptr(t)) {
2367
0
        pr_warn("map '%s': %s def is of unexpected kind %s.\n",
2368
0
          map_name, desc, btf_kind_str(t));
2369
0
        return -EINVAL;
2370
0
      }
2371
0
      t = skip_mods_and_typedefs(btf, t->type, NULL);
2372
0
      if (is_prog_array) {
2373
0
        if (!btf_is_func_proto(t)) {
2374
0
          pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2375
0
            map_name, btf_kind_str(t));
2376
0
          return -EINVAL;
2377
0
        }
2378
0
        continue;
2379
0
      }
2380
0
      if (!btf_is_struct(t)) {
2381
0
        pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2382
0
          map_name, btf_kind_str(t));
2383
0
        return -EINVAL;
2384
0
      }
2385
2386
0
      snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
2387
0
      err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
2388
0
      if (err)
2389
0
        return err;
2390
2391
0
      map_def->parts |= MAP_DEF_INNER_MAP;
2392
399
    } else if (strcmp(name, "pinning") == 0) {
2393
2
      __u32 val;
2394
2395
2
      if (is_inner) {
2396
0
        pr_warn("map '%s': inner def can't be pinned.\n", map_name);
2397
0
        return -EINVAL;
2398
0
      }
2399
2
      if (!get_map_field_int(map_name, btf, m, &val))
2400
2
        return -EINVAL;
2401
0
      if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
2402
0
        pr_warn("map '%s': invalid pinning value %u.\n",
2403
0
          map_name, val);
2404
0
        return -EINVAL;
2405
0
      }
2406
0
      map_def->pinning = val;
2407
0
      map_def->parts |= MAP_DEF_PINNING;
2408
397
    } else if (strcmp(name, "map_extra") == 0) {
2409
1
      __u32 map_extra;
2410
2411
1
      if (!get_map_field_int(map_name, btf, m, &map_extra))
2412
1
        return -EINVAL;
2413
0
      map_def->map_extra = map_extra;
2414
0
      map_def->parts |= MAP_DEF_MAP_EXTRA;
2415
396
    } else {
2416
396
      if (strict) {
2417
396
        pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2418
396
        return -ENOTSUP;
2419
396
      }
2420
0
      pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2421
0
    }
2422
417
  }
2423
2424
2
  if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2425
2
    pr_warn("map '%s': map type isn't specified.\n", map_name);
2426
2
    return -EINVAL;
2427
2
  }
2428
2429
0
  return 0;
2430
2
}
2431
2432
static size_t adjust_ringbuf_sz(size_t sz)
2433
0
{
2434
0
  __u32 page_sz = sysconf(_SC_PAGE_SIZE);
2435
0
  __u32 mul;
2436
2437
  /* if user forgot to set any size, make sure they see error */
2438
0
  if (sz == 0)
2439
0
    return 0;
2440
  /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
2441
   * a power-of-2 multiple of kernel's page size. If user diligently
2442
   * satisified these conditions, pass the size through.
2443
   */
2444
0
  if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
2445
0
    return sz;
2446
2447
  /* Otherwise find closest (page_sz * power_of_2) product bigger than
2448
   * user-set size to satisfy both user size request and kernel
2449
   * requirements and substitute correct max_entries for map creation.
2450
   */
2451
0
  for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
2452
0
    if (mul * page_sz > sz)
2453
0
      return mul * page_sz;
2454
0
  }
2455
2456
  /* if it's impossible to satisfy the conditions (i.e., user size is
2457
   * very close to UINT_MAX but is not a power-of-2 multiple of
2458
   * page_size) then just return original size and let kernel reject it
2459
   */
2460
0
  return sz;
2461
0
}
2462
2463
static bool map_is_ringbuf(const struct bpf_map *map)
2464
0
{
2465
0
  return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2466
0
         map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2467
0
}
2468
2469
static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
2470
0
{
2471
0
  map->def.type = def->map_type;
2472
0
  map->def.key_size = def->key_size;
2473
0
  map->def.value_size = def->value_size;
2474
0
  map->def.max_entries = def->max_entries;
2475
0
  map->def.map_flags = def->map_flags;
2476
0
  map->map_extra = def->map_extra;
2477
2478
0
  map->numa_node = def->numa_node;
2479
0
  map->btf_key_type_id = def->key_type_id;
2480
0
  map->btf_value_type_id = def->value_type_id;
2481
2482
  /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2483
0
  if (map_is_ringbuf(map))
2484
0
    map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2485
2486
0
  if (def->parts & MAP_DEF_MAP_TYPE)
2487
0
    pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2488
2489
0
  if (def->parts & MAP_DEF_KEY_TYPE)
2490
0
    pr_debug("map '%s': found key [%u], sz = %u.\n",
2491
0
       map->name, def->key_type_id, def->key_size);
2492
0
  else if (def->parts & MAP_DEF_KEY_SIZE)
2493
0
    pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2494
2495
0
  if (def->parts & MAP_DEF_VALUE_TYPE)
2496
0
    pr_debug("map '%s': found value [%u], sz = %u.\n",
2497
0
       map->name, def->value_type_id, def->value_size);
2498
0
  else if (def->parts & MAP_DEF_VALUE_SIZE)
2499
0
    pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2500
2501
0
  if (def->parts & MAP_DEF_MAX_ENTRIES)
2502
0
    pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2503
0
  if (def->parts & MAP_DEF_MAP_FLAGS)
2504
0
    pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2505
0
  if (def->parts & MAP_DEF_MAP_EXTRA)
2506
0
    pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2507
0
       (unsigned long long)def->map_extra);
2508
0
  if (def->parts & MAP_DEF_PINNING)
2509
0
    pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2510
0
  if (def->parts & MAP_DEF_NUMA_NODE)
2511
0
    pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2512
2513
0
  if (def->parts & MAP_DEF_INNER_MAP)
2514
0
    pr_debug("map '%s': found inner map definition.\n", map->name);
2515
0
}
2516
2517
static const char *btf_var_linkage_str(__u32 linkage)
2518
3
{
2519
3
  switch (linkage) {
2520
1
  case BTF_VAR_STATIC: return "static";
2521
0
  case BTF_VAR_GLOBAL_ALLOCATED: return "global";
2522
1
  case BTF_VAR_GLOBAL_EXTERN: return "extern";
2523
1
  default: return "unknown";
2524
3
  }
2525
3
}
2526
2527
static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2528
           const struct btf_type *sec,
2529
           int var_idx, int sec_idx,
2530
           const Elf_Data *data, bool strict,
2531
           const char *pin_root_path)
2532
529
{
2533
529
  struct btf_map_def map_def = {}, inner_def = {};
2534
529
  const struct btf_type *var, *def;
2535
529
  const struct btf_var_secinfo *vi;
2536
529
  const struct btf_var *var_extra;
2537
529
  const char *map_name;
2538
529
  struct bpf_map *map;
2539
529
  int err;
2540
2541
529
  vi = btf_var_secinfos(sec) + var_idx;
2542
529
  var = btf__type_by_id(obj->btf, vi->type);
2543
529
  var_extra = btf_var(var);
2544
529
  map_name = btf__name_by_offset(obj->btf, var->name_off);
2545
2546
529
  if (map_name == NULL || map_name[0] == '\0') {
2547
1
    pr_warn("map #%d: empty name.\n", var_idx);
2548
1
    return -EINVAL;
2549
1
  }
2550
528
  if ((__u64)vi->offset + vi->size > data->d_size) {
2551
46
    pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2552
46
    return -EINVAL;
2553
46
  }
2554
482
  if (!btf_is_var(var)) {
2555
0
    pr_warn("map '%s': unexpected var kind %s.\n",
2556
0
      map_name, btf_kind_str(var));
2557
0
    return -EINVAL;
2558
0
  }
2559
482
  if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2560
3
    pr_warn("map '%s': unsupported map linkage %s.\n",
2561
3
      map_name, btf_var_linkage_str(var_extra->linkage));
2562
3
    return -EOPNOTSUPP;
2563
3
  }
2564
2565
479
  def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2566
479
  if (!btf_is_struct(def)) {
2567
13
    pr_warn("map '%s': unexpected def kind %s.\n",
2568
13
      map_name, btf_kind_str(var));
2569
13
    return -EINVAL;
2570
13
  }
2571
466
  if (def->size > vi->size) {
2572
48
    pr_warn("map '%s': invalid def size.\n", map_name);
2573
48
    return -EINVAL;
2574
48
  }
2575
2576
418
  map = bpf_object__add_map(obj);
2577
418
  if (IS_ERR(map))
2578
0
    return PTR_ERR(map);
2579
418
  map->name = strdup(map_name);
2580
418
  if (!map->name) {
2581
0
    pr_warn("map '%s': failed to alloc map name.\n", map_name);
2582
0
    return -ENOMEM;
2583
0
  }
2584
418
  map->libbpf_type = LIBBPF_MAP_UNSPEC;
2585
418
  map->def.type = BPF_MAP_TYPE_UNSPEC;
2586
418
  map->sec_idx = sec_idx;
2587
418
  map->sec_offset = vi->offset;
2588
418
  map->btf_var_idx = var_idx;
2589
418
  pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2590
418
     map_name, map->sec_idx, map->sec_offset);
2591
2592
418
  err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2593
418
  if (err)
2594
418
    return err;
2595
2596
0
  fill_map_from_def(map, &map_def);
2597
2598
0
  if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
2599
0
    err = build_map_pin_path(map, pin_root_path);
2600
0
    if (err) {
2601
0
      pr_warn("map '%s': couldn't build pin path.\n", map->name);
2602
0
      return err;
2603
0
    }
2604
0
  }
2605
2606
0
  if (map_def.parts & MAP_DEF_INNER_MAP) {
2607
0
    map->inner_map = calloc(1, sizeof(*map->inner_map));
2608
0
    if (!map->inner_map)
2609
0
      return -ENOMEM;
2610
0
    map->inner_map->fd = -1;
2611
0
    map->inner_map->sec_idx = sec_idx;
2612
0
    map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2613
0
    if (!map->inner_map->name)
2614
0
      return -ENOMEM;
2615
0
    sprintf(map->inner_map->name, "%s.inner", map_name);
2616
2617
0
    fill_map_from_def(map->inner_map, &inner_def);
2618
0
  }
2619
2620
0
  err = map_fill_btf_type_info(obj, map);
2621
0
  if (err)
2622
0
    return err;
2623
2624
0
  return 0;
2625
0
}
2626
2627
static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2628
            const char *pin_root_path)
2629
2.74k
{
2630
2.74k
  const struct btf_type *sec = NULL;
2631
2.74k
  int nr_types, i, vlen, err;
2632
2.74k
  const struct btf_type *t;
2633
2.74k
  const char *name;
2634
2.74k
  Elf_Data *data;
2635
2.74k
  Elf_Scn *scn;
2636
2637
2.74k
  if (obj->efile.btf_maps_shndx < 0)
2638
2.15k
    return 0;
2639
2640
586
  scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2641
586
  data = elf_sec_data(obj, scn);
2642
586
  if (!scn || !data) {
2643
0
    pr_warn("elf: failed to get %s map definitions for %s\n",
2644
0
      MAPS_ELF_SEC, obj->path);
2645
0
    return -EINVAL;
2646
0
  }
2647
2648
586
  nr_types = btf__type_cnt(obj->btf);
2649
7.11k
  for (i = 1; i < nr_types; i++) {
2650
7.05k
    t = btf__type_by_id(obj->btf, i);
2651
7.05k
    if (!btf_is_datasec(t))
2652
5.95k
      continue;
2653
1.10k
    name = btf__name_by_offset(obj->btf, t->name_off);
2654
1.10k
    if (strcmp(name, MAPS_ELF_SEC) == 0) {
2655
530
      sec = t;
2656
530
      obj->efile.btf_maps_sec_btf_id = i;
2657
530
      break;
2658
530
    }
2659
1.10k
  }
2660
2661
586
  if (!sec) {
2662
56
    pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2663
56
    return -ENOENT;
2664
56
  }
2665
2666
530
  vlen = btf_vlen(sec);
2667
530
  for (i = 0; i < vlen; i++) {
2668
529
    err = bpf_object__init_user_btf_map(obj, sec, i,
2669
529
                obj->efile.btf_maps_shndx,
2670
529
                data, strict,
2671
529
                pin_root_path);
2672
529
    if (err)
2673
529
      return err;
2674
529
  }
2675
2676
1
  return 0;
2677
530
}
2678
2679
static int bpf_object__init_maps(struct bpf_object *obj,
2680
         const struct bpf_object_open_opts *opts)
2681
2.74k
{
2682
2.74k
  const char *pin_root_path;
2683
2.74k
  bool strict;
2684
2.74k
  int err = 0;
2685
2686
2.74k
  strict = !OPTS_GET(opts, relaxed_maps, false);
2687
2.74k
  pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
2688
2689
2.74k
  err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2690
2.74k
  err = err ?: bpf_object__init_global_data_maps(obj);
2691
2.74k
  err = err ?: bpf_object__init_kconfig_map(obj);
2692
2.74k
  err = err ?: bpf_object_init_struct_ops(obj);
2693
2694
2.74k
  return err;
2695
2.74k
}
2696
2697
static bool section_have_execinstr(struct bpf_object *obj, int idx)
2698
2.42k
{
2699
2.42k
  Elf64_Shdr *sh;
2700
2701
2.42k
  sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));
2702
2.42k
  if (!sh)
2703
0
    return false;
2704
2705
2.42k
  return sh->sh_flags & SHF_EXECINSTR;
2706
2.42k
}
2707
2708
static bool btf_needs_sanitization(struct bpf_object *obj)
2709
0
{
2710
0
  bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2711
0
  bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2712
0
  bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2713
0
  bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2714
0
  bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2715
0
  bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2716
0
  bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2717
2718
0
  return !has_func || !has_datasec || !has_func_global || !has_float ||
2719
0
         !has_decl_tag || !has_type_tag || !has_enum64;
2720
0
}
2721
2722
static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2723
0
{
2724
0
  bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2725
0
  bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2726
0
  bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2727
0
  bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2728
0
  bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2729
0
  bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2730
0
  bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2731
0
  int enum64_placeholder_id = 0;
2732
0
  struct btf_type *t;
2733
0
  int i, j, vlen;
2734
2735
0
  for (i = 1; i < btf__type_cnt(btf); i++) {
2736
0
    t = (struct btf_type *)btf__type_by_id(btf, i);
2737
2738
0
    if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) {
2739
      /* replace VAR/DECL_TAG with INT */
2740
0
      t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
2741
      /*
2742
       * using size = 1 is the safest choice, 4 will be too
2743
       * big and cause kernel BTF validation failure if
2744
       * original variable took less than 4 bytes
2745
       */
2746
0
      t->size = 1;
2747
0
      *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
2748
0
    } else if (!has_datasec && btf_is_datasec(t)) {
2749
      /* replace DATASEC with STRUCT */
2750
0
      const struct btf_var_secinfo *v = btf_var_secinfos(t);
2751
0
      struct btf_member *m = btf_members(t);
2752
0
      struct btf_type *vt;
2753
0
      char *name;
2754
2755
0
      name = (char *)btf__name_by_offset(btf, t->name_off);
2756
0
      while (*name) {
2757
0
        if (*name == '.')
2758
0
          *name = '_';
2759
0
        name++;
2760
0
      }
2761
2762
0
      vlen = btf_vlen(t);
2763
0
      t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
2764
0
      for (j = 0; j < vlen; j++, v++, m++) {
2765
        /* order of field assignments is important */
2766
0
        m->offset = v->offset * 8;
2767
0
        m->type = v->type;
2768
        /* preserve variable name as member name */
2769
0
        vt = (void *)btf__type_by_id(btf, v->type);
2770
0
        m->name_off = vt->name_off;
2771
0
      }
2772
0
    } else if (!has_func && btf_is_func_proto(t)) {
2773
      /* replace FUNC_PROTO with ENUM */
2774
0
      vlen = btf_vlen(t);
2775
0
      t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
2776
0
      t->size = sizeof(__u32); /* kernel enforced */
2777
0
    } else if (!has_func && btf_is_func(t)) {
2778
      /* replace FUNC with TYPEDEF */
2779
0
      t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
2780
0
    } else if (!has_func_global && btf_is_func(t)) {
2781
      /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
2782
0
      t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
2783
0
    } else if (!has_float && btf_is_float(t)) {
2784
      /* replace FLOAT with an equally-sized empty STRUCT;
2785
       * since C compilers do not accept e.g. "float" as a
2786
       * valid struct name, make it anonymous
2787
       */
2788
0
      t->name_off = 0;
2789
0
      t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
2790
0
    } else if (!has_type_tag && btf_is_type_tag(t)) {
2791
      /* replace TYPE_TAG with a CONST */
2792
0
      t->name_off = 0;
2793
0
      t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
2794
0
    } else if (!has_enum64 && btf_is_enum(t)) {
2795
      /* clear the kflag */
2796
0
      t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
2797
0
    } else if (!has_enum64 && btf_is_enum64(t)) {
2798
      /* replace ENUM64 with a union */
2799
0
      struct btf_member *m;
2800
2801
0
      if (enum64_placeholder_id == 0) {
2802
0
        enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
2803
0
        if (enum64_placeholder_id < 0)
2804
0
          return enum64_placeholder_id;
2805
2806
0
        t = (struct btf_type *)btf__type_by_id(btf, i);
2807
0
      }
2808
2809
0
      m = btf_members(t);
2810
0
      vlen = btf_vlen(t);
2811
0
      t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
2812
0
      for (j = 0; j < vlen; j++, m++) {
2813
0
        m->type = enum64_placeholder_id;
2814
0
        m->offset = 0;
2815
0
      }
2816
0
    }
2817
0
  }
2818
2819
0
  return 0;
2820
0
}
2821
2822
static bool libbpf_needs_btf(const struct bpf_object *obj)
2823
2.99k
{
2824
2.99k
  return obj->efile.btf_maps_shndx >= 0 ||
2825
2.99k
         obj->efile.st_ops_shndx >= 0 ||
2826
2.99k
         obj->efile.st_ops_link_shndx >= 0 ||
2827
2.99k
         obj->nr_extern > 0;
2828
2.99k
}
2829
2830
static bool kernel_needs_btf(const struct bpf_object *obj)
2831
0
{
2832
0
  return obj->efile.st_ops_shndx >= 0 || obj->efile.st_ops_link_shndx >= 0;
2833
0
}
2834
2835
static int bpf_object__init_btf(struct bpf_object *obj,
2836
        Elf_Data *btf_data,
2837
        Elf_Data *btf_ext_data)
2838
5.97k
{
2839
5.97k
  int err = -ENOENT;
2840
2841
5.97k
  if (btf_data) {
2842
4.17k
    obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2843
4.17k
    err = libbpf_get_error(obj->btf);
2844
4.17k
    if (err) {
2845
962
      obj->btf = NULL;
2846
962
      pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);
2847
962
      goto out;
2848
962
    }
2849
    /* enforce 8-byte pointers for BPF-targeted BTFs */
2850
3.21k
    btf__set_pointer_size(obj->btf, 8);
2851
3.21k
  }
2852
5.01k
  if (btf_ext_data) {
2853
349
    struct btf_ext_info *ext_segs[3];
2854
349
    int seg_num, sec_num;
2855
2856
349
    if (!obj->btf) {
2857
4
      pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
2858
4
         BTF_EXT_ELF_SEC, BTF_ELF_SEC);
2859
4
      goto out;
2860
4
    }
2861
345
    obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2862
345
    err = libbpf_get_error(obj->btf_ext);
2863
345
    if (err) {
2864
233
      pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",
2865
233
        BTF_EXT_ELF_SEC, err);
2866
233
      obj->btf_ext = NULL;
2867
233
      goto out;
2868
233
    }
2869
2870
    /* setup .BTF.ext to ELF section mapping */
2871
112
    ext_segs[0] = &obj->btf_ext->func_info;
2872
112
    ext_segs[1] = &obj->btf_ext->line_info;
2873
112
    ext_segs[2] = &obj->btf_ext->core_relo_info;
2874
448
    for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) {
2875
336
      struct btf_ext_info *seg = ext_segs[seg_num];
2876
336
      const struct btf_ext_info_sec *sec;
2877
336
      const char *sec_name;
2878
336
      Elf_Scn *scn;
2879
2880
336
      if (seg->sec_cnt == 0)
2881
226
        continue;
2882
2883
110
      seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
2884
110
      if (!seg->sec_idxs) {
2885
0
        err = -ENOMEM;
2886
0
        goto out;
2887
0
      }
2888
2889
110
      sec_num = 0;
2890
434
      for_each_btf_ext_sec(seg, sec) {
2891
        /* preventively increment index to avoid doing
2892
         * this before every continue below
2893
         */
2894
434
        sec_num++;
2895
2896
434
        sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
2897
434
        if (str_is_empty(sec_name))
2898
226
          continue;
2899
208
        scn = elf_sec_by_name(obj, sec_name);
2900
208
        if (!scn)
2901
186
          continue;
2902
2903
22
        seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
2904
22
      }
2905
110
    }
2906
112
  }
2907
5.97k
out:
2908
5.97k
  if (err && libbpf_needs_btf(obj)) {
2909
53
    pr_warn("BTF is required, but is missing or corrupted.\n");
2910
53
    return err;
2911
53
  }
2912
5.92k
  return 0;
2913
5.97k
}
2914
2915
static int compare_vsi_off(const void *_a, const void *_b)
2916
1.42k
{
2917
1.42k
  const struct btf_var_secinfo *a = _a;
2918
1.42k
  const struct btf_var_secinfo *b = _b;
2919
2920
1.42k
  return a->offset - b->offset;
2921
1.42k
}
2922
2923
static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
2924
           struct btf_type *t)
2925
3.45k
{
2926
3.45k
  __u32 size = 0, i, vars = btf_vlen(t);
2927
3.45k
  const char *sec_name = btf__name_by_offset(btf, t->name_off);
2928
3.45k
  struct btf_var_secinfo *vsi;
2929
3.45k
  bool fixup_offsets = false;
2930
3.45k
  int err;
2931
2932
3.45k
  if (!sec_name) {
2933
0
    pr_debug("No name found in string section for DATASEC kind.\n");
2934
0
    return -ENOENT;
2935
0
  }
2936
2937
  /* Extern-backing datasecs (.ksyms, .kconfig) have their size and
2938
   * variable offsets set at the previous step. Further, not every
2939
   * extern BTF VAR has corresponding ELF symbol preserved, so we skip
2940
   * all fixups altogether for such sections and go straight to sorting
2941
   * VARs within their DATASEC.
2942
   */
2943
3.45k
  if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0)
2944
1.01k
    goto sort_vars;
2945
2946
  /* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to
2947
   * fix this up. But BPF static linker already fixes this up and fills
2948
   * all the sizes and offsets during static linking. So this step has
2949
   * to be optional. But the STV_HIDDEN handling is non-optional for any
2950
   * non-extern DATASEC, so the variable fixup loop below handles both
2951
   * functions at the same time, paying the cost of BTF VAR <-> ELF
2952
   * symbol matching just once.
2953
   */
2954
2.44k
  if (t->size == 0) {
2955
447
    err = find_elf_sec_sz(obj, sec_name, &size);
2956
447
    if (err || !size) {
2957
157
      pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n",
2958
157
         sec_name, size, err);
2959
157
      return -ENOENT;
2960
157
    }
2961
2962
290
    t->size = size;
2963
290
    fixup_offsets = true;
2964
290
  }
2965
2966
3.20k
  for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
2967
1.34k
    const struct btf_type *t_var;
2968
1.34k
    struct btf_var *var;
2969
1.34k
    const char *var_name;
2970
1.34k
    Elf64_Sym *sym;
2971
2972
1.34k
    t_var = btf__type_by_id(btf, vsi->type);
2973
1.34k
    if (!t_var || !btf_is_var(t_var)) {
2974
143
      pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
2975
143
      return -EINVAL;
2976
143
    }
2977
2978
1.19k
    var = btf_var(t_var);
2979
1.19k
    if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
2980
215
      continue;
2981
2982
982
    var_name = btf__name_by_offset(btf, t_var->name_off);
2983
982
    if (!var_name) {
2984
0
      pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n",
2985
0
         sec_name, i);
2986
0
      return -ENOENT;
2987
0
    }
2988
2989
982
    sym = find_elf_var_sym(obj, var_name);
2990
982
    if (IS_ERR(sym)) {
2991
274
      pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n",
2992
274
         sec_name, var_name);
2993
274
      return -ENOENT;
2994
274
    }
2995
2996
708
    if (fixup_offsets)
2997
70
      vsi->offset = sym->st_value;
2998
2999
    /* if variable is a global/weak symbol, but has restricted
3000
     * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR
3001
     * as static. This follows similar logic for functions (BPF
3002
     * subprogs) and influences libbpf's further decisions about
3003
     * whether to make global data BPF array maps as
3004
     * BPF_F_MMAPABLE.
3005
     */
3006
708
    if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
3007
708
        || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
3008
35
      var->linkage = BTF_VAR_STATIC;
3009
708
  }
3010
3011
2.87k
sort_vars:
3012
2.87k
  qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
3013
2.87k
  return 0;
3014
2.28k
}
3015
3016
static int bpf_object_fixup_btf(struct bpf_object *obj)
3017
3.31k
{
3018
3.31k
  int i, n, err = 0;
3019
3020
3.31k
  if (!obj->btf)
3021
1.61k
    return 0;
3022
3023
1.70k
  n = btf__type_cnt(obj->btf);
3024
17.3k
  for (i = 1; i < n; i++) {
3025
16.2k
    struct btf_type *t = btf_type_by_id(obj->btf, i);
3026
3027
    /* Loader needs to fix up some of the things compiler
3028
     * couldn't get its hands on while emitting BTF. This
3029
     * is section size and global variable offset. We use
3030
     * the info from the ELF itself for this purpose.
3031
     */
3032
16.2k
    if (btf_is_datasec(t)) {
3033
3.45k
      err = btf_fixup_datasec(obj, obj->btf, t);
3034
3.45k
      if (err)
3035
574
        return err;
3036
3.45k
    }
3037
16.2k
  }
3038
3039
1.13k
  return 0;
3040
1.70k
}
3041
3042
static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
3043
0
{
3044
0
  if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3045
0
      prog->type == BPF_PROG_TYPE_LSM)
3046
0
    return true;
3047
3048
  /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
3049
   * also need vmlinux BTF
3050
   */
3051
0
  if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3052
0
    return true;
3053
3054
0
  return false;
3055
0
}
3056
3057
static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
3058
0
{
3059
0
  struct bpf_program *prog;
3060
0
  int i;
3061
3062
  /* CO-RE relocations need kernel BTF, only when btf_custom_path
3063
   * is not specified
3064
   */
3065
0
  if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3066
0
    return true;
3067
3068
  /* Support for typed ksyms needs kernel BTF */
3069
0
  for (i = 0; i < obj->nr_extern; i++) {
3070
0
    const struct extern_desc *ext;
3071
3072
0
    ext = &obj->externs[i];
3073
0
    if (ext->type == EXT_KSYM && ext->ksym.type_id)
3074
0
      return true;
3075
0
  }
3076
3077
0
  bpf_object__for_each_program(prog, obj) {
3078
0
    if (!prog->autoload)
3079
0
      continue;
3080
0
    if (prog_needs_vmlinux_btf(prog))
3081
0
      return true;
3082
0
  }
3083
3084
0
  return false;
3085
0
}
3086
3087
static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
3088
0
{
3089
0
  int err;
3090
3091
  /* btf_vmlinux could be loaded earlier */
3092
0
  if (obj->btf_vmlinux || obj->gen_loader)
3093
0
    return 0;
3094
3095
0
  if (!force && !obj_needs_vmlinux_btf(obj))
3096
0
    return 0;
3097
3098
0
  obj->btf_vmlinux = btf__load_vmlinux_btf();
3099
0
  err = libbpf_get_error(obj->btf_vmlinux);
3100
0
  if (err) {
3101
0
    pr_warn("Error loading vmlinux BTF: %d\n", err);
3102
0
    obj->btf_vmlinux = NULL;
3103
0
    return err;
3104
0
  }
3105
0
  return 0;
3106
0
}
3107
3108
static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
3109
0
{
3110
0
  struct btf *kern_btf = obj->btf;
3111
0
  bool btf_mandatory, sanitize;
3112
0
  int i, err = 0;
3113
3114
0
  if (!obj->btf)
3115
0
    return 0;
3116
3117
0
  if (!kernel_supports(obj, FEAT_BTF)) {
3118
0
    if (kernel_needs_btf(obj)) {
3119
0
      err = -EOPNOTSUPP;
3120
0
      goto report;
3121
0
    }
3122
0
    pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
3123
0
    return 0;
3124
0
  }
3125
3126
  /* Even though some subprogs are global/weak, user might prefer more
3127
   * permissive BPF verification process that BPF verifier performs for
3128
   * static functions, taking into account more context from the caller
3129
   * functions. In such case, they need to mark such subprogs with
3130
   * __attribute__((visibility("hidden"))) and libbpf will adjust
3131
   * corresponding FUNC BTF type to be marked as static and trigger more
3132
   * involved BPF verification process.
3133
   */
3134
0
  for (i = 0; i < obj->nr_programs; i++) {
3135
0
    struct bpf_program *prog = &obj->programs[i];
3136
0
    struct btf_type *t;
3137
0
    const char *name;
3138
0
    int j, n;
3139
3140
0
    if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3141
0
      continue;
3142
3143
0
    n = btf__type_cnt(obj->btf);
3144
0
    for (j = 1; j < n; j++) {
3145
0
      t = btf_type_by_id(obj->btf, j);
3146
0
      if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)
3147
0
        continue;
3148
3149
0
      name = btf__str_by_offset(obj->btf, t->name_off);
3150
0
      if (strcmp(name, prog->name) != 0)
3151
0
        continue;
3152
3153
0
      t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3154
0
      break;
3155
0
    }
3156
0
  }
3157
3158
0
  if (!kernel_supports(obj, FEAT_BTF_DECL_TAG))
3159
0
    goto skip_exception_cb;
3160
0
  for (i = 0; i < obj->nr_programs; i++) {
3161
0
    struct bpf_program *prog = &obj->programs[i];
3162
0
    int j, k, n;
3163
3164
0
    if (prog_is_subprog(obj, prog))
3165
0
      continue;
3166
0
    n = btf__type_cnt(obj->btf);
3167
0
    for (j = 1; j < n; j++) {
3168
0
      const char *str = "exception_callback:", *name;
3169
0
      size_t len = strlen(str);
3170
0
      struct btf_type *t;
3171
3172
0
      t = btf_type_by_id(obj->btf, j);
3173
0
      if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1)
3174
0
        continue;
3175
3176
0
      name = btf__str_by_offset(obj->btf, t->name_off);
3177
0
      if (strncmp(name, str, len))
3178
0
        continue;
3179
3180
0
      t = btf_type_by_id(obj->btf, t->type);
3181
0
      if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) {
3182
0
        pr_warn("prog '%s': exception_callback:<value> decl tag not applied to the main program\n",
3183
0
          prog->name);
3184
0
        return -EINVAL;
3185
0
      }
3186
0
      if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)))
3187
0
        continue;
3188
      /* Multiple callbacks are specified for the same prog,
3189
       * the verifier will eventually return an error for this
3190
       * case, hence simply skip appending a subprog.
3191
       */
3192
0
      if (prog->exception_cb_idx >= 0) {
3193
0
        prog->exception_cb_idx = -1;
3194
0
        break;
3195
0
      }
3196
3197
0
      name += len;
3198
0
      if (str_is_empty(name)) {
3199
0
        pr_warn("prog '%s': exception_callback:<value> decl tag contains empty value\n",
3200
0
          prog->name);
3201
0
        return -EINVAL;
3202
0
      }
3203
3204
0
      for (k = 0; k < obj->nr_programs; k++) {
3205
0
        struct bpf_program *subprog = &obj->programs[k];
3206
3207
0
        if (!prog_is_subprog(obj, subprog))
3208
0
          continue;
3209
0
        if (strcmp(name, subprog->name))
3210
0
          continue;
3211
        /* Enforce non-hidden, as from verifier point of
3212
         * view it expects global functions, whereas the
3213
         * mark_btf_static fixes up linkage as static.
3214
         */
3215
0
        if (!subprog->sym_global || subprog->mark_btf_static) {
3216
0
          pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",
3217
0
            prog->name, subprog->name);
3218
0
          return -EINVAL;
3219
0
        }
3220
        /* Let's see if we already saw a static exception callback with the same name */
3221
0
        if (prog->exception_cb_idx >= 0) {
3222
0
          pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n",
3223
0
                  prog->name, subprog->name);
3224
0
          return -EINVAL;
3225
0
        }
3226
0
        prog->exception_cb_idx = k;
3227
0
        break;
3228
0
      }
3229
3230
0
      if (prog->exception_cb_idx >= 0)
3231
0
        continue;
3232
0
      pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);
3233
0
      return -ENOENT;
3234
0
    }
3235
0
  }
3236
0
skip_exception_cb:
3237
3238
0
  sanitize = btf_needs_sanitization(obj);
3239
0
  if (sanitize) {
3240
0
    const void *raw_data;
3241
0
    __u32 sz;
3242
3243
    /* clone BTF to sanitize a copy and leave the original intact */
3244
0
    raw_data = btf__raw_data(obj->btf, &sz);
3245
0
    kern_btf = btf__new(raw_data, sz);
3246
0
    err = libbpf_get_error(kern_btf);
3247
0
    if (err)
3248
0
      return err;
3249
3250
    /* enforce 8-byte pointers for BPF-targeted BTFs */
3251
0
    btf__set_pointer_size(obj->btf, 8);
3252
0
    err = bpf_object__sanitize_btf(obj, kern_btf);
3253
0
    if (err)
3254
0
      return err;
3255
0
  }
3256
3257
0
  if (obj->gen_loader) {
3258
0
    __u32 raw_size = 0;
3259
0
    const void *raw_data = btf__raw_data(kern_btf, &raw_size);
3260
3261
0
    if (!raw_data)
3262
0
      return -ENOMEM;
3263
0
    bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3264
    /* Pretend to have valid FD to pass various fd >= 0 checks.
3265
     * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3266
     */
3267
0
    btf__set_fd(kern_btf, 0);
3268
0
  } else {
3269
    /* currently BPF_BTF_LOAD only supports log_level 1 */
3270
0
    err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3271
0
             obj->log_level ? 1 : 0);
3272
0
  }
3273
0
  if (sanitize) {
3274
0
    if (!err) {
3275
      /* move fd to libbpf's BTF */
3276
0
      btf__set_fd(obj->btf, btf__fd(kern_btf));
3277
0
      btf__set_fd(kern_btf, -1);
3278
0
    }
3279
0
    btf__free(kern_btf);
3280
0
  }
3281
0
report:
3282
0
  if (err) {
3283
0
    btf_mandatory = kernel_needs_btf(obj);
3284
0
    pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
3285
0
      btf_mandatory ? "BTF is mandatory, can't proceed."
3286
0
              : "BTF is optional, ignoring.");
3287
0
    if (!btf_mandatory)
3288
0
      err = 0;
3289
0
  }
3290
0
  return err;
3291
0
}
3292
3293
static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
3294
23.4k
{
3295
23.4k
  const char *name;
3296
3297
23.4k
  name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3298
23.4k
  if (!name) {
3299
7.45k
    pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3300
7.45k
      off, obj->path, elf_errmsg(-1));
3301
7.45k
    return NULL;
3302
7.45k
  }
3303
3304
15.9k
  return name;
3305
23.4k
}
3306
3307
static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
3308
45.5k
{
3309
45.5k
  const char *name;
3310
3311
45.5k
  name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3312
45.5k
  if (!name) {
3313
933
    pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3314
933
      off, obj->path, elf_errmsg(-1));
3315
933
    return NULL;
3316
933
  }
3317
3318
44.6k
  return name;
3319
45.5k
}
3320
3321
static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
3322
13.1k
{
3323
13.1k
  Elf_Scn *scn;
3324
3325
13.1k
  scn = elf_getscn(obj->efile.elf, idx);
3326
13.1k
  if (!scn) {
3327
0
    pr_warn("elf: failed to get section(%zu) from %s: %s\n",
3328
0
      idx, obj->path, elf_errmsg(-1));
3329
0
    return NULL;
3330
0
  }
3331
13.1k
  return scn;
3332
13.1k
}
3333
3334
static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
3335
655
{
3336
655
  Elf_Scn *scn = NULL;
3337
655
  Elf *elf = obj->efile.elf;
3338
655
  const char *sec_name;
3339
3340
4.42k
  while ((scn = elf_nextscn(elf, scn)) != NULL) {
3341
4.08k
    sec_name = elf_sec_name(obj, scn);
3342
4.08k
    if (!sec_name)
3343
0
      return NULL;
3344
3345
4.08k
    if (strcmp(sec_name, name) != 0)
3346
3.76k
      continue;
3347
3348
317
    return scn;
3349
4.08k
  }
3350
338
  return NULL;
3351
655
}
3352
3353
static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn)
3354
93.3k
{
3355
93.3k
  Elf64_Shdr *shdr;
3356
3357
93.3k
  if (!scn)
3358
0
    return NULL;
3359
3360
93.3k
  shdr = elf64_getshdr(scn);
3361
93.3k
  if (!shdr) {
3362
0
    pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
3363
0
      elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3364
0
    return NULL;
3365
0
  }
3366
3367
93.3k
  return shdr;
3368
93.3k
}
3369
3370
static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
3371
9.13k
{
3372
9.13k
  const char *name;
3373
9.13k
  Elf64_Shdr *sh;
3374
3375
9.13k
  if (!scn)
3376
0
    return NULL;
3377
3378
9.13k
  sh = elf_sec_hdr(obj, scn);
3379
9.13k
  if (!sh)
3380
0
    return NULL;
3381
3382
9.13k
  name = elf_sec_str(obj, sh->sh_name);
3383
9.13k
  if (!name) {
3384
837
    pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
3385
837
      elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3386
837
    return NULL;
3387
837
  }
3388
3389
8.29k
  return name;
3390
9.13k
}
3391
3392
static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
3393
36.4k
{
3394
36.4k
  Elf_Data *data;
3395
3396
36.4k
  if (!scn)
3397
152
    return NULL;
3398
3399
36.3k
  data = elf_getdata(scn, 0);
3400
36.3k
  if (!data) {
3401
827
    pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
3402
827
      elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
3403
827
      obj->path, elf_errmsg(-1));
3404
827
    return NULL;
3405
827
  }
3406
3407
35.4k
  return data;
3408
36.3k
}
3409
3410
static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx)
3411
595k
{
3412
595k
  if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3413
165
    return NULL;
3414
3415
595k
  return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3416
595k
}
3417
3418
static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx)
3419
6.54k
{
3420
6.54k
  if (idx >= data->d_size / sizeof(Elf64_Rel))
3421
0
    return NULL;
3422
3423
6.54k
  return (Elf64_Rel *)data->d_buf + idx;
3424
6.54k
}
3425
3426
static bool is_sec_name_dwarf(const char *name)
3427
31.6k
{
3428
  /* approximation, but the actual list is too long */
3429
31.6k
  return str_has_pfx(name, ".debug_");
3430
31.6k
}
3431
3432
static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name)
3433
34.9k
{
3434
  /* no special handling of .strtab */
3435
34.9k
  if (hdr->sh_type == SHT_STRTAB)
3436
5.15k
    return true;
3437
3438
  /* ignore .llvm_addrsig section as well */
3439
29.8k
  if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3440
23
    return true;
3441
3442
  /* no subprograms will lead to an empty .text section, ignore it */
3443
29.7k
  if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3444
29.7k
      strcmp(name, ".text") == 0)
3445
80
    return true;
3446
3447
  /* DWARF sections */
3448
29.7k
  if (is_sec_name_dwarf(name))
3449
2.90k
    return true;
3450
3451
26.8k
  if (str_has_pfx(name, ".rel")) {
3452
1.89k
    name += sizeof(".rel") - 1;
3453
    /* DWARF section relocations */
3454
1.89k
    if (is_sec_name_dwarf(name))
3455
233
      return true;
3456
3457
    /* .BTF and .BTF.ext don't need relocations */
3458
1.66k
    if (strcmp(name, BTF_ELF_SEC) == 0 ||
3459
1.66k
        strcmp(name, BTF_EXT_ELF_SEC) == 0)
3460
419
      return true;
3461
1.66k
  }
3462
3463
26.1k
  return false;
3464
26.8k
}
3465
3466
static int cmp_progs(const void *_a, const void *_b)
3467
43.2k
{
3468
43.2k
  const struct bpf_program *a = _a;
3469
43.2k
  const struct bpf_program *b = _b;
3470
3471
43.2k
  if (a->sec_idx != b->sec_idx)
3472
751
    return a->sec_idx < b->sec_idx ? -1 : 1;
3473
3474
  /* sec_insn_off can't be the same within the section */
3475
42.5k
  return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3476
43.2k
}
3477
3478
static int bpf_object__elf_collect(struct bpf_object *obj)
3479
7.95k
{
3480
7.95k
  struct elf_sec_desc *sec_desc;
3481
7.95k
  Elf *elf = obj->efile.elf;
3482
7.95k
  Elf_Data *btf_ext_data = NULL;
3483
7.95k
  Elf_Data *btf_data = NULL;
3484
7.95k
  int idx = 0, err = 0;
3485
7.95k
  const char *name;
3486
7.95k
  Elf_Data *data;
3487
7.95k
  Elf_Scn *scn;
3488
7.95k
  Elf64_Shdr *sh;
3489
3490
  /* ELF section indices are 0-based, but sec #0 is special "invalid"
3491
   * section. Since section count retrieved by elf_getshdrnum() does
3492
   * include sec #0, it is already the necessary size of an array to keep
3493
   * all the sections.
3494
   */
3495
7.95k
  if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3496
0
    pr_warn("elf: failed to get the number of sections for %s: %s\n",
3497
0
      obj->path, elf_errmsg(-1));
3498
0
    return -LIBBPF_ERRNO__FORMAT;
3499
0
  }
3500
7.95k
  obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3501
7.95k
  if (!obj->efile.secs)
3502
0
    return -ENOMEM;
3503
3504
  /* a bunch of ELF parsing functionality depends on processing symbols,
3505
   * so do the first pass and find the symbol table
3506
   */
3507
7.95k
  scn = NULL;
3508
48.4k
  while ((scn = elf_nextscn(elf, scn)) != NULL) {
3509
40.7k
    sh = elf_sec_hdr(obj, scn);
3510
40.7k
    if (!sh)
3511
0
      return -LIBBPF_ERRNO__FORMAT;
3512
3513
40.7k
    if (sh->sh_type == SHT_SYMTAB) {
3514
7.89k
      if (obj->efile.symbols) {
3515
3
        pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3516
3
        return -LIBBPF_ERRNO__FORMAT;
3517
3
      }
3518
3519
7.88k
      data = elf_sec_data(obj, scn);
3520
7.88k
      if (!data)
3521
240
        return -LIBBPF_ERRNO__FORMAT;
3522
3523
7.64k
      idx = elf_ndxscn(scn);
3524
3525
7.64k
      obj->efile.symbols = data;
3526
7.64k
      obj->efile.symbols_shndx = idx;
3527
7.64k
      obj->efile.strtabidx = sh->sh_link;
3528
7.64k
    }
3529
40.7k
  }
3530
3531
7.71k
  if (!obj->efile.symbols) {
3532
70
    pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
3533
70
      obj->path);
3534
70
    return -ENOENT;
3535
70
  }
3536
3537
7.64k
  scn = NULL;
3538
41.8k
  while ((scn = elf_nextscn(elf, scn)) != NULL) {
3539
35.0k
    idx = elf_ndxscn(scn);
3540
35.0k
    sec_desc = &obj->efile.secs[idx];
3541
3542
35.0k
    sh = elf_sec_hdr(obj, scn);
3543
35.0k
    if (!sh)
3544
0
      return -LIBBPF_ERRNO__FORMAT;
3545
3546
35.0k
    name = elf_sec_str(obj, sh->sh_name);
3547
35.0k
    if (!name)
3548
96
      return -LIBBPF_ERRNO__FORMAT;
3549
3550
34.9k
    if (ignore_elf_section(sh, name))
3551
8.81k
      continue;
3552
3553
26.1k
    data = elf_sec_data(obj, scn);
3554
26.1k
    if (!data)
3555
181
      return -LIBBPF_ERRNO__FORMAT;
3556
3557
25.9k
    pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
3558
25.9k
       idx, name, (unsigned long)data->d_size,
3559
25.9k
       (int)sh->sh_link, (unsigned long)sh->sh_flags,
3560
25.9k
       (int)sh->sh_type);
3561
3562
25.9k
    if (strcmp(name, "license") == 0) {
3563
229
      err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3564
229
      if (err)
3565
2
        return err;
3566
25.7k
    } else if (strcmp(name, "version") == 0) {
3567
12
      err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3568
12
      if (err)
3569
2
        return err;
3570
25.7k
    } else if (strcmp(name, "maps") == 0) {
3571
6
      pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");
3572
6
      return -ENOTSUP;
3573
25.7k
    } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
3574
822
      obj->efile.btf_maps_shndx = idx;
3575
24.9k
    } else if (strcmp(name, BTF_ELF_SEC) == 0) {
3576
4.31k
      if (sh->sh_type != SHT_PROGBITS)
3577
57
        return -LIBBPF_ERRNO__FORMAT;
3578
4.25k
      btf_data = data;
3579
20.5k
    } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
3580
531
      if (sh->sh_type != SHT_PROGBITS)
3581
53
        return -LIBBPF_ERRNO__FORMAT;
3582
478
      btf_ext_data = data;
3583
20.0k
    } else if (sh->sh_type == SHT_SYMTAB) {
3584
      /* already processed during the first pass above */
3585
13.2k
    } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3586
2.65k
      if (sh->sh_flags & SHF_EXECINSTR) {
3587
1.20k
        if (strcmp(name, ".text") == 0)
3588
194
          obj->efile.text_shndx = idx;
3589
1.20k
        err = bpf_object__add_programs(obj, data, name, idx);
3590
1.20k
        if (err)
3591
271
          return err;
3592
1.45k
      } else if (strcmp(name, DATA_SEC) == 0 ||
3593
1.45k
           str_has_pfx(name, DATA_SEC ".")) {
3594
514
        sec_desc->sec_type = SEC_DATA;
3595
514
        sec_desc->shdr = sh;
3596
514
        sec_desc->data = data;
3597
940
      } else if (strcmp(name, RODATA_SEC) == 0 ||
3598
940
           str_has_pfx(name, RODATA_SEC ".")) {
3599
246
        sec_desc->sec_type = SEC_RODATA;
3600
246
        sec_desc->shdr = sh;
3601
246
        sec_desc->data = data;
3602
694
      } else if (strcmp(name, STRUCT_OPS_SEC) == 0) {
3603
91
        obj->efile.st_ops_data = data;
3604
91
        obj->efile.st_ops_shndx = idx;
3605
603
      } else if (strcmp(name, STRUCT_OPS_LINK_SEC) == 0) {
3606
153
        obj->efile.st_ops_link_data = data;
3607
153
        obj->efile.st_ops_link_shndx = idx;
3608
450
      } else {
3609
450
        pr_info("elf: skipping unrecognized data section(%d) %s\n",
3610
450
          idx, name);
3611
450
      }
3612
10.5k
    } else if (sh->sh_type == SHT_REL) {
3613
2.60k
      int targ_sec_idx = sh->sh_info; /* points to other section */
3614
3615
2.60k
      if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3616
2.60k
          targ_sec_idx >= obj->efile.sec_cnt)
3617
183
        return -LIBBPF_ERRNO__FORMAT;
3618
3619
      /* Only do relo for section with exec instructions */
3620
2.42k
      if (!section_have_execinstr(obj, targ_sec_idx) &&
3621
2.42k
          strcmp(name, ".rel" STRUCT_OPS_SEC) &&
3622
2.42k
          strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) &&
3623
2.42k
          strcmp(name, ".rel" MAPS_ELF_SEC)) {
3624
598
        pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
3625
598
          idx, name, targ_sec_idx,
3626
598
          elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");
3627
598
        continue;
3628
598
      }
3629
3630
1.82k
      sec_desc->sec_type = SEC_RELO;
3631
1.82k
      sec_desc->shdr = sh;
3632
1.82k
      sec_desc->data = data;
3633
7.98k
    } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
3634
1.33k
               str_has_pfx(name, BSS_SEC "."))) {
3635
656
      sec_desc->sec_type = SEC_BSS;
3636
656
      sec_desc->shdr = sh;
3637
656
      sec_desc->data = data;
3638
7.32k
    } else {
3639
7.32k
      pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
3640
7.32k
        (size_t)sh->sh_size);
3641
7.32k
    }
3642
25.9k
  }
3643
3644
6.79k
  if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3645
820
    pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3646
820
    return -LIBBPF_ERRNO__FORMAT;
3647
820
  }
3648
3649
  /* sort BPF programs by section name and in-section instruction offset
3650
   * for faster search
3651
   */
3652
5.97k
  if (obj->nr_programs)
3653
614
    qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3654
3655
5.97k
  return bpf_object__init_btf(obj, btf_data, btf_ext_data);
3656
6.79k
}
3657
3658
static bool sym_is_extern(const Elf64_Sym *sym)
3659
322k
{
3660
322k
  int bind = ELF64_ST_BIND(sym->st_info);
3661
  /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
3662
322k
  return sym->st_shndx == SHN_UNDEF &&
3663
322k
         (bind == STB_GLOBAL || bind == STB_WEAK) &&
3664
322k
         ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3665
322k
}
3666
3667
static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx)
3668
779
{
3669
779
  int bind = ELF64_ST_BIND(sym->st_info);
3670
779
  int type = ELF64_ST_TYPE(sym->st_info);
3671
3672
  /* in .text section */
3673
779
  if (sym->st_shndx != text_shndx)
3674
475
    return false;
3675
3676
  /* local function */
3677
304
  if (bind == STB_LOCAL && type == STT_SECTION)
3678
101
    return true;
3679
3680
  /* global function */
3681
203
  return bind == STB_GLOBAL && type == STT_FUNC;
3682
304
}
3683
3684
static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
3685
1.88k
{
3686
1.88k
  const struct btf_type *t;
3687
1.88k
  const char *tname;
3688
1.88k
  int i, n;
3689
3690
1.88k
  if (!btf)
3691
41
    return -ESRCH;
3692
3693
1.83k
  n = btf__type_cnt(btf);
3694
14.9k
  for (i = 1; i < n; i++) {
3695
14.8k
    t = btf__type_by_id(btf, i);
3696
3697
14.8k
    if (!btf_is_var(t) && !btf_is_func(t))
3698
11.4k
      continue;
3699
3700
3.40k
    tname = btf__name_by_offset(btf, t->name_off);
3701
3.40k
    if (strcmp(tname, ext_name))
3702
1.73k
      continue;
3703
3704
1.66k
    if (btf_is_var(t) &&
3705
1.66k
        btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3706
44
      return -EINVAL;
3707
3708
1.61k
    if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
3709
16
      return -EINVAL;
3710
3711
1.60k
    return i;
3712
1.61k
  }
3713
3714
178
  return -ENOENT;
3715
1.83k
}
3716
3717
1.60k
static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
3718
1.60k
  const struct btf_var_secinfo *vs;
3719
1.60k
  const struct btf_type *t;
3720
1.60k
  int i, j, n;
3721
3722
1.60k
  if (!btf)
3723
0
    return -ESRCH;
3724
3725
1.60k
  n = btf__type_cnt(btf);
3726
14.5k
  for (i = 1; i < n; i++) {
3727
14.5k
    t = btf__type_by_id(btf, i);
3728
3729
14.5k
    if (!btf_is_datasec(t))
3730
11.0k
      continue;
3731
3732
3.47k
    vs = btf_var_secinfos(t);
3733
6.32k
    for (j = 0; j < btf_vlen(t); j++, vs++) {
3734
4.43k
      if (vs->type == ext_btf_id)
3735
1.58k
        return i;
3736
4.43k
    }
3737
3.47k
  }
3738
3739
12
  return -ENOENT;
3740
1.60k
}
3741
3742
static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
3743
             bool *is_signed)
3744
775
{
3745
775
  const struct btf_type *t;
3746
775
  const char *name;
3747
3748
775
  t = skip_mods_and_typedefs(btf, id, NULL);
3749
775
  name = btf__name_by_offset(btf, t->name_off);
3750
3751
775
  if (is_signed)
3752
611
    *is_signed = false;
3753
775
  switch (btf_kind(t)) {
3754
354
  case BTF_KIND_INT: {
3755
354
    int enc = btf_int_encoding(t);
3756
3757
354
    if (enc & BTF_INT_BOOL)
3758
57
      return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
3759
297
    if (is_signed)
3760
278
      *is_signed = enc & BTF_INT_SIGNED;
3761
297
    if (t->size == 1)
3762
96
      return KCFG_CHAR;
3763
201
    if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
3764
36
      return KCFG_UNKNOWN;
3765
165
    return KCFG_INT;
3766
201
  }
3767
130
  case BTF_KIND_ENUM:
3768
130
    if (t->size != 4)
3769
16
      return KCFG_UNKNOWN;
3770
114
    if (strcmp(name, "libbpf_tristate"))
3771
113
      return KCFG_UNKNOWN;
3772
1
    return KCFG_TRISTATE;
3773
92
  case BTF_KIND_ENUM64:
3774
92
    if (strcmp(name, "libbpf_tristate"))
3775
82
      return KCFG_UNKNOWN;
3776
10
    return KCFG_TRISTATE;
3777
164
  case BTF_KIND_ARRAY:
3778
164
    if (btf_array(t)->nelems == 0)
3779
0
      return KCFG_UNKNOWN;
3780
164
    if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
3781
146
      return KCFG_UNKNOWN;
3782
18
    return KCFG_CHAR_ARR;
3783
35
  default:
3784
35
    return KCFG_UNKNOWN;
3785
775
  }
3786
775
}
3787
3788
static int cmp_externs(const void *_a, const void *_b)
3789
851
{
3790
851
  const struct extern_desc *a = _a;
3791
851
  const struct extern_desc *b = _b;
3792
3793
851
  if (a->type != b->type)
3794
0
    return a->type < b->type ? -1 : 1;
3795
3796
851
  if (a->type == EXT_KCFG) {
3797
    /* descending order by alignment requirements */
3798
132
    if (a->kcfg.align != b->kcfg.align)
3799
0
      return a->kcfg.align > b->kcfg.align ? -1 : 1;
3800
    /* ascending order by size, within same alignment class */
3801
132
    if (a->kcfg.sz != b->kcfg.sz)
3802
0
      return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
3803
132
  }
3804
3805
  /* resolve ties by name */
3806
851
  return strcmp(a->name, b->name);
3807
851
}
3808
3809
static int find_int_btf_id(const struct btf *btf)
3810
724
{
3811
724
  const struct btf_type *t;
3812
724
  int i, n;
3813
3814
724
  n = btf__type_cnt(btf);
3815
10.1k
  for (i = 1; i < n; i++) {
3816
9.50k
    t = btf__type_by_id(btf, i);
3817
3818
9.50k
    if (btf_is_int(t) && btf_int_bits(t) == 32)
3819
31
      return i;
3820
9.50k
  }
3821
3822
693
  return 0;
3823
724
}
3824
3825
static int add_dummy_ksym_var(struct btf *btf)
3826
4.50k
{
3827
4.50k
  int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
3828
4.50k
  const struct btf_var_secinfo *vs;
3829
4.50k
  const struct btf_type *sec;
3830
3831
4.50k
  if (!btf)
3832
1.65k
    return 0;
3833
3834
2.85k
  sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
3835
2.85k
              BTF_KIND_DATASEC);
3836
2.85k
  if (sec_btf_id < 0)
3837
2.16k
    return 0;
3838
3839
685
  sec = btf__type_by_id(btf, sec_btf_id);
3840
685
  vs = btf_var_secinfos(sec);
3841
1.59k
  for (i = 0; i < btf_vlen(sec); i++, vs++) {
3842
1.41k
    const struct btf_type *vt;
3843
3844
1.41k
    vt = btf__type_by_id(btf, vs->type);
3845
1.41k
    if (btf_is_func(vt))
3846
508
      break;
3847
1.41k
  }
3848
3849
  /* No func in ksyms sec.  No need to add dummy var. */
3850
685
  if (i == btf_vlen(sec))
3851
177
    return 0;
3852
3853
508
  int_btf_id = find_int_btf_id(btf);
3854
508
  dummy_var_btf_id = btf__add_var(btf,
3855
508
          "dummy_ksym",
3856
508
          BTF_VAR_GLOBAL_ALLOCATED,
3857
508
          int_btf_id);
3858
508
  if (dummy_var_btf_id < 0)
3859
508
    pr_warn("cannot create a dummy_ksym var\n");
3860
3861
508
  return dummy_var_btf_id;
3862
685
}
3863
3864
static int bpf_object__collect_externs(struct bpf_object *obj)
3865
5.92k
{
3866
5.92k
  struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
3867
5.92k
  const struct btf_type *t;
3868
5.92k
  struct extern_desc *ext;
3869
5.92k
  int i, n, off, dummy_var_btf_id;
3870
5.92k
  const char *ext_name, *sec_name;
3871
5.92k
  size_t ext_essent_len;
3872
5.92k
  Elf_Scn *scn;
3873
5.92k
  Elf64_Shdr *sh;
3874
3875
5.92k
  if (!obj->efile.symbols)
3876
0
    return 0;
3877
3878
5.92k
  scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3879
5.92k
  sh = elf_sec_hdr(obj, scn);
3880
5.92k
  if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
3881
1.41k
    return -LIBBPF_ERRNO__FORMAT;
3882
3883
4.50k
  dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
3884
4.50k
  if (dummy_var_btf_id < 0)
3885
0
    return dummy_var_btf_id;
3886
3887
4.50k
  n = sh->sh_size / sh->sh_entsize;
3888
4.50k
  pr_debug("looking for externs among %d symbols...\n", n);
3889
3890
324k
  for (i = 0; i < n; i++) {
3891
321k
    Elf64_Sym *sym = elf_sym_by_idx(obj, i);
3892
3893
321k
    if (!sym)
3894
0
      return -LIBBPF_ERRNO__FORMAT;
3895
321k
    if (!sym_is_extern(sym))
3896
316k
      continue;
3897
5.10k
    ext_name = elf_sym_str(obj, sym->st_name);
3898
5.10k
    if (!ext_name || !ext_name[0])
3899
3.22k
      continue;
3900
3901
1.88k
    ext = obj->externs;
3902
1.88k
    ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3903
1.88k
    if (!ext)
3904
0
      return -ENOMEM;
3905
1.88k
    obj->externs = ext;
3906
1.88k
    ext = &ext[obj->nr_extern];
3907
1.88k
    memset(ext, 0, sizeof(*ext));
3908
1.88k
    obj->nr_extern++;
3909
3910
1.88k
    ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3911
1.88k
    if (ext->btf_id <= 0) {
3912
279
      pr_warn("failed to find BTF for extern '%s': %d\n",
3913
279
        ext_name, ext->btf_id);
3914
279
      return ext->btf_id;
3915
279
    }
3916
1.60k
    t = btf__type_by_id(obj->btf, ext->btf_id);
3917
1.60k
    ext->name = btf__name_by_offset(obj->btf, t->name_off);
3918
1.60k
    ext->sym_idx = i;
3919
1.60k
    ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
3920
3921
1.60k
    ext_essent_len = bpf_core_essential_name_len(ext->name);
3922
1.60k
    ext->essent_name = NULL;
3923
1.60k
    if (ext_essent_len != strlen(ext->name)) {
3924
43
      ext->essent_name = strndup(ext->name, ext_essent_len);
3925
43
      if (!ext->essent_name)
3926
0
        return -ENOMEM;
3927
43
    }
3928
3929
1.60k
    ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3930
1.60k
    if (ext->sec_btf_id <= 0) {
3931
12
      pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
3932
12
        ext_name, ext->btf_id, ext->sec_btf_id);
3933
12
      return ext->sec_btf_id;
3934
12
    }
3935
1.58k
    sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3936
1.58k
    sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3937
3938
1.58k
    if (strcmp(sec_name, KCONFIG_SEC) == 0) {
3939
841
      if (btf_is_func(t)) {
3940
1
        pr_warn("extern function %s is unsupported under %s section\n",
3941
1
          ext->name, KCONFIG_SEC);
3942
1
        return -ENOTSUP;
3943
1
      }
3944
840
      kcfg_sec = sec;
3945
840
      ext->type = EXT_KCFG;
3946
840
      ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3947
840
      if (ext->kcfg.sz <= 0) {
3948
217
        pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
3949
217
          ext_name, ext->kcfg.sz);
3950
217
        return ext->kcfg.sz;
3951
217
      }
3952
623
      ext->kcfg.align = btf__align_of(obj->btf, t->type);
3953
623
      if (ext->kcfg.align <= 0) {
3954
12
        pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
3955
12
          ext_name, ext->kcfg.align);
3956
12
        return -EINVAL;
3957
12
      }
3958
611
      ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3959
611
              &ext->kcfg.is_signed);
3960
611
      if (ext->kcfg.type == KCFG_UNKNOWN) {
3961
308
        pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);
3962
308
        return -ENOTSUP;
3963
308
      }
3964
748
    } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
3965
634
      ksym_sec = sec;
3966
634
      ext->type = EXT_KSYM;
3967
634
      skip_mods_and_typedefs(obj->btf, t->type,
3968
634
                 &ext->ksym.type_id);
3969
634
    } else {
3970
114
      pr_warn("unrecognized extern section '%s'\n", sec_name);
3971
114
      return -ENOTSUP;
3972
114
    }
3973
1.58k
  }
3974
3.56k
  pr_debug("collected %d externs total\n", obj->nr_extern);
3975
3976
3.56k
  if (!obj->nr_extern)
3977
3.15k
    return 0;
3978
3979
  /* sort externs by type, for kcfg ones also by (align, size, name) */
3980
408
  qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3981
3982
  /* for .ksyms section, we need to turn all externs into allocated
3983
   * variables in BTF to pass kernel verification; we do this by
3984
   * pretending that each extern is a 8-byte variable
3985
   */
3986
408
  if (ksym_sec) {
3987
    /* find existing 4-byte integer type in BTF to use for fake
3988
     * extern variables in DATASEC
3989
     */
3990
216
    int int_btf_id = find_int_btf_id(obj->btf);
3991
    /* For extern function, a dummy_var added earlier
3992
     * will be used to replace the vs->type and
3993
     * its name string will be used to refill
3994
     * the missing param's name.
3995
     */
3996
216
    const struct btf_type *dummy_var;
3997
3998
216
    dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
3999
810
    for (i = 0; i < obj->nr_extern; i++) {
4000
594
      ext = &obj->externs[i];
4001
594
      if (ext->type != EXT_KSYM)
4002
0
        continue;
4003
594
      pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
4004
594
         i, ext->sym_idx, ext->name);
4005
594
    }
4006
4007
216
    sec = ksym_sec;
4008
216
    n = btf_vlen(sec);
4009
427
    for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
4010
391
      struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
4011
391
      struct btf_type *vt;
4012
4013
391
      vt = (void *)btf__type_by_id(obj->btf, vs->type);
4014
391
      ext_name = btf__name_by_offset(obj->btf, vt->name_off);
4015
391
      ext = find_extern_by_name(obj, ext_name);
4016
391
      if (!ext) {
4017
180
        pr_warn("failed to find extern definition for BTF %s '%s'\n",
4018
180
          btf_kind_str(vt), ext_name);
4019
180
        return -ESRCH;
4020
180
      }
4021
211
      if (btf_is_func(vt)) {
4022
105
        const struct btf_type *func_proto;
4023
105
        struct btf_param *param;
4024
105
        int j;
4025
4026
105
        func_proto = btf__type_by_id(obj->btf,
4027
105
                   vt->type);
4028
105
        param = btf_params(func_proto);
4029
        /* Reuse the dummy_var string if the
4030
         * func proto does not have param name.
4031
         */
4032
217
        for (j = 0; j < btf_vlen(func_proto); j++)
4033
112
          if (param[j].type && !param[j].name_off)
4034
29
            param[j].name_off =
4035
29
              dummy_var->name_off;
4036
105
        vs->type = dummy_var_btf_id;
4037
105
        vt->info &= ~0xffff;
4038
105
        vt->info |= BTF_FUNC_GLOBAL;
4039
106
      } else {
4040
106
        btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4041
106
        vt->type = int_btf_id;
4042
106
      }
4043
211
      vs->offset = off;
4044
211
      vs->size = sizeof(int);
4045
211
    }
4046
36
    sec->size = off;
4047
36
  }
4048
4049
228
  if (kcfg_sec) {
4050
192
    sec = kcfg_sec;
4051
    /* for kcfg externs calculate their offsets within a .kconfig map */
4052
192
    off = 0;
4053
488
    for (i = 0; i < obj->nr_extern; i++) {
4054
296
      ext = &obj->externs[i];
4055
296
      if (ext->type != EXT_KCFG)
4056
0
        continue;
4057
4058
296
      ext->kcfg.data_off = roundup(off, ext->kcfg.align);
4059
296
      off = ext->kcfg.data_off + ext->kcfg.sz;
4060
296
      pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
4061
296
         i, ext->sym_idx, ext->kcfg.data_off, ext->name);
4062
296
    }
4063
192
    sec->size = off;
4064
192
    n = btf_vlen(sec);
4065
278
    for (i = 0; i < n; i++) {
4066
238
      struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
4067
4068
238
      t = btf__type_by_id(obj->btf, vs->type);
4069
238
      ext_name = btf__name_by_offset(obj->btf, t->name_off);
4070
238
      ext = find_extern_by_name(obj, ext_name);
4071
238
      if (!ext) {
4072
152
        pr_warn("failed to find extern definition for BTF var '%s'\n",
4073
152
          ext_name);
4074
152
        return -ESRCH;
4075
152
      }
4076
86
      btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4077
86
      vs->offset = ext->kcfg.data_off;
4078
86
    }
4079
192
  }
4080
76
  return 0;
4081
228
}
4082
4083
static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
4084
8.80k
{
4085
8.80k
  return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
4086
8.80k
}
4087
4088
struct bpf_program *
4089
bpf_object__find_program_by_name(const struct bpf_object *obj,
4090
         const char *name)
4091
0
{
4092
0
  struct bpf_program *prog;
4093
4094
0
  bpf_object__for_each_program(prog, obj) {
4095
0
    if (prog_is_subprog(obj, prog))
4096
0
      continue;
4097
0
    if (!strcmp(prog->name, name))
4098
0
      return prog;
4099
0
  }
4100
0
  return errno = ENOENT, NULL;
4101
0
}
4102
4103
static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
4104
              int shndx)
4105
465
{
4106
465
  switch (obj->efile.secs[shndx].sec_type) {
4107
45
  case SEC_BSS:
4108
342
  case SEC_DATA:
4109
463
  case SEC_RODATA:
4110
463
    return true;
4111
2
  default:
4112
2
    return false;
4113
465
  }
4114
465
}
4115
4116
static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
4117
              int shndx)
4118
21
{
4119
21
  return shndx == obj->efile.btf_maps_shndx;
4120
21
}
4121
4122
static enum libbpf_map_type
4123
bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
4124
486
{
4125
486
  if (shndx == obj->efile.symbols_shndx)
4126
2
    return LIBBPF_MAP_KCONFIG;
4127
4128
484
  switch (obj->efile.secs[shndx].sec_type) {
4129
45
  case SEC_BSS:
4130
45
    return LIBBPF_MAP_BSS;
4131
297
  case SEC_DATA:
4132
297
    return LIBBPF_MAP_DATA;
4133
121
  case SEC_RODATA:
4134
121
    return LIBBPF_MAP_RODATA;
4135
21
  default:
4136
21
    return LIBBPF_MAP_UNSPEC;
4137
484
  }
4138
484
}
4139
4140
static int bpf_program__record_reloc(struct bpf_program *prog,
4141
             struct reloc_desc *reloc_desc,
4142
             __u32 insn_idx, const char *sym_name,
4143
             const Elf64_Sym *sym, const Elf64_Rel *rel)
4144
981
{
4145
981
  struct bpf_insn *insn = &prog->insns[insn_idx];
4146
981
  size_t map_idx, nr_maps = prog->obj->nr_maps;
4147
981
  struct bpf_object *obj = prog->obj;
4148
981
  __u32 shdr_idx = sym->st_shndx;
4149
981
  enum libbpf_map_type type;
4150
981
  const char *sym_sec_name;
4151
981
  struct bpf_map *map;
4152
4153
981
  if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
4154
56
    pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
4155
56
      prog->name, sym_name, insn_idx, insn->code);
4156
56
    return -LIBBPF_ERRNO__RELOC;
4157
56
  }
4158
4159
925
  if (sym_is_extern(sym)) {
4160
2
    int sym_idx = ELF64_R_SYM(rel->r_info);
4161
2
    int i, n = obj->nr_extern;
4162
2
    struct extern_desc *ext;
4163
4164
2
    for (i = 0; i < n; i++) {
4165
0
      ext = &obj->externs[i];
4166
0
      if (ext->sym_idx == sym_idx)
4167
0
        break;
4168
0
    }
4169
2
    if (i >= n) {
4170
2
      pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
4171
2
        prog->name, sym_name, sym_idx);
4172
2
      return -LIBBPF_ERRNO__RELOC;
4173
2
    }
4174
0
    pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
4175
0
       prog->name, i, ext->name, ext->sym_idx, insn_idx);
4176
0
    if (insn->code == (BPF_JMP | BPF_CALL))
4177
0
      reloc_desc->type = RELO_EXTERN_CALL;
4178
0
    else
4179
0
      reloc_desc->type = RELO_EXTERN_LD64;
4180
0
    reloc_desc->insn_idx = insn_idx;
4181
0
    reloc_desc->ext_idx = i;
4182
0
    return 0;
4183
2
  }
4184
4185
  /* sub-program call relocation */
4186
923
  if (is_call_insn(insn)) {
4187
141
    if (insn->src_reg != BPF_PSEUDO_CALL) {
4188
6
      pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4189
6
      return -LIBBPF_ERRNO__RELOC;
4190
6
    }
4191
    /* text_shndx can be 0, if no default "main" program exists */
4192
135
    if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4193
14
      sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4194
14
      pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
4195
14
        prog->name, sym_name, sym_sec_name);
4196
14
      return -LIBBPF_ERRNO__RELOC;
4197
14
    }
4198
121
    if (sym->st_value % BPF_INSN_SZ) {
4199
3
      pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
4200
3
        prog->name, sym_name, (size_t)sym->st_value);
4201
3
      return -LIBBPF_ERRNO__RELOC;
4202
3
    }
4203
118
    reloc_desc->type = RELO_CALL;
4204
118
    reloc_desc->insn_idx = insn_idx;
4205
118
    reloc_desc->sym_off = sym->st_value;
4206
118
    return 0;
4207
121
  }
4208
4209
782
  if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
4210
3
    pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
4211
3
      prog->name, sym_name, shdr_idx);
4212
3
    return -LIBBPF_ERRNO__RELOC;
4213
3
  }
4214
4215
  /* loading subprog addresses */
4216
779
  if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4217
    /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4218
     * local_func: sym->st_value = 0, insn->imm = offset in the section.
4219
     */
4220
293
    if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4221
2
      pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4222
2
        prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4223
2
      return -LIBBPF_ERRNO__RELOC;
4224
2
    }
4225
4226
291
    reloc_desc->type = RELO_SUBPROG_ADDR;
4227
291
    reloc_desc->insn_idx = insn_idx;
4228
291
    reloc_desc->sym_off = sym->st_value;
4229
291
    return 0;
4230
293
  }
4231
4232
486
  type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
4233
486
  sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4234
4235
  /* generic map reference relocation */
4236
486
  if (type == LIBBPF_MAP_UNSPEC) {
4237
21
    if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
4238
21
      pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4239
21
        prog->name, sym_name, sym_sec_name);
4240
21
      return -LIBBPF_ERRNO__RELOC;
4241
21
    }
4242
0
    for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4243
0
      map = &obj->maps[map_idx];
4244
0
      if (map->libbpf_type != type ||
4245
0
          map->sec_idx != sym->st_shndx ||
4246
0
          map->sec_offset != sym->st_value)
4247
0
        continue;
4248
0
      pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
4249
0
         prog->name, map_idx, map->name, map->sec_idx,
4250
0
         map->sec_offset, insn_idx);
4251
0
      break;
4252
0
    }
4253
0
    if (map_idx >= nr_maps) {
4254
0
      pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4255
0
        prog->name, sym_sec_name, (size_t)sym->st_value);
4256
0
      return -LIBBPF_ERRNO__RELOC;
4257
0
    }
4258
0
    reloc_desc->type = RELO_LD64;
4259
0
    reloc_desc->insn_idx = insn_idx;
4260
0
    reloc_desc->map_idx = map_idx;
4261
0
    reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4262
0
    return 0;
4263
0
  }
4264
4265
  /* global data map relocation */
4266
465
  if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
4267
2
    pr_warn("prog '%s': bad data relo against section '%s'\n",
4268
2
      prog->name, sym_sec_name);
4269
2
    return -LIBBPF_ERRNO__RELOC;
4270
2
  }
4271
1.01k
  for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4272
1.00k
    map = &obj->maps[map_idx];
4273
1.00k
    if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4274
547
      continue;
4275
461
    pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
4276
461
       prog->name, map_idx, map->name, map->sec_idx,
4277
461
       map->sec_offset, insn_idx);
4278
461
    break;
4279
1.00k
  }
4280
463
  if (map_idx >= nr_maps) {
4281
2
    pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4282
2
      prog->name, sym_sec_name);
4283
2
    return -LIBBPF_ERRNO__RELOC;
4284
2
  }
4285
4286
461
  reloc_desc->type = RELO_DATA;
4287
461
  reloc_desc->insn_idx = insn_idx;
4288
461
  reloc_desc->map_idx = map_idx;
4289
461
  reloc_desc->sym_off = sym->st_value;
4290
461
  return 0;
4291
463
}
4292
4293
static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4294
1.43k
{
4295
1.43k
  return insn_idx >= prog->sec_insn_off &&
4296
1.43k
         insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4297
1.43k
}
4298
4299
static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
4300
             size_t sec_idx, size_t insn_idx)
4301
6.16k
{
4302
6.16k
  int l = 0, r = obj->nr_programs - 1, m;
4303
6.16k
  struct bpf_program *prog;
4304
4305
6.16k
  if (!obj->nr_programs)
4306
4.31k
    return NULL;
4307
4308
2.93k
  while (l < r) {
4309
1.08k
    m = l + (r - l + 1) / 2;
4310
1.08k
    prog = &obj->programs[m];
4311
4312
1.08k
    if (prog->sec_idx < sec_idx ||
4313
1.08k
        (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4314
516
      l = m;
4315
565
    else
4316
565
      r = m - 1;
4317
1.08k
  }
4318
  /* matching program could be at index l, but it still might be the
4319
   * wrong one, so we need to double check conditions for the last time
4320
   */
4321
1.85k
  prog = &obj->programs[l];
4322
1.85k
  if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4323
981
    return prog;
4324
874
  return NULL;
4325
1.85k
}
4326
4327
static int
4328
bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)
4329
1.37k
{
4330
1.37k
  const char *relo_sec_name, *sec_name;
4331
1.37k
  size_t sec_idx = shdr->sh_info, sym_idx;
4332
1.37k
  struct bpf_program *prog;
4333
1.37k
  struct reloc_desc *relos;
4334
1.37k
  int err, i, nrels;
4335
1.37k
  const char *sym_name;
4336
1.37k
  __u32 insn_idx;
4337
1.37k
  Elf_Scn *scn;
4338
1.37k
  Elf_Data *scn_data;
4339
1.37k
  Elf64_Sym *sym;
4340
1.37k
  Elf64_Rel *rel;
4341
4342
1.37k
  if (sec_idx >= obj->efile.sec_cnt)
4343
0
    return -EINVAL;
4344
4345
1.37k
  scn = elf_sec_by_idx(obj, sec_idx);
4346
1.37k
  scn_data = elf_sec_data(obj, scn);
4347
4348
1.37k
  relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4349
1.37k
  sec_name = elf_sec_name(obj, scn);
4350
1.37k
  if (!relo_sec_name || !sec_name)
4351
19
    return -EINVAL;
4352
4353
1.35k
  pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
4354
1.35k
     relo_sec_name, sec_idx, sec_name);
4355
1.35k
  nrels = shdr->sh_size / shdr->sh_entsize;
4356
4357
7.41k
  for (i = 0; i < nrels; i++) {
4358
6.54k
    rel = elf_rel_by_idx(data, i);
4359
6.54k
    if (!rel) {
4360
0
      pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
4361
0
      return -LIBBPF_ERRNO__FORMAT;
4362
0
    }
4363
4364
6.54k
    sym_idx = ELF64_R_SYM(rel->r_info);
4365
6.54k
    sym = elf_sym_by_idx(obj, sym_idx);
4366
6.54k
    if (!sym) {
4367
165
      pr_warn("sec '%s': symbol #%zu not found for relo #%d\n",
4368
165
        relo_sec_name, sym_idx, i);
4369
165
      return -LIBBPF_ERRNO__FORMAT;
4370
165
    }
4371
4372
6.37k
    if (sym->st_shndx >= obj->efile.sec_cnt) {
4373
18
      pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n",
4374
18
        relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4375
18
      return -LIBBPF_ERRNO__FORMAT;
4376
18
    }
4377
4378
6.35k
    if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4379
190
      pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
4380
190
        relo_sec_name, (size_t)rel->r_offset, i);
4381
190
      return -LIBBPF_ERRNO__FORMAT;
4382
190
    }
4383
4384
6.16k
    insn_idx = rel->r_offset / BPF_INSN_SZ;
4385
    /* relocations against static functions are recorded as
4386
     * relocations against the section that contains a function;
4387
     * in such case, symbol will be STT_SECTION and sym.st_name
4388
     * will point to empty string (0), so fetch section name
4389
     * instead
4390
     */
4391
6.16k
    if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4392
459
      sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4393
5.71k
    else
4394
5.71k
      sym_name = elf_sym_str(obj, sym->st_name);
4395
6.16k
    sym_name = sym_name ?: "<?";
4396
4397
6.16k
    pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
4398
6.16k
       relo_sec_name, i, insn_idx, sym_name);
4399
4400
6.16k
    prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4401
6.16k
    if (!prog) {
4402
5.18k
      pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
4403
5.18k
        relo_sec_name, i, sec_name, insn_idx);
4404
5.18k
      continue;
4405
5.18k
    }
4406
4407
981
    relos = libbpf_reallocarray(prog->reloc_desc,
4408
981
              prog->nr_reloc + 1, sizeof(*relos));
4409
981
    if (!relos)
4410
0
      return -ENOMEM;
4411
981
    prog->reloc_desc = relos;
4412
4413
    /* adjust insn_idx to local BPF program frame of reference */
4414
981
    insn_idx -= prog->sec_insn_off;
4415
981
    err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4416
981
            insn_idx, sym_name, sym, rel);
4417
981
    if (err)
4418
111
      return err;
4419
4420
870
    prog->nr_reloc++;
4421
870
  }
4422
868
  return 0;
4423
1.35k
}
4424
4425
static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map)
4426
1.42k
{
4427
1.42k
  int id;
4428
4429
1.42k
  if (!obj->btf)
4430
1.04k
    return -ENOENT;
4431
4432
  /* if it's BTF-defined map, we don't need to search for type IDs.
4433
   * For struct_ops map, it does not need btf_key_type_id and
4434
   * btf_value_type_id.
4435
   */
4436
377
  if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4437
0
    return 0;
4438
4439
  /*
4440
   * LLVM annotates global data differently in BTF, that is,
4441
   * only as '.data', '.bss' or '.rodata'.
4442
   */
4443
377
  if (!bpf_map__is_internal(map))
4444
0
    return -ENOENT;
4445
4446
377
  id = btf__find_by_name(obj->btf, map->real_name);
4447
377
  if (id < 0)
4448
190
    return id;
4449
4450
187
  map->btf_key_type_id = 0;
4451
187
  map->btf_value_type_id = id;
4452
187
  return 0;
4453
377
}
4454
4455
static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
4456
0
{
4457
0
  char file[PATH_MAX], buff[4096];
4458
0
  FILE *fp;
4459
0
  __u32 val;
4460
0
  int err;
4461
4462
0
  snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
4463
0
  memset(info, 0, sizeof(*info));
4464
4465
0
  fp = fopen(file, "re");
4466
0
  if (!fp) {
4467
0
    err = -errno;
4468
0
    pr_warn("failed to open %s: %d. No procfs support?\n", file,
4469
0
      err);
4470
0
    return err;
4471
0
  }
4472
4473
0
  while (fgets(buff, sizeof(buff), fp)) {
4474
0
    if (sscanf(buff, "map_type:\t%u", &val) == 1)
4475
0
      info->type = val;
4476
0
    else if (sscanf(buff, "key_size:\t%u", &val) == 1)
4477
0
      info->key_size = val;
4478
0
    else if (sscanf(buff, "value_size:\t%u", &val) == 1)
4479
0
      info->value_size = val;
4480
0
    else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
4481
0
      info->max_entries = val;
4482
0
    else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
4483
0
      info->map_flags = val;
4484
0
  }
4485
4486
0
  fclose(fp);
4487
4488
0
  return 0;
4489
0
}
4490
4491
bool bpf_map__autocreate(const struct bpf_map *map)
4492
0
{
4493
0
  return map->autocreate;
4494
0
}
4495
4496
int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
4497
0
{
4498
0
  if (map->obj->loaded)
4499
0
    return libbpf_err(-EBUSY);
4500
4501
0
  map->autocreate = autocreate;
4502
0
  return 0;
4503
0
}
4504
4505
int bpf_map__reuse_fd(struct bpf_map *map, int fd)
4506
0
{
4507
0
  struct bpf_map_info info;
4508
0
  __u32 len = sizeof(info), name_len;
4509
0
  int new_fd, err;
4510
0
  char *new_name;
4511
4512
0
  memset(&info, 0, len);
4513
0
  err = bpf_map_get_info_by_fd(fd, &info, &len);
4514
0
  if (err && errno == EINVAL)
4515
0
    err = bpf_get_map_info_from_fdinfo(fd, &info);
4516
0
  if (err)
4517
0
    return libbpf_err(err);
4518
4519
0
  name_len = strlen(info.name);
4520
0
  if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4521
0
    new_name = strdup(map->name);
4522
0
  else
4523
0
    new_name = strdup(info.name);
4524
4525
0
  if (!new_name)
4526
0
    return libbpf_err(-errno);
4527
4528
  /*
4529
   * Like dup(), but make sure new FD is >= 3 and has O_CLOEXEC set.
4530
   * This is similar to what we do in ensure_good_fd(), but without
4531
   * closing original FD.
4532
   */
4533
0
  new_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3);
4534
0
  if (new_fd < 0) {
4535
0
    err = -errno;
4536
0
    goto err_free_new_name;
4537
0
  }
4538
4539
0
  err = zclose(map->fd);
4540
0
  if (err) {
4541
0
    err = -errno;
4542
0
    goto err_close_new_fd;
4543
0
  }
4544
0
  free(map->name);
4545
4546
0
  map->fd = new_fd;
4547
0
  map->name = new_name;
4548
0
  map->def.type = info.type;
4549
0
  map->def.key_size = info.key_size;
4550
0
  map->def.value_size = info.value_size;
4551
0
  map->def.max_entries = info.max_entries;
4552
0
  map->def.map_flags = info.map_flags;
4553
0
  map->btf_key_type_id = info.btf_key_type_id;
4554
0
  map->btf_value_type_id = info.btf_value_type_id;
4555
0
  map->reused = true;
4556
0
  map->map_extra = info.map_extra;
4557
4558
0
  return 0;
4559
4560
0
err_close_new_fd:
4561
0
  close(new_fd);
4562
0
err_free_new_name:
4563
0
  free(new_name);
4564
0
  return libbpf_err(err);
4565
0
}
4566
4567
__u32 bpf_map__max_entries(const struct bpf_map *map)
4568
0
{
4569
0
  return map->def.max_entries;
4570
0
}
4571
4572
struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
4573
0
{
4574
0
  if (!bpf_map_type__is_map_in_map(map->def.type))
4575
0
    return errno = EINVAL, NULL;
4576
4577
0
  return map->inner_map;
4578
0
}
4579
4580
int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
4581
0
{
4582
0
  if (map->obj->loaded)
4583
0
    return libbpf_err(-EBUSY);
4584
4585
0
  map->def.max_entries = max_entries;
4586
4587
  /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4588
0
  if (map_is_ringbuf(map))
4589
0
    map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4590
4591
0
  return 0;
4592
0
}
4593
4594
static int
4595
bpf_object__probe_loading(struct bpf_object *obj)
4596
0
{
4597
0
  char *cp, errmsg[STRERR_BUFSIZE];
4598
0
  struct bpf_insn insns[] = {
4599
0
    BPF_MOV64_IMM(BPF_REG_0, 0),
4600
0
    BPF_EXIT_INSN(),
4601
0
  };
4602
0
  int ret, insn_cnt = ARRAY_SIZE(insns);
4603
4604
0
  if (obj->gen_loader)
4605
0
    return 0;
4606
4607
0
  ret = bump_rlimit_memlock();
4608
0
  if (ret)
4609
0
    pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);
4610
4611
  /* make sure basic loading works */
4612
0
  ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4613
0
  if (ret < 0)
4614
0
    ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
4615
0
  if (ret < 0) {
4616
0
    ret = errno;
4617
0
    cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4618
0
    pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
4619
0
      "program. Make sure your kernel supports BPF "
4620
0
      "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
4621
0
      "set to big enough value.\n", __func__, cp, ret);
4622
0
    return -ret;
4623
0
  }
4624
0
  close(ret);
4625
4626
0
  return 0;
4627
0
}
4628
4629
static int probe_fd(int fd)
4630
0
{
4631
0
  if (fd >= 0)
4632
0
    close(fd);
4633
0
  return fd >= 0;
4634
0
}
4635
4636
static int probe_kern_prog_name(void)
4637
0
{
4638
0
  const size_t attr_sz = offsetofend(union bpf_attr, prog_name);
4639
0
  struct bpf_insn insns[] = {
4640
0
    BPF_MOV64_IMM(BPF_REG_0, 0),
4641
0
    BPF_EXIT_INSN(),
4642
0
  };
4643
0
  union bpf_attr attr;
4644
0
  int ret;
4645
4646
0
  memset(&attr, 0, attr_sz);
4647
0
  attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER;
4648
0
  attr.license = ptr_to_u64("GPL");
4649
0
  attr.insns = ptr_to_u64(insns);
4650
0
  attr.insn_cnt = (__u32)ARRAY_SIZE(insns);
4651
0
  libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name));
4652
4653
  /* make sure loading with name works */
4654
0
  ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS);
4655
0
  return probe_fd(ret);
4656
0
}
4657
4658
static int probe_kern_global_data(void)
4659
0
{
4660
0
  char *cp, errmsg[STRERR_BUFSIZE];
4661
0
  struct bpf_insn insns[] = {
4662
0
    BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16),
4663
0
    BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42),
4664
0
    BPF_MOV64_IMM(BPF_REG_0, 0),
4665
0
    BPF_EXIT_INSN(),
4666
0
  };
4667
0
  int ret, map, insn_cnt = ARRAY_SIZE(insns);
4668
4669
0
  map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL);
4670
0
  if (map < 0) {
4671
0
    ret = -errno;
4672
0
    cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4673
0
    pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4674
0
      __func__, cp, -ret);
4675
0
    return ret;
4676
0
  }
4677
4678
0
  insns[0].imm = map;
4679
4680
0
  ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4681
0
  close(map);
4682
0
  return probe_fd(ret);
4683
0
}
4684
4685
static int probe_kern_btf(void)
4686
0
{
4687
0
  static const char strs[] = "\0int";
4688
0
  __u32 types[] = {
4689
    /* int */
4690
0
    BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4691
0
  };
4692
4693
0
  return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4694
0
               strs, sizeof(strs)));
4695
0
}
4696
4697
static int probe_kern_btf_func(void)
4698
0
{
4699
0
  static const char strs[] = "\0int\0x\0a";
4700
  /* void x(int a) {} */
4701
0
  __u32 types[] = {
4702
    /* int */
4703
0
    BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4704
    /* FUNC_PROTO */                                /* [2] */
4705
0
    BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4706
0
    BTF_PARAM_ENC(7, 1),
4707
    /* FUNC x */                                    /* [3] */
4708
0
    BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2),
4709
0
  };
4710
4711
0
  return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4712
0
               strs, sizeof(strs)));
4713
0
}
4714
4715
static int probe_kern_btf_func_global(void)
4716
0
{
4717
0
  static const char strs[] = "\0int\0x\0a";
4718
  /* static void x(int a) {} */
4719
0
  __u32 types[] = {
4720
    /* int */
4721
0
    BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4722
    /* FUNC_PROTO */                                /* [2] */
4723
0
    BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0),
4724
0
    BTF_PARAM_ENC(7, 1),
4725
    /* FUNC x BTF_FUNC_GLOBAL */                    /* [3] */
4726
0
    BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2),
4727
0
  };
4728
4729
0
  return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4730
0
               strs, sizeof(strs)));
4731
0
}
4732
4733
static int probe_kern_btf_datasec(void)
4734
0
{
4735
0
  static const char strs[] = "\0x\0.data";
4736
  /* static int a; */
4737
0
  __u32 types[] = {
4738
    /* int */
4739
0
    BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4740
    /* VAR x */                                     /* [2] */
4741
0
    BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4742
0
    BTF_VAR_STATIC,
4743
    /* DATASEC val */                               /* [3] */
4744
0
    BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4),
4745
0
    BTF_VAR_SECINFO_ENC(2, 0, 4),
4746
0
  };
4747
4748
0
  return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4749
0
               strs, sizeof(strs)));
4750
0
}
4751
4752
static int probe_kern_btf_float(void)
4753
0
{
4754
0
  static const char strs[] = "\0float";
4755
0
  __u32 types[] = {
4756
    /* float */
4757
0
    BTF_TYPE_FLOAT_ENC(1, 4),
4758
0
  };
4759
4760
0
  return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4761
0
               strs, sizeof(strs)));
4762
0
}
4763
4764
static int probe_kern_btf_decl_tag(void)
4765
0
{
4766
0
  static const char strs[] = "\0tag";
4767
0
  __u32 types[] = {
4768
    /* int */
4769
0
    BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),  /* [1] */
4770
    /* VAR x */                                     /* [2] */
4771
0
    BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1),
4772
0
    BTF_VAR_STATIC,
4773
    /* attr */
4774
0
    BTF_TYPE_DECL_TAG_ENC(1, 2, -1),
4775
0
  };
4776
4777
0
  return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4778
0
               strs, sizeof(strs)));
4779
0
}
4780
4781
static int probe_kern_btf_type_tag(void)
4782
0
{
4783
0
  static const char strs[] = "\0tag";
4784
0
  __u32 types[] = {
4785
    /* int */
4786
0
    BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),    /* [1] */
4787
    /* attr */
4788
0
    BTF_TYPE_TYPE_TAG_ENC(1, 1),       /* [2] */
4789
    /* ptr */
4790
0
    BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */
4791
0
  };
4792
4793
0
  return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4794
0
               strs, sizeof(strs)));
4795
0
}
4796
4797
static int probe_kern_array_mmap(void)
4798
0
{
4799
0
  LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE);
4800
0
  int fd;
4801
4802
0
  fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts);
4803
0
  return probe_fd(fd);
4804
0
}
4805
4806
static int probe_kern_exp_attach_type(void)
4807
0
{
4808
0
  LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE);
4809
0
  struct bpf_insn insns[] = {
4810
0
    BPF_MOV64_IMM(BPF_REG_0, 0),
4811
0
    BPF_EXIT_INSN(),
4812
0
  };
4813
0
  int fd, insn_cnt = ARRAY_SIZE(insns);
4814
4815
  /* use any valid combination of program type and (optional)
4816
   * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS)
4817
   * to see if kernel supports expected_attach_type field for
4818
   * BPF_PROG_LOAD command
4819
   */
4820
0
  fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts);
4821
0
  return probe_fd(fd);
4822
0
}
4823
4824
static int probe_kern_probe_read_kernel(void)
4825
0
{
4826
0
  struct bpf_insn insns[] = {
4827
0
    BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),  /* r1 = r10 (fp) */
4828
0
    BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */
4829
0
    BPF_MOV64_IMM(BPF_REG_2, 8),   /* r2 = 8 */
4830
0
    BPF_MOV64_IMM(BPF_REG_3, 0),   /* r3 = 0 */
4831
0
    BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel),
4832
0
    BPF_EXIT_INSN(),
4833
0
  };
4834
0
  int fd, insn_cnt = ARRAY_SIZE(insns);
4835
4836
0
  fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL);
4837
0
  return probe_fd(fd);
4838
0
}
4839
4840
static int probe_prog_bind_map(void)
4841
0
{
4842
0
  char *cp, errmsg[STRERR_BUFSIZE];
4843
0
  struct bpf_insn insns[] = {
4844
0
    BPF_MOV64_IMM(BPF_REG_0, 0),
4845
0
    BPF_EXIT_INSN(),
4846
0
  };
4847
0
  int ret, map, prog, insn_cnt = ARRAY_SIZE(insns);
4848
4849
0
  map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL);
4850
0
  if (map < 0) {
4851
0
    ret = -errno;
4852
0
    cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4853
0
    pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n",
4854
0
      __func__, cp, -ret);
4855
0
    return ret;
4856
0
  }
4857
4858
0
  prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4859
0
  if (prog < 0) {
4860
0
    close(map);
4861
0
    return 0;
4862
0
  }
4863
4864
0
  ret = bpf_prog_bind_map(prog, map, NULL);
4865
4866
0
  close(map);
4867
0
  close(prog);
4868
4869
0
  return ret >= 0;
4870
0
}
4871
4872
static int probe_module_btf(void)
4873
0
{
4874
0
  static const char strs[] = "\0int";
4875
0
  __u32 types[] = {
4876
    /* int */
4877
0
    BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4),
4878
0
  };
4879
0
  struct bpf_btf_info info;
4880
0
  __u32 len = sizeof(info);
4881
0
  char name[16];
4882
0
  int fd, err;
4883
4884
0
  fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs));
4885
0
  if (fd < 0)
4886
0
    return 0; /* BTF not supported at all */
4887
4888
0
  memset(&info, 0, sizeof(info));
4889
0
  info.name = ptr_to_u64(name);
4890
0
  info.name_len = sizeof(name);
4891
4892
  /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer;
4893
   * kernel's module BTF support coincides with support for
4894
   * name/name_len fields in struct bpf_btf_info.
4895
   */
4896
0
  err = bpf_btf_get_info_by_fd(fd, &info, &len);
4897
0
  close(fd);
4898
0
  return !err;
4899
0
}
4900
4901
static int probe_perf_link(void)
4902
0
{
4903
0
  struct bpf_insn insns[] = {
4904
0
    BPF_MOV64_IMM(BPF_REG_0, 0),
4905
0
    BPF_EXIT_INSN(),
4906
0
  };
4907
0
  int prog_fd, link_fd, err;
4908
4909
0
  prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL",
4910
0
        insns, ARRAY_SIZE(insns), NULL);
4911
0
  if (prog_fd < 0)
4912
0
    return -errno;
4913
4914
  /* use invalid perf_event FD to get EBADF, if link is supported;
4915
   * otherwise EINVAL should be returned
4916
   */
4917
0
  link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL);
4918
0
  err = -errno; /* close() can clobber errno */
4919
4920
0
  if (link_fd >= 0)
4921
0
    close(link_fd);
4922
0
  close(prog_fd);
4923
4924
0
  return link_fd < 0 && err == -EBADF;
4925
0
}
4926
4927
static int probe_uprobe_multi_link(void)
4928
0
{
4929
0
  LIBBPF_OPTS(bpf_prog_load_opts, load_opts,
4930
0
    .expected_attach_type = BPF_TRACE_UPROBE_MULTI,
4931
0
  );
4932
0
  LIBBPF_OPTS(bpf_link_create_opts, link_opts);
4933
0
  struct bpf_insn insns[] = {
4934
0
    BPF_MOV64_IMM(BPF_REG_0, 0),
4935
0
    BPF_EXIT_INSN(),
4936
0
  };
4937
0
  int prog_fd, link_fd, err;
4938
0
  unsigned long offset = 0;
4939
4940
0
  prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL",
4941
0
        insns, ARRAY_SIZE(insns), &load_opts);
4942
0
  if (prog_fd < 0)
4943
0
    return -errno;
4944
4945
  /* Creating uprobe in '/' binary should fail with -EBADF. */
4946
0
  link_opts.uprobe_multi.path = "/";
4947
0
  link_opts.uprobe_multi.offsets = &offset;
4948
0
  link_opts.uprobe_multi.cnt = 1;
4949
4950
0
  link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts);
4951
0
  err = -errno; /* close() can clobber errno */
4952
4953
0
  if (link_fd >= 0)
4954
0
    close(link_fd);
4955
0
  close(prog_fd);
4956
4957
0
  return link_fd < 0 && err == -EBADF;
4958
0
}
4959
4960
static int probe_kern_bpf_cookie(void)
4961
0
{
4962
0
  struct bpf_insn insns[] = {
4963
0
    BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie),
4964
0
    BPF_EXIT_INSN(),
4965
0
  };
4966
0
  int ret, insn_cnt = ARRAY_SIZE(insns);
4967
4968
0
  ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL);
4969
0
  return probe_fd(ret);
4970
0
}
4971
4972
static int probe_kern_btf_enum64(void)
4973
0
{
4974
0
  static const char strs[] = "\0enum64";
4975
0
  __u32 types[] = {
4976
0
    BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8),
4977
0
  };
4978
4979
0
  return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types),
4980
0
               strs, sizeof(strs)));
4981
0
}
4982
4983
static int probe_kern_syscall_wrapper(void);
4984
4985
enum kern_feature_result {
4986
  FEAT_UNKNOWN = 0,
4987
  FEAT_SUPPORTED = 1,
4988
  FEAT_MISSING = 2,
4989
};
4990
4991
typedef int (*feature_probe_fn)(void);
4992
4993
static struct kern_feature_desc {
4994
  const char *desc;
4995
  feature_probe_fn probe;
4996
  enum kern_feature_result res;
4997
} feature_probes[__FEAT_CNT] = {
4998
  [FEAT_PROG_NAME] = {
4999
    "BPF program name", probe_kern_prog_name,
5000
  },
5001
  [FEAT_GLOBAL_DATA] = {
5002
    "global variables", probe_kern_global_data,
5003
  },
5004
  [FEAT_BTF] = {
5005
    "minimal BTF", probe_kern_btf,
5006
  },
5007
  [FEAT_BTF_FUNC] = {
5008
    "BTF functions", probe_kern_btf_func,
5009
  },
5010
  [FEAT_BTF_GLOBAL_FUNC] = {
5011
    "BTF global function", probe_kern_btf_func_global,
5012
  },
5013
  [FEAT_BTF_DATASEC] = {
5014
    "BTF data section and variable", probe_kern_btf_datasec,
5015
  },
5016
  [FEAT_ARRAY_MMAP] = {
5017
    "ARRAY map mmap()", probe_kern_array_mmap,
5018
  },
5019
  [FEAT_EXP_ATTACH_TYPE] = {
5020
    "BPF_PROG_LOAD expected_attach_type attribute",
5021
    probe_kern_exp_attach_type,
5022
  },
5023
  [FEAT_PROBE_READ_KERN] = {
5024
    "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel,
5025
  },
5026
  [FEAT_PROG_BIND_MAP] = {
5027
    "BPF_PROG_BIND_MAP support", probe_prog_bind_map,
5028
  },
5029
  [FEAT_MODULE_BTF] = {
5030
    "module BTF support", probe_module_btf,
5031
  },
5032
  [FEAT_BTF_FLOAT] = {
5033
    "BTF_KIND_FLOAT support", probe_kern_btf_float,
5034
  },
5035
  [FEAT_PERF_LINK] = {
5036
    "BPF perf link support", probe_perf_link,
5037
  },
5038
  [FEAT_BTF_DECL_TAG] = {
5039
    "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag,
5040
  },
5041
  [FEAT_BTF_TYPE_TAG] = {
5042
    "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag,
5043
  },
5044
  [FEAT_MEMCG_ACCOUNT] = {
5045
    "memcg-based memory accounting", probe_memcg_account,
5046
  },
5047
  [FEAT_BPF_COOKIE] = {
5048
    "BPF cookie support", probe_kern_bpf_cookie,
5049
  },
5050
  [FEAT_BTF_ENUM64] = {
5051
    "BTF_KIND_ENUM64 support", probe_kern_btf_enum64,
5052
  },
5053
  [FEAT_SYSCALL_WRAPPER] = {
5054
    "Kernel using syscall wrapper", probe_kern_syscall_wrapper,
5055
  },
5056
  [FEAT_UPROBE_MULTI_LINK] = {
5057
    "BPF multi-uprobe link support", probe_uprobe_multi_link,
5058
  },
5059
};
5060
5061
bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
5062
0
{
5063
0
  struct kern_feature_desc *feat = &feature_probes[feat_id];
5064
0
  int ret;
5065
5066
0
  if (obj && obj->gen_loader)
5067
    /* To generate loader program assume the latest kernel
5068
     * to avoid doing extra prog_load, map_create syscalls.
5069
     */
5070
0
    return true;
5071
5072
0
  if (READ_ONCE(feat->res) == FEAT_UNKNOWN) {
5073
0
    ret = feat->probe();
5074
0
    if (ret > 0) {
5075
0
      WRITE_ONCE(feat->res, FEAT_SUPPORTED);
5076
0
    } else if (ret == 0) {
5077
0
      WRITE_ONCE(feat->res, FEAT_MISSING);
5078
0
    } else {
5079
0
      pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret);
5080
0
      WRITE_ONCE(feat->res, FEAT_MISSING);
5081
0
    }
5082
0
  }
5083
5084
0
  return READ_ONCE(feat->res) == FEAT_SUPPORTED;
5085
0
}
5086
5087
static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
5088
0
{
5089
0
  struct bpf_map_info map_info;
5090
0
  char msg[STRERR_BUFSIZE];
5091
0
  __u32 map_info_len = sizeof(map_info);
5092
0
  int err;
5093
5094
0
  memset(&map_info, 0, map_info_len);
5095
0
  err = bpf_map_get_info_by_fd(map_fd, &map_info, &map_info_len);
5096
0
  if (err && errno == EINVAL)
5097
0
    err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
5098
0
  if (err) {
5099
0
    pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
5100
0
      libbpf_strerror_r(errno, msg, sizeof(msg)));
5101
0
    return false;
5102
0
  }
5103
5104
0
  return (map_info.type == map->def.type &&
5105
0
    map_info.key_size == map->def.key_size &&
5106
0
    map_info.value_size == map->def.value_size &&
5107
0
    <