Coverage Report

Created: 2024-05-20 06:14

/src/libbpf/src/libbpf.c
Line
Count
Source (jump to first uncovered line)
1
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
2
3
/*
4
 * Common eBPF ELF object loading operations.
5
 *
6
 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org>
7
 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com>
8
 * Copyright (C) 2015 Huawei Inc.
9
 * Copyright (C) 2017 Nicira, Inc.
10
 * Copyright (C) 2019 Isovalent, Inc.
11
 */
12
13
#ifndef _GNU_SOURCE
14
#define _GNU_SOURCE
15
#endif
16
#include <stdlib.h>
17
#include <stdio.h>
18
#include <stdarg.h>
19
#include <libgen.h>
20
#include <inttypes.h>
21
#include <limits.h>
22
#include <string.h>
23
#include <unistd.h>
24
#include <endian.h>
25
#include <fcntl.h>
26
#include <errno.h>
27
#include <ctype.h>
28
#include <asm/unistd.h>
29
#include <linux/err.h>
30
#include <linux/kernel.h>
31
#include <linux/bpf.h>
32
#include <linux/btf.h>
33
#include <linux/filter.h>
34
#include <linux/limits.h>
35
#include <linux/perf_event.h>
36
#include <linux/bpf_perf_event.h>
37
#include <linux/ring_buffer.h>
38
#include <sys/epoll.h>
39
#include <sys/ioctl.h>
40
#include <sys/mman.h>
41
#include <sys/stat.h>
42
#include <sys/types.h>
43
#include <sys/vfs.h>
44
#include <sys/utsname.h>
45
#include <sys/resource.h>
46
#include <libelf.h>
47
#include <gelf.h>
48
#include <zlib.h>
49
50
#include "libbpf.h"
51
#include "bpf.h"
52
#include "btf.h"
53
#include "str_error.h"
54
#include "libbpf_internal.h"
55
#include "hashmap.h"
56
#include "bpf_gen_internal.h"
57
#include "zip.h"
58
59
#ifndef BPF_FS_MAGIC
60
0
#define BPF_FS_MAGIC    0xcafe4a11
61
#endif
62
63
1
#define BPF_FS_DEFAULT_PATH "/sys/fs/bpf"
64
65
125k
#define BPF_INSN_SZ (sizeof(struct bpf_insn))
66
67
/* vsprintf() in __base_pr() uses nonliteral format string. It may break
68
 * compilation if user enables corresponding warning. Disable it explicitly.
69
 */
70
#pragma GCC diagnostic ignored "-Wformat-nonliteral"
71
72
#define __printf(a, b)  __attribute__((format(printf, a, b)))
73
74
static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
75
static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
76
static int map_set_def_max_entries(struct bpf_map *map);
77
78
static const char * const attach_type_name[] = {
79
  [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress",
80
  [BPF_CGROUP_INET_EGRESS]  = "cgroup_inet_egress",
81
  [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create",
82
  [BPF_CGROUP_INET_SOCK_RELEASE]  = "cgroup_inet_sock_release",
83
  [BPF_CGROUP_SOCK_OPS]   = "cgroup_sock_ops",
84
  [BPF_CGROUP_DEVICE]   = "cgroup_device",
85
  [BPF_CGROUP_INET4_BIND]   = "cgroup_inet4_bind",
86
  [BPF_CGROUP_INET6_BIND]   = "cgroup_inet6_bind",
87
  [BPF_CGROUP_INET4_CONNECT]  = "cgroup_inet4_connect",
88
  [BPF_CGROUP_INET6_CONNECT]  = "cgroup_inet6_connect",
89
  [BPF_CGROUP_UNIX_CONNECT]       = "cgroup_unix_connect",
90
  [BPF_CGROUP_INET4_POST_BIND]  = "cgroup_inet4_post_bind",
91
  [BPF_CGROUP_INET6_POST_BIND]  = "cgroup_inet6_post_bind",
92
  [BPF_CGROUP_INET4_GETPEERNAME]  = "cgroup_inet4_getpeername",
93
  [BPF_CGROUP_INET6_GETPEERNAME]  = "cgroup_inet6_getpeername",
94
  [BPF_CGROUP_UNIX_GETPEERNAME] = "cgroup_unix_getpeername",
95
  [BPF_CGROUP_INET4_GETSOCKNAME]  = "cgroup_inet4_getsockname",
96
  [BPF_CGROUP_INET6_GETSOCKNAME]  = "cgroup_inet6_getsockname",
97
  [BPF_CGROUP_UNIX_GETSOCKNAME] = "cgroup_unix_getsockname",
98
  [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg",
99
  [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg",
100
  [BPF_CGROUP_UNIX_SENDMSG] = "cgroup_unix_sendmsg",
101
  [BPF_CGROUP_SYSCTL]   = "cgroup_sysctl",
102
  [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg",
103
  [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg",
104
  [BPF_CGROUP_UNIX_RECVMSG] = "cgroup_unix_recvmsg",
105
  [BPF_CGROUP_GETSOCKOPT]   = "cgroup_getsockopt",
106
  [BPF_CGROUP_SETSOCKOPT]   = "cgroup_setsockopt",
107
  [BPF_SK_SKB_STREAM_PARSER]  = "sk_skb_stream_parser",
108
  [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict",
109
  [BPF_SK_SKB_VERDICT]    = "sk_skb_verdict",
110
  [BPF_SK_MSG_VERDICT]    = "sk_msg_verdict",
111
  [BPF_LIRC_MODE2]    = "lirc_mode2",
112
  [BPF_FLOW_DISSECTOR]    = "flow_dissector",
113
  [BPF_TRACE_RAW_TP]    = "trace_raw_tp",
114
  [BPF_TRACE_FENTRY]    = "trace_fentry",
115
  [BPF_TRACE_FEXIT]   = "trace_fexit",
116
  [BPF_MODIFY_RETURN]   = "modify_return",
117
  [BPF_LSM_MAC]     = "lsm_mac",
118
  [BPF_LSM_CGROUP]    = "lsm_cgroup",
119
  [BPF_SK_LOOKUP]     = "sk_lookup",
120
  [BPF_TRACE_ITER]    = "trace_iter",
121
  [BPF_XDP_DEVMAP]    = "xdp_devmap",
122
  [BPF_XDP_CPUMAP]    = "xdp_cpumap",
123
  [BPF_XDP]     = "xdp",
124
  [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select",
125
  [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE]  = "sk_reuseport_select_or_migrate",
126
  [BPF_PERF_EVENT]    = "perf_event",
127
  [BPF_TRACE_KPROBE_MULTI]  = "trace_kprobe_multi",
128
  [BPF_STRUCT_OPS]    = "struct_ops",
129
  [BPF_NETFILTER]     = "netfilter",
130
  [BPF_TCX_INGRESS]   = "tcx_ingress",
131
  [BPF_TCX_EGRESS]    = "tcx_egress",
132
  [BPF_TRACE_UPROBE_MULTI]  = "trace_uprobe_multi",
133
  [BPF_NETKIT_PRIMARY]    = "netkit_primary",
134
  [BPF_NETKIT_PEER]   = "netkit_peer",
135
  [BPF_TRACE_KPROBE_SESSION]  = "trace_kprobe_session",
136
};
137
138
static const char * const link_type_name[] = {
139
  [BPF_LINK_TYPE_UNSPEC]      = "unspec",
140
  [BPF_LINK_TYPE_RAW_TRACEPOINT]    = "raw_tracepoint",
141
  [BPF_LINK_TYPE_TRACING]     = "tracing",
142
  [BPF_LINK_TYPE_CGROUP]      = "cgroup",
143
  [BPF_LINK_TYPE_ITER]      = "iter",
144
  [BPF_LINK_TYPE_NETNS]     = "netns",
145
  [BPF_LINK_TYPE_XDP]     = "xdp",
146
  [BPF_LINK_TYPE_PERF_EVENT]    = "perf_event",
147
  [BPF_LINK_TYPE_KPROBE_MULTI]    = "kprobe_multi",
148
  [BPF_LINK_TYPE_STRUCT_OPS]    = "struct_ops",
149
  [BPF_LINK_TYPE_NETFILTER]   = "netfilter",
150
  [BPF_LINK_TYPE_TCX]     = "tcx",
151
  [BPF_LINK_TYPE_UPROBE_MULTI]    = "uprobe_multi",
152
  [BPF_LINK_TYPE_NETKIT]      = "netkit",
153
  [BPF_LINK_TYPE_SOCKMAP]     = "sockmap",
154
};
155
156
static const char * const map_type_name[] = {
157
  [BPF_MAP_TYPE_UNSPEC]     = "unspec",
158
  [BPF_MAP_TYPE_HASH]     = "hash",
159
  [BPF_MAP_TYPE_ARRAY]      = "array",
160
  [BPF_MAP_TYPE_PROG_ARRAY]   = "prog_array",
161
  [BPF_MAP_TYPE_PERF_EVENT_ARRAY]   = "perf_event_array",
162
  [BPF_MAP_TYPE_PERCPU_HASH]    = "percpu_hash",
163
  [BPF_MAP_TYPE_PERCPU_ARRAY]   = "percpu_array",
164
  [BPF_MAP_TYPE_STACK_TRACE]    = "stack_trace",
165
  [BPF_MAP_TYPE_CGROUP_ARRAY]   = "cgroup_array",
166
  [BPF_MAP_TYPE_LRU_HASH]     = "lru_hash",
167
  [BPF_MAP_TYPE_LRU_PERCPU_HASH]    = "lru_percpu_hash",
168
  [BPF_MAP_TYPE_LPM_TRIE]     = "lpm_trie",
169
  [BPF_MAP_TYPE_ARRAY_OF_MAPS]    = "array_of_maps",
170
  [BPF_MAP_TYPE_HASH_OF_MAPS]   = "hash_of_maps",
171
  [BPF_MAP_TYPE_DEVMAP]     = "devmap",
172
  [BPF_MAP_TYPE_DEVMAP_HASH]    = "devmap_hash",
173
  [BPF_MAP_TYPE_SOCKMAP]      = "sockmap",
174
  [BPF_MAP_TYPE_CPUMAP]     = "cpumap",
175
  [BPF_MAP_TYPE_XSKMAP]     = "xskmap",
176
  [BPF_MAP_TYPE_SOCKHASH]     = "sockhash",
177
  [BPF_MAP_TYPE_CGROUP_STORAGE]   = "cgroup_storage",
178
  [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY]  = "reuseport_sockarray",
179
  [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE]  = "percpu_cgroup_storage",
180
  [BPF_MAP_TYPE_QUEUE]      = "queue",
181
  [BPF_MAP_TYPE_STACK]      = "stack",
182
  [BPF_MAP_TYPE_SK_STORAGE]   = "sk_storage",
183
  [BPF_MAP_TYPE_STRUCT_OPS]   = "struct_ops",
184
  [BPF_MAP_TYPE_RINGBUF]      = "ringbuf",
185
  [BPF_MAP_TYPE_INODE_STORAGE]    = "inode_storage",
186
  [BPF_MAP_TYPE_TASK_STORAGE]   = "task_storage",
187
  [BPF_MAP_TYPE_BLOOM_FILTER]   = "bloom_filter",
188
  [BPF_MAP_TYPE_USER_RINGBUF]             = "user_ringbuf",
189
  [BPF_MAP_TYPE_CGRP_STORAGE]   = "cgrp_storage",
190
  [BPF_MAP_TYPE_ARENA]      = "arena",
191
};
192
193
static const char * const prog_type_name[] = {
194
  [BPF_PROG_TYPE_UNSPEC]      = "unspec",
195
  [BPF_PROG_TYPE_SOCKET_FILTER]   = "socket_filter",
196
  [BPF_PROG_TYPE_KPROBE]      = "kprobe",
197
  [BPF_PROG_TYPE_SCHED_CLS]   = "sched_cls",
198
  [BPF_PROG_TYPE_SCHED_ACT]   = "sched_act",
199
  [BPF_PROG_TYPE_TRACEPOINT]    = "tracepoint",
200
  [BPF_PROG_TYPE_XDP]     = "xdp",
201
  [BPF_PROG_TYPE_PERF_EVENT]    = "perf_event",
202
  [BPF_PROG_TYPE_CGROUP_SKB]    = "cgroup_skb",
203
  [BPF_PROG_TYPE_CGROUP_SOCK]   = "cgroup_sock",
204
  [BPF_PROG_TYPE_LWT_IN]      = "lwt_in",
205
  [BPF_PROG_TYPE_LWT_OUT]     = "lwt_out",
206
  [BPF_PROG_TYPE_LWT_XMIT]    = "lwt_xmit",
207
  [BPF_PROG_TYPE_SOCK_OPS]    = "sock_ops",
208
  [BPF_PROG_TYPE_SK_SKB]      = "sk_skb",
209
  [BPF_PROG_TYPE_CGROUP_DEVICE]   = "cgroup_device",
210
  [BPF_PROG_TYPE_SK_MSG]      = "sk_msg",
211
  [BPF_PROG_TYPE_RAW_TRACEPOINT]    = "raw_tracepoint",
212
  [BPF_PROG_TYPE_CGROUP_SOCK_ADDR]  = "cgroup_sock_addr",
213
  [BPF_PROG_TYPE_LWT_SEG6LOCAL]   = "lwt_seg6local",
214
  [BPF_PROG_TYPE_LIRC_MODE2]    = "lirc_mode2",
215
  [BPF_PROG_TYPE_SK_REUSEPORT]    = "sk_reuseport",
216
  [BPF_PROG_TYPE_FLOW_DISSECTOR]    = "flow_dissector",
217
  [BPF_PROG_TYPE_CGROUP_SYSCTL]   = "cgroup_sysctl",
218
  [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable",
219
  [BPF_PROG_TYPE_CGROUP_SOCKOPT]    = "cgroup_sockopt",
220
  [BPF_PROG_TYPE_TRACING]     = "tracing",
221
  [BPF_PROG_TYPE_STRUCT_OPS]    = "struct_ops",
222
  [BPF_PROG_TYPE_EXT]     = "ext",
223
  [BPF_PROG_TYPE_LSM]     = "lsm",
224
  [BPF_PROG_TYPE_SK_LOOKUP]   = "sk_lookup",
225
  [BPF_PROG_TYPE_SYSCALL]     = "syscall",
226
  [BPF_PROG_TYPE_NETFILTER]   = "netfilter",
227
};
228
229
static int __base_pr(enum libbpf_print_level level, const char *format,
230
         va_list args)
231
0
{
232
0
  if (level == LIBBPF_DEBUG)
233
0
    return 0;
234
235
0
  return vfprintf(stderr, format, args);
236
0
}
237
238
static libbpf_print_fn_t __libbpf_pr = __base_pr;
239
240
libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn)
241
11.0k
{
242
11.0k
  libbpf_print_fn_t old_print_fn;
243
244
11.0k
  old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED);
245
246
11.0k
  return old_print_fn;
247
11.0k
}
248
249
__printf(2, 3)
250
void libbpf_print(enum libbpf_print_level level, const char *format, ...)
251
128k
{
252
128k
  va_list args;
253
128k
  int old_errno;
254
128k
  libbpf_print_fn_t print_fn;
255
256
128k
  print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED);
257
128k
  if (!print_fn)
258
0
    return;
259
260
128k
  old_errno = errno;
261
262
128k
  va_start(args, format);
263
128k
  __libbpf_pr(level, format, args);
264
128k
  va_end(args);
265
266
128k
  errno = old_errno;
267
128k
}
268
269
static void pr_perm_msg(int err)
270
0
{
271
0
  struct rlimit limit;
272
0
  char buf[100];
273
274
0
  if (err != -EPERM || geteuid() != 0)
275
0
    return;
276
277
0
  err = getrlimit(RLIMIT_MEMLOCK, &limit);
278
0
  if (err)
279
0
    return;
280
281
0
  if (limit.rlim_cur == RLIM_INFINITY)
282
0
    return;
283
284
0
  if (limit.rlim_cur < 1024)
285
0
    snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
286
0
  else if (limit.rlim_cur < 1024*1024)
287
0
    snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
288
0
  else
289
0
    snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
290
291
0
  pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
292
0
    buf);
293
0
}
294
295
#define STRERR_BUFSIZE  128
296
297
/* Copied from tools/perf/util/util.h */
298
#ifndef zfree
299
240k
# define zfree(ptr) ({ free(*ptr); *ptr = NULL; })
300
#endif
301
302
#ifndef zclose
303
42.4k
# define zclose(fd) ({      \
304
42.4k
  int ___err = 0;     \
305
42.4k
  if ((fd) >= 0)     \
306
42.4k
    ___err = close((fd)); \
307
42.4k
  fd = -1;      \
308
42.4k
  ___err; })
309
#endif
310
311
static inline __u64 ptr_to_u64(const void *ptr)
312
0
{
313
0
  return (__u64) (unsigned long) ptr;
314
0
}
315
316
int libbpf_set_strict_mode(enum libbpf_strict_mode mode)
317
0
{
318
  /* as of v1.0 libbpf_set_strict_mode() is a no-op */
319
0
  return 0;
320
0
}
321
322
__u32 libbpf_major_version(void)
323
0
{
324
0
  return LIBBPF_MAJOR_VERSION;
325
0
}
326
327
__u32 libbpf_minor_version(void)
328
0
{
329
0
  return LIBBPF_MINOR_VERSION;
330
0
}
331
332
const char *libbpf_version_string(void)
333
0
{
334
0
#define __S(X) #X
335
0
#define _S(X) __S(X)
336
0
  return  "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION);
337
0
#undef _S
338
0
#undef __S
339
0
}
340
341
enum reloc_type {
342
  RELO_LD64,
343
  RELO_CALL,
344
  RELO_DATA,
345
  RELO_EXTERN_LD64,
346
  RELO_EXTERN_CALL,
347
  RELO_SUBPROG_ADDR,
348
  RELO_CORE,
349
};
350
351
struct reloc_desc {
352
  enum reloc_type type;
353
  int insn_idx;
354
  union {
355
    const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */
356
    struct {
357
      int map_idx;
358
      int sym_off;
359
      int ext_idx;
360
    };
361
  };
362
};
363
364
/* stored as sec_def->cookie for all libbpf-supported SEC()s */
365
enum sec_def_flags {
366
  SEC_NONE = 0,
367
  /* expected_attach_type is optional, if kernel doesn't support that */
368
  SEC_EXP_ATTACH_OPT = 1,
369
  /* legacy, only used by libbpf_get_type_names() and
370
   * libbpf_attach_type_by_name(), not used by libbpf itself at all.
371
   * This used to be associated with cgroup (and few other) BPF programs
372
   * that were attachable through BPF_PROG_ATTACH command. Pretty
373
   * meaningless nowadays, though.
374
   */
375
  SEC_ATTACHABLE = 2,
376
  SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT,
377
  /* attachment target is specified through BTF ID in either kernel or
378
   * other BPF program's BTF object
379
   */
380
  SEC_ATTACH_BTF = 4,
381
  /* BPF program type allows sleeping/blocking in kernel */
382
  SEC_SLEEPABLE = 8,
383
  /* BPF program support non-linear XDP buffer */
384
  SEC_XDP_FRAGS = 16,
385
  /* Setup proper attach type for usdt probes. */
386
  SEC_USDT = 32,
387
};
388
389
struct bpf_sec_def {
390
  char *sec;
391
  enum bpf_prog_type prog_type;
392
  enum bpf_attach_type expected_attach_type;
393
  long cookie;
394
  int handler_id;
395
396
  libbpf_prog_setup_fn_t prog_setup_fn;
397
  libbpf_prog_prepare_load_fn_t prog_prepare_load_fn;
398
  libbpf_prog_attach_fn_t prog_attach_fn;
399
};
400
401
/*
402
 * bpf_prog should be a better name but it has been used in
403
 * linux/filter.h.
404
 */
405
struct bpf_program {
406
  char *name;
407
  char *sec_name;
408
  size_t sec_idx;
409
  const struct bpf_sec_def *sec_def;
410
  /* this program's instruction offset (in number of instructions)
411
   * within its containing ELF section
412
   */
413
  size_t sec_insn_off;
414
  /* number of original instructions in ELF section belonging to this
415
   * program, not taking into account subprogram instructions possible
416
   * appended later during relocation
417
   */
418
  size_t sec_insn_cnt;
419
  /* Offset (in number of instructions) of the start of instruction
420
   * belonging to this BPF program  within its containing main BPF
421
   * program. For the entry-point (main) BPF program, this is always
422
   * zero. For a sub-program, this gets reset before each of main BPF
423
   * programs are processed and relocated and is used to determined
424
   * whether sub-program was already appended to the main program, and
425
   * if yes, at which instruction offset.
426
   */
427
  size_t sub_insn_off;
428
429
  /* instructions that belong to BPF program; insns[0] is located at
430
   * sec_insn_off instruction within its ELF section in ELF file, so
431
   * when mapping ELF file instruction index to the local instruction,
432
   * one needs to subtract sec_insn_off; and vice versa.
433
   */
434
  struct bpf_insn *insns;
435
  /* actual number of instruction in this BPF program's image; for
436
   * entry-point BPF programs this includes the size of main program
437
   * itself plus all the used sub-programs, appended at the end
438
   */
439
  size_t insns_cnt;
440
441
  struct reloc_desc *reloc_desc;
442
  int nr_reloc;
443
444
  /* BPF verifier log settings */
445
  char *log_buf;
446
  size_t log_size;
447
  __u32 log_level;
448
449
  struct bpf_object *obj;
450
451
  int fd;
452
  bool autoload;
453
  bool autoattach;
454
  bool sym_global;
455
  bool mark_btf_static;
456
  enum bpf_prog_type type;
457
  enum bpf_attach_type expected_attach_type;
458
  int exception_cb_idx;
459
460
  int prog_ifindex;
461
  __u32 attach_btf_obj_fd;
462
  __u32 attach_btf_id;
463
  __u32 attach_prog_fd;
464
465
  void *func_info;
466
  __u32 func_info_rec_size;
467
  __u32 func_info_cnt;
468
469
  void *line_info;
470
  __u32 line_info_rec_size;
471
  __u32 line_info_cnt;
472
  __u32 prog_flags;
473
};
474
475
struct bpf_struct_ops {
476
  const char *tname;
477
  const struct btf_type *type;
478
  struct bpf_program **progs;
479
  __u32 *kern_func_off;
480
  /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */
481
  void *data;
482
  /* e.g. struct bpf_struct_ops_tcp_congestion_ops in
483
   *      btf_vmlinux's format.
484
   * struct bpf_struct_ops_tcp_congestion_ops {
485
   *  [... some other kernel fields ...]
486
   *  struct tcp_congestion_ops data;
487
   * }
488
   * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
489
   * bpf_map__init_kern_struct_ops() will populate the "kern_vdata"
490
   * from "data".
491
   */
492
  void *kern_vdata;
493
  __u32 type_id;
494
};
495
496
2.22k
#define DATA_SEC ".data"
497
2.18k
#define BSS_SEC ".bss"
498
1.70k
#define RODATA_SEC ".rodata"
499
7.25k
#define KCONFIG_SEC ".kconfig"
500
9.51k
#define KSYMS_SEC ".ksyms"
501
4.11k
#define STRUCT_OPS_SEC ".struct_ops"
502
3.81k
#define STRUCT_OPS_LINK_SEC ".struct_ops.link"
503
840
#define ARENA_SEC ".addr_space.1"
504
505
enum libbpf_map_type {
506
  LIBBPF_MAP_UNSPEC,
507
  LIBBPF_MAP_DATA,
508
  LIBBPF_MAP_BSS,
509
  LIBBPF_MAP_RODATA,
510
  LIBBPF_MAP_KCONFIG,
511
};
512
513
struct bpf_map_def {
514
  unsigned int type;
515
  unsigned int key_size;
516
  unsigned int value_size;
517
  unsigned int max_entries;
518
  unsigned int map_flags;
519
};
520
521
struct bpf_map {
522
  struct bpf_object *obj;
523
  char *name;
524
  /* real_name is defined for special internal maps (.rodata*,
525
   * .data*, .bss, .kconfig) and preserves their original ELF section
526
   * name. This is important to be able to find corresponding BTF
527
   * DATASEC information.
528
   */
529
  char *real_name;
530
  int fd;
531
  int sec_idx;
532
  size_t sec_offset;
533
  int map_ifindex;
534
  int inner_map_fd;
535
  struct bpf_map_def def;
536
  __u32 numa_node;
537
  __u32 btf_var_idx;
538
  int mod_btf_fd;
539
  __u32 btf_key_type_id;
540
  __u32 btf_value_type_id;
541
  __u32 btf_vmlinux_value_type_id;
542
  enum libbpf_map_type libbpf_type;
543
  void *mmaped;
544
  struct bpf_struct_ops *st_ops;
545
  struct bpf_map *inner_map;
546
  void **init_slots;
547
  int init_slots_sz;
548
  char *pin_path;
549
  bool pinned;
550
  bool reused;
551
  bool autocreate;
552
  __u64 map_extra;
553
};
554
555
enum extern_type {
556
  EXT_UNKNOWN,
557
  EXT_KCFG,
558
  EXT_KSYM,
559
};
560
561
enum kcfg_type {
562
  KCFG_UNKNOWN,
563
  KCFG_CHAR,
564
  KCFG_BOOL,
565
  KCFG_INT,
566
  KCFG_TRISTATE,
567
  KCFG_CHAR_ARR,
568
};
569
570
struct extern_desc {
571
  enum extern_type type;
572
  int sym_idx;
573
  int btf_id;
574
  int sec_btf_id;
575
  const char *name;
576
  char *essent_name;
577
  bool is_set;
578
  bool is_weak;
579
  union {
580
    struct {
581
      enum kcfg_type type;
582
      int sz;
583
      int align;
584
      int data_off;
585
      bool is_signed;
586
    } kcfg;
587
    struct {
588
      unsigned long long addr;
589
590
      /* target btf_id of the corresponding kernel var. */
591
      int kernel_btf_obj_fd;
592
      int kernel_btf_id;
593
594
      /* local btf_id of the ksym extern's type. */
595
      __u32 type_id;
596
      /* BTF fd index to be patched in for insn->off, this is
597
       * 0 for vmlinux BTF, index in obj->fd_array for module
598
       * BTF
599
       */
600
      __s16 btf_fd_idx;
601
    } ksym;
602
  };
603
};
604
605
struct module_btf {
606
  struct btf *btf;
607
  char *name;
608
  __u32 id;
609
  int fd;
610
  int fd_array_idx;
611
};
612
613
enum sec_type {
614
  SEC_UNUSED = 0,
615
  SEC_RELO,
616
  SEC_BSS,
617
  SEC_DATA,
618
  SEC_RODATA,
619
  SEC_ST_OPS,
620
};
621
622
struct elf_sec_desc {
623
  enum sec_type sec_type;
624
  Elf64_Shdr *shdr;
625
  Elf_Data *data;
626
};
627
628
struct elf_state {
629
  int fd;
630
  const void *obj_buf;
631
  size_t obj_buf_sz;
632
  Elf *elf;
633
  Elf64_Ehdr *ehdr;
634
  Elf_Data *symbols;
635
  Elf_Data *arena_data;
636
  size_t shstrndx; /* section index for section name strings */
637
  size_t strtabidx;
638
  struct elf_sec_desc *secs;
639
  size_t sec_cnt;
640
  int btf_maps_shndx;
641
  __u32 btf_maps_sec_btf_id;
642
  int text_shndx;
643
  int symbols_shndx;
644
  bool has_st_ops;
645
  int arena_data_shndx;
646
};
647
648
struct usdt_manager;
649
650
struct bpf_object {
651
  char name[BPF_OBJ_NAME_LEN];
652
  char license[64];
653
  __u32 kern_version;
654
655
  struct bpf_program *programs;
656
  size_t nr_programs;
657
  struct bpf_map *maps;
658
  size_t nr_maps;
659
  size_t maps_cap;
660
661
  char *kconfig;
662
  struct extern_desc *externs;
663
  int nr_extern;
664
  int kconfig_map_idx;
665
666
  bool loaded;
667
  bool has_subcalls;
668
  bool has_rodata;
669
670
  struct bpf_gen *gen_loader;
671
672
  /* Information when doing ELF related work. Only valid if efile.elf is not NULL */
673
  struct elf_state efile;
674
675
  struct btf *btf;
676
  struct btf_ext *btf_ext;
677
678
  /* Parse and load BTF vmlinux if any of the programs in the object need
679
   * it at load time.
680
   */
681
  struct btf *btf_vmlinux;
682
  /* Path to the custom BTF to be used for BPF CO-RE relocations as an
683
   * override for vmlinux BTF.
684
   */
685
  char *btf_custom_path;
686
  /* vmlinux BTF override for CO-RE relocations */
687
  struct btf *btf_vmlinux_override;
688
  /* Lazily initialized kernel module BTFs */
689
  struct module_btf *btf_modules;
690
  bool btf_modules_loaded;
691
  size_t btf_module_cnt;
692
  size_t btf_module_cap;
693
694
  /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */
695
  char *log_buf;
696
  size_t log_size;
697
  __u32 log_level;
698
699
  int *fd_array;
700
  size_t fd_array_cap;
701
  size_t fd_array_cnt;
702
703
  struct usdt_manager *usdt_man;
704
705
  struct bpf_map *arena_map;
706
  void *arena_data;
707
  size_t arena_data_sz;
708
709
  struct kern_feature_cache *feat_cache;
710
  char *token_path;
711
  int token_fd;
712
713
  char path[];
714
};
715
716
static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
717
static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
718
static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
719
static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
720
static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);
721
static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
722
static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
723
static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);
724
static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx);
725
726
void bpf_program__unload(struct bpf_program *prog)
727
28.6k
{
728
28.6k
  if (!prog)
729
0
    return;
730
731
28.6k
  zclose(prog->fd);
732
733
28.6k
  zfree(&prog->func_info);
734
28.6k
  zfree(&prog->line_info);
735
28.6k
}
736
737
static void bpf_program__exit(struct bpf_program *prog)
738
14.3k
{
739
14.3k
  if (!prog)
740
0
    return;
741
742
14.3k
  bpf_program__unload(prog);
743
14.3k
  zfree(&prog->name);
744
14.3k
  zfree(&prog->sec_name);
745
14.3k
  zfree(&prog->insns);
746
14.3k
  zfree(&prog->reloc_desc);
747
748
14.3k
  prog->nr_reloc = 0;
749
14.3k
  prog->insns_cnt = 0;
750
14.3k
  prog->sec_idx = -1;
751
14.3k
}
752
753
static bool insn_is_subprog_call(const struct bpf_insn *insn)
754
0
{
755
0
  return BPF_CLASS(insn->code) == BPF_JMP &&
756
0
         BPF_OP(insn->code) == BPF_CALL &&
757
0
         BPF_SRC(insn->code) == BPF_K &&
758
0
         insn->src_reg == BPF_PSEUDO_CALL &&
759
0
         insn->dst_reg == 0 &&
760
0
         insn->off == 0;
761
0
}
762
763
static bool is_call_insn(const struct bpf_insn *insn)
764
3.52k
{
765
3.52k
  return insn->code == (BPF_JMP | BPF_CALL);
766
3.52k
}
767
768
static bool insn_is_pseudo_func(struct bpf_insn *insn)
769
0
{
770
0
  return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;
771
0
}
772
773
static int
774
bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
775
          const char *name, size_t sec_idx, const char *sec_name,
776
          size_t sec_off, void *insn_data, size_t insn_data_sz)
777
14.4k
{
778
14.4k
  if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) {
779
58
    pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n",
780
58
      sec_name, name, sec_off, insn_data_sz);
781
58
    return -EINVAL;
782
58
  }
783
784
14.3k
  memset(prog, 0, sizeof(*prog));
785
14.3k
  prog->obj = obj;
786
787
14.3k
  prog->sec_idx = sec_idx;
788
14.3k
  prog->sec_insn_off = sec_off / BPF_INSN_SZ;
789
14.3k
  prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
790
  /* insns_cnt can later be increased by appending used subprograms */
791
14.3k
  prog->insns_cnt = prog->sec_insn_cnt;
792
793
14.3k
  prog->type = BPF_PROG_TYPE_UNSPEC;
794
14.3k
  prog->fd = -1;
795
14.3k
  prog->exception_cb_idx = -1;
796
797
  /* libbpf's convention for SEC("?abc...") is that it's just like
798
   * SEC("abc...") but the corresponding bpf_program starts out with
799
   * autoload set to false.
800
   */
801
14.3k
  if (sec_name[0] == '?') {
802
214
    prog->autoload = false;
803
    /* from now on forget there was ? in section name */
804
214
    sec_name++;
805
14.1k
  } else {
806
14.1k
    prog->autoload = true;
807
14.1k
  }
808
809
14.3k
  prog->autoattach = true;
810
811
  /* inherit object's log_level */
812
14.3k
  prog->log_level = obj->log_level;
813
814
14.3k
  prog->sec_name = strdup(sec_name);
815
14.3k
  if (!prog->sec_name)
816
0
    goto errout;
817
818
14.3k
  prog->name = strdup(name);
819
14.3k
  if (!prog->name)
820
0
    goto errout;
821
822
14.3k
  prog->insns = malloc(insn_data_sz);
823
14.3k
  if (!prog->insns)
824
21
    goto errout;
825
14.3k
  memcpy(prog->insns, insn_data, insn_data_sz);
826
827
14.3k
  return 0;
828
21
errout:
829
21
  pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
830
21
  bpf_program__exit(prog);
831
21
  return -ENOMEM;
832
14.3k
}
833
834
static int
835
bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
836
       const char *sec_name, int sec_idx)
837
1.12k
{
838
1.12k
  Elf_Data *symbols = obj->efile.symbols;
839
1.12k
  struct bpf_program *prog, *progs;
840
1.12k
  void *data = sec_data->d_buf;
841
1.12k
  size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;
842
1.12k
  int nr_progs, err, i;
843
1.12k
  const char *name;
844
1.12k
  Elf64_Sym *sym;
845
846
1.12k
  progs = obj->programs;
847
1.12k
  nr_progs = obj->nr_programs;
848
1.12k
  nr_syms = symbols->d_size / sizeof(Elf64_Sym);
849
850
124k
  for (i = 0; i < nr_syms; i++) {
851
123k
    sym = elf_sym_by_idx(obj, i);
852
853
123k
    if (sym->st_shndx != sec_idx)
854
107k
      continue;
855
16.1k
    if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)
856
1.54k
      continue;
857
858
14.5k
    prog_sz = sym->st_size;
859
14.5k
    sec_off = sym->st_value;
860
861
14.5k
    name = elf_sym_str(obj, sym->st_name);
862
14.5k
    if (!name) {
863
59
      pr_warn("sec '%s': failed to get symbol name for offset %zu\n",
864
59
        sec_name, sec_off);
865
59
      return -LIBBPF_ERRNO__FORMAT;
866
59
    }
867
868
14.5k
    if (sec_off + prog_sz > sec_sz) {
869
106
      pr_warn("sec '%s': program at offset %zu crosses section boundary\n",
870
106
        sec_name, sec_off);
871
106
      return -LIBBPF_ERRNO__FORMAT;
872
106
    }
873
874
14.4k
    if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
875
3
      pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name);
876
3
      return -ENOTSUP;
877
3
    }
878
879
14.4k
    pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n",
880
28.8k
       sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz);
881
882
14.4k
    progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs));
883
14.4k
    if (!progs) {
884
      /*
885
       * In this case the original obj->programs
886
       * is still valid, so don't need special treat for
887
       * bpf_close_object().
888
       */
889
0
      pr_warn("sec '%s': failed to alloc memory for new program '%s'\n",
890
0
        sec_name, name);
891
0
      return -ENOMEM;
892
0
    }
893
14.4k
    obj->programs = progs;
894
895
14.4k
    prog = &progs[nr_progs];
896
897
14.4k
    err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
898
14.4k
              sec_off, data + sec_off, prog_sz);
899
14.4k
    if (err)
900
79
      return err;
901
902
14.3k
    if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL)
903
14.1k
      prog->sym_global = true;
904
905
    /* if function is a global/weak symbol, but has restricted
906
     * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC
907
     * as static to enable more permissive BPF verification mode
908
     * with more outside context available to BPF verifier
909
     */
910
14.3k
    if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
911
14.1k
        || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))
912
10.8k
      prog->mark_btf_static = true;
913
914
14.3k
    nr_progs++;
915
14.3k
    obj->nr_programs = nr_progs;
916
14.3k
  }
917
918
881
  return 0;
919
1.12k
}
920
921
static const struct btf_member *
922
find_member_by_offset(const struct btf_type *t, __u32 bit_offset)
923
0
{
924
0
  struct btf_member *m;
925
0
  int i;
926
927
0
  for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
928
0
    if (btf_member_bit_offset(t, i) == bit_offset)
929
0
      return m;
930
0
  }
931
932
0
  return NULL;
933
0
}
934
935
static const struct btf_member *
936
find_member_by_name(const struct btf *btf, const struct btf_type *t,
937
        const char *name)
938
0
{
939
0
  struct btf_member *m;
940
0
  int i;
941
942
0
  for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) {
943
0
    if (!strcmp(btf__name_by_offset(btf, m->name_off), name))
944
0
      return m;
945
0
  }
946
947
0
  return NULL;
948
0
}
949
950
static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
951
          __u16 kind, struct btf **res_btf,
952
          struct module_btf **res_mod_btf);
953
954
0
#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_"
955
static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix,
956
           const char *name, __u32 kind);
957
958
static int
959
find_struct_ops_kern_types(struct bpf_object *obj, const char *tname_raw,
960
         struct module_btf **mod_btf,
961
         const struct btf_type **type, __u32 *type_id,
962
         const struct btf_type **vtype, __u32 *vtype_id,
963
         const struct btf_member **data_member)
964
0
{
965
0
  const struct btf_type *kern_type, *kern_vtype;
966
0
  const struct btf_member *kern_data_member;
967
0
  struct btf *btf;
968
0
  __s32 kern_vtype_id, kern_type_id;
969
0
  char tname[256];
970
0
  __u32 i;
971
972
0
  snprintf(tname, sizeof(tname), "%.*s",
973
0
     (int)bpf_core_essential_name_len(tname_raw), tname_raw);
974
975
0
  kern_type_id = find_ksym_btf_id(obj, tname, BTF_KIND_STRUCT,
976
0
          &btf, mod_btf);
977
0
  if (kern_type_id < 0) {
978
0
    pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n",
979
0
      tname);
980
0
    return kern_type_id;
981
0
  }
982
0
  kern_type = btf__type_by_id(btf, kern_type_id);
983
984
  /* Find the corresponding "map_value" type that will be used
985
   * in map_update(BPF_MAP_TYPE_STRUCT_OPS).  For example,
986
   * find "struct bpf_struct_ops_tcp_congestion_ops" from the
987
   * btf_vmlinux.
988
   */
989
0
  kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX,
990
0
            tname, BTF_KIND_STRUCT);
991
0
  if (kern_vtype_id < 0) {
992
0
    pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n",
993
0
      STRUCT_OPS_VALUE_PREFIX, tname);
994
0
    return kern_vtype_id;
995
0
  }
996
0
  kern_vtype = btf__type_by_id(btf, kern_vtype_id);
997
998
  /* Find "struct tcp_congestion_ops" from
999
   * struct bpf_struct_ops_tcp_congestion_ops {
1000
   *  [ ... ]
1001
   *  struct tcp_congestion_ops data;
1002
   * }
1003
   */
1004
0
  kern_data_member = btf_members(kern_vtype);
1005
0
  for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) {
1006
0
    if (kern_data_member->type == kern_type_id)
1007
0
      break;
1008
0
  }
1009
0
  if (i == btf_vlen(kern_vtype)) {
1010
0
    pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n",
1011
0
      tname, STRUCT_OPS_VALUE_PREFIX, tname);
1012
0
    return -EINVAL;
1013
0
  }
1014
1015
0
  *type = kern_type;
1016
0
  *type_id = kern_type_id;
1017
0
  *vtype = kern_vtype;
1018
0
  *vtype_id = kern_vtype_id;
1019
0
  *data_member = kern_data_member;
1020
1021
0
  return 0;
1022
0
}
1023
1024
static bool bpf_map__is_struct_ops(const struct bpf_map *map)
1025
282
{
1026
282
  return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;
1027
282
}
1028
1029
static bool is_valid_st_ops_program(struct bpf_object *obj,
1030
            const struct bpf_program *prog)
1031
0
{
1032
0
  int i;
1033
1034
0
  for (i = 0; i < obj->nr_programs; i++) {
1035
0
    if (&obj->programs[i] == prog)
1036
0
      return prog->type == BPF_PROG_TYPE_STRUCT_OPS;
1037
0
  }
1038
1039
0
  return false;
1040
0
}
1041
1042
/* For each struct_ops program P, referenced from some struct_ops map M,
1043
 * enable P.autoload if there are Ms for which M.autocreate is true,
1044
 * disable P.autoload if for all Ms M.autocreate is false.
1045
 * Don't change P.autoload for programs that are not referenced from any maps.
1046
 */
1047
static int bpf_object_adjust_struct_ops_autoload(struct bpf_object *obj)
1048
0
{
1049
0
  struct bpf_program *prog, *slot_prog;
1050
0
  struct bpf_map *map;
1051
0
  int i, j, k, vlen;
1052
1053
0
  for (i = 0; i < obj->nr_programs; ++i) {
1054
0
    int should_load = false;
1055
0
    int use_cnt = 0;
1056
1057
0
    prog = &obj->programs[i];
1058
0
    if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
1059
0
      continue;
1060
1061
0
    for (j = 0; j < obj->nr_maps; ++j) {
1062
0
      map = &obj->maps[j];
1063
0
      if (!bpf_map__is_struct_ops(map))
1064
0
        continue;
1065
1066
0
      vlen = btf_vlen(map->st_ops->type);
1067
0
      for (k = 0; k < vlen; ++k) {
1068
0
        slot_prog = map->st_ops->progs[k];
1069
0
        if (prog != slot_prog)
1070
0
          continue;
1071
1072
0
        use_cnt++;
1073
0
        if (map->autocreate)
1074
0
          should_load = true;
1075
0
      }
1076
0
    }
1077
0
    if (use_cnt)
1078
0
      prog->autoload = should_load;
1079
0
  }
1080
1081
0
  return 0;
1082
0
}
1083
1084
/* Init the map's fields that depend on kern_btf */
1085
static int bpf_map__init_kern_struct_ops(struct bpf_map *map)
1086
0
{
1087
0
  const struct btf_member *member, *kern_member, *kern_data_member;
1088
0
  const struct btf_type *type, *kern_type, *kern_vtype;
1089
0
  __u32 i, kern_type_id, kern_vtype_id, kern_data_off;
1090
0
  struct bpf_object *obj = map->obj;
1091
0
  const struct btf *btf = obj->btf;
1092
0
  struct bpf_struct_ops *st_ops;
1093
0
  const struct btf *kern_btf;
1094
0
  struct module_btf *mod_btf;
1095
0
  void *data, *kern_data;
1096
0
  const char *tname;
1097
0
  int err;
1098
1099
0
  st_ops = map->st_ops;
1100
0
  type = st_ops->type;
1101
0
  tname = st_ops->tname;
1102
0
  err = find_struct_ops_kern_types(obj, tname, &mod_btf,
1103
0
           &kern_type, &kern_type_id,
1104
0
           &kern_vtype, &kern_vtype_id,
1105
0
           &kern_data_member);
1106
0
  if (err)
1107
0
    return err;
1108
1109
0
  kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux;
1110
1111
0
  pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n",
1112
0
     map->name, st_ops->type_id, kern_type_id, kern_vtype_id);
1113
1114
0
  map->mod_btf_fd = mod_btf ? mod_btf->fd : -1;
1115
0
  map->def.value_size = kern_vtype->size;
1116
0
  map->btf_vmlinux_value_type_id = kern_vtype_id;
1117
1118
0
  st_ops->kern_vdata = calloc(1, kern_vtype->size);
1119
0
  if (!st_ops->kern_vdata)
1120
0
    return -ENOMEM;
1121
1122
0
  data = st_ops->data;
1123
0
  kern_data_off = kern_data_member->offset / 8;
1124
0
  kern_data = st_ops->kern_vdata + kern_data_off;
1125
1126
0
  member = btf_members(type);
1127
0
  for (i = 0; i < btf_vlen(type); i++, member++) {
1128
0
    const struct btf_type *mtype, *kern_mtype;
1129
0
    __u32 mtype_id, kern_mtype_id;
1130
0
    void *mdata, *kern_mdata;
1131
0
    struct bpf_program *prog;
1132
0
    __s64 msize, kern_msize;
1133
0
    __u32 moff, kern_moff;
1134
0
    __u32 kern_member_idx;
1135
0
    const char *mname;
1136
1137
0
    mname = btf__name_by_offset(btf, member->name_off);
1138
0
    moff = member->offset / 8;
1139
0
    mdata = data + moff;
1140
0
    msize = btf__resolve_size(btf, member->type);
1141
0
    if (msize < 0) {
1142
0
      pr_warn("struct_ops init_kern %s: failed to resolve the size of member %s\n",
1143
0
        map->name, mname);
1144
0
      return msize;
1145
0
    }
1146
1147
0
    kern_member = find_member_by_name(kern_btf, kern_type, mname);
1148
0
    if (!kern_member) {
1149
0
      if (!libbpf_is_mem_zeroed(mdata, msize)) {
1150
0
        pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n",
1151
0
          map->name, mname);
1152
0
        return -ENOTSUP;
1153
0
      }
1154
1155
0
      if (st_ops->progs[i]) {
1156
        /* If we had declaratively set struct_ops callback, we need to
1157
         * force its autoload to false, because it doesn't have
1158
         * a chance of succeeding from POV of the current struct_ops map.
1159
         * If this program is still referenced somewhere else, though,
1160
         * then bpf_object_adjust_struct_ops_autoload() will update its
1161
         * autoload accordingly.
1162
         */
1163
0
        st_ops->progs[i]->autoload = false;
1164
0
        st_ops->progs[i] = NULL;
1165
0
      }
1166
1167
      /* Skip all-zero/NULL fields if they are not present in the kernel BTF */
1168
0
      pr_info("struct_ops %s: member %s not found in kernel, skipping it as it's set to zero\n",
1169
0
        map->name, mname);
1170
0
      continue;
1171
0
    }
1172
1173
0
    kern_member_idx = kern_member - btf_members(kern_type);
1174
0
    if (btf_member_bitfield_size(type, i) ||
1175
0
        btf_member_bitfield_size(kern_type, kern_member_idx)) {
1176
0
      pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n",
1177
0
        map->name, mname);
1178
0
      return -ENOTSUP;
1179
0
    }
1180
1181
0
    kern_moff = kern_member->offset / 8;
1182
0
    kern_mdata = kern_data + kern_moff;
1183
1184
0
    mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);
1185
0
    kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,
1186
0
                &kern_mtype_id);
1187
0
    if (BTF_INFO_KIND(mtype->info) !=
1188
0
        BTF_INFO_KIND(kern_mtype->info)) {
1189
0
      pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n",
1190
0
        map->name, mname, BTF_INFO_KIND(mtype->info),
1191
0
        BTF_INFO_KIND(kern_mtype->info));
1192
0
      return -ENOTSUP;
1193
0
    }
1194
1195
0
    if (btf_is_ptr(mtype)) {
1196
0
      prog = *(void **)mdata;
1197
      /* just like for !kern_member case above, reset declaratively
1198
       * set (at compile time) program's autload to false,
1199
       * if user replaced it with another program or NULL
1200
       */
1201
0
      if (st_ops->progs[i] && st_ops->progs[i] != prog)
1202
0
        st_ops->progs[i]->autoload = false;
1203
1204
      /* Update the value from the shadow type */
1205
0
      st_ops->progs[i] = prog;
1206
0
      if (!prog)
1207
0
        continue;
1208
1209
0
      if (!is_valid_st_ops_program(obj, prog)) {
1210
0
        pr_warn("struct_ops init_kern %s: member %s is not a struct_ops program\n",
1211
0
          map->name, mname);
1212
0
        return -ENOTSUP;
1213
0
      }
1214
1215
0
      kern_mtype = skip_mods_and_typedefs(kern_btf,
1216
0
                  kern_mtype->type,
1217
0
                  &kern_mtype_id);
1218
1219
      /* mtype->type must be a func_proto which was
1220
       * guaranteed in bpf_object__collect_st_ops_relos(),
1221
       * so only check kern_mtype for func_proto here.
1222
       */
1223
0
      if (!btf_is_func_proto(kern_mtype)) {
1224
0
        pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n",
1225
0
          map->name, mname);
1226
0
        return -ENOTSUP;
1227
0
      }
1228
1229
0
      if (mod_btf)
1230
0
        prog->attach_btf_obj_fd = mod_btf->fd;
1231
1232
      /* if we haven't yet processed this BPF program, record proper
1233
       * attach_btf_id and member_idx
1234
       */
1235
0
      if (!prog->attach_btf_id) {
1236
0
        prog->attach_btf_id = kern_type_id;
1237
0
        prog->expected_attach_type = kern_member_idx;
1238
0
      }
1239
1240
      /* struct_ops BPF prog can be re-used between multiple
1241
       * .struct_ops & .struct_ops.link as long as it's the
1242
       * same struct_ops struct definition and the same
1243
       * function pointer field
1244
       */
1245
0
      if (prog->attach_btf_id != kern_type_id) {
1246
0
        pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: attach_btf_id %u != kern_type_id %u\n",
1247
0
          map->name, mname, prog->name, prog->sec_name, prog->type,
1248
0
          prog->attach_btf_id, kern_type_id);
1249
0
        return -EINVAL;
1250
0
      }
1251
0
      if (prog->expected_attach_type != kern_member_idx) {
1252
0
        pr_warn("struct_ops init_kern %s func ptr %s: invalid reuse of prog %s in sec %s with type %u: expected_attach_type %u != kern_member_idx %u\n",
1253
0
          map->name, mname, prog->name, prog->sec_name, prog->type,
1254
0
          prog->expected_attach_type, kern_member_idx);
1255
0
        return -EINVAL;
1256
0
      }
1257
1258
0
      st_ops->kern_func_off[i] = kern_data_off + kern_moff;
1259
1260
0
      pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1261
0
         map->name, mname, prog->name, moff,
1262
0
         kern_moff);
1263
1264
0
      continue;
1265
0
    }
1266
1267
0
    kern_msize = btf__resolve_size(kern_btf, kern_mtype_id);
1268
0
    if (kern_msize < 0 || msize != kern_msize) {
1269
0
      pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n",
1270
0
        map->name, mname, (ssize_t)msize,
1271
0
        (ssize_t)kern_msize);
1272
0
      return -ENOTSUP;
1273
0
    }
1274
1275
0
    pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n",
1276
0
       map->name, mname, (unsigned int)msize,
1277
0
       moff, kern_moff);
1278
0
    memcpy(kern_mdata, mdata, msize);
1279
0
  }
1280
1281
0
  return 0;
1282
0
}
1283
1284
static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
1285
0
{
1286
0
  struct bpf_map *map;
1287
0
  size_t i;
1288
0
  int err;
1289
1290
0
  for (i = 0; i < obj->nr_maps; i++) {
1291
0
    map = &obj->maps[i];
1292
1293
0
    if (!bpf_map__is_struct_ops(map))
1294
0
      continue;
1295
1296
0
    if (!map->autocreate)
1297
0
      continue;
1298
1299
0
    err = bpf_map__init_kern_struct_ops(map);
1300
0
    if (err)
1301
0
      return err;
1302
0
  }
1303
1304
0
  return 0;
1305
0
}
1306
1307
static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name,
1308
        int shndx, Elf_Data *data)
1309
175
{
1310
175
  const struct btf_type *type, *datasec;
1311
175
  const struct btf_var_secinfo *vsi;
1312
175
  struct bpf_struct_ops *st_ops;
1313
175
  const char *tname, *var_name;
1314
175
  __s32 type_id, datasec_id;
1315
175
  const struct btf *btf;
1316
175
  struct bpf_map *map;
1317
175
  __u32 i;
1318
1319
175
  if (shndx == -1)
1320
0
    return 0;
1321
1322
175
  btf = obj->btf;
1323
175
  datasec_id = btf__find_by_name_kind(btf, sec_name,
1324
175
              BTF_KIND_DATASEC);
1325
175
  if (datasec_id < 0) {
1326
81
    pr_warn("struct_ops init: DATASEC %s not found\n",
1327
81
      sec_name);
1328
81
    return -EINVAL;
1329
81
  }
1330
1331
94
  datasec = btf__type_by_id(btf, datasec_id);
1332
94
  vsi = btf_var_secinfos(datasec);
1333
122
  for (i = 0; i < btf_vlen(datasec); i++, vsi++) {
1334
70
    type = btf__type_by_id(obj->btf, vsi->type);
1335
70
    var_name = btf__name_by_offset(obj->btf, type->name_off);
1336
1337
70
    type_id = btf__resolve_type(obj->btf, vsi->type);
1338
70
    if (type_id < 0) {
1339
11
      pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n",
1340
11
        vsi->type, sec_name);
1341
11
      return -EINVAL;
1342
11
    }
1343
1344
59
    type = btf__type_by_id(obj->btf, type_id);
1345
59
    tname = btf__name_by_offset(obj->btf, type->name_off);
1346
59
    if (!tname[0]) {
1347
1
      pr_warn("struct_ops init: anonymous type is not supported\n");
1348
1
      return -ENOTSUP;
1349
1
    }
1350
58
    if (!btf_is_struct(type)) {
1351
10
      pr_warn("struct_ops init: %s is not a struct\n", tname);
1352
10
      return -EINVAL;
1353
10
    }
1354
1355
48
    map = bpf_object__add_map(obj);
1356
48
    if (IS_ERR(map))
1357
0
      return PTR_ERR(map);
1358
1359
48
    map->sec_idx = shndx;
1360
48
    map->sec_offset = vsi->offset;
1361
48
    map->name = strdup(var_name);
1362
48
    if (!map->name)
1363
0
      return -ENOMEM;
1364
48
    map->btf_value_type_id = type_id;
1365
1366
    /* Follow same convention as for programs autoload:
1367
     * SEC("?.struct_ops") means map is not created by default.
1368
     */
1369
48
    if (sec_name[0] == '?') {
1370
1
      map->autocreate = false;
1371
      /* from now on forget there was ? in section name */
1372
1
      sec_name++;
1373
1
    }
1374
1375
48
    map->def.type = BPF_MAP_TYPE_STRUCT_OPS;
1376
48
    map->def.key_size = sizeof(int);
1377
48
    map->def.value_size = type->size;
1378
48
    map->def.max_entries = 1;
1379
48
    map->def.map_flags = strcmp(sec_name, STRUCT_OPS_LINK_SEC) == 0 ? BPF_F_LINK : 0;
1380
1381
48
    map->st_ops = calloc(1, sizeof(*map->st_ops));
1382
48
    if (!map->st_ops)
1383
0
      return -ENOMEM;
1384
48
    st_ops = map->st_ops;
1385
48
    st_ops->data = malloc(type->size);
1386
48
    st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));
1387
48
    st_ops->kern_func_off = malloc(btf_vlen(type) *
1388
48
                 sizeof(*st_ops->kern_func_off));
1389
48
    if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)
1390
0
      return -ENOMEM;
1391
1392
48
    if (vsi->offset + type->size > data->d_size) {
1393
20
      pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n",
1394
20
        var_name, sec_name);
1395
20
      return -EINVAL;
1396
20
    }
1397
1398
28
    memcpy(st_ops->data,
1399
28
           data->d_buf + vsi->offset,
1400
28
           type->size);
1401
28
    st_ops->tname = tname;
1402
28
    st_ops->type = type;
1403
28
    st_ops->type_id = type_id;
1404
1405
28
    pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n",
1406
28
       tname, type_id, var_name, vsi->offset);
1407
28
  }
1408
1409
52
  return 0;
1410
94
}
1411
1412
static int bpf_object_init_struct_ops(struct bpf_object *obj)
1413
2.06k
{
1414
2.06k
  const char *sec_name;
1415
2.06k
  int sec_idx, err;
1416
1417
17.1k
  for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) {
1418
15.1k
    struct elf_sec_desc *desc = &obj->efile.secs[sec_idx];
1419
1420
15.1k
    if (desc->sec_type != SEC_ST_OPS)
1421
14.9k
      continue;
1422
1423
175
    sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1424
175
    if (!sec_name)
1425
0
      return -LIBBPF_ERRNO__FORMAT;
1426
1427
175
    err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data);
1428
175
    if (err)
1429
123
      return err;
1430
175
  }
1431
1432
1.94k
  return 0;
1433
2.06k
}
1434
1435
static struct bpf_object *bpf_object__new(const char *path,
1436
            const void *obj_buf,
1437
            size_t obj_buf_sz,
1438
            const char *obj_name)
1439
11.0k
{
1440
11.0k
  struct bpf_object *obj;
1441
11.0k
  char *end;
1442
1443
11.0k
  obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1444
11.0k
  if (!obj) {
1445
0
    pr_warn("alloc memory failed for %s\n", path);
1446
0
    return ERR_PTR(-ENOMEM);
1447
0
  }
1448
1449
11.0k
  strcpy(obj->path, path);
1450
11.0k
  if (obj_name) {
1451
11.0k
    libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1452
11.0k
  } else {
1453
    /* Using basename() GNU version which doesn't modify arg. */
1454
0
    libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1455
0
    end = strchr(obj->name, '.');
1456
0
    if (end)
1457
0
      *end = 0;
1458
0
  }
1459
1460
11.0k
  obj->efile.fd = -1;
1461
  /*
1462
   * Caller of this function should also call
1463
   * bpf_object__elf_finish() after data collection to return
1464
   * obj_buf to user. If not, we should duplicate the buffer to
1465
   * avoid user freeing them before elf finish.
1466
   */
1467
11.0k
  obj->efile.obj_buf = obj_buf;
1468
11.0k
  obj->efile.obj_buf_sz = obj_buf_sz;
1469
11.0k
  obj->efile.btf_maps_shndx = -1;
1470
11.0k
  obj->kconfig_map_idx = -1;
1471
1472
11.0k
  obj->kern_version = get_kernel_version();
1473
11.0k
  obj->loaded = false;
1474
1475
11.0k
  return obj;
1476
11.0k
}
1477
1478
static void bpf_object__elf_finish(struct bpf_object *obj)
1479
14.8k
{
1480
14.8k
  if (!obj->efile.elf)
1481
3.85k
    return;
1482
1483
10.9k
  elf_end(obj->efile.elf);
1484
10.9k
  obj->efile.elf = NULL;
1485
10.9k
  obj->efile.symbols = NULL;
1486
10.9k
  obj->efile.arena_data = NULL;
1487
1488
10.9k
  zfree(&obj->efile.secs);
1489
10.9k
  obj->efile.sec_cnt = 0;
1490
10.9k
  zclose(obj->efile.fd);
1491
10.9k
  obj->efile.obj_buf = NULL;
1492
10.9k
  obj->efile.obj_buf_sz = 0;
1493
10.9k
}
1494
1495
static int bpf_object__elf_init(struct bpf_object *obj)
1496
11.0k
{
1497
11.0k
  Elf64_Ehdr *ehdr;
1498
11.0k
  int err = 0;
1499
11.0k
  Elf *elf;
1500
1501
11.0k
  if (obj->efile.elf) {
1502
0
    pr_warn("elf: init internal error\n");
1503
0
    return -LIBBPF_ERRNO__LIBELF;
1504
0
  }
1505
1506
11.0k
  if (obj->efile.obj_buf_sz > 0) {
1507
    /* obj_buf should have been validated by bpf_object__open_mem(). */
1508
11.0k
    elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1509
11.0k
  } else {
1510
0
    obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1511
0
    if (obj->efile.fd < 0) {
1512
0
      char errmsg[STRERR_BUFSIZE], *cp;
1513
1514
0
      err = -errno;
1515
0
      cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
1516
0
      pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1517
0
      return err;
1518
0
    }
1519
1520
0
    elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1521
0
  }
1522
1523
11.0k
  if (!elf) {
1524
81
    pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1525
81
    err = -LIBBPF_ERRNO__LIBELF;
1526
81
    goto errout;
1527
81
  }
1528
1529
10.9k
  obj->efile.elf = elf;
1530
1531
10.9k
  if (elf_kind(elf) != ELF_K_ELF) {
1532
115
    err = -LIBBPF_ERRNO__FORMAT;
1533
115
    pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1534
115
    goto errout;
1535
115
  }
1536
1537
10.8k
  if (gelf_getclass(elf) != ELFCLASS64) {
1538
476
    err = -LIBBPF_ERRNO__FORMAT;
1539
476
    pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1540
476
    goto errout;
1541
476
  }
1542
1543
10.3k
  obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1544
10.3k
  if (!obj->efile.ehdr) {
1545
0
    pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1546
0
    err = -LIBBPF_ERRNO__FORMAT;
1547
0
    goto errout;
1548
0
  }
1549
1550
10.3k
  if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1551
42
    pr_warn("elf: failed to get section names section index for %s: %s\n",
1552
42
      obj->path, elf_errmsg(-1));
1553
42
    err = -LIBBPF_ERRNO__FORMAT;
1554
42
    goto errout;
1555
42
  }
1556
1557
  /* ELF is corrupted/truncated, avoid calling elf_strptr. */
1558
10.3k
  if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1559
1.09k
    pr_warn("elf: failed to get section names strings from %s: %s\n",
1560
1.09k
      obj->path, elf_errmsg(-1));
1561
1.09k
    err = -LIBBPF_ERRNO__FORMAT;
1562
1.09k
    goto errout;
1563
1.09k
  }
1564
1565
  /* Old LLVM set e_machine to EM_NONE */
1566
9.26k
  if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {
1567
440
    pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1568
440
    err = -LIBBPF_ERRNO__FORMAT;
1569
440
    goto errout;
1570
440
  }
1571
1572
8.82k
  return 0;
1573
2.24k
errout:
1574
2.24k
  bpf_object__elf_finish(obj);
1575
2.24k
  return err;
1576
9.26k
}
1577
1578
static int bpf_object__check_endianness(struct bpf_object *obj)
1579
8.82k
{
1580
8.82k
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1581
8.82k
  if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1582
8.79k
    return 0;
1583
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1584
  if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1585
    return 0;
1586
#else
1587
# error "Unrecognized __BYTE_ORDER__"
1588
#endif
1589
24
  pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1590
24
  return -LIBBPF_ERRNO__ENDIAN;
1591
8.82k
}
1592
1593
static int
1594
bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1595
611
{
1596
611
  if (!data) {
1597
2
    pr_warn("invalid license section in %s\n", obj->path);
1598
2
    return -LIBBPF_ERRNO__FORMAT;
1599
2
  }
1600
  /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't
1601
   * go over allowed ELF data section buffer
1602
   */
1603
609
  libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1604
609
  pr_debug("license of %s is %s\n", obj->path, obj->license);
1605
609
  return 0;
1606
611
}
1607
1608
static int
1609
bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1610
87
{
1611
87
  __u32 kver;
1612
1613
87
  if (!data || size != sizeof(kver)) {
1614
17
    pr_warn("invalid kver section in %s\n", obj->path);
1615
17
    return -LIBBPF_ERRNO__FORMAT;
1616
17
  }
1617
70
  memcpy(&kver, data, sizeof(kver));
1618
70
  obj->kern_version = kver;
1619
70
  pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1620
70
  return 0;
1621
87
}
1622
1623
static bool bpf_map_type__is_map_in_map(enum bpf_map_type type)
1624
133
{
1625
133
  if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
1626
133
      type == BPF_MAP_TYPE_HASH_OF_MAPS)
1627
47
    return true;
1628
86
  return false;
1629
133
}
1630
1631
static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size)
1632
401
{
1633
401
  Elf_Data *data;
1634
401
  Elf_Scn *scn;
1635
1636
401
  if (!name)
1637
0
    return -EINVAL;
1638
1639
401
  scn = elf_sec_by_name(obj, name);
1640
401
  data = elf_sec_data(obj, scn);
1641
401
  if (data) {
1642
260
    *size = data->d_size;
1643
260
    return 0; /* found it */
1644
260
  }
1645
1646
141
  return -ENOENT;
1647
401
}
1648
1649
static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name)
1650
1.75k
{
1651
1.75k
  Elf_Data *symbols = obj->efile.symbols;
1652
1.75k
  const char *sname;
1653
1.75k
  size_t si;
1654
1655
76.8k
  for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {
1656
76.5k
    Elf64_Sym *sym = elf_sym_by_idx(obj, si);
1657
1658
76.5k
    if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)
1659
69.1k
      continue;
1660
1661
7.45k
    if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&
1662
7.45k
        ELF64_ST_BIND(sym->st_info) != STB_WEAK)
1663
4.94k
      continue;
1664
1665
2.50k
    sname = elf_sym_str(obj, sym->st_name);
1666
2.50k
    if (!sname) {
1667
33
      pr_warn("failed to get sym name string for var %s\n", name);
1668
33
      return ERR_PTR(-EIO);
1669
33
    }
1670
2.47k
    if (strcmp(name, sname) == 0)
1671
1.49k
      return sym;
1672
2.47k
  }
1673
1674
224
  return ERR_PTR(-ENOENT);
1675
1.75k
}
1676
1677
/* Some versions of Android don't provide memfd_create() in their libc
1678
 * implementation, so avoid complications and just go straight to Linux
1679
 * syscall.
1680
 */
1681
static int sys_memfd_create(const char *name, unsigned flags)
1682
2.75k
{
1683
2.75k
  return syscall(__NR_memfd_create, name, flags);
1684
2.75k
}
1685
1686
#ifndef MFD_CLOEXEC
1687
#define MFD_CLOEXEC 0x0001U
1688
#endif
1689
1690
static int create_placeholder_fd(void)
1691
2.75k
{
1692
2.75k
  int fd;
1693
1694
2.75k
  fd = ensure_good_fd(sys_memfd_create("libbpf-placeholder-fd", MFD_CLOEXEC));
1695
2.75k
  if (fd < 0)
1696
0
    return -errno;
1697
2.75k
  return fd;
1698
2.75k
}
1699
1700
static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1701
2.75k
{
1702
2.75k
  struct bpf_map *map;
1703
2.75k
  int err;
1704
1705
2.75k
  err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1706
2.75k
        sizeof(*obj->maps), obj->nr_maps + 1);
1707
2.75k
  if (err)
1708
0
    return ERR_PTR(err);
1709
1710
2.75k
  map = &obj->maps[obj->nr_maps++];
1711
2.75k
  map->obj = obj;
1712
  /* Preallocate map FD without actually creating BPF map just yet.
1713
   * These map FD "placeholders" will be reused later without changing
1714
   * FD value when map is actually created in the kernel.
1715
   *
1716
   * This is useful to be able to perform BPF program relocations
1717
   * without having to create BPF maps before that step. This allows us
1718
   * to finalize and load BTF very late in BPF object's loading phase,
1719
   * right before BPF maps have to be created and BPF programs have to
1720
   * be loaded. By having these map FD placeholders we can perform all
1721
   * the sanitizations, relocations, and any other adjustments before we
1722
   * start creating actual BPF kernel objects (BTF, maps, progs).
1723
   */
1724
2.75k
  map->fd = create_placeholder_fd();
1725
2.75k
  if (map->fd < 0)
1726
0
    return ERR_PTR(map->fd);
1727
2.75k
  map->inner_map_fd = -1;
1728
2.75k
  map->autocreate = true;
1729
1730
2.75k
  return map;
1731
2.75k
}
1732
1733
static size_t array_map_mmap_sz(unsigned int value_sz, unsigned int max_entries)
1734
3.43k
{
1735
3.43k
  const long page_sz = sysconf(_SC_PAGE_SIZE);
1736
3.43k
  size_t map_sz;
1737
1738
3.43k
  map_sz = (size_t)roundup(value_sz, 8) * max_entries;
1739
3.43k
  map_sz = roundup(map_sz, page_sz);
1740
3.43k
  return map_sz;
1741
3.43k
}
1742
1743
static size_t bpf_map_mmap_sz(const struct bpf_map *map)
1744
3.43k
{
1745
3.43k
  const long page_sz = sysconf(_SC_PAGE_SIZE);
1746
1747
3.43k
  switch (map->def.type) {
1748
3.43k
  case BPF_MAP_TYPE_ARRAY:
1749
3.43k
    return array_map_mmap_sz(map->def.value_size, map->def.max_entries);
1750
0
  case BPF_MAP_TYPE_ARENA:
1751
0
    return page_sz * map->def.max_entries;
1752
0
  default:
1753
0
    return 0; /* not supported */
1754
3.43k
  }
1755
3.43k
}
1756
1757
static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz)
1758
0
{
1759
0
  void *mmaped;
1760
1761
0
  if (!map->mmaped)
1762
0
    return -EINVAL;
1763
1764
0
  if (old_sz == new_sz)
1765
0
    return 0;
1766
1767
0
  mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1768
0
  if (mmaped == MAP_FAILED)
1769
0
    return -errno;
1770
1771
0
  memcpy(mmaped, map->mmaped, min(old_sz, new_sz));
1772
0
  munmap(map->mmaped, old_sz);
1773
0
  map->mmaped = mmaped;
1774
0
  return 0;
1775
0
}
1776
1777
static char *internal_map_name(struct bpf_object *obj, const char *real_name)
1778
1.76k
{
1779
1.76k
  char map_name[BPF_OBJ_NAME_LEN], *p;
1780
1.76k
  int pfx_len, sfx_len = max((size_t)7, strlen(real_name));
1781
1782
  /* This is one of the more confusing parts of libbpf for various
1783
   * reasons, some of which are historical. The original idea for naming
1784
   * internal names was to include as much of BPF object name prefix as
1785
   * possible, so that it can be distinguished from similar internal
1786
   * maps of a different BPF object.
1787
   * As an example, let's say we have bpf_object named 'my_object_name'
1788
   * and internal map corresponding to '.rodata' ELF section. The final
1789
   * map name advertised to user and to the kernel will be
1790
   * 'my_objec.rodata', taking first 8 characters of object name and
1791
   * entire 7 characters of '.rodata'.
1792
   * Somewhat confusingly, if internal map ELF section name is shorter
1793
   * than 7 characters, e.g., '.bss', we still reserve 7 characters
1794
   * for the suffix, even though we only have 4 actual characters, and
1795
   * resulting map will be called 'my_objec.bss', not even using all 15
1796
   * characters allowed by the kernel. Oh well, at least the truncated
1797
   * object name is somewhat consistent in this case. But if the map
1798
   * name is '.kconfig', we'll still have entirety of '.kconfig' added
1799
   * (8 chars) and thus will be left with only first 7 characters of the
1800
   * object name ('my_obje'). Happy guessing, user, that the final map
1801
   * name will be "my_obje.kconfig".
1802
   * Now, with libbpf starting to support arbitrarily named .rodata.*
1803
   * and .data.* data sections, it's possible that ELF section name is
1804
   * longer than allowed 15 chars, so we now need to be careful to take
1805
   * only up to 15 first characters of ELF name, taking no BPF object
1806
   * name characters at all. So '.rodata.abracadabra' will result in
1807
   * '.rodata.abracad' kernel and user-visible name.
1808
   * We need to keep this convoluted logic intact for .data, .bss and
1809
   * .rodata maps, but for new custom .data.custom and .rodata.custom
1810
   * maps we use their ELF names as is, not prepending bpf_object name
1811
   * in front. We still need to truncate them to 15 characters for the
1812
   * kernel. Full name can be recovered for such maps by using DATASEC
1813
   * BTF type associated with such map's value type, though.
1814
   */
1815
1.76k
  if (sfx_len >= BPF_OBJ_NAME_LEN)
1816
348
    sfx_len = BPF_OBJ_NAME_LEN - 1;
1817
1818
  /* if there are two or more dots in map name, it's a custom dot map */
1819
1.76k
  if (strchr(real_name + 1, '.') != NULL)
1820
1.31k
    pfx_len = 0;
1821
451
  else
1822
451
    pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1823
1824
1.76k
  snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1825
1.76k
     sfx_len, real_name);
1826
1827
  /* sanitise map name to characters allowed by kernel */
1828
23.4k
  for (p = map_name; *p && p < map_name + sizeof(map_name); p++)
1829
21.6k
    if (!isalnum(*p) && *p != '_' && *p != '.')
1830
3.45k
      *p = '_';
1831
1832
1.76k
  return strdup(map_name);
1833
1.76k
}
1834
1835
static int
1836
map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map);
1837
1838
/* Internal BPF map is mmap()'able only if at least one of corresponding
1839
 * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL
1840
 * variable and it's not marked as __hidden (which turns it into, effectively,
1841
 * a STATIC variable).
1842
 */
1843
static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map)
1844
1.76k
{
1845
1.76k
  const struct btf_type *t, *vt;
1846
1.76k
  struct btf_var_secinfo *vsi;
1847
1.76k
  int i, n;
1848
1849
1.76k
  if (!map->btf_value_type_id)
1850
1.60k
    return false;
1851
1852
158
  t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1853
158
  if (!btf_is_datasec(t))
1854
28
    return false;
1855
1856
130
  vsi = btf_var_secinfos(t);
1857
215
  for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) {
1858
176
    vt = btf__type_by_id(obj->btf, vsi->type);
1859
176
    if (!btf_is_var(vt))
1860
64
      continue;
1861
1862
112
    if (btf_var(vt)->linkage != BTF_VAR_STATIC)
1863
91
      return true;
1864
112
  }
1865
1866
39
  return false;
1867
130
}
1868
1869
static int
1870
bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1871
            const char *real_name, int sec_idx, void *data, size_t data_sz)
1872
1.76k
{
1873
1.76k
  struct bpf_map_def *def;
1874
1.76k
  struct bpf_map *map;
1875
1.76k
  size_t mmap_sz;
1876
1.76k
  int err;
1877
1878
1.76k
  map = bpf_object__add_map(obj);
1879
1.76k
  if (IS_ERR(map))
1880
0
    return PTR_ERR(map);
1881
1882
1.76k
  map->libbpf_type = type;
1883
1.76k
  map->sec_idx = sec_idx;
1884
1.76k
  map->sec_offset = 0;
1885
1.76k
  map->real_name = strdup(real_name);
1886
1.76k
  map->name = internal_map_name(obj, real_name);
1887
1.76k
  if (!map->real_name || !map->name) {
1888
0
    zfree(&map->real_name);
1889
0
    zfree(&map->name);
1890
0
    return -ENOMEM;
1891
0
  }
1892
1893
1.76k
  def = &map->def;
1894
1.76k
  def->type = BPF_MAP_TYPE_ARRAY;
1895
1.76k
  def->key_size = sizeof(int);
1896
1.76k
  def->value_size = data_sz;
1897
1.76k
  def->max_entries = 1;
1898
1.76k
  def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
1899
1.76k
    ? BPF_F_RDONLY_PROG : 0;
1900
1901
  /* failures are fine because of maps like .rodata.str1.1 */
1902
1.76k
  (void) map_fill_btf_type_info(obj, map);
1903
1904
1.76k
  if (map_is_mmapable(obj, map))
1905
91
    def->map_flags |= BPF_F_MMAPABLE;
1906
1907
1.76k
  pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
1908
1.76k
     map->name, map->sec_idx, map->sec_offset, def->map_flags);
1909
1910
1.76k
  mmap_sz = bpf_map_mmap_sz(map);
1911
1.76k
  map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
1912
1.76k
         MAP_SHARED | MAP_ANONYMOUS, -1, 0);
1913
1.76k
  if (map->mmaped == MAP_FAILED) {
1914
94
    err = -errno;
1915
94
    map->mmaped = NULL;
1916
94
    pr_warn("failed to alloc map '%s' content buffer: %d\n",
1917
94
      map->name, err);
1918
94
    zfree(&map->real_name);
1919
94
    zfree(&map->name);
1920
94
    return err;
1921
94
  }
1922
1923
1.67k
  if (data)
1924
861
    memcpy(map->mmaped, data, data_sz);
1925
1926
1.67k
  pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1927
1.67k
  return 0;
1928
1.76k
}
1929
1930
static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1931
2.15k
{
1932
2.15k
  struct elf_sec_desc *sec_desc;
1933
2.15k
  const char *sec_name;
1934
2.15k
  int err = 0, sec_idx;
1935
1936
  /*
1937
   * Populate obj->maps with libbpf internal maps.
1938
   */
1939
17.1k
  for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1940
14.9k
    sec_desc = &obj->efile.secs[sec_idx];
1941
1942
    /* Skip recognized sections with size 0. */
1943
14.9k
    if (!sec_desc->data || sec_desc->data->d_size == 0)
1944
12.1k
      continue;
1945
1946
2.80k
    switch (sec_desc->sec_type) {
1947
502
    case SEC_DATA:
1948
502
      sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1949
502
      err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1950
502
                  sec_name, sec_idx,
1951
502
                  sec_desc->data->d_buf,
1952
502
                  sec_desc->data->d_size);
1953
502
      break;
1954
359
    case SEC_RODATA:
1955
359
      obj->has_rodata = true;
1956
359
      sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1957
359
      err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1958
359
                  sec_name, sec_idx,
1959
359
                  sec_desc->data->d_buf,
1960
359
                  sec_desc->data->d_size);
1961
359
      break;
1962
824
    case SEC_BSS:
1963
824
      sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1964
824
      err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1965
824
                  sec_name, sec_idx,
1966
824
                  NULL,
1967
824
                  sec_desc->data->d_size);
1968
824
      break;
1969
1.12k
    default:
1970
      /* skip */
1971
1.12k
      break;
1972
2.80k
    }
1973
2.80k
    if (err)
1974
29
      return err;
1975
2.80k
  }
1976
2.12k
  return 0;
1977
2.15k
}
1978
1979
1980
static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1981
                 const void *name)
1982
950
{
1983
950
  int i;
1984
1985
2.69k
  for (i = 0; i < obj->nr_extern; i++) {
1986
2.20k
    if (strcmp(obj->externs[i].name, name) == 0)
1987
458
      return &obj->externs[i];
1988
2.20k
  }
1989
492
  return NULL;
1990
950
}
1991
1992
static struct extern_desc *find_extern_by_name_with_len(const struct bpf_object *obj,
1993
              const void *name, int len)
1994
0
{
1995
0
  const char *ext_name;
1996
0
  int i;
1997
1998
0
  for (i = 0; i < obj->nr_extern; i++) {
1999
0
    ext_name = obj->externs[i].name;
2000
0
    if (strlen(ext_name) == len && strncmp(ext_name, name, len) == 0)
2001
0
      return &obj->externs[i];
2002
0
  }
2003
0
  return NULL;
2004
0
}
2005
2006
static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,
2007
            char value)
2008
0
{
2009
0
  switch (ext->kcfg.type) {
2010
0
  case KCFG_BOOL:
2011
0
    if (value == 'm') {
2012
0
      pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n",
2013
0
        ext->name, value);
2014
0
      return -EINVAL;
2015
0
    }
2016
0
    *(bool *)ext_val = value == 'y' ? true : false;
2017
0
    break;
2018
0
  case KCFG_TRISTATE:
2019
0
    if (value == 'y')
2020
0
      *(enum libbpf_tristate *)ext_val = TRI_YES;
2021
0
    else if (value == 'm')
2022
0
      *(enum libbpf_tristate *)ext_val = TRI_MODULE;
2023
0
    else /* value == 'n' */
2024
0
      *(enum libbpf_tristate *)ext_val = TRI_NO;
2025
0
    break;
2026
0
  case KCFG_CHAR:
2027
0
    *(char *)ext_val = value;
2028
0
    break;
2029
0
  case KCFG_UNKNOWN:
2030
0
  case KCFG_INT:
2031
0
  case KCFG_CHAR_ARR:
2032
0
  default:
2033
0
    pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n",
2034
0
      ext->name, value);
2035
0
    return -EINVAL;
2036
0
  }
2037
0
  ext->is_set = true;
2038
0
  return 0;
2039
0
}
2040
2041
static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,
2042
            const char *value)
2043
0
{
2044
0
  size_t len;
2045
2046
0
  if (ext->kcfg.type != KCFG_CHAR_ARR) {
2047
0
    pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n",
2048
0
      ext->name, value);
2049
0
    return -EINVAL;
2050
0
  }
2051
2052
0
  len = strlen(value);
2053
0
  if (value[len - 1] != '"') {
2054
0
    pr_warn("extern (kcfg) '%s': invalid string config '%s'\n",
2055
0
      ext->name, value);
2056
0
    return -EINVAL;
2057
0
  }
2058
2059
  /* strip quotes */
2060
0
  len -= 2;
2061
0
  if (len >= ext->kcfg.sz) {
2062
0
    pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n",
2063
0
      ext->name, value, len, ext->kcfg.sz - 1);
2064
0
    len = ext->kcfg.sz - 1;
2065
0
  }
2066
0
  memcpy(ext_val, value + 1, len);
2067
0
  ext_val[len] = '\0';
2068
0
  ext->is_set = true;
2069
0
  return 0;
2070
0
}
2071
2072
static int parse_u64(const char *value, __u64 *res)
2073
0
{
2074
0
  char *value_end;
2075
0
  int err;
2076
2077
0
  errno = 0;
2078
0
  *res = strtoull(value, &value_end, 0);
2079
0
  if (errno) {
2080
0
    err = -errno;
2081
0
    pr_warn("failed to parse '%s' as integer: %d\n", value, err);
2082
0
    return err;
2083
0
  }
2084
0
  if (*value_end) {
2085
0
    pr_warn("failed to parse '%s' as integer completely\n", value);
2086
0
    return -EINVAL;
2087
0
  }
2088
0
  return 0;
2089
0
}
2090
2091
static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)
2092
0
{
2093
0
  int bit_sz = ext->kcfg.sz * 8;
2094
2095
0
  if (ext->kcfg.sz == 8)
2096
0
    return true;
2097
2098
  /* Validate that value stored in u64 fits in integer of `ext->sz`
2099
   * bytes size without any loss of information. If the target integer
2100
   * is signed, we rely on the following limits of integer type of
2101
   * Y bits and subsequent transformation:
2102
   *
2103
   *     -2^(Y-1) <= X           <= 2^(Y-1) - 1
2104
   *            0 <= X + 2^(Y-1) <= 2^Y - 1
2105
   *            0 <= X + 2^(Y-1) <  2^Y
2106
   *
2107
   *  For unsigned target integer, check that all the (64 - Y) bits are
2108
   *  zero.
2109
   */
2110
0
  if (ext->kcfg.is_signed)
2111
0
    return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
2112
0
  else
2113
0
    return (v >> bit_sz) == 0;
2114
0
}
2115
2116
static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,
2117
            __u64 value)
2118
0
{
2119
0
  if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&
2120
0
      ext->kcfg.type != KCFG_BOOL) {
2121
0
    pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n",
2122
0
      ext->name, (unsigned long long)value);
2123
0
    return -EINVAL;
2124
0
  }
2125
0
  if (ext->kcfg.type == KCFG_BOOL && value > 1) {
2126
0
    pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n",
2127
0
      ext->name, (unsigned long long)value);
2128
0
    return -EINVAL;
2129
2130
0
  }
2131
0
  if (!is_kcfg_value_in_range(ext, value)) {
2132
0
    pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n",
2133
0
      ext->name, (unsigned long long)value, ext->kcfg.sz);
2134
0
    return -ERANGE;
2135
0
  }
2136
0
  switch (ext->kcfg.sz) {
2137
0
  case 1:
2138
0
    *(__u8 *)ext_val = value;
2139
0
    break;
2140
0
  case 2:
2141
0
    *(__u16 *)ext_val = value;
2142
0
    break;
2143
0
  case 4:
2144
0
    *(__u32 *)ext_val = value;
2145
0
    break;
2146
0
  case 8:
2147
0
    *(__u64 *)ext_val = value;
2148
0
    break;
2149
0
  default:
2150
0
    return -EINVAL;
2151
0
  }
2152
0
  ext->is_set = true;
2153
0
  return 0;
2154
0
}
2155
2156
static int bpf_object__process_kconfig_line(struct bpf_object *obj,
2157
              char *buf, void *data)
2158
0
{
2159
0
  struct extern_desc *ext;
2160
0
  char *sep, *value;
2161
0
  int len, err = 0;
2162
0
  void *ext_val;
2163
0
  __u64 num;
2164
2165
0
  if (!str_has_pfx(buf, "CONFIG_"))
2166
0
    return 0;
2167
2168
0
  sep = strchr(buf, '=');
2169
0
  if (!sep) {
2170
0
    pr_warn("failed to parse '%s': no separator\n", buf);
2171
0
    return -EINVAL;
2172
0
  }
2173
2174
  /* Trim ending '\n' */
2175
0
  len = strlen(buf);
2176
0
  if (buf[len - 1] == '\n')
2177
0
    buf[len - 1] = '\0';
2178
  /* Split on '=' and ensure that a value is present. */
2179
0
  *sep = '\0';
2180
0
  if (!sep[1]) {
2181
0
    *sep = '=';
2182
0
    pr_warn("failed to parse '%s': no value\n", buf);
2183
0
    return -EINVAL;
2184
0
  }
2185
2186
0
  ext = find_extern_by_name(obj, buf);
2187
0
  if (!ext || ext->is_set)
2188
0
    return 0;
2189
2190
0
  ext_val = data + ext->kcfg.data_off;
2191
0
  value = sep + 1;
2192
2193
0
  switch (*value) {
2194
0
  case 'y': case 'n': case 'm':
2195
0
    err = set_kcfg_value_tri(ext, ext_val, *value);
2196
0
    break;
2197
0
  case '"':
2198
0
    err = set_kcfg_value_str(ext, ext_val, value);
2199
0
    break;
2200
0
  default:
2201
    /* assume integer */
2202
0
    err = parse_u64(value, &num);
2203
0
    if (err) {
2204
0
      pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);
2205
0
      return err;
2206
0
    }
2207
0
    if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {
2208
0
      pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);
2209
0
      return -EINVAL;
2210
0
    }
2211
0
    err = set_kcfg_value_num(ext, ext_val, num);
2212
0
    break;
2213
0
  }
2214
0
  if (err)
2215
0
    return err;
2216
0
  pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);
2217
0
  return 0;
2218
0
}
2219
2220
static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
2221
0
{
2222
0
  char buf[PATH_MAX];
2223
0
  struct utsname uts;
2224
0
  int len, err = 0;
2225
0
  gzFile file;
2226
2227
0
  uname(&uts);
2228
0
  len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);
2229
0
  if (len < 0)
2230
0
    return -EINVAL;
2231
0
  else if (len >= PATH_MAX)
2232
0
    return -ENAMETOOLONG;
2233
2234
  /* gzopen also accepts uncompressed files. */
2235
0
  file = gzopen(buf, "re");
2236
0
  if (!file)
2237
0
    file = gzopen("/proc/config.gz", "re");
2238
2239
0
  if (!file) {
2240
0
    pr_warn("failed to open system Kconfig\n");
2241
0
    return -ENOENT;
2242
0
  }
2243
2244
0
  while (gzgets(file, buf, sizeof(buf))) {
2245
0
    err = bpf_object__process_kconfig_line(obj, buf, data);
2246
0
    if (err) {
2247
0
      pr_warn("error parsing system Kconfig line '%s': %d\n",
2248
0
        buf, err);
2249
0
      goto out;
2250
0
    }
2251
0
  }
2252
2253
0
out:
2254
0
  gzclose(file);
2255
0
  return err;
2256
0
}
2257
2258
static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
2259
          const char *config, void *data)
2260
0
{
2261
0
  char buf[PATH_MAX];
2262
0
  int err = 0;
2263
0
  FILE *file;
2264
2265
0
  file = fmemopen((void *)config, strlen(config), "r");
2266
0
  if (!file) {
2267
0
    err = -errno;
2268
0
    pr_warn("failed to open in-memory Kconfig: %d\n", err);
2269
0
    return err;
2270
0
  }
2271
2272
0
  while (fgets(buf, sizeof(buf), file)) {
2273
0
    err = bpf_object__process_kconfig_line(obj, buf, data);
2274
0
    if (err) {
2275
0
      pr_warn("error parsing in-memory Kconfig line '%s': %d\n",
2276
0
        buf, err);
2277
0
      break;
2278
0
    }
2279
0
  }
2280
2281
0
  fclose(file);
2282
0
  return err;
2283
0
}
2284
2285
static int bpf_object__init_kconfig_map(struct bpf_object *obj)
2286
2.12k
{
2287
2.12k
  struct extern_desc *last_ext = NULL, *ext;
2288
2.12k
  size_t map_sz;
2289
2.12k
  int i, err;
2290
2291
3.52k
  for (i = 0; i < obj->nr_extern; i++) {
2292
1.39k
    ext = &obj->externs[i];
2293
1.39k
    if (ext->type == EXT_KCFG)
2294
133
      last_ext = ext;
2295
1.39k
  }
2296
2297
2.12k
  if (!last_ext)
2298
2.04k
    return 0;
2299
2300
79
  map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;
2301
79
  err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
2302
79
              ".kconfig", obj->efile.symbols_shndx,
2303
79
              NULL, map_sz);
2304
79
  if (err)
2305
65
    return err;
2306
2307
14
  obj->kconfig_map_idx = obj->nr_maps - 1;
2308
2309
14
  return 0;
2310
79
}
2311
2312
const struct btf_type *
2313
skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id)
2314
5.29k
{
2315
5.29k
  const struct btf_type *t = btf__type_by_id(btf, id);
2316
2317
5.29k
  if (res_id)
2318
2.48k
    *res_id = id;
2319
2320
7.31k
  while (btf_is_mod(t) || btf_is_typedef(t)) {
2321
2.02k
    if (res_id)
2322
1.35k
      *res_id = t->type;
2323
2.02k
    t = btf__type_by_id(btf, t->type);
2324
2.02k
  }
2325
2326
5.29k
  return t;
2327
5.29k
}
2328
2329
static const struct btf_type *
2330
resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id)
2331
0
{
2332
0
  const struct btf_type *t;
2333
2334
0
  t = skip_mods_and_typedefs(btf, id, NULL);
2335
0
  if (!btf_is_ptr(t))
2336
0
    return NULL;
2337
2338
0
  t = skip_mods_and_typedefs(btf, t->type, res_id);
2339
2340
0
  return btf_is_func_proto(t) ? t : NULL;
2341
0
}
2342
2343
static const char *__btf_kind_str(__u16 kind)
2344
397
{
2345
397
  switch (kind) {
2346
48
  case BTF_KIND_UNKN: return "void";
2347
14
  case BTF_KIND_INT: return "int";
2348
3
  case BTF_KIND_PTR: return "ptr";
2349
10
  case BTF_KIND_ARRAY: return "array";
2350
7
  case BTF_KIND_STRUCT: return "struct";
2351
4
  case BTF_KIND_UNION: return "union";
2352
2
  case BTF_KIND_ENUM: return "enum";
2353
5
  case BTF_KIND_FWD: return "fwd";
2354
6
  case BTF_KIND_TYPEDEF: return "typedef";
2355
2
  case BTF_KIND_VOLATILE: return "volatile";
2356
4
  case BTF_KIND_CONST: return "const";
2357
3
  case BTF_KIND_RESTRICT: return "restrict";
2358
156
  case BTF_KIND_FUNC: return "func";
2359
13
  case BTF_KIND_FUNC_PROTO: return "func_proto";
2360
51
  case BTF_KIND_VAR: return "var";
2361
52
  case BTF_KIND_DATASEC: return "datasec";
2362
5
  case BTF_KIND_FLOAT: return "float";
2363
3
  case BTF_KIND_DECL_TAG: return "decl_tag";
2364
2
  case BTF_KIND_TYPE_TAG: return "type_tag";
2365
7
  case BTF_KIND_ENUM64: return "enum64";
2366
0
  default: return "unknown";
2367
397
  }
2368
397
}
2369
2370
const char *btf_kind_str(const struct btf_type *t)
2371
397
{
2372
397
  return __btf_kind_str(btf_kind(t));
2373
397
}
2374
2375
/*
2376
 * Fetch integer attribute of BTF map definition. Such attributes are
2377
 * represented using a pointer to an array, in which dimensionality of array
2378
 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY];
2379
 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF
2380
 * type definition, while using only sizeof(void *) space in ELF data section.
2381
 */
2382
static bool get_map_field_int(const char *map_name, const struct btf *btf,
2383
            const struct btf_member *m, __u32 *res)
2384
718
{
2385
718
  const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2386
718
  const char *name = btf__name_by_offset(btf, m->name_off);
2387
718
  const struct btf_array *arr_info;
2388
718
  const struct btf_type *arr_t;
2389
2390
718
  if (!btf_is_ptr(t)) {
2391
21
    pr_warn("map '%s': attr '%s': expected PTR, got %s.\n",
2392
21
      map_name, name, btf_kind_str(t));
2393
21
    return false;
2394
21
  }
2395
2396
697
  arr_t = btf__type_by_id(btf, t->type);
2397
697
  if (!arr_t) {
2398
0
    pr_warn("map '%s': attr '%s': type [%u] not found.\n",
2399
0
      map_name, name, t->type);
2400
0
    return false;
2401
0
  }
2402
697
  if (!btf_is_array(arr_t)) {
2403
12
    pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n",
2404
12
      map_name, name, btf_kind_str(arr_t));
2405
12
    return false;
2406
12
  }
2407
685
  arr_info = btf_array(arr_t);
2408
685
  *res = arr_info->nelems;
2409
685
  return true;
2410
697
}
2411
2412
static bool get_map_field_long(const char *map_name, const struct btf *btf,
2413
             const struct btf_member *m, __u64 *res)
2414
38
{
2415
38
  const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);
2416
38
  const char *name = btf__name_by_offset(btf, m->name_off);
2417
2418
38
  if (btf_is_ptr(t)) {
2419
22
    __u32 res32;
2420
22
    bool ret;
2421
2422
22
    ret = get_map_field_int(map_name, btf, m, &res32);
2423
22
    if (ret)
2424
21
      *res = (__u64)res32;
2425
22
    return ret;
2426
22
  }
2427
2428
16
  if (!btf_is_enum(t) && !btf_is_enum64(t)) {
2429
9
    pr_warn("map '%s': attr '%s': expected ENUM or ENUM64, got %s.\n",
2430
9
      map_name, name, btf_kind_str(t));
2431
9
    return false;
2432
9
  }
2433
2434
7
  if (btf_vlen(t) != 1) {
2435
1
    pr_warn("map '%s': attr '%s': invalid __ulong\n",
2436
1
      map_name, name);
2437
1
    return false;
2438
1
  }
2439
2440
6
  if (btf_is_enum(t)) {
2441
3
    const struct btf_enum *e = btf_enum(t);
2442
2443
3
    *res = e->val;
2444
3
  } else {
2445
3
    const struct btf_enum64 *e = btf_enum64(t);
2446
2447
3
    *res = btf_enum64_value(e);
2448
3
  }
2449
6
  return true;
2450
7
}
2451
2452
static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name)
2453
1
{
2454
1
  int len;
2455
2456
1
  len = snprintf(buf, buf_sz, "%s/%s", path, name);
2457
1
  if (len < 0)
2458
0
    return -EINVAL;
2459
1
  if (len >= buf_sz)
2460
0
    return -ENAMETOOLONG;
2461
2462
1
  return 0;
2463
1
}
2464
2465
static int build_map_pin_path(struct bpf_map *map, const char *path)
2466
1
{
2467
1
  char buf[PATH_MAX];
2468
1
  int err;
2469
2470
1
  if (!path)
2471
1
    path = BPF_FS_DEFAULT_PATH;
2472
2473
1
  err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map));
2474
1
  if (err)
2475
0
    return err;
2476
2477
1
  return bpf_map__set_pin_path(map, buf);
2478
1
}
2479
2480
/* should match definition in bpf_helpers.h */
2481
enum libbpf_pin_type {
2482
  LIBBPF_PIN_NONE,
2483
  /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */
2484
  LIBBPF_PIN_BY_NAME,
2485
};
2486
2487
int parse_btf_map_def(const char *map_name, struct btf *btf,
2488
          const struct btf_type *def_t, bool strict,
2489
          struct btf_map_def *map_def, struct btf_map_def *inner_def)
2490
949
{
2491
949
  const struct btf_type *t;
2492
949
  const struct btf_member *m;
2493
949
  bool is_inner = inner_def == NULL;
2494
949
  int vlen, i;
2495
2496
949
  vlen = btf_vlen(def_t);
2497
949
  m = btf_members(def_t);
2498
2.01k
  for (i = 0; i < vlen; i++, m++) {
2499
1.90k
    const char *name = btf__name_by_offset(btf, m->name_off);
2500
2501
1.90k
    if (!name) {
2502
0
      pr_warn("map '%s': invalid field #%d.\n", map_name, i);
2503
0
      return -EINVAL;
2504
0
    }
2505
1.90k
    if (strcmp(name, "type") == 0) {
2506
259
      if (!get_map_field_int(map_name, btf, m, &map_def->map_type))
2507
3
        return -EINVAL;
2508
256
      map_def->parts |= MAP_DEF_MAP_TYPE;
2509
1.64k
    } else if (strcmp(name, "max_entries") == 0) {
2510
26
      if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))
2511
1
        return -EINVAL;
2512
25
      map_def->parts |= MAP_DEF_MAX_ENTRIES;
2513
1.62k
    } else if (strcmp(name, "map_flags") == 0) {
2514
21
      if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))
2515
4
        return -EINVAL;
2516
17
      map_def->parts |= MAP_DEF_MAP_FLAGS;
2517
1.59k
    } else if (strcmp(name, "numa_node") == 0) {
2518
18
      if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))
2519
4
        return -EINVAL;
2520
14
      map_def->parts |= MAP_DEF_NUMA_NODE;
2521
1.58k
    } else if (strcmp(name, "key_size") == 0) {
2522
192
      __u32 sz;
2523
2524
192
      if (!get_map_field_int(map_name, btf, m, &sz))
2525
16
        return -EINVAL;
2526
176
      if (map_def->key_size && map_def->key_size != sz) {
2527
40
        pr_warn("map '%s': conflicting key size %u != %u.\n",
2528
40
          map_name, map_def->key_size, sz);
2529
40
        return -EINVAL;
2530
40
      }
2531
136
      map_def->key_size = sz;
2532
136
      map_def->parts |= MAP_DEF_KEY_SIZE;
2533
1.38k
    } else if (strcmp(name, "key") == 0) {
2534
199
      __s64 sz;
2535
2536
199
      t = btf__type_by_id(btf, m->type);
2537
199
      if (!t) {
2538
0
        pr_warn("map '%s': key type [%d] not found.\n",
2539
0
          map_name, m->type);
2540
0
        return -EINVAL;
2541
0
      }
2542
199
      if (!btf_is_ptr(t)) {
2543
4
        pr_warn("map '%s': key spec is not PTR: %s.\n",
2544
4
          map_name, btf_kind_str(t));
2545
4
        return -EINVAL;
2546
4
      }
2547
195
      sz = btf__resolve_size(btf, t->type);
2548
195
      if (sz < 0) {
2549
11
        pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n",
2550
11
          map_name, t->type, (ssize_t)sz);
2551
11
        return sz;
2552
11
      }
2553
184
      if (map_def->key_size && map_def->key_size != sz) {
2554
5
        pr_warn("map '%s': conflicting key size %u != %zd.\n",
2555
5
          map_name, map_def->key_size, (ssize_t)sz);
2556
5
        return -EINVAL;
2557
5
      }
2558
179
      map_def->key_size = sz;
2559
179
      map_def->key_type_id = t->type;
2560
179
      map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;
2561
1.19k
    } else if (strcmp(name, "value_size") == 0) {
2562
118
      __u32 sz;
2563
2564
118
      if (!get_map_field_int(map_name, btf, m, &sz))
2565
2
        return -EINVAL;
2566
116
      if (map_def->value_size && map_def->value_size != sz) {
2567
51
        pr_warn("map '%s': conflicting value size %u != %u.\n",
2568
51
          map_name, map_def->value_size, sz);
2569
51
        return -EINVAL;
2570
51
      }
2571
65
      map_def->value_size = sz;
2572
65
      map_def->parts |= MAP_DEF_VALUE_SIZE;
2573
1.07k
    } else if (strcmp(name, "value") == 0) {
2574
348
      __s64 sz;
2575
2576
348
      t = btf__type_by_id(btf, m->type);
2577
348
      if (!t) {
2578
0
        pr_warn("map '%s': value type [%d] not found.\n",
2579
0
          map_name, m->type);
2580
0
        return -EINVAL;
2581
0
      }
2582
348
      if (!btf_is_ptr(t)) {
2583
7
        pr_warn("map '%s': value spec is not PTR: %s.\n",
2584
7
          map_name, btf_kind_str(t));
2585
7
        return -EINVAL;
2586
7
      }
2587
341
      sz = btf__resolve_size(btf, t->type);
2588
341
      if (sz < 0) {
2589
9
        pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n",
2590
9
          map_name, t->type, (ssize_t)sz);
2591
9
        return sz;
2592
9
      }
2593
332
      if (map_def->value_size && map_def->value_size != sz) {
2594
2
        pr_warn("map '%s': conflicting value size %u != %zd.\n",
2595
2
          map_name, map_def->value_size, (ssize_t)sz);
2596
2
        return -EINVAL;
2597
2
      }
2598
330
      map_def->value_size = sz;
2599
330
      map_def->value_type_id = t->type;
2600
330
      map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;
2601
330
    }
2602
724
    else if (strcmp(name, "values") == 0) {
2603
133
      bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);
2604
133
      bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;
2605
133
      const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2606
133
      char inner_map_name[128];
2607
133
      int err;
2608
2609
133
      if (is_inner) {
2610
2
        pr_warn("map '%s': multi-level inner maps not supported.\n",
2611
2
          map_name);
2612
2
        return -ENOTSUP;
2613
2
      }
2614
131
      if (i != vlen - 1) {
2615
41
        pr_warn("map '%s': '%s' member should be last.\n",
2616
41
          map_name, name);
2617
41
        return -EINVAL;
2618
41
      }
2619
90
      if (!is_map_in_map && !is_prog_array) {
2620
23
        pr_warn("map '%s': should be map-in-map or prog-array.\n",
2621
23
          map_name);
2622
23
        return -ENOTSUP;
2623
23
      }
2624
67
      if (map_def->value_size && map_def->value_size != 4) {
2625
42
        pr_warn("map '%s': conflicting value size %u != 4.\n",
2626
42
          map_name, map_def->value_size);
2627
42
        return -EINVAL;
2628
42
      }
2629
25
      map_def->value_size = 4;
2630
25
      t = btf__type_by_id(btf, m->type);
2631
25
      if (!t) {
2632
0
        pr_warn("map '%s': %s type [%d] not found.\n",
2633
0
          map_name, desc, m->type);
2634
0
        return -EINVAL;
2635
0
      }
2636
25
      if (!btf_is_array(t) || btf_array(t)->nelems) {
2637
16
        pr_warn("map '%s': %s spec is not a zero-sized array.\n",
2638
16
          map_name, desc);
2639
16
        return -EINVAL;
2640
16
      }
2641
9
      t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);
2642
9
      if (!btf_is_ptr(t)) {
2643
2
        pr_warn("map '%s': %s def is of unexpected kind %s.\n",
2644
2
          map_name, desc, btf_kind_str(t));
2645
2
        return -EINVAL;
2646
2
      }
2647
7
      t = skip_mods_and_typedefs(btf, t->type, NULL);
2648
7
      if (is_prog_array) {
2649
1
        if (!btf_is_func_proto(t)) {
2650
1
          pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
2651
1
            map_name, btf_kind_str(t));
2652
1
          return -EINVAL;
2653
1
        }
2654
0
        continue;
2655
1
      }
2656
6
      if (!btf_is_struct(t)) {
2657
4
        pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",
2658
4
          map_name, btf_kind_str(t));
2659
4
        return -EINVAL;
2660
4
      }
2661
2662
2
      snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name);
2663
2
      err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL);
2664
2
      if (err)
2665
2
        return err;
2666
2667
0
      map_def->parts |= MAP_DEF_INNER_MAP;
2668
591
    } else if (strcmp(name, "pinning") == 0) {
2669
62
      __u32 val;
2670
2671
62
      if (is_inner) {
2672
0
        pr_warn("map '%s': inner def can't be pinned.\n", map_name);
2673
0
        return -EINVAL;
2674
0
      }
2675
62
      if (!get_map_field_int(map_name, btf, m, &val))
2676
2
        return -EINVAL;
2677
60
      if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) {
2678
48
        pr_warn("map '%s': invalid pinning value %u.\n",
2679
48
          map_name, val);
2680
48
        return -EINVAL;
2681
48
      }
2682
12
      map_def->pinning = val;
2683
12
      map_def->parts |= MAP_DEF_PINNING;
2684
529
    } else if (strcmp(name, "map_extra") == 0) {
2685
38
      __u64 map_extra;
2686
2687
38
      if (!get_map_field_long(map_name, btf, m, &map_extra))
2688
11
        return -EINVAL;
2689
27
      map_def->map_extra = map_extra;
2690
27
      map_def->parts |= MAP_DEF_MAP_EXTRA;
2691
491
    } else {
2692
491
      if (strict) {
2693
491
        pr_warn("map '%s': unknown field '%s'.\n", map_name, name);
2694
491
        return -ENOTSUP;
2695
491
      }
2696
0
      pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name);
2697
0
    }
2698
1.90k
  }
2699
2700
105
  if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {
2701
34
    pr_warn("map '%s': map type isn't specified.\n", map_name);
2702
34
    return -EINVAL;
2703
34
  }
2704
2705
71
  return 0;
2706
105
}
2707
2708
static size_t adjust_ringbuf_sz(size_t sz)
2709
3
{
2710
3
  __u32 page_sz = sysconf(_SC_PAGE_SIZE);
2711
3
  __u32 mul;
2712
2713
  /* if user forgot to set any size, make sure they see error */
2714
3
  if (sz == 0)
2715
1
    return 0;
2716
  /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be
2717
   * a power-of-2 multiple of kernel's page size. If user diligently
2718
   * satisified these conditions, pass the size through.
2719
   */
2720
2
  if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz))
2721
0
    return sz;
2722
2723
  /* Otherwise find closest (page_sz * power_of_2) product bigger than
2724
   * user-set size to satisfy both user size request and kernel
2725
   * requirements and substitute correct max_entries for map creation.
2726
   */
2727
2
  for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) {
2728
2
    if (mul * page_sz > sz)
2729
2
      return mul * page_sz;
2730
2
  }
2731
2732
  /* if it's impossible to satisfy the conditions (i.e., user size is
2733
   * very close to UINT_MAX but is not a power-of-2 multiple of
2734
   * page_size) then just return original size and let kernel reject it
2735
   */
2736
0
  return sz;
2737
2
}
2738
2739
static bool map_is_ringbuf(const struct bpf_map *map)
2740
71
{
2741
71
  return map->def.type == BPF_MAP_TYPE_RINGBUF ||
2742
71
         map->def.type == BPF_MAP_TYPE_USER_RINGBUF;
2743
71
}
2744
2745
static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def)
2746
71
{
2747
71
  map->def.type = def->map_type;
2748
71
  map->def.key_size = def->key_size;
2749
71
  map->def.value_size = def->value_size;
2750
71
  map->def.max_entries = def->max_entries;
2751
71
  map->def.map_flags = def->map_flags;
2752
71
  map->map_extra = def->map_extra;
2753
2754
71
  map->numa_node = def->numa_node;
2755
71
  map->btf_key_type_id = def->key_type_id;
2756
71
  map->btf_value_type_id = def->value_type_id;
2757
2758
  /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
2759
71
  if (map_is_ringbuf(map))
2760
3
    map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
2761
2762
71
  if (def->parts & MAP_DEF_MAP_TYPE)
2763
71
    pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);
2764
2765
71
  if (def->parts & MAP_DEF_KEY_TYPE)
2766
71
    pr_debug("map '%s': found key [%u], sz = %u.\n",
2767
64
       map->name, def->key_type_id, def->key_size);
2768
64
  else if (def->parts & MAP_DEF_KEY_SIZE)
2769
25
    pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);
2770
2771
71
  if (def->parts & MAP_DEF_VALUE_TYPE)
2772
71
    pr_debug("map '%s': found value [%u], sz = %u.\n",
2773
63
       map->name, def->value_type_id, def->value_size);
2774
63
  else if (def->parts & MAP_DEF_VALUE_SIZE)
2775
7
    pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);
2776
2777
71
  if (def->parts & MAP_DEF_MAX_ENTRIES)
2778
71
    pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);
2779
71
  if (def->parts & MAP_DEF_MAP_FLAGS)
2780
71
    pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);
2781
71
  if (def->parts & MAP_DEF_MAP_EXTRA)
2782
71
    pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,
2783
71
       (unsigned long long)def->map_extra);
2784
71
  if (def->parts & MAP_DEF_PINNING)
2785
71
    pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);
2786
71
  if (def->parts & MAP_DEF_NUMA_NODE)
2787
71
    pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);
2788
2789
71
  if (def->parts & MAP_DEF_INNER_MAP)
2790
71
    pr_debug("map '%s': found inner map definition.\n", map->name);
2791
71
}
2792
2793
static const char *btf_var_linkage_str(__u32 linkage)
2794
66
{
2795
66
  switch (linkage) {
2796
4
  case BTF_VAR_STATIC: return "static";
2797
0
  case BTF_VAR_GLOBAL_ALLOCATED: return "global";
2798
1
  case BTF_VAR_GLOBAL_EXTERN: return "extern";
2799
61
  default: return "unknown";
2800
66
  }
2801
66
}
2802
2803
static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2804
           const struct btf_type *sec,
2805
           int var_idx, int sec_idx,
2806
           const Elf_Data *data, bool strict,
2807
           const char *pin_root_path)
2808
1.06k
{
2809
1.06k
  struct btf_map_def map_def = {}, inner_def = {};
2810
1.06k
  const struct btf_type *var, *def;
2811
1.06k
  const struct btf_var_secinfo *vi;
2812
1.06k
  const struct btf_var *var_extra;
2813
1.06k
  const char *map_name;
2814
1.06k
  struct bpf_map *map;
2815
1.06k
  int err;
2816
2817
1.06k
  vi = btf_var_secinfos(sec) + var_idx;
2818
1.06k
  var = btf__type_by_id(obj->btf, vi->type);
2819
1.06k
  var_extra = btf_var(var);
2820
1.06k
  map_name = btf__name_by_offset(obj->btf, var->name_off);
2821
2822
1.06k
  if (map_name == NULL || map_name[0] == '\0') {
2823
1
    pr_warn("map #%d: empty name.\n", var_idx);
2824
1
    return -EINVAL;
2825
1
  }
2826
1.06k
  if ((__u64)vi->offset + vi->size > data->d_size) {
2827
1
    pr_warn("map '%s' BTF data is corrupted.\n", map_name);
2828
1
    return -EINVAL;
2829
1
  }
2830
1.06k
  if (!btf_is_var(var)) {
2831
0
    pr_warn("map '%s': unexpected var kind %s.\n",
2832
0
      map_name, btf_kind_str(var));
2833
0
    return -EINVAL;
2834
0
  }
2835
1.06k
  if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {
2836
66
    pr_warn("map '%s': unsupported map linkage %s.\n",
2837
66
      map_name, btf_var_linkage_str(var_extra->linkage));
2838
66
    return -EOPNOTSUPP;
2839
66
  }
2840
2841
1.00k
  def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2842
1.00k
  if (!btf_is_struct(def)) {
2843
14
    pr_warn("map '%s': unexpected def kind %s.\n",
2844
14
      map_name, btf_kind_str(var));
2845
14
    return -EINVAL;
2846
14
  }
2847
987
  if (def->size > vi->size) {
2848
40
    pr_warn("map '%s': invalid def size.\n", map_name);
2849
40
    return -EINVAL;
2850
40
  }
2851
2852
947
  map = bpf_object__add_map(obj);
2853
947
  if (IS_ERR(map))
2854
0
    return PTR_ERR(map);
2855
947
  map->name = strdup(map_name);
2856
947
  if (!map->name) {
2857
0
    pr_warn("map '%s': failed to alloc map name.\n", map_name);
2858
0
    return -ENOMEM;
2859
0
  }
2860
947
  map->libbpf_type = LIBBPF_MAP_UNSPEC;
2861
947
  map->def.type = BPF_MAP_TYPE_UNSPEC;
2862
947
  map->sec_idx = sec_idx;
2863
947
  map->sec_offset = vi->offset;
2864
947
  map->btf_var_idx = var_idx;
2865
947
  pr_debug("map '%s': at sec_idx %d, offset %zu.\n",
2866
947
     map_name, map->sec_idx, map->sec_offset);
2867
2868
947
  err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2869
947
  if (err)
2870
876
    return err;
2871
2872
71
  fill_map_from_def(map, &map_def);
2873
2874
71
  if (map_def.pinning == LIBBPF_PIN_BY_NAME) {
2875
1
    err = build_map_pin_path(map, pin_root_path);
2876
1
    if (err) {
2877
0
      pr_warn("map '%s': couldn't build pin path.\n", map->name);
2878
0
      return err;
2879
0
    }
2880
1
  }
2881
2882
71
  if (map_def.parts & MAP_DEF_INNER_MAP) {
2883
0
    map->inner_map = calloc(1, sizeof(*map->inner_map));
2884
0
    if (!map->inner_map)
2885
0
      return -ENOMEM;
2886
0
    map->inner_map->fd = create_placeholder_fd();
2887
0
    if (map->inner_map->fd < 0)
2888
0
      return map->inner_map->fd;
2889
0
    map->inner_map->sec_idx = sec_idx;
2890
0
    map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);
2891
0
    if (!map->inner_map->name)
2892
0
      return -ENOMEM;
2893
0
    sprintf(map->inner_map->name, "%s.inner", map_name);
2894
2895
0
    fill_map_from_def(map->inner_map, &inner_def);
2896
0
  }
2897
2898
71
  err = map_fill_btf_type_info(obj, map);
2899
71
  if (err)
2900
0
    return err;
2901
2902
71
  return 0;
2903
71
}
2904
2905
static int init_arena_map_data(struct bpf_object *obj, struct bpf_map *map,
2906
             const char *sec_name, int sec_idx,
2907
             void *data, size_t data_sz)
2908
0
{
2909
0
  const long page_sz = sysconf(_SC_PAGE_SIZE);
2910
0
  size_t mmap_sz;
2911
2912
0
  mmap_sz = bpf_map_mmap_sz(obj->arena_map);
2913
0
  if (roundup(data_sz, page_sz) > mmap_sz) {
2914
0
    pr_warn("elf: sec '%s': declared ARENA map size (%zu) is too small to hold global __arena variables of size %zu\n",
2915
0
      sec_name, mmap_sz, data_sz);
2916
0
    return -E2BIG;
2917
0
  }
2918
2919
0
  obj->arena_data = malloc(data_sz);
2920
0
  if (!obj->arena_data)
2921
0
    return -ENOMEM;
2922
0
  memcpy(obj->arena_data, data, data_sz);
2923
0
  obj->arena_data_sz = data_sz;
2924
2925
  /* make bpf_map__init_value() work for ARENA maps */
2926
0
  map->mmaped = obj->arena_data;
2927
2928
0
  return 0;
2929
0
}
2930
2931
static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2932
            const char *pin_root_path)
2933
3.20k
{
2934
3.20k
  const struct btf_type *sec = NULL;
2935
3.20k
  int nr_types, i, vlen, err;
2936
3.20k
  const struct btf_type *t;
2937
3.20k
  const char *name;
2938
3.20k
  Elf_Data *data;
2939
3.20k
  Elf_Scn *scn;
2940
2941
3.20k
  if (obj->efile.btf_maps_shndx < 0)
2942
2.08k
    return 0;
2943
2944
1.12k
  scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2945
1.12k
  data = elf_sec_data(obj, scn);
2946
1.12k
  if (!scn || !data) {
2947
0
    pr_warn("elf: failed to get %s map definitions for %s\n",
2948
0
      MAPS_ELF_SEC, obj->path);
2949
0
    return -EINVAL;
2950
0
  }
2951
2952
1.12k
  nr_types = btf__type_cnt(obj->btf);
2953
13.8k
  for (i = 1; i < nr_types; i++) {
2954
13.8k
    t = btf__type_by_id(obj->btf, i);
2955
13.8k
    if (!btf_is_datasec(t))
2956
12.0k
      continue;
2957
1.78k
    name = btf__name_by_offset(obj->btf, t->name_off);
2958
1.78k
    if (strcmp(name, MAPS_ELF_SEC) == 0) {
2959
1.07k
      sec = t;
2960
1.07k
      obj->efile.btf_maps_sec_btf_id = i;
2961
1.07k
      break;
2962
1.07k
    }
2963
1.78k
  }
2964
2965
1.12k
  if (!sec) {
2966
48
    pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC);
2967
48
    return -ENOENT;
2968
48
  }
2969
2970
1.07k
  vlen = btf_vlen(sec);
2971
1.14k
  for (i = 0; i < vlen; i++) {
2972
1.06k
    err = bpf_object__init_user_btf_map(obj, sec, i,
2973
1.06k
                obj->efile.btf_maps_shndx,
2974
1.06k
                data, strict,
2975
1.06k
                pin_root_path);
2976
1.06k
    if (err)
2977
998
      return err;
2978
1.06k
  }
2979
2980
147
  for (i = 0; i < obj->nr_maps; i++) {
2981
71
    struct bpf_map *map = &obj->maps[i];
2982
2983
71
    if (map->def.type != BPF_MAP_TYPE_ARENA)
2984
69
      continue;
2985
2986
2
    if (obj->arena_map) {
2987
0
      pr_warn("map '%s': only single ARENA map is supported (map '%s' is also ARENA)\n",
2988
0
        map->name, obj->arena_map->name);
2989
0
      return -EINVAL;
2990
0
    }
2991
2
    obj->arena_map = map;
2992
2993
2
    if (obj->efile.arena_data) {
2994
0
      err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx,
2995
0
              obj->efile.arena_data->d_buf,
2996
0
              obj->efile.arena_data->d_size);
2997
0
      if (err)
2998
0
        return err;
2999
0
    }
3000
2
  }
3001
76
  if (obj->efile.arena_data && !obj->arena_map) {
3002
1
    pr_warn("elf: sec '%s': to use global __arena variables the ARENA map should be explicitly declared in SEC(\".maps\")\n",
3003
1
      ARENA_SEC);
3004
1
    return -ENOENT;
3005
1
  }
3006
3007
75
  return 0;
3008
76
}
3009
3010
static int bpf_object__init_maps(struct bpf_object *obj,
3011
         const struct bpf_object_open_opts *opts)
3012
3.20k
{
3013
3.20k
  const char *pin_root_path;
3014
3.20k
  bool strict;
3015
3.20k
  int err = 0;
3016
3017
3.20k
  strict = !OPTS_GET(opts, relaxed_maps, false);
3018
3.20k
  pin_root_path = OPTS_GET(opts, pin_root_path, NULL);
3019
3020
3.20k
  err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
3021
3.20k
  err = err ?: bpf_object__init_global_data_maps(obj);
3022
3.20k
  err = err ?: bpf_object__init_kconfig_map(obj);
3023
18.4E
  err = err ?: bpf_object_init_struct_ops(obj);
3024
3025
18.4E
  return err;
3026
1.08k
}
3027
3028
static bool section_have_execinstr(struct bpf_object *obj, int idx)
3029
1.97k
{
3030
1.97k
  Elf64_Shdr *sh;
3031
3032
1.97k
  sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));
3033
1.97k
  if (!sh)
3034
0
    return false;
3035
3036
1.97k
  return sh->sh_flags & SHF_EXECINSTR;
3037
1.97k
}
3038
3039
static bool starts_with_qmark(const char *s)
3040
0
{
3041
0
  return s && s[0] == '?';
3042
0
}
3043
3044
static bool btf_needs_sanitization(struct bpf_object *obj)
3045
0
{
3046
0
  bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
3047
0
  bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
3048
0
  bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
3049
0
  bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
3050
0
  bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
3051
0
  bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
3052
0
  bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
3053
0
  bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC);
3054
3055
0
  return !has_func || !has_datasec || !has_func_global || !has_float ||
3056
0
         !has_decl_tag || !has_type_tag || !has_enum64 || !has_qmark_datasec;
3057
0
}
3058
3059
static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
3060
0
{
3061
0
  bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
3062
0
  bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
3063
0
  bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
3064
0
  bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
3065
0
  bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
3066
0
  bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
3067
0
  bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
3068
0
  bool has_qmark_datasec = kernel_supports(obj, FEAT_BTF_QMARK_DATASEC);
3069
0
  int enum64_placeholder_id = 0;
3070
0
  struct btf_type *t;
3071
0
  int i, j, vlen;
3072
3073
0
  for (i = 1; i < btf__type_cnt(btf); i++) {
3074
0
    t = (struct btf_type *)btf__type_by_id(btf, i);
3075
3076
0
    if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) {
3077
      /* replace VAR/DECL_TAG with INT */
3078
0
      t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);
3079
      /*
3080
       * using size = 1 is the safest choice, 4 will be too
3081
       * big and cause kernel BTF validation failure if
3082
       * original variable took less than 4 bytes
3083
       */
3084
0
      t->size = 1;
3085
0
      *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8);
3086
0
    } else if (!has_datasec && btf_is_datasec(t)) {
3087
      /* replace DATASEC with STRUCT */
3088
0
      const struct btf_var_secinfo *v = btf_var_secinfos(t);
3089
0
      struct btf_member *m = btf_members(t);
3090
0
      struct btf_type *vt;
3091
0
      char *name;
3092
3093
0
      name = (char *)btf__name_by_offset(btf, t->name_off);
3094
0
      while (*name) {
3095
0
        if (*name == '.' || *name == '?')
3096
0
          *name = '_';
3097
0
        name++;
3098
0
      }
3099
3100
0
      vlen = btf_vlen(t);
3101
0
      t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);
3102
0
      for (j = 0; j < vlen; j++, v++, m++) {
3103
        /* order of field assignments is important */
3104
0
        m->offset = v->offset * 8;
3105
0
        m->type = v->type;
3106
        /* preserve variable name as member name */
3107
0
        vt = (void *)btf__type_by_id(btf, v->type);
3108
0
        m->name_off = vt->name_off;
3109
0
      }
3110
0
    } else if (!has_qmark_datasec && btf_is_datasec(t) &&
3111
0
         starts_with_qmark(btf__name_by_offset(btf, t->name_off))) {
3112
      /* replace '?' prefix with '_' for DATASEC names */
3113
0
      char *name;
3114
3115
0
      name = (char *)btf__name_by_offset(btf, t->name_off);
3116
0
      if (name[0] == '?')
3117
0
        name[0] = '_';
3118
0
    } else if (!has_func && btf_is_func_proto(t)) {
3119
      /* replace FUNC_PROTO with ENUM */
3120
0
      vlen = btf_vlen(t);
3121
0
      t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);
3122
0
      t->size = sizeof(__u32); /* kernel enforced */
3123
0
    } else if (!has_func && btf_is_func(t)) {
3124
      /* replace FUNC with TYPEDEF */
3125
0
      t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);
3126
0
    } else if (!has_func_global && btf_is_func(t)) {
3127
      /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */
3128
0
      t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);
3129
0
    } else if (!has_float && btf_is_float(t)) {
3130
      /* replace FLOAT with an equally-sized empty STRUCT;
3131
       * since C compilers do not accept e.g. "float" as a
3132
       * valid struct name, make it anonymous
3133
       */
3134
0
      t->name_off = 0;
3135
0
      t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);
3136
0
    } else if (!has_type_tag && btf_is_type_tag(t)) {
3137
      /* replace TYPE_TAG with a CONST */
3138
0
      t->name_off = 0;
3139
0
      t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);
3140
0
    } else if (!has_enum64 && btf_is_enum(t)) {
3141
      /* clear the kflag */
3142
0
      t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);
3143
0
    } else if (!has_enum64 && btf_is_enum64(t)) {
3144
      /* replace ENUM64 with a union */
3145
0
      struct btf_member *m;
3146
3147
0
      if (enum64_placeholder_id == 0) {
3148
0
        enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0);
3149
0
        if (enum64_placeholder_id < 0)
3150
0
          return enum64_placeholder_id;
3151
3152
0
        t = (struct btf_type *)btf__type_by_id(btf, i);
3153
0
      }
3154
3155
0
      m = btf_members(t);
3156
0
      vlen = btf_vlen(t);
3157
0
      t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);
3158
0
      for (j = 0; j < vlen; j++, m++) {
3159
0
        m->type = enum64_placeholder_id;
3160
0
        m->offset = 0;
3161
0
      }
3162
0
    }
3163
0
  }
3164
3165
0
  return 0;
3166
0
}
3167
3168
static bool libbpf_needs_btf(const struct bpf_object *obj)
3169
2.85k
{
3170
2.85k
  return obj->efile.btf_maps_shndx >= 0 ||
3171
2.85k
         obj->efile.has_st_ops ||
3172
2.85k
         obj->nr_extern > 0;
3173
2.85k
}
3174
3175
static bool kernel_needs_btf(const struct bpf_object *obj)
3176
0
{
3177
0
  return obj->efile.has_st_ops;
3178
0
}
3179
3180
static int bpf_object__init_btf(struct bpf_object *obj,
3181
        Elf_Data *btf_data,
3182
        Elf_Data *btf_ext_data)
3183
6.54k
{
3184
6.54k
  int err = -ENOENT;
3185
3186
6.54k
  if (btf_data) {
3187
4.86k
    obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
3188
4.86k
    err = libbpf_get_error(obj->btf);
3189
4.86k
    if (err) {
3190
927
      obj->btf = NULL;
3191
927
      pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err);
3192
927
      goto out;
3193
927
    }
3194
    /* enforce 8-byte pointers for BPF-targeted BTFs */
3195
3.93k
    btf__set_pointer_size(obj->btf, 8);
3196
3.93k
  }
3197
5.61k
  if (btf_ext_data) {
3198
347
    struct btf_ext_info *ext_segs[3];
3199
347
    int seg_num, sec_num;
3200
3201
347
    if (!obj->btf) {
3202
3
      pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n",
3203
3
         BTF_EXT_ELF_SEC, BTF_ELF_SEC);
3204
3
      goto out;
3205
3
    }
3206
344
    obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
3207
344
    err = libbpf_get_error(obj->btf_ext);
3208
344
    if (err) {
3209
247
      pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n",
3210
247
        BTF_EXT_ELF_SEC, err);
3211
247
      obj->btf_ext = NULL;
3212
247
      goto out;
3213
247
    }
3214
3215
    /* setup .BTF.ext to ELF section mapping */
3216
97
    ext_segs[0] = &obj->btf_ext->func_info;
3217
97
    ext_segs[1] = &obj->btf_ext->line_info;
3218
97
    ext_segs[2] = &obj->btf_ext->core_relo_info;
3219
388
    for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) {
3220
291
      struct btf_ext_info *seg = ext_segs[seg_num];
3221
291
      const struct btf_ext_info_sec *sec;
3222
291
      const char *sec_name;
3223
291
      Elf_Scn *scn;
3224
3225
291
      if (seg->sec_cnt == 0)
3226
188
        continue;
3227
3228
103
      seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));
3229
103
      if (!seg->sec_idxs) {
3230
0
        err = -ENOMEM;
3231
0
        goto out;
3232
0
      }
3233
3234
103
      sec_num = 0;
3235
403
      for_each_btf_ext_sec(seg, sec) {
3236
        /* preventively increment index to avoid doing
3237
         * this before every continue below
3238
         */
3239
403
        sec_num++;
3240
3241
403
        sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
3242
403
        if (str_is_empty(sec_name))
3243
194
          continue;
3244
209
        scn = elf_sec_by_name(obj, sec_name);
3245
209
        if (!scn)
3246
191
          continue;
3247
3248
18
        seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);
3249
18
      }
3250
103
    }
3251
97
  }
3252
6.54k
out:
3253
6.54k
  if (err && libbpf_needs_btf(obj)) {
3254
60
    pr_warn("BTF is required, but is missing or corrupted.\n");
3255
60
    return err;
3256
60
  }
3257
6.48k
  return 0;
3258
6.54k
}
3259
3260
static int compare_vsi_off(const void *_a, const void *_b)
3261
1.49k
{
3262
1.49k
  const struct btf_var_secinfo *a = _a;
3263
1.49k
  const struct btf_var_secinfo *b = _b;
3264
3265
1.49k
  return a->offset - b->offset;
3266
1.49k
}
3267
3268
static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
3269
           struct btf_type *t)
3270
3.48k
{
3271
3.48k
  __u32 size = 0, i, vars = btf_vlen(t);
3272
3.48k
  const char *sec_name = btf__name_by_offset(btf, t->name_off);
3273
3.48k
  struct btf_var_secinfo *vsi;
3274
3.48k
  bool fixup_offsets = false;
3275
3.48k
  int err;
3276
3277
3.48k
  if (!sec_name) {
3278
0
    pr_debug("No name found in string section for DATASEC kind.\n");
3279
0
    return -ENOENT;
3280
0
  }
3281
3282
  /* Extern-backing datasecs (.ksyms, .kconfig) have their size and
3283
   * variable offsets set at the previous step. Further, not every
3284
   * extern BTF VAR has corresponding ELF symbol preserved, so we skip
3285
   * all fixups altogether for such sections and go straight to sorting
3286
   * VARs within their DATASEC.
3287
   */
3288
3.48k
  if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0)
3289
434
    goto sort_vars;
3290
3291
  /* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to
3292
   * fix this up. But BPF static linker already fixes this up and fills
3293
   * all the sizes and offsets during static linking. So this step has
3294
   * to be optional. But the STV_HIDDEN handling is non-optional for any
3295
   * non-extern DATASEC, so the variable fixup loop below handles both
3296
   * functions at the same time, paying the cost of BTF VAR <-> ELF
3297
   * symbol matching just once.
3298
   */
3299
3.05k
  if (t->size == 0) {
3300
401
    err = find_elf_sec_sz(obj, sec_name, &size);
3301
401
    if (err || !size) {
3302
143
      pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n",
3303
143
         sec_name, size, err);
3304
143
      return -ENOENT;
3305
143
    }
3306
3307
258
    t->size = size;
3308
258
    fixup_offsets = true;
3309
258
  }
3310
3311
4.58k
  for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) {
3312
2.05k
    const struct btf_type *t_var;
3313
2.05k
    struct btf_var *var;
3314
2.05k
    const char *var_name;
3315
2.05k
    Elf64_Sym *sym;
3316
3317
2.05k
    t_var = btf__type_by_id(btf, vsi->type);
3318
2.05k
    if (!t_var || !btf_is_var(t_var)) {
3319
118
      pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);
3320
118
      return -EINVAL;
3321
118
    }
3322
3323
1.93k
    var = btf_var(t_var);
3324
1.93k
    if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)
3325
183
      continue;
3326
3327
1.75k
    var_name = btf__name_by_offset(btf, t_var->name_off);
3328
1.75k
    if (!var_name) {
3329
0
      pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n",
3330
0
         sec_name, i);
3331
0
      return -ENOENT;
3332
0
    }
3333
3334
1.75k
    sym = find_elf_var_sym(obj, var_name);
3335
1.75k
    if (IS_ERR(sym)) {
3336
257
      pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n",
3337
257
         sec_name, var_name);
3338
257
      return -ENOENT;
3339
257
    }
3340
3341
1.49k
    if (fixup_offsets)
3342
69
      vsi->offset = sym->st_value;
3343
3344
    /* if variable is a global/weak symbol, but has restricted
3345
     * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR
3346
     * as static. This follows similar logic for functions (BPF
3347
     * subprogs) and influences libbpf's further decisions about
3348
     * whether to make global data BPF array maps as
3349
     * BPF_F_MMAPABLE.
3350
     */
3351
1.49k
    if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
3352
1.49k
        || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)
3353
66
      var->linkage = BTF_VAR_STATIC;
3354
1.49k
  }
3355
3356
2.96k
sort_vars:
3357
2.96k
  qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
3358
2.96k
  return 0;
3359
2.90k
}
3360
3361
static int bpf_object_fixup_btf(struct bpf_object *obj)
3362
3.72k
{
3363
3.72k
  int i, n, err = 0;
3364
3365
3.72k
  if (!obj->btf)
3366
1.51k
    return 0;
3367
3368
2.21k
  n = btf__type_cnt(obj->btf);
3369
24.0k
  for (i = 1; i < n; i++) {
3370
22.3k
    struct btf_type *t = btf_type_by_id(obj->btf, i);
3371
3372
    /* Loader needs to fix up some of the things compiler
3373
     * couldn't get its hands on while emitting BTF. This
3374
     * is section size and global variable offset. We use
3375
     * the info from the ELF itself for this purpose.
3376
     */
3377
22.3k
    if (btf_is_datasec(t)) {
3378
3.48k
      err = btf_fixup_datasec(obj, obj->btf, t);
3379
3.48k
      if (err)
3380
518
        return err;
3381
3.48k
    }
3382
22.3k
  }
3383
3384
1.69k
  return 0;
3385
2.21k
}
3386
3387
static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
3388
0
{
3389
0
  if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3390
0
      prog->type == BPF_PROG_TYPE_LSM)
3391
0
    return true;
3392
3393
  /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs
3394
   * also need vmlinux BTF
3395
   */
3396
0
  if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3397
0
    return true;
3398
3399
0
  return false;
3400
0
}
3401
3402
static bool map_needs_vmlinux_btf(struct bpf_map *map)
3403
0
{
3404
0
  return bpf_map__is_struct_ops(map);
3405
0
}
3406
3407
static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
3408
0
{
3409
0
  struct bpf_program *prog;
3410
0
  struct bpf_map *map;
3411
0
  int i;
3412
3413
  /* CO-RE relocations need kernel BTF, only when btf_custom_path
3414
   * is not specified
3415
   */
3416
0
  if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3417
0
    return true;
3418
3419
  /* Support for typed ksyms needs kernel BTF */
3420
0
  for (i = 0; i < obj->nr_extern; i++) {
3421
0
    const struct extern_desc *ext;
3422
3423
0
    ext = &obj->externs[i];
3424
0
    if (ext->type == EXT_KSYM && ext->ksym.type_id)
3425
0
      return true;
3426
0
  }
3427
3428
0
  bpf_object__for_each_program(prog, obj) {
3429
0
    if (!prog->autoload)
3430
0
      continue;
3431
0
    if (prog_needs_vmlinux_btf(prog))
3432
0
      return true;
3433
0
  }
3434
3435
0
  bpf_object__for_each_map(map, obj) {
3436
0
    if (map_needs_vmlinux_btf(map))
3437
0
      return true;
3438
0
  }
3439
3440
0
  return false;
3441
0
}
3442
3443
static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
3444
0
{
3445
0
  int err;
3446
3447
  /* btf_vmlinux could be loaded earlier */
3448
0
  if (obj->btf_vmlinux || obj->gen_loader)
3449
0
    return 0;
3450
3451
0
  if (!force && !obj_needs_vmlinux_btf(obj))
3452
0
    return 0;
3453
3454
0
  obj->btf_vmlinux = btf__load_vmlinux_btf();
3455
0
  err = libbpf_get_error(obj->btf_vmlinux);
3456
0
  if (err) {
3457
0
    pr_warn("Error loading vmlinux BTF: %d\n", err);
3458
0
    obj->btf_vmlinux = NULL;
3459
0
    return err;
3460
0
  }
3461
0
  return 0;
3462
0
}
3463
3464
static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
3465
0
{
3466
0
  struct btf *kern_btf = obj->btf;
3467
0
  bool btf_mandatory, sanitize;
3468
0
  int i, err = 0;
3469
3470
0
  if (!obj->btf)
3471
0
    return 0;
3472
3473
0
  if (!kernel_supports(obj, FEAT_BTF)) {
3474
0
    if (kernel_needs_btf(obj)) {
3475
0
      err = -EOPNOTSUPP;
3476
0
      goto report;
3477
0
    }
3478
0
    pr_debug("Kernel doesn't support BTF, skipping uploading it.\n");
3479
0
    return 0;
3480
0
  }
3481
3482
  /* Even though some subprogs are global/weak, user might prefer more
3483
   * permissive BPF verification process that BPF verifier performs for
3484
   * static functions, taking into account more context from the caller
3485
   * functions. In such case, they need to mark such subprogs with
3486
   * __attribute__((visibility("hidden"))) and libbpf will adjust
3487
   * corresponding FUNC BTF type to be marked as static and trigger more
3488
   * involved BPF verification process.
3489
   */
3490
0
  for (i = 0; i < obj->nr_programs; i++) {
3491
0
    struct bpf_program *prog = &obj->programs[i];
3492
0
    struct btf_type *t;
3493
0
    const char *name;
3494
0
    int j, n;
3495
3496
0
    if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3497
0
      continue;
3498
3499
0
    n = btf__type_cnt(obj->btf);
3500
0
    for (j = 1; j < n; j++) {
3501
0
      t = btf_type_by_id(obj->btf, j);
3502
0
      if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL)
3503
0
        continue;
3504
3505
0
      name = btf__str_by_offset(obj->btf, t->name_off);
3506
0
      if (strcmp(name, prog->name) != 0)
3507
0
        continue;
3508
3509
0
      t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);
3510
0
      break;
3511
0
    }
3512
0
  }
3513
3514
0
  sanitize = btf_needs_sanitization(obj);
3515
0
  if (sanitize) {
3516
0
    const void *raw_data;
3517
0
    __u32 sz;
3518
3519
    /* clone BTF to sanitize a copy and leave the original intact */
3520
0
    raw_data = btf__raw_data(obj->btf, &sz);
3521
0
    kern_btf = btf__new(raw_data, sz);
3522
0
    err = libbpf_get_error(kern_btf);
3523
0
    if (err)
3524
0
      return err;
3525
3526
    /* enforce 8-byte pointers for BPF-targeted BTFs */
3527
0
    btf__set_pointer_size(obj->btf, 8);
3528
0
    err = bpf_object__sanitize_btf(obj, kern_btf);
3529
0
    if (err)
3530
0
      return err;
3531
0
  }
3532
3533
0
  if (obj->gen_loader) {
3534
0
    __u32 raw_size = 0;
3535
0
    const void *raw_data = btf__raw_data(kern_btf, &raw_size);
3536
3537
0
    if (!raw_data)
3538
0
      return -ENOMEM;
3539
0
    bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3540
    /* Pretend to have valid FD to pass various fd >= 0 checks.
3541
     * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.
3542
     */
3543
0
    btf__set_fd(kern_btf, 0);
3544
0
  } else {
3545
    /* currently BPF_BTF_LOAD only supports log_level 1 */
3546
0
    err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3547
0
             obj->log_level ? 1 : 0, obj->token_fd);
3548
0
  }
3549
0
  if (sanitize) {
3550
0
    if (!err) {
3551
      /* move fd to libbpf's BTF */
3552
0
      btf__set_fd(obj->btf, btf__fd(kern_btf));
3553
0
      btf__set_fd(kern_btf, -1);
3554
0
    }
3555
0
    btf__free(kern_btf);
3556
0
  }
3557
0
report:
3558
0
  if (err) {
3559
0
    btf_mandatory = kernel_needs_btf(obj);
3560
0
    pr_warn("Error loading .BTF into kernel: %d. %s\n", err,
3561
0
      btf_mandatory ? "BTF is mandatory, can't proceed."
3562
0
              : "BTF is optional, ignoring.");
3563
0
    if (!btf_mandatory)
3564
0
      err = 0;
3565
0
  }
3566
0
  return err;
3567
0
}
3568
3569
static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
3570
31.0k
{
3571
31.0k
  const char *name;
3572
3573
31.0k
  name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3574
31.0k
  if (!name) {
3575
7.94k
    pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3576
7.94k
      off, obj->path, elf_errmsg(-1));
3577
7.94k
    return NULL;
3578
7.94k
  }
3579
3580
23.0k
  return name;
3581
31.0k
}
3582
3583
static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
3584
50.1k
{
3585
50.1k
  const char *name;
3586
3587
50.1k
  name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3588
50.1k
  if (!name) {
3589
859
    pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n",
3590
859
      off, obj->path, elf_errmsg(-1));
3591
859
    return NULL;
3592
859
  }
3593
3594
49.3k
  return name;
3595
50.1k
}
3596
3597
static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
3598
13.9k
{
3599
13.9k
  Elf_Scn *scn;
3600
3601
13.9k
  scn = elf_getscn(obj->efile.elf, idx);
3602
13.9k
  if (!scn) {
3603
0
    pr_warn("elf: failed to get section(%zu) from %s: %s\n",
3604
0
      idx, obj->path, elf_errmsg(-1));
3605
0
    return NULL;
3606
0
  }
3607
13.9k
  return scn;
3608
13.9k
}
3609
3610
static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
3611
610
{
3612
610
  Elf_Scn *scn = NULL;
3613
610
  Elf *elf = obj->efile.elf;
3614
610
  const char *sec_name;
3615
3616
4.38k
  while ((scn = elf_nextscn(elf, scn)) != NULL) {
3617
4.05k
    sec_name = elf_sec_name(obj, scn);
3618
4.05k
    if (!sec_name)
3619
0
      return NULL;
3620
3621
4.05k
    if (strcmp(sec_name, name) != 0)
3622
3.77k
      continue;
3623
3624
279
    return scn;
3625
4.05k
  }
3626
331
  return NULL;
3627
610
}
3628
3629
static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn)
3630
103k
{
3631
103k
  Elf64_Shdr *shdr;
3632
3633
103k
  if (!scn)
3634
0
    return NULL;
3635
3636
103k
  shdr = elf64_getshdr(scn);
3637
103k
  if (!shdr) {
3638
0
    pr_warn("elf: failed to get section(%zu) header from %s: %s\n",
3639
0
      elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3640
0
    return NULL;
3641
0
  }
3642
3643
103k
  return shdr;
3644
103k
}
3645
3646
static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
3647
8.81k
{
3648
8.81k
  const char *name;
3649
8.81k
  Elf64_Shdr *sh;
3650
3651
8.81k
  if (!scn)
3652
0
    return NULL;
3653
3654
8.81k
  sh = elf_sec_hdr(obj, scn);
3655
8.81k
  if (!sh)
3656
0
    return NULL;
3657
3658
8.81k
  name = elf_sec_str(obj, sh->sh_name);
3659
8.81k
  if (!name) {
3660
777
    pr_warn("elf: failed to get section(%zu) name from %s: %s\n",
3661
777
      elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3662
777
    return NULL;
3663
777
  }
3664
3665
8.03k
  return name;
3666
8.81k
}
3667
3668
static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
3669
42.3k
{
3670
42.3k
  Elf_Data *data;
3671
3672
42.3k
  if (!scn)
3673
140
    return NULL;
3674
3675
42.2k
  data = elf_getdata(scn, 0);
3676
42.2k
  if (!data) {
3677
431
    pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n",
3678
431
      elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
3679
431
      obj->path, elf_errmsg(-1));
3680
431
    return NULL;
3681
431
  }
3682
3683
41.8k
  return data;
3684
42.2k
}
3685
3686
static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx)
3687
565k
{
3688
565k
  if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3689
126
    return NULL;
3690
3691
565k
  return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3692
565k
}
3693
3694
static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx)
3695
7.66k
{
3696
7.66k
  if (idx >= data->d_size / sizeof(Elf64_Rel))
3697
0
    return NULL;
3698
3699
7.66k
  return (Elf64_Rel *)data->d_buf + idx;
3700
7.66k
}
3701
3702
static bool is_sec_name_dwarf(const char *name)
3703
36.7k
{
3704
  /* approximation, but the actual list is too long */
3705
36.7k
  return str_has_pfx(name, ".debug_");
3706
36.7k
}
3707
3708
static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name)
3709
40.4k
{
3710
  /* no special handling of .strtab */
3711
40.4k
  if (hdr->sh_type == SHT_STRTAB)
3712
5.54k
    return true;
3713
3714
  /* ignore .llvm_addrsig section as well */
3715
34.8k
  if (hdr->sh_type == SHT_LLVM_ADDRSIG)
3716
41
    return true;
3717
3718
  /* no subprograms will lead to an empty .text section, ignore it */
3719
34.8k
  if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&
3720
34.8k
      strcmp(name, ".text") == 0)
3721
77
    return true;
3722
3723
  /* DWARF sections */
3724
34.7k
  if (is_sec_name_dwarf(name))
3725
2.90k
    return true;
3726
3727
31.8k
  if (str_has_pfx(name, ".rel")) {
3728
1.97k
    name += sizeof(".rel") - 1;
3729
    /* DWARF section relocations */
3730
1.97k
    if (is_sec_name_dwarf(name))
3731
207
      return true;
3732
3733
    /* .BTF and .BTF.ext don't need relocations */
3734
1.77k
    if (strcmp(name, BTF_ELF_SEC) == 0 ||
3735
1.77k
        strcmp(name, BTF_EXT_ELF_SEC) == 0)
3736
453
      return true;
3737
1.77k
  }
3738
3739
31.2k
  return false;
3740
31.8k
}
3741
3742
static int cmp_progs(const void *_a, const void *_b)
3743
72.8k
{
3744
72.8k
  const struct bpf_program *a = _a;
3745
72.8k
  const struct bpf_program *b = _b;
3746
3747
72.8k
  if (a->sec_idx != b->sec_idx)
3748
513
    return a->sec_idx < b->sec_idx ? -1 : 1;
3749
3750
  /* sec_insn_off can't be the same within the section */
3751
72.3k
  return a->sec_insn_off < b->sec_insn_off ? -1 : 1;
3752
72.8k
}
3753
3754
static int bpf_object__elf_collect(struct bpf_object *obj)
3755
8.79k
{
3756
8.79k
  struct elf_sec_desc *sec_desc;
3757
8.79k
  Elf *elf = obj->efile.elf;
3758
8.79k
  Elf_Data *btf_ext_data = NULL;
3759
8.79k
  Elf_Data *btf_data = NULL;
3760
8.79k
  int idx = 0, err = 0;
3761
8.79k
  const char *name;
3762
8.79k
  Elf_Data *data;
3763
8.79k
  Elf_Scn *scn;
3764
8.79k
  Elf64_Shdr *sh;
3765
3766
  /* ELF section indices are 0-based, but sec #0 is special "invalid"
3767
   * section. Since section count retrieved by elf_getshdrnum() does
3768
   * include sec #0, it is already the necessary size of an array to keep
3769
   * all the sections.
3770
   */
3771
8.79k
  if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3772
0
    pr_warn("elf: failed to get the number of sections for %s: %s\n",
3773
0
      obj->path, elf_errmsg(-1));
3774
0
    return -LIBBPF_ERRNO__FORMAT;
3775
0
  }
3776
8.79k
  obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3777
8.79k
  if (!obj->efile.secs)
3778
0
    return -ENOMEM;
3779
3780
  /* a bunch of ELF parsing functionality depends on processing symbols,
3781
   * so do the first pass and find the symbol table
3782
   */
3783
8.79k
  scn = NULL;
3784
53.9k
  while ((scn = elf_nextscn(elf, scn)) != NULL) {
3785
45.3k
    sh = elf_sec_hdr(obj, scn);
3786
45.3k
    if (!sh)
3787
0
      return -LIBBPF_ERRNO__FORMAT;
3788
3789
45.3k
    if (sh->sh_type == SHT_SYMTAB) {
3790
8.77k
      if (obj->efile.symbols) {
3791
1
        pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3792
1
        return -LIBBPF_ERRNO__FORMAT;
3793
1
      }
3794
3795
8.77k
      data = elf_sec_data(obj, scn);
3796
8.77k
      if (!data)
3797
239
        return -LIBBPF_ERRNO__FORMAT;
3798
3799
8.53k
      idx = elf_ndxscn(scn);
3800
3801
8.53k
      obj->efile.symbols = data;
3802
8.53k
      obj->efile.symbols_shndx = idx;
3803
8.53k
      obj->efile.strtabidx = sh->sh_link;
3804
8.53k
    }
3805
45.3k
  }
3806
3807
8.55k
  if (!obj->efile.symbols) {
3808
21
    pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n",
3809
21
      obj->path);
3810
21
    return -ENOENT;
3811
21
  }
3812
3813
8.53k
  scn = NULL;
3814
47.9k
  while ((scn = elf_nextscn(elf, scn)) != NULL) {
3815
40.5k
    idx = elf_ndxscn(scn);
3816
40.5k
    sec_desc = &obj->efile.secs[idx];
3817
3818
40.5k
    sh = elf_sec_hdr(obj, scn);
3819
40.5k
    if (!sh)
3820
0
      return -LIBBPF_ERRNO__FORMAT;
3821
3822
40.5k
    name = elf_sec_str(obj, sh->sh_name);
3823
40.5k
    if (!name)
3824
82
      return -LIBBPF_ERRNO__FORMAT;
3825
3826
40.4k
    if (ignore_elf_section(sh, name))
3827
9.22k
      continue;
3828
3829
31.2k
    data = elf_sec_data(obj, scn);
3830
31.2k
    if (!data)
3831
175
      return -LIBBPF_ERRNO__FORMAT;
3832
3833
31.0k
    pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n",
3834
31.0k
       idx, name, (unsigned long)data->d_size,
3835
31.0k
       (int)sh->sh_link, (unsigned long)sh->sh_flags,
3836
31.0k
       (int)sh->sh_type);
3837
3838
31.0k
    if (strcmp(name, "license") == 0) {
3839
611
      err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3840
611
      if (err)
3841
2
        return err;
3842
30.4k
    } else if (strcmp(name, "version") == 0) {
3843
87
      err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3844
87
      if (err)
3845
17
        return err;
3846
30.3k
    } else if (strcmp(name, "maps") == 0) {
3847
7
      pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n");
3848
7
      return -ENOTSUP;
3849
30.3k
    } else if (strcmp(name, MAPS_ELF_SEC) == 0) {
3850
1.63k
      obj->efile.btf_maps_shndx = idx;
3851
28.7k
    } else if (strcmp(name, BTF_ELF_SEC) == 0) {
3852
5.01k
      if (sh->sh_type != SHT_PROGBITS)
3853
62
        return -LIBBPF_ERRNO__FORMAT;
3854
4.94k
      btf_data = data;
3855
23.6k
    } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) {
3856
452
      if (sh->sh_type != SHT_PROGBITS)
3857
38
        return -LIBBPF_ERRNO__FORMAT;
3858
414
      btf_ext_data = data;
3859
23.2k
    } else if (sh->sh_type == SHT_SYMTAB) {
3860
      /* already processed during the first pass above */
3861
16.0k
    } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {
3862
3.35k
      if (sh->sh_flags & SHF_EXECINSTR) {
3863
1.12k
        if (strcmp(name, ".text") == 0)
3864
226
          obj->efile.text_shndx = idx;
3865
1.12k
        err = bpf_object__add_programs(obj, data, name, idx);
3866
1.12k
        if (err)
3867
247
          return err;
3868
2.22k
      } else if (strcmp(name, DATA_SEC) == 0 ||
3869
2.22k
           str_has_pfx(name, DATA_SEC ".")) {
3870
519
        sec_desc->sec_type = SEC_DATA;
3871
519
        sec_desc->shdr = sh;
3872
519
        sec_desc->data = data;
3873
1.70k
      } else if (strcmp(name, RODATA_SEC) == 0 ||
3874
1.70k
           str_has_pfx(name, RODATA_SEC ".")) {
3875
373
        sec_desc->sec_type = SEC_RODATA;
3876
373
        sec_desc->shdr = sh;
3877
373
        sec_desc->data = data;
3878
1.33k
      } else if (strcmp(name, STRUCT_OPS_SEC) == 0 ||
3879
1.33k
           strcmp(name, STRUCT_OPS_LINK_SEC) == 0 ||
3880
1.33k
           strcmp(name, "?" STRUCT_OPS_SEC) == 0 ||
3881
1.33k
           strcmp(name, "?" STRUCT_OPS_LINK_SEC) == 0) {
3882
491
        sec_desc->sec_type = SEC_ST_OPS;
3883
491
        sec_desc->shdr = sh;
3884
491
        sec_desc->data = data;
3885
491
        obj->efile.has_st_ops = true;
3886
839
      } else if (strcmp(name, ARENA_SEC) == 0) {
3887
78
        obj->efile.arena_data = data;
3888
78
        obj->efile.arena_data_shndx = idx;
3889
761
      } else {
3890
761
        pr_info("elf: skipping unrecognized data section(%d) %s\n",
3891
761
          idx, name);
3892
761
      }
3893
12.6k
    } else if (sh->sh_type == SHT_REL) {
3894
2.13k
      int targ_sec_idx = sh->sh_info; /* points to other section */
3895
3896
2.13k
      if (sh->sh_entsize != sizeof(Elf64_Rel) ||
3897
2.13k
          targ_sec_idx >= obj->efile.sec_cnt)
3898
163
        return -LIBBPF_ERRNO__FORMAT;
3899
3900
      /* Only do relo for section with exec instructions */
3901
1.97k
      if (!section_have_execinstr(obj, targ_sec_idx) &&
3902
1.97k
          strcmp(name, ".rel" STRUCT_OPS_SEC) &&
3903
1.97k
          strcmp(name, ".rel" STRUCT_OPS_LINK_SEC) &&
3904
1.97k
          strcmp(name, ".rel?" STRUCT_OPS_SEC) &&
3905
1.97k
          strcmp(name, ".rel?" STRUCT_OPS_LINK_SEC) &&
3906
1.97k
          strcmp(name, ".rel" MAPS_ELF_SEC)) {
3907
566
        pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n",
3908
566
          idx, name, targ_sec_idx,
3909
566
          elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");
3910
566
        continue;
3911
566
      }
3912
3913
1.40k
      sec_desc->sec_type = SEC_RELO;
3914
1.40k
      sec_desc->shdr = sh;
3915
1.40k
      sec_desc->data = data;
3916
10.5k
    } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||
3917
2.18k
               str_has_pfx(name, BSS_SEC "."))) {
3918
1.35k
      sec_desc->sec_type = SEC_BSS;
3919
1.35k
      sec_desc->shdr = sh;
3920
1.35k
      sec_desc->data = data;
3921
9.19k
    } else {
3922
9.19k
      pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name,
3923
9.19k
        (size_t)sh->sh_size);
3924
9.19k
    }
3925
31.0k
  }
3926
3927
7.64k
  if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3928
1.20k
    pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3929
1.20k
    return -LIBBPF_ERRNO__FORMAT;
3930
1.20k
  }
3931
3932
  /* sort BPF programs by section name and in-section instruction offset
3933
   * for faster search
3934
   */
3935
6.23k
  if (obj->nr_programs)
3936
569
    qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3937
3938
6.23k
  return bpf_object__init_btf(obj, btf_data, btf_ext_data);
3939
7.43k
}
3940
3941
static bool sym_is_extern(const Elf64_Sym *sym)
3942
358k
{
3943
358k
  int bind = ELF64_ST_BIND(sym->st_info);
3944
  /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */
3945
358k
  return sym->st_shndx == SHN_UNDEF &&
3946
358k
         (bind == STB_GLOBAL || bind == STB_WEAK) &&
3947
358k
         ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;
3948
358k
}
3949
3950
static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx)
3951
1.43k
{
3952
1.43k
  int bind = ELF64_ST_BIND(sym->st_info);
3953
1.43k
  int type = ELF64_ST_TYPE(sym->st_info);
3954
3955
  /* in .text section */
3956
1.43k
  if (sym->st_shndx != text_shndx)
3957
485
    return false;
3958
3959
  /* local function */
3960
954
  if (bind == STB_LOCAL && type == STT_SECTION)
3961
647
    return true;
3962
3963
  /* global function */
3964
307
  return bind == STB_GLOBAL && type == STT_FUNC;
3965
954
}
3966
3967
static int find_extern_btf_id(const struct btf *btf, const char *ext_name)
3968
4.06k
{
3969
4.06k
  const struct btf_type *t;
3970
4.06k
  const char *tname;
3971
4.06k
  int i, n;
3972
3973
4.06k
  if (!btf)
3974
38
    return -ESRCH;
3975
3976
4.02k
  n = btf__type_cnt(btf);
3977
30.3k
  for (i = 1; i < n; i++) {
3978
30.1k
    t = btf__type_by_id(btf, i);
3979
3980
30.1k
    if (!btf_is_var(t) && !btf_is_func(t))
3981
22.4k
      continue;
3982
3983
7.71k
    tname = btf__name_by_offset(btf, t->name_off);
3984
7.71k
    if (strcmp(tname, ext_name))
3985
3.83k
      continue;
3986
3987
3.87k
    if (btf_is_var(t) &&
3988
3.87k
        btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)
3989
53
      return -EINVAL;
3990
3991
3.81k
    if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN)
3992
12
      return -EINVAL;
3993
3994
3.80k
    return i;
3995
3.81k
  }
3996
3997
153
  return -ENOENT;
3998
4.02k
}
3999
4000
3.80k
static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) {
4001
3.80k
  const struct btf_var_secinfo *vs;
4002
3.80k
  const struct btf_type *t;
4003
3.80k
  int i, j, n;
4004
4005
3.80k
  if (!btf)
4006
0
    return -ESRCH;
4007
4008
3.80k
  n = btf__type_cnt(btf);
4009
22.6k
  for (i = 1; i < n; i++) {
4010
22.5k
    t = btf__type_by_id(btf, i);
4011
4012
22.5k
    if (!btf_is_datasec(t))
4013
15.0k
      continue;
4014
4015
7.48k
    vs = btf_var_secinfos(t);
4016
15.3k
    for (j = 0; j < btf_vlen(t); j++, vs++) {
4017
11.6k
      if (vs->type == ext_btf_id)
4018
3.77k
        return i;
4019
11.6k
    }
4020
7.48k
  }
4021
4022
35
  return -ENOENT;
4023
3.80k
}
4024
4025
static enum kcfg_type find_kcfg_type(const struct btf *btf, int id,
4026
             bool *is_signed)
4027
1.03k
{
4028
1.03k
  const struct btf_type *t;
4029
1.03k
  const char *name;
4030
4031
1.03k
  t = skip_mods_and_typedefs(btf, id, NULL);
4032
1.03k
  name = btf__name_by_offset(btf, t->name_off);
4033
4034
1.03k
  if (is_signed)
4035
970
    *is_signed = false;
4036
1.03k
  switch (btf_kind(t)) {
4037
666
  case BTF_KIND_INT: {
4038
666
    int enc = btf_int_encoding(t);
4039
4040
666
    if (enc & BTF_INT_BOOL)
4041
257
      return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;
4042
409
    if (is_signed)
4043
371
      *is_signed = enc & BTF_INT_SIGNED;
4044
409
    if (t->size == 1)
4045
189
      return KCFG_CHAR;
4046
220
    if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))
4047
36
      return KCFG_UNKNOWN;
4048
184
    return KCFG_INT;
4049
220
  }
4050
163
  case BTF_KIND_ENUM:
4051
163
    if (t->size != 4)
4052
34
      return KCFG_UNKNOWN;
4053
129
    if (strcmp(name, "libbpf_tristate"))
4054
119
      return KCFG_UNKNOWN;
4055
10
    return KCFG_TRISTATE;
4056
97
  case BTF_KIND_ENUM64:
4057
97
    if (strcmp(name, "libbpf_tristate"))
4058
87
      return KCFG_UNKNOWN;
4059
10
    return KCFG_TRISTATE;
4060
63
  case BTF_KIND_ARRAY:
4061
63
    if (btf_array(t)->nelems == 0)
4062
0
      return KCFG_UNKNOWN;
4063
63
    if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)
4064
33
      return KCFG_UNKNOWN;
4065
30
    return KCFG_CHAR_ARR;
4066
44
  default:
4067
44
    return KCFG_UNKNOWN;
4068
1.03k
  }
4069
1.03k
}
4070
4071
static int cmp_externs(const void *_a, const void *_b)
4072
6.39k
{
4073
6.39k
  const struct extern_desc *a = _a;
4074
6.39k
  const struct extern_desc *b = _b;
4075
4076
6.39k
  if (a->type != b->type)
4077
0
    return a->type < b->type ? -1 : 1;
4078
4079
6.39k
  if (a->type == EXT_KCFG) {
4080
    /* descending order by alignment requirements */
4081
994
    if (a->kcfg.align != b->kcfg.align)
4082
0
      return a->kcfg.align > b->kcfg.align ? -1 : 1;
4083
    /* ascending order by size, within same alignment class */
4084
994
    if (a->kcfg.sz != b->kcfg.sz)
4085
0
      return a->kcfg.sz < b->kcfg.sz ? -1 : 1;
4086
994
  }
4087
4088
  /* resolve ties by name */
4089
6.39k
  return strcmp(a->name, b->name);
4090
6.39k
}
4091
4092
static int find_int_btf_id(const struct btf *btf)
4093
916
{
4094
916
  const struct btf_type *t;
4095
916
  int i, n;
4096
4097
916
  n = btf__type_cnt(btf);
4098
9.84k
  for (i = 1; i < n; i++) {
4099
9.01k
    t = btf__type_by_id(btf, i);
4100
4101
9.01k
    if (btf_is_int(t) && btf_int_bits(t) == 32)
4102
80
      return i;
4103
9.01k
  }
4104
4105
836
  return 0;
4106
916
}
4107
4108
static int add_dummy_ksym_var(struct btf *btf)
4109
5.10k
{
4110
5.10k
  int i, int_btf_id, sec_btf_id, dummy_var_btf_id;
4111
5.10k
  const struct btf_var_secinfo *vs;
4112
5.10k
  const struct btf_type *sec;
4113
4114
5.10k
  if (!btf)
4115
1.54k
    return 0;
4116
4117
3.55k
  sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC,
4118
3.55k
              BTF_KIND_DATASEC);
4119
3.55k
  if (sec_btf_id < 0)
4120
2.81k
    return 0;
4121
4122
743
  sec = btf__type_by_id(btf, sec_btf_id);
4123
743
  vs = btf_var_secinfos(sec);
4124
1.68k
  for (i = 0; i < btf_vlen(sec); i++, vs++) {
4125
1.48k
    const struct btf_type *vt;
4126
4127
1.48k
    vt = btf__type_by_id(btf, vs->type);
4128
1.48k
    if (btf_is_func(vt))
4129
545
      break;
4130
1.48k
  }
4131
4132
  /* No func in ksyms sec.  No need to add dummy var. */
4133
743
  if (i == btf_vlen(sec))
4134
198
    return 0;
4135
4136
545
  int_btf_id = find_int_btf_id(btf);
4137
545
  dummy_var_btf_id = btf__add_var(btf,
4138
545
          "dummy_ksym",
4139
545
          BTF_VAR_GLOBAL_ALLOCATED,
4140
545
          int_btf_id);
4141
545
  if (dummy_var_btf_id < 0)
4142
545
    pr_warn("cannot create a dummy_ksym var\n");
4143
4144
545
  return dummy_var_btf_id;
4145
743
}
4146
4147
static int bpf_object__collect_externs(struct bpf_object *obj)
4148
6.48k
{
4149
6.48k
  struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL;
4150
6.48k
  const struct btf_type *t;
4151
6.48k
  struct extern_desc *ext;
4152
6.48k
  int i, n, off, dummy_var_btf_id;
4153
6.48k
  const char *ext_name, *sec_name;
4154
6.48k
  size_t ext_essent_len;
4155
6.48k
  Elf_Scn *scn;
4156
6.48k
  Elf64_Shdr *sh;
4157
4158
6.48k
  if (!obj->efile.symbols)
4159
0
    return 0;
4160
4161
6.48k
  scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
4162
6.48k
  sh = elf_sec_hdr(obj, scn);
4163
6.48k
  if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))
4164
1.37k
    return -LIBBPF_ERRNO__FORMAT;
4165
4166
5.10k
  dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
4167
5.10k
  if (dummy_var_btf_id < 0)
4168
0
    return dummy_var_btf_id;
4169
4170
5.10k
  n = sh->sh_size / sh->sh_entsize;
4171
5.10k
  pr_debug("looking for externs among %d symbols...\n", n);
4172
4173
361k
  for (i = 0; i < n; i++) {
4174
357k
    Elf64_Sym *sym = elf_sym_by_idx(obj, i);
4175
4176
357k
    if (!sym)
4177
0
      return -LIBBPF_ERRNO__FORMAT;
4178
357k
    if (!sym_is_extern(sym))
4179
349k
      continue;
4180
7.09k
    ext_name = elf_sym_str(obj, sym->st_name);
4181
7.09k
    if (!ext_name || !ext_name[0])
4182
3.03k
      continue;
4183
4184
4.06k
    ext = obj->externs;
4185
4.06k
    ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
4186
4.06k
    if (!ext)
4187
0
      return -ENOMEM;
4188
4.06k
    obj->externs = ext;
4189
4.06k
    ext = &ext[obj->nr_extern];
4190
4.06k
    memset(ext, 0, sizeof(*ext));
4191
4.06k
    obj->nr_extern++;
4192
4193
4.06k
    ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
4194
4.06k
    if (ext->btf_id <= 0) {
4195
256
      pr_warn("failed to find BTF for extern '%s': %d\n",
4196
256
        ext_name, ext->btf_id);
4197
256
      return ext->btf_id;
4198
256
    }
4199
3.80k
    t = btf__type_by_id(obj->btf, ext->btf_id);
4200
3.80k
    ext->name = btf__name_by_offset(obj->btf, t->name_off);
4201
3.80k
    ext->sym_idx = i;
4202
3.80k
    ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;
4203
4204
3.80k
    ext_essent_len = bpf_core_essential_name_len(ext->name);
4205
3.80k
    ext->essent_name = NULL;
4206
3.80k
    if (ext_essent_len != strlen(ext->name)) {
4207
246
      ext->essent_name = strndup(ext->name, ext_essent_len);
4208
246
      if (!ext->essent_name)
4209
0
        return -ENOMEM;
4210
246
    }
4211
4212
3.80k
    ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
4213
3.80k
    if (ext->sec_btf_id <= 0) {
4214
35
      pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n",
4215
35
        ext_name, ext->btf_id, ext->sec_btf_id);
4216
35
      return ext->sec_btf_id;
4217
35
    }
4218
3.77k
    sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
4219
3.77k
    sec_name = btf__name_by_offset(obj->btf, sec->name_off);
4220
4221
3.77k
    if (strcmp(sec_name, KCONFIG_SEC) == 0) {
4222
1.18k
      if (btf_is_func(t)) {
4223
1
        pr_warn("extern function %s is unsupported under %s section\n",
4224
1
          ext->name, KCONFIG_SEC);
4225
1
        return -ENOTSUP;
4226
1
      }
4227
1.18k
      kcfg_sec = sec;
4228
1.18k
      ext->type = EXT_KCFG;
4229
1.18k
      ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
4230
1.18k
      if (ext->kcfg.sz <= 0) {
4231
170
        pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n",
4232
170
          ext_name, ext->kcfg.sz);
4233
170
        return ext->kcfg.sz;
4234
170
      }
4235
1.01k
      ext->kcfg.align = btf__align_of(obj->btf, t->type);
4236
1.01k
      if (ext->kcfg.align <= 0) {
4237
40
        pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n",
4238
40
          ext_name, ext->kcfg.align);
4239
40
        return -EINVAL;
4240
40
      }
4241
970
      ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
4242
970
              &ext->kcfg.is_signed);
4243
970
      if (ext->kcfg.type == KCFG_UNKNOWN) {
4244
348
        pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name);
4245
348
        return -ENOTSUP;
4246
348
      }
4247
2.59k
    } else if (strcmp(sec_name, KSYMS_SEC) == 0) {
4248
2.48k
      ksym_sec = sec;
4249
2.48k
      ext->type = EXT_KSYM;
4250
2.48k
      skip_mods_and_typedefs(obj->btf, t->type,
4251
2.48k
                 &ext->ksym.type_id);
4252
2.48k
    } else {
4253
107
      pr_warn("unrecognized extern section '%s'\n", sec_name);
4254
107
      return -ENOTSUP;
4255
107
    }
4256
3.77k
  }
4257
4.14k
  pr_debug("collected %d externs total\n", obj->nr_extern);
4258
4259
4.14k
  if (!obj->nr_extern)
4260
3.59k
    return 0;
4261
4262
  /* sort externs by type, for kcfg ones also by (align, size, name) */
4263
555
  qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
4264
4265
  /* for .ksyms section, we need to turn all externs into allocated
4266
   * variables in BTF to pass kernel verification; we do this by
4267
   * pretending that each extern is a 8-byte variable
4268
   */
4269
555
  if (ksym_sec) {
4270
    /* find existing 4-byte integer type in BTF to use for fake
4271
     * extern variables in DATASEC
4272
     */
4273
371
    int int_btf_id = find_int_btf_id(obj->btf);
4274
    /* For extern function, a dummy_var added earlier
4275
     * will be used to replace the vs->type and
4276
     * its name string will be used to refill
4277
     * the missing param's name.
4278
     */
4279
371
    const struct btf_type *dummy_var;
4280
4281
371
    dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
4282
2.69k
    for (i = 0; i < obj->nr_extern; i++) {
4283
2.32k
      ext = &obj->externs[i];
4284
2.32k
      if (ext->type != EXT_KSYM)
4285
0
        continue;
4286
2.32k
      pr_debug("extern (ksym) #%d: symbol %d, name %s\n",
4287
2.32k
         i, ext->sym_idx, ext->name);
4288
2.32k
    }
4289
4290
371
    sec = ksym_sec;
4291
371
    n = btf_vlen(sec);
4292
779
    for (i = 0, off = 0; i < n; i++, off += sizeof(int)) {
4293
731
      struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
4294
731
      struct btf_type *vt;
4295
4296
731
      vt = (void *)btf__type_by_id(obj->btf, vs->type);
4297
731
      ext_name = btf__name_by_offset(obj->btf, vt->name_off);
4298
731
      ext = find_extern_by_name(obj, ext_name);
4299
731
      if (!ext) {
4300
323
        pr_warn("failed to find extern definition for BTF %s '%s'\n",
4301
323
          btf_kind_str(vt), ext_name);
4302
323
        return -ESRCH;
4303
323
      }
4304
408
      if (btf_is_func(vt)) {
4305
63
        const struct btf_type *func_proto;
4306
63
        struct btf_param *param;
4307
63
        int j;
4308
4309
63
        func_proto = btf__type_by_id(obj->btf,
4310
63
                   vt->type);
4311
63
        param = btf_params(func_proto);
4312
        /* Reuse the dummy_var string if the
4313
         * func proto does not have param name.
4314
         */
4315
156
        for (j = 0; j < btf_vlen(func_proto); j++)
4316
93
          if (param[j].type && !param[j].name_off)
4317
23
            param[j].name_off =
4318
23
              dummy_var->name_off;
4319
63
        vs->type = dummy_var_btf_id;
4320
63
        vt->info &= ~0xffff;
4321
63
        vt->info |= BTF_FUNC_GLOBAL;
4322
345
      } else {
4323
345
        btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4324
345
        vt->type = int_btf_id;
4325
345
      }
4326
408
      vs->offset = off;
4327
408
      vs->size = sizeof(int);
4328
408
    }
4329
48
    sec->size = off;
4330
48
  }
4331
4332
232
  if (kcfg_sec) {
4333
184
    sec = kcfg_sec;
4334
    /* for kcfg externs calculate their offsets within a .kconfig map */
4335
184
    off = 0;
4336
799
    for (i = 0; i < obj->nr_extern; i++) {
4337
615
      ext = &obj->externs[i];
4338
615
      if (ext->type != EXT_KCFG)
4339
0
        continue;
4340
4341
615
      ext->kcfg.data_off = roundup(off, ext->kcfg.align);
4342
615
      off = ext->kcfg.data_off + ext->kcfg.sz;
4343
615
      pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n",
4344
615
         i, ext->sym_idx, ext->kcfg.data_off, ext->name);
4345
615
    }
4346
184
    sec->size = off;
4347
184
    n = btf_vlen(sec);
4348
234
    for (i = 0; i < n; i++) {
4349
219
      struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i;
4350
4351
219
      t = btf__type_by_id(obj->btf, vs->type);
4352
219
      ext_name = btf__name_by_offset(obj->btf, t->name_off);
4353
219
      ext = find_extern_by_name(obj, ext_name);
4354
219
      if (!ext) {
4355
169
        pr_warn("failed to find extern definition for BTF var '%s'\n",
4356
169
          ext_name);
4357
169
        return -ESRCH;
4358
169
      }
4359
50
      btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;
4360
50
      vs->offset = ext->kcfg.data_off;
4361
50
    }
4362
184
  }
4363
63
  return 0;
4364
232
}
4365
4366
static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
4367
13.4k
{
4368
13.4k
  return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
4369
13.4k
}
4370
4371
struct bpf_program *
4372
bpf_object__find_program_by_name(const struct bpf_object *obj,
4373
         const char *name)
4374
0
{
4375
0
  struct bpf_program *prog;
4376
4377
0
  bpf_object__for_each_program(prog, obj) {
4378
0
    if (prog_is_subprog(obj, prog))
4379
0
      continue;
4380
0
    if (!strcmp(prog->name, name))
4381
0
      return prog;
4382
0
  }
4383
0
  return errno = ENOENT, NULL;
4384
0
}
4385
4386
static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
4387
              int shndx)
4388
401
{
4389
401
  switch (obj->efile.secs[shndx].sec_type) {
4390
90
  case SEC_BSS:
4391
276
  case SEC_DATA:
4392
399
  case SEC_RODATA:
4393
399
    return true;
4394
2
  default:
4395
2
    return false;
4396
401
  }
4397
401
}
4398
4399
static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
4400
              int shndx)
4401
27
{
4402
27
  return shndx == obj->efile.btf_maps_shndx;
4403
27
}
4404
4405
static enum libbpf_map_type
4406
bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
4407
499
{
4408
499
  if (shndx == obj->efile.symbols_shndx)
4409
2
    return LIBBPF_MAP_KCONFIG;
4410
4411
497
  switch (obj->efile.secs[shndx].sec_type) {
4412
90
  case SEC_BSS:
4413
90
    return LIBBPF_MAP_BSS;
4414
186
  case SEC_DATA:
4415
186
    return LIBBPF_MAP_DATA;
4416
123
  case SEC_RODATA:
4417
123
    return LIBBPF_MAP_RODATA;
4418
98
  default:
4419
98
    return LIBBPF_MAP_UNSPEC;
4420
497
  }
4421
497
}
4422
4423
static int bpf_program__record_reloc(struct bpf_program *prog,
4424
             struct reloc_desc *reloc_desc,
4425
             __u32 insn_idx, const char *sym_name,
4426
             const Elf64_Sym *sym, const Elf64_Rel *rel)
4427
1.78k
{
4428
1.78k
  struct bpf_insn *insn = &prog->insns[insn_idx];
4429
1.78k
  size_t map_idx, nr_maps = prog->obj->nr_maps;
4430
1.78k
  struct bpf_object *obj = prog->obj;
4431
1.78k
  __u32 shdr_idx = sym->st_shndx;
4432
1.78k
  enum libbpf_map_type type;
4433
1.78k
  const char *sym_sec_name;
4434
1.78k
  struct bpf_map *map;
4435
4436
1.78k
  if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) {
4437
38
    pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
4438
38
      prog->name, sym_name, insn_idx, insn->code);
4439
38
    return -LIBBPF_ERRNO__RELOC;
4440
38
  }
4441
4442
1.74k
  if (sym_is_extern(sym)) {
4443
1
    int sym_idx = ELF64_R_SYM(rel->r_info);
4444
1
    int i, n = obj->nr_extern;
4445
1
    struct extern_desc *ext;
4446
4447
1
    for (i = 0; i < n; i++) {
4448
0
      ext = &obj->externs[i];
4449
0
      if (ext->sym_idx == sym_idx)
4450
0
        break;
4451
0
    }
4452
1
    if (i >= n) {
4453
1
      pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
4454
1
        prog->name, sym_name, sym_idx);
4455
1
      return -LIBBPF_ERRNO__RELOC;
4456
1
    }
4457
0
    pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
4458
0
       prog->name, i, ext->name, ext->sym_idx, insn_idx);
4459
0
    if (insn->code == (BPF_JMP | BPF_CALL))
4460
0
      reloc_desc->type = RELO_EXTERN_CALL;
4461
0
    else
4462
0
      reloc_desc->type = RELO_EXTERN_LD64;
4463
0
    reloc_desc->insn_idx = insn_idx;
4464
0
    reloc_desc->ext_idx = i;
4465
0
    return 0;
4466
1
  }
4467
4468
  /* sub-program call relocation */
4469
1.74k
  if (is_call_insn(insn)) {
4470
298
    if (insn->src_reg != BPF_PSEUDO_CALL) {
4471
3
      pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4472
3
      return -LIBBPF_ERRNO__RELOC;
4473
3
    }
4474
    /* text_shndx can be 0, if no default "main" program exists */
4475
295
    if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4476
10
      sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4477
10
      pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
4478
10
        prog->name, sym_name, sym_sec_name);
4479
10
      return -LIBBPF_ERRNO__RELOC;
4480
10
    }
4481
285
    if (sym->st_value % BPF_INSN_SZ) {
4482
3
      pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
4483
3
        prog->name, sym_name, (size_t)sym->st_value);
4484
3
      return -LIBBPF_ERRNO__RELOC;
4485
3
    }
4486
282
    reloc_desc->type = RELO_CALL;
4487
282
    reloc_desc->insn_idx = insn_idx;
4488
282
    reloc_desc->sym_off = sym->st_value;
4489
282
    return 0;
4490
285
  }
4491
4492
1.44k
  if (!shdr_idx || shdr_idx >= SHN_LORESERVE) {
4493
4
    pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
4494
4
      prog->name, sym_name, shdr_idx);
4495
4
    return -LIBBPF_ERRNO__RELOC;
4496
4
  }
4497
4498
  /* loading subprog addresses */
4499
1.43k
  if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4500
    /* global_func: sym->st_value = offset in the section, insn->imm = 0.
4501
     * local_func: sym->st_value = 0, insn->imm = offset in the section.
4502
     */
4503
940
    if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {
4504
6
      pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4505
6
        prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4506
6
      return -LIBBPF_ERRNO__RELOC;
4507
6
    }
4508
4509
934
    reloc_desc->type = RELO_SUBPROG_ADDR;
4510
934
    reloc_desc->insn_idx = insn_idx;
4511
934
    reloc_desc->sym_off = sym->st_value;
4512
934
    return 0;
4513
940
  }
4514
4515
499
  type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
4516
499
  sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4517
4518
  /* arena data relocation */
4519
499
  if (shdr_idx == obj->efile.arena_data_shndx) {
4520
71
    reloc_desc->type = RELO_DATA;
4521
71
    reloc_desc->insn_idx = insn_idx;
4522
71
    reloc_desc->map_idx = obj->arena_map - obj->maps;
4523
71
    reloc_desc->sym_off = sym->st_value;
4524
71
    return 0;
4525
71
  }
4526
4527
  /* generic map reference relocation */
4528
428
  if (type == LIBBPF_MAP_UNSPEC) {
4529
27
    if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
4530
27
      pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4531
27
        prog->name, sym_name, sym_sec_name);
4532
27
      return -LIBBPF_ERRNO__RELOC;
4533
27
    }
4534
0
    for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4535
0
      map = &obj->maps[map_idx];
4536
0
      if (map->libbpf_type != type ||
4537
0
          map->sec_idx != sym->st_shndx ||
4538
0
          map->sec_offset != sym->st_value)
4539
0
        continue;
4540
0
      pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
4541
0
         prog->name, map_idx, map->name, map->sec_idx,
4542
0
         map->sec_offset, insn_idx);
4543
0
      break;
4544
0
    }
4545
0
    if (map_idx >= nr_maps) {
4546
0
      pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4547
0
        prog->name, sym_sec_name, (size_t)sym->st_value);
4548
0
      return -LIBBPF_ERRNO__RELOC;
4549
0
    }
4550
0
    reloc_desc->type = RELO_LD64;
4551
0
    reloc_desc->insn_idx = insn_idx;
4552
0
    reloc_desc->map_idx = map_idx;
4553
0
    reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */
4554
0
    return 0;
4555
0
  }
4556
4557
  /* global data map relocation */
4558
401
  if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
4559
2
    pr_warn("prog '%s': bad data relo against section '%s'\n",
4560
2
      prog->name, sym_sec_name);
4561
2
    return -LIBBPF_ERRNO__RELOC;
4562
2
  }
4563
582
  for (map_idx = 0; map_idx < nr_maps; map_idx++) {
4564
580
    map = &obj->maps[map_idx];
4565
580
    if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)
4566
183
      continue;
4567
397
    pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
4568
397
       prog->name, map_idx, map->name, map->sec_idx,
4569
397
       map->sec_offset, insn_idx);
4570
397
    break;
4571
580
  }
4572
399
  if (map_idx >= nr_maps) {
4573
2
    pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4574
2
      prog->name, sym_sec_name);
4575
2
    return -LIBBPF_ERRNO__RELOC;
4576
2
  }
4577
4578
397
  reloc_desc->type = RELO_DATA;
4579
397
  reloc_desc->insn_idx = insn_idx;
4580
397
  reloc_desc->map_idx = map_idx;
4581
397
  reloc_desc->sym_off = sym->st_value;
4582
397
  return 0;
4583
399
}
4584
4585
static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4586
2.50k
{
4587
2.50k
  return insn_idx >= prog->sec_insn_off &&
4588
2.50k
         insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4589
2.50k
}
4590
4591
static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
4592
             size_t sec_idx, size_t insn_idx)
4593
7.38k
{
4594
7.38k
  int l = 0, r = obj->nr_programs - 1, m;
4595
7.38k
  struct bpf_program *prog;
4596
4597
7.38k
  if (!obj->nr_programs)
4598
4.48k
    return NULL;
4599
4600
4.35k
  while (l < r) {
4601
1.46k
    m = l + (r - l + 1) / 2;
4602
1.46k
    prog = &obj->programs[m];
4603
4604
1.46k
    if (prog->sec_idx < sec_idx ||
4605
1.46k
        (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4606
577
      l = m;
4607
883
    else
4608
883
      r = m - 1;
4609
1.46k
  }
4610
  /* matching program could be at index l, but it still might be the
4611
   * wrong one, so we need to double check conditions for the last time
4612
   */
4613
2.89k
  prog = &obj->programs[l];
4614
2.89k
  if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4615
1.78k
    return prog;
4616
1.11k
  return NULL;
4617
2.89k
}
4618
4619
static int
4620
bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)
4621
860
{
4622
860
  const char *relo_sec_name, *sec_name;
4623
860
  size_t sec_idx = shdr->sh_info, sym_idx;
4624
860
  struct bpf_program *prog;
4625
860
  struct reloc_desc *relos;
4626
860
  int err, i, nrels;
4627
860
  const char *sym_name;
4628
860
  __u32 insn_idx;
4629
860
  Elf_Scn *scn;
4630
860
  Elf_Data *scn_data;
4631
860
  Elf64_Sym *sym;
4632
860
  Elf64_Rel *rel;
4633
4634
860
  if (sec_idx >= obj->efile.sec_cnt)
4635
0
    return -EINVAL;
4636
4637
860
  scn = elf_sec_by_idx(obj, sec_idx);
4638
860
  scn_data = elf_sec_data(obj, scn);
4639
860
  if (!scn_data)
4640
16
    return -LIBBPF_ERRNO__FORMAT;
4641
4642
844
  relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4643
844
  sec_name = elf_sec_name(obj, scn);
4644
844
  if (!relo_sec_name || !sec_name)
4645
20
    return -EINVAL;
4646
4647
824
  pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n",
4648
824
     relo_sec_name, sec_idx, sec_name);
4649
824
  nrels = shdr->sh_size / shdr->sh_entsize;
4650
4651
7.66k
  for (i = 0; i < nrels; i++) {
4652
7.66k
    rel = elf_rel_by_idx(data, i);
4653
7.66k
    if (!rel) {
4654
0
      pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i);
4655
0
      return -LIBBPF_ERRNO__FORMAT;
4656
0
    }
4657
4658
7.66k
    sym_idx = ELF64_R_SYM(rel->r_info);
4659
7.66k
    sym = elf_sym_by_idx(obj, sym_idx);
4660
7.66k
    if (!sym) {
4661
126
      pr_warn("sec '%s': symbol #%zu not found for relo #%d\n",
4662
126
        relo_sec_name, sym_idx, i);
4663
126
      return -LIBBPF_ERRNO__FORMAT;
4664
126
    }
4665
4666
7.53k
    if (sym->st_shndx >= obj->efile.sec_cnt) {
4667
28
      pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n",
4668
28
        relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);
4669
28
      return -LIBBPF_ERRNO__FORMAT;
4670
28
    }
4671
4672
7.51k
    if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {
4673
127
      pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n",
4674
127
        relo_sec_name, (size_t)rel->r_offset, i);
4675
127
      return -LIBBPF_ERRNO__FORMAT;
4676
127
    }
4677
4678
7.38k
    insn_idx = rel->r_offset / BPF_INSN_SZ;
4679
    /* relocations against static functions are recorded as
4680
     * relocations against the section that contains a function;
4681
     * in such case, symbol will be STT_SECTION and sym.st_name
4682
     * will point to empty string (0), so fetch section name
4683
     * instead
4684
     */
4685
7.38k
    if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)
4686
544
      sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4687
6.83k
    else
4688
6.83k
      sym_name = elf_sym_str(obj, sym->st_name);
4689
7.38k
    sym_name = sym_name ?: "<?";
4690
4691
7.38k
    pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n",
4692
5.70k
       relo_sec_name, i, insn_idx, sym_name);
4693
4694
5.70k
    prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4695
5.70k
    if (!prog) {
4696
5.60k
      pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n",
4697
5.60k
        relo_sec_name, i, sec_name, insn_idx);
4698
5.60k
      continue;
4699
5.60k
    }
4700
4701
103
    relos = libbpf_reallocarray(prog->reloc_desc,
4702
103
              prog->nr_reloc + 1, sizeof(*relos));
4703
103
    if (!relos)
4704
0
      return -ENOMEM;
4705
103
    prog->reloc_desc = relos;
4706
4707
    /* adjust insn_idx to local BPF program frame of reference */
4708
103
    insn_idx -= prog->sec_insn_off;
4709
103
    err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4710
103
            insn_idx, sym_name, sym, rel);
4711
103
    if (err)
4712
96
      return err;
4713
4714
7
    prog->nr_reloc++;
4715
7
  }
4716
18.4E
  return 0;
4717
824
}
4718
4719
static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map)
4720
1.83k
{
4721
1.83k
  int id;
4722
4723
1.83k
  if (!obj->btf)
4724
1.48k
    return -ENOENT;
4725
4726
  /* if it's BTF-defined map, we don't need to search for type IDs.
4727
   * For struct_ops map, it does not need btf_key_type_id and
4728
   * btf_value_type_id.
4729
   */
4730
353
  if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4731
71
    return 0;
4732
4733
  /*
4734
   * LLVM annotates global data differently in BTF, that is,
4735
   * only as '.data', '.bss' or '.rodata'.
4736
   */
4737
282
  if (!bpf_map__is_internal(map))
4738
0
    return -ENOENT;
4739
4740
282
  id = btf__find_by_name(obj->btf, map->real_name);
4741
282
  if (id < 0)
4742
124
    return id;
4743
4744
158
  map->btf_key_type_id = 0;
4745
158
  map->btf_value_type_id = id;
4746
158
  return 0;
4747
282
}
4748
4749
static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info)
4750
0
{
4751
0
  char file[PATH_MAX], buff[4096];
4752
0
  FILE *fp;
4753
0
  __u32 val;
4754
0
  int err;
4755
4756
0
  snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd);
4757
0
  memset(info, 0, sizeof(*info));
4758
4759
0
  fp = fopen(file, "re");
4760
0
  if (!fp) {
4761
0
    err = -errno;
4762
0
    pr_warn("failed to open %s: %d. No procfs support?\n", file,
4763
0
      err);
4764
0
    return err;
4765
0
  }
4766
4767
0
  while (fgets(buff, sizeof(buff), fp)) {
4768
0
    if (sscanf(buff, "map_type:\t%u", &val) == 1)
4769
0
      info->type = val;
4770
0
    else if (sscanf(buff, "key_size:\t%u", &val) == 1)
4771
0
      info->key_size = val;
4772
0
    else if (sscanf(buff, "value_size:\t%u", &val) == 1)
4773
0
      info->value_size = val;
4774
0
    else if (sscanf(buff, "max_entries:\t%u", &val) == 1)
4775
0
      info->max_entries = val;
4776
0
    else if (sscanf(buff, "map_flags:\t%i", &val) == 1)
4777
0
      info->map_flags = val;
4778
0
  }
4779
4780
0
  fclose(fp);
4781
4782
0
  return 0;
4783
0
}
4784
4785
bool bpf_map__autocreate(const struct bpf_map *map)
4786
0
{
4787
0
  return map->autocreate;
4788
0
}
4789
4790
int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate)
4791
0
{
4792
0
  if (map->obj->loaded)
4793
0
    return libbpf_err(-EBUSY);
4794
4795
0
  map->autocreate = autocreate;
4796
0
  return 0;
4797
0
}
4798
4799
int bpf_map__reuse_fd(struct bpf_map *map, int fd)
4800
0
{
4801
0
  struct bpf_map_info info;
4802
0
  __u32 len = sizeof(info), name_len;
4803
0
  int new_fd, err;
4804
0
  char *new_name;
4805
4806
0
  memset(&info, 0, len);
4807
0
  err = bpf_map_get_info_by_fd(fd, &info, &len);
4808
0
  if (err && errno == EINVAL)
4809
0
    err = bpf_get_map_info_from_fdinfo(fd, &info);
4810
0
  if (err)
4811
0
    return libbpf_err(err);
4812
4813
0
  name_len = strlen(info.name);
4814
0
  if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)
4815
0
    new_name = strdup(map->name);
4816
0
  else
4817
0
    new_name = strdup(info.name);
4818
4819
0
  if (!new_name)
4820
0
    return libbpf_err(-errno);
4821
4822
  /*
4823
   * Like dup(), but make sure new FD is >= 3 and has O_CLOEXEC set.
4824
   * This is similar to what we do in ensure_good_fd(), but without
4825
   * closing original FD.
4826
   */
4827
0
  new_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3);
4828
0
  if (new_fd < 0) {
4829
0
    err = -errno;
4830
0
    goto err_free_new_name;
4831
0
  }
4832
4833
0
  err = reuse_fd(map->fd, new_fd);
4834
0
  if (err)
4835
0
    goto err_free_new_name;
4836
4837
0
  free(map->name);
4838
4839
0
  map->name = new_name;
4840
0
  map->def.type = info.type;
4841
0
  map->def.key_size = info.key_size;
4842
0
  map->def.value_size = info.value_size;
4843
0
  map->def.max_entries = info.max_entries;
4844
0
  map->def.map_flags = info.map_flags;
4845
0
  map->btf_key_type_id = info.btf_key_type_id;
4846
0
  map->btf_value_type_id = info.btf_value_type_id;
4847
0
  map->reused = true;
4848
0
  map->map_extra = info.map_extra;
4849
4850
0
  return 0;
4851
4852
0
err_free_new_name:
4853
0
  free(new_name);
4854
0
  return libbpf_err(err);
4855
0
}
4856
4857
__u32 bpf_map__max_entries(const struct bpf_map *map)
4858
0
{
4859
0
  return map->def.max_entries;
4860
0
}
4861
4862
struct bpf_map *bpf_map__inner_map(struct bpf_map *map)
4863
0
{
4864
0
  if (!bpf_map_type__is_map_in_map(map->def.type))
4865
0
    return errno = EINVAL, NULL;
4866
4867
0
  return map->inner_map;
4868
0
}
4869
4870
int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries)
4871
0
{
4872
0
  if (map->obj->loaded)
4873
0
    return libbpf_err(-EBUSY);
4874
4875
0
  map->def.max_entries = max_entries;
4876
4877
  /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */
4878
0
  if (map_is_ringbuf(map))
4879
0
    map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);
4880
4881
0
  return 0;
4882
0
}
4883
4884
static int bpf_object_prepare_token(struct bpf_object *obj)
4885
0
{
4886
0
  const char *bpffs_path;
4887
0
  int bpffs_fd = -1, token_fd, err;
4888
0
  bool mandatory;
4889
0
  enum libbpf_print_level level;
4890
4891
  /* token is explicitly prevented */
4892
0
  if (obj->token_path && obj->token_path[0] == '\0') {
4893
0
    pr_debug("object '%s': token is prevented, skipping...\n", obj->name);
4894
0
    return 0;
4895
0
  }
4896
4897
0
  mandatory = obj->token_path != NULL;
4898
0
  level = mandatory ? LIBBPF_WARN : LIBBPF_DEBUG;
4899
4900
0
  bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH;
4901
0
  bpffs_fd = open(bpffs_path, O_DIRECTORY, O_RDWR);
4902
0
  if (bpffs_fd < 0) {
4903
0
    err = -errno;
4904
0
    __pr(level, "object '%s': failed (%d) to open BPF FS mount at '%s'%s\n",
4905
0
         obj->name, err, bpffs_path,
4906
0
         mandatory ? "" : ", skipping optional step...");
4907
0
    return mandatory ? err : 0;
4908
0
  }
4909
4910
0
  token_fd = bpf_token_create(bpffs_fd, 0);
4911
0
  close(bpffs_fd);
4912
0
  if (token_fd < 0) {
4913
0
    if (!mandatory && token_fd == -ENOENT) {
4914
0
      pr_debug("object '%s': BPF FS at '%s' doesn't have BPF token delegation set up, skipping...\n",
4915
0
         obj->name, bpffs_path);
4916
0
      return 0;
4917
0
    }
4918
0
    __pr(level, "object '%s': failed (%d) to create BPF token from '%s'%s\n",
4919
0
         obj->name, token_fd, bpffs_path,
4920
0
         mandatory ? "" : ", skipping optional step...");
4921
0
    return mandatory ? token_fd : 0;
4922
0
  }
4923
4924
0
  obj->feat_cache = calloc(1, sizeof(*obj->feat_cache));
4925
0
  if (!obj->feat_cache) {
4926
0
    close(token_fd);
4927
0
    return -ENOMEM;
4928
0
  }
4929
4930
0
  obj->token_fd = token_fd;
4931
0
  obj->feat_cache->token_fd = token_fd;
4932
4933
0
  return 0;
4934
0
}
4935
4936
static int
4937
bpf_object__probe_loading(struct bpf_object *obj)
4938
0
{
4939
0
  char *cp, errmsg[STRERR_BUFSIZE];
4940
0
  struct bpf_insn insns[] = {
4941
0
    BPF_MOV64_IMM(BPF_REG_0, 0),
4942
0
    BPF_EXIT_INSN(),
4943
0
  };
4944
0
  int ret, insn_cnt = ARRAY_SIZE(insns);
4945
0
  LIBBPF_OPTS(bpf_prog_load_opts, opts,
4946
0
    .token_fd = obj->token_fd,
4947
0
    .prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0,
4948
0
  );
4949
4950
0
  if (obj->gen_loader)
4951
0
    return 0;
4952
4953
0
  ret = bump_rlimit_memlock();
4954
0
  if (ret)
4955
0
    pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret);
4956
4957
  /* make sure basic loading works */
4958
0
  ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, &opts);
4959
0
  if (ret < 0)
4960
0
    ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, &opts);
4961
0
  if (ret < 0) {
4962
0
    ret = errno;
4963
0
    cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg));
4964
0
    pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF "
4965
0
      "program. Make sure your kernel supports BPF "
4966
0
      "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is "
4967
0
      "set to big enough value.\n", __func__, cp, ret);
4968
0
    return -ret;
4969
0
  }
4970
0
  close(ret);
4971
4972
0
  return 0;
4973
0
}
4974
4975
bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
4976
0
{
4977
0
  if (obj->gen_loader)
4978
    /* To generate loader program assume the latest kernel
4979
     * to avoid doing extra prog_load, map_create syscalls.
4980
     */
4981
0
    return true;
4982
4983
0
  if (obj->token_fd)
4984
0
    return feat_supported(obj->feat_cache, feat_id);
4985
4986
0
  return feat_supported(NULL, feat_id);
4987
0
}
4988
4989
static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd)
4990
0
{
4991
0
  struct bpf_map_info map_info;
4992
0
  char msg[STRERR_BUFSIZE];
4993
0
  __u32 map_info_len = sizeof(map_info);
4994
0
  int err;
4995
4996
0
  memset(&map_info, 0, map_info_len);
4997
0
  err = bpf_map_get_info_by_fd(map_fd, &map_info, &map_info_len);
4998
0
  if (err && errno == EINVAL)
4999
0
    err = bpf_get_map_info_from_fdinfo(map_fd, &map_info);
5000
0
  if (err) {
5001
0
    pr_warn("failed to get map info for map FD %d: %s\n", map_fd,
5002
0
      libbpf_strerror_r(errno, msg, sizeof(msg)));
5003
0
    return false;
5004
0
  }
5005
5006
0
  return (map_info.type == map->def.type &&
5007
0
    map_info.key_size == map->def.key_size &&
5008
0
    map_info.value_size == map->def.value_size &&
5009
0
    map_info.max_entries == map->def.max_entries &&
5010
0
    map_info.map_flags == map->def.map_flags &&
5011
0
    map_info.map_extra == map->map_extra);
5012
0
}
5013
5014
static int
5015
bpf_object__reuse_map(struct bpf_map *map)
5016
0
{
5017
0
  char *cp, errmsg[STRERR_BUFSIZE];
5018
0
  int err, pin_fd;
5019
5020
0
  pin_fd = bpf_obj_get(map->pin_path);
5021
0
  if (pin_fd < 0) {
5022
0
    err = -errno;
5023
0
    if (err == -ENOENT) {
5024
0
      pr_debug("found no pinned map to reuse at '%s'\n",
5025
0
         map->pin_path);
5026
0
      return 0;
5027
0
    }
5028
5029
0
    cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));
5030
0
    pr_warn("couldn't retrieve pinned map '%s': %s\n",
5031
0
      map->pin_path, cp);
5032
0
    return err;
5033
0
  }
5034
5035
0
  if (!map_is_reuse_compat(map, pin_fd)) {
5036
0
    pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n",
5037
0
      map->pin_path);
5038
0
    close(pin_fd);
5039
0
    return -EINVAL;
5040
0
  }
5041
5042
0
  err = bpf_map__reuse_fd(map, pin_fd);
5043
0
  close(pin_fd);
5044
0
  if (err)
5045
0
    return err;
5046
5047
0
  map->pinned = true;
5048
0
  pr_debug("reused pinned map at '%s'\n", map->pin_path);
5049
5050
0
  return 0;
5051
0
}
5052
5053
static int
5054
bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
5055
0
{
5056
0
  enum libbpf_map_type map_type = map->libbpf_type;
5057
0
  char *cp, errmsg[STRERR_BUFSIZE];
5058
0
  int err, zero = 0;
5059
5060
0
  if (obj->gen_loader) {
5061
0
    bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
5062
0
           map->mmaped, map->def.value_size);
5063
0
    if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG)
5064
0
      bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5065
0
    return 0;
5066
0
  }
5067
5068
0
  err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);
5069
0
  if (err) {
5070
0
    err = -errno;
5071
0
    cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5072
0
    pr_warn("Error setting initial map(%s) contents: %s\n",
5073
0
      map->name, cp);
5074
0
    return err;
5075
0
  }
5076
5077
  /* Freeze .rodata and .kconfig map as read-only from syscall side. */
5078
0
  if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) {
5079
0
    err = bpf_map_freeze(map->fd);
5080
0
    if (err) {
5081
0
      err = -errno;
5082
0
      cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg));
5083
0
      pr_warn("Error freezing map(%s) as read-only: %s\n",
5084
0
        map->name, cp);
5085
0
      return err;
5086
0
    }
5087
0
  }
5088
0
  return 0;
5089
0
}
5090
5091
static void bpf_map__destroy(struct bpf_map *map);
5092
5093
static bool map_is_created(const struct bpf_map *map)
5094
0
{
5095
0
  return map->obj->loaded || map->reused;
5096
0
}
5097
5098
static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
5099
0
{
5100
0
  LIBBPF_OPTS(bpf_map_create_opts, create_attr);
5101
0
  struct bpf_map_def *def = &map->def;
5102
0
  const char *map_name = NULL;
5103
0
  int err = 0, map_fd;
5104
5105
0
  if (kernel_supports(obj, FEAT_PROG_NAME))
5106
0
    map_name = map->name;
5107
0
  create_attr.map_ifindex = map->map_ifindex;
5108
0
  create_attr.map_flags = def->map_flags;
5109
0
  create_attr.numa_node = map->numa_node;
5110
0
  create_attr.map_extra = map->map_extra;
5111
0
  create_attr.token_fd = obj->token_fd;
5112
0
  if (obj->token_fd)
5113
0
    create_attr.map_flags |= BPF_F_TOKEN_FD;
5114
5115
0
  if (bpf_map__is_struct_ops(map)) {
5116
0
    create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;
5117
0
    if (map->mod_btf_fd >= 0) {
5118
0
      create_attr.value_type_btf_obj_fd = map->mod_btf_fd;
5119
0
      create_attr.map_flags |= BPF_F_VTYPE_BTF_OBJ_FD;
5120
0
    }
5121
0
  }
5122
5123
0
  if (obj->btf && btf__fd(obj->btf) >= 0) {
5124
0
    create_attr.btf_fd = btf__fd(obj->btf);
5125
0
    create_attr.btf_key_type_id = map->btf_key_type_id;
5126
0
    create_attr.btf_value_type_id = map->btf_value_type_id;
5127
0
  }
5128
5129
0
  if (bpf_map_type__is_map_in_map(def->type)) {
5130
0
    if (map->inner_map) {
5131
0
      err = map_set_def_max_entries(map->inner_map);
5132
0
      if (err)
5133
0
        return err;
5134
0
      err = bpf_object__create_map(obj, map->inner_map, true);
5135
0
      if (err) {
5136
0
        pr_warn("map '%s': failed to create inner map: %d\n",
5137
0
          map->name, err);
5138
0
        return err;
5139
0
      }
5140
0
      map->inner_map_fd = map->inner_map->fd;
5141
0
    }
5142
0
    if (map->inner_map_fd >= 0)
5143
0
      create_attr.inner_map_fd = map->inner_map_fd;
5144
0
  }
5145
5146
0
  switch (def->type) {
5147
0
  case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
5148
0
  case BPF_MAP_TYPE_CGROUP_ARRAY:
5149
0
  case BPF_MAP_TYPE_STACK_TRACE:
5150
0
  case BPF_MAP_TYPE_ARRAY_OF_MAPS:
5151
0
  case BPF_MAP_TYPE_HASH_OF_MAPS:
5152
0
  case BPF_MAP_TYPE_DEVMAP:
5153
0
  case BPF_MAP_TYPE_DEVMAP_HASH:
5154
0
  case BPF_MAP_TYPE_CPUMAP:
5155
0
  case BPF_MAP_TYPE_XSKMAP:
5156
0
  case BPF_MAP_TYPE_SOCKMAP:
5157
0
  case BPF_MAP_TYPE_SOCKHASH:
5158
0
  case BPF_MAP_TYPE_QUEUE:
5159
0
  case BPF_MAP_TYPE_STACK:
5160
0
  case BPF_MAP_TYPE_ARENA:
5161
0
    create_attr.btf_fd = 0;
5162
0
    create_attr.btf_key_type_id = 0;</