1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com 3 * Copyright (c) 2016 Facebook 4 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io 5 */ 6 #include <uapi/linux/btf.h> 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/slab.h> 10 #include <linux/bpf.h> 11 #include <linux/btf.h> 12 #include <linux/bpf_verifier.h> 13 #include <linux/filter.h> 14 #include <net/netlink.h> 15 #include <linux/file.h> 16 #include <linux/vmalloc.h> 17 #include <linux/stringify.h> 18 #include <linux/bsearch.h> 19 #include <linux/sort.h> 20 #include <linux/perf_event.h> 21 #include <linux/ctype.h> 22 23 #include "disasm.h" 24 25 static const struct bpf_verifier_ops * const bpf_verifier_ops[] = { 26 #define BPF_PROG_TYPE(_id, _name) \ 27 [_id] = & _name ## _verifier_ops, 28 #define BPF_MAP_TYPE(_id, _ops) 29 #include <linux/bpf_types.h> 30 #undef BPF_PROG_TYPE 31 #undef BPF_MAP_TYPE 32 }; 33 34 /* bpf_check() is a static code analyzer that walks eBPF program 35 * instruction by instruction and updates register/stack state. 36 * All paths of conditional branches are analyzed until 'bpf_exit' insn. 37 * 38 * The first pass is depth-first-search to check that the program is a DAG. 39 * It rejects the following programs: 40 * - larger than BPF_MAXINSNS insns 41 * - if loop is present (detected via back-edge) 42 * - unreachable insns exist (shouldn't be a forest. program = one function) 43 * - out of bounds or malformed jumps 44 * The second pass is all possible path descent from the 1st insn. 45 * Since it's analyzing all pathes through the program, the length of the 46 * analysis is limited to 64k insn, which may be hit even if total number of 47 * insn is less then 4K, but there are too many branches that change stack/regs. 48 * Number of 'branches to be analyzed' is limited to 1k 49 * 50 * On entry to each instruction, each register has a type, and the instruction 51 * changes the types of the registers depending on instruction semantics. 52 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is 53 * copied to R1. 54 * 55 * All registers are 64-bit. 56 * R0 - return register 57 * R1-R5 argument passing registers 58 * R6-R9 callee saved registers 59 * R10 - frame pointer read-only 60 * 61 * At the start of BPF program the register R1 contains a pointer to bpf_context 62 * and has type PTR_TO_CTX. 63 * 64 * Verifier tracks arithmetic operations on pointers in case: 65 * BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), 66 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20), 67 * 1st insn copies R10 (which has FRAME_PTR) type into R1 68 * and 2nd arithmetic instruction is pattern matched to recognize 69 * that it wants to construct a pointer to some element within stack. 70 * So after 2nd insn, the register R1 has type PTR_TO_STACK 71 * (and -20 constant is saved for further stack bounds checking). 72 * Meaning that this reg is a pointer to stack plus known immediate constant. 73 * 74 * Most of the time the registers have SCALAR_VALUE type, which 75 * means the register has some value, but it's not a valid pointer. 76 * (like pointer plus pointer becomes SCALAR_VALUE type) 77 * 78 * When verifier sees load or store instructions the type of base register 79 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are 80 * four pointer types recognized by check_mem_access() function. 81 * 82 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value' 83 * and the range of [ptr, ptr + map's value_size) is accessible. 84 * 85 * registers used to pass values to function calls are checked against 86 * function argument constraints. 87 * 88 * ARG_PTR_TO_MAP_KEY is one of such argument constraints. 89 * It means that the register type passed to this function must be 90 * PTR_TO_STACK and it will be used inside the function as 91 * 'pointer to map element key' 92 * 93 * For example the argument constraints for bpf_map_lookup_elem(): 94 * .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 95 * .arg1_type = ARG_CONST_MAP_PTR, 96 * .arg2_type = ARG_PTR_TO_MAP_KEY, 97 * 98 * ret_type says that this function returns 'pointer to map elem value or null' 99 * function expects 1st argument to be a const pointer to 'struct bpf_map' and 100 * 2nd argument should be a pointer to stack, which will be used inside 101 * the helper function as a pointer to map element key. 102 * 103 * On the kernel side the helper function looks like: 104 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5) 105 * { 106 * struct bpf_map *map = (struct bpf_map *) (unsigned long) r1; 107 * void *key = (void *) (unsigned long) r2; 108 * void *value; 109 * 110 * here kernel can access 'key' and 'map' pointers safely, knowing that 111 * [key, key + map->key_size) bytes are valid and were initialized on 112 * the stack of eBPF program. 113 * } 114 * 115 * Corresponding eBPF program may look like: 116 * BPF_MOV64_REG(BPF_REG_2, BPF_REG_10), // after this insn R2 type is FRAME_PTR 117 * BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK 118 * BPF_LD_MAP_FD(BPF_REG_1, map_fd), // after this insn R1 type is CONST_PTR_TO_MAP 119 * BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem), 120 * here verifier looks at prototype of map_lookup_elem() and sees: 121 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok, 122 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes 123 * 124 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far, 125 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits 126 * and were initialized prior to this call. 127 * If it's ok, then verifier allows this BPF_CALL insn and looks at 128 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets 129 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function 130 * returns ether pointer to map value or NULL. 131 * 132 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off' 133 * insn, the register holding that pointer in the true branch changes state to 134 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false 135 * branch. See check_cond_jmp_op(). 136 * 137 * After the call R0 is set to return type of the function and registers R1-R5 138 * are set to NOT_INIT to indicate that they are no longer readable. 139 * 140 * The following reference types represent a potential reference to a kernel 141 * resource which, after first being allocated, must be checked and freed by 142 * the BPF program: 143 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET 144 * 145 * When the verifier sees a helper call return a reference type, it allocates a 146 * pointer id for the reference and stores it in the current function state. 147 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into 148 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type 149 * passes through a NULL-check conditional. For the branch wherein the state is 150 * changed to CONST_IMM, the verifier releases the reference. 151 * 152 * For each helper function that allocates a reference, such as 153 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as 154 * bpf_sk_release(). When a reference type passes into the release function, 155 * the verifier also releases the reference. If any unchecked or unreleased 156 * reference remains at the end of the program, the verifier rejects it. 157 */ 158 159 /* verifier_state + insn_idx are pushed to stack when branch is encountered */ 160 struct bpf_verifier_stack_elem { 161 /* verifer state is 'st' 162 * before processing instruction 'insn_idx' 163 * and after processing instruction 'prev_insn_idx' 164 */ 165 struct bpf_verifier_state st; 166 int insn_idx; 167 int prev_insn_idx; 168 struct bpf_verifier_stack_elem *next; 169 }; 170 171 #define BPF_COMPLEXITY_LIMIT_STACK 1024 172 #define BPF_COMPLEXITY_LIMIT_STATES 64 173 174 #define BPF_MAP_PTR_UNPRIV 1UL 175 #define BPF_MAP_PTR_POISON ((void *)((0xeB9FUL << 1) + \ 176 POISON_POINTER_DELTA)) 177 #define BPF_MAP_PTR(X) ((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV)) 178 179 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) 180 { 181 return BPF_MAP_PTR(aux->map_state) == BPF_MAP_PTR_POISON; 182 } 183 184 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) 185 { 186 return aux->map_state & BPF_MAP_PTR_UNPRIV; 187 } 188 189 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, 190 const struct bpf_map *map, bool unpriv) 191 { 192 BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV); 193 unpriv |= bpf_map_ptr_unpriv(aux); 194 aux->map_state = (unsigned long)map | 195 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL); 196 } 197 198 struct bpf_call_arg_meta { 199 struct bpf_map *map_ptr; 200 bool raw_mode; 201 bool pkt_access; 202 int regno; 203 int access_size; 204 s64 msize_smax_value; 205 u64 msize_umax_value; 206 int ref_obj_id; 207 int func_id; 208 }; 209 210 static DEFINE_MUTEX(bpf_verifier_lock); 211 212 static const struct bpf_line_info * 213 find_linfo(const struct bpf_verifier_env *env, u32 insn_off) 214 { 215 const struct bpf_line_info *linfo; 216 const struct bpf_prog *prog; 217 u32 i, nr_linfo; 218 219 prog = env->prog; 220 nr_linfo = prog->aux->nr_linfo; 221 222 if (!nr_linfo || insn_off >= prog->len) 223 return NULL; 224 225 linfo = prog->aux->linfo; 226 for (i = 1; i < nr_linfo; i++) 227 if (insn_off < linfo[i].insn_off) 228 break; 229 230 return &linfo[i - 1]; 231 } 232 233 void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, 234 va_list args) 235 { 236 unsigned int n; 237 238 n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args); 239 240 WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1, 241 "verifier log line truncated - local buffer too short\n"); 242 243 n = min(log->len_total - log->len_used - 1, n); 244 log->kbuf[n] = '\0'; 245 246 if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1)) 247 log->len_used += n; 248 else 249 log->ubuf = NULL; 250 } 251 252 /* log_level controls verbosity level of eBPF verifier. 253 * bpf_verifier_log_write() is used to dump the verification trace to the log, 254 * so the user can figure out what's wrong with the program 255 */ 256 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, 257 const char *fmt, ...) 258 { 259 va_list args; 260 261 if (!bpf_verifier_log_needed(&env->log)) 262 return; 263 264 va_start(args, fmt); 265 bpf_verifier_vlog(&env->log, fmt, args); 266 va_end(args); 267 } 268 EXPORT_SYMBOL_GPL(bpf_verifier_log_write); 269 270 __printf(2, 3) static void verbose(void *private_data, const char *fmt, ...) 271 { 272 struct bpf_verifier_env *env = private_data; 273 va_list args; 274 275 if (!bpf_verifier_log_needed(&env->log)) 276 return; 277 278 va_start(args, fmt); 279 bpf_verifier_vlog(&env->log, fmt, args); 280 va_end(args); 281 } 282 283 static const char *ltrim(const char *s) 284 { 285 while (isspace(*s)) 286 s++; 287 288 return s; 289 } 290 291 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, 292 u32 insn_off, 293 const char *prefix_fmt, ...) 294 { 295 const struct bpf_line_info *linfo; 296 297 if (!bpf_verifier_log_needed(&env->log)) 298 return; 299 300 linfo = find_linfo(env, insn_off); 301 if (!linfo || linfo == env->prev_linfo) 302 return; 303 304 if (prefix_fmt) { 305 va_list args; 306 307 va_start(args, prefix_fmt); 308 bpf_verifier_vlog(&env->log, prefix_fmt, args); 309 va_end(args); 310 } 311 312 verbose(env, "%s\n", 313 ltrim(btf_name_by_offset(env->prog->aux->btf, 314 linfo->line_off))); 315 316 env->prev_linfo = linfo; 317 } 318 319 static bool type_is_pkt_pointer(enum bpf_reg_type type) 320 { 321 return type == PTR_TO_PACKET || 322 type == PTR_TO_PACKET_META; 323 } 324 325 static bool type_is_sk_pointer(enum bpf_reg_type type) 326 { 327 return type == PTR_TO_SOCKET || 328 type == PTR_TO_SOCK_COMMON || 329 type == PTR_TO_TCP_SOCK; 330 } 331 332 static bool reg_type_may_be_null(enum bpf_reg_type type) 333 { 334 return type == PTR_TO_MAP_VALUE_OR_NULL || 335 type == PTR_TO_SOCKET_OR_NULL || 336 type == PTR_TO_SOCK_COMMON_OR_NULL || 337 type == PTR_TO_TCP_SOCK_OR_NULL; 338 } 339 340 static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg) 341 { 342 return reg->type == PTR_TO_MAP_VALUE && 343 map_value_has_spin_lock(reg->map_ptr); 344 } 345 346 static bool reg_type_may_be_refcounted_or_null(enum bpf_reg_type type) 347 { 348 return type == PTR_TO_SOCKET || 349 type == PTR_TO_SOCKET_OR_NULL || 350 type == PTR_TO_TCP_SOCK || 351 type == PTR_TO_TCP_SOCK_OR_NULL; 352 } 353 354 static bool arg_type_may_be_refcounted(enum bpf_arg_type type) 355 { 356 return type == ARG_PTR_TO_SOCK_COMMON; 357 } 358 359 /* Determine whether the function releases some resources allocated by another 360 * function call. The first reference type argument will be assumed to be 361 * released by release_reference(). 362 */ 363 static bool is_release_function(enum bpf_func_id func_id) 364 { 365 return func_id == BPF_FUNC_sk_release; 366 } 367 368 static bool is_acquire_function(enum bpf_func_id func_id) 369 { 370 return func_id == BPF_FUNC_sk_lookup_tcp || 371 func_id == BPF_FUNC_sk_lookup_udp || 372 func_id == BPF_FUNC_skc_lookup_tcp; 373 } 374 375 static bool is_ptr_cast_function(enum bpf_func_id func_id) 376 { 377 return func_id == BPF_FUNC_tcp_sock || 378 func_id == BPF_FUNC_sk_fullsock; 379 } 380 381 /* string representation of 'enum bpf_reg_type' */ 382 static const char * const reg_type_str[] = { 383 [NOT_INIT] = "?", 384 [SCALAR_VALUE] = "inv", 385 [PTR_TO_CTX] = "ctx", 386 [CONST_PTR_TO_MAP] = "map_ptr", 387 [PTR_TO_MAP_VALUE] = "map_value", 388 [PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null", 389 [PTR_TO_STACK] = "fp", 390 [PTR_TO_PACKET] = "pkt", 391 [PTR_TO_PACKET_META] = "pkt_meta", 392 [PTR_TO_PACKET_END] = "pkt_end", 393 [PTR_TO_FLOW_KEYS] = "flow_keys", 394 [PTR_TO_SOCKET] = "sock", 395 [PTR_TO_SOCKET_OR_NULL] = "sock_or_null", 396 [PTR_TO_SOCK_COMMON] = "sock_common", 397 [PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null", 398 [PTR_TO_TCP_SOCK] = "tcp_sock", 399 [PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null", 400 [PTR_TO_TP_BUFFER] = "tp_buffer", 401 }; 402 403 static char slot_type_char[] = { 404 [STACK_INVALID] = '?', 405 [STACK_SPILL] = 'r', 406 [STACK_MISC] = 'm', 407 [STACK_ZERO] = '0', 408 }; 409 410 static void print_liveness(struct bpf_verifier_env *env, 411 enum bpf_reg_liveness live) 412 { 413 if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE)) 414 verbose(env, "_"); 415 if (live & REG_LIVE_READ) 416 verbose(env, "r"); 417 if (live & REG_LIVE_WRITTEN) 418 verbose(env, "w"); 419 if (live & REG_LIVE_DONE) 420 verbose(env, "D"); 421 } 422 423 static struct bpf_func_state *func(struct bpf_verifier_env *env, 424 const struct bpf_reg_state *reg) 425 { 426 struct bpf_verifier_state *cur = env->cur_state; 427 428 return cur->frame[reg->frameno]; 429 } 430 431 static void print_verifier_state(struct bpf_verifier_env *env, 432 const struct bpf_func_state *state) 433 { 434 const struct bpf_reg_state *reg; 435 enum bpf_reg_type t; 436 int i; 437 438 if (state->frameno) 439 verbose(env, " frame%d:", state->frameno); 440 for (i = 0; i < MAX_BPF_REG; i++) { 441 reg = &state->regs[i]; 442 t = reg->type; 443 if (t == NOT_INIT) 444 continue; 445 verbose(env, " R%d", i); 446 print_liveness(env, reg->live); 447 verbose(env, "=%s", reg_type_str[t]); 448 if ((t == SCALAR_VALUE || t == PTR_TO_STACK) && 449 tnum_is_const(reg->var_off)) { 450 /* reg->off should be 0 for SCALAR_VALUE */ 451 verbose(env, "%lld", reg->var_off.value + reg->off); 452 if (t == PTR_TO_STACK) 453 verbose(env, ",call_%d", func(env, reg)->callsite); 454 } else { 455 verbose(env, "(id=%d", reg->id); 456 if (reg_type_may_be_refcounted_or_null(t)) 457 verbose(env, ",ref_obj_id=%d", reg->ref_obj_id); 458 if (t != SCALAR_VALUE) 459 verbose(env, ",off=%d", reg->off); 460 if (type_is_pkt_pointer(t)) 461 verbose(env, ",r=%d", reg->range); 462 else if (t == CONST_PTR_TO_MAP || 463 t == PTR_TO_MAP_VALUE || 464 t == PTR_TO_MAP_VALUE_OR_NULL) 465 verbose(env, ",ks=%d,vs=%d", 466 reg->map_ptr->key_size, 467 reg->map_ptr->value_size); 468 if (tnum_is_const(reg->var_off)) { 469 /* Typically an immediate SCALAR_VALUE, but 470 * could be a pointer whose offset is too big 471 * for reg->off 472 */ 473 verbose(env, ",imm=%llx", reg->var_off.value); 474 } else { 475 if (reg->smin_value != reg->umin_value && 476 reg->smin_value != S64_MIN) 477 verbose(env, ",smin_value=%lld", 478 (long long)reg->smin_value); 479 if (reg->smax_value != reg->umax_value && 480 reg->smax_value != S64_MAX) 481 verbose(env, ",smax_value=%lld", 482 (long long)reg->smax_value); 483 if (reg->umin_value != 0) 484 verbose(env, ",umin_value=%llu", 485 (unsigned long long)reg->umin_value); 486 if (reg->umax_value != U64_MAX) 487 verbose(env, ",umax_value=%llu", 488 (unsigned long long)reg->umax_value); 489 if (!tnum_is_unknown(reg->var_off)) { 490 char tn_buf[48]; 491 492 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 493 verbose(env, ",var_off=%s", tn_buf); 494 } 495 } 496 verbose(env, ")"); 497 } 498 } 499 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) { 500 char types_buf[BPF_REG_SIZE + 1]; 501 bool valid = false; 502 int j; 503 504 for (j = 0; j < BPF_REG_SIZE; j++) { 505 if (state->stack[i].slot_type[j] != STACK_INVALID) 506 valid = true; 507 types_buf[j] = slot_type_char[ 508 state->stack[i].slot_type[j]]; 509 } 510 types_buf[BPF_REG_SIZE] = 0; 511 if (!valid) 512 continue; 513 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE); 514 print_liveness(env, state->stack[i].spilled_ptr.live); 515 if (state->stack[i].slot_type[0] == STACK_SPILL) 516 verbose(env, "=%s", 517 reg_type_str[state->stack[i].spilled_ptr.type]); 518 else 519 verbose(env, "=%s", types_buf); 520 } 521 if (state->acquired_refs && state->refs[0].id) { 522 verbose(env, " refs=%d", state->refs[0].id); 523 for (i = 1; i < state->acquired_refs; i++) 524 if (state->refs[i].id) 525 verbose(env, ",%d", state->refs[i].id); 526 } 527 verbose(env, "\n"); 528 } 529 530 #define COPY_STATE_FN(NAME, COUNT, FIELD, SIZE) \ 531 static int copy_##NAME##_state(struct bpf_func_state *dst, \ 532 const struct bpf_func_state *src) \ 533 { \ 534 if (!src->FIELD) \ 535 return 0; \ 536 if (WARN_ON_ONCE(dst->COUNT < src->COUNT)) { \ 537 /* internal bug, make state invalid to reject the program */ \ 538 memset(dst, 0, sizeof(*dst)); \ 539 return -EFAULT; \ 540 } \ 541 memcpy(dst->FIELD, src->FIELD, \ 542 sizeof(*src->FIELD) * (src->COUNT / SIZE)); \ 543 return 0; \ 544 } 545 /* copy_reference_state() */ 546 COPY_STATE_FN(reference, acquired_refs, refs, 1) 547 /* copy_stack_state() */ 548 COPY_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) 549 #undef COPY_STATE_FN 550 551 #define REALLOC_STATE_FN(NAME, COUNT, FIELD, SIZE) \ 552 static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \ 553 bool copy_old) \ 554 { \ 555 u32 old_size = state->COUNT; \ 556 struct bpf_##NAME##_state *new_##FIELD; \ 557 int slot = size / SIZE; \ 558 \ 559 if (size <= old_size || !size) { \ 560 if (copy_old) \ 561 return 0; \ 562 state->COUNT = slot * SIZE; \ 563 if (!size && old_size) { \ 564 kfree(state->FIELD); \ 565 state->FIELD = NULL; \ 566 } \ 567 return 0; \ 568 } \ 569 new_##FIELD = kmalloc_array(slot, sizeof(struct bpf_##NAME##_state), \ 570 GFP_KERNEL); \ 571 if (!new_##FIELD) \ 572 return -ENOMEM; \ 573 if (copy_old) { \ 574 if (state->FIELD) \ 575 memcpy(new_##FIELD, state->FIELD, \ 576 sizeof(*new_##FIELD) * (old_size / SIZE)); \ 577 memset(new_##FIELD + old_size / SIZE, 0, \ 578 sizeof(*new_##FIELD) * (size - old_size) / SIZE); \ 579 } \ 580 state->COUNT = slot * SIZE; \ 581 kfree(state->FIELD); \ 582 state->FIELD = new_##FIELD; \ 583 return 0; \ 584 } 585 /* realloc_reference_state() */ 586 REALLOC_STATE_FN(reference, acquired_refs, refs, 1) 587 /* realloc_stack_state() */ 588 REALLOC_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE) 589 #undef REALLOC_STATE_FN 590 591 /* do_check() starts with zero-sized stack in struct bpf_verifier_state to 592 * make it consume minimal amount of memory. check_stack_write() access from 593 * the program calls into realloc_func_state() to grow the stack size. 594 * Note there is a non-zero 'parent' pointer inside bpf_verifier_state 595 * which realloc_stack_state() copies over. It points to previous 596 * bpf_verifier_state which is never reallocated. 597 */ 598 static int realloc_func_state(struct bpf_func_state *state, int stack_size, 599 int refs_size, bool copy_old) 600 { 601 int err = realloc_reference_state(state, refs_size, copy_old); 602 if (err) 603 return err; 604 return realloc_stack_state(state, stack_size, copy_old); 605 } 606 607 /* Acquire a pointer id from the env and update the state->refs to include 608 * this new pointer reference. 609 * On success, returns a valid pointer id to associate with the register 610 * On failure, returns a negative errno. 611 */ 612 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx) 613 { 614 struct bpf_func_state *state = cur_func(env); 615 int new_ofs = state->acquired_refs; 616 int id, err; 617 618 err = realloc_reference_state(state, state->acquired_refs + 1, true); 619 if (err) 620 return err; 621 id = ++env->id_gen; 622 state->refs[new_ofs].id = id; 623 state->refs[new_ofs].insn_idx = insn_idx; 624 625 return id; 626 } 627 628 /* release function corresponding to acquire_reference_state(). Idempotent. */ 629 static int release_reference_state(struct bpf_func_state *state, int ptr_id) 630 { 631 int i, last_idx; 632 633 last_idx = state->acquired_refs - 1; 634 for (i = 0; i < state->acquired_refs; i++) { 635 if (state->refs[i].id == ptr_id) { 636 if (last_idx && i != last_idx) 637 memcpy(&state->refs[i], &state->refs[last_idx], 638 sizeof(*state->refs)); 639 memset(&state->refs[last_idx], 0, sizeof(*state->refs)); 640 state->acquired_refs--; 641 return 0; 642 } 643 } 644 return -EINVAL; 645 } 646 647 static int transfer_reference_state(struct bpf_func_state *dst, 648 struct bpf_func_state *src) 649 { 650 int err = realloc_reference_state(dst, src->acquired_refs, false); 651 if (err) 652 return err; 653 err = copy_reference_state(dst, src); 654 if (err) 655 return err; 656 return 0; 657 } 658 659 static void free_func_state(struct bpf_func_state *state) 660 { 661 if (!state) 662 return; 663 kfree(state->refs); 664 kfree(state->stack); 665 kfree(state); 666 } 667 668 static void free_verifier_state(struct bpf_verifier_state *state, 669 bool free_self) 670 { 671 int i; 672 673 for (i = 0; i <= state->curframe; i++) { 674 free_func_state(state->frame[i]); 675 state->frame[i] = NULL; 676 } 677 if (free_self) 678 kfree(state); 679 } 680 681 /* copy verifier state from src to dst growing dst stack space 682 * when necessary to accommodate larger src stack 683 */ 684 static int copy_func_state(struct bpf_func_state *dst, 685 const struct bpf_func_state *src) 686 { 687 int err; 688 689 err = realloc_func_state(dst, src->allocated_stack, src->acquired_refs, 690 false); 691 if (err) 692 return err; 693 memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs)); 694 err = copy_reference_state(dst, src); 695 if (err) 696 return err; 697 return copy_stack_state(dst, src); 698 } 699 700 static int copy_verifier_state(struct bpf_verifier_state *dst_state, 701 const struct bpf_verifier_state *src) 702 { 703 struct bpf_func_state *dst; 704 int i, err; 705 706 /* if dst has more stack frames then src frame, free them */ 707 for (i = src->curframe + 1; i <= dst_state->curframe; i++) { 708 free_func_state(dst_state->frame[i]); 709 dst_state->frame[i] = NULL; 710 } 711 dst_state->speculative = src->speculative; 712 dst_state->curframe = src->curframe; 713 dst_state->active_spin_lock = src->active_spin_lock; 714 for (i = 0; i <= src->curframe; i++) { 715 dst = dst_state->frame[i]; 716 if (!dst) { 717 dst = kzalloc(sizeof(*dst), GFP_KERNEL); 718 if (!dst) 719 return -ENOMEM; 720 dst_state->frame[i] = dst; 721 } 722 err = copy_func_state(dst, src->frame[i]); 723 if (err) 724 return err; 725 } 726 return 0; 727 } 728 729 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, 730 int *insn_idx) 731 { 732 struct bpf_verifier_state *cur = env->cur_state; 733 struct bpf_verifier_stack_elem *elem, *head = env->head; 734 int err; 735 736 if (env->head == NULL) 737 return -ENOENT; 738 739 if (cur) { 740 err = copy_verifier_state(cur, &head->st); 741 if (err) 742 return err; 743 } 744 if (insn_idx) 745 *insn_idx = head->insn_idx; 746 if (prev_insn_idx) 747 *prev_insn_idx = head->prev_insn_idx; 748 elem = head->next; 749 free_verifier_state(&head->st, false); 750 kfree(head); 751 env->head = elem; 752 env->stack_size--; 753 return 0; 754 } 755 756 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, 757 int insn_idx, int prev_insn_idx, 758 bool speculative) 759 { 760 struct bpf_verifier_state *cur = env->cur_state; 761 struct bpf_verifier_stack_elem *elem; 762 int err; 763 764 elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL); 765 if (!elem) 766 goto err; 767 768 elem->insn_idx = insn_idx; 769 elem->prev_insn_idx = prev_insn_idx; 770 elem->next = env->head; 771 env->head = elem; 772 env->stack_size++; 773 err = copy_verifier_state(&elem->st, cur); 774 if (err) 775 goto err; 776 elem->st.speculative |= speculative; 777 if (env->stack_size > BPF_COMPLEXITY_LIMIT_STACK) { 778 verbose(env, "BPF program is too complex\n"); 779 goto err; 780 } 781 return &elem->st; 782 err: 783 free_verifier_state(env->cur_state, true); 784 env->cur_state = NULL; 785 /* pop all elements and return */ 786 while (!pop_stack(env, NULL, NULL)); 787 return NULL; 788 } 789 790 #define CALLER_SAVED_REGS 6 791 static const int caller_saved[CALLER_SAVED_REGS] = { 792 BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5 793 }; 794 795 static void __mark_reg_not_init(struct bpf_reg_state *reg); 796 797 /* Mark the unknown part of a register (variable offset or scalar value) as 798 * known to have the value @imm. 799 */ 800 static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm) 801 { 802 /* Clear id, off, and union(map_ptr, range) */ 803 memset(((u8 *)reg) + sizeof(reg->type), 0, 804 offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type)); 805 reg->var_off = tnum_const(imm); 806 reg->smin_value = (s64)imm; 807 reg->smax_value = (s64)imm; 808 reg->umin_value = imm; 809 reg->umax_value = imm; 810 } 811 812 /* Mark the 'variable offset' part of a register as zero. This should be 813 * used only on registers holding a pointer type. 814 */ 815 static void __mark_reg_known_zero(struct bpf_reg_state *reg) 816 { 817 __mark_reg_known(reg, 0); 818 } 819 820 static void __mark_reg_const_zero(struct bpf_reg_state *reg) 821 { 822 __mark_reg_known(reg, 0); 823 reg->type = SCALAR_VALUE; 824 } 825 826 static void mark_reg_known_zero(struct bpf_verifier_env *env, 827 struct bpf_reg_state *regs, u32 regno) 828 { 829 if (WARN_ON(regno >= MAX_BPF_REG)) { 830 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno); 831 /* Something bad happened, let's kill all regs */ 832 for (regno = 0; regno < MAX_BPF_REG; regno++) 833 __mark_reg_not_init(regs + regno); 834 return; 835 } 836 __mark_reg_known_zero(regs + regno); 837 } 838 839 static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg) 840 { 841 return type_is_pkt_pointer(reg->type); 842 } 843 844 static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg) 845 { 846 return reg_is_pkt_pointer(reg) || 847 reg->type == PTR_TO_PACKET_END; 848 } 849 850 /* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */ 851 static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg, 852 enum bpf_reg_type which) 853 { 854 /* The register can already have a range from prior markings. 855 * This is fine as long as it hasn't been advanced from its 856 * origin. 857 */ 858 return reg->type == which && 859 reg->id == 0 && 860 reg->off == 0 && 861 tnum_equals_const(reg->var_off, 0); 862 } 863 864 /* Attempts to improve min/max values based on var_off information */ 865 static void __update_reg_bounds(struct bpf_reg_state *reg) 866 { 867 /* min signed is max(sign bit) | min(other bits) */ 868 reg->smin_value = max_t(s64, reg->smin_value, 869 reg->var_off.value | (reg->var_off.mask & S64_MIN)); 870 /* max signed is min(sign bit) | max(other bits) */ 871 reg->smax_value = min_t(s64, reg->smax_value, 872 reg->var_off.value | (reg->var_off.mask & S64_MAX)); 873 reg->umin_value = max(reg->umin_value, reg->var_off.value); 874 reg->umax_value = min(reg->umax_value, 875 reg->var_off.value | reg->var_off.mask); 876 } 877 878 /* Uses signed min/max values to inform unsigned, and vice-versa */ 879 static void __reg_deduce_bounds(struct bpf_reg_state *reg) 880 { 881 /* Learn sign from signed bounds. 882 * If we cannot cross the sign boundary, then signed and unsigned bounds 883 * are the same, so combine. This works even in the negative case, e.g. 884 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff. 885 */ 886 if (reg->smin_value >= 0 || reg->smax_value < 0) { 887 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 888 reg->umin_value); 889 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 890 reg->umax_value); 891 return; 892 } 893 /* Learn sign from unsigned bounds. Signed bounds cross the sign 894 * boundary, so we must be careful. 895 */ 896 if ((s64)reg->umax_value >= 0) { 897 /* Positive. We can't learn anything from the smin, but smax 898 * is positive, hence safe. 899 */ 900 reg->smin_value = reg->umin_value; 901 reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value, 902 reg->umax_value); 903 } else if ((s64)reg->umin_value < 0) { 904 /* Negative. We can't learn anything from the smax, but smin 905 * is negative, hence safe. 906 */ 907 reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value, 908 reg->umin_value); 909 reg->smax_value = reg->umax_value; 910 } 911 } 912 913 /* Attempts to improve var_off based on unsigned min/max information */ 914 static void __reg_bound_offset(struct bpf_reg_state *reg) 915 { 916 reg->var_off = tnum_intersect(reg->var_off, 917 tnum_range(reg->umin_value, 918 reg->umax_value)); 919 } 920 921 /* Reset the min/max bounds of a register */ 922 static void __mark_reg_unbounded(struct bpf_reg_state *reg) 923 { 924 reg->smin_value = S64_MIN; 925 reg->smax_value = S64_MAX; 926 reg->umin_value = 0; 927 reg->umax_value = U64_MAX; 928 } 929 930 /* Mark a register as having a completely unknown (scalar) value. */ 931 static void __mark_reg_unknown(struct bpf_reg_state *reg) 932 { 933 /* 934 * Clear type, id, off, and union(map_ptr, range) and 935 * padding between 'type' and union 936 */ 937 memset(reg, 0, offsetof(struct bpf_reg_state, var_off)); 938 reg->type = SCALAR_VALUE; 939 reg->var_off = tnum_unknown; 940 reg->frameno = 0; 941 __mark_reg_unbounded(reg); 942 } 943 944 static void mark_reg_unknown(struct bpf_verifier_env *env, 945 struct bpf_reg_state *regs, u32 regno) 946 { 947 if (WARN_ON(regno >= MAX_BPF_REG)) { 948 verbose(env, "mark_reg_unknown(regs, %u)\n", regno); 949 /* Something bad happened, let's kill all regs except FP */ 950 for (regno = 0; regno < BPF_REG_FP; regno++) 951 __mark_reg_not_init(regs + regno); 952 return; 953 } 954 __mark_reg_unknown(regs + regno); 955 } 956 957 static void __mark_reg_not_init(struct bpf_reg_state *reg) 958 { 959 __mark_reg_unknown(reg); 960 reg->type = NOT_INIT; 961 } 962 963 static void mark_reg_not_init(struct bpf_verifier_env *env, 964 struct bpf_reg_state *regs, u32 regno) 965 { 966 if (WARN_ON(regno >= MAX_BPF_REG)) { 967 verbose(env, "mark_reg_not_init(regs, %u)\n", regno); 968 /* Something bad happened, let's kill all regs except FP */ 969 for (regno = 0; regno < BPF_REG_FP; regno++) 970 __mark_reg_not_init(regs + regno); 971 return; 972 } 973 __mark_reg_not_init(regs + regno); 974 } 975 976 static void init_reg_state(struct bpf_verifier_env *env, 977 struct bpf_func_state *state) 978 { 979 struct bpf_reg_state *regs = state->regs; 980 int i; 981 982 for (i = 0; i < MAX_BPF_REG; i++) { 983 mark_reg_not_init(env, regs, i); 984 regs[i].live = REG_LIVE_NONE; 985 regs[i].parent = NULL; 986 } 987 988 /* frame pointer */ 989 regs[BPF_REG_FP].type = PTR_TO_STACK; 990 mark_reg_known_zero(env, regs, BPF_REG_FP); 991 regs[BPF_REG_FP].frameno = state->frameno; 992 993 /* 1st arg to a function */ 994 regs[BPF_REG_1].type = PTR_TO_CTX; 995 mark_reg_known_zero(env, regs, BPF_REG_1); 996 } 997 998 #define BPF_MAIN_FUNC (-1) 999 static void init_func_state(struct bpf_verifier_env *env, 1000 struct bpf_func_state *state, 1001 int callsite, int frameno, int subprogno) 1002 { 1003 state->callsite = callsite; 1004 state->frameno = frameno; 1005 state->subprogno = subprogno; 1006 init_reg_state(env, state); 1007 } 1008 1009 enum reg_arg_type { 1010 SRC_OP, /* register is used as source operand */ 1011 DST_OP, /* register is used as destination operand */ 1012 DST_OP_NO_MARK /* same as above, check only, don't mark */ 1013 }; 1014 1015 static int cmp_subprogs(const void *a, const void *b) 1016 { 1017 return ((struct bpf_subprog_info *)a)->start - 1018 ((struct bpf_subprog_info *)b)->start; 1019 } 1020 1021 static int find_subprog(struct bpf_verifier_env *env, int off) 1022 { 1023 struct bpf_subprog_info *p; 1024 1025 p = bsearch(&off, env->subprog_info, env->subprog_cnt, 1026 sizeof(env->subprog_info[0]), cmp_subprogs); 1027 if (!p) 1028 return -ENOENT; 1029 return p - env->subprog_info; 1030 1031 } 1032 1033 static int add_subprog(struct bpf_verifier_env *env, int off) 1034 { 1035 int insn_cnt = env->prog->len; 1036 int ret; 1037 1038 if (off >= insn_cnt || off < 0) { 1039 verbose(env, "call to invalid destination\n"); 1040 return -EINVAL; 1041 } 1042 ret = find_subprog(env, off); 1043 if (ret >= 0) 1044 return 0; 1045 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) { 1046 verbose(env, "too many subprograms\n"); 1047 return -E2BIG; 1048 } 1049 env->subprog_info[env->subprog_cnt++].start = off; 1050 sort(env->subprog_info, env->subprog_cnt, 1051 sizeof(env->subprog_info[0]), cmp_subprogs, NULL); 1052 return 0; 1053 } 1054 1055 static int check_subprogs(struct bpf_verifier_env *env) 1056 { 1057 int i, ret, subprog_start, subprog_end, off, cur_subprog = 0; 1058 struct bpf_subprog_info *subprog = env->subprog_info; 1059 struct bpf_insn *insn = env->prog->insnsi; 1060 int insn_cnt = env->prog->len; 1061 1062 /* Add entry function. */ 1063 ret = add_subprog(env, 0); 1064 if (ret < 0) 1065 return ret; 1066 1067 /* determine subprog starts. The end is one before the next starts */ 1068 for (i = 0; i < insn_cnt; i++) { 1069 if (insn[i].code != (BPF_JMP | BPF_CALL)) 1070 continue; 1071 if (insn[i].src_reg != BPF_PSEUDO_CALL) 1072 continue; 1073 if (!env->allow_ptr_leaks) { 1074 verbose(env, "function calls to other bpf functions are allowed for root only\n"); 1075 return -EPERM; 1076 } 1077 ret = add_subprog(env, i + insn[i].imm + 1); 1078 if (ret < 0) 1079 return ret; 1080 } 1081 1082 /* Add a fake 'exit' subprog which could simplify subprog iteration 1083 * logic. 'subprog_cnt' should not be increased. 1084 */ 1085 subprog[env->subprog_cnt].start = insn_cnt; 1086 1087 if (env->log.level & BPF_LOG_LEVEL2) 1088 for (i = 0; i < env->subprog_cnt; i++) 1089 verbose(env, "func#%d @%d\n", i, subprog[i].start); 1090 1091 /* now check that all jumps are within the same subprog */ 1092 subprog_start = subprog[cur_subprog].start; 1093 subprog_end = subprog[cur_subprog + 1].start; 1094 for (i = 0; i < insn_cnt; i++) { 1095 u8 code = insn[i].code; 1096 1097 if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32) 1098 goto next; 1099 if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL) 1100 goto next; 1101 off = i + insn[i].off + 1; 1102 if (off < subprog_start || off >= subprog_end) { 1103 verbose(env, "jump out of range from insn %d to %d\n", i, off); 1104 return -EINVAL; 1105 } 1106 next: 1107 if (i == subprog_end - 1) { 1108 /* to avoid fall-through from one subprog into another 1109 * the last insn of the subprog should be either exit 1110 * or unconditional jump back 1111 */ 1112 if (code != (BPF_JMP | BPF_EXIT) && 1113 code != (BPF_JMP | BPF_JA)) { 1114 verbose(env, "last insn is not an exit or jmp\n"); 1115 return -EINVAL; 1116 } 1117 subprog_start = subprog_end; 1118 cur_subprog++; 1119 if (cur_subprog < env->subprog_cnt) 1120 subprog_end = subprog[cur_subprog + 1].start; 1121 } 1122 } 1123 return 0; 1124 } 1125 1126 /* Parentage chain of this register (or stack slot) should take care of all 1127 * issues like callee-saved registers, stack slot allocation time, etc. 1128 */ 1129 static int mark_reg_read(struct bpf_verifier_env *env, 1130 const struct bpf_reg_state *state, 1131 struct bpf_reg_state *parent) 1132 { 1133 bool writes = parent == state->parent; /* Observe write marks */ 1134 int cnt = 0; 1135 1136 while (parent) { 1137 /* if read wasn't screened by an earlier write ... */ 1138 if (writes && state->live & REG_LIVE_WRITTEN) 1139 break; 1140 if (parent->live & REG_LIVE_DONE) { 1141 verbose(env, "verifier BUG type %s var_off %lld off %d\n", 1142 reg_type_str[parent->type], 1143 parent->var_off.value, parent->off); 1144 return -EFAULT; 1145 } 1146 if (parent->live & REG_LIVE_READ) 1147 /* The parentage chain never changes and 1148 * this parent was already marked as LIVE_READ. 1149 * There is no need to keep walking the chain again and 1150 * keep re-marking all parents as LIVE_READ. 1151 * This case happens when the same register is read 1152 * multiple times without writes into it in-between. 1153 */ 1154 break; 1155 /* ... then we depend on parent's value */ 1156 parent->live |= REG_LIVE_READ; 1157 state = parent; 1158 parent = state->parent; 1159 writes = true; 1160 cnt++; 1161 } 1162 1163 if (env->longest_mark_read_walk < cnt) 1164 env->longest_mark_read_walk = cnt; 1165 return 0; 1166 } 1167 1168 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, 1169 enum reg_arg_type t) 1170 { 1171 struct bpf_verifier_state *vstate = env->cur_state; 1172 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 1173 struct bpf_reg_state *reg, *regs = state->regs; 1174 1175 if (regno >= MAX_BPF_REG) { 1176 verbose(env, "R%d is invalid\n", regno); 1177 return -EINVAL; 1178 } 1179 1180 reg = ®s[regno]; 1181 if (t == SRC_OP) { 1182 /* check whether register used as source operand can be read */ 1183 if (reg->type == NOT_INIT) { 1184 verbose(env, "R%d !read_ok\n", regno); 1185 return -EACCES; 1186 } 1187 /* We don't need to worry about FP liveness because it's read-only */ 1188 if (regno == BPF_REG_FP) 1189 return 0; 1190 1191 return mark_reg_read(env, reg, reg->parent); 1192 } else { 1193 /* check whether register used as dest operand can be written to */ 1194 if (regno == BPF_REG_FP) { 1195 verbose(env, "frame pointer is read only\n"); 1196 return -EACCES; 1197 } 1198 reg->live |= REG_LIVE_WRITTEN; 1199 if (t == DST_OP) 1200 mark_reg_unknown(env, regs, regno); 1201 } 1202 return 0; 1203 } 1204 1205 static bool is_spillable_regtype(enum bpf_reg_type type) 1206 { 1207 switch (type) { 1208 case PTR_TO_MAP_VALUE: 1209 case PTR_TO_MAP_VALUE_OR_NULL: 1210 case PTR_TO_STACK: 1211 case PTR_TO_CTX: 1212 case PTR_TO_PACKET: 1213 case PTR_TO_PACKET_META: 1214 case PTR_TO_PACKET_END: 1215 case PTR_TO_FLOW_KEYS: 1216 case CONST_PTR_TO_MAP: 1217 case PTR_TO_SOCKET: 1218 case PTR_TO_SOCKET_OR_NULL: 1219 case PTR_TO_SOCK_COMMON: 1220 case PTR_TO_SOCK_COMMON_OR_NULL: 1221 case PTR_TO_TCP_SOCK: 1222 case PTR_TO_TCP_SOCK_OR_NULL: 1223 return true; 1224 default: 1225 return false; 1226 } 1227 } 1228 1229 /* Does this register contain a constant zero? */ 1230 static bool register_is_null(struct bpf_reg_state *reg) 1231 { 1232 return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); 1233 } 1234 1235 /* check_stack_read/write functions track spill/fill of registers, 1236 * stack boundary and alignment are checked in check_mem_access() 1237 */ 1238 static int check_stack_write(struct bpf_verifier_env *env, 1239 struct bpf_func_state *state, /* func where register points to */ 1240 int off, int size, int value_regno, int insn_idx) 1241 { 1242 struct bpf_func_state *cur; /* state of the current function */ 1243 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err; 1244 enum bpf_reg_type type; 1245 1246 err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE), 1247 state->acquired_refs, true); 1248 if (err) 1249 return err; 1250 /* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0, 1251 * so it's aligned access and [off, off + size) are within stack limits 1252 */ 1253 if (!env->allow_ptr_leaks && 1254 state->stack[spi].slot_type[0] == STACK_SPILL && 1255 size != BPF_REG_SIZE) { 1256 verbose(env, "attempt to corrupt spilled pointer on stack\n"); 1257 return -EACCES; 1258 } 1259 1260 cur = env->cur_state->frame[env->cur_state->curframe]; 1261 if (value_regno >= 0 && 1262 is_spillable_regtype((type = cur->regs[value_regno].type))) { 1263 1264 /* register containing pointer is being spilled into stack */ 1265 if (size != BPF_REG_SIZE) { 1266 verbose(env, "invalid size of register spill\n"); 1267 return -EACCES; 1268 } 1269 1270 if (state != cur && type == PTR_TO_STACK) { 1271 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n"); 1272 return -EINVAL; 1273 } 1274 1275 /* save register state */ 1276 state->stack[spi].spilled_ptr = cur->regs[value_regno]; 1277 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 1278 1279 for (i = 0; i < BPF_REG_SIZE; i++) { 1280 if (state->stack[spi].slot_type[i] == STACK_MISC && 1281 !env->allow_ptr_leaks) { 1282 int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off; 1283 int soff = (-spi - 1) * BPF_REG_SIZE; 1284 1285 /* detected reuse of integer stack slot with a pointer 1286 * which means either llvm is reusing stack slot or 1287 * an attacker is trying to exploit CVE-2018-3639 1288 * (speculative store bypass) 1289 * Have to sanitize that slot with preemptive 1290 * store of zero. 1291 */ 1292 if (*poff && *poff != soff) { 1293 /* disallow programs where single insn stores 1294 * into two different stack slots, since verifier 1295 * cannot sanitize them 1296 */ 1297 verbose(env, 1298 "insn %d cannot access two stack slots fp%d and fp%d", 1299 insn_idx, *poff, soff); 1300 return -EINVAL; 1301 } 1302 *poff = soff; 1303 } 1304 state->stack[spi].slot_type[i] = STACK_SPILL; 1305 } 1306 } else { 1307 u8 type = STACK_MISC; 1308 1309 /* regular write of data into stack destroys any spilled ptr */ 1310 state->stack[spi].spilled_ptr.type = NOT_INIT; 1311 /* Mark slots as STACK_MISC if they belonged to spilled ptr. */ 1312 if (state->stack[spi].slot_type[0] == STACK_SPILL) 1313 for (i = 0; i < BPF_REG_SIZE; i++) 1314 state->stack[spi].slot_type[i] = STACK_MISC; 1315 1316 /* only mark the slot as written if all 8 bytes were written 1317 * otherwise read propagation may incorrectly stop too soon 1318 * when stack slots are partially written. 1319 * This heuristic means that read propagation will be 1320 * conservative, since it will add reg_live_read marks 1321 * to stack slots all the way to first state when programs 1322 * writes+reads less than 8 bytes 1323 */ 1324 if (size == BPF_REG_SIZE) 1325 state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN; 1326 1327 /* when we zero initialize stack slots mark them as such */ 1328 if (value_regno >= 0 && 1329 register_is_null(&cur->regs[value_regno])) 1330 type = STACK_ZERO; 1331 1332 /* Mark slots affected by this stack write. */ 1333 for (i = 0; i < size; i++) 1334 state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] = 1335 type; 1336 } 1337 return 0; 1338 } 1339 1340 static int check_stack_read(struct bpf_verifier_env *env, 1341 struct bpf_func_state *reg_state /* func where register points to */, 1342 int off, int size, int value_regno) 1343 { 1344 struct bpf_verifier_state *vstate = env->cur_state; 1345 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 1346 int i, slot = -off - 1, spi = slot / BPF_REG_SIZE; 1347 u8 *stype; 1348 1349 if (reg_state->allocated_stack <= slot) { 1350 verbose(env, "invalid read from stack off %d+0 size %d\n", 1351 off, size); 1352 return -EACCES; 1353 } 1354 stype = reg_state->stack[spi].slot_type; 1355 1356 if (stype[0] == STACK_SPILL) { 1357 if (size != BPF_REG_SIZE) { 1358 verbose(env, "invalid size of register spill\n"); 1359 return -EACCES; 1360 } 1361 for (i = 1; i < BPF_REG_SIZE; i++) { 1362 if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) { 1363 verbose(env, "corrupted spill memory\n"); 1364 return -EACCES; 1365 } 1366 } 1367 1368 if (value_regno >= 0) { 1369 /* restore register state from stack */ 1370 state->regs[value_regno] = reg_state->stack[spi].spilled_ptr; 1371 /* mark reg as written since spilled pointer state likely 1372 * has its liveness marks cleared by is_state_visited() 1373 * which resets stack/reg liveness for state transitions 1374 */ 1375 state->regs[value_regno].live |= REG_LIVE_WRITTEN; 1376 } 1377 mark_reg_read(env, ®_state->stack[spi].spilled_ptr, 1378 reg_state->stack[spi].spilled_ptr.parent); 1379 return 0; 1380 } else { 1381 int zeros = 0; 1382 1383 for (i = 0; i < size; i++) { 1384 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC) 1385 continue; 1386 if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) { 1387 zeros++; 1388 continue; 1389 } 1390 verbose(env, "invalid read from stack off %d+%d size %d\n", 1391 off, i, size); 1392 return -EACCES; 1393 } 1394 mark_reg_read(env, ®_state->stack[spi].spilled_ptr, 1395 reg_state->stack[spi].spilled_ptr.parent); 1396 if (value_regno >= 0) { 1397 if (zeros == size) { 1398 /* any size read into register is zero extended, 1399 * so the whole register == const_zero 1400 */ 1401 __mark_reg_const_zero(&state->regs[value_regno]); 1402 } else { 1403 /* have read misc data from the stack */ 1404 mark_reg_unknown(env, state->regs, value_regno); 1405 } 1406 state->regs[value_regno].live |= REG_LIVE_WRITTEN; 1407 } 1408 return 0; 1409 } 1410 } 1411 1412 static int check_stack_access(struct bpf_verifier_env *env, 1413 const struct bpf_reg_state *reg, 1414 int off, int size) 1415 { 1416 /* Stack accesses must be at a fixed offset, so that we 1417 * can determine what type of data were returned. See 1418 * check_stack_read(). 1419 */ 1420 if (!tnum_is_const(reg->var_off)) { 1421 char tn_buf[48]; 1422 1423 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 1424 verbose(env, "variable stack access var_off=%s off=%d size=%d\n", 1425 tn_buf, off, size); 1426 return -EACCES; 1427 } 1428 1429 if (off >= 0 || off < -MAX_BPF_STACK) { 1430 verbose(env, "invalid stack off=%d size=%d\n", off, size); 1431 return -EACCES; 1432 } 1433 1434 return 0; 1435 } 1436 1437 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, 1438 int off, int size, enum bpf_access_type type) 1439 { 1440 struct bpf_reg_state *regs = cur_regs(env); 1441 struct bpf_map *map = regs[regno].map_ptr; 1442 u32 cap = bpf_map_flags_to_cap(map); 1443 1444 if (type == BPF_WRITE && !(cap & BPF_MAP_CAN_WRITE)) { 1445 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", 1446 map->value_size, off, size); 1447 return -EACCES; 1448 } 1449 1450 if (type == BPF_READ && !(cap & BPF_MAP_CAN_READ)) { 1451 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", 1452 map->value_size, off, size); 1453 return -EACCES; 1454 } 1455 1456 return 0; 1457 } 1458 1459 /* check read/write into map element returned by bpf_map_lookup_elem() */ 1460 static int __check_map_access(struct bpf_verifier_env *env, u32 regno, int off, 1461 int size, bool zero_size_allowed) 1462 { 1463 struct bpf_reg_state *regs = cur_regs(env); 1464 struct bpf_map *map = regs[regno].map_ptr; 1465 1466 if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) || 1467 off + size > map->value_size) { 1468 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", 1469 map->value_size, off, size); 1470 return -EACCES; 1471 } 1472 return 0; 1473 } 1474 1475 /* check read/write into a map element with possible variable offset */ 1476 static int check_map_access(struct bpf_verifier_env *env, u32 regno, 1477 int off, int size, bool zero_size_allowed) 1478 { 1479 struct bpf_verifier_state *vstate = env->cur_state; 1480 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 1481 struct bpf_reg_state *reg = &state->regs[regno]; 1482 int err; 1483 1484 /* We may have adjusted the register to this map value, so we 1485 * need to try adding each of min_value and max_value to off 1486 * to make sure our theoretical access will be safe. 1487 */ 1488 if (env->log.level & BPF_LOG_LEVEL) 1489 print_verifier_state(env, state); 1490 1491 /* The minimum value is only important with signed 1492 * comparisons where we can't assume the floor of a 1493 * value is 0. If we are using signed variables for our 1494 * index'es we need to make sure that whatever we use 1495 * will have a set floor within our range. 1496 */ 1497 if (reg->smin_value < 0 && 1498 (reg->smin_value == S64_MIN || 1499 (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) || 1500 reg->smin_value + off < 0)) { 1501 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 1502 regno); 1503 return -EACCES; 1504 } 1505 err = __check_map_access(env, regno, reg->smin_value + off, size, 1506 zero_size_allowed); 1507 if (err) { 1508 verbose(env, "R%d min value is outside of the array range\n", 1509 regno); 1510 return err; 1511 } 1512 1513 /* If we haven't set a max value then we need to bail since we can't be 1514 * sure we won't do bad things. 1515 * If reg->umax_value + off could overflow, treat that as unbounded too. 1516 */ 1517 if (reg->umax_value >= BPF_MAX_VAR_OFF) { 1518 verbose(env, "R%d unbounded memory access, make sure to bounds check any array access into a map\n", 1519 regno); 1520 return -EACCES; 1521 } 1522 err = __check_map_access(env, regno, reg->umax_value + off, size, 1523 zero_size_allowed); 1524 if (err) 1525 verbose(env, "R%d max value is outside of the array range\n", 1526 regno); 1527 1528 if (map_value_has_spin_lock(reg->map_ptr)) { 1529 u32 lock = reg->map_ptr->spin_lock_off; 1530 1531 /* if any part of struct bpf_spin_lock can be touched by 1532 * load/store reject this program. 1533 * To check that [x1, x2) overlaps with [y1, y2) 1534 * it is sufficient to check x1 < y2 && y1 < x2. 1535 */ 1536 if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) && 1537 lock < reg->umax_value + off + size) { 1538 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n"); 1539 return -EACCES; 1540 } 1541 } 1542 return err; 1543 } 1544 1545 #define MAX_PACKET_OFF 0xffff 1546 1547 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, 1548 const struct bpf_call_arg_meta *meta, 1549 enum bpf_access_type t) 1550 { 1551 switch (env->prog->type) { 1552 /* Program types only with direct read access go here! */ 1553 case BPF_PROG_TYPE_LWT_IN: 1554 case BPF_PROG_TYPE_LWT_OUT: 1555 case BPF_PROG_TYPE_LWT_SEG6LOCAL: 1556 case BPF_PROG_TYPE_SK_REUSEPORT: 1557 case BPF_PROG_TYPE_FLOW_DISSECTOR: 1558 case BPF_PROG_TYPE_CGROUP_SKB: 1559 if (t == BPF_WRITE) 1560 return false; 1561 /* fallthrough */ 1562 1563 /* Program types with direct read + write access go here! */ 1564 case BPF_PROG_TYPE_SCHED_CLS: 1565 case BPF_PROG_TYPE_SCHED_ACT: 1566 case BPF_PROG_TYPE_XDP: 1567 case BPF_PROG_TYPE_LWT_XMIT: 1568 case BPF_PROG_TYPE_SK_SKB: 1569 case BPF_PROG_TYPE_SK_MSG: 1570 if (meta) 1571 return meta->pkt_access; 1572 1573 env->seen_direct_write = true; 1574 return true; 1575 default: 1576 return false; 1577 } 1578 } 1579 1580 static int __check_packet_access(struct bpf_verifier_env *env, u32 regno, 1581 int off, int size, bool zero_size_allowed) 1582 { 1583 struct bpf_reg_state *regs = cur_regs(env); 1584 struct bpf_reg_state *reg = ®s[regno]; 1585 1586 if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) || 1587 (u64)off + size > reg->range) { 1588 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", 1589 off, size, regno, reg->id, reg->off, reg->range); 1590 return -EACCES; 1591 } 1592 return 0; 1593 } 1594 1595 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, 1596 int size, bool zero_size_allowed) 1597 { 1598 struct bpf_reg_state *regs = cur_regs(env); 1599 struct bpf_reg_state *reg = ®s[regno]; 1600 int err; 1601 1602 /* We may have added a variable offset to the packet pointer; but any 1603 * reg->range we have comes after that. We are only checking the fixed 1604 * offset. 1605 */ 1606 1607 /* We don't allow negative numbers, because we aren't tracking enough 1608 * detail to prove they're safe. 1609 */ 1610 if (reg->smin_value < 0) { 1611 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 1612 regno); 1613 return -EACCES; 1614 } 1615 err = __check_packet_access(env, regno, off, size, zero_size_allowed); 1616 if (err) { 1617 verbose(env, "R%d offset is outside of the packet\n", regno); 1618 return err; 1619 } 1620 1621 /* __check_packet_access has made sure "off + size - 1" is within u16. 1622 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff, 1623 * otherwise find_good_pkt_pointers would have refused to set range info 1624 * that __check_packet_access would have rejected this pkt access. 1625 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32. 1626 */ 1627 env->prog->aux->max_pkt_offset = 1628 max_t(u32, env->prog->aux->max_pkt_offset, 1629 off + reg->umax_value + size - 1); 1630 1631 return err; 1632 } 1633 1634 /* check access to 'struct bpf_context' fields. Supports fixed offsets only */ 1635 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, 1636 enum bpf_access_type t, enum bpf_reg_type *reg_type) 1637 { 1638 struct bpf_insn_access_aux info = { 1639 .reg_type = *reg_type, 1640 }; 1641 1642 if (env->ops->is_valid_access && 1643 env->ops->is_valid_access(off, size, t, env->prog, &info)) { 1644 /* A non zero info.ctx_field_size indicates that this field is a 1645 * candidate for later verifier transformation to load the whole 1646 * field and then apply a mask when accessed with a narrower 1647 * access than actual ctx access size. A zero info.ctx_field_size 1648 * will only allow for whole field access and rejects any other 1649 * type of narrower access. 1650 */ 1651 *reg_type = info.reg_type; 1652 1653 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size; 1654 /* remember the offset of last byte accessed in ctx */ 1655 if (env->prog->aux->max_ctx_offset < off + size) 1656 env->prog->aux->max_ctx_offset = off + size; 1657 return 0; 1658 } 1659 1660 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size); 1661 return -EACCES; 1662 } 1663 1664 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, 1665 int size) 1666 { 1667 if (size < 0 || off < 0 || 1668 (u64)off + size > sizeof(struct bpf_flow_keys)) { 1669 verbose(env, "invalid access to flow keys off=%d size=%d\n", 1670 off, size); 1671 return -EACCES; 1672 } 1673 return 0; 1674 } 1675 1676 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, 1677 u32 regno, int off, int size, 1678 enum bpf_access_type t) 1679 { 1680 struct bpf_reg_state *regs = cur_regs(env); 1681 struct bpf_reg_state *reg = ®s[regno]; 1682 struct bpf_insn_access_aux info = {}; 1683 bool valid; 1684 1685 if (reg->smin_value < 0) { 1686 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", 1687 regno); 1688 return -EACCES; 1689 } 1690 1691 switch (reg->type) { 1692 case PTR_TO_SOCK_COMMON: 1693 valid = bpf_sock_common_is_valid_access(off, size, t, &info); 1694 break; 1695 case PTR_TO_SOCKET: 1696 valid = bpf_sock_is_valid_access(off, size, t, &info); 1697 break; 1698 case PTR_TO_TCP_SOCK: 1699 valid = bpf_tcp_sock_is_valid_access(off, size, t, &info); 1700 break; 1701 default: 1702 valid = false; 1703 } 1704 1705 1706 if (valid) { 1707 env->insn_aux_data[insn_idx].ctx_field_size = 1708 info.ctx_field_size; 1709 return 0; 1710 } 1711 1712 verbose(env, "R%d invalid %s access off=%d size=%d\n", 1713 regno, reg_type_str[reg->type], off, size); 1714 1715 return -EACCES; 1716 } 1717 1718 static bool __is_pointer_value(bool allow_ptr_leaks, 1719 const struct bpf_reg_state *reg) 1720 { 1721 if (allow_ptr_leaks) 1722 return false; 1723 1724 return reg->type != SCALAR_VALUE; 1725 } 1726 1727 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno) 1728 { 1729 return cur_regs(env) + regno; 1730 } 1731 1732 static bool is_pointer_value(struct bpf_verifier_env *env, int regno) 1733 { 1734 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno)); 1735 } 1736 1737 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno) 1738 { 1739 const struct bpf_reg_state *reg = reg_state(env, regno); 1740 1741 return reg->type == PTR_TO_CTX; 1742 } 1743 1744 static bool is_sk_reg(struct bpf_verifier_env *env, int regno) 1745 { 1746 const struct bpf_reg_state *reg = reg_state(env, regno); 1747 1748 return type_is_sk_pointer(reg->type); 1749 } 1750 1751 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno) 1752 { 1753 const struct bpf_reg_state *reg = reg_state(env, regno); 1754 1755 return type_is_pkt_pointer(reg->type); 1756 } 1757 1758 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno) 1759 { 1760 const struct bpf_reg_state *reg = reg_state(env, regno); 1761 1762 /* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */ 1763 return reg->type == PTR_TO_FLOW_KEYS; 1764 } 1765 1766 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, 1767 const struct bpf_reg_state *reg, 1768 int off, int size, bool strict) 1769 { 1770 struct tnum reg_off; 1771 int ip_align; 1772 1773 /* Byte size accesses are always allowed. */ 1774 if (!strict || size == 1) 1775 return 0; 1776 1777 /* For platforms that do not have a Kconfig enabling 1778 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of 1779 * NET_IP_ALIGN is universally set to '2'. And on platforms 1780 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get 1781 * to this code only in strict mode where we want to emulate 1782 * the NET_IP_ALIGN==2 checking. Therefore use an 1783 * unconditional IP align value of '2'. 1784 */ 1785 ip_align = 2; 1786 1787 reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off)); 1788 if (!tnum_is_aligned(reg_off, size)) { 1789 char tn_buf[48]; 1790 1791 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 1792 verbose(env, 1793 "misaligned packet access off %d+%s+%d+%d size %d\n", 1794 ip_align, tn_buf, reg->off, off, size); 1795 return -EACCES; 1796 } 1797 1798 return 0; 1799 } 1800 1801 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, 1802 const struct bpf_reg_state *reg, 1803 const char *pointer_desc, 1804 int off, int size, bool strict) 1805 { 1806 struct tnum reg_off; 1807 1808 /* Byte size accesses are always allowed. */ 1809 if (!strict || size == 1) 1810 return 0; 1811 1812 reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off)); 1813 if (!tnum_is_aligned(reg_off, size)) { 1814 char tn_buf[48]; 1815 1816 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 1817 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", 1818 pointer_desc, tn_buf, reg->off, off, size); 1819 return -EACCES; 1820 } 1821 1822 return 0; 1823 } 1824 1825 static int check_ptr_alignment(struct bpf_verifier_env *env, 1826 const struct bpf_reg_state *reg, int off, 1827 int size, bool strict_alignment_once) 1828 { 1829 bool strict = env->strict_alignment || strict_alignment_once; 1830 const char *pointer_desc = ""; 1831 1832 switch (reg->type) { 1833 case PTR_TO_PACKET: 1834 case PTR_TO_PACKET_META: 1835 /* Special case, because of NET_IP_ALIGN. Given metadata sits 1836 * right in front, treat it the very same way. 1837 */ 1838 return check_pkt_ptr_alignment(env, reg, off, size, strict); 1839 case PTR_TO_FLOW_KEYS: 1840 pointer_desc = "flow keys "; 1841 break; 1842 case PTR_TO_MAP_VALUE: 1843 pointer_desc = "value "; 1844 break; 1845 case PTR_TO_CTX: 1846 pointer_desc = "context "; 1847 break; 1848 case PTR_TO_STACK: 1849 pointer_desc = "stack "; 1850 /* The stack spill tracking logic in check_stack_write() 1851 * and check_stack_read() relies on stack accesses being 1852 * aligned. 1853 */ 1854 strict = true; 1855 break; 1856 case PTR_TO_SOCKET: 1857 pointer_desc = "sock "; 1858 break; 1859 case PTR_TO_SOCK_COMMON: 1860 pointer_desc = "sock_common "; 1861 break; 1862 case PTR_TO_TCP_SOCK: 1863 pointer_desc = "tcp_sock "; 1864 break; 1865 default: 1866 break; 1867 } 1868 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, 1869 strict); 1870 } 1871 1872 static int update_stack_depth(struct bpf_verifier_env *env, 1873 const struct bpf_func_state *func, 1874 int off) 1875 { 1876 u16 stack = env->subprog_info[func->subprogno].stack_depth; 1877 1878 if (stack >= -off) 1879 return 0; 1880 1881 /* update known max for given subprogram */ 1882 env->subprog_info[func->subprogno].stack_depth = -off; 1883 return 0; 1884 } 1885 1886 /* starting from main bpf function walk all instructions of the function 1887 * and recursively walk all callees that given function can call. 1888 * Ignore jump and exit insns. 1889 * Since recursion is prevented by check_cfg() this algorithm 1890 * only needs a local stack of MAX_CALL_FRAMES to remember callsites 1891 */ 1892 static int check_max_stack_depth(struct bpf_verifier_env *env) 1893 { 1894 int depth = 0, frame = 0, idx = 0, i = 0, subprog_end; 1895 struct bpf_subprog_info *subprog = env->subprog_info; 1896 struct bpf_insn *insn = env->prog->insnsi; 1897 int ret_insn[MAX_CALL_FRAMES]; 1898 int ret_prog[MAX_CALL_FRAMES]; 1899 1900 process_func: 1901 /* round up to 32-bytes, since this is granularity 1902 * of interpreter stack size 1903 */ 1904 depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 1905 if (depth > MAX_BPF_STACK) { 1906 verbose(env, "combined stack size of %d calls is %d. Too large\n", 1907 frame + 1, depth); 1908 return -EACCES; 1909 } 1910 continue_func: 1911 subprog_end = subprog[idx + 1].start; 1912 for (; i < subprog_end; i++) { 1913 if (insn[i].code != (BPF_JMP | BPF_CALL)) 1914 continue; 1915 if (insn[i].src_reg != BPF_PSEUDO_CALL) 1916 continue; 1917 /* remember insn and function to return to */ 1918 ret_insn[frame] = i + 1; 1919 ret_prog[frame] = idx; 1920 1921 /* find the callee */ 1922 i = i + insn[i].imm + 1; 1923 idx = find_subprog(env, i); 1924 if (idx < 0) { 1925 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 1926 i); 1927 return -EFAULT; 1928 } 1929 frame++; 1930 if (frame >= MAX_CALL_FRAMES) { 1931 verbose(env, "the call stack of %d frames is too deep !\n", 1932 frame); 1933 return -E2BIG; 1934 } 1935 goto process_func; 1936 } 1937 /* end of for() loop means the last insn of the 'subprog' 1938 * was reached. Doesn't matter whether it was JA or EXIT 1939 */ 1940 if (frame == 0) 1941 return 0; 1942 depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32); 1943 frame--; 1944 i = ret_insn[frame]; 1945 idx = ret_prog[frame]; 1946 goto continue_func; 1947 } 1948 1949 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 1950 static int get_callee_stack_depth(struct bpf_verifier_env *env, 1951 const struct bpf_insn *insn, int idx) 1952 { 1953 int start = idx + insn->imm + 1, subprog; 1954 1955 subprog = find_subprog(env, start); 1956 if (subprog < 0) { 1957 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 1958 start); 1959 return -EFAULT; 1960 } 1961 return env->subprog_info[subprog].stack_depth; 1962 } 1963 #endif 1964 1965 static int check_ctx_reg(struct bpf_verifier_env *env, 1966 const struct bpf_reg_state *reg, int regno) 1967 { 1968 /* Access to ctx or passing it to a helper is only allowed in 1969 * its original, unmodified form. 1970 */ 1971 1972 if (reg->off) { 1973 verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n", 1974 regno, reg->off); 1975 return -EACCES; 1976 } 1977 1978 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 1979 char tn_buf[48]; 1980 1981 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 1982 verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf); 1983 return -EACCES; 1984 } 1985 1986 return 0; 1987 } 1988 1989 static int check_tp_buffer_access(struct bpf_verifier_env *env, 1990 const struct bpf_reg_state *reg, 1991 int regno, int off, int size) 1992 { 1993 if (off < 0) { 1994 verbose(env, 1995 "R%d invalid tracepoint buffer access: off=%d, size=%d", 1996 regno, off, size); 1997 return -EACCES; 1998 } 1999 if (!tnum_is_const(reg->var_off) || reg->var_off.value) { 2000 char tn_buf[48]; 2001 2002 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 2003 verbose(env, 2004 "R%d invalid variable buffer offset: off=%d, var_off=%s", 2005 regno, off, tn_buf); 2006 return -EACCES; 2007 } 2008 if (off + size > env->prog->aux->max_tp_access) 2009 env->prog->aux->max_tp_access = off + size; 2010 2011 return 0; 2012 } 2013 2014 2015 /* truncate register to smaller size (in bytes) 2016 * must be called with size < BPF_REG_SIZE 2017 */ 2018 static void coerce_reg_to_size(struct bpf_reg_state *reg, int size) 2019 { 2020 u64 mask; 2021 2022 /* clear high bits in bit representation */ 2023 reg->var_off = tnum_cast(reg->var_off, size); 2024 2025 /* fix arithmetic bounds */ 2026 mask = ((u64)1 << (size * 8)) - 1; 2027 if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) { 2028 reg->umin_value &= mask; 2029 reg->umax_value &= mask; 2030 } else { 2031 reg->umin_value = 0; 2032 reg->umax_value = mask; 2033 } 2034 reg->smin_value = reg->umin_value; 2035 reg->smax_value = reg->umax_value; 2036 } 2037 2038 /* check whether memory at (regno + off) is accessible for t = (read | write) 2039 * if t==write, value_regno is a register which value is stored into memory 2040 * if t==read, value_regno is a register which will receive the value from memory 2041 * if t==write && value_regno==-1, some unknown value is stored into memory 2042 * if t==read && value_regno==-1, don't care what we read from memory 2043 */ 2044 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, 2045 int off, int bpf_size, enum bpf_access_type t, 2046 int value_regno, bool strict_alignment_once) 2047 { 2048 struct bpf_reg_state *regs = cur_regs(env); 2049 struct bpf_reg_state *reg = regs + regno; 2050 struct bpf_func_state *state; 2051 int size, err = 0; 2052 2053 size = bpf_size_to_bytes(bpf_size); 2054 if (size < 0) 2055 return size; 2056 2057 /* alignment checks will add in reg->off themselves */ 2058 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once); 2059 if (err) 2060 return err; 2061 2062 /* for access checks, reg->off is just part of off */ 2063 off += reg->off; 2064 2065 if (reg->type == PTR_TO_MAP_VALUE) { 2066 if (t == BPF_WRITE && value_regno >= 0 && 2067 is_pointer_value(env, value_regno)) { 2068 verbose(env, "R%d leaks addr into map\n", value_regno); 2069 return -EACCES; 2070 } 2071 err = check_map_access_type(env, regno, off, size, t); 2072 if (err) 2073 return err; 2074 err = check_map_access(env, regno, off, size, false); 2075 if (!err && t == BPF_READ && value_regno >= 0) 2076 mark_reg_unknown(env, regs, value_regno); 2077 2078 } else if (reg->type == PTR_TO_CTX) { 2079 enum bpf_reg_type reg_type = SCALAR_VALUE; 2080 2081 if (t == BPF_WRITE && value_regno >= 0 && 2082 is_pointer_value(env, value_regno)) { 2083 verbose(env, "R%d leaks addr into ctx\n", value_regno); 2084 return -EACCES; 2085 } 2086 2087 err = check_ctx_reg(env, reg, regno); 2088 if (err < 0) 2089 return err; 2090 2091 err = check_ctx_access(env, insn_idx, off, size, t, ®_type); 2092 if (!err && t == BPF_READ && value_regno >= 0) { 2093 /* ctx access returns either a scalar, or a 2094 * PTR_TO_PACKET[_META,_END]. In the latter 2095 * case, we know the offset is zero. 2096 */ 2097 if (reg_type == SCALAR_VALUE) { 2098 mark_reg_unknown(env, regs, value_regno); 2099 } else { 2100 mark_reg_known_zero(env, regs, 2101 value_regno); 2102 if (reg_type_may_be_null(reg_type)) 2103 regs[value_regno].id = ++env->id_gen; 2104 } 2105 regs[value_regno].type = reg_type; 2106 } 2107 2108 } else if (reg->type == PTR_TO_STACK) { 2109 off += reg->var_off.value; 2110 err = check_stack_access(env, reg, off, size); 2111 if (err) 2112 return err; 2113 2114 state = func(env, reg); 2115 err = update_stack_depth(env, state, off); 2116 if (err) 2117 return err; 2118 2119 if (t == BPF_WRITE) 2120 err = check_stack_write(env, state, off, size, 2121 value_regno, insn_idx); 2122 else 2123 err = check_stack_read(env, state, off, size, 2124 value_regno); 2125 } else if (reg_is_pkt_pointer(reg)) { 2126 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) { 2127 verbose(env, "cannot write into packet\n"); 2128 return -EACCES; 2129 } 2130 if (t == BPF_WRITE && value_regno >= 0 && 2131 is_pointer_value(env, value_regno)) { 2132 verbose(env, "R%d leaks addr into packet\n", 2133 value_regno); 2134 return -EACCES; 2135 } 2136 err = check_packet_access(env, regno, off, size, false); 2137 if (!err && t == BPF_READ && value_regno >= 0) 2138 mark_reg_unknown(env, regs, value_regno); 2139 } else if (reg->type == PTR_TO_FLOW_KEYS) { 2140 if (t == BPF_WRITE && value_regno >= 0 && 2141 is_pointer_value(env, value_regno)) { 2142 verbose(env, "R%d leaks addr into flow keys\n", 2143 value_regno); 2144 return -EACCES; 2145 } 2146 2147 err = check_flow_keys_access(env, off, size); 2148 if (!err && t == BPF_READ && value_regno >= 0) 2149 mark_reg_unknown(env, regs, value_regno); 2150 } else if (type_is_sk_pointer(reg->type)) { 2151 if (t == BPF_WRITE) { 2152 verbose(env, "R%d cannot write into %s\n", 2153 regno, reg_type_str[reg->type]); 2154 return -EACCES; 2155 } 2156 err = check_sock_access(env, insn_idx, regno, off, size, t); 2157 if (!err && value_regno >= 0) 2158 mark_reg_unknown(env, regs, value_regno); 2159 } else if (reg->type == PTR_TO_TP_BUFFER) { 2160 err = check_tp_buffer_access(env, reg, regno, off, size); 2161 if (!err && t == BPF_READ && value_regno >= 0) 2162 mark_reg_unknown(env, regs, value_regno); 2163 } else { 2164 verbose(env, "R%d invalid mem access '%s'\n", regno, 2165 reg_type_str[reg->type]); 2166 return -EACCES; 2167 } 2168 2169 if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ && 2170 regs[value_regno].type == SCALAR_VALUE) { 2171 /* b/h/w load zero-extends, mark upper bits as known 0 */ 2172 coerce_reg_to_size(®s[value_regno], size); 2173 } 2174 return err; 2175 } 2176 2177 static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn) 2178 { 2179 int err; 2180 2181 if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) || 2182 insn->imm != 0) { 2183 verbose(env, "BPF_XADD uses reserved fields\n"); 2184 return -EINVAL; 2185 } 2186 2187 /* check src1 operand */ 2188 err = check_reg_arg(env, insn->src_reg, SRC_OP); 2189 if (err) 2190 return err; 2191 2192 /* check src2 operand */ 2193 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 2194 if (err) 2195 return err; 2196 2197 if (is_pointer_value(env, insn->src_reg)) { 2198 verbose(env, "R%d leaks addr into mem\n", insn->src_reg); 2199 return -EACCES; 2200 } 2201 2202 if (is_ctx_reg(env, insn->dst_reg) || 2203 is_pkt_reg(env, insn->dst_reg) || 2204 is_flow_key_reg(env, insn->dst_reg) || 2205 is_sk_reg(env, insn->dst_reg)) { 2206 verbose(env, "BPF_XADD stores into R%d %s is not allowed\n", 2207 insn->dst_reg, 2208 reg_type_str[reg_state(env, insn->dst_reg)->type]); 2209 return -EACCES; 2210 } 2211 2212 /* check whether atomic_add can read the memory */ 2213 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 2214 BPF_SIZE(insn->code), BPF_READ, -1, true); 2215 if (err) 2216 return err; 2217 2218 /* check whether atomic_add can write into the same memory */ 2219 return check_mem_access(env, insn_idx, insn->dst_reg, insn->off, 2220 BPF_SIZE(insn->code), BPF_WRITE, -1, true); 2221 } 2222 2223 static int __check_stack_boundary(struct bpf_verifier_env *env, u32 regno, 2224 int off, int access_size, 2225 bool zero_size_allowed) 2226 { 2227 struct bpf_reg_state *reg = reg_state(env, regno); 2228 2229 if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 || 2230 access_size < 0 || (access_size == 0 && !zero_size_allowed)) { 2231 if (tnum_is_const(reg->var_off)) { 2232 verbose(env, "invalid stack type R%d off=%d access_size=%d\n", 2233 regno, off, access_size); 2234 } else { 2235 char tn_buf[48]; 2236 2237 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 2238 verbose(env, "invalid stack type R%d var_off=%s access_size=%d\n", 2239 regno, tn_buf, access_size); 2240 } 2241 return -EACCES; 2242 } 2243 return 0; 2244 } 2245 2246 /* when register 'regno' is passed into function that will read 'access_size' 2247 * bytes from that pointer, make sure that it's within stack boundary 2248 * and all elements of stack are initialized. 2249 * Unlike most pointer bounds-checking functions, this one doesn't take an 2250 * 'off' argument, so it has to add in reg->off itself. 2251 */ 2252 static int check_stack_boundary(struct bpf_verifier_env *env, int regno, 2253 int access_size, bool zero_size_allowed, 2254 struct bpf_call_arg_meta *meta) 2255 { 2256 struct bpf_reg_state *reg = reg_state(env, regno); 2257 struct bpf_func_state *state = func(env, reg); 2258 int err, min_off, max_off, i, slot, spi; 2259 2260 if (reg->type != PTR_TO_STACK) { 2261 /* Allow zero-byte read from NULL, regardless of pointer type */ 2262 if (zero_size_allowed && access_size == 0 && 2263 register_is_null(reg)) 2264 return 0; 2265 2266 verbose(env, "R%d type=%s expected=%s\n", regno, 2267 reg_type_str[reg->type], 2268 reg_type_str[PTR_TO_STACK]); 2269 return -EACCES; 2270 } 2271 2272 if (tnum_is_const(reg->var_off)) { 2273 min_off = max_off = reg->var_off.value + reg->off; 2274 err = __check_stack_boundary(env, regno, min_off, access_size, 2275 zero_size_allowed); 2276 if (err) 2277 return err; 2278 } else { 2279 /* Variable offset is prohibited for unprivileged mode for 2280 * simplicity since it requires corresponding support in 2281 * Spectre masking for stack ALU. 2282 * See also retrieve_ptr_limit(). 2283 */ 2284 if (!env->allow_ptr_leaks) { 2285 char tn_buf[48]; 2286 2287 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 2288 verbose(env, "R%d indirect variable offset stack access prohibited for !root, var_off=%s\n", 2289 regno, tn_buf); 2290 return -EACCES; 2291 } 2292 /* Only initialized buffer on stack is allowed to be accessed 2293 * with variable offset. With uninitialized buffer it's hard to 2294 * guarantee that whole memory is marked as initialized on 2295 * helper return since specific bounds are unknown what may 2296 * cause uninitialized stack leaking. 2297 */ 2298 if (meta && meta->raw_mode) 2299 meta = NULL; 2300 2301 if (reg->smax_value >= BPF_MAX_VAR_OFF || 2302 reg->smax_value <= -BPF_MAX_VAR_OFF) { 2303 verbose(env, "R%d unbounded indirect variable offset stack access\n", 2304 regno); 2305 return -EACCES; 2306 } 2307 min_off = reg->smin_value + reg->off; 2308 max_off = reg->smax_value + reg->off; 2309 err = __check_stack_boundary(env, regno, min_off, access_size, 2310 zero_size_allowed); 2311 if (err) { 2312 verbose(env, "R%d min value is outside of stack bound\n", 2313 regno); 2314 return err; 2315 } 2316 err = __check_stack_boundary(env, regno, max_off, access_size, 2317 zero_size_allowed); 2318 if (err) { 2319 verbose(env, "R%d max value is outside of stack bound\n", 2320 regno); 2321 return err; 2322 } 2323 } 2324 2325 if (meta && meta->raw_mode) { 2326 meta->access_size = access_size; 2327 meta->regno = regno; 2328 return 0; 2329 } 2330 2331 for (i = min_off; i < max_off + access_size; i++) { 2332 u8 *stype; 2333 2334 slot = -i - 1; 2335 spi = slot / BPF_REG_SIZE; 2336 if (state->allocated_stack <= slot) 2337 goto err; 2338 stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE]; 2339 if (*stype == STACK_MISC) 2340 goto mark; 2341 if (*stype == STACK_ZERO) { 2342 /* helper can write anything into the stack */ 2343 *stype = STACK_MISC; 2344 goto mark; 2345 } 2346 err: 2347 if (tnum_is_const(reg->var_off)) { 2348 verbose(env, "invalid indirect read from stack off %d+%d size %d\n", 2349 min_off, i - min_off, access_size); 2350 } else { 2351 char tn_buf[48]; 2352 2353 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 2354 verbose(env, "invalid indirect read from stack var_off %s+%d size %d\n", 2355 tn_buf, i - min_off, access_size); 2356 } 2357 return -EACCES; 2358 mark: 2359 /* reading any byte out of 8-byte 'spill_slot' will cause 2360 * the whole slot to be marked as 'read' 2361 */ 2362 mark_reg_read(env, &state->stack[spi].spilled_ptr, 2363 state->stack[spi].spilled_ptr.parent); 2364 } 2365 return update_stack_depth(env, state, min_off); 2366 } 2367 2368 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, 2369 int access_size, bool zero_size_allowed, 2370 struct bpf_call_arg_meta *meta) 2371 { 2372 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 2373 2374 switch (reg->type) { 2375 case PTR_TO_PACKET: 2376 case PTR_TO_PACKET_META: 2377 return check_packet_access(env, regno, reg->off, access_size, 2378 zero_size_allowed); 2379 case PTR_TO_MAP_VALUE: 2380 if (check_map_access_type(env, regno, reg->off, access_size, 2381 meta && meta->raw_mode ? BPF_WRITE : 2382 BPF_READ)) 2383 return -EACCES; 2384 return check_map_access(env, regno, reg->off, access_size, 2385 zero_size_allowed); 2386 default: /* scalar_value|ptr_to_stack or invalid ptr */ 2387 return check_stack_boundary(env, regno, access_size, 2388 zero_size_allowed, meta); 2389 } 2390 } 2391 2392 /* Implementation details: 2393 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL 2394 * Two bpf_map_lookups (even with the same key) will have different reg->id. 2395 * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after 2396 * value_or_null->value transition, since the verifier only cares about 2397 * the range of access to valid map value pointer and doesn't care about actual 2398 * address of the map element. 2399 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps 2400 * reg->id > 0 after value_or_null->value transition. By doing so 2401 * two bpf_map_lookups will be considered two different pointers that 2402 * point to different bpf_spin_locks. 2403 * The verifier allows taking only one bpf_spin_lock at a time to avoid 2404 * dead-locks. 2405 * Since only one bpf_spin_lock is allowed the checks are simpler than 2406 * reg_is_refcounted() logic. The verifier needs to remember only 2407 * one spin_lock instead of array of acquired_refs. 2408 * cur_state->active_spin_lock remembers which map value element got locked 2409 * and clears it after bpf_spin_unlock. 2410 */ 2411 static int process_spin_lock(struct bpf_verifier_env *env, int regno, 2412 bool is_lock) 2413 { 2414 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 2415 struct bpf_verifier_state *cur = env->cur_state; 2416 bool is_const = tnum_is_const(reg->var_off); 2417 struct bpf_map *map = reg->map_ptr; 2418 u64 val = reg->var_off.value; 2419 2420 if (reg->type != PTR_TO_MAP_VALUE) { 2421 verbose(env, "R%d is not a pointer to map_value\n", regno); 2422 return -EINVAL; 2423 } 2424 if (!is_const) { 2425 verbose(env, 2426 "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", 2427 regno); 2428 return -EINVAL; 2429 } 2430 if (!map->btf) { 2431 verbose(env, 2432 "map '%s' has to have BTF in order to use bpf_spin_lock\n", 2433 map->name); 2434 return -EINVAL; 2435 } 2436 if (!map_value_has_spin_lock(map)) { 2437 if (map->spin_lock_off == -E2BIG) 2438 verbose(env, 2439 "map '%s' has more than one 'struct bpf_spin_lock'\n", 2440 map->name); 2441 else if (map->spin_lock_off == -ENOENT) 2442 verbose(env, 2443 "map '%s' doesn't have 'struct bpf_spin_lock'\n", 2444 map->name); 2445 else 2446 verbose(env, 2447 "map '%s' is not a struct type or bpf_spin_lock is mangled\n", 2448 map->name); 2449 return -EINVAL; 2450 } 2451 if (map->spin_lock_off != val + reg->off) { 2452 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", 2453 val + reg->off); 2454 return -EINVAL; 2455 } 2456 if (is_lock) { 2457 if (cur->active_spin_lock) { 2458 verbose(env, 2459 "Locking two bpf_spin_locks are not allowed\n"); 2460 return -EINVAL; 2461 } 2462 cur->active_spin_lock = reg->id; 2463 } else { 2464 if (!cur->active_spin_lock) { 2465 verbose(env, "bpf_spin_unlock without taking a lock\n"); 2466 return -EINVAL; 2467 } 2468 if (cur->active_spin_lock != reg->id) { 2469 verbose(env, "bpf_spin_unlock of different lock\n"); 2470 return -EINVAL; 2471 } 2472 cur->active_spin_lock = 0; 2473 } 2474 return 0; 2475 } 2476 2477 static bool arg_type_is_mem_ptr(enum bpf_arg_type type) 2478 { 2479 return type == ARG_PTR_TO_MEM || 2480 type == ARG_PTR_TO_MEM_OR_NULL || 2481 type == ARG_PTR_TO_UNINIT_MEM; 2482 } 2483 2484 static bool arg_type_is_mem_size(enum bpf_arg_type type) 2485 { 2486 return type == ARG_CONST_SIZE || 2487 type == ARG_CONST_SIZE_OR_ZERO; 2488 } 2489 2490 static bool arg_type_is_int_ptr(enum bpf_arg_type type) 2491 { 2492 return type == ARG_PTR_TO_INT || 2493 type == ARG_PTR_TO_LONG; 2494 } 2495 2496 static int int_ptr_type_to_size(enum bpf_arg_type type) 2497 { 2498 if (type == ARG_PTR_TO_INT) 2499 return sizeof(u32); 2500 else if (type == ARG_PTR_TO_LONG) 2501 return sizeof(u64); 2502 2503 return -EINVAL; 2504 } 2505 2506 static int check_func_arg(struct bpf_verifier_env *env, u32 regno, 2507 enum bpf_arg_type arg_type, 2508 struct bpf_call_arg_meta *meta) 2509 { 2510 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno]; 2511 enum bpf_reg_type expected_type, type = reg->type; 2512 int err = 0; 2513 2514 if (arg_type == ARG_DONTCARE) 2515 return 0; 2516 2517 err = check_reg_arg(env, regno, SRC_OP); 2518 if (err) 2519 return err; 2520 2521 if (arg_type == ARG_ANYTHING) { 2522 if (is_pointer_value(env, regno)) { 2523 verbose(env, "R%d leaks addr into helper function\n", 2524 regno); 2525 return -EACCES; 2526 } 2527 return 0; 2528 } 2529 2530 if (type_is_pkt_pointer(type) && 2531 !may_access_direct_pkt_data(env, meta, BPF_READ)) { 2532 verbose(env, "helper access to the packet is not allowed\n"); 2533 return -EACCES; 2534 } 2535 2536 if (arg_type == ARG_PTR_TO_MAP_KEY || 2537 arg_type == ARG_PTR_TO_MAP_VALUE || 2538 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE || 2539 arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) { 2540 expected_type = PTR_TO_STACK; 2541 if (register_is_null(reg) && 2542 arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL) 2543 /* final test in check_stack_boundary() */; 2544 else if (!type_is_pkt_pointer(type) && 2545 type != PTR_TO_MAP_VALUE && 2546 type != expected_type) 2547 goto err_type; 2548 } else if (arg_type == ARG_CONST_SIZE || 2549 arg_type == ARG_CONST_SIZE_OR_ZERO) { 2550 expected_type = SCALAR_VALUE; 2551 if (type != expected_type) 2552 goto err_type; 2553 } else if (arg_type == ARG_CONST_MAP_PTR) { 2554 expected_type = CONST_PTR_TO_MAP; 2555 if (type != expected_type) 2556 goto err_type; 2557 } else if (arg_type == ARG_PTR_TO_CTX) { 2558 expected_type = PTR_TO_CTX; 2559 if (type != expected_type) 2560 goto err_type; 2561 err = check_ctx_reg(env, reg, regno); 2562 if (err < 0) 2563 return err; 2564 } else if (arg_type == ARG_PTR_TO_SOCK_COMMON) { 2565 expected_type = PTR_TO_SOCK_COMMON; 2566 /* Any sk pointer can be ARG_PTR_TO_SOCK_COMMON */ 2567 if (!type_is_sk_pointer(type)) 2568 goto err_type; 2569 if (reg->ref_obj_id) { 2570 if (meta->ref_obj_id) { 2571 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", 2572 regno, reg->ref_obj_id, 2573 meta->ref_obj_id); 2574 return -EFAULT; 2575 } 2576 meta->ref_obj_id = reg->ref_obj_id; 2577 } 2578 } else if (arg_type == ARG_PTR_TO_SOCKET) { 2579 expected_type = PTR_TO_SOCKET; 2580 if (type != expected_type) 2581 goto err_type; 2582 } else if (arg_type == ARG_PTR_TO_SPIN_LOCK) { 2583 if (meta->func_id == BPF_FUNC_spin_lock) { 2584 if (process_spin_lock(env, regno, true)) 2585 return -EACCES; 2586 } else if (meta->func_id == BPF_FUNC_spin_unlock) { 2587 if (process_spin_lock(env, regno, false)) 2588 return -EACCES; 2589 } else { 2590 verbose(env, "verifier internal error\n"); 2591 return -EFAULT; 2592 } 2593 } else if (arg_type_is_mem_ptr(arg_type)) { 2594 expected_type = PTR_TO_STACK; 2595 /* One exception here. In case function allows for NULL to be 2596 * passed in as argument, it's a SCALAR_VALUE type. Final test 2597 * happens during stack boundary checking. 2598 */ 2599 if (register_is_null(reg) && 2600 arg_type == ARG_PTR_TO_MEM_OR_NULL) 2601 /* final test in check_stack_boundary() */; 2602 else if (!type_is_pkt_pointer(type) && 2603 type != PTR_TO_MAP_VALUE && 2604 type != expected_type) 2605 goto err_type; 2606 meta->raw_mode = arg_type == ARG_PTR_TO_UNINIT_MEM; 2607 } else if (arg_type_is_int_ptr(arg_type)) { 2608 expected_type = PTR_TO_STACK; 2609 if (!type_is_pkt_pointer(type) && 2610 type != PTR_TO_MAP_VALUE && 2611 type != expected_type) 2612 goto err_type; 2613 } else { 2614 verbose(env, "unsupported arg_type %d\n", arg_type); 2615 return -EFAULT; 2616 } 2617 2618 if (arg_type == ARG_CONST_MAP_PTR) { 2619 /* bpf_map_xxx(map_ptr) call: remember that map_ptr */ 2620 meta->map_ptr = reg->map_ptr; 2621 } else if (arg_type == ARG_PTR_TO_MAP_KEY) { 2622 /* bpf_map_xxx(..., map_ptr, ..., key) call: 2623 * check that [key, key + map->key_size) are within 2624 * stack limits and initialized 2625 */ 2626 if (!meta->map_ptr) { 2627 /* in function declaration map_ptr must come before 2628 * map_key, so that it's verified and known before 2629 * we have to check map_key here. Otherwise it means 2630 * that kernel subsystem misconfigured verifier 2631 */ 2632 verbose(env, "invalid map_ptr to access map->key\n"); 2633 return -EACCES; 2634 } 2635 err = check_helper_mem_access(env, regno, 2636 meta->map_ptr->key_size, false, 2637 NULL); 2638 } else if (arg_type == ARG_PTR_TO_MAP_VALUE || 2639 (arg_type == ARG_PTR_TO_MAP_VALUE_OR_NULL && 2640 !register_is_null(reg)) || 2641 arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) { 2642 /* bpf_map_xxx(..., map_ptr, ..., value) call: 2643 * check [value, value + map->value_size) validity 2644 */ 2645 if (!meta->map_ptr) { 2646 /* kernel subsystem misconfigured verifier */ 2647 verbose(env, "invalid map_ptr to access map->value\n"); 2648 return -EACCES; 2649 } 2650 meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE); 2651 err = check_helper_mem_access(env, regno, 2652 meta->map_ptr->value_size, false, 2653 meta); 2654 } else if (arg_type_is_mem_size(arg_type)) { 2655 bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO); 2656 2657 /* remember the mem_size which may be used later 2658 * to refine return values. 2659 */ 2660 meta->msize_smax_value = reg->smax_value; 2661 meta->msize_umax_value = reg->umax_value; 2662 2663 /* The register is SCALAR_VALUE; the access check 2664 * happens using its boundaries. 2665 */ 2666 if (!tnum_is_const(reg->var_off)) 2667 /* For unprivileged variable accesses, disable raw 2668 * mode so that the program is required to 2669 * initialize all the memory that the helper could 2670 * just partially fill up. 2671 */ 2672 meta = NULL; 2673 2674 if (reg->smin_value < 0) { 2675 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", 2676 regno); 2677 return -EACCES; 2678 } 2679 2680 if (reg->umin_value == 0) { 2681 err = check_helper_mem_access(env, regno - 1, 0, 2682 zero_size_allowed, 2683 meta); 2684 if (err) 2685 return err; 2686 } 2687 2688 if (reg->umax_value >= BPF_MAX_VAR_SIZ) { 2689 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", 2690 regno); 2691 return -EACCES; 2692 } 2693 err = check_helper_mem_access(env, regno - 1, 2694 reg->umax_value, 2695 zero_size_allowed, meta); 2696 } else if (arg_type_is_int_ptr(arg_type)) { 2697 int size = int_ptr_type_to_size(arg_type); 2698 2699 err = check_helper_mem_access(env, regno, size, false, meta); 2700 if (err) 2701 return err; 2702 err = check_ptr_alignment(env, reg, 0, size, true); 2703 } 2704 2705 return err; 2706 err_type: 2707 verbose(env, "R%d type=%s expected=%s\n", regno, 2708 reg_type_str[type], reg_type_str[expected_type]); 2709 return -EACCES; 2710 } 2711 2712 static int check_map_func_compatibility(struct bpf_verifier_env *env, 2713 struct bpf_map *map, int func_id) 2714 { 2715 if (!map) 2716 return 0; 2717 2718 /* We need a two way check, first is from map perspective ... */ 2719 switch (map->map_type) { 2720 case BPF_MAP_TYPE_PROG_ARRAY: 2721 if (func_id != BPF_FUNC_tail_call) 2722 goto error; 2723 break; 2724 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 2725 if (func_id != BPF_FUNC_perf_event_read && 2726 func_id != BPF_FUNC_perf_event_output && 2727 func_id != BPF_FUNC_perf_event_read_value) 2728 goto error; 2729 break; 2730 case BPF_MAP_TYPE_STACK_TRACE: 2731 if (func_id != BPF_FUNC_get_stackid) 2732 goto error; 2733 break; 2734 case BPF_MAP_TYPE_CGROUP_ARRAY: 2735 if (func_id != BPF_FUNC_skb_under_cgroup && 2736 func_id != BPF_FUNC_current_task_under_cgroup) 2737 goto error; 2738 break; 2739 case BPF_MAP_TYPE_CGROUP_STORAGE: 2740 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: 2741 if (func_id != BPF_FUNC_get_local_storage) 2742 goto error; 2743 break; 2744 /* devmap returns a pointer to a live net_device ifindex that we cannot 2745 * allow to be modified from bpf side. So do not allow lookup elements 2746 * for now. 2747 */ 2748 case BPF_MAP_TYPE_DEVMAP: 2749 if (func_id != BPF_FUNC_redirect_map) 2750 goto error; 2751 break; 2752 /* Restrict bpf side of cpumap and xskmap, open when use-cases 2753 * appear. 2754 */ 2755 case BPF_MAP_TYPE_CPUMAP: 2756 case BPF_MAP_TYPE_XSKMAP: 2757 if (func_id != BPF_FUNC_redirect_map) 2758 goto error; 2759 break; 2760 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 2761 case BPF_MAP_TYPE_HASH_OF_MAPS: 2762 if (func_id != BPF_FUNC_map_lookup_elem) 2763 goto error; 2764 break; 2765 case BPF_MAP_TYPE_SOCKMAP: 2766 if (func_id != BPF_FUNC_sk_redirect_map && 2767 func_id != BPF_FUNC_sock_map_update && 2768 func_id != BPF_FUNC_map_delete_elem && 2769 func_id != BPF_FUNC_msg_redirect_map) 2770 goto error; 2771 break; 2772 case BPF_MAP_TYPE_SOCKHASH: 2773 if (func_id != BPF_FUNC_sk_redirect_hash && 2774 func_id != BPF_FUNC_sock_hash_update && 2775 func_id != BPF_FUNC_map_delete_elem && 2776 func_id != BPF_FUNC_msg_redirect_hash) 2777 goto error; 2778 break; 2779 case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY: 2780 if (func_id != BPF_FUNC_sk_select_reuseport) 2781 goto error; 2782 break; 2783 case BPF_MAP_TYPE_QUEUE: 2784 case BPF_MAP_TYPE_STACK: 2785 if (func_id != BPF_FUNC_map_peek_elem && 2786 func_id != BPF_FUNC_map_pop_elem && 2787 func_id != BPF_FUNC_map_push_elem) 2788 goto error; 2789 break; 2790 case BPF_MAP_TYPE_SK_STORAGE: 2791 if (func_id != BPF_FUNC_sk_storage_get && 2792 func_id != BPF_FUNC_sk_storage_delete) 2793 goto error; 2794 break; 2795 default: 2796 break; 2797 } 2798 2799 /* ... and second from the function itself. */ 2800 switch (func_id) { 2801 case BPF_FUNC_tail_call: 2802 if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY) 2803 goto error; 2804 if (env->subprog_cnt > 1) { 2805 verbose(env, "tail_calls are not allowed in programs with bpf-to-bpf calls\n"); 2806 return -EINVAL; 2807 } 2808 break; 2809 case BPF_FUNC_perf_event_read: 2810 case BPF_FUNC_perf_event_output: 2811 case BPF_FUNC_perf_event_read_value: 2812 if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) 2813 goto error; 2814 break; 2815 case BPF_FUNC_get_stackid: 2816 if (map->map_type != BPF_MAP_TYPE_STACK_TRACE) 2817 goto error; 2818 break; 2819 case BPF_FUNC_current_task_under_cgroup: 2820 case BPF_FUNC_skb_under_cgroup: 2821 if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY) 2822 goto error; 2823 break; 2824 case BPF_FUNC_redirect_map: 2825 if (map->map_type != BPF_MAP_TYPE_DEVMAP && 2826 map->map_type != BPF_MAP_TYPE_CPUMAP && 2827 map->map_type != BPF_MAP_TYPE_XSKMAP) 2828 goto error; 2829 break; 2830 case BPF_FUNC_sk_redirect_map: 2831 case BPF_FUNC_msg_redirect_map: 2832 case BPF_FUNC_sock_map_update: 2833 if (map->map_type != BPF_MAP_TYPE_SOCKMAP) 2834 goto error; 2835 break; 2836 case BPF_FUNC_sk_redirect_hash: 2837 case BPF_FUNC_msg_redirect_hash: 2838 case BPF_FUNC_sock_hash_update: 2839 if (map->map_type != BPF_MAP_TYPE_SOCKHASH) 2840 goto error; 2841 break; 2842 case BPF_FUNC_get_local_storage: 2843 if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE && 2844 map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) 2845 goto error; 2846 break; 2847 case BPF_FUNC_sk_select_reuseport: 2848 if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) 2849 goto error; 2850 break; 2851 case BPF_FUNC_map_peek_elem: 2852 case BPF_FUNC_map_pop_elem: 2853 case BPF_FUNC_map_push_elem: 2854 if (map->map_type != BPF_MAP_TYPE_QUEUE && 2855 map->map_type != BPF_MAP_TYPE_STACK) 2856 goto error; 2857 break; 2858 case BPF_FUNC_sk_storage_get: 2859 case BPF_FUNC_sk_storage_delete: 2860 if (map->map_type != BPF_MAP_TYPE_SK_STORAGE) 2861 goto error; 2862 break; 2863 default: 2864 break; 2865 } 2866 2867 return 0; 2868 error: 2869 verbose(env, "cannot pass map_type %d into func %s#%d\n", 2870 map->map_type, func_id_name(func_id), func_id); 2871 return -EINVAL; 2872 } 2873 2874 static bool check_raw_mode_ok(const struct bpf_func_proto *fn) 2875 { 2876 int count = 0; 2877 2878 if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM) 2879 count++; 2880 if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM) 2881 count++; 2882 if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM) 2883 count++; 2884 if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM) 2885 count++; 2886 if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM) 2887 count++; 2888 2889 /* We only support one arg being in raw mode at the moment, 2890 * which is sufficient for the helper functions we have 2891 * right now. 2892 */ 2893 return count <= 1; 2894 } 2895 2896 static bool check_args_pair_invalid(enum bpf_arg_type arg_curr, 2897 enum bpf_arg_type arg_next) 2898 { 2899 return (arg_type_is_mem_ptr(arg_curr) && 2900 !arg_type_is_mem_size(arg_next)) || 2901 (!arg_type_is_mem_ptr(arg_curr) && 2902 arg_type_is_mem_size(arg_next)); 2903 } 2904 2905 static bool check_arg_pair_ok(const struct bpf_func_proto *fn) 2906 { 2907 /* bpf_xxx(..., buf, len) call will access 'len' 2908 * bytes from memory 'buf'. Both arg types need 2909 * to be paired, so make sure there's no buggy 2910 * helper function specification. 2911 */ 2912 if (arg_type_is_mem_size(fn->arg1_type) || 2913 arg_type_is_mem_ptr(fn->arg5_type) || 2914 check_args_pair_invalid(fn->arg1_type, fn->arg2_type) || 2915 check_args_pair_invalid(fn->arg2_type, fn->arg3_type) || 2916 check_args_pair_invalid(fn->arg3_type, fn->arg4_type) || 2917 check_args_pair_invalid(fn->arg4_type, fn->arg5_type)) 2918 return false; 2919 2920 return true; 2921 } 2922 2923 static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id) 2924 { 2925 int count = 0; 2926 2927 if (arg_type_may_be_refcounted(fn->arg1_type)) 2928 count++; 2929 if (arg_type_may_be_refcounted(fn->arg2_type)) 2930 count++; 2931 if (arg_type_may_be_refcounted(fn->arg3_type)) 2932 count++; 2933 if (arg_type_may_be_refcounted(fn->arg4_type)) 2934 count++; 2935 if (arg_type_may_be_refcounted(fn->arg5_type)) 2936 count++; 2937 2938 /* A reference acquiring function cannot acquire 2939 * another refcounted ptr. 2940 */ 2941 if (is_acquire_function(func_id) && count) 2942 return false; 2943 2944 /* We only support one arg being unreferenced at the moment, 2945 * which is sufficient for the helper functions we have right now. 2946 */ 2947 return count <= 1; 2948 } 2949 2950 static int check_func_proto(const struct bpf_func_proto *fn, int func_id) 2951 { 2952 return check_raw_mode_ok(fn) && 2953 check_arg_pair_ok(fn) && 2954 check_refcount_ok(fn, func_id) ? 0 : -EINVAL; 2955 } 2956 2957 /* Packet data might have moved, any old PTR_TO_PACKET[_META,_END] 2958 * are now invalid, so turn them into unknown SCALAR_VALUE. 2959 */ 2960 static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, 2961 struct bpf_func_state *state) 2962 { 2963 struct bpf_reg_state *regs = state->regs, *reg; 2964 int i; 2965 2966 for (i = 0; i < MAX_BPF_REG; i++) 2967 if (reg_is_pkt_pointer_any(®s[i])) 2968 mark_reg_unknown(env, regs, i); 2969 2970 bpf_for_each_spilled_reg(i, state, reg) { 2971 if (!reg) 2972 continue; 2973 if (reg_is_pkt_pointer_any(reg)) 2974 __mark_reg_unknown(reg); 2975 } 2976 } 2977 2978 static void clear_all_pkt_pointers(struct bpf_verifier_env *env) 2979 { 2980 struct bpf_verifier_state *vstate = env->cur_state; 2981 int i; 2982 2983 for (i = 0; i <= vstate->curframe; i++) 2984 __clear_all_pkt_pointers(env, vstate->frame[i]); 2985 } 2986 2987 static void release_reg_references(struct bpf_verifier_env *env, 2988 struct bpf_func_state *state, 2989 int ref_obj_id) 2990 { 2991 struct bpf_reg_state *regs = state->regs, *reg; 2992 int i; 2993 2994 for (i = 0; i < MAX_BPF_REG; i++) 2995 if (regs[i].ref_obj_id == ref_obj_id) 2996 mark_reg_unknown(env, regs, i); 2997 2998 bpf_for_each_spilled_reg(i, state, reg) { 2999 if (!reg) 3000 continue; 3001 if (reg->ref_obj_id == ref_obj_id) 3002 __mark_reg_unknown(reg); 3003 } 3004 } 3005 3006 /* The pointer with the specified id has released its reference to kernel 3007 * resources. Identify all copies of the same pointer and clear the reference. 3008 */ 3009 static int release_reference(struct bpf_verifier_env *env, 3010 int ref_obj_id) 3011 { 3012 struct bpf_verifier_state *vstate = env->cur_state; 3013 int err; 3014 int i; 3015 3016 err = release_reference_state(cur_func(env), ref_obj_id); 3017 if (err) 3018 return err; 3019 3020 for (i = 0; i <= vstate->curframe; i++) 3021 release_reg_references(env, vstate->frame[i], ref_obj_id); 3022 3023 return 0; 3024 } 3025 3026 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, 3027 int *insn_idx) 3028 { 3029 struct bpf_verifier_state *state = env->cur_state; 3030 struct bpf_func_state *caller, *callee; 3031 int i, err, subprog, target_insn; 3032 3033 if (state->curframe + 1 >= MAX_CALL_FRAMES) { 3034 verbose(env, "the call stack of %d frames is too deep\n", 3035 state->curframe + 2); 3036 return -E2BIG; 3037 } 3038 3039 target_insn = *insn_idx + insn->imm; 3040 subprog = find_subprog(env, target_insn + 1); 3041 if (subprog < 0) { 3042 verbose(env, "verifier bug. No program starts at insn %d\n", 3043 target_insn + 1); 3044 return -EFAULT; 3045 } 3046 3047 caller = state->frame[state->curframe]; 3048 if (state->frame[state->curframe + 1]) { 3049 verbose(env, "verifier bug. Frame %d already allocated\n", 3050 state->curframe + 1); 3051 return -EFAULT; 3052 } 3053 3054 callee = kzalloc(sizeof(*callee), GFP_KERNEL); 3055 if (!callee) 3056 return -ENOMEM; 3057 state->frame[state->curframe + 1] = callee; 3058 3059 /* callee cannot access r0, r6 - r9 for reading and has to write 3060 * into its own stack before reading from it. 3061 * callee can read/write into caller's stack 3062 */ 3063 init_func_state(env, callee, 3064 /* remember the callsite, it will be used by bpf_exit */ 3065 *insn_idx /* callsite */, 3066 state->curframe + 1 /* frameno within this callchain */, 3067 subprog /* subprog number within this prog */); 3068 3069 /* Transfer references to the callee */ 3070 err = transfer_reference_state(callee, caller); 3071 if (err) 3072 return err; 3073 3074 /* copy r1 - r5 args that callee can access. The copy includes parent 3075 * pointers, which connects us up to the liveness chain 3076 */ 3077 for (i = BPF_REG_1; i <= BPF_REG_5; i++) 3078 callee->regs[i] = caller->regs[i]; 3079 3080 /* after the call registers r0 - r5 were scratched */ 3081 for (i = 0; i < CALLER_SAVED_REGS; i++) { 3082 mark_reg_not_init(env, caller->regs, caller_saved[i]); 3083 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 3084 } 3085 3086 /* only increment it after check_reg_arg() finished */ 3087 state->curframe++; 3088 3089 /* and go analyze first insn of the callee */ 3090 *insn_idx = target_insn; 3091 3092 if (env->log.level & BPF_LOG_LEVEL) { 3093 verbose(env, "caller:\n"); 3094 print_verifier_state(env, caller); 3095 verbose(env, "callee:\n"); 3096 print_verifier_state(env, callee); 3097 } 3098 return 0; 3099 } 3100 3101 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) 3102 { 3103 struct bpf_verifier_state *state = env->cur_state; 3104 struct bpf_func_state *caller, *callee; 3105 struct bpf_reg_state *r0; 3106 int err; 3107 3108 callee = state->frame[state->curframe]; 3109 r0 = &callee->regs[BPF_REG_0]; 3110 if (r0->type == PTR_TO_STACK) { 3111 /* technically it's ok to return caller's stack pointer 3112 * (or caller's caller's pointer) back to the caller, 3113 * since these pointers are valid. Only current stack 3114 * pointer will be invalid as soon as function exits, 3115 * but let's be conservative 3116 */ 3117 verbose(env, "cannot return stack pointer to the caller\n"); 3118 return -EINVAL; 3119 } 3120 3121 state->curframe--; 3122 caller = state->frame[state->curframe]; 3123 /* return to the caller whatever r0 had in the callee */ 3124 caller->regs[BPF_REG_0] = *r0; 3125 3126 /* Transfer references to the caller */ 3127 err = transfer_reference_state(caller, callee); 3128 if (err) 3129 return err; 3130 3131 *insn_idx = callee->callsite + 1; 3132 if (env->log.level & BPF_LOG_LEVEL) { 3133 verbose(env, "returning from callee:\n"); 3134 print_verifier_state(env, callee); 3135 verbose(env, "to caller at %d:\n", *insn_idx); 3136 print_verifier_state(env, caller); 3137 } 3138 /* clear everything in the callee */ 3139 free_func_state(callee); 3140 state->frame[state->curframe + 1] = NULL; 3141 return 0; 3142 } 3143 3144 static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type, 3145 int func_id, 3146 struct bpf_call_arg_meta *meta) 3147 { 3148 struct bpf_reg_state *ret_reg = ®s[BPF_REG_0]; 3149 3150 if (ret_type != RET_INTEGER || 3151 (func_id != BPF_FUNC_get_stack && 3152 func_id != BPF_FUNC_probe_read_str)) 3153 return; 3154 3155 ret_reg->smax_value = meta->msize_smax_value; 3156 ret_reg->umax_value = meta->msize_umax_value; 3157 __reg_deduce_bounds(ret_reg); 3158 __reg_bound_offset(ret_reg); 3159 } 3160 3161 static int 3162 record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, 3163 int func_id, int insn_idx) 3164 { 3165 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx]; 3166 struct bpf_map *map = meta->map_ptr; 3167 3168 if (func_id != BPF_FUNC_tail_call && 3169 func_id != BPF_FUNC_map_lookup_elem && 3170 func_id != BPF_FUNC_map_update_elem && 3171 func_id != BPF_FUNC_map_delete_elem && 3172 func_id != BPF_FUNC_map_push_elem && 3173 func_id != BPF_FUNC_map_pop_elem && 3174 func_id != BPF_FUNC_map_peek_elem) 3175 return 0; 3176 3177 if (map == NULL) { 3178 verbose(env, "kernel subsystem misconfigured verifier\n"); 3179 return -EINVAL; 3180 } 3181 3182 /* In case of read-only, some additional restrictions 3183 * need to be applied in order to prevent altering the 3184 * state of the map from program side. 3185 */ 3186 if ((map->map_flags & BPF_F_RDONLY_PROG) && 3187 (func_id == BPF_FUNC_map_delete_elem || 3188 func_id == BPF_FUNC_map_update_elem || 3189 func_id == BPF_FUNC_map_push_elem || 3190 func_id == BPF_FUNC_map_pop_elem)) { 3191 verbose(env, "write into map forbidden\n"); 3192 return -EACCES; 3193 } 3194 3195 if (!BPF_MAP_PTR(aux->map_state)) 3196 bpf_map_ptr_store(aux, meta->map_ptr, 3197 meta->map_ptr->unpriv_array); 3198 else if (BPF_MAP_PTR(aux->map_state) != meta->map_ptr) 3199 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, 3200 meta->map_ptr->unpriv_array); 3201 return 0; 3202 } 3203 3204 static int check_reference_leak(struct bpf_verifier_env *env) 3205 { 3206 struct bpf_func_state *state = cur_func(env); 3207 int i; 3208 3209 for (i = 0; i < state->acquired_refs; i++) { 3210 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", 3211 state->refs[i].id, state->refs[i].insn_idx); 3212 } 3213 return state->acquired_refs ? -EINVAL : 0; 3214 } 3215 3216 static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx) 3217 { 3218 const struct bpf_func_proto *fn = NULL; 3219 struct bpf_reg_state *regs; 3220 struct bpf_call_arg_meta meta; 3221 bool changes_data; 3222 int i, err; 3223 3224 /* find function prototype */ 3225 if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) { 3226 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), 3227 func_id); 3228 return -EINVAL; 3229 } 3230 3231 if (env->ops->get_func_proto) 3232 fn = env->ops->get_func_proto(func_id, env->prog); 3233 if (!fn) { 3234 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), 3235 func_id); 3236 return -EINVAL; 3237 } 3238 3239 /* eBPF programs must be GPL compatible to use GPL-ed functions */ 3240 if (!env->prog->gpl_compatible && fn->gpl_only) { 3241 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n"); 3242 return -EINVAL; 3243 } 3244 3245 /* With LD_ABS/IND some JITs save/restore skb from r1. */ 3246 changes_data = bpf_helper_changes_pkt_data(fn->func); 3247 if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) { 3248 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", 3249 func_id_name(func_id), func_id); 3250 return -EINVAL; 3251 } 3252 3253 memset(&meta, 0, sizeof(meta)); 3254 meta.pkt_access = fn->pkt_access; 3255 3256 err = check_func_proto(fn, func_id); 3257 if (err) { 3258 verbose(env, "kernel subsystem misconfigured func %s#%d\n", 3259 func_id_name(func_id), func_id); 3260 return err; 3261 } 3262 3263 meta.func_id = func_id; 3264 /* check args */ 3265 err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &meta); 3266 if (err) 3267 return err; 3268 err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta); 3269 if (err) 3270 return err; 3271 err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta); 3272 if (err) 3273 return err; 3274 err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &meta); 3275 if (err) 3276 return err; 3277 err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &meta); 3278 if (err) 3279 return err; 3280 3281 err = record_func_map(env, &meta, func_id, insn_idx); 3282 if (err) 3283 return err; 3284 3285 /* Mark slots with STACK_MISC in case of raw mode, stack offset 3286 * is inferred from register state. 3287 */ 3288 for (i = 0; i < meta.access_size; i++) { 3289 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, 3290 BPF_WRITE, -1, false); 3291 if (err) 3292 return err; 3293 } 3294 3295 if (func_id == BPF_FUNC_tail_call) { 3296 err = check_reference_leak(env); 3297 if (err) { 3298 verbose(env, "tail_call would lead to reference leak\n"); 3299 return err; 3300 } 3301 } else if (is_release_function(func_id)) { 3302 err = release_reference(env, meta.ref_obj_id); 3303 if (err) { 3304 verbose(env, "func %s#%d reference has not been acquired before\n", 3305 func_id_name(func_id), func_id); 3306 return err; 3307 } 3308 } 3309 3310 regs = cur_regs(env); 3311 3312 /* check that flags argument in get_local_storage(map, flags) is 0, 3313 * this is required because get_local_storage() can't return an error. 3314 */ 3315 if (func_id == BPF_FUNC_get_local_storage && 3316 !register_is_null(®s[BPF_REG_2])) { 3317 verbose(env, "get_local_storage() doesn't support non-zero flags\n"); 3318 return -EINVAL; 3319 } 3320 3321 /* reset caller saved regs */ 3322 for (i = 0; i < CALLER_SAVED_REGS; i++) { 3323 mark_reg_not_init(env, regs, caller_saved[i]); 3324 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 3325 } 3326 3327 /* update return register (already marked as written above) */ 3328 if (fn->ret_type == RET_INTEGER) { 3329 /* sets type to SCALAR_VALUE */ 3330 mark_reg_unknown(env, regs, BPF_REG_0); 3331 } else if (fn->ret_type == RET_VOID) { 3332 regs[BPF_REG_0].type = NOT_INIT; 3333 } else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL || 3334 fn->ret_type == RET_PTR_TO_MAP_VALUE) { 3335 /* There is no offset yet applied, variable or fixed */ 3336 mark_reg_known_zero(env, regs, BPF_REG_0); 3337 /* remember map_ptr, so that check_map_access() 3338 * can check 'value_size' boundary of memory access 3339 * to map element returned from bpf_map_lookup_elem() 3340 */ 3341 if (meta.map_ptr == NULL) { 3342 verbose(env, 3343 "kernel subsystem misconfigured verifier\n"); 3344 return -EINVAL; 3345 } 3346 regs[BPF_REG_0].map_ptr = meta.map_ptr; 3347 if (fn->ret_type == RET_PTR_TO_MAP_VALUE) { 3348 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE; 3349 if (map_value_has_spin_lock(meta.map_ptr)) 3350 regs[BPF_REG_0].id = ++env->id_gen; 3351 } else { 3352 regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL; 3353 regs[BPF_REG_0].id = ++env->id_gen; 3354 } 3355 } else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) { 3356 mark_reg_known_zero(env, regs, BPF_REG_0); 3357 regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL; 3358 regs[BPF_REG_0].id = ++env->id_gen; 3359 } else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) { 3360 mark_reg_known_zero(env, regs, BPF_REG_0); 3361 regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL; 3362 regs[BPF_REG_0].id = ++env->id_gen; 3363 } else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) { 3364 mark_reg_known_zero(env, regs, BPF_REG_0); 3365 regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL; 3366 regs[BPF_REG_0].id = ++env->id_gen; 3367 } else { 3368 verbose(env, "unknown return type %d of func %s#%d\n", 3369 fn->ret_type, func_id_name(func_id), func_id); 3370 return -EINVAL; 3371 } 3372 3373 if (is_ptr_cast_function(func_id)) { 3374 /* For release_reference() */ 3375 regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id; 3376 } else if (is_acquire_function(func_id)) { 3377 int id = acquire_reference_state(env, insn_idx); 3378 3379 if (id < 0) 3380 return id; 3381 /* For mark_ptr_or_null_reg() */ 3382 regs[BPF_REG_0].id = id; 3383 /* For release_reference() */ 3384 regs[BPF_REG_0].ref_obj_id = id; 3385 } 3386 3387 do_refine_retval_range(regs, fn->ret_type, func_id, &meta); 3388 3389 err = check_map_func_compatibility(env, meta.map_ptr, func_id); 3390 if (err) 3391 return err; 3392 3393 if (func_id == BPF_FUNC_get_stack && !env->prog->has_callchain_buf) { 3394 const char *err_str; 3395 3396 #ifdef CONFIG_PERF_EVENTS 3397 err = get_callchain_buffers(sysctl_perf_event_max_stack); 3398 err_str = "cannot get callchain buffer for func %s#%d\n"; 3399 #else 3400 err = -ENOTSUPP; 3401 err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n"; 3402 #endif 3403 if (err) { 3404 verbose(env, err_str, func_id_name(func_id), func_id); 3405 return err; 3406 } 3407 3408 env->prog->has_callchain_buf = true; 3409 } 3410 3411 if (changes_data) 3412 clear_all_pkt_pointers(env); 3413 return 0; 3414 } 3415 3416 static bool signed_add_overflows(s64 a, s64 b) 3417 { 3418 /* Do the add in u64, where overflow is well-defined */ 3419 s64 res = (s64)((u64)a + (u64)b); 3420 3421 if (b < 0) 3422 return res > a; 3423 return res < a; 3424 } 3425 3426 static bool signed_sub_overflows(s64 a, s64 b) 3427 { 3428 /* Do the sub in u64, where overflow is well-defined */ 3429 s64 res = (s64)((u64)a - (u64)b); 3430 3431 if (b < 0) 3432 return res < a; 3433 return res > a; 3434 } 3435 3436 static bool check_reg_sane_offset(struct bpf_verifier_env *env, 3437 const struct bpf_reg_state *reg, 3438 enum bpf_reg_type type) 3439 { 3440 bool known = tnum_is_const(reg->var_off); 3441 s64 val = reg->var_off.value; 3442 s64 smin = reg->smin_value; 3443 3444 if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) { 3445 verbose(env, "math between %s pointer and %lld is not allowed\n", 3446 reg_type_str[type], val); 3447 return false; 3448 } 3449 3450 if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) { 3451 verbose(env, "%s pointer offset %d is not allowed\n", 3452 reg_type_str[type], reg->off); 3453 return false; 3454 } 3455 3456 if (smin == S64_MIN) { 3457 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n", 3458 reg_type_str[type]); 3459 return false; 3460 } 3461 3462 if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) { 3463 verbose(env, "value %lld makes %s pointer be out of bounds\n", 3464 smin, reg_type_str[type]); 3465 return false; 3466 } 3467 3468 return true; 3469 } 3470 3471 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env) 3472 { 3473 return &env->insn_aux_data[env->insn_idx]; 3474 } 3475 3476 static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg, 3477 u32 *ptr_limit, u8 opcode, bool off_is_neg) 3478 { 3479 bool mask_to_left = (opcode == BPF_ADD && off_is_neg) || 3480 (opcode == BPF_SUB && !off_is_neg); 3481 u32 off; 3482 3483 switch (ptr_reg->type) { 3484 case PTR_TO_STACK: 3485 /* Indirect variable offset stack access is prohibited in 3486 * unprivileged mode so it's not handled here. 3487 */ 3488 off = ptr_reg->off + ptr_reg->var_off.value; 3489 if (mask_to_left) 3490 *ptr_limit = MAX_BPF_STACK + off; 3491 else 3492 *ptr_limit = -off; 3493 return 0; 3494 case PTR_TO_MAP_VALUE: 3495 if (mask_to_left) { 3496 *ptr_limit = ptr_reg->umax_value + ptr_reg->off; 3497 } else { 3498 off = ptr_reg->smin_value + ptr_reg->off; 3499 *ptr_limit = ptr_reg->map_ptr->value_size - off; 3500 } 3501 return 0; 3502 default: 3503 return -EINVAL; 3504 } 3505 } 3506 3507 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, 3508 const struct bpf_insn *insn) 3509 { 3510 return env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K; 3511 } 3512 3513 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, 3514 u32 alu_state, u32 alu_limit) 3515 { 3516 /* If we arrived here from different branches with different 3517 * state or limits to sanitize, then this won't work. 3518 */ 3519 if (aux->alu_state && 3520 (aux->alu_state != alu_state || 3521 aux->alu_limit != alu_limit)) 3522 return -EACCES; 3523 3524 /* Corresponding fixup done in fixup_bpf_calls(). */ 3525 aux->alu_state = alu_state; 3526 aux->alu_limit = alu_limit; 3527 return 0; 3528 } 3529 3530 static int sanitize_val_alu(struct bpf_verifier_env *env, 3531 struct bpf_insn *insn) 3532 { 3533 struct bpf_insn_aux_data *aux = cur_aux(env); 3534 3535 if (can_skip_alu_sanitation(env, insn)) 3536 return 0; 3537 3538 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0); 3539 } 3540 3541 static int sanitize_ptr_alu(struct bpf_verifier_env *env, 3542 struct bpf_insn *insn, 3543 const struct bpf_reg_state *ptr_reg, 3544 struct bpf_reg_state *dst_reg, 3545 bool off_is_neg) 3546 { 3547 struct bpf_verifier_state *vstate = env->cur_state; 3548 struct bpf_insn_aux_data *aux = cur_aux(env); 3549 bool ptr_is_dst_reg = ptr_reg == dst_reg; 3550 u8 opcode = BPF_OP(insn->code); 3551 u32 alu_state, alu_limit; 3552 struct bpf_reg_state tmp; 3553 bool ret; 3554 3555 if (can_skip_alu_sanitation(env, insn)) 3556 return 0; 3557 3558 /* We already marked aux for masking from non-speculative 3559 * paths, thus we got here in the first place. We only care 3560 * to explore bad access from here. 3561 */ 3562 if (vstate->speculative) 3563 goto do_sim; 3564 3565 alu_state = off_is_neg ? BPF_ALU_NEG_VALUE : 0; 3566 alu_state |= ptr_is_dst_reg ? 3567 BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST; 3568 3569 if (retrieve_ptr_limit(ptr_reg, &alu_limit, opcode, off_is_neg)) 3570 return 0; 3571 if (update_alu_sanitation_state(aux, alu_state, alu_limit)) 3572 return -EACCES; 3573 do_sim: 3574 /* Simulate and find potential out-of-bounds access under 3575 * speculative execution from truncation as a result of 3576 * masking when off was not within expected range. If off 3577 * sits in dst, then we temporarily need to move ptr there 3578 * to simulate dst (== 0) +/-= ptr. Needed, for example, 3579 * for cases where we use K-based arithmetic in one direction 3580 * and truncated reg-based in the other in order to explore 3581 * bad access. 3582 */ 3583 if (!ptr_is_dst_reg) { 3584 tmp = *dst_reg; 3585 *dst_reg = *ptr_reg; 3586 } 3587 ret = push_stack(env, env->insn_idx + 1, env->insn_idx, true); 3588 if (!ptr_is_dst_reg && ret) 3589 *dst_reg = tmp; 3590 return !ret ? -EFAULT : 0; 3591 } 3592 3593 /* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off. 3594 * Caller should also handle BPF_MOV case separately. 3595 * If we return -EACCES, caller may want to try again treating pointer as a 3596 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks. 3597 */ 3598 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, 3599 struct bpf_insn *insn, 3600 const struct bpf_reg_state *ptr_reg, 3601 const struct bpf_reg_state *off_reg) 3602 { 3603 struct bpf_verifier_state *vstate = env->cur_state; 3604 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 3605 struct bpf_reg_state *regs = state->regs, *dst_reg; 3606 bool known = tnum_is_const(off_reg->var_off); 3607 s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value, 3608 smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value; 3609 u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value, 3610 umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value; 3611 u32 dst = insn->dst_reg, src = insn->src_reg; 3612 u8 opcode = BPF_OP(insn->code); 3613 int ret; 3614 3615 dst_reg = ®s[dst]; 3616 3617 if ((known && (smin_val != smax_val || umin_val != umax_val)) || 3618 smin_val > smax_val || umin_val > umax_val) { 3619 /* Taint dst register if offset had invalid bounds derived from 3620 * e.g. dead branches. 3621 */ 3622 __mark_reg_unknown(dst_reg); 3623 return 0; 3624 } 3625 3626 if (BPF_CLASS(insn->code) != BPF_ALU64) { 3627 /* 32-bit ALU ops on pointers produce (meaningless) scalars */ 3628 verbose(env, 3629 "R%d 32-bit pointer arithmetic prohibited\n", 3630 dst); 3631 return -EACCES; 3632 } 3633 3634 switch (ptr_reg->type) { 3635 case PTR_TO_MAP_VALUE_OR_NULL: 3636 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", 3637 dst, reg_type_str[ptr_reg->type]); 3638 return -EACCES; 3639 case CONST_PTR_TO_MAP: 3640 case PTR_TO_PACKET_END: 3641 case PTR_TO_SOCKET: 3642 case PTR_TO_SOCKET_OR_NULL: 3643 case PTR_TO_SOCK_COMMON: 3644 case PTR_TO_SOCK_COMMON_OR_NULL: 3645 case PTR_TO_TCP_SOCK: 3646 case PTR_TO_TCP_SOCK_OR_NULL: 3647 verbose(env, "R%d pointer arithmetic on %s prohibited\n", 3648 dst, reg_type_str[ptr_reg->type]); 3649 return -EACCES; 3650 case PTR_TO_MAP_VALUE: 3651 if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) { 3652 verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n", 3653 off_reg == dst_reg ? dst : src); 3654 return -EACCES; 3655 } 3656 /* fall-through */ 3657 default: 3658 break; 3659 } 3660 3661 /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. 3662 * The id may be overwritten later if we create a new variable offset. 3663 */ 3664 dst_reg->type = ptr_reg->type; 3665 dst_reg->id = ptr_reg->id; 3666 3667 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || 3668 !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) 3669 return -EINVAL; 3670 3671 switch (opcode) { 3672 case BPF_ADD: 3673 ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0); 3674 if (ret < 0) { 3675 verbose(env, "R%d tried to add from different maps or paths\n", dst); 3676 return ret; 3677 } 3678 /* We can take a fixed offset as long as it doesn't overflow 3679 * the s32 'off' field 3680 */ 3681 if (known && (ptr_reg->off + smin_val == 3682 (s64)(s32)(ptr_reg->off + smin_val))) { 3683 /* pointer += K. Accumulate it into fixed offset */ 3684 dst_reg->smin_value = smin_ptr; 3685 dst_reg->smax_value = smax_ptr; 3686 dst_reg->umin_value = umin_ptr; 3687 dst_reg->umax_value = umax_ptr; 3688 dst_reg->var_off = ptr_reg->var_off; 3689 dst_reg->off = ptr_reg->off + smin_val; 3690 dst_reg->raw = ptr_reg->raw; 3691 break; 3692 } 3693 /* A new variable offset is created. Note that off_reg->off 3694 * == 0, since it's a scalar. 3695 * dst_reg gets the pointer type and since some positive 3696 * integer value was added to the pointer, give it a new 'id' 3697 * if it's a PTR_TO_PACKET. 3698 * this creates a new 'base' pointer, off_reg (variable) gets 3699 * added into the variable offset, and we copy the fixed offset 3700 * from ptr_reg. 3701 */ 3702 if (signed_add_overflows(smin_ptr, smin_val) || 3703 signed_add_overflows(smax_ptr, smax_val)) { 3704 dst_reg->smin_value = S64_MIN; 3705 dst_reg->smax_value = S64_MAX; 3706 } else { 3707 dst_reg->smin_value = smin_ptr + smin_val; 3708 dst_reg->smax_value = smax_ptr + smax_val; 3709 } 3710 if (umin_ptr + umin_val < umin_ptr || 3711 umax_ptr + umax_val < umax_ptr) { 3712 dst_reg->umin_value = 0; 3713 dst_reg->umax_value = U64_MAX; 3714 } else { 3715 dst_reg->umin_value = umin_ptr + umin_val; 3716 dst_reg->umax_value = umax_ptr + umax_val; 3717 } 3718 dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off); 3719 dst_reg->off = ptr_reg->off; 3720 dst_reg->raw = ptr_reg->raw; 3721 if (reg_is_pkt_pointer(ptr_reg)) { 3722 dst_reg->id = ++env->id_gen; 3723 /* something was added to pkt_ptr, set range to zero */ 3724 dst_reg->raw = 0; 3725 } 3726 break; 3727 case BPF_SUB: 3728 ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0); 3729 if (ret < 0) { 3730 verbose(env, "R%d tried to sub from different maps or paths\n", dst); 3731 return ret; 3732 } 3733 if (dst_reg == off_reg) { 3734 /* scalar -= pointer. Creates an unknown scalar */ 3735 verbose(env, "R%d tried to subtract pointer from scalar\n", 3736 dst); 3737 return -EACCES; 3738 } 3739 /* We don't allow subtraction from FP, because (according to 3740 * test_verifier.c test "invalid fp arithmetic", JITs might not 3741 * be able to deal with it. 3742 */ 3743 if (ptr_reg->type == PTR_TO_STACK) { 3744 verbose(env, "R%d subtraction from stack pointer prohibited\n", 3745 dst); 3746 return -EACCES; 3747 } 3748 if (known && (ptr_reg->off - smin_val == 3749 (s64)(s32)(ptr_reg->off - smin_val))) { 3750 /* pointer -= K. Subtract it from fixed offset */ 3751 dst_reg->smin_value = smin_ptr; 3752 dst_reg->smax_value = smax_ptr; 3753 dst_reg->umin_value = umin_ptr; 3754 dst_reg->umax_value = umax_ptr; 3755 dst_reg->var_off = ptr_reg->var_off; 3756 dst_reg->id = ptr_reg->id; 3757 dst_reg->off = ptr_reg->off - smin_val; 3758 dst_reg->raw = ptr_reg->raw; 3759 break; 3760 } 3761 /* A new variable offset is created. If the subtrahend is known 3762 * nonnegative, then any reg->range we had before is still good. 3763 */ 3764 if (signed_sub_overflows(smin_ptr, smax_val) || 3765 signed_sub_overflows(smax_ptr, smin_val)) { 3766 /* Overflow possible, we know nothing */ 3767 dst_reg->smin_value = S64_MIN; 3768 dst_reg->smax_value = S64_MAX; 3769 } else { 3770 dst_reg->smin_value = smin_ptr - smax_val; 3771 dst_reg->smax_value = smax_ptr - smin_val; 3772 } 3773 if (umin_ptr < umax_val) { 3774 /* Overflow possible, we know nothing */ 3775 dst_reg->umin_value = 0; 3776 dst_reg->umax_value = U64_MAX; 3777 } else { 3778 /* Cannot overflow (as long as bounds are consistent) */ 3779 dst_reg->umin_value = umin_ptr - umax_val; 3780 dst_reg->umax_value = umax_ptr - umin_val; 3781 } 3782 dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off); 3783 dst_reg->off = ptr_reg->off; 3784 dst_reg->raw = ptr_reg->raw; 3785 if (reg_is_pkt_pointer(ptr_reg)) { 3786 dst_reg->id = ++env->id_gen; 3787 /* something was added to pkt_ptr, set range to zero */ 3788 if (smin_val < 0) 3789 dst_reg->raw = 0; 3790 } 3791 break; 3792 case BPF_AND: 3793 case BPF_OR: 3794 case BPF_XOR: 3795 /* bitwise ops on pointers are troublesome, prohibit. */ 3796 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", 3797 dst, bpf_alu_string[opcode >> 4]); 3798 return -EACCES; 3799 default: 3800 /* other operators (e.g. MUL,LSH) produce non-pointer results */ 3801 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", 3802 dst, bpf_alu_string[opcode >> 4]); 3803 return -EACCES; 3804 } 3805 3806 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) 3807 return -EINVAL; 3808 3809 __update_reg_bounds(dst_reg); 3810 __reg_deduce_bounds(dst_reg); 3811 __reg_bound_offset(dst_reg); 3812 3813 /* For unprivileged we require that resulting offset must be in bounds 3814 * in order to be able to sanitize access later on. 3815 */ 3816 if (!env->allow_ptr_leaks) { 3817 if (dst_reg->type == PTR_TO_MAP_VALUE && 3818 check_map_access(env, dst, dst_reg->off, 1, false)) { 3819 verbose(env, "R%d pointer arithmetic of map value goes out of range, " 3820 "prohibited for !root\n", dst); 3821 return -EACCES; 3822 } else if (dst_reg->type == PTR_TO_STACK && 3823 check_stack_access(env, dst_reg, dst_reg->off + 3824 dst_reg->var_off.value, 1)) { 3825 verbose(env, "R%d stack pointer arithmetic goes out of range, " 3826 "prohibited for !root\n", dst); 3827 return -EACCES; 3828 } 3829 } 3830 3831 return 0; 3832 } 3833 3834 /* WARNING: This function does calculations on 64-bit values, but the actual 3835 * execution may occur on 32-bit values. Therefore, things like bitshifts 3836 * need extra checks in the 32-bit case. 3837 */ 3838 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, 3839 struct bpf_insn *insn, 3840 struct bpf_reg_state *dst_reg, 3841 struct bpf_reg_state src_reg) 3842 { 3843 struct bpf_reg_state *regs = cur_regs(env); 3844 u8 opcode = BPF_OP(insn->code); 3845 bool src_known, dst_known; 3846 s64 smin_val, smax_val; 3847 u64 umin_val, umax_val; 3848 u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32; 3849 u32 dst = insn->dst_reg; 3850 int ret; 3851 3852 if (insn_bitness == 32) { 3853 /* Relevant for 32-bit RSH: Information can propagate towards 3854 * LSB, so it isn't sufficient to only truncate the output to 3855 * 32 bits. 3856 */ 3857 coerce_reg_to_size(dst_reg, 4); 3858 coerce_reg_to_size(&src_reg, 4); 3859 } 3860 3861 smin_val = src_reg.smin_value; 3862 smax_val = src_reg.smax_value; 3863 umin_val = src_reg.umin_value; 3864 umax_val = src_reg.umax_value; 3865 src_known = tnum_is_const(src_reg.var_off); 3866 dst_known = tnum_is_const(dst_reg->var_off); 3867 3868 if ((src_known && (smin_val != smax_val || umin_val != umax_val)) || 3869 smin_val > smax_val || umin_val > umax_val) { 3870 /* Taint dst register if offset had invalid bounds derived from 3871 * e.g. dead branches. 3872 */ 3873 __mark_reg_unknown(dst_reg); 3874 return 0; 3875 } 3876 3877 if (!src_known && 3878 opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) { 3879 __mark_reg_unknown(dst_reg); 3880 return 0; 3881 } 3882 3883 switch (opcode) { 3884 case BPF_ADD: 3885 ret = sanitize_val_alu(env, insn); 3886 if (ret < 0) { 3887 verbose(env, "R%d tried to add from different pointers or scalars\n", dst); 3888 return ret; 3889 } 3890 if (signed_add_overflows(dst_reg->smin_value, smin_val) || 3891 signed_add_overflows(dst_reg->smax_value, smax_val)) { 3892 dst_reg->smin_value = S64_MIN; 3893 dst_reg->smax_value = S64_MAX; 3894 } else { 3895 dst_reg->smin_value += smin_val; 3896 dst_reg->smax_value += smax_val; 3897 } 3898 if (dst_reg->umin_value + umin_val < umin_val || 3899 dst_reg->umax_value + umax_val < umax_val) { 3900 dst_reg->umin_value = 0; 3901 dst_reg->umax_value = U64_MAX; 3902 } else { 3903 dst_reg->umin_value += umin_val; 3904 dst_reg->umax_value += umax_val; 3905 } 3906 dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off); 3907 break; 3908 case BPF_SUB: 3909 ret = sanitize_val_alu(env, insn); 3910 if (ret < 0) { 3911 verbose(env, "R%d tried to sub from different pointers or scalars\n", dst); 3912 return ret; 3913 } 3914 if (signed_sub_overflows(dst_reg->smin_value, smax_val) || 3915 signed_sub_overflows(dst_reg->smax_value, smin_val)) { 3916 /* Overflow possible, we know nothing */ 3917 dst_reg->smin_value = S64_MIN; 3918 dst_reg->smax_value = S64_MAX; 3919 } else { 3920 dst_reg->smin_value -= smax_val; 3921 dst_reg->smax_value -= smin_val; 3922 } 3923 if (dst_reg->umin_value < umax_val) { 3924 /* Overflow possible, we know nothing */ 3925 dst_reg->umin_value = 0; 3926 dst_reg->umax_value = U64_MAX; 3927 } else { 3928 /* Cannot overflow (as long as bounds are consistent) */ 3929 dst_reg->umin_value -= umax_val; 3930 dst_reg->umax_value -= umin_val; 3931 } 3932 dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off); 3933 break; 3934 case BPF_MUL: 3935 dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off); 3936 if (smin_val < 0 || dst_reg->smin_value < 0) { 3937 /* Ain't nobody got time to multiply that sign */ 3938 __mark_reg_unbounded(dst_reg); 3939 __update_reg_bounds(dst_reg); 3940 break; 3941 } 3942 /* Both values are positive, so we can work with unsigned and 3943 * copy the result to signed (unless it exceeds S64_MAX). 3944 */ 3945 if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) { 3946 /* Potential overflow, we know nothing */ 3947 __mark_reg_unbounded(dst_reg); 3948 /* (except what we can learn from the var_off) */ 3949 __update_reg_bounds(dst_reg); 3950 break; 3951 } 3952 dst_reg->umin_value *= umin_val; 3953 dst_reg->umax_value *= umax_val; 3954 if (dst_reg->umax_value > S64_MAX) { 3955 /* Overflow possible, we know nothing */ 3956 dst_reg->smin_value = S64_MIN; 3957 dst_reg->smax_value = S64_MAX; 3958 } else { 3959 dst_reg->smin_value = dst_reg->umin_value; 3960 dst_reg->smax_value = dst_reg->umax_value; 3961 } 3962 break; 3963 case BPF_AND: 3964 if (src_known && dst_known) { 3965 __mark_reg_known(dst_reg, dst_reg->var_off.value & 3966 src_reg.var_off.value); 3967 break; 3968 } 3969 /* We get our minimum from the var_off, since that's inherently 3970 * bitwise. Our maximum is the minimum of the operands' maxima. 3971 */ 3972 dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off); 3973 dst_reg->umin_value = dst_reg->var_off.value; 3974 dst_reg->umax_value = min(dst_reg->umax_value, umax_val); 3975 if (dst_reg->smin_value < 0 || smin_val < 0) { 3976 /* Lose signed bounds when ANDing negative numbers, 3977 * ain't nobody got time for that. 3978 */ 3979 dst_reg->smin_value = S64_MIN; 3980 dst_reg->smax_value = S64_MAX; 3981 } else { 3982 /* ANDing two positives gives a positive, so safe to 3983 * cast result into s64. 3984 */ 3985 dst_reg->smin_value = dst_reg->umin_value; 3986 dst_reg->smax_value = dst_reg->umax_value; 3987 } 3988 /* We may learn something more from the var_off */ 3989 __update_reg_bounds(dst_reg); 3990 break; 3991 case BPF_OR: 3992 if (src_known && dst_known) { 3993 __mark_reg_known(dst_reg, dst_reg->var_off.value | 3994 src_reg.var_off.value); 3995 break; 3996 } 3997 /* We get our maximum from the var_off, and our minimum is the 3998 * maximum of the operands' minima 3999 */ 4000 dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off); 4001 dst_reg->umin_value = max(dst_reg->umin_value, umin_val); 4002 dst_reg->umax_value = dst_reg->var_off.value | 4003 dst_reg->var_off.mask; 4004 if (dst_reg->smin_value < 0 || smin_val < 0) { 4005 /* Lose signed bounds when ORing negative numbers, 4006 * ain't nobody got time for that. 4007 */ 4008 dst_reg->smin_value = S64_MIN; 4009 dst_reg->smax_value = S64_MAX; 4010 } else { 4011 /* ORing two positives gives a positive, so safe to 4012 * cast result into s64. 4013 */ 4014 dst_reg->smin_value = dst_reg->umin_value; 4015 dst_reg->smax_value = dst_reg->umax_value; 4016 } 4017 /* We may learn something more from the var_off */ 4018 __update_reg_bounds(dst_reg); 4019 break; 4020 case BPF_LSH: 4021 if (umax_val >= insn_bitness) { 4022 /* Shifts greater than 31 or 63 are undefined. 4023 * This includes shifts by a negative number. 4024 */ 4025 mark_reg_unknown(env, regs, insn->dst_reg); 4026 break; 4027 } 4028 /* We lose all sign bit information (except what we can pick 4029 * up from var_off) 4030 */ 4031 dst_reg->smin_value = S64_MIN; 4032 dst_reg->smax_value = S64_MAX; 4033 /* If we might shift our top bit out, then we know nothing */ 4034 if (dst_reg->umax_value > 1ULL << (63 - umax_val)) { 4035 dst_reg->umin_value = 0; 4036 dst_reg->umax_value = U64_MAX; 4037 } else { 4038 dst_reg->umin_value <<= umin_val; 4039 dst_reg->umax_value <<= umax_val; 4040 } 4041 dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val); 4042 /* We may learn something more from the var_off */ 4043 __update_reg_bounds(dst_reg); 4044 break; 4045 case BPF_RSH: 4046 if (umax_val >= insn_bitness) { 4047 /* Shifts greater than 31 or 63 are undefined. 4048 * This includes shifts by a negative number. 4049 */ 4050 mark_reg_unknown(env, regs, insn->dst_reg); 4051 break; 4052 } 4053 /* BPF_RSH is an unsigned shift. If the value in dst_reg might 4054 * be negative, then either: 4055 * 1) src_reg might be zero, so the sign bit of the result is 4056 * unknown, so we lose our signed bounds 4057 * 2) it's known negative, thus the unsigned bounds capture the 4058 * signed bounds 4059 * 3) the signed bounds cross zero, so they tell us nothing 4060 * about the result 4061 * If the value in dst_reg is known nonnegative, then again the 4062 * unsigned bounts capture the signed bounds. 4063 * Thus, in all cases it suffices to blow away our signed bounds 4064 * and rely on inferring new ones from the unsigned bounds and 4065 * var_off of the result. 4066 */ 4067 dst_reg->smin_value = S64_MIN; 4068 dst_reg->smax_value = S64_MAX; 4069 dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val); 4070 dst_reg->umin_value >>= umax_val; 4071 dst_reg->umax_value >>= umin_val; 4072 /* We may learn something more from the var_off */ 4073 __update_reg_bounds(dst_reg); 4074 break; 4075 case BPF_ARSH: 4076 if (umax_val >= insn_bitness) { 4077 /* Shifts greater than 31 or 63 are undefined. 4078 * This includes shifts by a negative number. 4079 */ 4080 mark_reg_unknown(env, regs, insn->dst_reg); 4081 break; 4082 } 4083 4084 /* Upon reaching here, src_known is true and 4085 * umax_val is equal to umin_val. 4086 */ 4087 dst_reg->smin_value >>= umin_val; 4088 dst_reg->smax_value >>= umin_val; 4089 dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val); 4090 4091 /* blow away the dst_reg umin_value/umax_value and rely on 4092 * dst_reg var_off to refine the result. 4093 */ 4094 dst_reg->umin_value = 0; 4095 dst_reg->umax_value = U64_MAX; 4096 __update_reg_bounds(dst_reg); 4097 break; 4098 default: 4099 mark_reg_unknown(env, regs, insn->dst_reg); 4100 break; 4101 } 4102 4103 if (BPF_CLASS(insn->code) != BPF_ALU64) { 4104 /* 32-bit ALU ops are (32,32)->32 */ 4105 coerce_reg_to_size(dst_reg, 4); 4106 } 4107 4108 __reg_deduce_bounds(dst_reg); 4109 __reg_bound_offset(dst_reg); 4110 return 0; 4111 } 4112 4113 /* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max 4114 * and var_off. 4115 */ 4116 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, 4117 struct bpf_insn *insn) 4118 { 4119 struct bpf_verifier_state *vstate = env->cur_state; 4120 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4121 struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg; 4122 struct bpf_reg_state *ptr_reg = NULL, off_reg = {0}; 4123 u8 opcode = BPF_OP(insn->code); 4124 4125 dst_reg = ®s[insn->dst_reg]; 4126 src_reg = NULL; 4127 if (dst_reg->type != SCALAR_VALUE) 4128 ptr_reg = dst_reg; 4129 if (BPF_SRC(insn->code) == BPF_X) { 4130 src_reg = ®s[insn->src_reg]; 4131 if (src_reg->type != SCALAR_VALUE) { 4132 if (dst_reg->type != SCALAR_VALUE) { 4133 /* Combining two pointers by any ALU op yields 4134 * an arbitrary scalar. Disallow all math except 4135 * pointer subtraction 4136 */ 4137 if (opcode == BPF_SUB && env->allow_ptr_leaks) { 4138 mark_reg_unknown(env, regs, insn->dst_reg); 4139 return 0; 4140 } 4141 verbose(env, "R%d pointer %s pointer prohibited\n", 4142 insn->dst_reg, 4143 bpf_alu_string[opcode >> 4]); 4144 return -EACCES; 4145 } else { 4146 /* scalar += pointer 4147 * This is legal, but we have to reverse our 4148 * src/dest handling in computing the range 4149 */ 4150 return adjust_ptr_min_max_vals(env, insn, 4151 src_reg, dst_reg); 4152 } 4153 } else if (ptr_reg) { 4154 /* pointer += scalar */ 4155 return adjust_ptr_min_max_vals(env, insn, 4156 dst_reg, src_reg); 4157 } 4158 } else { 4159 /* Pretend the src is a reg with a known value, since we only 4160 * need to be able to read from this state. 4161 */ 4162 off_reg.type = SCALAR_VALUE; 4163 __mark_reg_known(&off_reg, insn->imm); 4164 src_reg = &off_reg; 4165 if (ptr_reg) /* pointer += K */ 4166 return adjust_ptr_min_max_vals(env, insn, 4167 ptr_reg, src_reg); 4168 } 4169 4170 /* Got here implies adding two SCALAR_VALUEs */ 4171 if (WARN_ON_ONCE(ptr_reg)) { 4172 print_verifier_state(env, state); 4173 verbose(env, "verifier internal error: unexpected ptr_reg\n"); 4174 return -EINVAL; 4175 } 4176 if (WARN_ON(!src_reg)) { 4177 print_verifier_state(env, state); 4178 verbose(env, "verifier internal error: no src_reg\n"); 4179 return -EINVAL; 4180 } 4181 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg); 4182 } 4183 4184 /* check validity of 32-bit and 64-bit arithmetic operations */ 4185 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn) 4186 { 4187 struct bpf_reg_state *regs = cur_regs(env); 4188 u8 opcode = BPF_OP(insn->code); 4189 int err; 4190 4191 if (opcode == BPF_END || opcode == BPF_NEG) { 4192 if (opcode == BPF_NEG) { 4193 if (BPF_SRC(insn->code) != 0 || 4194 insn->src_reg != BPF_REG_0 || 4195 insn->off != 0 || insn->imm != 0) { 4196 verbose(env, "BPF_NEG uses reserved fields\n"); 4197 return -EINVAL; 4198 } 4199 } else { 4200 if (insn->src_reg != BPF_REG_0 || insn->off != 0 || 4201 (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) || 4202 BPF_CLASS(insn->code) == BPF_ALU64) { 4203 verbose(env, "BPF_END uses reserved fields\n"); 4204 return -EINVAL; 4205 } 4206 } 4207 4208 /* check src operand */ 4209 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 4210 if (err) 4211 return err; 4212 4213 if (is_pointer_value(env, insn->dst_reg)) { 4214 verbose(env, "R%d pointer arithmetic prohibited\n", 4215 insn->dst_reg); 4216 return -EACCES; 4217 } 4218 4219 /* check dest operand */ 4220 err = check_reg_arg(env, insn->dst_reg, DST_OP); 4221 if (err) 4222 return err; 4223 4224 } else if (opcode == BPF_MOV) { 4225 4226 if (BPF_SRC(insn->code) == BPF_X) { 4227 if (insn->imm != 0 || insn->off != 0) { 4228 verbose(env, "BPF_MOV uses reserved fields\n"); 4229 return -EINVAL; 4230 } 4231 4232 /* check src operand */ 4233 err = check_reg_arg(env, insn->src_reg, SRC_OP); 4234 if (err) 4235 return err; 4236 } else { 4237 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 4238 verbose(env, "BPF_MOV uses reserved fields\n"); 4239 return -EINVAL; 4240 } 4241 } 4242 4243 /* check dest operand, mark as required later */ 4244 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 4245 if (err) 4246 return err; 4247 4248 if (BPF_SRC(insn->code) == BPF_X) { 4249 struct bpf_reg_state *src_reg = regs + insn->src_reg; 4250 struct bpf_reg_state *dst_reg = regs + insn->dst_reg; 4251 4252 if (BPF_CLASS(insn->code) == BPF_ALU64) { 4253 /* case: R1 = R2 4254 * copy register state to dest reg 4255 */ 4256 *dst_reg = *src_reg; 4257 dst_reg->live |= REG_LIVE_WRITTEN; 4258 } else { 4259 /* R1 = (u32) R2 */ 4260 if (is_pointer_value(env, insn->src_reg)) { 4261 verbose(env, 4262 "R%d partial copy of pointer\n", 4263 insn->src_reg); 4264 return -EACCES; 4265 } else if (src_reg->type == SCALAR_VALUE) { 4266 *dst_reg = *src_reg; 4267 dst_reg->live |= REG_LIVE_WRITTEN; 4268 } else { 4269 mark_reg_unknown(env, regs, 4270 insn->dst_reg); 4271 } 4272 coerce_reg_to_size(dst_reg, 4); 4273 } 4274 } else { 4275 /* case: R = imm 4276 * remember the value we stored into this reg 4277 */ 4278 /* clear any state __mark_reg_known doesn't set */ 4279 mark_reg_unknown(env, regs, insn->dst_reg); 4280 regs[insn->dst_reg].type = SCALAR_VALUE; 4281 if (BPF_CLASS(insn->code) == BPF_ALU64) { 4282 __mark_reg_known(regs + insn->dst_reg, 4283 insn->imm); 4284 } else { 4285 __mark_reg_known(regs + insn->dst_reg, 4286 (u32)insn->imm); 4287 } 4288 } 4289 4290 } else if (opcode > BPF_END) { 4291 verbose(env, "invalid BPF_ALU opcode %x\n", opcode); 4292 return -EINVAL; 4293 4294 } else { /* all other ALU ops: and, sub, xor, add, ... */ 4295 4296 if (BPF_SRC(insn->code) == BPF_X) { 4297 if (insn->imm != 0 || insn->off != 0) { 4298 verbose(env, "BPF_ALU uses reserved fields\n"); 4299 return -EINVAL; 4300 } 4301 /* check src1 operand */ 4302 err = check_reg_arg(env, insn->src_reg, SRC_OP); 4303 if (err) 4304 return err; 4305 } else { 4306 if (insn->src_reg != BPF_REG_0 || insn->off != 0) { 4307 verbose(env, "BPF_ALU uses reserved fields\n"); 4308 return -EINVAL; 4309 } 4310 } 4311 4312 /* check src2 operand */ 4313 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 4314 if (err) 4315 return err; 4316 4317 if ((opcode == BPF_MOD || opcode == BPF_DIV) && 4318 BPF_SRC(insn->code) == BPF_K && insn->imm == 0) { 4319 verbose(env, "div by zero\n"); 4320 return -EINVAL; 4321 } 4322 4323 if ((opcode == BPF_LSH || opcode == BPF_RSH || 4324 opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) { 4325 int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32; 4326 4327 if (insn->imm < 0 || insn->imm >= size) { 4328 verbose(env, "invalid shift %d\n", insn->imm); 4329 return -EINVAL; 4330 } 4331 } 4332 4333 /* check dest operand */ 4334 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 4335 if (err) 4336 return err; 4337 4338 return adjust_reg_min_max_vals(env, insn); 4339 } 4340 4341 return 0; 4342 } 4343 4344 static void __find_good_pkt_pointers(struct bpf_func_state *state, 4345 struct bpf_reg_state *dst_reg, 4346 enum bpf_reg_type type, u16 new_range) 4347 { 4348 struct bpf_reg_state *reg; 4349 int i; 4350 4351 for (i = 0; i < MAX_BPF_REG; i++) { 4352 reg = &state->regs[i]; 4353 if (reg->type == type && reg->id == dst_reg->id) 4354 /* keep the maximum range already checked */ 4355 reg->range = max(reg->range, new_range); 4356 } 4357 4358 bpf_for_each_spilled_reg(i, state, reg) { 4359 if (!reg) 4360 continue; 4361 if (reg->type == type && reg->id == dst_reg->id) 4362 reg->range = max(reg->range, new_range); 4363 } 4364 } 4365 4366 static void find_good_pkt_pointers(struct bpf_verifier_state *vstate, 4367 struct bpf_reg_state *dst_reg, 4368 enum bpf_reg_type type, 4369 bool range_right_open) 4370 { 4371 u16 new_range; 4372 int i; 4373 4374 if (dst_reg->off < 0 || 4375 (dst_reg->off == 0 && range_right_open)) 4376 /* This doesn't give us any range */ 4377 return; 4378 4379 if (dst_reg->umax_value > MAX_PACKET_OFF || 4380 dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF) 4381 /* Risk of overflow. For instance, ptr + (1<<63) may be less 4382 * than pkt_end, but that's because it's also less than pkt. 4383 */ 4384 return; 4385 4386 new_range = dst_reg->off; 4387 if (range_right_open) 4388 new_range--; 4389 4390 /* Examples for register markings: 4391 * 4392 * pkt_data in dst register: 4393 * 4394 * r2 = r3; 4395 * r2 += 8; 4396 * if (r2 > pkt_end) goto <handle exception> 4397 * <access okay> 4398 * 4399 * r2 = r3; 4400 * r2 += 8; 4401 * if (r2 < pkt_end) goto <access okay> 4402 * <handle exception> 4403 * 4404 * Where: 4405 * r2 == dst_reg, pkt_end == src_reg 4406 * r2=pkt(id=n,off=8,r=0) 4407 * r3=pkt(id=n,off=0,r=0) 4408 * 4409 * pkt_data in src register: 4410 * 4411 * r2 = r3; 4412 * r2 += 8; 4413 * if (pkt_end >= r2) goto <access okay> 4414 * <handle exception> 4415 * 4416 * r2 = r3; 4417 * r2 += 8; 4418 * if (pkt_end <= r2) goto <handle exception> 4419 * <access okay> 4420 * 4421 * Where: 4422 * pkt_end == dst_reg, r2 == src_reg 4423 * r2=pkt(id=n,off=8,r=0) 4424 * r3=pkt(id=n,off=0,r=0) 4425 * 4426 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8) 4427 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8) 4428 * and [r3, r3 + 8-1) respectively is safe to access depending on 4429 * the check. 4430 */ 4431 4432 /* If our ids match, then we must have the same max_value. And we 4433 * don't care about the other reg's fixed offset, since if it's too big 4434 * the range won't allow anything. 4435 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16. 4436 */ 4437 for (i = 0; i <= vstate->curframe; i++) 4438 __find_good_pkt_pointers(vstate->frame[i], dst_reg, type, 4439 new_range); 4440 } 4441 4442 /* compute branch direction of the expression "if (reg opcode val) goto target;" 4443 * and return: 4444 * 1 - branch will be taken and "goto target" will be executed 4445 * 0 - branch will not be taken and fall-through to next insn 4446 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value range [0,10] 4447 */ 4448 static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode, 4449 bool is_jmp32) 4450 { 4451 struct bpf_reg_state reg_lo; 4452 s64 sval; 4453 4454 if (__is_pointer_value(false, reg)) 4455 return -1; 4456 4457 if (is_jmp32) { 4458 reg_lo = *reg; 4459 reg = ®_lo; 4460 /* For JMP32, only low 32 bits are compared, coerce_reg_to_size 4461 * could truncate high bits and update umin/umax according to 4462 * information of low bits. 4463 */ 4464 coerce_reg_to_size(reg, 4); 4465 /* smin/smax need special handling. For example, after coerce, 4466 * if smin_value is 0x00000000ffffffffLL, the value is -1 when 4467 * used as operand to JMP32. It is a negative number from s32's 4468 * point of view, while it is a positive number when seen as 4469 * s64. The smin/smax are kept as s64, therefore, when used with 4470 * JMP32, they need to be transformed into s32, then sign 4471 * extended back to s64. 4472 * 4473 * Also, smin/smax were copied from umin/umax. If umin/umax has 4474 * different sign bit, then min/max relationship doesn't 4475 * maintain after casting into s32, for this case, set smin/smax 4476 * to safest range. 4477 */ 4478 if ((reg->umax_value ^ reg->umin_value) & 4479 (1ULL << 31)) { 4480 reg->smin_value = S32_MIN; 4481 reg->smax_value = S32_MAX; 4482 } 4483 reg->smin_value = (s64)(s32)reg->smin_value; 4484 reg->smax_value = (s64)(s32)reg->smax_value; 4485 4486 val = (u32)val; 4487 sval = (s64)(s32)val; 4488 } else { 4489 sval = (s64)val; 4490 } 4491 4492 switch (opcode) { 4493 case BPF_JEQ: 4494 if (tnum_is_const(reg->var_off)) 4495 return !!tnum_equals_const(reg->var_off, val); 4496 break; 4497 case BPF_JNE: 4498 if (tnum_is_const(reg->var_off)) 4499 return !tnum_equals_const(reg->var_off, val); 4500 break; 4501 case BPF_JSET: 4502 if ((~reg->var_off.mask & reg->var_off.value) & val) 4503 return 1; 4504 if (!((reg->var_off.mask | reg->var_off.value) & val)) 4505 return 0; 4506 break; 4507 case BPF_JGT: 4508 if (reg->umin_value > val) 4509 return 1; 4510 else if (reg->umax_value <= val) 4511 return 0; 4512 break; 4513 case BPF_JSGT: 4514 if (reg->smin_value > sval) 4515 return 1; 4516 else if (reg->smax_value < sval) 4517 return 0; 4518 break; 4519 case BPF_JLT: 4520 if (reg->umax_value < val) 4521 return 1; 4522 else if (reg->umin_value >= val) 4523 return 0; 4524 break; 4525 case BPF_JSLT: 4526 if (reg->smax_value < sval) 4527 return 1; 4528 else if (reg->smin_value >= sval) 4529 return 0; 4530 break; 4531 case BPF_JGE: 4532 if (reg->umin_value >= val) 4533 return 1; 4534 else if (reg->umax_value < val) 4535 return 0; 4536 break; 4537 case BPF_JSGE: 4538 if (reg->smin_value >= sval) 4539 return 1; 4540 else if (reg->smax_value < sval) 4541 return 0; 4542 break; 4543 case BPF_JLE: 4544 if (reg->umax_value <= val) 4545 return 1; 4546 else if (reg->umin_value > val) 4547 return 0; 4548 break; 4549 case BPF_JSLE: 4550 if (reg->smax_value <= sval) 4551 return 1; 4552 else if (reg->smin_value > sval) 4553 return 0; 4554 break; 4555 } 4556 4557 return -1; 4558 } 4559 4560 /* Generate min value of the high 32-bit from TNUM info. */ 4561 static u64 gen_hi_min(struct tnum var) 4562 { 4563 return var.value & ~0xffffffffULL; 4564 } 4565 4566 /* Generate max value of the high 32-bit from TNUM info. */ 4567 static u64 gen_hi_max(struct tnum var) 4568 { 4569 return (var.value | var.mask) & ~0xffffffffULL; 4570 } 4571 4572 /* Return true if VAL is compared with a s64 sign extended from s32, and they 4573 * are with the same signedness. 4574 */ 4575 static bool cmp_val_with_extended_s64(s64 sval, struct bpf_reg_state *reg) 4576 { 4577 return ((s32)sval >= 0 && 4578 reg->smin_value >= 0 && reg->smax_value <= S32_MAX) || 4579 ((s32)sval < 0 && 4580 reg->smax_value <= 0 && reg->smin_value >= S32_MIN); 4581 } 4582 4583 /* Adjusts the register min/max values in the case that the dst_reg is the 4584 * variable register that we are working on, and src_reg is a constant or we're 4585 * simply doing a BPF_K check. 4586 * In JEQ/JNE cases we also adjust the var_off values. 4587 */ 4588 static void reg_set_min_max(struct bpf_reg_state *true_reg, 4589 struct bpf_reg_state *false_reg, u64 val, 4590 u8 opcode, bool is_jmp32) 4591 { 4592 s64 sval; 4593 4594 /* If the dst_reg is a pointer, we can't learn anything about its 4595 * variable offset from the compare (unless src_reg were a pointer into 4596 * the same object, but we don't bother with that. 4597 * Since false_reg and true_reg have the same type by construction, we 4598 * only need to check one of them for pointerness. 4599 */ 4600 if (__is_pointer_value(false, false_reg)) 4601 return; 4602 4603 val = is_jmp32 ? (u32)val : val; 4604 sval = is_jmp32 ? (s64)(s32)val : (s64)val; 4605 4606 switch (opcode) { 4607 case BPF_JEQ: 4608 case BPF_JNE: 4609 { 4610 struct bpf_reg_state *reg = 4611 opcode == BPF_JEQ ? true_reg : false_reg; 4612 4613 /* For BPF_JEQ, if this is false we know nothing Jon Snow, but 4614 * if it is true we know the value for sure. Likewise for 4615 * BPF_JNE. 4616 */ 4617 if (is_jmp32) { 4618 u64 old_v = reg->var_off.value; 4619 u64 hi_mask = ~0xffffffffULL; 4620 4621 reg->var_off.value = (old_v & hi_mask) | val; 4622 reg->var_off.mask &= hi_mask; 4623 } else { 4624 __mark_reg_known(reg, val); 4625 } 4626 break; 4627 } 4628 case BPF_JSET: 4629 false_reg->var_off = tnum_and(false_reg->var_off, 4630 tnum_const(~val)); 4631 if (is_power_of_2(val)) 4632 true_reg->var_off = tnum_or(true_reg->var_off, 4633 tnum_const(val)); 4634 break; 4635 case BPF_JGE: 4636 case BPF_JGT: 4637 { 4638 u64 false_umax = opcode == BPF_JGT ? val : val - 1; 4639 u64 true_umin = opcode == BPF_JGT ? val + 1 : val; 4640 4641 if (is_jmp32) { 4642 false_umax += gen_hi_max(false_reg->var_off); 4643 true_umin += gen_hi_min(true_reg->var_off); 4644 } 4645 false_reg->umax_value = min(false_reg->umax_value, false_umax); 4646 true_reg->umin_value = max(true_reg->umin_value, true_umin); 4647 break; 4648 } 4649 case BPF_JSGE: 4650 case BPF_JSGT: 4651 { 4652 s64 false_smax = opcode == BPF_JSGT ? sval : sval - 1; 4653 s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval; 4654 4655 /* If the full s64 was not sign-extended from s32 then don't 4656 * deduct further info. 4657 */ 4658 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) 4659 break; 4660 false_reg->smax_value = min(false_reg->smax_value, false_smax); 4661 true_reg->smin_value = max(true_reg->smin_value, true_smin); 4662 break; 4663 } 4664 case BPF_JLE: 4665 case BPF_JLT: 4666 { 4667 u64 false_umin = opcode == BPF_JLT ? val : val + 1; 4668 u64 true_umax = opcode == BPF_JLT ? val - 1 : val; 4669 4670 if (is_jmp32) { 4671 false_umin += gen_hi_min(false_reg->var_off); 4672 true_umax += gen_hi_max(true_reg->var_off); 4673 } 4674 false_reg->umin_value = max(false_reg->umin_value, false_umin); 4675 true_reg->umax_value = min(true_reg->umax_value, true_umax); 4676 break; 4677 } 4678 case BPF_JSLE: 4679 case BPF_JSLT: 4680 { 4681 s64 false_smin = opcode == BPF_JSLT ? sval : sval + 1; 4682 s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval; 4683 4684 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) 4685 break; 4686 false_reg->smin_value = max(false_reg->smin_value, false_smin); 4687 true_reg->smax_value = min(true_reg->smax_value, true_smax); 4688 break; 4689 } 4690 default: 4691 break; 4692 } 4693 4694 __reg_deduce_bounds(false_reg); 4695 __reg_deduce_bounds(true_reg); 4696 /* We might have learned some bits from the bounds. */ 4697 __reg_bound_offset(false_reg); 4698 __reg_bound_offset(true_reg); 4699 /* Intersecting with the old var_off might have improved our bounds 4700 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 4701 * then new var_off is (0; 0x7f...fc) which improves our umax. 4702 */ 4703 __update_reg_bounds(false_reg); 4704 __update_reg_bounds(true_reg); 4705 } 4706 4707 /* Same as above, but for the case that dst_reg holds a constant and src_reg is 4708 * the variable reg. 4709 */ 4710 static void reg_set_min_max_inv(struct bpf_reg_state *true_reg, 4711 struct bpf_reg_state *false_reg, u64 val, 4712 u8 opcode, bool is_jmp32) 4713 { 4714 s64 sval; 4715 4716 if (__is_pointer_value(false, false_reg)) 4717 return; 4718 4719 val = is_jmp32 ? (u32)val : val; 4720 sval = is_jmp32 ? (s64)(s32)val : (s64)val; 4721 4722 switch (opcode) { 4723 case BPF_JEQ: 4724 case BPF_JNE: 4725 { 4726 struct bpf_reg_state *reg = 4727 opcode == BPF_JEQ ? true_reg : false_reg; 4728 4729 if (is_jmp32) { 4730 u64 old_v = reg->var_off.value; 4731 u64 hi_mask = ~0xffffffffULL; 4732 4733 reg->var_off.value = (old_v & hi_mask) | val; 4734 reg->var_off.mask &= hi_mask; 4735 } else { 4736 __mark_reg_known(reg, val); 4737 } 4738 break; 4739 } 4740 case BPF_JSET: 4741 false_reg->var_off = tnum_and(false_reg->var_off, 4742 tnum_const(~val)); 4743 if (is_power_of_2(val)) 4744 true_reg->var_off = tnum_or(true_reg->var_off, 4745 tnum_const(val)); 4746 break; 4747 case BPF_JGE: 4748 case BPF_JGT: 4749 { 4750 u64 false_umin = opcode == BPF_JGT ? val : val + 1; 4751 u64 true_umax = opcode == BPF_JGT ? val - 1 : val; 4752 4753 if (is_jmp32) { 4754 false_umin += gen_hi_min(false_reg->var_off); 4755 true_umax += gen_hi_max(true_reg->var_off); 4756 } 4757 false_reg->umin_value = max(false_reg->umin_value, false_umin); 4758 true_reg->umax_value = min(true_reg->umax_value, true_umax); 4759 break; 4760 } 4761 case BPF_JSGE: 4762 case BPF_JSGT: 4763 { 4764 s64 false_smin = opcode == BPF_JSGT ? sval : sval + 1; 4765 s64 true_smax = opcode == BPF_JSGT ? sval - 1 : sval; 4766 4767 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) 4768 break; 4769 false_reg->smin_value = max(false_reg->smin_value, false_smin); 4770 true_reg->smax_value = min(true_reg->smax_value, true_smax); 4771 break; 4772 } 4773 case BPF_JLE: 4774 case BPF_JLT: 4775 { 4776 u64 false_umax = opcode == BPF_JLT ? val : val - 1; 4777 u64 true_umin = opcode == BPF_JLT ? val + 1 : val; 4778 4779 if (is_jmp32) { 4780 false_umax += gen_hi_max(false_reg->var_off); 4781 true_umin += gen_hi_min(true_reg->var_off); 4782 } 4783 false_reg->umax_value = min(false_reg->umax_value, false_umax); 4784 true_reg->umin_value = max(true_reg->umin_value, true_umin); 4785 break; 4786 } 4787 case BPF_JSLE: 4788 case BPF_JSLT: 4789 { 4790 s64 false_smax = opcode == BPF_JSLT ? sval : sval - 1; 4791 s64 true_smin = opcode == BPF_JSLT ? sval + 1 : sval; 4792 4793 if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg)) 4794 break; 4795 false_reg->smax_value = min(false_reg->smax_value, false_smax); 4796 true_reg->smin_value = max(true_reg->smin_value, true_smin); 4797 break; 4798 } 4799 default: 4800 break; 4801 } 4802 4803 __reg_deduce_bounds(false_reg); 4804 __reg_deduce_bounds(true_reg); 4805 /* We might have learned some bits from the bounds. */ 4806 __reg_bound_offset(false_reg); 4807 __reg_bound_offset(true_reg); 4808 /* Intersecting with the old var_off might have improved our bounds 4809 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 4810 * then new var_off is (0; 0x7f...fc) which improves our umax. 4811 */ 4812 __update_reg_bounds(false_reg); 4813 __update_reg_bounds(true_reg); 4814 } 4815 4816 /* Regs are known to be equal, so intersect their min/max/var_off */ 4817 static void __reg_combine_min_max(struct bpf_reg_state *src_reg, 4818 struct bpf_reg_state *dst_reg) 4819 { 4820 src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value, 4821 dst_reg->umin_value); 4822 src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value, 4823 dst_reg->umax_value); 4824 src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value, 4825 dst_reg->smin_value); 4826 src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value, 4827 dst_reg->smax_value); 4828 src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off, 4829 dst_reg->var_off); 4830 /* We might have learned new bounds from the var_off. */ 4831 __update_reg_bounds(src_reg); 4832 __update_reg_bounds(dst_reg); 4833 /* We might have learned something about the sign bit. */ 4834 __reg_deduce_bounds(src_reg); 4835 __reg_deduce_bounds(dst_reg); 4836 /* We might have learned some bits from the bounds. */ 4837 __reg_bound_offset(src_reg); 4838 __reg_bound_offset(dst_reg); 4839 /* Intersecting with the old var_off might have improved our bounds 4840 * slightly. e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc), 4841 * then new var_off is (0; 0x7f...fc) which improves our umax. 4842 */ 4843 __update_reg_bounds(src_reg); 4844 __update_reg_bounds(dst_reg); 4845 } 4846 4847 static void reg_combine_min_max(struct bpf_reg_state *true_src, 4848 struct bpf_reg_state *true_dst, 4849 struct bpf_reg_state *false_src, 4850 struct bpf_reg_state *false_dst, 4851 u8 opcode) 4852 { 4853 switch (opcode) { 4854 case BPF_JEQ: 4855 __reg_combine_min_max(true_src, true_dst); 4856 break; 4857 case BPF_JNE: 4858 __reg_combine_min_max(false_src, false_dst); 4859 break; 4860 } 4861 } 4862 4863 static void mark_ptr_or_null_reg(struct bpf_func_state *state, 4864 struct bpf_reg_state *reg, u32 id, 4865 bool is_null) 4866 { 4867 if (reg_type_may_be_null(reg->type) && reg->id == id) { 4868 /* Old offset (both fixed and variable parts) should 4869 * have been known-zero, because we don't allow pointer 4870 * arithmetic on pointers that might be NULL. 4871 */ 4872 if (WARN_ON_ONCE(reg->smin_value || reg->smax_value || 4873 !tnum_equals_const(reg->var_off, 0) || 4874 reg->off)) { 4875 __mark_reg_known_zero(reg); 4876 reg->off = 0; 4877 } 4878 if (is_null) { 4879 reg->type = SCALAR_VALUE; 4880 } else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) { 4881 if (reg->map_ptr->inner_map_meta) { 4882 reg->type = CONST_PTR_TO_MAP; 4883 reg->map_ptr = reg->map_ptr->inner_map_meta; 4884 } else { 4885 reg->type = PTR_TO_MAP_VALUE; 4886 } 4887 } else if (reg->type == PTR_TO_SOCKET_OR_NULL) { 4888 reg->type = PTR_TO_SOCKET; 4889 } else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) { 4890 reg->type = PTR_TO_SOCK_COMMON; 4891 } else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) { 4892 reg->type = PTR_TO_TCP_SOCK; 4893 } 4894 if (is_null) { 4895 /* We don't need id and ref_obj_id from this point 4896 * onwards anymore, thus we should better reset it, 4897 * so that state pruning has chances to take effect. 4898 */ 4899 reg->id = 0; 4900 reg->ref_obj_id = 0; 4901 } else if (!reg_may_point_to_spin_lock(reg)) { 4902 /* For not-NULL ptr, reg->ref_obj_id will be reset 4903 * in release_reg_references(). 4904 * 4905 * reg->id is still used by spin_lock ptr. Other 4906 * than spin_lock ptr type, reg->id can be reset. 4907 */ 4908 reg->id = 0; 4909 } 4910 } 4911 } 4912 4913 static void __mark_ptr_or_null_regs(struct bpf_func_state *state, u32 id, 4914 bool is_null) 4915 { 4916 struct bpf_reg_state *reg; 4917 int i; 4918 4919 for (i = 0; i < MAX_BPF_REG; i++) 4920 mark_ptr_or_null_reg(state, &state->regs[i], id, is_null); 4921 4922 bpf_for_each_spilled_reg(i, state, reg) { 4923 if (!reg) 4924 continue; 4925 mark_ptr_or_null_reg(state, reg, id, is_null); 4926 } 4927 } 4928 4929 /* The logic is similar to find_good_pkt_pointers(), both could eventually 4930 * be folded together at some point. 4931 */ 4932 static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno, 4933 bool is_null) 4934 { 4935 struct bpf_func_state *state = vstate->frame[vstate->curframe]; 4936 struct bpf_reg_state *regs = state->regs; 4937 u32 ref_obj_id = regs[regno].ref_obj_id; 4938 u32 id = regs[regno].id; 4939 int i; 4940 4941 if (ref_obj_id && ref_obj_id == id && is_null) 4942 /* regs[regno] is in the " == NULL" branch. 4943 * No one could have freed the reference state before 4944 * doing the NULL check. 4945 */ 4946 WARN_ON_ONCE(release_reference_state(state, id)); 4947 4948 for (i = 0; i <= vstate->curframe; i++) 4949 __mark_ptr_or_null_regs(vstate->frame[i], id, is_null); 4950 } 4951 4952 static bool try_match_pkt_pointers(const struct bpf_insn *insn, 4953 struct bpf_reg_state *dst_reg, 4954 struct bpf_reg_state *src_reg, 4955 struct bpf_verifier_state *this_branch, 4956 struct bpf_verifier_state *other_branch) 4957 { 4958 if (BPF_SRC(insn->code) != BPF_X) 4959 return false; 4960 4961 /* Pointers are always 64-bit. */ 4962 if (BPF_CLASS(insn->code) == BPF_JMP32) 4963 return false; 4964 4965 switch (BPF_OP(insn->code)) { 4966 case BPF_JGT: 4967 if ((dst_reg->type == PTR_TO_PACKET && 4968 src_reg->type == PTR_TO_PACKET_END) || 4969 (dst_reg->type == PTR_TO_PACKET_META && 4970 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 4971 /* pkt_data' > pkt_end, pkt_meta' > pkt_data */ 4972 find_good_pkt_pointers(this_branch, dst_reg, 4973 dst_reg->type, false); 4974 } else if ((dst_reg->type == PTR_TO_PACKET_END && 4975 src_reg->type == PTR_TO_PACKET) || 4976 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 4977 src_reg->type == PTR_TO_PACKET_META)) { 4978 /* pkt_end > pkt_data', pkt_data > pkt_meta' */ 4979 find_good_pkt_pointers(other_branch, src_reg, 4980 src_reg->type, true); 4981 } else { 4982 return false; 4983 } 4984 break; 4985 case BPF_JLT: 4986 if ((dst_reg->type == PTR_TO_PACKET && 4987 src_reg->type == PTR_TO_PACKET_END) || 4988 (dst_reg->type == PTR_TO_PACKET_META && 4989 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 4990 /* pkt_data' < pkt_end, pkt_meta' < pkt_data */ 4991 find_good_pkt_pointers(other_branch, dst_reg, 4992 dst_reg->type, true); 4993 } else if ((dst_reg->type == PTR_TO_PACKET_END && 4994 src_reg->type == PTR_TO_PACKET) || 4995 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 4996 src_reg->type == PTR_TO_PACKET_META)) { 4997 /* pkt_end < pkt_data', pkt_data > pkt_meta' */ 4998 find_good_pkt_pointers(this_branch, src_reg, 4999 src_reg->type, false); 5000 } else { 5001 return false; 5002 } 5003 break; 5004 case BPF_JGE: 5005 if ((dst_reg->type == PTR_TO_PACKET && 5006 src_reg->type == PTR_TO_PACKET_END) || 5007 (dst_reg->type == PTR_TO_PACKET_META && 5008 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 5009 /* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */ 5010 find_good_pkt_pointers(this_branch, dst_reg, 5011 dst_reg->type, true); 5012 } else if ((dst_reg->type == PTR_TO_PACKET_END && 5013 src_reg->type == PTR_TO_PACKET) || 5014 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 5015 src_reg->type == PTR_TO_PACKET_META)) { 5016 /* pkt_end >= pkt_data', pkt_data >= pkt_meta' */ 5017 find_good_pkt_pointers(other_branch, src_reg, 5018 src_reg->type, false); 5019 } else { 5020 return false; 5021 } 5022 break; 5023 case BPF_JLE: 5024 if ((dst_reg->type == PTR_TO_PACKET && 5025 src_reg->type == PTR_TO_PACKET_END) || 5026 (dst_reg->type == PTR_TO_PACKET_META && 5027 reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) { 5028 /* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */ 5029 find_good_pkt_pointers(other_branch, dst_reg, 5030 dst_reg->type, false); 5031 } else if ((dst_reg->type == PTR_TO_PACKET_END && 5032 src_reg->type == PTR_TO_PACKET) || 5033 (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) && 5034 src_reg->type == PTR_TO_PACKET_META)) { 5035 /* pkt_end <= pkt_data', pkt_data <= pkt_meta' */ 5036 find_good_pkt_pointers(this_branch, src_reg, 5037 src_reg->type, true); 5038 } else { 5039 return false; 5040 } 5041 break; 5042 default: 5043 return false; 5044 } 5045 5046 return true; 5047 } 5048 5049 static int check_cond_jmp_op(struct bpf_verifier_env *env, 5050 struct bpf_insn *insn, int *insn_idx) 5051 { 5052 struct bpf_verifier_state *this_branch = env->cur_state; 5053 struct bpf_verifier_state *other_branch; 5054 struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs; 5055 struct bpf_reg_state *dst_reg, *other_branch_regs; 5056 u8 opcode = BPF_OP(insn->code); 5057 bool is_jmp32; 5058 int err; 5059 5060 /* Only conditional jumps are expected to reach here. */ 5061 if (opcode == BPF_JA || opcode > BPF_JSLE) { 5062 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode); 5063 return -EINVAL; 5064 } 5065 5066 if (BPF_SRC(insn->code) == BPF_X) { 5067 if (insn->imm != 0) { 5068 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 5069 return -EINVAL; 5070 } 5071 5072 /* check src1 operand */ 5073 err = check_reg_arg(env, insn->src_reg, SRC_OP); 5074 if (err) 5075 return err; 5076 5077 if (is_pointer_value(env, insn->src_reg)) { 5078 verbose(env, "R%d pointer comparison prohibited\n", 5079 insn->src_reg); 5080 return -EACCES; 5081 } 5082 } else { 5083 if (insn->src_reg != BPF_REG_0) { 5084 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n"); 5085 return -EINVAL; 5086 } 5087 } 5088 5089 /* check src2 operand */ 5090 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 5091 if (err) 5092 return err; 5093 5094 dst_reg = ®s[insn->dst_reg]; 5095 is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32; 5096 5097 if (BPF_SRC(insn->code) == BPF_K) { 5098 int pred = is_branch_taken(dst_reg, insn->imm, opcode, 5099 is_jmp32); 5100 5101 if (pred == 1) { 5102 /* only follow the goto, ignore fall-through */ 5103 *insn_idx += insn->off; 5104 return 0; 5105 } else if (pred == 0) { 5106 /* only follow fall-through branch, since 5107 * that's where the program will go 5108 */ 5109 return 0; 5110 } 5111 } 5112 5113 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, 5114 false); 5115 if (!other_branch) 5116 return -EFAULT; 5117 other_branch_regs = other_branch->frame[other_branch->curframe]->regs; 5118 5119 /* detect if we are comparing against a constant value so we can adjust 5120 * our min/max values for our dst register. 5121 * this is only legit if both are scalars (or pointers to the same 5122 * object, I suppose, but we don't support that right now), because 5123 * otherwise the different base pointers mean the offsets aren't 5124 * comparable. 5125 */ 5126 if (BPF_SRC(insn->code) == BPF_X) { 5127 struct bpf_reg_state *src_reg = ®s[insn->src_reg]; 5128 struct bpf_reg_state lo_reg0 = *dst_reg; 5129 struct bpf_reg_state lo_reg1 = *src_reg; 5130 struct bpf_reg_state *src_lo, *dst_lo; 5131 5132 dst_lo = &lo_reg0; 5133 src_lo = &lo_reg1; 5134 coerce_reg_to_size(dst_lo, 4); 5135 coerce_reg_to_size(src_lo, 4); 5136 5137 if (dst_reg->type == SCALAR_VALUE && 5138 src_reg->type == SCALAR_VALUE) { 5139 if (tnum_is_const(src_reg->var_off) || 5140 (is_jmp32 && tnum_is_const(src_lo->var_off))) 5141 reg_set_min_max(&other_branch_regs[insn->dst_reg], 5142 dst_reg, 5143 is_jmp32 5144 ? src_lo->var_off.value 5145 : src_reg->var_off.value, 5146 opcode, is_jmp32); 5147 else if (tnum_is_const(dst_reg->var_off) || 5148 (is_jmp32 && tnum_is_const(dst_lo->var_off))) 5149 reg_set_min_max_inv(&other_branch_regs[insn->src_reg], 5150 src_reg, 5151 is_jmp32 5152 ? dst_lo->var_off.value 5153 : dst_reg->var_off.value, 5154 opcode, is_jmp32); 5155 else if (!is_jmp32 && 5156 (opcode == BPF_JEQ || opcode == BPF_JNE)) 5157 /* Comparing for equality, we can combine knowledge */ 5158 reg_combine_min_max(&other_branch_regs[insn->src_reg], 5159 &other_branch_regs[insn->dst_reg], 5160 src_reg, dst_reg, opcode); 5161 } 5162 } else if (dst_reg->type == SCALAR_VALUE) { 5163 reg_set_min_max(&other_branch_regs[insn->dst_reg], 5164 dst_reg, insn->imm, opcode, is_jmp32); 5165 } 5166 5167 /* detect if R == 0 where R is returned from bpf_map_lookup_elem(). 5168 * NOTE: these optimizations below are related with pointer comparison 5169 * which will never be JMP32. 5170 */ 5171 if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K && 5172 insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) && 5173 reg_type_may_be_null(dst_reg->type)) { 5174 /* Mark all identical registers in each branch as either 5175 * safe or unknown depending R == 0 or R != 0 conditional. 5176 */ 5177 mark_ptr_or_null_regs(this_branch, insn->dst_reg, 5178 opcode == BPF_JNE); 5179 mark_ptr_or_null_regs(other_branch, insn->dst_reg, 5180 opcode == BPF_JEQ); 5181 } else if (!try_match_pkt_pointers(insn, dst_reg, ®s[insn->src_reg], 5182 this_branch, other_branch) && 5183 is_pointer_value(env, insn->dst_reg)) { 5184 verbose(env, "R%d pointer comparison prohibited\n", 5185 insn->dst_reg); 5186 return -EACCES; 5187 } 5188 if (env->log.level & BPF_LOG_LEVEL) 5189 print_verifier_state(env, this_branch->frame[this_branch->curframe]); 5190 return 0; 5191 } 5192 5193 /* verify BPF_LD_IMM64 instruction */ 5194 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn) 5195 { 5196 struct bpf_insn_aux_data *aux = cur_aux(env); 5197 struct bpf_reg_state *regs = cur_regs(env); 5198 struct bpf_map *map; 5199 int err; 5200 5201 if (BPF_SIZE(insn->code) != BPF_DW) { 5202 verbose(env, "invalid BPF_LD_IMM insn\n"); 5203 return -EINVAL; 5204 } 5205 if (insn->off != 0) { 5206 verbose(env, "BPF_LD_IMM64 uses reserved fields\n"); 5207 return -EINVAL; 5208 } 5209 5210 err = check_reg_arg(env, insn->dst_reg, DST_OP); 5211 if (err) 5212 return err; 5213 5214 if (insn->src_reg == 0) { 5215 u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm; 5216 5217 regs[insn->dst_reg].type = SCALAR_VALUE; 5218 __mark_reg_known(®s[insn->dst_reg], imm); 5219 return 0; 5220 } 5221 5222 map = env->used_maps[aux->map_index]; 5223 mark_reg_known_zero(env, regs, insn->dst_reg); 5224 regs[insn->dst_reg].map_ptr = map; 5225 5226 if (insn->src_reg == BPF_PSEUDO_MAP_VALUE) { 5227 regs[insn->dst_reg].type = PTR_TO_MAP_VALUE; 5228 regs[insn->dst_reg].off = aux->map_off; 5229 if (map_value_has_spin_lock(map)) 5230 regs[insn->dst_reg].id = ++env->id_gen; 5231 } else if (insn->src_reg == BPF_PSEUDO_MAP_FD) { 5232 regs[insn->dst_reg].type = CONST_PTR_TO_MAP; 5233 } else { 5234 verbose(env, "bpf verifier is misconfigured\n"); 5235 return -EINVAL; 5236 } 5237 5238 return 0; 5239 } 5240 5241 static bool may_access_skb(enum bpf_prog_type type) 5242 { 5243 switch (type) { 5244 case BPF_PROG_TYPE_SOCKET_FILTER: 5245 case BPF_PROG_TYPE_SCHED_CLS: 5246 case BPF_PROG_TYPE_SCHED_ACT: 5247 return true; 5248 default: 5249 return false; 5250 } 5251 } 5252 5253 /* verify safety of LD_ABS|LD_IND instructions: 5254 * - they can only appear in the programs where ctx == skb 5255 * - since they are wrappers of function calls, they scratch R1-R5 registers, 5256 * preserve R6-R9, and store return value into R0 5257 * 5258 * Implicit input: 5259 * ctx == skb == R6 == CTX 5260 * 5261 * Explicit input: 5262 * SRC == any register 5263 * IMM == 32-bit immediate 5264 * 5265 * Output: 5266 * R0 - 8/16/32-bit skb data converted to cpu endianness 5267 */ 5268 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn) 5269 { 5270 struct bpf_reg_state *regs = cur_regs(env); 5271 u8 mode = BPF_MODE(insn->code); 5272 int i, err; 5273 5274 if (!may_access_skb(env->prog->type)) { 5275 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n"); 5276 return -EINVAL; 5277 } 5278 5279 if (!env->ops->gen_ld_abs) { 5280 verbose(env, "bpf verifier is misconfigured\n"); 5281 return -EINVAL; 5282 } 5283 5284 if (env->subprog_cnt > 1) { 5285 /* when program has LD_ABS insn JITs and interpreter assume 5286 * that r1 == ctx == skb which is not the case for callees 5287 * that can have arbitrary arguments. It's problematic 5288 * for main prog as well since JITs would need to analyze 5289 * all functions in order to make proper register save/restore 5290 * decisions in the main prog. Hence disallow LD_ABS with calls 5291 */ 5292 verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n"); 5293 return -EINVAL; 5294 } 5295 5296 if (insn->dst_reg != BPF_REG_0 || insn->off != 0 || 5297 BPF_SIZE(insn->code) == BPF_DW || 5298 (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) { 5299 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n"); 5300 return -EINVAL; 5301 } 5302 5303 /* check whether implicit source operand (register R6) is readable */ 5304 err = check_reg_arg(env, BPF_REG_6, SRC_OP); 5305 if (err) 5306 return err; 5307 5308 /* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as 5309 * gen_ld_abs() may terminate the program at runtime, leading to 5310 * reference leak. 5311 */ 5312 err = check_reference_leak(env); 5313 if (err) { 5314 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n"); 5315 return err; 5316 } 5317 5318 if (env->cur_state->active_spin_lock) { 5319 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n"); 5320 return -EINVAL; 5321 } 5322 5323 if (regs[BPF_REG_6].type != PTR_TO_CTX) { 5324 verbose(env, 5325 "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n"); 5326 return -EINVAL; 5327 } 5328 5329 if (mode == BPF_IND) { 5330 /* check explicit source operand */ 5331 err = check_reg_arg(env, insn->src_reg, SRC_OP); 5332 if (err) 5333 return err; 5334 } 5335 5336 /* reset caller saved regs to unreadable */ 5337 for (i = 0; i < CALLER_SAVED_REGS; i++) { 5338 mark_reg_not_init(env, regs, caller_saved[i]); 5339 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK); 5340 } 5341 5342 /* mark destination R0 register as readable, since it contains 5343 * the value fetched from the packet. 5344 * Already marked as written above. 5345 */ 5346 mark_reg_unknown(env, regs, BPF_REG_0); 5347 return 0; 5348 } 5349 5350 static int check_return_code(struct bpf_verifier_env *env) 5351 { 5352 struct bpf_reg_state *reg; 5353 struct tnum range = tnum_range(0, 1); 5354 5355 switch (env->prog->type) { 5356 case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: 5357 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG || 5358 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG) 5359 range = tnum_range(1, 1); 5360 case BPF_PROG_TYPE_CGROUP_SKB: 5361 case BPF_PROG_TYPE_CGROUP_SOCK: 5362 case BPF_PROG_TYPE_SOCK_OPS: 5363 case BPF_PROG_TYPE_CGROUP_DEVICE: 5364 case BPF_PROG_TYPE_CGROUP_SYSCTL: 5365 break; 5366 default: 5367 return 0; 5368 } 5369 5370 reg = cur_regs(env) + BPF_REG_0; 5371 if (reg->type != SCALAR_VALUE) { 5372 verbose(env, "At program exit the register R0 is not a known value (%s)\n", 5373 reg_type_str[reg->type]); 5374 return -EINVAL; 5375 } 5376 5377 if (!tnum_in(range, reg->var_off)) { 5378 char tn_buf[48]; 5379 5380 verbose(env, "At program exit the register R0 "); 5381 if (!tnum_is_unknown(reg->var_off)) { 5382 tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off); 5383 verbose(env, "has value %s", tn_buf); 5384 } else { 5385 verbose(env, "has unknown scalar value"); 5386 } 5387 tnum_strn(tn_buf, sizeof(tn_buf), range); 5388 verbose(env, " should have been in %s\n", tn_buf); 5389 return -EINVAL; 5390 } 5391 return 0; 5392 } 5393 5394 /* non-recursive DFS pseudo code 5395 * 1 procedure DFS-iterative(G,v): 5396 * 2 label v as discovered 5397 * 3 let S be a stack 5398 * 4 S.push(v) 5399 * 5 while S is not empty 5400 * 6 t <- S.pop() 5401 * 7 if t is what we're looking for: 5402 * 8 return t 5403 * 9 for all edges e in G.adjacentEdges(t) do 5404 * 10 if edge e is already labelled 5405 * 11 continue with the next edge 5406 * 12 w <- G.adjacentVertex(t,e) 5407 * 13 if vertex w is not discovered and not explored 5408 * 14 label e as tree-edge 5409 * 15 label w as discovered 5410 * 16 S.push(w) 5411 * 17 continue at 5 5412 * 18 else if vertex w is discovered 5413 * 19 label e as back-edge 5414 * 20 else 5415 * 21 // vertex w is explored 5416 * 22 label e as forward- or cross-edge 5417 * 23 label t as explored 5418 * 24 S.pop() 5419 * 5420 * convention: 5421 * 0x10 - discovered 5422 * 0x11 - discovered and fall-through edge labelled 5423 * 0x12 - discovered and fall-through and branch edges labelled 5424 * 0x20 - explored 5425 */ 5426 5427 enum { 5428 DISCOVERED = 0x10, 5429 EXPLORED = 0x20, 5430 FALLTHROUGH = 1, 5431 BRANCH = 2, 5432 }; 5433 5434 #define STATE_LIST_MARK ((struct bpf_verifier_state_list *) -1L) 5435 5436 /* t, w, e - match pseudo-code above: 5437 * t - index of current instruction 5438 * w - next instruction 5439 * e - edge 5440 */ 5441 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env) 5442 { 5443 int *insn_stack = env->cfg.insn_stack; 5444 int *insn_state = env->cfg.insn_state; 5445 5446 if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH)) 5447 return 0; 5448 5449 if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH)) 5450 return 0; 5451 5452 if (w < 0 || w >= env->prog->len) { 5453 verbose_linfo(env, t, "%d: ", t); 5454 verbose(env, "jump out of range from insn %d to %d\n", t, w); 5455 return -EINVAL; 5456 } 5457 5458 if (e == BRANCH) 5459 /* mark branch target for state pruning */ 5460 env->explored_states[w] = STATE_LIST_MARK; 5461 5462 if (insn_state[w] == 0) { 5463 /* tree-edge */ 5464 insn_state[t] = DISCOVERED | e; 5465 insn_state[w] = DISCOVERED; 5466 if (env->cfg.cur_stack >= env->prog->len) 5467 return -E2BIG; 5468 insn_stack[env->cfg.cur_stack++] = w; 5469 return 1; 5470 } else if ((insn_state[w] & 0xF0) == DISCOVERED) { 5471 verbose_linfo(env, t, "%d: ", t); 5472 verbose_linfo(env, w, "%d: ", w); 5473 verbose(env, "back-edge from insn %d to %d\n", t, w); 5474 return -EINVAL; 5475 } else if (insn_state[w] == EXPLORED) { 5476 /* forward- or cross-edge */ 5477 insn_state[t] = DISCOVERED | e; 5478 } else { 5479 verbose(env, "insn state internal bug\n"); 5480 return -EFAULT; 5481 } 5482 return 0; 5483 } 5484 5485 /* non-recursive depth-first-search to detect loops in BPF program 5486 * loop == back-edge in directed graph 5487 */ 5488 static int check_cfg(struct bpf_verifier_env *env) 5489 { 5490 struct bpf_insn *insns = env->prog->insnsi; 5491 int insn_cnt = env->prog->len; 5492 int *insn_stack, *insn_state; 5493 int ret = 0; 5494 int i, t; 5495 5496 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 5497 if (!insn_state) 5498 return -ENOMEM; 5499 5500 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL); 5501 if (!insn_stack) { 5502 kvfree(insn_state); 5503 return -ENOMEM; 5504 } 5505 5506 insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */ 5507 insn_stack[0] = 0; /* 0 is the first instruction */ 5508 env->cfg.cur_stack = 1; 5509 5510 peek_stack: 5511 if (env->cfg.cur_stack == 0) 5512 goto check_state; 5513 t = insn_stack[env->cfg.cur_stack - 1]; 5514 5515 if (BPF_CLASS(insns[t].code) == BPF_JMP || 5516 BPF_CLASS(insns[t].code) == BPF_JMP32) { 5517 u8 opcode = BPF_OP(insns[t].code); 5518 5519 if (opcode == BPF_EXIT) { 5520 goto mark_explored; 5521 } else if (opcode == BPF_CALL) { 5522 ret = push_insn(t, t + 1, FALLTHROUGH, env); 5523 if (ret == 1) 5524 goto peek_stack; 5525 else if (ret < 0) 5526 goto err_free; 5527 if (t + 1 < insn_cnt) 5528 env->explored_states[t + 1] = STATE_LIST_MARK; 5529 if (insns[t].src_reg == BPF_PSEUDO_CALL) { 5530 env->explored_states[t] = STATE_LIST_MARK; 5531 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env); 5532 if (ret == 1) 5533 goto peek_stack; 5534 else if (ret < 0) 5535 goto err_free; 5536 } 5537 } else if (opcode == BPF_JA) { 5538 if (BPF_SRC(insns[t].code) != BPF_K) { 5539 ret = -EINVAL; 5540 goto err_free; 5541 } 5542 /* unconditional jump with single edge */ 5543 ret = push_insn(t, t + insns[t].off + 1, 5544 FALLTHROUGH, env); 5545 if (ret == 1) 5546 goto peek_stack; 5547 else if (ret < 0) 5548 goto err_free; 5549 /* tell verifier to check for equivalent states 5550 * after every call and jump 5551 */ 5552 if (t + 1 < insn_cnt) 5553 env->explored_states[t + 1] = STATE_LIST_MARK; 5554 } else { 5555 /* conditional jump with two edges */ 5556 env->explored_states[t] = STATE_LIST_MARK; 5557 ret = push_insn(t, t + 1, FALLTHROUGH, env); 5558 if (ret == 1) 5559 goto peek_stack; 5560 else if (ret < 0) 5561 goto err_free; 5562 5563 ret = push_insn(t, t + insns[t].off + 1, BRANCH, env); 5564 if (ret == 1) 5565 goto peek_stack; 5566 else if (ret < 0) 5567 goto err_free; 5568 } 5569 } else { 5570 /* all other non-branch instructions with single 5571 * fall-through edge 5572 */ 5573 ret = push_insn(t, t + 1, FALLTHROUGH, env); 5574 if (ret == 1) 5575 goto peek_stack; 5576 else if (ret < 0) 5577 goto err_free; 5578 } 5579 5580 mark_explored: 5581 insn_state[t] = EXPLORED; 5582 if (env->cfg.cur_stack-- <= 0) { 5583 verbose(env, "pop stack internal bug\n"); 5584 ret = -EFAULT; 5585 goto err_free; 5586 } 5587 goto peek_stack; 5588 5589 check_state: 5590 for (i = 0; i < insn_cnt; i++) { 5591 if (insn_state[i] != EXPLORED) { 5592 verbose(env, "unreachable insn %d\n", i); 5593 ret = -EINVAL; 5594 goto err_free; 5595 } 5596 } 5597 ret = 0; /* cfg looks good */ 5598 5599 err_free: 5600 kvfree(insn_state); 5601 kvfree(insn_stack); 5602 env->cfg.insn_state = env->cfg.insn_stack = NULL; 5603 return ret; 5604 } 5605 5606 /* The minimum supported BTF func info size */ 5607 #define MIN_BPF_FUNCINFO_SIZE 8 5608 #define MAX_FUNCINFO_REC_SIZE 252 5609 5610 static int check_btf_func(struct bpf_verifier_env *env, 5611 const union bpf_attr *attr, 5612 union bpf_attr __user *uattr) 5613 { 5614 u32 i, nfuncs, urec_size, min_size; 5615 u32 krec_size = sizeof(struct bpf_func_info); 5616 struct bpf_func_info *krecord; 5617 const struct btf_type *type; 5618 struct bpf_prog *prog; 5619 const struct btf *btf; 5620 void __user *urecord; 5621 u32 prev_offset = 0; 5622 int ret = 0; 5623 5624 nfuncs = attr->func_info_cnt; 5625 if (!nfuncs) 5626 return 0; 5627 5628 if (nfuncs != env->subprog_cnt) { 5629 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n"); 5630 return -EINVAL; 5631 } 5632 5633 urec_size = attr->func_info_rec_size; 5634 if (urec_size < MIN_BPF_FUNCINFO_SIZE || 5635 urec_size > MAX_FUNCINFO_REC_SIZE || 5636 urec_size % sizeof(u32)) { 5637 verbose(env, "invalid func info rec size %u\n", urec_size); 5638 return -EINVAL; 5639 } 5640 5641 prog = env->prog; 5642 btf = prog->aux->btf; 5643 5644 urecord = u64_to_user_ptr(attr->func_info); 5645 min_size = min_t(u32, krec_size, urec_size); 5646 5647 krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN); 5648 if (!krecord) 5649 return -ENOMEM; 5650 5651 for (i = 0; i < nfuncs; i++) { 5652 ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size); 5653 if (ret) { 5654 if (ret == -E2BIG) { 5655 verbose(env, "nonzero tailing record in func info"); 5656 /* set the size kernel expects so loader can zero 5657 * out the rest of the record. 5658 */ 5659 if (put_user(min_size, &uattr->func_info_rec_size)) 5660 ret = -EFAULT; 5661 } 5662 goto err_free; 5663 } 5664 5665 if (copy_from_user(&krecord[i], urecord, min_size)) { 5666 ret = -EFAULT; 5667 goto err_free; 5668 } 5669 5670 /* check insn_off */ 5671 if (i == 0) { 5672 if (krecord[i].insn_off) { 5673 verbose(env, 5674 "nonzero insn_off %u for the first func info record", 5675 krecord[i].insn_off); 5676 ret = -EINVAL; 5677 goto err_free; 5678 } 5679 } else if (krecord[i].insn_off <= prev_offset) { 5680 verbose(env, 5681 "same or smaller insn offset (%u) than previous func info record (%u)", 5682 krecord[i].insn_off, prev_offset); 5683 ret = -EINVAL; 5684 goto err_free; 5685 } 5686 5687 if (env->subprog_info[i].start != krecord[i].insn_off) { 5688 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n"); 5689 ret = -EINVAL; 5690 goto err_free; 5691 } 5692 5693 /* check type_id */ 5694 type = btf_type_by_id(btf, krecord[i].type_id); 5695 if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) { 5696 verbose(env, "invalid type id %d in func info", 5697 krecord[i].type_id); 5698 ret = -EINVAL; 5699 goto err_free; 5700 } 5701 5702 prev_offset = krecord[i].insn_off; 5703 urecord += urec_size; 5704 } 5705 5706 prog->aux->func_info = krecord; 5707 prog->aux->func_info_cnt = nfuncs; 5708 return 0; 5709 5710 err_free: 5711 kvfree(krecord); 5712 return ret; 5713 } 5714 5715 static void adjust_btf_func(struct bpf_verifier_env *env) 5716 { 5717 int i; 5718 5719 if (!env->prog->aux->func_info) 5720 return; 5721 5722 for (i = 0; i < env->subprog_cnt; i++) 5723 env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start; 5724 } 5725 5726 #define MIN_BPF_LINEINFO_SIZE (offsetof(struct bpf_line_info, line_col) + \ 5727 sizeof(((struct bpf_line_info *)(0))->line_col)) 5728 #define MAX_LINEINFO_REC_SIZE MAX_FUNCINFO_REC_SIZE 5729 5730 static int check_btf_line(struct bpf_verifier_env *env, 5731 const union bpf_attr *attr, 5732 union bpf_attr __user *uattr) 5733 { 5734 u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0; 5735 struct bpf_subprog_info *sub; 5736 struct bpf_line_info *linfo; 5737 struct bpf_prog *prog; 5738 const struct btf *btf; 5739 void __user *ulinfo; 5740 int err; 5741 5742 nr_linfo = attr->line_info_cnt; 5743 if (!nr_linfo) 5744 return 0; 5745 5746 rec_size = attr->line_info_rec_size; 5747 if (rec_size < MIN_BPF_LINEINFO_SIZE || 5748 rec_size > MAX_LINEINFO_REC_SIZE || 5749 rec_size & (sizeof(u32) - 1)) 5750 return -EINVAL; 5751 5752 /* Need to zero it in case the userspace may 5753 * pass in a smaller bpf_line_info object. 5754 */ 5755 linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info), 5756 GFP_KERNEL | __GFP_NOWARN); 5757 if (!linfo) 5758 return -ENOMEM; 5759 5760 prog = env->prog; 5761 btf = prog->aux->btf; 5762 5763 s = 0; 5764 sub = env->subprog_info; 5765 ulinfo = u64_to_user_ptr(attr->line_info); 5766 expected_size = sizeof(struct bpf_line_info); 5767 ncopy = min_t(u32, expected_size, rec_size); 5768 for (i = 0; i < nr_linfo; i++) { 5769 err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size); 5770 if (err) { 5771 if (err == -E2BIG) { 5772 verbose(env, "nonzero tailing record in line_info"); 5773 if (put_user(expected_size, 5774 &uattr->line_info_rec_size)) 5775 err = -EFAULT; 5776 } 5777 goto err_free; 5778 } 5779 5780 if (copy_from_user(&linfo[i], ulinfo, ncopy)) { 5781 err = -EFAULT; 5782 goto err_free; 5783 } 5784 5785 /* 5786 * Check insn_off to ensure 5787 * 1) strictly increasing AND 5788 * 2) bounded by prog->len 5789 * 5790 * The linfo[0].insn_off == 0 check logically falls into 5791 * the later "missing bpf_line_info for func..." case 5792 * because the first linfo[0].insn_off must be the 5793 * first sub also and the first sub must have 5794 * subprog_info[0].start == 0. 5795 */ 5796 if ((i && linfo[i].insn_off <= prev_offset) || 5797 linfo[i].insn_off >= prog->len) { 5798 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", 5799 i, linfo[i].insn_off, prev_offset, 5800 prog->len); 5801 err = -EINVAL; 5802 goto err_free; 5803 } 5804 5805 if (!prog->insnsi[linfo[i].insn_off].code) { 5806 verbose(env, 5807 "Invalid insn code at line_info[%u].insn_off\n", 5808 i); 5809 err = -EINVAL; 5810 goto err_free; 5811 } 5812 5813 if (!btf_name_by_offset(btf, linfo[i].line_off) || 5814 !btf_name_by_offset(btf, linfo[i].file_name_off)) { 5815 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i); 5816 err = -EINVAL; 5817 goto err_free; 5818 } 5819 5820 if (s != env->subprog_cnt) { 5821 if (linfo[i].insn_off == sub[s].start) { 5822 sub[s].linfo_idx = i; 5823 s++; 5824 } else if (sub[s].start < linfo[i].insn_off) { 5825 verbose(env, "missing bpf_line_info for func#%u\n", s); 5826 err = -EINVAL; 5827 goto err_free; 5828 } 5829 } 5830 5831 prev_offset = linfo[i].insn_off; 5832 ulinfo += rec_size; 5833 } 5834 5835 if (s != env->subprog_cnt) { 5836 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", 5837 env->subprog_cnt - s, s); 5838 err = -EINVAL; 5839 goto err_free; 5840 } 5841 5842 prog->aux->linfo = linfo; 5843 prog->aux->nr_linfo = nr_linfo; 5844 5845 return 0; 5846 5847 err_free: 5848 kvfree(linfo); 5849 return err; 5850 } 5851 5852 static int check_btf_info(struct bpf_verifier_env *env, 5853 const union bpf_attr *attr, 5854 union bpf_attr __user *uattr) 5855 { 5856 struct btf *btf; 5857 int err; 5858 5859 if (!attr->func_info_cnt && !attr->line_info_cnt) 5860 return 0; 5861 5862 btf = btf_get_by_fd(attr->prog_btf_fd); 5863 if (IS_ERR(btf)) 5864 return PTR_ERR(btf); 5865 env->prog->aux->btf = btf; 5866 5867 err = check_btf_func(env, attr, uattr); 5868 if (err) 5869 return err; 5870 5871 err = check_btf_line(env, attr, uattr); 5872 if (err) 5873 return err; 5874 5875 return 0; 5876 } 5877 5878 /* check %cur's range satisfies %old's */ 5879 static bool range_within(struct bpf_reg_state *old, 5880 struct bpf_reg_state *cur) 5881 { 5882 return old->umin_value <= cur->umin_value && 5883 old->umax_value >= cur->umax_value && 5884 old->smin_value <= cur->smin_value && 5885 old->smax_value >= cur->smax_value; 5886 } 5887 5888 /* Maximum number of register states that can exist at once */ 5889 #define ID_MAP_SIZE (MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) 5890 struct idpair { 5891 u32 old; 5892 u32 cur; 5893 }; 5894 5895 /* If in the old state two registers had the same id, then they need to have 5896 * the same id in the new state as well. But that id could be different from 5897 * the old state, so we need to track the mapping from old to new ids. 5898 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent 5899 * regs with old id 5 must also have new id 9 for the new state to be safe. But 5900 * regs with a different old id could still have new id 9, we don't care about 5901 * that. 5902 * So we look through our idmap to see if this old id has been seen before. If 5903 * so, we require the new id to match; otherwise, we add the id pair to the map. 5904 */ 5905 static bool check_ids(u32 old_id, u32 cur_id, struct idpair *idmap) 5906 { 5907 unsigned int i; 5908 5909 for (i = 0; i < ID_MAP_SIZE; i++) { 5910 if (!idmap[i].old) { 5911 /* Reached an empty slot; haven't seen this id before */ 5912 idmap[i].old = old_id; 5913 idmap[i].cur = cur_id; 5914 return true; 5915 } 5916 if (idmap[i].old == old_id) 5917 return idmap[i].cur == cur_id; 5918 } 5919 /* We ran out of idmap slots, which should be impossible */ 5920 WARN_ON_ONCE(1); 5921 return false; 5922 } 5923 5924 static void clean_func_state(struct bpf_verifier_env *env, 5925 struct bpf_func_state *st) 5926 { 5927 enum bpf_reg_liveness live; 5928 int i, j; 5929 5930 for (i = 0; i < BPF_REG_FP; i++) { 5931 live = st->regs[i].live; 5932 /* liveness must not touch this register anymore */ 5933 st->regs[i].live |= REG_LIVE_DONE; 5934 if (!(live & REG_LIVE_READ)) 5935 /* since the register is unused, clear its state 5936 * to make further comparison simpler 5937 */ 5938 __mark_reg_not_init(&st->regs[i]); 5939 } 5940 5941 for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) { 5942 live = st->stack[i].spilled_ptr.live; 5943 /* liveness must not touch this stack slot anymore */ 5944 st->stack[i].spilled_ptr.live |= REG_LIVE_DONE; 5945 if (!(live & REG_LIVE_READ)) { 5946 __mark_reg_not_init(&st->stack[i].spilled_ptr); 5947 for (j = 0; j < BPF_REG_SIZE; j++) 5948 st->stack[i].slot_type[j] = STACK_INVALID; 5949 } 5950 } 5951 } 5952 5953 static void clean_verifier_state(struct bpf_verifier_env *env, 5954 struct bpf_verifier_state *st) 5955 { 5956 int i; 5957 5958 if (st->frame[0]->regs[0].live & REG_LIVE_DONE) 5959 /* all regs in this state in all frames were already marked */ 5960 return; 5961 5962 for (i = 0; i <= st->curframe; i++) 5963 clean_func_state(env, st->frame[i]); 5964 } 5965 5966 /* the parentage chains form a tree. 5967 * the verifier states are added to state lists at given insn and 5968 * pushed into state stack for future exploration. 5969 * when the verifier reaches bpf_exit insn some of the verifer states 5970 * stored in the state lists have their final liveness state already, 5971 * but a lot of states will get revised from liveness point of view when 5972 * the verifier explores other branches. 5973 * Example: 5974 * 1: r0 = 1 5975 * 2: if r1 == 100 goto pc+1 5976 * 3: r0 = 2 5977 * 4: exit 5978 * when the verifier reaches exit insn the register r0 in the state list of 5979 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch 5980 * of insn 2 and goes exploring further. At the insn 4 it will walk the 5981 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ. 5982 * 5983 * Since the verifier pushes the branch states as it sees them while exploring 5984 * the program the condition of walking the branch instruction for the second 5985 * time means that all states below this branch were already explored and 5986 * their final liveness markes are already propagated. 5987 * Hence when the verifier completes the search of state list in is_state_visited() 5988 * we can call this clean_live_states() function to mark all liveness states 5989 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state' 5990 * will not be used. 5991 * This function also clears the registers and stack for states that !READ 5992 * to simplify state merging. 5993 * 5994 * Important note here that walking the same branch instruction in the callee 5995 * doesn't meant that the states are DONE. The verifier has to compare 5996 * the callsites 5997 */ 5998 static void clean_live_states(struct bpf_verifier_env *env, int insn, 5999 struct bpf_verifier_state *cur) 6000 { 6001 struct bpf_verifier_state_list *sl; 6002 int i; 6003 6004 sl = env->explored_states[insn]; 6005 if (!sl) 6006 return; 6007 6008 while (sl != STATE_LIST_MARK) { 6009 if (sl->state.curframe != cur->curframe) 6010 goto next; 6011 for (i = 0; i <= cur->curframe; i++) 6012 if (sl->state.frame[i]->callsite != cur->frame[i]->callsite) 6013 goto next; 6014 clean_verifier_state(env, &sl->state); 6015 next: 6016 sl = sl->next; 6017 } 6018 } 6019 6020 /* Returns true if (rold safe implies rcur safe) */ 6021 static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur, 6022 struct idpair *idmap) 6023 { 6024 bool equal; 6025 6026 if (!(rold->live & REG_LIVE_READ)) 6027 /* explored state didn't use this */ 6028 return true; 6029 6030 equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0; 6031 6032 if (rold->type == PTR_TO_STACK) 6033 /* two stack pointers are equal only if they're pointing to 6034 * the same stack frame, since fp-8 in foo != fp-8 in bar 6035 */ 6036 return equal && rold->frameno == rcur->frameno; 6037 6038 if (equal) 6039 return true; 6040 6041 if (rold->type == NOT_INIT) 6042 /* explored state can't have used this */ 6043 return true; 6044 if (rcur->type == NOT_INIT) 6045 return false; 6046 switch (rold->type) { 6047 case SCALAR_VALUE: 6048 if (rcur->type == SCALAR_VALUE) { 6049 /* new val must satisfy old val knowledge */ 6050 return range_within(rold, rcur) && 6051 tnum_in(rold->var_off, rcur->var_off); 6052 } else { 6053 /* We're trying to use a pointer in place of a scalar. 6054 * Even if the scalar was unbounded, this could lead to 6055 * pointer leaks because scalars are allowed to leak 6056 * while pointers are not. We could make this safe in 6057 * special cases if root is calling us, but it's 6058 * probably not worth the hassle. 6059 */ 6060 return false; 6061 } 6062 case PTR_TO_MAP_VALUE: 6063 /* If the new min/max/var_off satisfy the old ones and 6064 * everything else matches, we are OK. 6065 * 'id' is not compared, since it's only used for maps with 6066 * bpf_spin_lock inside map element and in such cases if 6067 * the rest of the prog is valid for one map element then 6068 * it's valid for all map elements regardless of the key 6069 * used in bpf_map_lookup() 6070 */ 6071 return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 && 6072 range_within(rold, rcur) && 6073 tnum_in(rold->var_off, rcur->var_off); 6074 case PTR_TO_MAP_VALUE_OR_NULL: 6075 /* a PTR_TO_MAP_VALUE could be safe to use as a 6076 * PTR_TO_MAP_VALUE_OR_NULL into the same map. 6077 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL- 6078 * checked, doing so could have affected others with the same 6079 * id, and we can't check for that because we lost the id when 6080 * we converted to a PTR_TO_MAP_VALUE. 6081 */ 6082 if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL) 6083 return false; 6084 if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id))) 6085 return false; 6086 /* Check our ids match any regs they're supposed to */ 6087 return check_ids(rold->id, rcur->id, idmap); 6088 case PTR_TO_PACKET_META: 6089 case PTR_TO_PACKET: 6090 if (rcur->type != rold->type) 6091 return false; 6092 /* We must have at least as much range as the old ptr 6093 * did, so that any accesses which were safe before are 6094 * still safe. This is true even if old range < old off, 6095 * since someone could have accessed through (ptr - k), or 6096 * even done ptr -= k in a register, to get a safe access. 6097 */ 6098 if (rold->range > rcur->range) 6099 return false; 6100 /* If the offsets don't match, we can't trust our alignment; 6101 * nor can we be sure that we won't fall out of range. 6102 */ 6103 if (rold->off != rcur->off) 6104 return false; 6105 /* id relations must be preserved */ 6106 if (rold->id && !check_ids(rold->id, rcur->id, idmap)) 6107 return false; 6108 /* new val must satisfy old val knowledge */ 6109 return range_within(rold, rcur) && 6110 tnum_in(rold->var_off, rcur->var_off); 6111 case PTR_TO_CTX: 6112 case CONST_PTR_TO_MAP: 6113 case PTR_TO_PACKET_END: 6114 case PTR_TO_FLOW_KEYS: 6115 case PTR_TO_SOCKET: 6116 case PTR_TO_SOCKET_OR_NULL: 6117 case PTR_TO_SOCK_COMMON: 6118 case PTR_TO_SOCK_COMMON_OR_NULL: 6119 case PTR_TO_TCP_SOCK: 6120 case PTR_TO_TCP_SOCK_OR_NULL: 6121 /* Only valid matches are exact, which memcmp() above 6122 * would have accepted 6123 */ 6124 default: 6125 /* Don't know what's going on, just say it's not safe */ 6126 return false; 6127 } 6128 6129 /* Shouldn't get here; if we do, say it's not safe */ 6130 WARN_ON_ONCE(1); 6131 return false; 6132 } 6133 6134 static bool stacksafe(struct bpf_func_state *old, 6135 struct bpf_func_state *cur, 6136 struct idpair *idmap) 6137 { 6138 int i, spi; 6139 6140 /* walk slots of the explored stack and ignore any additional 6141 * slots in the current stack, since explored(safe) state 6142 * didn't use them 6143 */ 6144 for (i = 0; i < old->allocated_stack; i++) { 6145 spi = i / BPF_REG_SIZE; 6146 6147 if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) { 6148 i += BPF_REG_SIZE - 1; 6149 /* explored state didn't use this */ 6150 continue; 6151 } 6152 6153 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID) 6154 continue; 6155 6156 /* explored stack has more populated slots than current stack 6157 * and these slots were used 6158 */ 6159 if (i >= cur->allocated_stack) 6160 return false; 6161 6162 /* if old state was safe with misc data in the stack 6163 * it will be safe with zero-initialized stack. 6164 * The opposite is not true 6165 */ 6166 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC && 6167 cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO) 6168 continue; 6169 if (old->stack[spi].slot_type[i % BPF_REG_SIZE] != 6170 cur->stack[spi].slot_type[i % BPF_REG_SIZE]) 6171 /* Ex: old explored (safe) state has STACK_SPILL in 6172 * this stack slot, but current has has STACK_MISC -> 6173 * this verifier states are not equivalent, 6174 * return false to continue verification of this path 6175 */ 6176 return false; 6177 if (i % BPF_REG_SIZE) 6178 continue; 6179 if (old->stack[spi].slot_type[0] != STACK_SPILL) 6180 continue; 6181 if (!regsafe(&old->stack[spi].spilled_ptr, 6182 &cur->stack[spi].spilled_ptr, 6183 idmap)) 6184 /* when explored and current stack slot are both storing 6185 * spilled registers, check that stored pointers types 6186 * are the same as well. 6187 * Ex: explored safe path could have stored 6188 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8} 6189 * but current path has stored: 6190 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16} 6191 * such verifier states are not equivalent. 6192 * return false to continue verification of this path 6193 */ 6194 return false; 6195 } 6196 return true; 6197 } 6198 6199 static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur) 6200 { 6201 if (old->acquired_refs != cur->acquired_refs) 6202 return false; 6203 return !memcmp(old->refs, cur->refs, 6204 sizeof(*old->refs) * old->acquired_refs); 6205 } 6206 6207 /* compare two verifier states 6208 * 6209 * all states stored in state_list are known to be valid, since 6210 * verifier reached 'bpf_exit' instruction through them 6211 * 6212 * this function is called when verifier exploring different branches of 6213 * execution popped from the state stack. If it sees an old state that has 6214 * more strict register state and more strict stack state then this execution 6215 * branch doesn't need to be explored further, since verifier already 6216 * concluded that more strict state leads to valid finish. 6217 * 6218 * Therefore two states are equivalent if register state is more conservative 6219 * and explored stack state is more conservative than the current one. 6220 * Example: 6221 * explored current 6222 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC) 6223 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC) 6224 * 6225 * In other words if current stack state (one being explored) has more 6226 * valid slots than old one that already passed validation, it means 6227 * the verifier can stop exploring and conclude that current state is valid too 6228 * 6229 * Similarly with registers. If explored state has register type as invalid 6230 * whereas register type in current state is meaningful, it means that 6231 * the current state will reach 'bpf_exit' instruction safely 6232 */ 6233 static bool func_states_equal(struct bpf_func_state *old, 6234 struct bpf_func_state *cur) 6235 { 6236 struct idpair *idmap; 6237 bool ret = false; 6238 int i; 6239 6240 idmap = kcalloc(ID_MAP_SIZE, sizeof(struct idpair), GFP_KERNEL); 6241 /* If we failed to allocate the idmap, just say it's not safe */ 6242 if (!idmap) 6243 return false; 6244 6245 for (i = 0; i < MAX_BPF_REG; i++) { 6246 if (!regsafe(&old->regs[i], &cur->regs[i], idmap)) 6247 goto out_free; 6248 } 6249 6250 if (!stacksafe(old, cur, idmap)) 6251 goto out_free; 6252 6253 if (!refsafe(old, cur)) 6254 goto out_free; 6255 ret = true; 6256 out_free: 6257 kfree(idmap); 6258 return ret; 6259 } 6260 6261 static bool states_equal(struct bpf_verifier_env *env, 6262 struct bpf_verifier_state *old, 6263 struct bpf_verifier_state *cur) 6264 { 6265 int i; 6266 6267 if (old->curframe != cur->curframe) 6268 return false; 6269 6270 /* Verification state from speculative execution simulation 6271 * must never prune a non-speculative execution one. 6272 */ 6273 if (old->speculative && !cur->speculative) 6274 return false; 6275 6276 if (old->active_spin_lock != cur->active_spin_lock) 6277 return false; 6278 6279 /* for states to be equal callsites have to be the same 6280 * and all frame states need to be equivalent 6281 */ 6282 for (i = 0; i <= old->curframe; i++) { 6283 if (old->frame[i]->callsite != cur->frame[i]->callsite) 6284 return false; 6285 if (!func_states_equal(old->frame[i], cur->frame[i])) 6286 return false; 6287 } 6288 return true; 6289 } 6290 6291 static int propagate_liveness_reg(struct bpf_verifier_env *env, 6292 struct bpf_reg_state *reg, 6293 struct bpf_reg_state *parent_reg) 6294 { 6295 int err; 6296 6297 if (parent_reg->live & REG_LIVE_READ || !(reg->live & REG_LIVE_READ)) 6298 return 0; 6299 6300 err = mark_reg_read(env, reg, parent_reg); 6301 if (err) 6302 return err; 6303 6304 return 0; 6305 } 6306 6307 /* A write screens off any subsequent reads; but write marks come from the 6308 * straight-line code between a state and its parent. When we arrive at an 6309 * equivalent state (jump target or such) we didn't arrive by the straight-line 6310 * code, so read marks in the state must propagate to the parent regardless 6311 * of the state's write marks. That's what 'parent == state->parent' comparison 6312 * in mark_reg_read() is for. 6313 */ 6314 static int propagate_liveness(struct bpf_verifier_env *env, 6315 const struct bpf_verifier_state *vstate, 6316 struct bpf_verifier_state *vparent) 6317 { 6318 struct bpf_reg_state *state_reg, *parent_reg; 6319 struct bpf_func_state *state, *parent; 6320 int i, frame, err = 0; 6321 6322 if (vparent->curframe != vstate->curframe) { 6323 WARN(1, "propagate_live: parent frame %d current frame %d\n", 6324 vparent->curframe, vstate->curframe); 6325 return -EFAULT; 6326 } 6327 /* Propagate read liveness of registers... */ 6328 BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG); 6329 for (frame = 0; frame <= vstate->curframe; frame++) { 6330 parent = vparent->frame[frame]; 6331 state = vstate->frame[frame]; 6332 parent_reg = parent->regs; 6333 state_reg = state->regs; 6334 /* We don't need to worry about FP liveness, it's read-only */ 6335 for (i = frame < vstate->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) { 6336 err = propagate_liveness_reg(env, &state_reg[i], 6337 &parent_reg[i]); 6338 if (err) 6339 return err; 6340 } 6341 6342 /* Propagate stack slots. */ 6343 for (i = 0; i < state->allocated_stack / BPF_REG_SIZE && 6344 i < parent->allocated_stack / BPF_REG_SIZE; i++) { 6345 parent_reg = &parent->stack[i].spilled_ptr; 6346 state_reg = &state->stack[i].spilled_ptr; 6347 err = propagate_liveness_reg(env, state_reg, 6348 parent_reg); 6349 if (err) 6350 return err; 6351 } 6352 } 6353 return err; 6354 } 6355 6356 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx) 6357 { 6358 struct bpf_verifier_state_list *new_sl; 6359 struct bpf_verifier_state_list *sl, **pprev; 6360 struct bpf_verifier_state *cur = env->cur_state, *new; 6361 int i, j, err, states_cnt = 0; 6362 6363 pprev = &env->explored_states[insn_idx]; 6364 sl = *pprev; 6365 6366 if (!sl) 6367 /* this 'insn_idx' instruction wasn't marked, so we will not 6368 * be doing state search here 6369 */ 6370 return 0; 6371 6372 clean_live_states(env, insn_idx, cur); 6373 6374 while (sl != STATE_LIST_MARK) { 6375 if (states_equal(env, &sl->state, cur)) { 6376 sl->hit_cnt++; 6377 /* reached equivalent register/stack state, 6378 * prune the search. 6379 * Registers read by the continuation are read by us. 6380 * If we have any write marks in env->cur_state, they 6381 * will prevent corresponding reads in the continuation 6382 * from reaching our parent (an explored_state). Our 6383 * own state will get the read marks recorded, but 6384 * they'll be immediately forgotten as we're pruning 6385 * this state and will pop a new one. 6386 */ 6387 err = propagate_liveness(env, &sl->state, cur); 6388 if (err) 6389 return err; 6390 return 1; 6391 } 6392 states_cnt++; 6393 sl->miss_cnt++; 6394 /* heuristic to determine whether this state is beneficial 6395 * to keep checking from state equivalence point of view. 6396 * Higher numbers increase max_states_per_insn and verification time, 6397 * but do not meaningfully decrease insn_processed. 6398 */ 6399 if (sl->miss_cnt > sl->hit_cnt * 3 + 3) { 6400 /* the state is unlikely to be useful. Remove it to 6401 * speed up verification 6402 */ 6403 *pprev = sl->next; 6404 if (sl->state.frame[0]->regs[0].live & REG_LIVE_DONE) { 6405 free_verifier_state(&sl->state, false); 6406 kfree(sl); 6407 env->peak_states--; 6408 } else { 6409 /* cannot free this state, since parentage chain may 6410 * walk it later. Add it for free_list instead to 6411 * be freed at the end of verification 6412 */ 6413 sl->next = env->free_list; 6414 env->free_list = sl; 6415 } 6416 sl = *pprev; 6417 continue; 6418 } 6419 pprev = &sl->next; 6420 sl = *pprev; 6421 } 6422 6423 if (env->max_states_per_insn < states_cnt) 6424 env->max_states_per_insn = states_cnt; 6425 6426 if (!env->allow_ptr_leaks && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) 6427 return 0; 6428 6429 /* there were no equivalent states, remember current one. 6430 * technically the current state is not proven to be safe yet, 6431 * but it will either reach outer most bpf_exit (which means it's safe) 6432 * or it will be rejected. Since there are no loops, we won't be 6433 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx) 6434 * again on the way to bpf_exit 6435 */ 6436 new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL); 6437 if (!new_sl) 6438 return -ENOMEM; 6439 env->total_states++; 6440 env->peak_states++; 6441 6442 /* add new state to the head of linked list */ 6443 new = &new_sl->state; 6444 err = copy_verifier_state(new, cur); 6445 if (err) { 6446 free_verifier_state(new, false); 6447 kfree(new_sl); 6448 return err; 6449 } 6450 new_sl->next = env->explored_states[insn_idx]; 6451 env->explored_states[insn_idx] = new_sl; 6452 /* connect new state to parentage chain. Current frame needs all 6453 * registers connected. Only r6 - r9 of the callers are alive (pushed 6454 * to the stack implicitly by JITs) so in callers' frames connect just 6455 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to 6456 * the state of the call instruction (with WRITTEN set), and r0 comes 6457 * from callee with its full parentage chain, anyway. 6458 */ 6459 for (j = 0; j <= cur->curframe; j++) 6460 for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++) 6461 cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i]; 6462 /* clear write marks in current state: the writes we did are not writes 6463 * our child did, so they don't screen off its reads from us. 6464 * (There are no read marks in current state, because reads always mark 6465 * their parent and current state never has children yet. Only 6466 * explored_states can get read marks.) 6467 */ 6468 for (i = 0; i < BPF_REG_FP; i++) 6469 cur->frame[cur->curframe]->regs[i].live = REG_LIVE_NONE; 6470 6471 /* all stack frames are accessible from callee, clear them all */ 6472 for (j = 0; j <= cur->curframe; j++) { 6473 struct bpf_func_state *frame = cur->frame[j]; 6474 struct bpf_func_state *newframe = new->frame[j]; 6475 6476 for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) { 6477 frame->stack[i].spilled_ptr.live = REG_LIVE_NONE; 6478 frame->stack[i].spilled_ptr.parent = 6479 &newframe->stack[i].spilled_ptr; 6480 } 6481 } 6482 return 0; 6483 } 6484 6485 /* Return true if it's OK to have the same insn return a different type. */ 6486 static bool reg_type_mismatch_ok(enum bpf_reg_type type) 6487 { 6488 switch (type) { 6489 case PTR_TO_CTX: 6490 case PTR_TO_SOCKET: 6491 case PTR_TO_SOCKET_OR_NULL: 6492 case PTR_TO_SOCK_COMMON: 6493 case PTR_TO_SOCK_COMMON_OR_NULL: 6494 case PTR_TO_TCP_SOCK: 6495 case PTR_TO_TCP_SOCK_OR_NULL: 6496 return false; 6497 default: 6498 return true; 6499 } 6500 } 6501 6502 /* If an instruction was previously used with particular pointer types, then we 6503 * need to be careful to avoid cases such as the below, where it may be ok 6504 * for one branch accessing the pointer, but not ok for the other branch: 6505 * 6506 * R1 = sock_ptr 6507 * goto X; 6508 * ... 6509 * R1 = some_other_valid_ptr; 6510 * goto X; 6511 * ... 6512 * R2 = *(u32 *)(R1 + 0); 6513 */ 6514 static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev) 6515 { 6516 return src != prev && (!reg_type_mismatch_ok(src) || 6517 !reg_type_mismatch_ok(prev)); 6518 } 6519 6520 static int do_check(struct bpf_verifier_env *env) 6521 { 6522 struct bpf_verifier_state *state; 6523 struct bpf_insn *insns = env->prog->insnsi; 6524 struct bpf_reg_state *regs; 6525 int insn_cnt = env->prog->len; 6526 bool do_print_state = false; 6527 6528 env->prev_linfo = NULL; 6529 6530 state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL); 6531 if (!state) 6532 return -ENOMEM; 6533 state->curframe = 0; 6534 state->speculative = false; 6535 state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL); 6536 if (!state->frame[0]) { 6537 kfree(state); 6538 return -ENOMEM; 6539 } 6540 env->cur_state = state; 6541 init_func_state(env, state->frame[0], 6542 BPF_MAIN_FUNC /* callsite */, 6543 0 /* frameno */, 6544 0 /* subprogno, zero == main subprog */); 6545 6546 for (;;) { 6547 struct bpf_insn *insn; 6548 u8 class; 6549 int err; 6550 6551 if (env->insn_idx >= insn_cnt) { 6552 verbose(env, "invalid insn idx %d insn_cnt %d\n", 6553 env->insn_idx, insn_cnt); 6554 return -EFAULT; 6555 } 6556 6557 insn = &insns[env->insn_idx]; 6558 class = BPF_CLASS(insn->code); 6559 6560 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) { 6561 verbose(env, 6562 "BPF program is too large. Processed %d insn\n", 6563 env->insn_processed); 6564 return -E2BIG; 6565 } 6566 6567 err = is_state_visited(env, env->insn_idx); 6568 if (err < 0) 6569 return err; 6570 if (err == 1) { 6571 /* found equivalent state, can prune the search */ 6572 if (env->log.level & BPF_LOG_LEVEL) { 6573 if (do_print_state) 6574 verbose(env, "\nfrom %d to %d%s: safe\n", 6575 env->prev_insn_idx, env->insn_idx, 6576 env->cur_state->speculative ? 6577 " (speculative execution)" : ""); 6578 else 6579 verbose(env, "%d: safe\n", env->insn_idx); 6580 } 6581 goto process_bpf_exit; 6582 } 6583 6584 if (signal_pending(current)) 6585 return -EAGAIN; 6586 6587 if (need_resched()) 6588 cond_resched(); 6589 6590 if (env->log.level & BPF_LOG_LEVEL2 || 6591 (env->log.level & BPF_LOG_LEVEL && do_print_state)) { 6592 if (env->log.level & BPF_LOG_LEVEL2) 6593 verbose(env, "%d:", env->insn_idx); 6594 else 6595 verbose(env, "\nfrom %d to %d%s:", 6596 env->prev_insn_idx, env->insn_idx, 6597 env->cur_state->speculative ? 6598 " (speculative execution)" : ""); 6599 print_verifier_state(env, state->frame[state->curframe]); 6600 do_print_state = false; 6601 } 6602 6603 if (env->log.level & BPF_LOG_LEVEL) { 6604 const struct bpf_insn_cbs cbs = { 6605 .cb_print = verbose, 6606 .private_data = env, 6607 }; 6608 6609 verbose_linfo(env, env->insn_idx, "; "); 6610 verbose(env, "%d: ", env->insn_idx); 6611 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks); 6612 } 6613 6614 if (bpf_prog_is_dev_bound(env->prog->aux)) { 6615 err = bpf_prog_offload_verify_insn(env, env->insn_idx, 6616 env->prev_insn_idx); 6617 if (err) 6618 return err; 6619 } 6620 6621 regs = cur_regs(env); 6622 env->insn_aux_data[env->insn_idx].seen = true; 6623 6624 if (class == BPF_ALU || class == BPF_ALU64) { 6625 err = check_alu_op(env, insn); 6626 if (err) 6627 return err; 6628 6629 } else if (class == BPF_LDX) { 6630 enum bpf_reg_type *prev_src_type, src_reg_type; 6631 6632 /* check for reserved fields is already done */ 6633 6634 /* check src operand */ 6635 err = check_reg_arg(env, insn->src_reg, SRC_OP); 6636 if (err) 6637 return err; 6638 6639 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK); 6640 if (err) 6641 return err; 6642 6643 src_reg_type = regs[insn->src_reg].type; 6644 6645 /* check that memory (src_reg + off) is readable, 6646 * the state of dst_reg will be updated by this func 6647 */ 6648 err = check_mem_access(env, env->insn_idx, insn->src_reg, 6649 insn->off, BPF_SIZE(insn->code), 6650 BPF_READ, insn->dst_reg, false); 6651 if (err) 6652 return err; 6653 6654 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type; 6655 6656 if (*prev_src_type == NOT_INIT) { 6657 /* saw a valid insn 6658 * dst_reg = *(u32 *)(src_reg + off) 6659 * save type to validate intersecting paths 6660 */ 6661 *prev_src_type = src_reg_type; 6662 6663 } else if (reg_type_mismatch(src_reg_type, *prev_src_type)) { 6664 /* ABuser program is trying to use the same insn 6665 * dst_reg = *(u32*) (src_reg + off) 6666 * with different pointer types: 6667 * src_reg == ctx in one branch and 6668 * src_reg == stack|map in some other branch. 6669 * Reject it. 6670 */ 6671 verbose(env, "same insn cannot be used with different pointers\n"); 6672 return -EINVAL; 6673 } 6674 6675 } else if (class == BPF_STX) { 6676 enum bpf_reg_type *prev_dst_type, dst_reg_type; 6677 6678 if (BPF_MODE(insn->code) == BPF_XADD) { 6679 err = check_xadd(env, env->insn_idx, insn); 6680 if (err) 6681 return err; 6682 env->insn_idx++; 6683 continue; 6684 } 6685 6686 /* check src1 operand */ 6687 err = check_reg_arg(env, insn->src_reg, SRC_OP); 6688 if (err) 6689 return err; 6690 /* check src2 operand */ 6691 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 6692 if (err) 6693 return err; 6694 6695 dst_reg_type = regs[insn->dst_reg].type; 6696 6697 /* check that memory (dst_reg + off) is writeable */ 6698 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 6699 insn->off, BPF_SIZE(insn->code), 6700 BPF_WRITE, insn->src_reg, false); 6701 if (err) 6702 return err; 6703 6704 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type; 6705 6706 if (*prev_dst_type == NOT_INIT) { 6707 *prev_dst_type = dst_reg_type; 6708 } else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) { 6709 verbose(env, "same insn cannot be used with different pointers\n"); 6710 return -EINVAL; 6711 } 6712 6713 } else if (class == BPF_ST) { 6714 if (BPF_MODE(insn->code) != BPF_MEM || 6715 insn->src_reg != BPF_REG_0) { 6716 verbose(env, "BPF_ST uses reserved fields\n"); 6717 return -EINVAL; 6718 } 6719 /* check src operand */ 6720 err = check_reg_arg(env, insn->dst_reg, SRC_OP); 6721 if (err) 6722 return err; 6723 6724 if (is_ctx_reg(env, insn->dst_reg)) { 6725 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", 6726 insn->dst_reg, 6727 reg_type_str[reg_state(env, insn->dst_reg)->type]); 6728 return -EACCES; 6729 } 6730 6731 /* check that memory (dst_reg + off) is writeable */ 6732 err = check_mem_access(env, env->insn_idx, insn->dst_reg, 6733 insn->off, BPF_SIZE(insn->code), 6734 BPF_WRITE, -1, false); 6735 if (err) 6736 return err; 6737 6738 } else if (class == BPF_JMP || class == BPF_JMP32) { 6739 u8 opcode = BPF_OP(insn->code); 6740 6741 if (opcode == BPF_CALL) { 6742 if (BPF_SRC(insn->code) != BPF_K || 6743 insn->off != 0 || 6744 (insn->src_reg != BPF_REG_0 && 6745 insn->src_reg != BPF_PSEUDO_CALL) || 6746 insn->dst_reg != BPF_REG_0 || 6747 class == BPF_JMP32) { 6748 verbose(env, "BPF_CALL uses reserved fields\n"); 6749 return -EINVAL; 6750 } 6751 6752 if (env->cur_state->active_spin_lock && 6753 (insn->src_reg == BPF_PSEUDO_CALL || 6754 insn->imm != BPF_FUNC_spin_unlock)) { 6755 verbose(env, "function calls are not allowed while holding a lock\n"); 6756 return -EINVAL; 6757 } 6758 if (insn->src_reg == BPF_PSEUDO_CALL) 6759 err = check_func_call(env, insn, &env->insn_idx); 6760 else 6761 err = check_helper_call(env, insn->imm, env->insn_idx); 6762 if (err) 6763 return err; 6764 6765 } else if (opcode == BPF_JA) { 6766 if (BPF_SRC(insn->code) != BPF_K || 6767 insn->imm != 0 || 6768 insn->src_reg != BPF_REG_0 || 6769 insn->dst_reg != BPF_REG_0 || 6770 class == BPF_JMP32) { 6771 verbose(env, "BPF_JA uses reserved fields\n"); 6772 return -EINVAL; 6773 } 6774 6775 env->insn_idx += insn->off + 1; 6776 continue; 6777 6778 } else if (opcode == BPF_EXIT) { 6779 if (BPF_SRC(insn->code) != BPF_K || 6780 insn->imm != 0 || 6781 insn->src_reg != BPF_REG_0 || 6782 insn->dst_reg != BPF_REG_0 || 6783 class == BPF_JMP32) { 6784 verbose(env, "BPF_EXIT uses reserved fields\n"); 6785 return -EINVAL; 6786 } 6787 6788 if (env->cur_state->active_spin_lock) { 6789 verbose(env, "bpf_spin_unlock is missing\n"); 6790 return -EINVAL; 6791 } 6792 6793 if (state->curframe) { 6794 /* exit from nested function */ 6795 env->prev_insn_idx = env->insn_idx; 6796 err = prepare_func_exit(env, &env->insn_idx); 6797 if (err) 6798 return err; 6799 do_print_state = true; 6800 continue; 6801 } 6802 6803 err = check_reference_leak(env); 6804 if (err) 6805 return err; 6806 6807 /* eBPF calling convetion is such that R0 is used 6808 * to return the value from eBPF program. 6809 * Make sure that it's readable at this time 6810 * of bpf_exit, which means that program wrote 6811 * something into it earlier 6812 */ 6813 err = check_reg_arg(env, BPF_REG_0, SRC_OP); 6814 if (err) 6815 return err; 6816 6817 if (is_pointer_value(env, BPF_REG_0)) { 6818 verbose(env, "R0 leaks addr as return value\n"); 6819 return -EACCES; 6820 } 6821 6822 err = check_return_code(env); 6823 if (err) 6824 return err; 6825 process_bpf_exit: 6826 err = pop_stack(env, &env->prev_insn_idx, 6827 &env->insn_idx); 6828 if (err < 0) { 6829 if (err != -ENOENT) 6830 return err; 6831 break; 6832 } else { 6833 do_print_state = true; 6834 continue; 6835 } 6836 } else { 6837 err = check_cond_jmp_op(env, insn, &env->insn_idx); 6838 if (err) 6839 return err; 6840 } 6841 } else if (class == BPF_LD) { 6842 u8 mode = BPF_MODE(insn->code); 6843 6844 if (mode == BPF_ABS || mode == BPF_IND) { 6845 err = check_ld_abs(env, insn); 6846 if (err) 6847 return err; 6848 6849 } else if (mode == BPF_IMM) { 6850 err = check_ld_imm(env, insn); 6851 if (err) 6852 return err; 6853 6854 env->insn_idx++; 6855 env->insn_aux_data[env->insn_idx].seen = true; 6856 } else { 6857 verbose(env, "invalid BPF_LD mode\n"); 6858 return -EINVAL; 6859 } 6860 } else { 6861 verbose(env, "unknown insn class %d\n", class); 6862 return -EINVAL; 6863 } 6864 6865 env->insn_idx++; 6866 } 6867 6868 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth; 6869 return 0; 6870 } 6871 6872 static int check_map_prealloc(struct bpf_map *map) 6873 { 6874 return (map->map_type != BPF_MAP_TYPE_HASH && 6875 map->map_type != BPF_MAP_TYPE_PERCPU_HASH && 6876 map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) || 6877 !(map->map_flags & BPF_F_NO_PREALLOC); 6878 } 6879 6880 static bool is_tracing_prog_type(enum bpf_prog_type type) 6881 { 6882 switch (type) { 6883 case BPF_PROG_TYPE_KPROBE: 6884 case BPF_PROG_TYPE_TRACEPOINT: 6885 case BPF_PROG_TYPE_PERF_EVENT: 6886 case BPF_PROG_TYPE_RAW_TRACEPOINT: 6887 return true; 6888 default: 6889 return false; 6890 } 6891 } 6892 6893 static int check_map_prog_compatibility(struct bpf_verifier_env *env, 6894 struct bpf_map *map, 6895 struct bpf_prog *prog) 6896 6897 { 6898 /* Make sure that BPF_PROG_TYPE_PERF_EVENT programs only use 6899 * preallocated hash maps, since doing memory allocation 6900 * in overflow_handler can crash depending on where nmi got 6901 * triggered. 6902 */ 6903 if (prog->type == BPF_PROG_TYPE_PERF_EVENT) { 6904 if (!check_map_prealloc(map)) { 6905 verbose(env, "perf_event programs can only use preallocated hash map\n"); 6906 return -EINVAL; 6907 } 6908 if (map->inner_map_meta && 6909 !check_map_prealloc(map->inner_map_meta)) { 6910 verbose(env, "perf_event programs can only use preallocated inner hash map\n"); 6911 return -EINVAL; 6912 } 6913 } 6914 6915 if ((is_tracing_prog_type(prog->type) || 6916 prog->type == BPF_PROG_TYPE_SOCKET_FILTER) && 6917 map_value_has_spin_lock(map)) { 6918 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n"); 6919 return -EINVAL; 6920 } 6921 6922 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && 6923 !bpf_offload_prog_map_match(prog, map)) { 6924 verbose(env, "offload device mismatch between prog and map\n"); 6925 return -EINVAL; 6926 } 6927 6928 return 0; 6929 } 6930 6931 static bool bpf_map_is_cgroup_storage(struct bpf_map *map) 6932 { 6933 return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE || 6934 map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE); 6935 } 6936 6937 /* look for pseudo eBPF instructions that access map FDs and 6938 * replace them with actual map pointers 6939 */ 6940 static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env) 6941 { 6942 struct bpf_insn *insn = env->prog->insnsi; 6943 int insn_cnt = env->prog->len; 6944 int i, j, err; 6945 6946 err = bpf_prog_calc_tag(env->prog); 6947 if (err) 6948 return err; 6949 6950 for (i = 0; i < insn_cnt; i++, insn++) { 6951 if (BPF_CLASS(insn->code) == BPF_LDX && 6952 (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) { 6953 verbose(env, "BPF_LDX uses reserved fields\n"); 6954 return -EINVAL; 6955 } 6956 6957 if (BPF_CLASS(insn->code) == BPF_STX && 6958 ((BPF_MODE(insn->code) != BPF_MEM && 6959 BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) { 6960 verbose(env, "BPF_STX uses reserved fields\n"); 6961 return -EINVAL; 6962 } 6963 6964 if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) { 6965 struct bpf_insn_aux_data *aux; 6966 struct bpf_map *map; 6967 struct fd f; 6968 u64 addr; 6969 6970 if (i == insn_cnt - 1 || insn[1].code != 0 || 6971 insn[1].dst_reg != 0 || insn[1].src_reg != 0 || 6972 insn[1].off != 0) { 6973 verbose(env, "invalid bpf_ld_imm64 insn\n"); 6974 return -EINVAL; 6975 } 6976 6977 if (insn[0].src_reg == 0) 6978 /* valid generic load 64-bit imm */ 6979 goto next_insn; 6980 6981 /* In final convert_pseudo_ld_imm64() step, this is 6982 * converted into regular 64-bit imm load insn. 6983 */ 6984 if ((insn[0].src_reg != BPF_PSEUDO_MAP_FD && 6985 insn[0].src_reg != BPF_PSEUDO_MAP_VALUE) || 6986 (insn[0].src_reg == BPF_PSEUDO_MAP_FD && 6987 insn[1].imm != 0)) { 6988 verbose(env, 6989 "unrecognized bpf_ld_imm64 insn\n"); 6990 return -EINVAL; 6991 } 6992 6993 f = fdget(insn[0].imm); 6994 map = __bpf_map_get(f); 6995 if (IS_ERR(map)) { 6996 verbose(env, "fd %d is not pointing to valid bpf_map\n", 6997 insn[0].imm); 6998 return PTR_ERR(map); 6999 } 7000 7001 err = check_map_prog_compatibility(env, map, env->prog); 7002 if (err) { 7003 fdput(f); 7004 return err; 7005 } 7006 7007 aux = &env->insn_aux_data[i]; 7008 if (insn->src_reg == BPF_PSEUDO_MAP_FD) { 7009 addr = (unsigned long)map; 7010 } else { 7011 u32 off = insn[1].imm; 7012 7013 if (off >= BPF_MAX_VAR_OFF) { 7014 verbose(env, "direct value offset of %u is not allowed\n", off); 7015 fdput(f); 7016 return -EINVAL; 7017 } 7018 7019 if (!map->ops->map_direct_value_addr) { 7020 verbose(env, "no direct value access support for this map type\n"); 7021 fdput(f); 7022 return -EINVAL; 7023 } 7024 7025 err = map->ops->map_direct_value_addr(map, &addr, off); 7026 if (err) { 7027 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", 7028 map->value_size, off); 7029 fdput(f); 7030 return err; 7031 } 7032 7033 aux->map_off = off; 7034 addr += off; 7035 } 7036 7037 insn[0].imm = (u32)addr; 7038 insn[1].imm = addr >> 32; 7039 7040 /* check whether we recorded this map already */ 7041 for (j = 0; j < env->used_map_cnt; j++) { 7042 if (env->used_maps[j] == map) { 7043 aux->map_index = j; 7044 fdput(f); 7045 goto next_insn; 7046 } 7047 } 7048 7049 if (env->used_map_cnt >= MAX_USED_MAPS) { 7050 fdput(f); 7051 return -E2BIG; 7052 } 7053 7054 /* hold the map. If the program is rejected by verifier, 7055 * the map will be released by release_maps() or it 7056 * will be used by the valid program until it's unloaded 7057 * and all maps are released in free_used_maps() 7058 */ 7059 map = bpf_map_inc(map, false); 7060 if (IS_ERR(map)) { 7061 fdput(f); 7062 return PTR_ERR(map); 7063 } 7064 7065 aux->map_index = env->used_map_cnt; 7066 env->used_maps[env->used_map_cnt++] = map; 7067 7068 if (bpf_map_is_cgroup_storage(map) && 7069 bpf_cgroup_storage_assign(env->prog, map)) { 7070 verbose(env, "only one cgroup storage of each type is allowed\n"); 7071 fdput(f); 7072 return -EBUSY; 7073 } 7074 7075 fdput(f); 7076 next_insn: 7077 insn++; 7078 i++; 7079 continue; 7080 } 7081 7082 /* Basic sanity check before we invest more work here. */ 7083 if (!bpf_opcode_in_insntable(insn->code)) { 7084 verbose(env, "unknown opcode %02x\n", insn->code); 7085 return -EINVAL; 7086 } 7087 } 7088 7089 /* now all pseudo BPF_LD_IMM64 instructions load valid 7090 * 'struct bpf_map *' into a register instead of user map_fd. 7091 * These pointers will be used later by verifier to validate map access. 7092 */ 7093 return 0; 7094 } 7095 7096 /* drop refcnt of maps used by the rejected program */ 7097 static void release_maps(struct bpf_verifier_env *env) 7098 { 7099 enum bpf_cgroup_storage_type stype; 7100 int i; 7101 7102 for_each_cgroup_storage_type(stype) { 7103 if (!env->prog->aux->cgroup_storage[stype]) 7104 continue; 7105 bpf_cgroup_storage_release(env->prog, 7106 env->prog->aux->cgroup_storage[stype]); 7107 } 7108 7109 for (i = 0; i < env->used_map_cnt; i++) 7110 bpf_map_put(env->used_maps[i]); 7111 } 7112 7113 /* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */ 7114 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env) 7115 { 7116 struct bpf_insn *insn = env->prog->insnsi; 7117 int insn_cnt = env->prog->len; 7118 int i; 7119 7120 for (i = 0; i < insn_cnt; i++, insn++) 7121 if (insn->code == (BPF_LD | BPF_IMM | BPF_DW)) 7122 insn->src_reg = 0; 7123 } 7124 7125 /* single env->prog->insni[off] instruction was replaced with the range 7126 * insni[off, off + cnt). Adjust corresponding insn_aux_data by copying 7127 * [0, off) and [off, end) to new locations, so the patched range stays zero 7128 */ 7129 static int adjust_insn_aux_data(struct bpf_verifier_env *env, u32 prog_len, 7130 u32 off, u32 cnt) 7131 { 7132 struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data; 7133 int i; 7134 7135 if (cnt == 1) 7136 return 0; 7137 new_data = vzalloc(array_size(prog_len, 7138 sizeof(struct bpf_insn_aux_data))); 7139 if (!new_data) 7140 return -ENOMEM; 7141 memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off); 7142 memcpy(new_data + off + cnt - 1, old_data + off, 7143 sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1)); 7144 for (i = off; i < off + cnt - 1; i++) 7145 new_data[i].seen = true; 7146 env->insn_aux_data = new_data; 7147 vfree(old_data); 7148 return 0; 7149 } 7150 7151 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len) 7152 { 7153 int i; 7154 7155 if (len == 1) 7156 return; 7157 /* NOTE: fake 'exit' subprog should be updated as well. */ 7158 for (i = 0; i <= env->subprog_cnt; i++) { 7159 if (env->subprog_info[i].start <= off) 7160 continue; 7161 env->subprog_info[i].start += len - 1; 7162 } 7163 } 7164 7165 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, 7166 const struct bpf_insn *patch, u32 len) 7167 { 7168 struct bpf_prog *new_prog; 7169 7170 new_prog = bpf_patch_insn_single(env->prog, off, patch, len); 7171 if (IS_ERR(new_prog)) { 7172 if (PTR_ERR(new_prog) == -ERANGE) 7173 verbose(env, 7174 "insn %d cannot be patched due to 16-bit range\n", 7175 env->insn_aux_data[off].orig_idx); 7176 return NULL; 7177 } 7178 if (adjust_insn_aux_data(env, new_prog->len, off, len)) 7179 return NULL; 7180 adjust_subprog_starts(env, off, len); 7181 return new_prog; 7182 } 7183 7184 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, 7185 u32 off, u32 cnt) 7186 { 7187 int i, j; 7188 7189 /* find first prog starting at or after off (first to remove) */ 7190 for (i = 0; i < env->subprog_cnt; i++) 7191 if (env->subprog_info[i].start >= off) 7192 break; 7193 /* find first prog starting at or after off + cnt (first to stay) */ 7194 for (j = i; j < env->subprog_cnt; j++) 7195 if (env->subprog_info[j].start >= off + cnt) 7196 break; 7197 /* if j doesn't start exactly at off + cnt, we are just removing 7198 * the front of previous prog 7199 */ 7200 if (env->subprog_info[j].start != off + cnt) 7201 j--; 7202 7203 if (j > i) { 7204 struct bpf_prog_aux *aux = env->prog->aux; 7205 int move; 7206 7207 /* move fake 'exit' subprog as well */ 7208 move = env->subprog_cnt + 1 - j; 7209 7210 memmove(env->subprog_info + i, 7211 env->subprog_info + j, 7212 sizeof(*env->subprog_info) * move); 7213 env->subprog_cnt -= j - i; 7214 7215 /* remove func_info */ 7216 if (aux->func_info) { 7217 move = aux->func_info_cnt - j; 7218 7219 memmove(aux->func_info + i, 7220 aux->func_info + j, 7221 sizeof(*aux->func_info) * move); 7222 aux->func_info_cnt -= j - i; 7223 /* func_info->insn_off is set after all code rewrites, 7224 * in adjust_btf_func() - no need to adjust 7225 */ 7226 } 7227 } else { 7228 /* convert i from "first prog to remove" to "first to adjust" */ 7229 if (env->subprog_info[i].start == off) 7230 i++; 7231 } 7232 7233 /* update fake 'exit' subprog as well */ 7234 for (; i <= env->subprog_cnt; i++) 7235 env->subprog_info[i].start -= cnt; 7236 7237 return 0; 7238 } 7239 7240 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, 7241 u32 cnt) 7242 { 7243 struct bpf_prog *prog = env->prog; 7244 u32 i, l_off, l_cnt, nr_linfo; 7245 struct bpf_line_info *linfo; 7246 7247 nr_linfo = prog->aux->nr_linfo; 7248 if (!nr_linfo) 7249 return 0; 7250 7251 linfo = prog->aux->linfo; 7252 7253 /* find first line info to remove, count lines to be removed */ 7254 for (i = 0; i < nr_linfo; i++) 7255 if (linfo[i].insn_off >= off) 7256 break; 7257 7258 l_off = i; 7259 l_cnt = 0; 7260 for (; i < nr_linfo; i++) 7261 if (linfo[i].insn_off < off + cnt) 7262 l_cnt++; 7263 else 7264 break; 7265 7266 /* First live insn doesn't match first live linfo, it needs to "inherit" 7267 * last removed linfo. prog is already modified, so prog->len == off 7268 * means no live instructions after (tail of the program was removed). 7269 */ 7270 if (prog->len != off && l_cnt && 7271 (i == nr_linfo || linfo[i].insn_off != off + cnt)) { 7272 l_cnt--; 7273 linfo[--i].insn_off = off + cnt; 7274 } 7275 7276 /* remove the line info which refer to the removed instructions */ 7277 if (l_cnt) { 7278 memmove(linfo + l_off, linfo + i, 7279 sizeof(*linfo) * (nr_linfo - i)); 7280 7281 prog->aux->nr_linfo -= l_cnt; 7282 nr_linfo = prog->aux->nr_linfo; 7283 } 7284 7285 /* pull all linfo[i].insn_off >= off + cnt in by cnt */ 7286 for (i = l_off; i < nr_linfo; i++) 7287 linfo[i].insn_off -= cnt; 7288 7289 /* fix up all subprogs (incl. 'exit') which start >= off */ 7290 for (i = 0; i <= env->subprog_cnt; i++) 7291 if (env->subprog_info[i].linfo_idx > l_off) { 7292 /* program may have started in the removed region but 7293 * may not be fully removed 7294 */ 7295 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) 7296 env->subprog_info[i].linfo_idx -= l_cnt; 7297 else 7298 env->subprog_info[i].linfo_idx = l_off; 7299 } 7300 7301 return 0; 7302 } 7303 7304 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt) 7305 { 7306 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 7307 unsigned int orig_prog_len = env->prog->len; 7308 int err; 7309 7310 if (bpf_prog_is_dev_bound(env->prog->aux)) 7311 bpf_prog_offload_remove_insns(env, off, cnt); 7312 7313 err = bpf_remove_insns(env->prog, off, cnt); 7314 if (err) 7315 return err; 7316 7317 err = adjust_subprog_starts_after_remove(env, off, cnt); 7318 if (err) 7319 return err; 7320 7321 err = bpf_adj_linfo_after_remove(env, off, cnt); 7322 if (err) 7323 return err; 7324 7325 memmove(aux_data + off, aux_data + off + cnt, 7326 sizeof(*aux_data) * (orig_prog_len - off - cnt)); 7327 7328 return 0; 7329 } 7330 7331 /* The verifier does more data flow analysis than llvm and will not 7332 * explore branches that are dead at run time. Malicious programs can 7333 * have dead code too. Therefore replace all dead at-run-time code 7334 * with 'ja -1'. 7335 * 7336 * Just nops are not optimal, e.g. if they would sit at the end of the 7337 * program and through another bug we would manage to jump there, then 7338 * we'd execute beyond program memory otherwise. Returning exception 7339 * code also wouldn't work since we can have subprogs where the dead 7340 * code could be located. 7341 */ 7342 static void sanitize_dead_code(struct bpf_verifier_env *env) 7343 { 7344 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 7345 struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1); 7346 struct bpf_insn *insn = env->prog->insnsi; 7347 const int insn_cnt = env->prog->len; 7348 int i; 7349 7350 for (i = 0; i < insn_cnt; i++) { 7351 if (aux_data[i].seen) 7352 continue; 7353 memcpy(insn + i, &trap, sizeof(trap)); 7354 } 7355 } 7356 7357 static bool insn_is_cond_jump(u8 code) 7358 { 7359 u8 op; 7360 7361 if (BPF_CLASS(code) == BPF_JMP32) 7362 return true; 7363 7364 if (BPF_CLASS(code) != BPF_JMP) 7365 return false; 7366 7367 op = BPF_OP(code); 7368 return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL; 7369 } 7370 7371 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env) 7372 { 7373 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 7374 struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 7375 struct bpf_insn *insn = env->prog->insnsi; 7376 const int insn_cnt = env->prog->len; 7377 int i; 7378 7379 for (i = 0; i < insn_cnt; i++, insn++) { 7380 if (!insn_is_cond_jump(insn->code)) 7381 continue; 7382 7383 if (!aux_data[i + 1].seen) 7384 ja.off = insn->off; 7385 else if (!aux_data[i + 1 + insn->off].seen) 7386 ja.off = 0; 7387 else 7388 continue; 7389 7390 if (bpf_prog_is_dev_bound(env->prog->aux)) 7391 bpf_prog_offload_replace_insn(env, i, &ja); 7392 7393 memcpy(insn, &ja, sizeof(ja)); 7394 } 7395 } 7396 7397 static int opt_remove_dead_code(struct bpf_verifier_env *env) 7398 { 7399 struct bpf_insn_aux_data *aux_data = env->insn_aux_data; 7400 int insn_cnt = env->prog->len; 7401 int i, err; 7402 7403 for (i = 0; i < insn_cnt; i++) { 7404 int j; 7405 7406 j = 0; 7407 while (i + j < insn_cnt && !aux_data[i + j].seen) 7408 j++; 7409 if (!j) 7410 continue; 7411 7412 err = verifier_remove_insns(env, i, j); 7413 if (err) 7414 return err; 7415 insn_cnt = env->prog->len; 7416 } 7417 7418 return 0; 7419 } 7420 7421 static int opt_remove_nops(struct bpf_verifier_env *env) 7422 { 7423 const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0); 7424 struct bpf_insn *insn = env->prog->insnsi; 7425 int insn_cnt = env->prog->len; 7426 int i, err; 7427 7428 for (i = 0; i < insn_cnt; i++) { 7429 if (memcmp(&insn[i], &ja, sizeof(ja))) 7430 continue; 7431 7432 err = verifier_remove_insns(env, i, 1); 7433 if (err) 7434 return err; 7435 insn_cnt--; 7436 i--; 7437 } 7438 7439 return 0; 7440 } 7441 7442 /* convert load instructions that access fields of a context type into a 7443 * sequence of instructions that access fields of the underlying structure: 7444 * struct __sk_buff -> struct sk_buff 7445 * struct bpf_sock_ops -> struct sock 7446 */ 7447 static int convert_ctx_accesses(struct bpf_verifier_env *env) 7448 { 7449 const struct bpf_verifier_ops *ops = env->ops; 7450 int i, cnt, size, ctx_field_size, delta = 0; 7451 const int insn_cnt = env->prog->len; 7452 struct bpf_insn insn_buf[16], *insn; 7453 u32 target_size, size_default, off; 7454 struct bpf_prog *new_prog; 7455 enum bpf_access_type type; 7456 bool is_narrower_load; 7457 7458 if (ops->gen_prologue || env->seen_direct_write) { 7459 if (!ops->gen_prologue) { 7460 verbose(env, "bpf verifier is misconfigured\n"); 7461 return -EINVAL; 7462 } 7463 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, 7464 env->prog); 7465 if (cnt >= ARRAY_SIZE(insn_buf)) { 7466 verbose(env, "bpf verifier is misconfigured\n"); 7467 return -EINVAL; 7468 } else if (cnt) { 7469 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt); 7470 if (!new_prog) 7471 return -ENOMEM; 7472 7473 env->prog = new_prog; 7474 delta += cnt - 1; 7475 } 7476 } 7477 7478 if (bpf_prog_is_dev_bound(env->prog->aux)) 7479 return 0; 7480 7481 insn = env->prog->insnsi + delta; 7482 7483 for (i = 0; i < insn_cnt; i++, insn++) { 7484 bpf_convert_ctx_access_t convert_ctx_access; 7485 7486 if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) || 7487 insn->code == (BPF_LDX | BPF_MEM | BPF_H) || 7488 insn->code == (BPF_LDX | BPF_MEM | BPF_W) || 7489 insn->code == (BPF_LDX | BPF_MEM | BPF_DW)) 7490 type = BPF_READ; 7491 else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) || 7492 insn->code == (BPF_STX | BPF_MEM | BPF_H) || 7493 insn->code == (BPF_STX | BPF_MEM | BPF_W) || 7494 insn->code == (BPF_STX | BPF_MEM | BPF_DW)) 7495 type = BPF_WRITE; 7496 else 7497 continue; 7498 7499 if (type == BPF_WRITE && 7500 env->insn_aux_data[i + delta].sanitize_stack_off) { 7501 struct bpf_insn patch[] = { 7502 /* Sanitize suspicious stack slot with zero. 7503 * There are no memory dependencies for this store, 7504 * since it's only using frame pointer and immediate 7505 * constant of zero 7506 */ 7507 BPF_ST_MEM(BPF_DW, BPF_REG_FP, 7508 env->insn_aux_data[i + delta].sanitize_stack_off, 7509 0), 7510 /* the original STX instruction will immediately 7511 * overwrite the same stack slot with appropriate value 7512 */ 7513 *insn, 7514 }; 7515 7516 cnt = ARRAY_SIZE(patch); 7517 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt); 7518 if (!new_prog) 7519 return -ENOMEM; 7520 7521 delta += cnt - 1; 7522 env->prog = new_prog; 7523 insn = new_prog->insnsi + i + delta; 7524 continue; 7525 } 7526 7527 switch (env->insn_aux_data[i + delta].ptr_type) { 7528 case PTR_TO_CTX: 7529 if (!ops->convert_ctx_access) 7530 continue; 7531 convert_ctx_access = ops->convert_ctx_access; 7532 break; 7533 case PTR_TO_SOCKET: 7534 case PTR_TO_SOCK_COMMON: 7535 convert_ctx_access = bpf_sock_convert_ctx_access; 7536 break; 7537 case PTR_TO_TCP_SOCK: 7538 convert_ctx_access = bpf_tcp_sock_convert_ctx_access; 7539 break; 7540 default: 7541 continue; 7542 } 7543 7544 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size; 7545 size = BPF_LDST_BYTES(insn); 7546 7547 /* If the read access is a narrower load of the field, 7548 * convert to a 4/8-byte load, to minimum program type specific 7549 * convert_ctx_access changes. If conversion is successful, 7550 * we will apply proper mask to the result. 7551 */ 7552 is_narrower_load = size < ctx_field_size; 7553 size_default = bpf_ctx_off_adjust_machine(ctx_field_size); 7554 off = insn->off; 7555 if (is_narrower_load) { 7556 u8 size_code; 7557 7558 if (type == BPF_WRITE) { 7559 verbose(env, "bpf verifier narrow ctx access misconfigured\n"); 7560 return -EINVAL; 7561 } 7562 7563 size_code = BPF_H; 7564 if (ctx_field_size == 4) 7565 size_code = BPF_W; 7566 else if (ctx_field_size == 8) 7567 size_code = BPF_DW; 7568 7569 insn->off = off & ~(size_default - 1); 7570 insn->code = BPF_LDX | BPF_MEM | size_code; 7571 } 7572 7573 target_size = 0; 7574 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, 7575 &target_size); 7576 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) || 7577 (ctx_field_size && !target_size)) { 7578 verbose(env, "bpf verifier is misconfigured\n"); 7579 return -EINVAL; 7580 } 7581 7582 if (is_narrower_load && size < target_size) { 7583 u8 shift = (off & (size_default - 1)) * 8; 7584 7585 if (ctx_field_size <= 4) { 7586 if (shift) 7587 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH, 7588 insn->dst_reg, 7589 shift); 7590 insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg, 7591 (1 << size * 8) - 1); 7592 } else { 7593 if (shift) 7594 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH, 7595 insn->dst_reg, 7596 shift); 7597 insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg, 7598 (1ULL << size * 8) - 1); 7599 } 7600 } 7601 7602 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 7603 if (!new_prog) 7604 return -ENOMEM; 7605 7606 delta += cnt - 1; 7607 7608 /* keep walking new program and skip insns we just inserted */ 7609 env->prog = new_prog; 7610 insn = new_prog->insnsi + i + delta; 7611 } 7612 7613 return 0; 7614 } 7615 7616 static int jit_subprogs(struct bpf_verifier_env *env) 7617 { 7618 struct bpf_prog *prog = env->prog, **func, *tmp; 7619 int i, j, subprog_start, subprog_end = 0, len, subprog; 7620 struct bpf_insn *insn; 7621 void *old_bpf_func; 7622 int err; 7623 7624 if (env->subprog_cnt <= 1) 7625 return 0; 7626 7627 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 7628 if (insn->code != (BPF_JMP | BPF_CALL) || 7629 insn->src_reg != BPF_PSEUDO_CALL) 7630 continue; 7631 /* Upon error here we cannot fall back to interpreter but 7632 * need a hard reject of the program. Thus -EFAULT is 7633 * propagated in any case. 7634 */ 7635 subprog = find_subprog(env, i + insn->imm + 1); 7636 if (subprog < 0) { 7637 WARN_ONCE(1, "verifier bug. No program starts at insn %d\n", 7638 i + insn->imm + 1); 7639 return -EFAULT; 7640 } 7641 /* temporarily remember subprog id inside insn instead of 7642 * aux_data, since next loop will split up all insns into funcs 7643 */ 7644 insn->off = subprog; 7645 /* remember original imm in case JIT fails and fallback 7646 * to interpreter will be needed 7647 */ 7648 env->insn_aux_data[i].call_imm = insn->imm; 7649 /* point imm to __bpf_call_base+1 from JITs point of view */ 7650 insn->imm = 1; 7651 } 7652 7653 err = bpf_prog_alloc_jited_linfo(prog); 7654 if (err) 7655 goto out_undo_insn; 7656 7657 err = -ENOMEM; 7658 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL); 7659 if (!func) 7660 goto out_undo_insn; 7661 7662 for (i = 0; i < env->subprog_cnt; i++) { 7663 subprog_start = subprog_end; 7664 subprog_end = env->subprog_info[i + 1].start; 7665 7666 len = subprog_end - subprog_start; 7667 /* BPF_PROG_RUN doesn't call subprogs directly, 7668 * hence main prog stats include the runtime of subprogs. 7669 * subprogs don't have IDs and not reachable via prog_get_next_id 7670 * func[i]->aux->stats will never be accessed and stays NULL 7671 */ 7672 func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER); 7673 if (!func[i]) 7674 goto out_free; 7675 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], 7676 len * sizeof(struct bpf_insn)); 7677 func[i]->type = prog->type; 7678 func[i]->len = len; 7679 if (bpf_prog_calc_tag(func[i])) 7680 goto out_free; 7681 func[i]->is_func = 1; 7682 func[i]->aux->func_idx = i; 7683 /* the btf and func_info will be freed only at prog->aux */ 7684 func[i]->aux->btf = prog->aux->btf; 7685 func[i]->aux->func_info = prog->aux->func_info; 7686 7687 /* Use bpf_prog_F_tag to indicate functions in stack traces. 7688 * Long term would need debug info to populate names 7689 */ 7690 func[i]->aux->name[0] = 'F'; 7691 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth; 7692 func[i]->jit_requested = 1; 7693 func[i]->aux->linfo = prog->aux->linfo; 7694 func[i]->aux->nr_linfo = prog->aux->nr_linfo; 7695 func[i]->aux->jited_linfo = prog->aux->jited_linfo; 7696 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx; 7697 func[i] = bpf_int_jit_compile(func[i]); 7698 if (!func[i]->jited) { 7699 err = -ENOTSUPP; 7700 goto out_free; 7701 } 7702 cond_resched(); 7703 } 7704 /* at this point all bpf functions were successfully JITed 7705 * now populate all bpf_calls with correct addresses and 7706 * run last pass of JIT 7707 */ 7708 for (i = 0; i < env->subprog_cnt; i++) { 7709 insn = func[i]->insnsi; 7710 for (j = 0; j < func[i]->len; j++, insn++) { 7711 if (insn->code != (BPF_JMP | BPF_CALL) || 7712 insn->src_reg != BPF_PSEUDO_CALL) 7713 continue; 7714 subprog = insn->off; 7715 insn->imm = BPF_CAST_CALL(func[subprog]->bpf_func) - 7716 __bpf_call_base; 7717 } 7718 7719 /* we use the aux data to keep a list of the start addresses 7720 * of the JITed images for each function in the program 7721 * 7722 * for some architectures, such as powerpc64, the imm field 7723 * might not be large enough to hold the offset of the start 7724 * address of the callee's JITed image from __bpf_call_base 7725 * 7726 * in such cases, we can lookup the start address of a callee 7727 * by using its subprog id, available from the off field of 7728 * the call instruction, as an index for this list 7729 */ 7730 func[i]->aux->func = func; 7731 func[i]->aux->func_cnt = env->subprog_cnt; 7732 } 7733 for (i = 0; i < env->subprog_cnt; i++) { 7734 old_bpf_func = func[i]->bpf_func; 7735 tmp = bpf_int_jit_compile(func[i]); 7736 if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) { 7737 verbose(env, "JIT doesn't support bpf-to-bpf calls\n"); 7738 err = -ENOTSUPP; 7739 goto out_free; 7740 } 7741 cond_resched(); 7742 } 7743 7744 /* finally lock prog and jit images for all functions and 7745 * populate kallsysm 7746 */ 7747 for (i = 0; i < env->subprog_cnt; i++) { 7748 bpf_prog_lock_ro(func[i]); 7749 bpf_prog_kallsyms_add(func[i]); 7750 } 7751 7752 /* Last step: make now unused interpreter insns from main 7753 * prog consistent for later dump requests, so they can 7754 * later look the same as if they were interpreted only. 7755 */ 7756 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 7757 if (insn->code != (BPF_JMP | BPF_CALL) || 7758 insn->src_reg != BPF_PSEUDO_CALL) 7759 continue; 7760 insn->off = env->insn_aux_data[i].call_imm; 7761 subprog = find_subprog(env, i + insn->off + 1); 7762 insn->imm = subprog; 7763 } 7764 7765 prog->jited = 1; 7766 prog->bpf_func = func[0]->bpf_func; 7767 prog->aux->func = func; 7768 prog->aux->func_cnt = env->subprog_cnt; 7769 bpf_prog_free_unused_jited_linfo(prog); 7770 return 0; 7771 out_free: 7772 for (i = 0; i < env->subprog_cnt; i++) 7773 if (func[i]) 7774 bpf_jit_free(func[i]); 7775 kfree(func); 7776 out_undo_insn: 7777 /* cleanup main prog to be interpreted */ 7778 prog->jit_requested = 0; 7779 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) { 7780 if (insn->code != (BPF_JMP | BPF_CALL) || 7781 insn->src_reg != BPF_PSEUDO_CALL) 7782 continue; 7783 insn->off = 0; 7784 insn->imm = env->insn_aux_data[i].call_imm; 7785 } 7786 bpf_prog_free_jited_linfo(prog); 7787 return err; 7788 } 7789 7790 static int fixup_call_args(struct bpf_verifier_env *env) 7791 { 7792 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 7793 struct bpf_prog *prog = env->prog; 7794 struct bpf_insn *insn = prog->insnsi; 7795 int i, depth; 7796 #endif 7797 int err = 0; 7798 7799 if (env->prog->jit_requested && 7800 !bpf_prog_is_dev_bound(env->prog->aux)) { 7801 err = jit_subprogs(env); 7802 if (err == 0) 7803 return 0; 7804 if (err == -EFAULT) 7805 return err; 7806 } 7807 #ifndef CONFIG_BPF_JIT_ALWAYS_ON 7808 for (i = 0; i < prog->len; i++, insn++) { 7809 if (insn->code != (BPF_JMP | BPF_CALL) || 7810 insn->src_reg != BPF_PSEUDO_CALL) 7811 continue; 7812 depth = get_callee_stack_depth(env, insn, i); 7813 if (depth < 0) 7814 return depth; 7815 bpf_patch_call_args(insn, depth); 7816 } 7817 err = 0; 7818 #endif 7819 return err; 7820 } 7821 7822 /* fixup insn->imm field of bpf_call instructions 7823 * and inline eligible helpers as explicit sequence of BPF instructions 7824 * 7825 * this function is called after eBPF program passed verification 7826 */ 7827 static int fixup_bpf_calls(struct bpf_verifier_env *env) 7828 { 7829 struct bpf_prog *prog = env->prog; 7830 struct bpf_insn *insn = prog->insnsi; 7831 const struct bpf_func_proto *fn; 7832 const int insn_cnt = prog->len; 7833 const struct bpf_map_ops *ops; 7834 struct bpf_insn_aux_data *aux; 7835 struct bpf_insn insn_buf[16]; 7836 struct bpf_prog *new_prog; 7837 struct bpf_map *map_ptr; 7838 int i, cnt, delta = 0; 7839 7840 for (i = 0; i < insn_cnt; i++, insn++) { 7841 if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) || 7842 insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 7843 insn->code == (BPF_ALU | BPF_MOD | BPF_X) || 7844 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 7845 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; 7846 struct bpf_insn mask_and_div[] = { 7847 BPF_MOV32_REG(insn->src_reg, insn->src_reg), 7848 /* Rx div 0 -> 0 */ 7849 BPF_JMP_IMM(BPF_JNE, insn->src_reg, 0, 2), 7850 BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg), 7851 BPF_JMP_IMM(BPF_JA, 0, 0, 1), 7852 *insn, 7853 }; 7854 struct bpf_insn mask_and_mod[] = { 7855 BPF_MOV32_REG(insn->src_reg, insn->src_reg), 7856 /* Rx mod 0 -> Rx */ 7857 BPF_JMP_IMM(BPF_JEQ, insn->src_reg, 0, 1), 7858 *insn, 7859 }; 7860 struct bpf_insn *patchlet; 7861 7862 if (insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) || 7863 insn->code == (BPF_ALU | BPF_DIV | BPF_X)) { 7864 patchlet = mask_and_div + (is64 ? 1 : 0); 7865 cnt = ARRAY_SIZE(mask_and_div) - (is64 ? 1 : 0); 7866 } else { 7867 patchlet = mask_and_mod + (is64 ? 1 : 0); 7868 cnt = ARRAY_SIZE(mask_and_mod) - (is64 ? 1 : 0); 7869 } 7870 7871 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt); 7872 if (!new_prog) 7873 return -ENOMEM; 7874 7875 delta += cnt - 1; 7876 env->prog = prog = new_prog; 7877 insn = new_prog->insnsi + i + delta; 7878 continue; 7879 } 7880 7881 if (BPF_CLASS(insn->code) == BPF_LD && 7882 (BPF_MODE(insn->code) == BPF_ABS || 7883 BPF_MODE(insn->code) == BPF_IND)) { 7884 cnt = env->ops->gen_ld_abs(insn, insn_buf); 7885 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 7886 verbose(env, "bpf verifier is misconfigured\n"); 7887 return -EINVAL; 7888 } 7889 7890 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 7891 if (!new_prog) 7892 return -ENOMEM; 7893 7894 delta += cnt - 1; 7895 env->prog = prog = new_prog; 7896 insn = new_prog->insnsi + i + delta; 7897 continue; 7898 } 7899 7900 if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) || 7901 insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) { 7902 const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X; 7903 const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X; 7904 struct bpf_insn insn_buf[16]; 7905 struct bpf_insn *patch = &insn_buf[0]; 7906 bool issrc, isneg; 7907 u32 off_reg; 7908 7909 aux = &env->insn_aux_data[i + delta]; 7910 if (!aux->alu_state || 7911 aux->alu_state == BPF_ALU_NON_POINTER) 7912 continue; 7913 7914 isneg = aux->alu_state & BPF_ALU_NEG_VALUE; 7915 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == 7916 BPF_ALU_SANITIZE_SRC; 7917 7918 off_reg = issrc ? insn->src_reg : insn->dst_reg; 7919 if (isneg) 7920 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 7921 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit - 1); 7922 *patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg); 7923 *patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg); 7924 *patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0); 7925 *patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63); 7926 if (issrc) { 7927 *patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, 7928 off_reg); 7929 insn->src_reg = BPF_REG_AX; 7930 } else { 7931 *patch++ = BPF_ALU64_REG(BPF_AND, off_reg, 7932 BPF_REG_AX); 7933 } 7934 if (isneg) 7935 insn->code = insn->code == code_add ? 7936 code_sub : code_add; 7937 *patch++ = *insn; 7938 if (issrc && isneg) 7939 *patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1); 7940 cnt = patch - insn_buf; 7941 7942 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 7943 if (!new_prog) 7944 return -ENOMEM; 7945 7946 delta += cnt - 1; 7947 env->prog = prog = new_prog; 7948 insn = new_prog->insnsi + i + delta; 7949 continue; 7950 } 7951 7952 if (insn->code != (BPF_JMP | BPF_CALL)) 7953 continue; 7954 if (insn->src_reg == BPF_PSEUDO_CALL) 7955 continue; 7956 7957 if (insn->imm == BPF_FUNC_get_route_realm) 7958 prog->dst_needed = 1; 7959 if (insn->imm == BPF_FUNC_get_prandom_u32) 7960 bpf_user_rnd_init_once(); 7961 if (insn->imm == BPF_FUNC_override_return) 7962 prog->kprobe_override = 1; 7963 if (insn->imm == BPF_FUNC_tail_call) { 7964 /* If we tail call into other programs, we 7965 * cannot make any assumptions since they can 7966 * be replaced dynamically during runtime in 7967 * the program array. 7968 */ 7969 prog->cb_access = 1; 7970 env->prog->aux->stack_depth = MAX_BPF_STACK; 7971 env->prog->aux->max_pkt_offset = MAX_PACKET_OFF; 7972 7973 /* mark bpf_tail_call as different opcode to avoid 7974 * conditional branch in the interpeter for every normal 7975 * call and to prevent accidental JITing by JIT compiler 7976 * that doesn't support bpf_tail_call yet 7977 */ 7978 insn->imm = 0; 7979 insn->code = BPF_JMP | BPF_TAIL_CALL; 7980 7981 aux = &env->insn_aux_data[i + delta]; 7982 if (!bpf_map_ptr_unpriv(aux)) 7983 continue; 7984 7985 /* instead of changing every JIT dealing with tail_call 7986 * emit two extra insns: 7987 * if (index >= max_entries) goto out; 7988 * index &= array->index_mask; 7989 * to avoid out-of-bounds cpu speculation 7990 */ 7991 if (bpf_map_ptr_poisoned(aux)) { 7992 verbose(env, "tail_call abusing map_ptr\n"); 7993 return -EINVAL; 7994 } 7995 7996 map_ptr = BPF_MAP_PTR(aux->map_state); 7997 insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3, 7998 map_ptr->max_entries, 2); 7999 insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3, 8000 container_of(map_ptr, 8001 struct bpf_array, 8002 map)->index_mask); 8003 insn_buf[2] = *insn; 8004 cnt = 3; 8005 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt); 8006 if (!new_prog) 8007 return -ENOMEM; 8008 8009 delta += cnt - 1; 8010 env->prog = prog = new_prog; 8011 insn = new_prog->insnsi + i + delta; 8012 continue; 8013 } 8014 8015 /* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup 8016 * and other inlining handlers are currently limited to 64 bit 8017 * only. 8018 */ 8019 if (prog->jit_requested && BITS_PER_LONG == 64 && 8020 (insn->imm == BPF_FUNC_map_lookup_elem || 8021 insn->imm == BPF_FUNC_map_update_elem || 8022 insn->imm == BPF_FUNC_map_delete_elem || 8023 insn->imm == BPF_FUNC_map_push_elem || 8024 insn->imm == BPF_FUNC_map_pop_elem || 8025 insn->imm == BPF_FUNC_map_peek_elem)) { 8026 aux = &env->insn_aux_data[i + delta]; 8027 if (bpf_map_ptr_poisoned(aux)) 8028 goto patch_call_imm; 8029 8030 map_ptr = BPF_MAP_PTR(aux->map_state); 8031 ops = map_ptr->ops; 8032 if (insn->imm == BPF_FUNC_map_lookup_elem && 8033 ops->map_gen_lookup) { 8034 cnt = ops->map_gen_lookup(map_ptr, insn_buf); 8035 if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) { 8036 verbose(env, "bpf verifier is misconfigured\n"); 8037 return -EINVAL; 8038 } 8039 8040 new_prog = bpf_patch_insn_data(env, i + delta, 8041 insn_buf, cnt); 8042 if (!new_prog) 8043 return -ENOMEM; 8044 8045 delta += cnt - 1; 8046 env->prog = prog = new_prog; 8047 insn = new_prog->insnsi + i + delta; 8048 continue; 8049 } 8050 8051 BUILD_BUG_ON(!__same_type(ops->map_lookup_elem, 8052 (void *(*)(struct bpf_map *map, void *key))NULL)); 8053 BUILD_BUG_ON(!__same_type(ops->map_delete_elem, 8054 (int (*)(struct bpf_map *map, void *key))NULL)); 8055 BUILD_BUG_ON(!__same_type(ops->map_update_elem, 8056 (int (*)(struct bpf_map *map, void *key, void *value, 8057 u64 flags))NULL)); 8058 BUILD_BUG_ON(!__same_type(ops->map_push_elem, 8059 (int (*)(struct bpf_map *map, void *value, 8060 u64 flags))NULL)); 8061 BUILD_BUG_ON(!__same_type(ops->map_pop_elem, 8062 (int (*)(struct bpf_map *map, void *value))NULL)); 8063 BUILD_BUG_ON(!__same_type(ops->map_peek_elem, 8064 (int (*)(struct bpf_map *map, void *value))NULL)); 8065 8066 switch (insn->imm) { 8067 case BPF_FUNC_map_lookup_elem: 8068 insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) - 8069 __bpf_call_base; 8070 continue; 8071 case BPF_FUNC_map_update_elem: 8072 insn->imm = BPF_CAST_CALL(ops->map_update_elem) - 8073 __bpf_call_base; 8074 continue; 8075 case BPF_FUNC_map_delete_elem: 8076 insn->imm = BPF_CAST_CALL(ops->map_delete_elem) - 8077 __bpf_call_base; 8078 continue; 8079 case BPF_FUNC_map_push_elem: 8080 insn->imm = BPF_CAST_CALL(ops->map_push_elem) - 8081 __bpf_call_base; 8082 continue; 8083 case BPF_FUNC_map_pop_elem: 8084 insn->imm = BPF_CAST_CALL(ops->map_pop_elem) - 8085 __bpf_call_base; 8086 continue; 8087 case BPF_FUNC_map_peek_elem: 8088 insn->imm = BPF_CAST_CALL(ops->map_peek_elem) - 8089 __bpf_call_base; 8090 continue; 8091 } 8092 8093 goto patch_call_imm; 8094 } 8095 8096 patch_call_imm: 8097 fn = env->ops->get_func_proto(insn->imm, env->prog); 8098 /* all functions that have prototype and verifier allowed 8099 * programs to call them, must be real in-kernel functions 8100 */ 8101 if (!fn->func) { 8102 verbose(env, 8103 "kernel subsystem misconfigured func %s#%d\n", 8104 func_id_name(insn->imm), insn->imm); 8105 return -EFAULT; 8106 } 8107 insn->imm = fn->func - __bpf_call_base; 8108 } 8109 8110 return 0; 8111 } 8112 8113 static void free_states(struct bpf_verifier_env *env) 8114 { 8115 struct bpf_verifier_state_list *sl, *sln; 8116 int i; 8117 8118 sl = env->free_list; 8119 while (sl) { 8120 sln = sl->next; 8121 free_verifier_state(&sl->state, false); 8122 kfree(sl); 8123 sl = sln; 8124 } 8125 8126 if (!env->explored_states) 8127 return; 8128 8129 for (i = 0; i < env->prog->len; i++) { 8130 sl = env->explored_states[i]; 8131 8132 if (sl) 8133 while (sl != STATE_LIST_MARK) { 8134 sln = sl->next; 8135 free_verifier_state(&sl->state, false); 8136 kfree(sl); 8137 sl = sln; 8138 } 8139 } 8140 8141 kvfree(env->explored_states); 8142 } 8143 8144 static void print_verification_stats(struct bpf_verifier_env *env) 8145 { 8146 int i; 8147 8148 if (env->log.level & BPF_LOG_STATS) { 8149 verbose(env, "verification time %lld usec\n", 8150 div_u64(env->verification_time, 1000)); 8151 verbose(env, "stack depth "); 8152 for (i = 0; i < env->subprog_cnt; i++) { 8153 u32 depth = env->subprog_info[i].stack_depth; 8154 8155 verbose(env, "%d", depth); 8156 if (i + 1 < env->subprog_cnt) 8157 verbose(env, "+"); 8158 } 8159 verbose(env, "\n"); 8160 } 8161 verbose(env, "processed %d insns (limit %d) max_states_per_insn %d " 8162 "total_states %d peak_states %d mark_read %d\n", 8163 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, 8164 env->max_states_per_insn, env->total_states, 8165 env->peak_states, env->longest_mark_read_walk); 8166 } 8167 8168 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, 8169 union bpf_attr __user *uattr) 8170 { 8171 u64 start_time = ktime_get_ns(); 8172 struct bpf_verifier_env *env; 8173 struct bpf_verifier_log *log; 8174 int i, len, ret = -EINVAL; 8175 bool is_priv; 8176 8177 /* no program is valid */ 8178 if (ARRAY_SIZE(bpf_verifier_ops) == 0) 8179 return -EINVAL; 8180 8181 /* 'struct bpf_verifier_env' can be global, but since it's not small, 8182 * allocate/free it every time bpf_check() is called 8183 */ 8184 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL); 8185 if (!env) 8186 return -ENOMEM; 8187 log = &env->log; 8188 8189 len = (*prog)->len; 8190 env->insn_aux_data = 8191 vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len)); 8192 ret = -ENOMEM; 8193 if (!env->insn_aux_data) 8194 goto err_free_env; 8195 for (i = 0; i < len; i++) 8196 env->insn_aux_data[i].orig_idx = i; 8197 env->prog = *prog; 8198 env->ops = bpf_verifier_ops[env->prog->type]; 8199 is_priv = capable(CAP_SYS_ADMIN); 8200 8201 /* grab the mutex to protect few globals used by verifier */ 8202 if (!is_priv) 8203 mutex_lock(&bpf_verifier_lock); 8204 8205 if (attr->log_level || attr->log_buf || attr->log_size) { 8206 /* user requested verbose verifier output 8207 * and supplied buffer to store the verification trace 8208 */ 8209 log->level = attr->log_level; 8210 log->ubuf = (char __user *) (unsigned long) attr->log_buf; 8211 log->len_total = attr->log_size; 8212 8213 ret = -EINVAL; 8214 /* log attributes have to be sane */ 8215 if (log->len_total < 128 || log->len_total > UINT_MAX >> 2 || 8216 !log->level || !log->ubuf || log->level & ~BPF_LOG_MASK) 8217 goto err_unlock; 8218 } 8219 8220 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT); 8221 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 8222 env->strict_alignment = true; 8223 if (attr->prog_flags & BPF_F_ANY_ALIGNMENT) 8224 env->strict_alignment = false; 8225 8226 env->allow_ptr_leaks = is_priv; 8227 8228 ret = replace_map_fd_with_map_ptr(env); 8229 if (ret < 0) 8230 goto skip_full_check; 8231 8232 if (bpf_prog_is_dev_bound(env->prog->aux)) { 8233 ret = bpf_prog_offload_verifier_prep(env->prog); 8234 if (ret) 8235 goto skip_full_check; 8236 } 8237 8238 env->explored_states = kvcalloc(env->prog->len, 8239 sizeof(struct bpf_verifier_state_list *), 8240 GFP_USER); 8241 ret = -ENOMEM; 8242 if (!env->explored_states) 8243 goto skip_full_check; 8244 8245 ret = check_subprogs(env); 8246 if (ret < 0) 8247 goto skip_full_check; 8248 8249 ret = check_btf_info(env, attr, uattr); 8250 if (ret < 0) 8251 goto skip_full_check; 8252 8253 ret = check_cfg(env); 8254 if (ret < 0) 8255 goto skip_full_check; 8256 8257 ret = do_check(env); 8258 if (env->cur_state) { 8259 free_verifier_state(env->cur_state, true); 8260 env->cur_state = NULL; 8261 } 8262 8263 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) 8264 ret = bpf_prog_offload_finalize(env); 8265 8266 skip_full_check: 8267 while (!pop_stack(env, NULL, NULL)); 8268 free_states(env); 8269 8270 if (ret == 0) 8271 ret = check_max_stack_depth(env); 8272 8273 /* instruction rewrites happen after this point */ 8274 if (is_priv) { 8275 if (ret == 0) 8276 opt_hard_wire_dead_code_branches(env); 8277 if (ret == 0) 8278 ret = opt_remove_dead_code(env); 8279 if (ret == 0) 8280 ret = opt_remove_nops(env); 8281 } else { 8282 if (ret == 0) 8283 sanitize_dead_code(env); 8284 } 8285 8286 if (ret == 0) 8287 /* program is valid, convert *(u32*)(ctx + off) accesses */ 8288 ret = convert_ctx_accesses(env); 8289 8290 if (ret == 0) 8291 ret = fixup_bpf_calls(env); 8292 8293 if (ret == 0) 8294 ret = fixup_call_args(env); 8295 8296 env->verification_time = ktime_get_ns() - start_time; 8297 print_verification_stats(env); 8298 8299 if (log->level && bpf_verifier_log_full(log)) 8300 ret = -ENOSPC; 8301 if (log->level && !log->ubuf) { 8302 ret = -EFAULT; 8303 goto err_release_maps; 8304 } 8305 8306 if (ret == 0 && env->used_map_cnt) { 8307 /* if program passed verifier, update used_maps in bpf_prog_info */ 8308 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, 8309 sizeof(env->used_maps[0]), 8310 GFP_KERNEL); 8311 8312 if (!env->prog->aux->used_maps) { 8313 ret = -ENOMEM; 8314 goto err_release_maps; 8315 } 8316 8317 memcpy(env->prog->aux->used_maps, env->used_maps, 8318 sizeof(env->used_maps[0]) * env->used_map_cnt); 8319 env->prog->aux->used_map_cnt = env->used_map_cnt; 8320 8321 /* program is valid. Convert pseudo bpf_ld_imm64 into generic 8322 * bpf_ld_imm64 instructions 8323 */ 8324 convert_pseudo_ld_imm64(env); 8325 } 8326 8327 if (ret == 0) 8328 adjust_btf_func(env); 8329 8330 err_release_maps: 8331 if (!env->prog->aux->used_maps) 8332 /* if we didn't copy map pointers into bpf_prog_info, release 8333 * them now. Otherwise free_used_maps() will release them. 8334 */ 8335 release_maps(env); 8336 *prog = env->prog; 8337 err_unlock: 8338 if (!is_priv) 8339 mutex_unlock(&bpf_verifier_lock); 8340 vfree(env->insn_aux_data); 8341 err_free_env: 8342 kfree(env); 8343 return ret; 8344 } 8345