verifier.c 228 KB
Newer Older
1
/* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com
2
 * Copyright (c) 2016 Facebook
3
 * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
4 5 6 7 8 9 10 11 12 13
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of version 2 of the GNU General Public
 * License as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 */
14
#include <uapi/linux/btf.h>
15 16 17 18
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/bpf.h>
19
#include <linux/btf.h>
20
#include <linux/bpf_verifier.h>
21 22 23 24
#include <linux/filter.h>
#include <net/netlink.h>
#include <linux/file.h>
#include <linux/vmalloc.h>
25
#include <linux/stringify.h>
26 27
#include <linux/bsearch.h>
#include <linux/sort.h>
28
#include <linux/perf_event.h>
29
#include <linux/ctype.h>
30

31 32
#include "disasm.h"

33 34 35 36 37 38 39 40 41
static const struct bpf_verifier_ops * const bpf_verifier_ops[] = {
#define BPF_PROG_TYPE(_id, _name) \
	[_id] = & _name ## _verifier_ops,
#define BPF_MAP_TYPE(_id, _ops)
#include <linux/bpf_types.h>
#undef BPF_PROG_TYPE
#undef BPF_MAP_TYPE
};

42 43 44 45 46 47 48 49 50 51 52 53
/* bpf_check() is a static code analyzer that walks eBPF program
 * instruction by instruction and updates register/stack state.
 * All paths of conditional branches are analyzed until 'bpf_exit' insn.
 *
 * The first pass is depth-first-search to check that the program is a DAG.
 * It rejects the following programs:
 * - larger than BPF_MAXINSNS insns
 * - if loop is present (detected via back-edge)
 * - unreachable insns exist (shouldn't be a forest. program = one function)
 * - out of bounds or malformed jumps
 * The second pass is all possible path descent from the 1st insn.
 * Since it's analyzing all pathes through the program, the length of the
54
 * analysis is limited to 64k insn, which may be hit even if total number of
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
 * insn is less then 4K, but there are too many branches that change stack/regs.
 * Number of 'branches to be analyzed' is limited to 1k
 *
 * On entry to each instruction, each register has a type, and the instruction
 * changes the types of the registers depending on instruction semantics.
 * If instruction is BPF_MOV64_REG(BPF_REG_1, BPF_REG_5), then type of R5 is
 * copied to R1.
 *
 * All registers are 64-bit.
 * R0 - return register
 * R1-R5 argument passing registers
 * R6-R9 callee saved registers
 * R10 - frame pointer read-only
 *
 * At the start of BPF program the register R1 contains a pointer to bpf_context
 * and has type PTR_TO_CTX.
 *
 * Verifier tracks arithmetic operations on pointers in case:
 *    BPF_MOV64_REG(BPF_REG_1, BPF_REG_10),
 *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -20),
 * 1st insn copies R10 (which has FRAME_PTR) type into R1
 * and 2nd arithmetic instruction is pattern matched to recognize
 * that it wants to construct a pointer to some element within stack.
 * So after 2nd insn, the register R1 has type PTR_TO_STACK
 * (and -20 constant is saved for further stack bounds checking).
 * Meaning that this reg is a pointer to stack plus known immediate constant.
 *
82
 * Most of the time the registers have SCALAR_VALUE type, which
83
 * means the register has some value, but it's not a valid pointer.
84
 * (like pointer plus pointer becomes SCALAR_VALUE type)
85 86
 *
 * When verifier sees load or store instructions the type of base register
87 88
 * can be: PTR_TO_MAP_VALUE, PTR_TO_CTX, PTR_TO_STACK, PTR_TO_SOCKET. These are
 * four pointer types recognized by check_mem_access() function.
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
 *
 * PTR_TO_MAP_VALUE means that this register is pointing to 'map element value'
 * and the range of [ptr, ptr + map's value_size) is accessible.
 *
 * registers used to pass values to function calls are checked against
 * function argument constraints.
 *
 * ARG_PTR_TO_MAP_KEY is one of such argument constraints.
 * It means that the register type passed to this function must be
 * PTR_TO_STACK and it will be used inside the function as
 * 'pointer to map element key'
 *
 * For example the argument constraints for bpf_map_lookup_elem():
 *   .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
 *   .arg1_type = ARG_CONST_MAP_PTR,
 *   .arg2_type = ARG_PTR_TO_MAP_KEY,
 *
 * ret_type says that this function returns 'pointer to map elem value or null'
 * function expects 1st argument to be a const pointer to 'struct bpf_map' and
 * 2nd argument should be a pointer to stack, which will be used inside
 * the helper function as a pointer to map element key.
 *
 * On the kernel side the helper function looks like:
 * u64 bpf_map_lookup_elem(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5)
 * {
 *    struct bpf_map *map = (struct bpf_map *) (unsigned long) r1;
 *    void *key = (void *) (unsigned long) r2;
 *    void *value;
 *
 *    here kernel can access 'key' and 'map' pointers safely, knowing that
 *    [key, key + map->key_size) bytes are valid and were initialized on
 *    the stack of eBPF program.
 * }
 *
 * Corresponding eBPF program may look like:
 *    BPF_MOV64_REG(BPF_REG_2, BPF_REG_10),  // after this insn R2 type is FRAME_PTR
 *    BPF_ALU64_IMM(BPF_ADD, BPF_REG_2, -4), // after this insn R2 type is PTR_TO_STACK
 *    BPF_LD_MAP_FD(BPF_REG_1, map_fd),      // after this insn R1 type is CONST_PTR_TO_MAP
 *    BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_map_lookup_elem),
 * here verifier looks at prototype of map_lookup_elem() and sees:
 * .arg1_type == ARG_CONST_MAP_PTR and R1->type == CONST_PTR_TO_MAP, which is ok,
 * Now verifier knows that this map has key of R1->map_ptr->key_size bytes
 *
 * Then .arg2_type == ARG_PTR_TO_MAP_KEY and R2->type == PTR_TO_STACK, ok so far,
 * Now verifier checks that [R2, R2 + map's key_size) are within stack limits
 * and were initialized prior to this call.
 * If it's ok, then verifier allows this BPF_CALL insn and looks at
 * .ret_type which is RET_PTR_TO_MAP_VALUE_OR_NULL, so it sets
 * R0->type = PTR_TO_MAP_VALUE_OR_NULL which means bpf_map_lookup_elem() function
 * returns ether pointer to map value or NULL.
 *
 * When type PTR_TO_MAP_VALUE_OR_NULL passes through 'if (reg != 0) goto +off'
 * insn, the register holding that pointer in the true branch changes state to
 * PTR_TO_MAP_VALUE and the same register changes state to CONST_IMM in the false
 * branch. See check_cond_jmp_op().
 *
 * After the call R0 is set to return type of the function and registers R1-R5
 * are set to NOT_INIT to indicate that they are no longer readable.
147 148 149 150 151 152 153 154 155 156 157 158
 *
 * The following reference types represent a potential reference to a kernel
 * resource which, after first being allocated, must be checked and freed by
 * the BPF program:
 * - PTR_TO_SOCKET_OR_NULL, PTR_TO_SOCKET
 *
 * When the verifier sees a helper call return a reference type, it allocates a
 * pointer id for the reference and stores it in the current function state.
 * Similar to the way that PTR_TO_MAP_VALUE_OR_NULL is converted into
 * PTR_TO_MAP_VALUE, PTR_TO_SOCKET_OR_NULL becomes PTR_TO_SOCKET when the type
 * passes through a NULL-check conditional. For the branch wherein the state is
 * changed to CONST_IMM, the verifier releases the reference.
159 160 161 162 163 164
 *
 * For each helper function that allocates a reference, such as
 * bpf_sk_lookup_tcp(), there is a corresponding release function, such as
 * bpf_sk_release(). When a reference type passes into the release function,
 * the verifier also releases the reference. If any unchecked or unreleased
 * reference remains at the end of the program, the verifier rejects it.
165 166
 */

167
/* verifier_state + insn_idx are pushed to stack when branch is encountered */
168
struct bpf_verifier_stack_elem {
169 170 171 172
	/* verifer state is 'st'
	 * before processing instruction 'insn_idx'
	 * and after processing instruction 'prev_insn_idx'
	 */
173
	struct bpf_verifier_state st;
174 175
	int insn_idx;
	int prev_insn_idx;
176
	struct bpf_verifier_stack_elem *next;
177 178
};

179
#define BPF_COMPLEXITY_LIMIT_INSNS	131072
180
#define BPF_COMPLEXITY_LIMIT_STACK	1024
181
#define BPF_COMPLEXITY_LIMIT_STATES	64
182

183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
#define BPF_MAP_PTR_UNPRIV	1UL
#define BPF_MAP_PTR_POISON	((void *)((0xeB9FUL << 1) +	\
					  POISON_POINTER_DELTA))
#define BPF_MAP_PTR(X)		((struct bpf_map *)((X) & ~BPF_MAP_PTR_UNPRIV))

static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
{
	return BPF_MAP_PTR(aux->map_state) == BPF_MAP_PTR_POISON;
}

static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
{
	return aux->map_state & BPF_MAP_PTR_UNPRIV;
}

static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux,
			      const struct bpf_map *map, bool unpriv)
{
	BUILD_BUG_ON((unsigned long)BPF_MAP_PTR_POISON & BPF_MAP_PTR_UNPRIV);
	unpriv |= bpf_map_ptr_unpriv(aux);
	aux->map_state = (unsigned long)map |
			 (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
}
206

207 208
struct bpf_call_arg_meta {
	struct bpf_map *map_ptr;
209
	bool raw_mode;
210
	bool pkt_access;
211 212
	int regno;
	int access_size;
213 214
	s64 msize_smax_value;
	u64 msize_umax_value;
215
	int ref_obj_id;
216
	int func_id;
217 218
};

219 220
static DEFINE_MUTEX(bpf_verifier_lock);

221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
static const struct bpf_line_info *
find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
{
	const struct bpf_line_info *linfo;
	const struct bpf_prog *prog;
	u32 i, nr_linfo;

	prog = env->prog;
	nr_linfo = prog->aux->nr_linfo;

	if (!nr_linfo || insn_off >= prog->len)
		return NULL;

	linfo = prog->aux->linfo;
	for (i = 1; i < nr_linfo; i++)
		if (insn_off < linfo[i].insn_off)
			break;

	return &linfo[i - 1];
}

242 243
void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
		       va_list args)
244
{
245
	unsigned int n;
246

247 248 249 250 251 252 253 254 255 256 257 258
	n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);

	WARN_ONCE(n >= BPF_VERIFIER_TMP_LOG_SIZE - 1,
		  "verifier log line truncated - local buffer too short\n");

	n = min(log->len_total - log->len_used - 1, n);
	log->kbuf[n] = '\0';

	if (!copy_to_user(log->ubuf + log->len_used, log->kbuf, n + 1))
		log->len_used += n;
	else
		log->ubuf = NULL;
259
}
260 261 262 263

/* log_level controls verbosity level of eBPF verifier.
 * bpf_verifier_log_write() is used to dump the verification trace to the log,
 * so the user can figure out what's wrong with the program
264
 */
265 266 267 268 269
__printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
					   const char *fmt, ...)
{
	va_list args;

270 271 272
	if (!bpf_verifier_log_needed(&env->log))
		return;

273
	va_start(args, fmt);
274
	bpf_verifier_vlog(&env->log, fmt, args);
275 276 277 278 279 280
	va_end(args);
}
EXPORT_SYMBOL_GPL(bpf_verifier_log_write);

__printf(2, 3) static void verbose(void *private_data, const char *fmt, ...)
{
281
	struct bpf_verifier_env *env = private_data;
282 283
	va_list args;

284 285 286
	if (!bpf_verifier_log_needed(&env->log))
		return;

287
	va_start(args, fmt);
288
	bpf_verifier_vlog(&env->log, fmt, args);
289 290
	va_end(args);
}
291

292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
static const char *ltrim(const char *s)
{
	while (isspace(*s))
		s++;

	return s;
}

__printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env,
					 u32 insn_off,
					 const char *prefix_fmt, ...)
{
	const struct bpf_line_info *linfo;

	if (!bpf_verifier_log_needed(&env->log))
		return;

	linfo = find_linfo(env, insn_off);
	if (!linfo || linfo == env->prev_linfo)
		return;

	if (prefix_fmt) {
		va_list args;

		va_start(args, prefix_fmt);
		bpf_verifier_vlog(&env->log, prefix_fmt, args);
		va_end(args);
	}

	verbose(env, "%s\n",
		ltrim(btf_name_by_offset(env->prog->aux->btf,
					 linfo->line_off)));

	env->prev_linfo = linfo;
}

328 329 330 331 332 333
static bool type_is_pkt_pointer(enum bpf_reg_type type)
{
	return type == PTR_TO_PACKET ||
	       type == PTR_TO_PACKET_META;
}

334 335 336
static bool type_is_sk_pointer(enum bpf_reg_type type)
{
	return type == PTR_TO_SOCKET ||
337 338
		type == PTR_TO_SOCK_COMMON ||
		type == PTR_TO_TCP_SOCK;
339 340
}

341 342
static bool reg_type_may_be_null(enum bpf_reg_type type)
{
343
	return type == PTR_TO_MAP_VALUE_OR_NULL ||
344
	       type == PTR_TO_SOCKET_OR_NULL ||
345 346
	       type == PTR_TO_SOCK_COMMON_OR_NULL ||
	       type == PTR_TO_TCP_SOCK_OR_NULL;
347 348
}

349 350 351 352 353 354
static bool reg_may_point_to_spin_lock(const struct bpf_reg_state *reg)
{
	return reg->type == PTR_TO_MAP_VALUE &&
		map_value_has_spin_lock(reg->map_ptr);
}

355
static bool arg_type_may_be_refcounted(enum bpf_arg_type type)
356
{
357
	return type == ARG_PTR_TO_SOCK_COMMON;
358 359 360 361 362 363 364 365
}

/* Determine whether the function releases some resources allocated by another
 * function call. The first reference type argument will be assumed to be
 * released by release_reference().
 */
static bool is_release_function(enum bpf_func_id func_id)
{
366
	return func_id == BPF_FUNC_sk_release;
367 368
}

369 370 371
static bool is_acquire_function(enum bpf_func_id func_id)
{
	return func_id == BPF_FUNC_sk_lookup_tcp ||
372 373
		func_id == BPF_FUNC_sk_lookup_udp ||
		func_id == BPF_FUNC_skc_lookup_tcp;
374 375
}

376 377 378 379 380 381
static bool is_ptr_cast_function(enum bpf_func_id func_id)
{
	return func_id == BPF_FUNC_tcp_sock ||
		func_id == BPF_FUNC_sk_fullsock;
}

382 383 384
/* string representation of 'enum bpf_reg_type' */
static const char * const reg_type_str[] = {
	[NOT_INIT]		= "?",
385
	[SCALAR_VALUE]		= "inv",
386 387 388 389 390
	[PTR_TO_CTX]		= "ctx",
	[CONST_PTR_TO_MAP]	= "map_ptr",
	[PTR_TO_MAP_VALUE]	= "map_value",
	[PTR_TO_MAP_VALUE_OR_NULL] = "map_value_or_null",
	[PTR_TO_STACK]		= "fp",
391
	[PTR_TO_PACKET]		= "pkt",
392
	[PTR_TO_PACKET_META]	= "pkt_meta",
393
	[PTR_TO_PACKET_END]	= "pkt_end",
394
	[PTR_TO_FLOW_KEYS]	= "flow_keys",
395 396
	[PTR_TO_SOCKET]		= "sock",
	[PTR_TO_SOCKET_OR_NULL] = "sock_or_null",
397 398
	[PTR_TO_SOCK_COMMON]	= "sock_common",
	[PTR_TO_SOCK_COMMON_OR_NULL] = "sock_common_or_null",
399 400
	[PTR_TO_TCP_SOCK]	= "tcp_sock",
	[PTR_TO_TCP_SOCK_OR_NULL] = "tcp_sock_or_null",
401 402
};

403 404 405 406 407 408 409
static char slot_type_char[] = {
	[STACK_INVALID]	= '?',
	[STACK_SPILL]	= 'r',
	[STACK_MISC]	= 'm',
	[STACK_ZERO]	= '0',
};

410 411 412
static void print_liveness(struct bpf_verifier_env *env,
			   enum bpf_reg_liveness live)
{
413
	if (live & (REG_LIVE_READ | REG_LIVE_WRITTEN | REG_LIVE_DONE))
414 415 416 417 418
	    verbose(env, "_");
	if (live & REG_LIVE_READ)
		verbose(env, "r");
	if (live & REG_LIVE_WRITTEN)
		verbose(env, "w");
419 420
	if (live & REG_LIVE_DONE)
		verbose(env, "D");
421 422
}

423 424 425 426 427 428 429 430
static struct bpf_func_state *func(struct bpf_verifier_env *env,
				   const struct bpf_reg_state *reg)
{
	struct bpf_verifier_state *cur = env->cur_state;

	return cur->frame[reg->frameno];
}

431
static void print_verifier_state(struct bpf_verifier_env *env,
432
				 const struct bpf_func_state *state)
433
{
434
	const struct bpf_reg_state *reg;
435 436 437
	enum bpf_reg_type t;
	int i;

438 439
	if (state->frameno)
		verbose(env, " frame%d:", state->frameno);
440
	for (i = 0; i < MAX_BPF_REG; i++) {
441 442
		reg = &state->regs[i];
		t = reg->type;
443 444
		if (t == NOT_INIT)
			continue;
445 446 447
		verbose(env, " R%d", i);
		print_liveness(env, reg->live);
		verbose(env, "=%s", reg_type_str[t]);
448 449 450
		if ((t == SCALAR_VALUE || t == PTR_TO_STACK) &&
		    tnum_is_const(reg->var_off)) {
			/* reg->off should be 0 for SCALAR_VALUE */
451
			verbose(env, "%lld", reg->var_off.value + reg->off);
452 453
			if (t == PTR_TO_STACK)
				verbose(env, ",call_%d", func(env, reg)->callsite);
454
		} else {
455 456
			verbose(env, "(id=%d ref_obj_id=%d", reg->id,
				reg->ref_obj_id);
457
			if (t != SCALAR_VALUE)
458
				verbose(env, ",off=%d", reg->off);
459
			if (type_is_pkt_pointer(t))
460
				verbose(env, ",r=%d", reg->range);
461 462 463
			else if (t == CONST_PTR_TO_MAP ||
				 t == PTR_TO_MAP_VALUE ||
				 t == PTR_TO_MAP_VALUE_OR_NULL)
464
				verbose(env, ",ks=%d,vs=%d",
465 466
					reg->map_ptr->key_size,
					reg->map_ptr->value_size);
467 468 469 470 471
			if (tnum_is_const(reg->var_off)) {
				/* Typically an immediate SCALAR_VALUE, but
				 * could be a pointer whose offset is too big
				 * for reg->off
				 */
472
				verbose(env, ",imm=%llx", reg->var_off.value);
473 474 475
			} else {
				if (reg->smin_value != reg->umin_value &&
				    reg->smin_value != S64_MIN)
476
					verbose(env, ",smin_value=%lld",
477 478 479
						(long long)reg->smin_value);
				if (reg->smax_value != reg->umax_value &&
				    reg->smax_value != S64_MAX)
480
					verbose(env, ",smax_value=%lld",
481 482
						(long long)reg->smax_value);
				if (reg->umin_value != 0)
483
					verbose(env, ",umin_value=%llu",
484 485
						(unsigned long long)reg->umin_value);
				if (reg->umax_value != U64_MAX)
486
					verbose(env, ",umax_value=%llu",
487 488 489
						(unsigned long long)reg->umax_value);
				if (!tnum_is_unknown(reg->var_off)) {
					char tn_buf[48];
490

491
					tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
492
					verbose(env, ",var_off=%s", tn_buf);
493
				}
494
			}
495
			verbose(env, ")");
496
		}
497
	}
498
	for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
		char types_buf[BPF_REG_SIZE + 1];
		bool valid = false;
		int j;

		for (j = 0; j < BPF_REG_SIZE; j++) {
			if (state->stack[i].slot_type[j] != STACK_INVALID)
				valid = true;
			types_buf[j] = slot_type_char[
					state->stack[i].slot_type[j]];
		}
		types_buf[BPF_REG_SIZE] = 0;
		if (!valid)
			continue;
		verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
		print_liveness(env, state->stack[i].spilled_ptr.live);
		if (state->stack[i].slot_type[0] == STACK_SPILL)
515
			verbose(env, "=%s",
516
				reg_type_str[state->stack[i].spilled_ptr.type]);
517 518
		else
			verbose(env, "=%s", types_buf);
519
	}
520 521 522 523 524 525
	if (state->acquired_refs && state->refs[0].id) {
		verbose(env, " refs=%d", state->refs[0].id);
		for (i = 1; i < state->acquired_refs; i++)
			if (state->refs[i].id)
				verbose(env, ",%d", state->refs[i].id);
	}
526
	verbose(env, "\n");
527 528
}

529 530 531 532 533 534 535 536 537 538 539 540 541 542
#define COPY_STATE_FN(NAME, COUNT, FIELD, SIZE)				\
static int copy_##NAME##_state(struct bpf_func_state *dst,		\
			       const struct bpf_func_state *src)	\
{									\
	if (!src->FIELD)						\
		return 0;						\
	if (WARN_ON_ONCE(dst->COUNT < src->COUNT)) {			\
		/* internal bug, make state invalid to reject the program */ \
		memset(dst, 0, sizeof(*dst));				\
		return -EFAULT;						\
	}								\
	memcpy(dst->FIELD, src->FIELD,					\
	       sizeof(*src->FIELD) * (src->COUNT / SIZE));		\
	return 0;							\
543
}
544 545
/* copy_reference_state() */
COPY_STATE_FN(reference, acquired_refs, refs, 1)
546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583
/* copy_stack_state() */
COPY_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE)
#undef COPY_STATE_FN

#define REALLOC_STATE_FN(NAME, COUNT, FIELD, SIZE)			\
static int realloc_##NAME##_state(struct bpf_func_state *state, int size, \
				  bool copy_old)			\
{									\
	u32 old_size = state->COUNT;					\
	struct bpf_##NAME##_state *new_##FIELD;				\
	int slot = size / SIZE;						\
									\
	if (size <= old_size || !size) {				\
		if (copy_old)						\
			return 0;					\
		state->COUNT = slot * SIZE;				\
		if (!size && old_size) {				\
			kfree(state->FIELD);				\
			state->FIELD = NULL;				\
		}							\
		return 0;						\
	}								\
	new_##FIELD = kmalloc_array(slot, sizeof(struct bpf_##NAME##_state), \
				    GFP_KERNEL);			\
	if (!new_##FIELD)						\
		return -ENOMEM;						\
	if (copy_old) {							\
		if (state->FIELD)					\
			memcpy(new_##FIELD, state->FIELD,		\
			       sizeof(*new_##FIELD) * (old_size / SIZE)); \
		memset(new_##FIELD + old_size / SIZE, 0,		\
		       sizeof(*new_##FIELD) * (size - old_size) / SIZE); \
	}								\
	state->COUNT = slot * SIZE;					\
	kfree(state->FIELD);						\
	state->FIELD = new_##FIELD;					\
	return 0;							\
}
584 585
/* realloc_reference_state() */
REALLOC_STATE_FN(reference, acquired_refs, refs, 1)
586 587 588
/* realloc_stack_state() */
REALLOC_STATE_FN(stack, allocated_stack, stack, BPF_REG_SIZE)
#undef REALLOC_STATE_FN
589 590 591

/* do_check() starts with zero-sized stack in struct bpf_verifier_state to
 * make it consume minimal amount of memory. check_stack_write() access from
592
 * the program calls into realloc_func_state() to grow the stack size.
593 594 595
 * Note there is a non-zero 'parent' pointer inside bpf_verifier_state
 * which realloc_stack_state() copies over. It points to previous
 * bpf_verifier_state which is never reallocated.
596
 */
597 598
static int realloc_func_state(struct bpf_func_state *state, int stack_size,
			      int refs_size, bool copy_old)
599
{
600 601 602 603 604 605 606 607 608 609
	int err = realloc_reference_state(state, refs_size, copy_old);
	if (err)
		return err;
	return realloc_stack_state(state, stack_size, copy_old);
}

/* Acquire a pointer id from the env and update the state->refs to include
 * this new pointer reference.
 * On success, returns a valid pointer id to associate with the register
 * On failure, returns a negative errno.
610
 */
611
static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
612
{
613 614 615 616 617 618 619 620 621 622
	struct bpf_func_state *state = cur_func(env);
	int new_ofs = state->acquired_refs;
	int id, err;

	err = realloc_reference_state(state, state->acquired_refs + 1, true);
	if (err)
		return err;
	id = ++env->id_gen;
	state->refs[new_ofs].id = id;
	state->refs[new_ofs].insn_idx = insn_idx;
623

624 625 626 627
	return id;
}

/* release function corresponding to acquire_reference_state(). Idempotent. */
628
static int release_reference_state(struct bpf_func_state *state, int ptr_id)
629 630 631 632 633 634 635 636 637 638 639
{
	int i, last_idx;

	last_idx = state->acquired_refs - 1;
	for (i = 0; i < state->acquired_refs; i++) {
		if (state->refs[i].id == ptr_id) {
			if (last_idx && i != last_idx)
				memcpy(&state->refs[i], &state->refs[last_idx],
				       sizeof(*state->refs));
			memset(&state->refs[last_idx], 0, sizeof(*state->refs));
			state->acquired_refs--;
640 641 642
			return 0;
		}
	}
643
	return -EINVAL;
644 645 646 647 648 649 650 651 652 653 654
}

static int transfer_reference_state(struct bpf_func_state *dst,
				    struct bpf_func_state *src)
{
	int err = realloc_reference_state(dst, src->acquired_refs, false);
	if (err)
		return err;
	err = copy_reference_state(dst, src);
	if (err)
		return err;
655 656 657
	return 0;
}

658 659
static void free_func_state(struct bpf_func_state *state)
{
660 661
	if (!state)
		return;
662
	kfree(state->refs);
663 664 665 666
	kfree(state->stack);
	kfree(state);
}

667 668
static void free_verifier_state(struct bpf_verifier_state *state,
				bool free_self)
669
{
670 671 672 673 674 675
	int i;

	for (i = 0; i <= state->curframe; i++) {
		free_func_state(state->frame[i]);
		state->frame[i] = NULL;
	}
676 677
	if (free_self)
		kfree(state);
678 679 680 681 682
}

/* copy verifier state from src to dst growing dst stack space
 * when necessary to accommodate larger src stack
 */
683 684
static int copy_func_state(struct bpf_func_state *dst,
			   const struct bpf_func_state *src)
685 686 687
{
	int err;

688 689 690 691 692 693
	err = realloc_func_state(dst, src->allocated_stack, src->acquired_refs,
				 false);
	if (err)
		return err;
	memcpy(dst, src, offsetof(struct bpf_func_state, acquired_refs));
	err = copy_reference_state(dst, src);
694 695 696 697 698
	if (err)
		return err;
	return copy_stack_state(dst, src);
}

699 700 701 702 703 704 705 706 707 708 709
static int copy_verifier_state(struct bpf_verifier_state *dst_state,
			       const struct bpf_verifier_state *src)
{
	struct bpf_func_state *dst;
	int i, err;

	/* if dst has more stack frames then src frame, free them */
	for (i = src->curframe + 1; i <= dst_state->curframe; i++) {
		free_func_state(dst_state->frame[i]);
		dst_state->frame[i] = NULL;
	}
710
	dst_state->speculative = src->speculative;
711
	dst_state->curframe = src->curframe;
712
	dst_state->active_spin_lock = src->active_spin_lock;
713 714 715 716 717 718 719 720 721 722 723 724 725 726 727
	for (i = 0; i <= src->curframe; i++) {
		dst = dst_state->frame[i];
		if (!dst) {
			dst = kzalloc(sizeof(*dst), GFP_KERNEL);
			if (!dst)
				return -ENOMEM;
			dst_state->frame[i] = dst;
		}
		err = copy_func_state(dst, src->frame[i]);
		if (err)
			return err;
	}
	return 0;
}

728 729 730 731 732 733
static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx,
		     int *insn_idx)
{
	struct bpf_verifier_state *cur = env->cur_state;
	struct bpf_verifier_stack_elem *elem, *head = env->head;
	int err;
734 735

	if (env->head == NULL)
736
		return -ENOENT;
737

738 739 740 741 742 743 744
	if (cur) {
		err = copy_verifier_state(cur, &head->st);
		if (err)
			return err;
	}
	if (insn_idx)
		*insn_idx = head->insn_idx;
745
	if (prev_insn_idx)
746 747
		*prev_insn_idx = head->prev_insn_idx;
	elem = head->next;
748
	free_verifier_state(&head->st, false);
749
	kfree(head);
750 751
	env->head = elem;
	env->stack_size--;
752
	return 0;
753 754
}

755
static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
756 757
					     int insn_idx, int prev_insn_idx,
					     bool speculative)
758
{
759
	struct bpf_verifier_state *cur = env->cur_state;
760
	struct bpf_verifier_stack_elem *elem;
761
	int err;
762

763
	elem = kzalloc(sizeof(struct bpf_verifier_stack_elem), GFP_KERNEL);
764 765 766 767 768 769 770 771
	if (!elem)
		goto err;

	elem->insn_idx = insn_idx;
	elem->prev_insn_idx = prev_insn_idx;
	elem->next = env->head;
	env->head = elem;
	env->stack_size++;
772 773 774
	err = copy_verifier_state(&elem->st, cur);
	if (err)
		goto err;
775
	elem->st.speculative |= speculative;
776
	if (env->stack_size > BPF_COMPLEXITY_LIMIT_STACK) {
777
		verbose(env, "BPF program is too complex\n");
778 779 780 781
		goto err;
	}
	return &elem->st;
err:
782 783
	free_verifier_state(env->cur_state, true);
	env->cur_state = NULL;
784
	/* pop all elements and return */
785
	while (!pop_stack(env, NULL, NULL));
786 787 788 789 790 791 792 793
	return NULL;
}

#define CALLER_SAVED_REGS 6
static const int caller_saved[CALLER_SAVED_REGS] = {
	BPF_REG_0, BPF_REG_1, BPF_REG_2, BPF_REG_3, BPF_REG_4, BPF_REG_5
};

794 795
static void __mark_reg_not_init(struct bpf_reg_state *reg);

796 797 798 799 800
/* Mark the unknown part of a register (variable offset or scalar value) as
 * known to have the value @imm.
 */
static void __mark_reg_known(struct bpf_reg_state *reg, u64 imm)
{
801 802 803
	/* Clear id, off, and union(map_ptr, range) */
	memset(((u8 *)reg) + sizeof(reg->type), 0,
	       offsetof(struct bpf_reg_state, var_off) - sizeof(reg->type));
804 805 806 807 808 809 810
	reg->var_off = tnum_const(imm);
	reg->smin_value = (s64)imm;
	reg->smax_value = (s64)imm;
	reg->umin_value = imm;
	reg->umax_value = imm;
}

811 812 813 814
/* Mark the 'variable offset' part of a register as zero.  This should be
 * used only on registers holding a pointer type.
 */
static void __mark_reg_known_zero(struct bpf_reg_state *reg)
815
{
816
	__mark_reg_known(reg, 0);
817
}
818

819 820 821 822 823 824
static void __mark_reg_const_zero(struct bpf_reg_state *reg)
{
	__mark_reg_known(reg, 0);
	reg->type = SCALAR_VALUE;
}

825 826
static void mark_reg_known_zero(struct bpf_verifier_env *env,
				struct bpf_reg_state *regs, u32 regno)
827 828
{
	if (WARN_ON(regno >= MAX_BPF_REG)) {
829
		verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
830 831 832 833 834 835 836 837
		/* Something bad happened, let's kill all regs */
		for (regno = 0; regno < MAX_BPF_REG; regno++)
			__mark_reg_not_init(regs + regno);
		return;
	}
	__mark_reg_known_zero(regs + regno);
}

838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
static bool reg_is_pkt_pointer(const struct bpf_reg_state *reg)
{
	return type_is_pkt_pointer(reg->type);
}

static bool reg_is_pkt_pointer_any(const struct bpf_reg_state *reg)
{
	return reg_is_pkt_pointer(reg) ||
	       reg->type == PTR_TO_PACKET_END;
}

/* Unmodified PTR_TO_PACKET[_META,_END] register from ctx access. */
static bool reg_is_init_pkt_pointer(const struct bpf_reg_state *reg,
				    enum bpf_reg_type which)
{
	/* The register can already have a range from prior markings.
	 * This is fine as long as it hasn't been advanced from its
	 * origin.
	 */
	return reg->type == which &&
	       reg->id == 0 &&
	       reg->off == 0 &&
	       tnum_equals_const(reg->var_off, 0);
}

863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928
/* Attempts to improve min/max values based on var_off information */
static void __update_reg_bounds(struct bpf_reg_state *reg)
{
	/* min signed is max(sign bit) | min(other bits) */
	reg->smin_value = max_t(s64, reg->smin_value,
				reg->var_off.value | (reg->var_off.mask & S64_MIN));
	/* max signed is min(sign bit) | max(other bits) */
	reg->smax_value = min_t(s64, reg->smax_value,
				reg->var_off.value | (reg->var_off.mask & S64_MAX));
	reg->umin_value = max(reg->umin_value, reg->var_off.value);
	reg->umax_value = min(reg->umax_value,
			      reg->var_off.value | reg->var_off.mask);
}

/* Uses signed min/max values to inform unsigned, and vice-versa */
static void __reg_deduce_bounds(struct bpf_reg_state *reg)
{
	/* Learn sign from signed bounds.
	 * If we cannot cross the sign boundary, then signed and unsigned bounds
	 * are the same, so combine.  This works even in the negative case, e.g.
	 * -3 s<= x s<= -1 implies 0xf...fd u<= x u<= 0xf...ff.
	 */
	if (reg->smin_value >= 0 || reg->smax_value < 0) {
		reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
							  reg->umin_value);
		reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
							  reg->umax_value);
		return;
	}
	/* Learn sign from unsigned bounds.  Signed bounds cross the sign
	 * boundary, so we must be careful.
	 */
	if ((s64)reg->umax_value >= 0) {
		/* Positive.  We can't learn anything from the smin, but smax
		 * is positive, hence safe.
		 */
		reg->smin_value = reg->umin_value;
		reg->smax_value = reg->umax_value = min_t(u64, reg->smax_value,
							  reg->umax_value);
	} else if ((s64)reg->umin_value < 0) {
		/* Negative.  We can't learn anything from the smax, but smin
		 * is negative, hence safe.
		 */
		reg->smin_value = reg->umin_value = max_t(u64, reg->smin_value,
							  reg->umin_value);
		reg->smax_value = reg->umax_value;
	}
}

/* Attempts to improve var_off based on unsigned min/max information */
static void __reg_bound_offset(struct bpf_reg_state *reg)
{
	reg->var_off = tnum_intersect(reg->var_off,
				      tnum_range(reg->umin_value,
						 reg->umax_value));
}

/* Reset the min/max bounds of a register */
static void __mark_reg_unbounded(struct bpf_reg_state *reg)
{
	reg->smin_value = S64_MIN;
	reg->smax_value = S64_MAX;
	reg->umin_value = 0;
	reg->umax_value = U64_MAX;
}

929 930 931
/* Mark a register as having a completely unknown (scalar) value. */
static void __mark_reg_unknown(struct bpf_reg_state *reg)
{
932 933 934 935 936
	/*
	 * Clear type, id, off, and union(map_ptr, range) and
	 * padding between 'type' and union
	 */
	memset(reg, 0, offsetof(struct bpf_reg_state, var_off));
937 938
	reg->type = SCALAR_VALUE;
	reg->var_off = tnum_unknown;
939
	reg->frameno = 0;
940
	__mark_reg_unbounded(reg);
941 942
}

943 944
static void mark_reg_unknown(struct bpf_verifier_env *env,
			     struct bpf_reg_state *regs, u32 regno)
945 946
{
	if (WARN_ON(regno >= MAX_BPF_REG)) {
947
		verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
948 949
		/* Something bad happened, let's kill all regs except FP */
		for (regno = 0; regno < BPF_REG_FP; regno++)
950 951 952 953 954 955 956 957 958 959 960 961
			__mark_reg_not_init(regs + regno);
		return;
	}
	__mark_reg_unknown(regs + regno);
}

static void __mark_reg_not_init(struct bpf_reg_state *reg)
{
	__mark_reg_unknown(reg);
	reg->type = NOT_INIT;
}

962 963
static void mark_reg_not_init(struct bpf_verifier_env *env,
			      struct bpf_reg_state *regs, u32 regno)
964 965
{
	if (WARN_ON(regno >= MAX_BPF_REG)) {
966
		verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
967 968
		/* Something bad happened, let's kill all regs except FP */
		for (regno = 0; regno < BPF_REG_FP; regno++)
969 970 971 972
			__mark_reg_not_init(regs + regno);
		return;
	}
	__mark_reg_not_init(regs + regno);
973 974
}

975
static void init_reg_state(struct bpf_verifier_env *env,
976
			   struct bpf_func_state *state)
977
{
978
	struct bpf_reg_state *regs = state->regs;
979 980
	int i;

981
	for (i = 0; i < MAX_BPF_REG; i++) {
982
		mark_reg_not_init(env, regs, i);
983
		regs[i].live = REG_LIVE_NONE;
984
		regs[i].parent = NULL;
985
	}
986 987

	/* frame pointer */
988
	regs[BPF_REG_FP].type = PTR_TO_STACK;
989
	mark_reg_known_zero(env, regs, BPF_REG_FP);
990
	regs[BPF_REG_FP].frameno = state->frameno;
991 992 993

	/* 1st arg to a function */
	regs[BPF_REG_1].type = PTR_TO_CTX;
994
	mark_reg_known_zero(env, regs, BPF_REG_1);
995 996
}

997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
#define BPF_MAIN_FUNC (-1)
static void init_func_state(struct bpf_verifier_env *env,
			    struct bpf_func_state *state,
			    int callsite, int frameno, int subprogno)
{
	state->callsite = callsite;
	state->frameno = frameno;
	state->subprogno = subprogno;
	init_reg_state(env, state);
}

1008 1009 1010 1011 1012 1013
enum reg_arg_type {
	SRC_OP,		/* register is used as source operand */
	DST_OP,		/* register is used as destination operand */
	DST_OP_NO_MARK	/* same as above, check only, don't mark */
};

1014 1015
static int cmp_subprogs(const void *a, const void *b)
{
1016 1017
	return ((struct bpf_subprog_info *)a)->start -
	       ((struct bpf_subprog_info *)b)->start;
1018 1019 1020 1021
}

static int find_subprog(struct bpf_verifier_env *env, int off)
{
1022
	struct bpf_subprog_info *p;
1023

1024 1025
	p = bsearch(&off, env->subprog_info, env->subprog_cnt,
		    sizeof(env->subprog_info[0]), cmp_subprogs);
1026 1027
	if (!p)
		return -ENOENT;
1028
	return p - env->subprog_info;
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043

}

static int add_subprog(struct bpf_verifier_env *env, int off)
{
	int insn_cnt = env->prog->len;
	int ret;

	if (off >= insn_cnt || off < 0) {
		verbose(env, "call to invalid destination\n");
		return -EINVAL;
	}
	ret = find_subprog(env, off);
	if (ret >= 0)
		return 0;
1044
	if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
1045 1046 1047
		verbose(env, "too many subprograms\n");
		return -E2BIG;
	}
1048 1049 1050
	env->subprog_info[env->subprog_cnt++].start = off;
	sort(env->subprog_info, env->subprog_cnt,
	     sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
1051 1052 1053 1054 1055 1056
	return 0;
}

static int check_subprogs(struct bpf_verifier_env *env)
{
	int i, ret, subprog_start, subprog_end, off, cur_subprog = 0;
1057
	struct bpf_subprog_info *subprog = env->subprog_info;
1058 1059 1060
	struct bpf_insn *insn = env->prog->insnsi;
	int insn_cnt = env->prog->len;

1061 1062 1063 1064 1065
	/* Add entry function. */
	ret = add_subprog(env, 0);
	if (ret < 0)
		return ret;

1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080
	/* determine subprog starts. The end is one before the next starts */
	for (i = 0; i < insn_cnt; i++) {
		if (insn[i].code != (BPF_JMP | BPF_CALL))
			continue;
		if (insn[i].src_reg != BPF_PSEUDO_CALL)
			continue;
		if (!env->allow_ptr_leaks) {
			verbose(env, "function calls to other bpf functions are allowed for root only\n");
			return -EPERM;
		}
		ret = add_subprog(env, i + insn[i].imm + 1);
		if (ret < 0)
			return ret;
	}

1081 1082 1083 1084 1085
	/* Add a fake 'exit' subprog which could simplify subprog iteration
	 * logic. 'subprog_cnt' should not be increased.
	 */
	subprog[env->subprog_cnt].start = insn_cnt;

1086 1087
	if (env->log.level > 1)
		for (i = 0; i < env->subprog_cnt; i++)
1088
			verbose(env, "func#%d @%d\n", i, subprog[i].start);
1089 1090

	/* now check that all jumps are within the same subprog */
1091 1092
	subprog_start = subprog[cur_subprog].start;
	subprog_end = subprog[cur_subprog + 1].start;
1093 1094 1095
	for (i = 0; i < insn_cnt; i++) {
		u8 code = insn[i].code;

Jiong Wang's avatar
Jiong Wang committed
1096
		if (BPF_CLASS(code) != BPF_JMP && BPF_CLASS(code) != BPF_JMP32)
1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
			goto next;
		if (BPF_OP(code) == BPF_EXIT || BPF_OP(code) == BPF_CALL)
			goto next;
		off = i + insn[i].off + 1;
		if (off < subprog_start || off >= subprog_end) {
			verbose(env, "jump out of range from insn %d to %d\n", i, off);
			return -EINVAL;
		}
next:
		if (i == subprog_end - 1) {
			/* to avoid fall-through from one subprog into another
			 * the last insn of the subprog should be either exit
			 * or unconditional jump back
			 */
			if (code != (BPF_JMP | BPF_EXIT) &&
			    code != (BPF_JMP | BPF_JA)) {
				verbose(env, "last insn is not an exit or jmp\n");
				return -EINVAL;
			}
			subprog_start = subprog_end;
1117 1118
			cur_subprog++;
			if (cur_subprog < env->subprog_cnt)
1119
				subprog_end = subprog[cur_subprog + 1].start;
1120 1121 1122 1123 1124
		}
	}
	return 0;
}

1125 1126 1127
/* Parentage chain of this register (or stack slot) should take care of all
 * issues like callee-saved registers, stack slot allocation time, etc.
 */
1128
static int mark_reg_read(struct bpf_verifier_env *env,
1129 1130
			 const struct bpf_reg_state *state,
			 struct bpf_reg_state *parent)
1131 1132
{
	bool writes = parent == state->parent; /* Observe write marks */
1133 1134 1135

	while (parent) {
		/* if read wasn't screened by an earlier write ... */
1136
		if (writes && state->live & REG_LIVE_WRITTEN)
1137
			break;
1138 1139 1140 1141 1142 1143
		if (parent->live & REG_LIVE_DONE) {
			verbose(env, "verifier BUG type %s var_off %lld off %d\n",
				reg_type_str[parent->type],
				parent->var_off.value, parent->off);
			return -EFAULT;
		}
1144
		/* ... then we depend on parent's value */
1145
		parent->live |= REG_LIVE_READ;
1146 1147
		state = parent;
		parent = state->parent;
1148
		writes = true;
1149
	}
1150
	return 0;
1151 1152 1153
}

static int check_reg_arg(struct bpf_verifier_env *env, u32 regno,
1154 1155
			 enum reg_arg_type t)
{
1156 1157 1158
	struct bpf_verifier_state *vstate = env->cur_state;
	struct bpf_func_state *state = vstate->frame[vstate->curframe];
	struct bpf_reg_state *regs = state->regs;
1159

1160
	if (regno >= MAX_BPF_REG) {
1161
		verbose(env, "R%d is invalid\n", regno);
1162 1163 1164 1165 1166 1167
		return -EINVAL;
	}

	if (t == SRC_OP) {
		/* check whether register used as source operand can be read */
		if (regs[regno].type == NOT_INIT) {
1168
			verbose(env, "R%d !read_ok\n", regno);
1169 1170
			return -EACCES;
		}
1171 1172 1173 1174
		/* We don't need to worry about FP liveness because it's read-only */
		if (regno != BPF_REG_FP)
			return mark_reg_read(env, &regs[regno],
					     regs[regno].parent);
1175 1176 1177
	} else {
		/* check whether register used as dest operand can be written to */
		if (regno == BPF_REG_FP) {
1178
			verbose(env, "frame pointer is read only\n");
1179 1180
			return -EACCES;
		}
1181
		regs[regno].live |= REG_LIVE_WRITTEN;
1182
		if (t == DST_OP)
1183
			mark_reg_unknown(env, regs, regno);
1184 1185 1186 1187
	}
	return 0;
}

1188 1189 1190 1191 1192 1193 1194
static bool is_spillable_regtype(enum bpf_reg_type type)
{
	switch (type) {
	case PTR_TO_MAP_VALUE:
	case PTR_TO_MAP_VALUE_OR_NULL:
	case PTR_TO_STACK:
	case PTR_TO_CTX:
1195
	case PTR_TO_PACKET:
1196
	case PTR_TO_PACKET_META:
1197
	case PTR_TO_PACKET_END:
1198
	case PTR_TO_FLOW_KEYS:
1199
	case CONST_PTR_TO_MAP:
1200 1201
	case PTR_TO_SOCKET:
	case PTR_TO_SOCKET_OR_NULL:
1202 1203
	case PTR_TO_SOCK_COMMON:
	case PTR_TO_SOCK_COMMON_OR_NULL:
1204 1205
	case PTR_TO_TCP_SOCK:
	case PTR_TO_TCP_SOCK_OR_NULL:
1206 1207 1208 1209 1210 1211
		return true;
	default:
		return false;
	}
}

1212 1213 1214 1215 1216 1217
/* Does this register contain a constant zero? */
static bool register_is_null(struct bpf_reg_state *reg)
{
	return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0);
}

1218 1219 1220
/* check_stack_read/write functions track spill/fill of registers,
 * stack boundary and alignment are checked in check_mem_access()
 */
1221
static int check_stack_write(struct bpf_verifier_env *env,
1222
			     struct bpf_func_state *state, /* func where register points to */
1223
			     int off, int size, int value_regno, int insn_idx)
1224
{
1225
	struct bpf_func_state *cur; /* state of the current function */
1226
	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE, err;
1227
	enum bpf_reg_type type;
1228

1229
	err = realloc_func_state(state, round_up(slot + 1, BPF_REG_SIZE),
1230
				 state->acquired_refs, true);
1231 1232
	if (err)
		return err;
1233 1234 1235
	/* caller checked that off % size == 0 and -MAX_BPF_STACK <= off < 0,
	 * so it's aligned access and [off, off + size) are within stack limits
	 */
1236 1237 1238 1239 1240 1241
	if (!env->allow_ptr_leaks &&
	    state->stack[spi].slot_type[0] == STACK_SPILL &&
	    size != BPF_REG_SIZE) {
		verbose(env, "attempt to corrupt spilled pointer on stack\n");
		return -EACCES;
	}
1242

1243
	cur = env->cur_state->frame[env->cur_state->curframe];
1244
	if (value_regno >= 0 &&
1245
	    is_spillable_regtype((type = cur->regs[value_regno].type))) {
1246 1247

		/* register containing pointer is being spilled into stack */
1248
		if (size != BPF_REG_SIZE) {
1249
			verbose(env, "invalid size of register spill\n");
1250 1251 1252
			return -EACCES;
		}

1253 1254 1255 1256 1257
		if (state != cur && type == PTR_TO_STACK) {
			verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
			return -EINVAL;
		}

1258
		/* save register state */
1259
		state->stack[spi].spilled_ptr = cur->regs[value_regno];
1260
		state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;
1261

1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286
		for (i = 0; i < BPF_REG_SIZE; i++) {
			if (state->stack[spi].slot_type[i] == STACK_MISC &&
			    !env->allow_ptr_leaks) {
				int *poff = &env->insn_aux_data[insn_idx].sanitize_stack_off;
				int soff = (-spi - 1) * BPF_REG_SIZE;

				/* detected reuse of integer stack slot with a pointer
				 * which means either llvm is reusing stack slot or
				 * an attacker is trying to exploit CVE-2018-3639
				 * (speculative store bypass)
				 * Have to sanitize that slot with preemptive
				 * store of zero.
				 */
				if (*poff && *poff != soff) {
					/* disallow programs where single insn stores
					 * into two different stack slots, since verifier
					 * cannot sanitize them
					 */
					verbose(env,
						"insn %d cannot access two stack slots fp%d and fp%d",
						insn_idx, *poff, soff);
					return -EINVAL;
				}
				*poff = soff;
			}
1287
			state->stack[spi].slot_type[i] = STACK_SPILL;
1288
		}
1289
	} else {
1290 1291
		u8 type = STACK_MISC;

1292 1293
		/* regular write of data into stack destroys any spilled ptr */
		state->stack[spi].spilled_ptr.type = NOT_INIT;
1294 1295 1296 1297
		/* Mark slots as STACK_MISC if they belonged to spilled ptr. */
		if (state->stack[spi].slot_type[0] == STACK_SPILL)
			for (i = 0; i < BPF_REG_SIZE; i++)
				state->stack[spi].slot_type[i] = STACK_MISC;
1298

1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
		/* only mark the slot as written if all 8 bytes were written
		 * otherwise read propagation may incorrectly stop too soon
		 * when stack slots are partially written.
		 * This heuristic means that read propagation will be
		 * conservative, since it will add reg_live_read marks
		 * to stack slots all the way to first state when programs
		 * writes+reads less than 8 bytes
		 */
		if (size == BPF_REG_SIZE)
			state->stack[spi].spilled_ptr.live |= REG_LIVE_WRITTEN;

		/* when we zero initialize stack slots mark them as such */
		if (value_regno >= 0 &&
		    register_is_null(&cur->regs[value_regno]))
			type = STACK_ZERO;

1315
		/* Mark slots affected by this stack write. */
1316
		for (i = 0; i < size; i++)
1317
			state->stack[spi].slot_type[(slot - i) % BPF_REG_SIZE] =
1318
				type;
1319 1320 1321 1322
	}
	return 0;
}

1323
static int check_stack_read(struct bpf_verifier_env *env,
1324 1325
			    struct bpf_func_state *reg_state /* func where register points to */,
			    int off, int size, int value_regno)
1326
{
1327 1328
	struct bpf_verifier_state *vstate = env->cur_state;
	struct bpf_func_state *state = vstate->frame[vstate->curframe];
1329 1330
	int i, slot = -off - 1, spi = slot / BPF_REG_SIZE;
	u8 *stype;
1331

1332
	if (reg_state->allocated_stack <= slot) {
1333 1334 1335 1336
		verbose(env, "invalid read from stack off %d+0 size %d\n",
			off, size);
		return -EACCES;
	}
1337
	stype = reg_state->stack[spi].slot_type;
1338

1339
	if (stype[0] == STACK_SPILL) {
1340
		if (size != BPF_REG_SIZE) {
1341
			verbose(env, "invalid size of register spill\n");
1342 1343
			return -EACCES;
		}
1344
		for (i = 1; i < BPF_REG_SIZE; i++) {
1345
			if (stype[(slot - i) % BPF_REG_SIZE] != STACK_SPILL) {
1346
				verbose(env, "corrupted spill memory\n");
1347 1348 1349 1350
				return -EACCES;
			}
		}

1351
		if (value_regno >= 0) {
1352
			/* restore register state from stack */
1353
			state->regs[value_regno] = reg_state->stack[spi].spilled_ptr;
1354 1355 1356 1357 1358
			/* mark reg as written since spilled pointer state likely
			 * has its liveness marks cleared by is_state_visited()
			 * which resets stack/reg liveness for state transitions
			 */
			state->regs[value_regno].live |= REG_LIVE_WRITTEN;
1359
		}
1360 1361
		mark_reg_read(env, &reg_state->stack[spi].spilled_ptr,
			      reg_state->stack[spi].spilled_ptr.parent);
1362 1363
		return 0;
	} else {
1364 1365
		int zeros = 0;

1366
		for (i = 0; i < size; i++) {
1367 1368 1369 1370 1371
			if (stype[(slot - i) % BPF_REG_SIZE] == STACK_MISC)
				continue;
			if (stype[(slot - i) % BPF_REG_SIZE] == STACK_ZERO) {
				zeros++;
				continue;
1372
			}
1373 1374 1375 1376
			verbose(env, "invalid read from stack off %d+%d size %d\n",
				off, i, size);
			return -EACCES;
		}
1377 1378
		mark_reg_read(env, &reg_state->stack[spi].spilled_ptr,
			      reg_state->stack[spi].spilled_ptr.parent);
1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
		if (value_regno >= 0) {
			if (zeros == size) {
				/* any size read into register is zero extended,
				 * so the whole register == const_zero
				 */
				__mark_reg_const_zero(&state->regs[value_regno]);
			} else {
				/* have read misc data from the stack */
				mark_reg_unknown(env, state->regs, value_regno);
			}
			state->regs[value_regno].live |= REG_LIVE_WRITTEN;
1390 1391 1392 1393 1394
		}
		return 0;
	}
}

1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419
static int check_stack_access(struct bpf_verifier_env *env,
			      const struct bpf_reg_state *reg,
			      int off, int size)
{
	/* Stack accesses must be at a fixed offset, so that we
	 * can determine what type of data were returned. See
	 * check_stack_read().
	 */
	if (!tnum_is_const(reg->var_off)) {
		char tn_buf[48];

		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
		verbose(env, "variable stack access var_off=%s off=%d size=%d",
			tn_buf, off, size);
		return -EACCES;
	}

	if (off >= 0 || off < -MAX_BPF_STACK) {
		verbose(env, "invalid stack off=%d size=%d\n", off, size);
		return -EACCES;
	}

	return 0;
}

1420
/* check read/write into map element returned by bpf_map_lookup_elem() */
1421
static int __check_map_access(struct bpf_verifier_env *env, u32 regno, int off,
1422
			      int size, bool zero_size_allowed)
1423
{
1424 1425
	struct bpf_reg_state *regs = cur_regs(env);
	struct bpf_map *map = regs[regno].map_ptr;
1426

1427 1428
	if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) ||
	    off + size > map->value_size) {
1429
		verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n",
1430 1431 1432 1433 1434 1435
			map->value_size, off, size);
		return -EACCES;
	}
	return 0;
}

1436 1437
/* check read/write into a map element with possible variable offset */
static int check_map_access(struct bpf_verifier_env *env, u32 regno,
1438
			    int off, int size, bool zero_size_allowed)
1439
{
1440 1441
	struct bpf_verifier_state *vstate = env->cur_state;
	struct bpf_func_state *state = vstate->frame[vstate->curframe];
1442 1443 1444
	struct bpf_reg_state *reg = &state->regs[regno];
	int err;

1445 1446 1447
	/* We may have adjusted the register to this map value, so we
	 * need to try adding each of min_value and max_value to off
	 * to make sure our theoretical access will be safe.
1448
	 */
1449 1450
	if (env->log.level)
		print_verifier_state(env, state);
1451

1452 1453 1454 1455 1456 1457
	/* The minimum value is only important with signed
	 * comparisons where we can't assume the floor of a
	 * value is 0.  If we are using signed variables for our
	 * index'es we need to make sure that whatever we use
	 * will have a set floor within our range.
	 */
1458 1459 1460 1461
	if (reg->smin_value < 0 &&
	    (reg->smin_value == S64_MIN ||
	     (off + reg->smin_value != (s64)(s32)(off + reg->smin_value)) ||
	      reg->smin_value + off < 0)) {
1462
		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
1463 1464 1465
			regno);
		return -EACCES;
	}
1466 1467
	err = __check_map_access(env, regno, reg->smin_value + off, size,
				 zero_size_allowed);
1468
	if (err) {
1469 1470
		verbose(env, "R%d min value is outside of the array range\n",
			regno);
1471 1472 1473
		return err;
	}

1474 1475 1476
	/* If we haven't set a max value then we need to bail since we can't be
	 * sure we won't do bad things.
	 * If reg->umax_value + off could overflow, treat that as unbounded too.
1477
	 */
1478
	if (reg->umax_value >= BPF_MAX_VAR_OFF) {
1479
		verbose(env, "R%d unbounded memory access, make sure to bounds check any array access into a map\n",
1480 1481 1482
			regno);
		return -EACCES;
	}
1483 1484
	err = __check_map_access(env, regno, reg->umax_value + off, size,
				 zero_size_allowed);
1485
	if (err)
1486 1487
		verbose(env, "R%d max value is outside of the array range\n",
			regno);
1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502

	if (map_value_has_spin_lock(reg->map_ptr)) {
		u32 lock = reg->map_ptr->spin_lock_off;

		/* if any part of struct bpf_spin_lock can be touched by
		 * load/store reject this program.
		 * To check that [x1, x2) overlaps with [y1, y2)
		 * it is sufficient to check x1 < y2 && y1 < x2.
		 */
		if (reg->smin_value + off < lock + sizeof(struct bpf_spin_lock) &&
		     lock < reg->umax_value + off + size) {
			verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n");
			return -EACCES;
		}
	}
1503
	return err;
1504 1505
}

1506 1507
#define MAX_PACKET_OFF 0xffff

1508
static bool may_access_direct_pkt_data(struct bpf_verifier_env *env,
1509 1510
				       const struct bpf_call_arg_meta *meta,
				       enum bpf_access_type t)
1511
{
1512
	switch (env->prog->type) {
1513
	/* Program types only with direct read access go here! */
1514 1515
	case BPF_PROG_TYPE_LWT_IN:
	case BPF_PROG_TYPE_LWT_OUT:
1516
	case BPF_PROG_TYPE_LWT_SEG6LOCAL:
1517
	case BPF_PROG_TYPE_SK_REUSEPORT:
1518
	case BPF_PROG_TYPE_FLOW_DISSECTOR:
1519
	case BPF_PROG_TYPE_CGROUP_SKB:
1520 1521
		if (t == BPF_WRITE)
			return false;
1522
		/* fallthrough */
1523 1524

	/* Program types with direct read + write access go here! */
1525 1526
	case BPF_PROG_TYPE_SCHED_CLS:
	case BPF_PROG_TYPE_SCHED_ACT:
1527
	case BPF_PROG_TYPE_XDP:
1528
	case BPF_PROG_TYPE_LWT_XMIT:
1529
	case BPF_PROG_TYPE_SK_SKB:
1530
	case BPF_PROG_TYPE_SK_MSG:
1531 1532 1533 1534
		if (meta)
			return meta->pkt_access;

		env->seen_direct_write = true;
1535 1536 1537 1538 1539 1540
		return true;
	default:
		return false;
	}
}

1541
static int __check_packet_access(struct bpf_verifier_env *env, u32 regno,
1542
				 int off, int size, bool zero_size_allowed)
1543
{
1544
	struct bpf_reg_state *regs = cur_regs(env);
1545
	struct bpf_reg_state *reg = &regs[regno];
1546

1547 1548
	if (off < 0 || size < 0 || (size == 0 && !zero_size_allowed) ||
	    (u64)off + size > reg->range) {
1549
		verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n",
1550
			off, size, regno, reg->id, reg->off, reg->range);
1551 1552 1553 1554 1555
		return -EACCES;
	}
	return 0;
}

1556
static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off,
1557
			       int size, bool zero_size_allowed)
1558
{
1559
	struct bpf_reg_state *regs = cur_regs(env);
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570
	struct bpf_reg_state *reg = &regs[regno];
	int err;

	/* We may have added a variable offset to the packet pointer; but any
	 * reg->range we have comes after that.  We are only checking the fixed
	 * offset.
	 */

	/* We don't allow negative numbers, because we aren't tracking enough
	 * detail to prove they're safe.
	 */
1571
	if (reg->smin_value < 0) {
1572
		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
1573 1574 1575
			regno);
		return -EACCES;
	}
1576
	err = __check_packet_access(env, regno, off, size, zero_size_allowed);
1577
	if (err) {
1578
		verbose(env, "R%d offset is outside of the packet\n", regno);
1579 1580
		return err;
	}
1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591

	/* __check_packet_access has made sure "off + size - 1" is within u16.
	 * reg->umax_value can't be bigger than MAX_PACKET_OFF which is 0xffff,
	 * otherwise find_good_pkt_pointers would have refused to set range info
	 * that __check_packet_access would have rejected this pkt access.
	 * Therefore, "off + reg->umax_value + size - 1" won't overflow u32.
	 */
	env->prog->aux->max_pkt_offset =
		max_t(u32, env->prog->aux->max_pkt_offset,
		      off + reg->umax_value + size - 1);

1592 1593 1594 1595
	return err;
}

/* check access to 'struct bpf_context' fields.  Supports fixed offsets only */
1596
static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size,
1597
			    enum bpf_access_type t, enum bpf_reg_type *reg_type)
1598
{
1599 1600 1601
	struct bpf_insn_access_aux info = {
		.reg_type = *reg_type,
	};
1602

1603
	if (env->ops->is_valid_access &&
1604
	    env->ops->is_valid_access(off, size, t, env->prog, &info)) {
1605 1606 1607 1608 1609 1610
		/* A non zero info.ctx_field_size indicates that this field is a
		 * candidate for later verifier transformation to load the whole
		 * field and then apply a mask when accessed with a narrower
		 * access than actual ctx access size. A zero info.ctx_field_size
		 * will only allow for whole field access and rejects any other
		 * type of narrower access.
1611
		 */
1612
		*reg_type = info.reg_type;
1613

1614
		env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
1615 1616 1617
		/* remember the offset of last byte accessed in ctx */
		if (env->prog->aux->max_ctx_offset < off + size)
			env->prog->aux->max_ctx_offset = off + size;
1618
		return 0;
1619
	}
1620

1621
	verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
1622 1623 1624
	return -EACCES;
}

1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636
static int check_flow_keys_access(struct bpf_verifier_env *env, int off,
				  int size)
{
	if (size < 0 || off < 0 ||
	    (u64)off + size > sizeof(struct bpf_flow_keys)) {
		verbose(env, "invalid access to flow keys off=%d size=%d\n",
			off, size);
		return -EACCES;
	}
	return 0;
}

1637 1638 1639
static int check_sock_access(struct bpf_verifier_env *env, int insn_idx,
			     u32 regno, int off, int size,
			     enum bpf_access_type t)
1640 1641 1642
{
	struct bpf_reg_state *regs = cur_regs(env);
	struct bpf_reg_state *reg = &regs[regno];
1643
	struct bpf_insn_access_aux info = {};
1644
	bool valid;
1645 1646 1647 1648 1649 1650 1651

	if (reg->smin_value < 0) {
		verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n",
			regno);
		return -EACCES;
	}

1652 1653 1654 1655 1656 1657 1658
	switch (reg->type) {
	case PTR_TO_SOCK_COMMON:
		valid = bpf_sock_common_is_valid_access(off, size, t, &info);
		break;
	case PTR_TO_SOCKET:
		valid = bpf_sock_is_valid_access(off, size, t, &info);
		break;
1659 1660 1661
	case PTR_TO_TCP_SOCK:
		valid = bpf_tcp_sock_is_valid_access(off, size, t, &info);
		break;
1662 1663
	default:
		valid = false;
1664 1665
	}

1666

1667 1668 1669 1670 1671 1672 1673 1674 1675 1676
	if (valid) {
		env->insn_aux_data[insn_idx].ctx_field_size =
			info.ctx_field_size;
		return 0;
	}

	verbose(env, "R%d invalid %s access off=%d size=%d\n",
		regno, reg_type_str[reg->type], off, size);

	return -EACCES;
1677 1678
}

1679 1680
static bool __is_pointer_value(bool allow_ptr_leaks,
			       const struct bpf_reg_state *reg)
1681
{
1682
	if (allow_ptr_leaks)
1683 1684
		return false;

1685
	return reg->type != SCALAR_VALUE;
1686 1687
}

1688 1689 1690 1691 1692
static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
{
	return cur_regs(env) + regno;
}

1693 1694
static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
{
1695
	return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
1696 1697
}

1698 1699
static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
{
1700
	const struct bpf_reg_state *reg = reg_state(env, regno);
1701

1702 1703 1704 1705 1706 1707 1708 1709
	return reg->type == PTR_TO_CTX;
}

static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
{
	const struct bpf_reg_state *reg = reg_state(env, regno);

	return type_is_sk_pointer(reg->type);
1710 1711
}

1712 1713
static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
{
1714
	const struct bpf_reg_state *reg = reg_state(env, regno);
1715 1716 1717 1718

	return type_is_pkt_pointer(reg->type);
}

1719 1720 1721 1722 1723 1724 1725 1726
static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
{
	const struct bpf_reg_state *reg = reg_state(env, regno);

	/* Separate to is_ctx_reg() since we still want to allow BPF_ST here. */
	return reg->type == PTR_TO_FLOW_KEYS;
}

1727 1728
static int check_pkt_ptr_alignment(struct bpf_verifier_env *env,
				   const struct bpf_reg_state *reg,
1729
				   int off, int size, bool strict)
1730
{
1731
	struct tnum reg_off;
1732
	int ip_align;
1733 1734 1735 1736 1737

	/* Byte size accesses are always allowed. */
	if (!strict || size == 1)
		return 0;

1738 1739 1740 1741 1742 1743 1744
	/* For platforms that do not have a Kconfig enabling
	 * CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS the value of
	 * NET_IP_ALIGN is universally set to '2'.  And on platforms
	 * that do set CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS, we get
	 * to this code only in strict mode where we want to emulate
	 * the NET_IP_ALIGN==2 checking.  Therefore use an
	 * unconditional IP align value of '2'.
1745
	 */
1746
	ip_align = 2;
1747 1748 1749 1750 1751 1752

	reg_off = tnum_add(reg->var_off, tnum_const(ip_align + reg->off + off));
	if (!tnum_is_aligned(reg_off, size)) {
		char tn_buf[48];

		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
1753 1754
		verbose(env,
			"misaligned packet access off %d+%s+%d+%d size %d\n",
1755
			ip_align, tn_buf, reg->off, off, size);
1756 1757
		return -EACCES;
	}
1758

1759 1760 1761
	return 0;
}

1762 1763
static int check_generic_ptr_alignment(struct bpf_verifier_env *env,
				       const struct bpf_reg_state *reg,
1764 1765
				       const char *pointer_desc,
				       int off, int size, bool strict)
1766
{
1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
	struct tnum reg_off;

	/* Byte size accesses are always allowed. */
	if (!strict || size == 1)
		return 0;

	reg_off = tnum_add(reg->var_off, tnum_const(reg->off + off));
	if (!tnum_is_aligned(reg_off, size)) {
		char tn_buf[48];

		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
1778
		verbose(env, "misaligned %saccess off %s+%d+%d size %d\n",
1779
			pointer_desc, tn_buf, reg->off, off, size);
1780 1781 1782
		return -EACCES;
	}

1783 1784 1785
	return 0;
}

1786
static int check_ptr_alignment(struct bpf_verifier_env *env,
1787 1788
			       const struct bpf_reg_state *reg, int off,
			       int size, bool strict_alignment_once)
1789
{
1790
	bool strict = env->strict_alignment || strict_alignment_once;
1791
	const char *pointer_desc = "";
1792

1793 1794
	switch (reg->type) {
	case PTR_TO_PACKET:
1795 1796 1797 1798
	case PTR_TO_PACKET_META:
		/* Special case, because of NET_IP_ALIGN. Given metadata sits
		 * right in front, treat it the very same way.
		 */
1799
		return check_pkt_ptr_alignment(env, reg, off, size, strict);
1800 1801 1802
	case PTR_TO_FLOW_KEYS:
		pointer_desc = "flow keys ";
		break;
1803 1804 1805 1806 1807 1808 1809 1810
	case PTR_TO_MAP_VALUE:
		pointer_desc = "value ";
		break;
	case PTR_TO_CTX:
		pointer_desc = "context ";
		break;
	case PTR_TO_STACK:
		pointer_desc = "stack ";
1811 1812 1813 1814 1815
		/* The stack spill tracking logic in check_stack_write()
		 * and check_stack_read() relies on stack accesses being
		 * aligned.
		 */
		strict = true;
1816
		break;
1817 1818 1819
	case PTR_TO_SOCKET:
		pointer_desc = "sock ";
		break;
1820 1821 1822
	case PTR_TO_SOCK_COMMON:
		pointer_desc = "sock_common ";
		break;
1823 1824 1825
	case PTR_TO_TCP_SOCK:
		pointer_desc = "tcp_sock ";
		break;
1826
	default:
1827
		break;
1828
	}
1829 1830
	return check_generic_ptr_alignment(env, reg, pointer_desc, off, size,
					   strict);
1831 1832
}

1833 1834 1835 1836
static int update_stack_depth(struct bpf_verifier_env *env,
			      const struct bpf_func_state *func,
			      int off)
{
1837
	u16 stack = env->subprog_info[func->subprogno].stack_depth;
1838 1839 1840 1841 1842

	if (stack >= -off)
		return 0;

	/* update known max for given subprogram */
1843
	env->subprog_info[func->subprogno].stack_depth = -off;
1844 1845
	return 0;
}
1846

1847 1848 1849 1850 1851 1852 1853 1854
/* starting from main bpf function walk all instructions of the function
 * and recursively walk all callees that given function can call.
 * Ignore jump and exit insns.
 * Since recursion is prevented by check_cfg() this algorithm
 * only needs a local stack of MAX_CALL_FRAMES to remember callsites
 */
static int check_max_stack_depth(struct bpf_verifier_env *env)
{
1855 1856
	int depth = 0, frame = 0, idx = 0, i = 0, subprog_end;
	struct bpf_subprog_info *subprog = env->subprog_info;
1857 1858 1859
	struct bpf_insn *insn = env->prog->insnsi;
	int ret_insn[MAX_CALL_FRAMES];
	int ret_prog[MAX_CALL_FRAMES];
1860

1861 1862 1863 1864
process_func:
	/* round up to 32-bytes, since this is granularity
	 * of interpreter stack size
	 */
1865
	depth += round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
1866
	if (depth > MAX_BPF_STACK) {
1867
		verbose(env, "combined stack size of %d calls is %d. Too large\n",
1868
			frame + 1, depth);
1869 1870
		return -EACCES;
	}
1871
continue_func:
1872
	subprog_end = subprog[idx + 1].start;
1873 1874 1875 1876 1877 1878 1879
	for (; i < subprog_end; i++) {
		if (insn[i].code != (BPF_JMP | BPF_CALL))
			continue;
		if (insn[i].src_reg != BPF_PSEUDO_CALL)
			continue;
		/* remember insn and function to return to */
		ret_insn[frame] = i + 1;
1880
		ret_prog[frame] = idx;
1881 1882 1883

		/* find the callee */
		i = i + insn[i].imm + 1;
1884 1885
		idx = find_subprog(env, i);
		if (idx < 0) {
1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901
			WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
				  i);
			return -EFAULT;
		}
		frame++;
		if (frame >= MAX_CALL_FRAMES) {
			WARN_ONCE(1, "verifier bug. Call stack is too deep\n");
			return -EFAULT;
		}
		goto process_func;
	}
	/* end of for() loop means the last insn of the 'subprog'
	 * was reached. Doesn't matter whether it was JA or EXIT
	 */
	if (frame == 0)
		return 0;
1902
	depth -= round_up(max_t(u32, subprog[idx].stack_depth, 1), 32);
1903 1904
	frame--;
	i = ret_insn[frame];
1905
	idx = ret_prog[frame];
1906
	goto continue_func;
1907 1908
}

1909
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920
static int get_callee_stack_depth(struct bpf_verifier_env *env,
				  const struct bpf_insn *insn, int idx)
{
	int start = idx + insn->imm + 1, subprog;

	subprog = find_subprog(env, start);
	if (subprog < 0) {
		WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
			  start);
		return -EFAULT;
	}
1921
	return env->subprog_info[subprog].stack_depth;
1922
}
1923
#endif
1924

1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948
static int check_ctx_reg(struct bpf_verifier_env *env,
			 const struct bpf_reg_state *reg, int regno)
{
	/* Access to ctx or passing it to a helper is only allowed in
	 * its original, unmodified form.
	 */

	if (reg->off) {
		verbose(env, "dereference of modified ctx ptr R%d off=%d disallowed\n",
			regno, reg->off);
		return -EACCES;
	}

	if (!tnum_is_const(reg->var_off) || reg->var_off.value) {
		char tn_buf[48];

		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
		verbose(env, "variable ctx access var_off=%s disallowed\n", tn_buf);
		return -EACCES;
	}

	return 0;
}

1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
/* truncate register to smaller size (in bytes)
 * must be called with size < BPF_REG_SIZE
 */
static void coerce_reg_to_size(struct bpf_reg_state *reg, int size)
{
	u64 mask;

	/* clear high bits in bit representation */
	reg->var_off = tnum_cast(reg->var_off, size);

	/* fix arithmetic bounds */
	mask = ((u64)1 << (size * 8)) - 1;
	if ((reg->umin_value & ~mask) == (reg->umax_value & ~mask)) {
		reg->umin_value &= mask;
		reg->umax_value &= mask;
	} else {
		reg->umin_value = 0;
		reg->umax_value = mask;
	}
	reg->smin_value = reg->umin_value;
	reg->smax_value = reg->umax_value;
}

1972 1973 1974 1975 1976 1977
/* check whether memory at (regno + off) is accessible for t = (read | write)
 * if t==write, value_regno is a register which value is stored into memory
 * if t==read, value_regno is a register which will receive the value from memory
 * if t==write && value_regno==-1, some unknown value is stored into memory
 * if t==read && value_regno==-1, don't care what we read from memory
 */
1978 1979 1980
static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno,
			    int off, int bpf_size, enum bpf_access_type t,
			    int value_regno, bool strict_alignment_once)
1981
{
1982 1983
	struct bpf_reg_state *regs = cur_regs(env);
	struct bpf_reg_state *reg = regs + regno;
1984
	struct bpf_func_state *state;
1985 1986 1987 1988 1989 1990
	int size, err = 0;

	size = bpf_size_to_bytes(bpf_size);
	if (size < 0)
		return size;

1991
	/* alignment checks will add in reg->off themselves */
1992
	err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
1993 1994
	if (err)
		return err;
1995

1996 1997 1998 1999
	/* for access checks, reg->off is just part of off */
	off += reg->off;

	if (reg->type == PTR_TO_MAP_VALUE) {
2000 2001
		if (t == BPF_WRITE && value_regno >= 0 &&
		    is_pointer_value(env, value_regno)) {
2002
			verbose(env, "R%d leaks addr into map\n", value_regno);
2003 2004
			return -EACCES;
		}
2005

2006
		err = check_map_access(env, regno, off, size, false);
2007
		if (!err && t == BPF_READ && value_regno >= 0)
2008
			mark_reg_unknown(env, regs, value_regno);
2009

2010
	} else if (reg->type == PTR_TO_CTX) {
2011
		enum bpf_reg_type reg_type = SCALAR_VALUE;
2012

2013 2014
		if (t == BPF_WRITE && value_regno >= 0 &&
		    is_pointer_value(env, value_regno)) {
2015
			verbose(env, "R%d leaks addr into ctx\n", value_regno);
2016 2017
			return -EACCES;
		}
2018

2019 2020 2021 2022
		err = check_ctx_reg(env, reg, regno);
		if (err < 0)
			return err;

2023
		err = check_ctx_access(env, insn_idx, off, size, t, &reg_type);
2024
		if (!err && t == BPF_READ && value_regno >= 0) {
2025
			/* ctx access returns either a scalar, or a
2026 2027
			 * PTR_TO_PACKET[_META,_END]. In the latter
			 * case, we know the offset is zero.
2028
			 */
2029
			if (reg_type == SCALAR_VALUE) {
2030
				mark_reg_unknown(env, regs, value_regno);
2031
			} else {
2032
				mark_reg_known_zero(env, regs,
2033
						    value_regno);
2034 2035 2036
				if (reg_type_may_be_null(reg_type))
					regs[value_regno].id = ++env->id_gen;
			}
2037
			regs[value_regno].type = reg_type;
2038
		}
2039

2040 2041
	} else if (reg->type == PTR_TO_STACK) {
		off += reg->var_off.value;
2042 2043 2044
		err = check_stack_access(env, reg, off, size);
		if (err)
			return err;
2045

2046 2047 2048 2049
		state = func(env, reg);
		err = update_stack_depth(env, state, off);
		if (err)
			return err;
2050

2051
		if (t == BPF_WRITE)
2052
			err = check_stack_write(env, state, off, size,
2053
						value_regno, insn_idx);
2054
		else
2055 2056
			err = check_stack_read(env, state, off, size,
					       value_regno);
2057
	} else if (reg_is_pkt_pointer(reg)) {
2058
		if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
2059
			verbose(env, "cannot write into packet\n");
2060 2061
			return -EACCES;
		}
2062 2063
		if (t == BPF_WRITE && value_regno >= 0 &&
		    is_pointer_value(env, value_regno)) {
2064 2065
			verbose(env, "R%d leaks addr into packet\n",
				value_regno);
2066 2067
			return -EACCES;
		}
2068
		err = check_packet_access(env, regno, off, size, false);
2069
		if (!err && t == BPF_READ && value_regno >= 0)
2070
			mark_reg_unknown(env, regs, value_regno);
2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081
	} else if (reg->type == PTR_TO_FLOW_KEYS) {
		if (t == BPF_WRITE && value_regno >= 0 &&
		    is_pointer_value(env, value_regno)) {
			verbose(env, "R%d leaks addr into flow keys\n",
				value_regno);
			return -EACCES;
		}

		err = check_flow_keys_access(env, off, size);
		if (!err && t == BPF_READ && value_regno >= 0)
			mark_reg_unknown(env, regs, value_regno);
2082
	} else if (type_is_sk_pointer(reg->type)) {
2083
		if (t == BPF_WRITE) {
2084 2085
			verbose(env, "R%d cannot write into %s\n",
				regno, reg_type_str[reg->type]);
2086 2087
			return -EACCES;
		}
2088
		err = check_sock_access(env, insn_idx, regno, off, size, t);
2089 2090
		if (!err && value_regno >= 0)
			mark_reg_unknown(env, regs, value_regno);
2091
	} else {
2092 2093
		verbose(env, "R%d invalid mem access '%s'\n", regno,
			reg_type_str[reg->type]);
2094 2095
		return -EACCES;
	}
2096

2097
	if (!err && size < BPF_REG_SIZE && value_regno >= 0 && t == BPF_READ &&
2098
	    regs[value_regno].type == SCALAR_VALUE) {
2099
		/* b/h/w load zero-extends, mark upper bits as known 0 */
2100
		coerce_reg_to_size(&regs[value_regno], size);
2101
	}
2102 2103 2104
	return err;
}

2105
static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
2106 2107 2108 2109 2110
{
	int err;

	if ((BPF_SIZE(insn->code) != BPF_W && BPF_SIZE(insn->code) != BPF_DW) ||
	    insn->imm != 0) {
2111
		verbose(env, "BPF_XADD uses reserved fields\n");
2112 2113 2114 2115
		return -EINVAL;
	}

	/* check src1 operand */
2116
	err = check_reg_arg(env, insn->src_reg, SRC_OP);
2117 2118 2119 2120
	if (err)
		return err;

	/* check src2 operand */
2121
	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
2122 2123 2124
	if (err)
		return err;

2125
	if (is_pointer_value(env, insn->src_reg)) {
2126
		verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
2127 2128 2129
		return -EACCES;
	}

2130
	if (is_ctx_reg(env, insn->dst_reg) ||
2131
	    is_pkt_reg(env, insn->dst_reg) ||
2132 2133
	    is_flow_key_reg(env, insn->dst_reg) ||
	    is_sk_reg(env, insn->dst_reg)) {
2134
		verbose(env, "BPF_XADD stores into R%d %s is not allowed\n",
2135 2136
			insn->dst_reg,
			reg_type_str[reg_state(env, insn->dst_reg)->type]);
2137 2138 2139
		return -EACCES;
	}

2140
	/* check whether atomic_add can read the memory */
2141
	err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
2142
			       BPF_SIZE(insn->code), BPF_READ, -1, true);
2143 2144 2145 2146
	if (err)
		return err;

	/* check whether atomic_add can write into the same memory */
2147
	return check_mem_access(env, insn_idx, insn->dst_reg, insn->off,
2148
				BPF_SIZE(insn->code), BPF_WRITE, -1, true);
2149 2150 2151 2152
}

/* when register 'regno' is passed into function that will read 'access_size'
 * bytes from that pointer, make sure that it's within stack boundary
2153 2154 2155
 * and all elements of stack are initialized.
 * Unlike most pointer bounds-checking functions, this one doesn't take an
 * 'off' argument, so it has to add in reg->off itself.
2156
 */
2157
static int check_stack_boundary(struct bpf_verifier_env *env, int regno,
2158 2159
				int access_size, bool zero_size_allowed,
				struct bpf_call_arg_meta *meta)
2160
{
2161
	struct bpf_reg_state *reg = reg_state(env, regno);
2162
	struct bpf_func_state *state = func(env, reg);
2163
	int off, i, slot, spi;
2164

2165
	if (reg->type != PTR_TO_STACK) {
2166
		/* Allow zero-byte read from NULL, regardless of pointer type */
2167
		if (zero_size_allowed && access_size == 0 &&
2168
		    register_is_null(reg))
2169 2170
			return 0;

2171
		verbose(env, "R%d type=%s expected=%s\n", regno,
2172
			reg_type_str[reg->type],
2173
			reg_type_str[PTR_TO_STACK]);
2174
		return -EACCES;
2175
	}
2176

2177
	/* Only allow fixed-offset stack reads */
2178
	if (!tnum_is_const(reg->var_off)) {
2179 2180
		char tn_buf[48];

2181
		tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
2182
		verbose(env, "invalid variable stack read R%d var_off=%s\n",
2183
			regno, tn_buf);
2184
		return -EACCES;
2185
	}
2186
	off = reg->off + reg->var_off.value;
2187
	if (off >= 0 || off < -MAX_BPF_STACK || off + access_size > 0 ||
2188
	    access_size < 0 || (access_size == 0 && !zero_size_allowed)) {
2189
		verbose(env, "invalid stack type R%d off=%d access_size=%d\n",
2190 2191 2192 2193
			regno, off, access_size);
		return -EACCES;
	}

2194 2195 2196 2197 2198 2199
	if (meta && meta->raw_mode) {
		meta->access_size = access_size;
		meta->regno = regno;
		return 0;
	}

2200
	for (i = 0; i < access_size; i++) {
2201 2202
		u8 *stype;

2203 2204
		slot = -(off + i) - 1;
		spi = slot / BPF_REG_SIZE;
2205 2206 2207 2208 2209 2210 2211 2212 2213
		if (state->allocated_stack <= slot)
			goto err;
		stype = &state->stack[spi].slot_type[slot % BPF_REG_SIZE];
		if (*stype == STACK_MISC)
			goto mark;
		if (*stype == STACK_ZERO) {
			/* helper can write anything into the stack */
			*stype = STACK_MISC;
			goto mark;
2214
		}
2215 2216 2217 2218 2219 2220 2221 2222
err:
		verbose(env, "invalid indirect read from stack off %d+%d size %d\n",
			off, i, access_size);
		return -EACCES;
mark:
		/* reading any byte out of 8-byte 'spill_slot' will cause
		 * the whole slot to be marked as 'read'
		 */
2223 2224
		mark_reg_read(env, &state->stack[spi].spilled_ptr,
			      state->stack[spi].spilled_ptr.parent);
2225
	}
2226
	return update_stack_depth(env, state, off);
2227 2228
}

2229 2230 2231 2232
static int check_helper_mem_access(struct bpf_verifier_env *env, int regno,
				   int access_size, bool zero_size_allowed,
				   struct bpf_call_arg_meta *meta)
{
2233
	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
2234

2235
	switch (reg->type) {
2236
	case PTR_TO_PACKET:
2237
	case PTR_TO_PACKET_META:
2238 2239
		return check_packet_access(env, regno, reg->off, access_size,
					   zero_size_allowed);
2240
	case PTR_TO_MAP_VALUE:
2241 2242
		return check_map_access(env, regno, reg->off, access_size,
					zero_size_allowed);
2243
	default: /* scalar_value|ptr_to_stack or invalid ptr */
2244 2245 2246 2247 2248
		return check_stack_boundary(env, regno, access_size,
					    zero_size_allowed, meta);
	}
}

2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333
/* Implementation details:
 * bpf_map_lookup returns PTR_TO_MAP_VALUE_OR_NULL
 * Two bpf_map_lookups (even with the same key) will have different reg->id.
 * For traditional PTR_TO_MAP_VALUE the verifier clears reg->id after
 * value_or_null->value transition, since the verifier only cares about
 * the range of access to valid map value pointer and doesn't care about actual
 * address of the map element.
 * For maps with 'struct bpf_spin_lock' inside map value the verifier keeps
 * reg->id > 0 after value_or_null->value transition. By doing so
 * two bpf_map_lookups will be considered two different pointers that
 * point to different bpf_spin_locks.
 * The verifier allows taking only one bpf_spin_lock at a time to avoid
 * dead-locks.
 * Since only one bpf_spin_lock is allowed the checks are simpler than
 * reg_is_refcounted() logic. The verifier needs to remember only
 * one spin_lock instead of array of acquired_refs.
 * cur_state->active_spin_lock remembers which map value element got locked
 * and clears it after bpf_spin_unlock.
 */
static int process_spin_lock(struct bpf_verifier_env *env, int regno,
			     bool is_lock)
{
	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
	struct bpf_verifier_state *cur = env->cur_state;
	bool is_const = tnum_is_const(reg->var_off);
	struct bpf_map *map = reg->map_ptr;
	u64 val = reg->var_off.value;

	if (reg->type != PTR_TO_MAP_VALUE) {
		verbose(env, "R%d is not a pointer to map_value\n", regno);
		return -EINVAL;
	}
	if (!is_const) {
		verbose(env,
			"R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n",
			regno);
		return -EINVAL;
	}
	if (!map->btf) {
		verbose(env,
			"map '%s' has to have BTF in order to use bpf_spin_lock\n",
			map->name);
		return -EINVAL;
	}
	if (!map_value_has_spin_lock(map)) {
		if (map->spin_lock_off == -E2BIG)
			verbose(env,
				"map '%s' has more than one 'struct bpf_spin_lock'\n",
				map->name);
		else if (map->spin_lock_off == -ENOENT)
			verbose(env,
				"map '%s' doesn't have 'struct bpf_spin_lock'\n",
				map->name);
		else
			verbose(env,
				"map '%s' is not a struct type or bpf_spin_lock is mangled\n",
				map->name);
		return -EINVAL;
	}
	if (map->spin_lock_off != val + reg->off) {
		verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n",
			val + reg->off);
		return -EINVAL;
	}
	if (is_lock) {
		if (cur->active_spin_lock) {
			verbose(env,
				"Locking two bpf_spin_locks are not allowed\n");
			return -EINVAL;
		}
		cur->active_spin_lock = reg->id;
	} else {
		if (!cur->active_spin_lock) {
			verbose(env, "bpf_spin_unlock without taking a lock\n");
			return -EINVAL;
		}
		if (cur->active_spin_lock != reg->id) {
			verbose(env, "bpf_spin_unlock of different lock\n");
			return -EINVAL;
		}
		cur->active_spin_lock = 0;
	}
	return 0;
}

2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346
static bool arg_type_is_mem_ptr(enum bpf_arg_type type)
{
	return type == ARG_PTR_TO_MEM ||
	       type == ARG_PTR_TO_MEM_OR_NULL ||
	       type == ARG_PTR_TO_UNINIT_MEM;
}

static bool arg_type_is_mem_size(enum bpf_arg_type type)
{
	return type == ARG_CONST_SIZE ||
	       type == ARG_CONST_SIZE_OR_ZERO;
}

2347
static int check_func_arg(struct bpf_verifier_env *env, u32 regno,
2348 2349
			  enum bpf_arg_type arg_type,
			  struct bpf_call_arg_meta *meta)
2350
{
2351
	struct bpf_reg_state *regs = cur_regs(env), *reg = &regs[regno];
2352
	enum bpf_reg_type expected_type, type = reg->type;
2353 2354
	int err = 0;

2355
	if (arg_type == ARG_DONTCARE)
2356 2357
		return 0;

2358 2359 2360
	err = check_reg_arg(env, regno, SRC_OP);
	if (err)
		return err;
2361

2362 2363
	if (arg_type == ARG_ANYTHING) {
		if (is_pointer_value(env, regno)) {
2364 2365
			verbose(env, "R%d leaks addr into helper function\n",
				regno);
2366 2367
			return -EACCES;
		}
2368
		return 0;
2369
	}
2370

2371
	if (type_is_pkt_pointer(type) &&
2372
	    !may_access_direct_pkt_data(env, meta, BPF_READ)) {
2373
		verbose(env, "helper access to the packet is not allowed\n");
2374 2375 2376
		return -EACCES;
	}

2377
	if (arg_type == ARG_PTR_TO_MAP_KEY ||
2378 2379
	    arg_type == ARG_PTR_TO_MAP_VALUE ||
	    arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
2380
		expected_type = PTR_TO_STACK;
2381
		if (!type_is_pkt_pointer(type) && type != PTR_TO_MAP_VALUE &&
2382
		    type != expected_type)
2383
			goto err_type;
2384 2385
	} else if (arg_type == ARG_CONST_SIZE ||
		   arg_type == ARG_CONST_SIZE_OR_ZERO) {
2386 2387
		expected_type = SCALAR_VALUE;
		if (type != expected_type)
2388
			goto err_type;
2389 2390
	} else if (arg_type == ARG_CONST_MAP_PTR) {
		expected_type = CONST_PTR_TO_MAP;
2391 2392
		if (type != expected_type)
			goto err_type;
2393 2394
	} else if (arg_type == ARG_PTR_TO_CTX) {
		expected_type = PTR_TO_CTX;
2395 2396
		if (type != expected_type)
			goto err_type;
2397 2398 2399
		err = check_ctx_reg(env, reg, regno);
		if (err < 0)
			return err;
2400 2401 2402 2403 2404
	} else if (arg_type == ARG_PTR_TO_SOCK_COMMON) {
		expected_type = PTR_TO_SOCK_COMMON;
		/* Any sk pointer can be ARG_PTR_TO_SOCK_COMMON */
		if (!type_is_sk_pointer(type))
			goto err_type;
2405 2406 2407 2408 2409 2410 2411 2412
		if (reg->ref_obj_id) {
			if (meta->ref_obj_id) {
				verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n",
					regno, reg->ref_obj_id,
					meta->ref_obj_id);
				return -EFAULT;
			}
			meta->ref_obj_id = reg->ref_obj_id;
2413
		}
2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424
	} else if (arg_type == ARG_PTR_TO_SPIN_LOCK) {
		if (meta->func_id == BPF_FUNC_spin_lock) {
			if (process_spin_lock(env, regno, true))
				return -EACCES;
		} else if (meta->func_id == BPF_FUNC_spin_unlock) {
			if (process_spin_lock(env, regno, false))
				return -EACCES;
		} else {
			verbose(env, "verifier internal error\n");
			return -EFAULT;
		}
2425
	} else if (arg_type_is_mem_ptr(arg_type)) {
2426 2427
		expected_type = PTR_TO_STACK;
		/* One exception here. In case function allows for NULL to be
2428
		 * passed in as argument, it's a SCALAR_VALUE type. Final test
2429 2430
		 * happens during stack boundary checking.
		 */
2431
		if (register_is_null(reg) &&
2432
		    arg_type == ARG_PTR_TO_MEM_OR_NULL)
2433
			/* final test in check_stack_boundary() */;
2434 2435
		else if (!type_is_pkt_pointer(type) &&
			 type != PTR_TO_MAP_VALUE &&
2436
			 type != expected_type)
2437
			goto err_type;
2438
		meta->raw_mode = arg_type == ARG_PTR_TO_UNINIT_MEM;
2439
	} else {
2440
		verbose(env, "unsupported arg_type %d\n", arg_type);
2441 2442 2443 2444 2445
		return -EFAULT;
	}

	if (arg_type == ARG_CONST_MAP_PTR) {
		/* bpf_map_xxx(map_ptr) call: remember that map_ptr */
2446
		meta->map_ptr = reg->map_ptr;
2447 2448 2449 2450 2451
	} else if (arg_type == ARG_PTR_TO_MAP_KEY) {
		/* bpf_map_xxx(..., map_ptr, ..., key) call:
		 * check that [key, key + map->key_size) are within
		 * stack limits and initialized
		 */
2452
		if (!meta->map_ptr) {
2453 2454 2455 2456 2457
			/* in function declaration map_ptr must come before
			 * map_key, so that it's verified and known before
			 * we have to check map_key here. Otherwise it means
			 * that kernel subsystem misconfigured verifier
			 */
2458
			verbose(env, "invalid map_ptr to access map->key\n");
2459 2460
			return -EACCES;
		}
2461 2462 2463
		err = check_helper_mem_access(env, regno,
					      meta->map_ptr->key_size, false,
					      NULL);
2464 2465
	} else if (arg_type == ARG_PTR_TO_MAP_VALUE ||
		   arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE) {
2466 2467 2468
		/* bpf_map_xxx(..., map_ptr, ..., value) call:
		 * check [value, value + map->value_size) validity
		 */
2469
		if (!meta->map_ptr) {
2470
			/* kernel subsystem misconfigured verifier */
2471
			verbose(env, "invalid map_ptr to access map->value\n");
2472 2473
			return -EACCES;
		}
2474
		meta->raw_mode = (arg_type == ARG_PTR_TO_UNINIT_MAP_VALUE);
2475 2476
		err = check_helper_mem_access(env, regno,
					      meta->map_ptr->value_size, false,
2477
					      meta);
2478
	} else if (arg_type_is_mem_size(arg_type)) {
2479
		bool zero_size_allowed = (arg_type == ARG_CONST_SIZE_OR_ZERO);
2480

2481 2482 2483 2484 2485 2486
		/* remember the mem_size which may be used later
		 * to refine return values.
		 */
		meta->msize_smax_value = reg->smax_value;
		meta->msize_umax_value = reg->umax_value;

2487 2488
		/* The register is SCALAR_VALUE; the access check
		 * happens using its boundaries.
2489
		 */
2490
		if (!tnum_is_const(reg->var_off))
2491 2492 2493 2494 2495 2496 2497
			/* For unprivileged variable accesses, disable raw
			 * mode so that the program is required to
			 * initialize all the memory that the helper could
			 * just partially fill up.
			 */
			meta = NULL;

2498
		if (reg->smin_value < 0) {
2499
			verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n",
2500 2501 2502
				regno);
			return -EACCES;
		}
2503

2504
		if (reg->umin_value == 0) {
2505 2506 2507
			err = check_helper_mem_access(env, regno - 1, 0,
						      zero_size_allowed,
						      meta);
2508 2509 2510
			if (err)
				return err;
		}
2511

2512
		if (reg->umax_value >= BPF_MAX_VAR_SIZ) {
2513
			verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n",
2514 2515 2516 2517
				regno);
			return -EACCES;
		}
		err = check_helper_mem_access(env, regno - 1,
2518
					      reg->umax_value,
2519
					      zero_size_allowed, meta);
2520 2521 2522
	}

	return err;
2523
err_type:
2524
	verbose(env, "R%d type=%s expected=%s\n", regno,
2525 2526
		reg_type_str[type], reg_type_str[expected_type]);
	return -EACCES;
2527 2528
}

2529 2530
static int check_map_func_compatibility(struct bpf_verifier_env *env,
					struct bpf_map *map, int func_id)
2531 2532 2533 2534
{
	if (!map)
		return 0;

2535 2536 2537 2538 2539 2540 2541 2542
	/* We need a two way check, first is from map perspective ... */
	switch (map->map_type) {
	case BPF_MAP_TYPE_PROG_ARRAY:
		if (func_id != BPF_FUNC_tail_call)
			goto error;
		break;
	case BPF_MAP_TYPE_PERF_EVENT_ARRAY:
		if (func_id != BPF_FUNC_perf_event_read &&
2543 2544
		    func_id != BPF_FUNC_perf_event_output &&
		    func_id != BPF_FUNC_perf_event_read_value)
2545 2546 2547 2548 2549 2550
			goto error;
		break;
	case BPF_MAP_TYPE_STACK_TRACE:
		if (func_id != BPF_FUNC_get_stackid)
			goto error;
		break;
2551
	case BPF_MAP_TYPE_CGROUP_ARRAY:
2552
		if (func_id != BPF_FUNC_skb_under_cgroup &&
2553
		    func_id != BPF_FUNC_current_task_under_cgroup)
2554 2555
			goto error;
		break;
2556
	case BPF_MAP_TYPE_CGROUP_STORAGE:
2557
	case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE:
2558 2559 2560
		if (func_id != BPF_FUNC_get_local_storage)
			goto error;
		break;
2561 2562 2563 2564 2565
	/* devmap returns a pointer to a live net_device ifindex that we cannot
	 * allow to be modified from bpf side. So do not allow lookup elements
	 * for now.
	 */
	case BPF_MAP_TYPE_DEVMAP:
2566
		if (func_id != BPF_FUNC_redirect_map)
2567 2568
			goto error;
		break;
2569 2570 2571
	/* Restrict bpf side of cpumap and xskmap, open when use-cases
	 * appear.
	 */
2572
	case BPF_MAP_TYPE_CPUMAP:
2573
	case BPF_MAP_TYPE_XSKMAP:
2574 2575 2576
		if (func_id != BPF_FUNC_redirect_map)
			goto error;
		break;
2577
	case BPF_MAP_TYPE_ARRAY_OF_MAPS:
2578
	case BPF_MAP_TYPE_HASH_OF_MAPS:
2579 2580
		if (func_id != BPF_FUNC_map_lookup_elem)
			goto error;
2581
		break;
2582 2583 2584
	case BPF_MAP_TYPE_SOCKMAP:
		if (func_id != BPF_FUNC_sk_redirect_map &&
		    func_id != BPF_FUNC_sock_map_update &&
2585 2586
		    func_id != BPF_FUNC_map_delete_elem &&
		    func_id != BPF_FUNC_msg_redirect_map)
2587 2588
			goto error;
		break;
2589 2590 2591 2592 2593 2594 2595
	case BPF_MAP_TYPE_SOCKHASH:
		if (func_id != BPF_FUNC_sk_redirect_hash &&
		    func_id != BPF_FUNC_sock_hash_update &&
		    func_id != BPF_FUNC_map_delete_elem &&
		    func_id != BPF_FUNC_msg_redirect_hash)
			goto error;
		break;
2596 2597 2598 2599
	case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
		if (func_id != BPF_FUNC_sk_select_reuseport)
			goto error;
		break;
2600 2601 2602 2603 2604 2605 2606
	case BPF_MAP_TYPE_QUEUE:
	case BPF_MAP_TYPE_STACK:
		if (func_id != BPF_FUNC_map_peek_elem &&
		    func_id != BPF_FUNC_map_pop_elem &&
		    func_id != BPF_FUNC_map_push_elem)
			goto error;
		break;
2607 2608 2609 2610 2611 2612 2613 2614 2615
	default:
		break;
	}

	/* ... and second from the function itself. */
	switch (func_id) {
	case BPF_FUNC_tail_call:
		if (map->map_type != BPF_MAP_TYPE_PROG_ARRAY)
			goto error;
2616
		if (env->subprog_cnt > 1) {
2617 2618 2619
			verbose(env, "tail_calls are not allowed in programs with bpf-to-bpf calls\n");
			return -EINVAL;
		}
2620 2621 2622
		break;
	case BPF_FUNC_perf_event_read:
	case BPF_FUNC_perf_event_output:
2623
	case BPF_FUNC_perf_event_read_value:
2624 2625 2626 2627 2628 2629 2630
		if (map->map_type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
			goto error;
		break;
	case BPF_FUNC_get_stackid:
		if (map->map_type != BPF_MAP_TYPE_STACK_TRACE)
			goto error;
		break;
2631
	case BPF_FUNC_current_task_under_cgroup:
2632
	case BPF_FUNC_skb_under_cgroup:
2633 2634 2635
		if (map->map_type != BPF_MAP_TYPE_CGROUP_ARRAY)
			goto error;
		break;
2636
	case BPF_FUNC_redirect_map:
2637
		if (map->map_type != BPF_MAP_TYPE_DEVMAP &&
2638 2639
		    map->map_type != BPF_MAP_TYPE_CPUMAP &&
		    map->map_type != BPF_MAP_TYPE_XSKMAP)
2640 2641
			goto error;
		break;
2642
	case BPF_FUNC_sk_redirect_map:
2643
	case BPF_FUNC_msg_redirect_map:
2644
	case BPF_FUNC_sock_map_update:
2645 2646 2647
		if (map->map_type != BPF_MAP_TYPE_SOCKMAP)
			goto error;
		break;
2648 2649 2650 2651
	case BPF_FUNC_sk_redirect_hash:
	case BPF_FUNC_msg_redirect_hash:
	case BPF_FUNC_sock_hash_update:
		if (map->map_type != BPF_MAP_TYPE_SOCKHASH)
2652 2653
			goto error;
		break;
2654
	case BPF_FUNC_get_local_storage:
2655 2656
		if (map->map_type != BPF_MAP_TYPE_CGROUP_STORAGE &&
		    map->map_type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE)
2657 2658
			goto error;
		break;
2659 2660 2661 2662
	case BPF_FUNC_sk_select_reuseport:
		if (map->map_type != BPF_MAP_TYPE_REUSEPORT_SOCKARRAY)
			goto error;
		break;
2663 2664 2665 2666 2667 2668 2669
	case BPF_FUNC_map_peek_elem:
	case BPF_FUNC_map_pop_elem:
	case BPF_FUNC_map_push_elem:
		if (map->map_type != BPF_MAP_TYPE_QUEUE &&
		    map->map_type != BPF_MAP_TYPE_STACK)
			goto error;
		break;
2670 2671
	default:
		break;
2672 2673 2674
	}

	return 0;
2675
error:
2676
	verbose(env, "cannot pass map_type %d into func %s#%d\n",
2677
		map->map_type, func_id_name(func_id), func_id);
2678
	return -EINVAL;
2679 2680
}

2681
static bool check_raw_mode_ok(const struct bpf_func_proto *fn)
2682 2683 2684
{
	int count = 0;

2685
	if (fn->arg1_type == ARG_PTR_TO_UNINIT_MEM)
2686
		count++;
2687
	if (fn->arg2_type == ARG_PTR_TO_UNINIT_MEM)
2688
		count++;
2689
	if (fn->arg3_type == ARG_PTR_TO_UNINIT_MEM)
2690
		count++;
2691
	if (fn->arg4_type == ARG_PTR_TO_UNINIT_MEM)
2692
		count++;
2693
	if (fn->arg5_type == ARG_PTR_TO_UNINIT_MEM)
2694 2695
		count++;

2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729
	/* We only support one arg being in raw mode at the moment,
	 * which is sufficient for the helper functions we have
	 * right now.
	 */
	return count <= 1;
}

static bool check_args_pair_invalid(enum bpf_arg_type arg_curr,
				    enum bpf_arg_type arg_next)
{
	return (arg_type_is_mem_ptr(arg_curr) &&
	        !arg_type_is_mem_size(arg_next)) ||
	       (!arg_type_is_mem_ptr(arg_curr) &&
		arg_type_is_mem_size(arg_next));
}

static bool check_arg_pair_ok(const struct bpf_func_proto *fn)
{
	/* bpf_xxx(..., buf, len) call will access 'len'
	 * bytes from memory 'buf'. Both arg types need
	 * to be paired, so make sure there's no buggy
	 * helper function specification.
	 */
	if (arg_type_is_mem_size(fn->arg1_type) ||
	    arg_type_is_mem_ptr(fn->arg5_type)  ||
	    check_args_pair_invalid(fn->arg1_type, fn->arg2_type) ||
	    check_args_pair_invalid(fn->arg2_type, fn->arg3_type) ||
	    check_args_pair_invalid(fn->arg3_type, fn->arg4_type) ||
	    check_args_pair_invalid(fn->arg4_type, fn->arg5_type))
		return false;

	return true;
}

2730
static bool check_refcount_ok(const struct bpf_func_proto *fn, int func_id)
2731 2732 2733
{
	int count = 0;

2734
	if (arg_type_may_be_refcounted(fn->arg1_type))
2735
		count++;
2736
	if (arg_type_may_be_refcounted(fn->arg2_type))
2737
		count++;
2738
	if (arg_type_may_be_refcounted(fn->arg3_type))
2739
		count++;
2740
	if (arg_type_may_be_refcounted(fn->arg4_type))
2741
		count++;
2742
	if (arg_type_may_be_refcounted(fn->arg5_type))
2743 2744
		count++;

2745 2746 2747 2748 2749 2750
	/* A reference acquiring function cannot acquire
	 * another refcounted ptr.
	 */
	if (is_acquire_function(func_id) && count)
		return false;

2751 2752 2753 2754 2755 2756
	/* We only support one arg being unreferenced at the moment,
	 * which is sufficient for the helper functions we have right now.
	 */
	return count <= 1;
}

2757
static int check_func_proto(const struct bpf_func_proto *fn, int func_id)
2758 2759
{
	return check_raw_mode_ok(fn) &&
2760
	       check_arg_pair_ok(fn) &&
2761
	       check_refcount_ok(fn, func_id) ? 0 : -EINVAL;
2762 2763
}

2764 2765
/* Packet data might have moved, any old PTR_TO_PACKET[_META,_END]
 * are now invalid, so turn them into unknown SCALAR_VALUE.
2766
 */
2767 2768
static void __clear_all_pkt_pointers(struct bpf_verifier_env *env,
				     struct bpf_func_state *state)
2769
{
2770
	struct bpf_reg_state *regs = state->regs, *reg;
2771 2772 2773
	int i;

	for (i = 0; i < MAX_BPF_REG; i++)
2774
		if (reg_is_pkt_pointer_any(&regs[i]))
2775
			mark_reg_unknown(env, regs, i);
2776

2777 2778
	bpf_for_each_spilled_reg(i, state, reg) {
		if (!reg)
2779
			continue;
2780 2781
		if (reg_is_pkt_pointer_any(reg))
			__mark_reg_unknown(reg);
2782 2783 2784
	}
}

2785 2786 2787 2788 2789 2790 2791 2792 2793
static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
{
	struct bpf_verifier_state *vstate = env->cur_state;
	int i;

	for (i = 0; i <= vstate->curframe; i++)
		__clear_all_pkt_pointers(env, vstate->frame[i]);
}

2794
static void release_reg_references(struct bpf_verifier_env *env,
2795 2796
				   struct bpf_func_state *state,
				   int ref_obj_id)
2797 2798 2799 2800 2801
{
	struct bpf_reg_state *regs = state->regs, *reg;
	int i;

	for (i = 0; i < MAX_BPF_REG; i++)
2802
		if (regs[i].ref_obj_id == ref_obj_id)
2803 2804 2805 2806 2807
			mark_reg_unknown(env, regs, i);

	bpf_for_each_spilled_reg(i, state, reg) {
		if (!reg)
			continue;
2808
		if (reg->ref_obj_id == ref_obj_id)
2809 2810 2811 2812 2813 2814 2815 2816
			__mark_reg_unknown(reg);
	}
}

/* The pointer with the specified id has released its reference to kernel
 * resources. Identify all copies of the same pointer and clear the reference.
 */
static int release_reference(struct bpf_verifier_env *env,
2817
			     int ref_obj_id)
2818 2819
{
	struct bpf_verifier_state *vstate = env->cur_state;
2820
	int err;
2821 2822
	int i;

2823 2824 2825 2826
	err = release_reference_state(cur_func(env), ref_obj_id);
	if (err)
		return err;

2827
	for (i = 0; i <= vstate->curframe; i++)
2828
		release_reg_references(env, vstate->frame[i], ref_obj_id);
2829

2830
	return 0;
2831 2832
}

2833 2834 2835 2836 2837
static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn,
			   int *insn_idx)
{
	struct bpf_verifier_state *state = env->cur_state;
	struct bpf_func_state *caller, *callee;
2838
	int i, err, subprog, target_insn;
2839

2840
	if (state->curframe + 1 >= MAX_CALL_FRAMES) {
2841
		verbose(env, "the call stack of %d frames is too deep\n",
2842
			state->curframe + 2);
2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873
		return -E2BIG;
	}

	target_insn = *insn_idx + insn->imm;
	subprog = find_subprog(env, target_insn + 1);
	if (subprog < 0) {
		verbose(env, "verifier bug. No program starts at insn %d\n",
			target_insn + 1);
		return -EFAULT;
	}

	caller = state->frame[state->curframe];
	if (state->frame[state->curframe + 1]) {
		verbose(env, "verifier bug. Frame %d already allocated\n",
			state->curframe + 1);
		return -EFAULT;
	}

	callee = kzalloc(sizeof(*callee), GFP_KERNEL);
	if (!callee)
		return -ENOMEM;
	state->frame[state->curframe + 1] = callee;

	/* callee cannot access r0, r6 - r9 for reading and has to write
	 * into its own stack before reading from it.
	 * callee can read/write into caller's stack
	 */
	init_func_state(env, callee,
			/* remember the callsite, it will be used by bpf_exit */
			*insn_idx /* callsite */,
			state->curframe + 1 /* frameno within this callchain */,
2874
			subprog /* subprog number within this prog */);
2875

2876 2877 2878 2879 2880
	/* Transfer references to the callee */
	err = transfer_reference_state(callee, caller);
	if (err)
		return err;

2881 2882 2883
	/* copy r1 - r5 args that callee can access.  The copy includes parent
	 * pointers, which connects us up to the liveness chain
	 */
2884 2885 2886
	for (i = BPF_REG_1; i <= BPF_REG_5; i++)
		callee->regs[i] = caller->regs[i];

2887
	/* after the call registers r0 - r5 were scratched */
2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912
	for (i = 0; i < CALLER_SAVED_REGS; i++) {
		mark_reg_not_init(env, caller->regs, caller_saved[i]);
		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
	}

	/* only increment it after check_reg_arg() finished */
	state->curframe++;

	/* and go analyze first insn of the callee */
	*insn_idx = target_insn;

	if (env->log.level) {
		verbose(env, "caller:\n");
		print_verifier_state(env, caller);
		verbose(env, "callee:\n");
		print_verifier_state(env, callee);
	}
	return 0;
}

static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
{
	struct bpf_verifier_state *state = env->cur_state;
	struct bpf_func_state *caller, *callee;
	struct bpf_reg_state *r0;
2913
	int err;
2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932

	callee = state->frame[state->curframe];
	r0 = &callee->regs[BPF_REG_0];
	if (r0->type == PTR_TO_STACK) {
		/* technically it's ok to return caller's stack pointer
		 * (or caller's caller's pointer) back to the caller,
		 * since these pointers are valid. Only current stack
		 * pointer will be invalid as soon as function exits,
		 * but let's be conservative
		 */
		verbose(env, "cannot return stack pointer to the caller\n");
		return -EINVAL;
	}

	state->curframe--;
	caller = state->frame[state->curframe];
	/* return to the caller whatever r0 had in the callee */
	caller->regs[BPF_REG_0] = *r0;

2933 2934 2935 2936 2937
	/* Transfer references to the caller */
	err = transfer_reference_state(caller, callee);
	if (err)
		return err;

2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950
	*insn_idx = callee->callsite + 1;
	if (env->log.level) {
		verbose(env, "returning from callee:\n");
		print_verifier_state(env, callee);
		verbose(env, "to caller at %d:\n", *insn_idx);
		print_verifier_state(env, caller);
	}
	/* clear everything in the callee */
	free_func_state(callee);
	state->frame[state->curframe + 1] = NULL;
	return 0;
}

2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967
static void do_refine_retval_range(struct bpf_reg_state *regs, int ret_type,
				   int func_id,
				   struct bpf_call_arg_meta *meta)
{
	struct bpf_reg_state *ret_reg = &regs[BPF_REG_0];

	if (ret_type != RET_INTEGER ||
	    (func_id != BPF_FUNC_get_stack &&
	     func_id != BPF_FUNC_probe_read_str))
		return;

	ret_reg->smax_value = meta->msize_smax_value;
	ret_reg->umax_value = meta->msize_umax_value;
	__reg_deduce_bounds(ret_reg);
	__reg_bound_offset(ret_reg);
}

2968 2969 2970 2971 2972 2973 2974
static int
record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta,
		int func_id, int insn_idx)
{
	struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];

	if (func_id != BPF_FUNC_tail_call &&
2975 2976
	    func_id != BPF_FUNC_map_lookup_elem &&
	    func_id != BPF_FUNC_map_update_elem &&
2977 2978 2979 2980
	    func_id != BPF_FUNC_map_delete_elem &&
	    func_id != BPF_FUNC_map_push_elem &&
	    func_id != BPF_FUNC_map_pop_elem &&
	    func_id != BPF_FUNC_map_peek_elem)
2981
		return 0;
2982

2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996
	if (meta->map_ptr == NULL) {
		verbose(env, "kernel subsystem misconfigured verifier\n");
		return -EINVAL;
	}

	if (!BPF_MAP_PTR(aux->map_state))
		bpf_map_ptr_store(aux, meta->map_ptr,
				  meta->map_ptr->unpriv_array);
	else if (BPF_MAP_PTR(aux->map_state) != meta->map_ptr)
		bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON,
				  meta->map_ptr->unpriv_array);
	return 0;
}

2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008
static int check_reference_leak(struct bpf_verifier_env *env)
{
	struct bpf_func_state *state = cur_func(env);
	int i;

	for (i = 0; i < state->acquired_refs; i++) {
		verbose(env, "Unreleased reference id=%d alloc_insn=%d\n",
			state->refs[i].id, state->refs[i].insn_idx);
	}
	return state->acquired_refs ? -EINVAL : 0;
}

3009
static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx)
3010 3011
{
	const struct bpf_func_proto *fn = NULL;
3012
	struct bpf_reg_state *regs;
3013
	struct bpf_call_arg_meta meta;
3014
	bool changes_data;
3015 3016 3017 3018
	int i, err;

	/* find function prototype */
	if (func_id < 0 || func_id >= __BPF_FUNC_MAX_ID) {
3019 3020
		verbose(env, "invalid func %s#%d\n", func_id_name(func_id),
			func_id);
3021 3022 3023
		return -EINVAL;
	}

3024
	if (env->ops->get_func_proto)
3025
		fn = env->ops->get_func_proto(func_id, env->prog);
3026
	if (!fn) {
3027 3028
		verbose(env, "unknown func %s#%d\n", func_id_name(func_id),
			func_id);
3029 3030 3031 3032
		return -EINVAL;
	}

	/* eBPF programs must be GPL compatible to use GPL-ed functions */
3033
	if (!env->prog->gpl_compatible && fn->gpl_only) {
3034
		verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
3035 3036 3037
		return -EINVAL;
	}

3038
	/* With LD_ABS/IND some JITs save/restore skb from r1. */
3039
	changes_data = bpf_helper_changes_pkt_data(fn->func);
3040 3041 3042 3043 3044
	if (changes_data && fn->arg1_type != ARG_PTR_TO_CTX) {
		verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n",
			func_id_name(func_id), func_id);
		return -EINVAL;
	}
3045

3046
	memset(&meta, 0, sizeof(meta));
3047
	meta.pkt_access = fn->pkt_access;
3048

3049
	err = check_func_proto(fn, func_id);
3050
	if (err) {
3051
		verbose(env, "kernel subsystem misconfigured func %s#%d\n",
3052
			func_id_name(func_id), func_id);
3053 3054 3055
		return err;
	}

3056
	meta.func_id = func_id;
3057
	/* check args */
3058
	err = check_func_arg(env, BPF_REG_1, fn->arg1_type, &meta);
3059 3060
	if (err)
		return err;
3061
	err = check_func_arg(env, BPF_REG_2, fn->arg2_type, &meta);
3062 3063
	if (err)
		return err;
3064
	err = check_func_arg(env, BPF_REG_3, fn->arg3_type, &meta);
3065 3066
	if (err)
		return err;
3067
	err = check_func_arg(env, BPF_REG_4, fn->arg4_type, &meta);
3068 3069
	if (err)
		return err;
3070
	err = check_func_arg(env, BPF_REG_5, fn->arg5_type, &meta);
3071 3072 3073
	if (err)
		return err;

3074 3075 3076 3077
	err = record_func_map(env, &meta, func_id, insn_idx);
	if (err)
		return err;

3078 3079 3080 3081
	/* Mark slots with STACK_MISC in case of raw mode, stack offset
	 * is inferred from register state.
	 */
	for (i = 0; i < meta.access_size; i++) {
3082 3083
		err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B,
				       BPF_WRITE, -1, false);
3084 3085 3086 3087
		if (err)
			return err;
	}

3088 3089 3090 3091 3092 3093 3094
	if (func_id == BPF_FUNC_tail_call) {
		err = check_reference_leak(env);
		if (err) {
			verbose(env, "tail_call would lead to reference leak\n");
			return err;
		}
	} else if (is_release_function(func_id)) {
3095
		err = release_reference(env, meta.ref_obj_id);
3096 3097 3098
		if (err) {
			verbose(env, "func %s#%d reference has not been acquired before\n",
				func_id_name(func_id), func_id);
3099
			return err;
3100
		}
3101 3102
	}

3103
	regs = cur_regs(env);
3104 3105 3106 3107 3108 3109 3110 3111 3112 3113

	/* check that flags argument in get_local_storage(map, flags) is 0,
	 * this is required because get_local_storage() can't return an error.
	 */
	if (func_id == BPF_FUNC_get_local_storage &&
	    !register_is_null(&regs[BPF_REG_2])) {
		verbose(env, "get_local_storage() doesn't support non-zero flags\n");
		return -EINVAL;
	}

3114
	/* reset caller saved regs */
3115
	for (i = 0; i < CALLER_SAVED_REGS; i++) {
3116
		mark_reg_not_init(env, regs, caller_saved[i]);
3117 3118
		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
	}
3119

3120
	/* update return register (already marked as written above) */
3121
	if (fn->ret_type == RET_INTEGER) {
3122
		/* sets type to SCALAR_VALUE */
3123
		mark_reg_unknown(env, regs, BPF_REG_0);
3124 3125
	} else if (fn->ret_type == RET_VOID) {
		regs[BPF_REG_0].type = NOT_INIT;
3126 3127
	} else if (fn->ret_type == RET_PTR_TO_MAP_VALUE_OR_NULL ||
		   fn->ret_type == RET_PTR_TO_MAP_VALUE) {
3128
		/* There is no offset yet applied, variable or fixed */
3129
		mark_reg_known_zero(env, regs, BPF_REG_0);
3130 3131 3132 3133
		/* remember map_ptr, so that check_map_access()
		 * can check 'value_size' boundary of memory access
		 * to map element returned from bpf_map_lookup_elem()
		 */
3134
		if (meta.map_ptr == NULL) {
3135 3136
			verbose(env,
				"kernel subsystem misconfigured verifier\n");
3137 3138
			return -EINVAL;
		}
3139
		regs[BPF_REG_0].map_ptr = meta.map_ptr;
3140 3141
		if (fn->ret_type == RET_PTR_TO_MAP_VALUE) {
			regs[BPF_REG_0].type = PTR_TO_MAP_VALUE;
3142 3143
			if (map_value_has_spin_lock(meta.map_ptr))
				regs[BPF_REG_0].id = ++env->id_gen;
3144 3145 3146 3147
		} else {
			regs[BPF_REG_0].type = PTR_TO_MAP_VALUE_OR_NULL;
			regs[BPF_REG_0].id = ++env->id_gen;
		}
3148 3149 3150
	} else if (fn->ret_type == RET_PTR_TO_SOCKET_OR_NULL) {
		mark_reg_known_zero(env, regs, BPF_REG_0);
		regs[BPF_REG_0].type = PTR_TO_SOCKET_OR_NULL;
3151
		regs[BPF_REG_0].id = ++env->id_gen;
3152 3153 3154 3155
	} else if (fn->ret_type == RET_PTR_TO_SOCK_COMMON_OR_NULL) {
		mark_reg_known_zero(env, regs, BPF_REG_0);
		regs[BPF_REG_0].type = PTR_TO_SOCK_COMMON_OR_NULL;
		regs[BPF_REG_0].id = ++env->id_gen;
3156 3157 3158 3159
	} else if (fn->ret_type == RET_PTR_TO_TCP_SOCK_OR_NULL) {
		mark_reg_known_zero(env, regs, BPF_REG_0);
		regs[BPF_REG_0].type = PTR_TO_TCP_SOCK_OR_NULL;
		regs[BPF_REG_0].id = ++env->id_gen;
3160
	} else {
3161
		verbose(env, "unknown return type %d of func %s#%d\n",
3162
			fn->ret_type, func_id_name(func_id), func_id);
3163 3164
		return -EINVAL;
	}
3165

3166
	if (is_ptr_cast_function(func_id)) {
3167 3168
		/* For release_reference() */
		regs[BPF_REG_0].ref_obj_id = meta.ref_obj_id;
3169 3170 3171 3172 3173 3174 3175 3176 3177 3178
	} else if (is_acquire_function(func_id)) {
		int id = acquire_reference_state(env, insn_idx);

		if (id < 0)
			return id;
		/* For mark_ptr_or_null_reg() */
		regs[BPF_REG_0].id = id;
		/* For release_reference() */
		regs[BPF_REG_0].ref_obj_id = id;
	}
3179

3180 3181
	do_refine_retval_range(regs, fn->ret_type, func_id, &meta);

3182
	err = check_map_func_compatibility(env, meta.map_ptr, func_id);
3183 3184
	if (err)
		return err;
3185

3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203
	if (func_id == BPF_FUNC_get_stack && !env->prog->has_callchain_buf) {
		const char *err_str;

#ifdef CONFIG_PERF_EVENTS
		err = get_callchain_buffers(sysctl_perf_event_max_stack);
		err_str = "cannot get callchain buffer for func %s#%d\n";
#else
		err = -ENOTSUPP;
		err_str = "func %s#%d not supported without CONFIG_PERF_EVENTS\n";
#endif
		if (err) {
			verbose(env, err_str, func_id_name(func_id), func_id);
			return err;
		}

		env->prog->has_callchain_buf = true;
	}

3204 3205 3206 3207 3208
	if (changes_data)
		clear_all_pkt_pointers(env);
	return 0;
}

3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226
static bool signed_add_overflows(s64 a, s64 b)
{
	/* Do the add in u64, where overflow is well-defined */
	s64 res = (s64)((u64)a + (u64)b);

	if (b < 0)
		return res > a;
	return res < a;
}

static bool signed_sub_overflows(s64 a, s64 b)
{
	/* Do the sub in u64, where overflow is well-defined */
	s64 res = (s64)((u64)a - (u64)b);

	if (b < 0)
		return res < a;
	return res > a;
3227 3228
}

3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263
static bool check_reg_sane_offset(struct bpf_verifier_env *env,
				  const struct bpf_reg_state *reg,
				  enum bpf_reg_type type)
{
	bool known = tnum_is_const(reg->var_off);
	s64 val = reg->var_off.value;
	s64 smin = reg->smin_value;

	if (known && (val >= BPF_MAX_VAR_OFF || val <= -BPF_MAX_VAR_OFF)) {
		verbose(env, "math between %s pointer and %lld is not allowed\n",
			reg_type_str[type], val);
		return false;
	}

	if (reg->off >= BPF_MAX_VAR_OFF || reg->off <= -BPF_MAX_VAR_OFF) {
		verbose(env, "%s pointer offset %d is not allowed\n",
			reg_type_str[type], reg->off);
		return false;
	}

	if (smin == S64_MIN) {
		verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
			reg_type_str[type]);
		return false;
	}

	if (smin >= BPF_MAX_VAR_OFF || smin <= -BPF_MAX_VAR_OFF) {
		verbose(env, "value %lld makes %s pointer be out of bounds\n",
			smin, reg_type_str[type]);
		return false;
	}

	return true;
}

3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295 3296
static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
{
	return &env->insn_aux_data[env->insn_idx];
}

static int retrieve_ptr_limit(const struct bpf_reg_state *ptr_reg,
			      u32 *ptr_limit, u8 opcode, bool off_is_neg)
{
	bool mask_to_left = (opcode == BPF_ADD &&  off_is_neg) ||
			    (opcode == BPF_SUB && !off_is_neg);
	u32 off;

	switch (ptr_reg->type) {
	case PTR_TO_STACK:
		off = ptr_reg->off + ptr_reg->var_off.value;
		if (mask_to_left)
			*ptr_limit = MAX_BPF_STACK + off;
		else
			*ptr_limit = -off;
		return 0;
	case PTR_TO_MAP_VALUE:
		if (mask_to_left) {
			*ptr_limit = ptr_reg->umax_value + ptr_reg->off;
		} else {
			off = ptr_reg->smin_value + ptr_reg->off;
			*ptr_limit = ptr_reg->map_ptr->value_size - off;
		}
		return 0;
	default:
		return -EINVAL;
	}
}

3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330
static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env,
				    const struct bpf_insn *insn)
{
	return env->allow_ptr_leaks || BPF_SRC(insn->code) == BPF_K;
}

static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux,
				       u32 alu_state, u32 alu_limit)
{
	/* If we arrived here from different branches with different
	 * state or limits to sanitize, then this won't work.
	 */
	if (aux->alu_state &&
	    (aux->alu_state != alu_state ||
	     aux->alu_limit != alu_limit))
		return -EACCES;

	/* Corresponding fixup done in fixup_bpf_calls(). */
	aux->alu_state = alu_state;
	aux->alu_limit = alu_limit;
	return 0;
}

static int sanitize_val_alu(struct bpf_verifier_env *env,
			    struct bpf_insn *insn)
{
	struct bpf_insn_aux_data *aux = cur_aux(env);

	if (can_skip_alu_sanitation(env, insn))
		return 0;

	return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
}

3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344
static int sanitize_ptr_alu(struct bpf_verifier_env *env,
			    struct bpf_insn *insn,
			    const struct bpf_reg_state *ptr_reg,
			    struct bpf_reg_state *dst_reg,
			    bool off_is_neg)
{
	struct bpf_verifier_state *vstate = env->cur_state;
	struct bpf_insn_aux_data *aux = cur_aux(env);
	bool ptr_is_dst_reg = ptr_reg == dst_reg;
	u8 opcode = BPF_OP(insn->code);
	u32 alu_state, alu_limit;
	struct bpf_reg_state tmp;
	bool ret;

3345
	if (can_skip_alu_sanitation(env, insn))
3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360
		return 0;

	/* We already marked aux for masking from non-speculative
	 * paths, thus we got here in the first place. We only care
	 * to explore bad access from here.
	 */
	if (vstate->speculative)
		goto do_sim;

	alu_state  = off_is_neg ? BPF_ALU_NEG_VALUE : 0;
	alu_state |= ptr_is_dst_reg ?
		     BPF_ALU_SANITIZE_SRC : BPF_ALU_SANITIZE_DST;

	if (retrieve_ptr_limit(ptr_reg, &alu_limit, opcode, off_is_neg))
		return 0;
3361
	if (update_alu_sanitation_state(aux, alu_state, alu_limit))
3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380 3381 3382
		return -EACCES;
do_sim:
	/* Simulate and find potential out-of-bounds access under
	 * speculative execution from truncation as a result of
	 * masking when off was not within expected range. If off
	 * sits in dst, then we temporarily need to move ptr there
	 * to simulate dst (== 0) +/-= ptr. Needed, for example,
	 * for cases where we use K-based arithmetic in one direction
	 * and truncated reg-based in the other in order to explore
	 * bad access.
	 */
	if (!ptr_is_dst_reg) {
		tmp = *dst_reg;
		*dst_reg = *ptr_reg;
	}
	ret = push_stack(env, env->insn_idx + 1, env->insn_idx, true);
	if (!ptr_is_dst_reg)
		*dst_reg = tmp;
	return !ret ? -EFAULT : 0;
}

3383 3384 3385 3386 3387 3388 3389 3390 3391
/* Handles arithmetic on a pointer and a scalar: computes new min/max and var_off.
 * Caller should also handle BPF_MOV case separately.
 * If we return -EACCES, caller may want to try again treating pointer as a
 * scalar.  So we only emit a diagnostic if !env->allow_ptr_leaks.
 */
static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env,
				   struct bpf_insn *insn,
				   const struct bpf_reg_state *ptr_reg,
				   const struct bpf_reg_state *off_reg)
3392
{
3393 3394 3395
	struct bpf_verifier_state *vstate = env->cur_state;
	struct bpf_func_state *state = vstate->frame[vstate->curframe];
	struct bpf_reg_state *regs = state->regs, *dst_reg;
3396
	bool known = tnum_is_const(off_reg->var_off);
3397 3398 3399 3400
	s64 smin_val = off_reg->smin_value, smax_val = off_reg->smax_value,
	    smin_ptr = ptr_reg->smin_value, smax_ptr = ptr_reg->smax_value;
	u64 umin_val = off_reg->umin_value, umax_val = off_reg->umax_value,
	    umin_ptr = ptr_reg->umin_value, umax_ptr = ptr_reg->umax_value;
3401
	u32 dst = insn->dst_reg, src = insn->src_reg;
3402
	u8 opcode = BPF_OP(insn->code);
3403
	int ret;
3404

3405
	dst_reg = &regs[dst];
3406

3407 3408 3409 3410 3411 3412 3413
	if ((known && (smin_val != smax_val || umin_val != umax_val)) ||
	    smin_val > smax_val || umin_val > umax_val) {
		/* Taint dst register if offset had invalid bounds derived from
		 * e.g. dead branches.
		 */
		__mark_reg_unknown(dst_reg);
		return 0;
3414 3415 3416 3417
	}

	if (BPF_CLASS(insn->code) != BPF_ALU64) {
		/* 32-bit ALU ops on pointers produce (meaningless) scalars */
3418 3419 3420
		verbose(env,
			"R%d 32-bit pointer arithmetic prohibited\n",
			dst);
3421
		return -EACCES;
3422 3423
	}

3424 3425 3426 3427
	switch (ptr_reg->type) {
	case PTR_TO_MAP_VALUE_OR_NULL:
		verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n",
			dst, reg_type_str[ptr_reg->type]);
3428
		return -EACCES;
3429 3430
	case CONST_PTR_TO_MAP:
	case PTR_TO_PACKET_END:
3431 3432
	case PTR_TO_SOCKET:
	case PTR_TO_SOCKET_OR_NULL:
3433 3434
	case PTR_TO_SOCK_COMMON:
	case PTR_TO_SOCK_COMMON_OR_NULL:
3435 3436
	case PTR_TO_TCP_SOCK:
	case PTR_TO_TCP_SOCK_OR_NULL:
3437 3438
		verbose(env, "R%d pointer arithmetic on %s prohibited\n",
			dst, reg_type_str[ptr_reg->type]);
3439
		return -EACCES;
3440 3441 3442 3443 3444 3445 3446
	case PTR_TO_MAP_VALUE:
		if (!env->allow_ptr_leaks && !known && (smin_val < 0) != (smax_val < 0)) {
			verbose(env, "R%d has unknown scalar with mixed signed bounds, pointer arithmetic with it prohibited for !root\n",
				off_reg == dst_reg ? dst : src);
			return -EACCES;
		}
		/* fall-through */
3447 3448
	default:
		break;
3449 3450 3451 3452
	}

	/* In case of 'scalar += pointer', dst_reg inherits pointer type and id.
	 * The id may be overwritten later if we create a new variable offset.
3453
	 */
3454 3455
	dst_reg->type = ptr_reg->type;
	dst_reg->id = ptr_reg->id;
3456

3457 3458 3459 3460
	if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) ||
	    !check_reg_sane_offset(env, ptr_reg, ptr_reg->type))
		return -EINVAL;

3461 3462
	switch (opcode) {
	case BPF_ADD:
3463 3464 3465 3466 3467
		ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0);
		if (ret < 0) {
			verbose(env, "R%d tried to add from different maps or paths\n", dst);
			return ret;
		}
3468 3469
		/* We can take a fixed offset as long as it doesn't overflow
		 * the s32 'off' field
3470
		 */
3471 3472
		if (known && (ptr_reg->off + smin_val ==
			      (s64)(s32)(ptr_reg->off + smin_val))) {
3473
			/* pointer += K.  Accumulate it into fixed offset */
3474 3475 3476 3477
			dst_reg->smin_value = smin_ptr;
			dst_reg->smax_value = smax_ptr;
			dst_reg->umin_value = umin_ptr;
			dst_reg->umax_value = umax_ptr;
3478
			dst_reg->var_off = ptr_reg->var_off;
3479
			dst_reg->off = ptr_reg->off + smin_val;
3480
			dst_reg->raw = ptr_reg->raw;
3481 3482 3483 3484 3485 3486 3487 3488 3489 3490
			break;
		}
		/* A new variable offset is created.  Note that off_reg->off
		 * == 0, since it's a scalar.
		 * dst_reg gets the pointer type and since some positive
		 * integer value was added to the pointer, give it a new 'id'
		 * if it's a PTR_TO_PACKET.
		 * this creates a new 'base' pointer, off_reg (variable) gets
		 * added into the variable offset, and we copy the fixed offset
		 * from ptr_reg.
3491
		 */
3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507
		if (signed_add_overflows(smin_ptr, smin_val) ||
		    signed_add_overflows(smax_ptr, smax_val)) {
			dst_reg->smin_value = S64_MIN;
			dst_reg->smax_value = S64_MAX;
		} else {
			dst_reg->smin_value = smin_ptr + smin_val;
			dst_reg->smax_value = smax_ptr + smax_val;
		}
		if (umin_ptr + umin_val < umin_ptr ||
		    umax_ptr + umax_val < umax_ptr) {
			dst_reg->umin_value = 0;
			dst_reg->umax_value = U64_MAX;
		} else {
			dst_reg->umin_value = umin_ptr + umin_val;
			dst_reg->umax_value = umax_ptr + umax_val;
		}
3508 3509
		dst_reg->var_off = tnum_add(ptr_reg->var_off, off_reg->var_off);
		dst_reg->off = ptr_reg->off;
3510
		dst_reg->raw = ptr_reg->raw;
3511
		if (reg_is_pkt_pointer(ptr_reg)) {
3512 3513
			dst_reg->id = ++env->id_gen;
			/* something was added to pkt_ptr, set range to zero */
3514
			dst_reg->raw = 0;
3515 3516 3517
		}
		break;
	case BPF_SUB:
3518 3519 3520 3521 3522
		ret = sanitize_ptr_alu(env, insn, ptr_reg, dst_reg, smin_val < 0);
		if (ret < 0) {
			verbose(env, "R%d tried to sub from different maps or paths\n", dst);
			return ret;
		}
3523 3524
		if (dst_reg == off_reg) {
			/* scalar -= pointer.  Creates an unknown scalar */
3525 3526
			verbose(env, "R%d tried to subtract pointer from scalar\n",
				dst);
3527 3528 3529 3530 3531
			return -EACCES;
		}
		/* We don't allow subtraction from FP, because (according to
		 * test_verifier.c test "invalid fp arithmetic", JITs might not
		 * be able to deal with it.
3532
		 */
3533
		if (ptr_reg->type == PTR_TO_STACK) {
3534 3535
			verbose(env, "R%d subtraction from stack pointer prohibited\n",
				dst);
3536 3537
			return -EACCES;
		}
3538 3539
		if (known && (ptr_reg->off - smin_val ==
			      (s64)(s32)(ptr_reg->off - smin_val))) {
3540
			/* pointer -= K.  Subtract it from fixed offset */
3541 3542 3543 3544
			dst_reg->smin_value = smin_ptr;
			dst_reg->smax_value = smax_ptr;
			dst_reg->umin_value = umin_ptr;
			dst_reg->umax_value = umax_ptr;
3545 3546
			dst_reg->var_off = ptr_reg->var_off;
			dst_reg->id = ptr_reg->id;
3547
			dst_reg->off = ptr_reg->off - smin_val;
3548
			dst_reg->raw = ptr_reg->raw;
3549 3550 3551 3552
			break;
		}
		/* A new variable offset is created.  If the subtrahend is known
		 * nonnegative, then any reg->range we had before is still good.
3553
		 */
3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571
		if (signed_sub_overflows(smin_ptr, smax_val) ||
		    signed_sub_overflows(smax_ptr, smin_val)) {
			/* Overflow possible, we know nothing */
			dst_reg->smin_value = S64_MIN;
			dst_reg->smax_value = S64_MAX;
		} else {
			dst_reg->smin_value = smin_ptr - smax_val;
			dst_reg->smax_value = smax_ptr - smin_val;
		}
		if (umin_ptr < umax_val) {
			/* Overflow possible, we know nothing */
			dst_reg->umin_value = 0;
			dst_reg->umax_value = U64_MAX;
		} else {
			/* Cannot overflow (as long as bounds are consistent) */
			dst_reg->umin_value = umin_ptr - umax_val;
			dst_reg->umax_value = umax_ptr - umin_val;
		}
3572 3573
		dst_reg->var_off = tnum_sub(ptr_reg->var_off, off_reg->var_off);
		dst_reg->off = ptr_reg->off;
3574
		dst_reg->raw = ptr_reg->raw;
3575
		if (reg_is_pkt_pointer(ptr_reg)) {
3576 3577
			dst_reg->id = ++env->id_gen;
			/* something was added to pkt_ptr, set range to zero */
3578
			if (smin_val < 0)
3579
				dst_reg->raw = 0;
3580
		}
3581 3582 3583 3584
		break;
	case BPF_AND:
	case BPF_OR:
	case BPF_XOR:
3585 3586 3587
		/* bitwise ops on pointers are troublesome, prohibit. */
		verbose(env, "R%d bitwise operator %s on pointer prohibited\n",
			dst, bpf_alu_string[opcode >> 4]);
3588 3589 3590
		return -EACCES;
	default:
		/* other operators (e.g. MUL,LSH) produce non-pointer results */
3591 3592
		verbose(env, "R%d pointer arithmetic with %s operator prohibited\n",
			dst, bpf_alu_string[opcode >> 4]);
3593
		return -EACCES;
3594 3595
	}

3596 3597 3598
	if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type))
		return -EINVAL;

3599 3600 3601
	__update_reg_bounds(dst_reg);
	__reg_deduce_bounds(dst_reg);
	__reg_bound_offset(dst_reg);
3602 3603 3604 3605

	/* For unprivileged we require that resulting offset must be in bounds
	 * in order to be able to sanitize access later on.
	 */
3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618
	if (!env->allow_ptr_leaks) {
		if (dst_reg->type == PTR_TO_MAP_VALUE &&
		    check_map_access(env, dst, dst_reg->off, 1, false)) {
			verbose(env, "R%d pointer arithmetic of map value goes out of range, "
				"prohibited for !root\n", dst);
			return -EACCES;
		} else if (dst_reg->type == PTR_TO_STACK &&
			   check_stack_access(env, dst_reg, dst_reg->off +
					      dst_reg->var_off.value, 1)) {
			verbose(env, "R%d stack pointer arithmetic goes out of range, "
				"prohibited for !root\n", dst);
			return -EACCES;
		}
3619 3620
	}

3621 3622 3623
	return 0;
}

3624 3625 3626 3627
/* WARNING: This function does calculations on 64-bit values, but the actual
 * execution may occur on 32-bit values. Therefore, things like bitshifts
 * need extra checks in the 32-bit case.
 */
3628 3629 3630 3631
static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env,
				      struct bpf_insn *insn,
				      struct bpf_reg_state *dst_reg,
				      struct bpf_reg_state src_reg)
3632
{
3633
	struct bpf_reg_state *regs = cur_regs(env);
3634
	u8 opcode = BPF_OP(insn->code);
3635
	bool src_known, dst_known;
3636 3637
	s64 smin_val, smax_val;
	u64 umin_val, umax_val;
3638
	u64 insn_bitness = (BPF_CLASS(insn->code) == BPF_ALU64) ? 64 : 32;
3639 3640
	u32 dst = insn->dst_reg;
	int ret;
3641

3642 3643 3644 3645 3646 3647 3648 3649 3650
	if (insn_bitness == 32) {
		/* Relevant for 32-bit RSH: Information can propagate towards
		 * LSB, so it isn't sufficient to only truncate the output to
		 * 32 bits.
		 */
		coerce_reg_to_size(dst_reg, 4);
		coerce_reg_to_size(&src_reg, 4);
	}

3651 3652 3653 3654
	smin_val = src_reg.smin_value;
	smax_val = src_reg.smax_value;
	umin_val = src_reg.umin_value;
	umax_val = src_reg.umax_value;
3655 3656
	src_known = tnum_is_const(src_reg.var_off);
	dst_known = tnum_is_const(dst_reg->var_off);
3657

3658 3659 3660 3661 3662 3663 3664 3665 3666
	if ((src_known && (smin_val != smax_val || umin_val != umax_val)) ||
	    smin_val > smax_val || umin_val > umax_val) {
		/* Taint dst register if offset had invalid bounds derived from
		 * e.g. dead branches.
		 */
		__mark_reg_unknown(dst_reg);
		return 0;
	}

3667 3668 3669 3670 3671 3672
	if (!src_known &&
	    opcode != BPF_ADD && opcode != BPF_SUB && opcode != BPF_AND) {
		__mark_reg_unknown(dst_reg);
		return 0;
	}

3673 3674
	switch (opcode) {
	case BPF_ADD:
3675 3676 3677 3678 3679
		ret = sanitize_val_alu(env, insn);
		if (ret < 0) {
			verbose(env, "R%d tried to add from different pointers or scalars\n", dst);
			return ret;
		}
3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695
		if (signed_add_overflows(dst_reg->smin_value, smin_val) ||
		    signed_add_overflows(dst_reg->smax_value, smax_val)) {
			dst_reg->smin_value = S64_MIN;
			dst_reg->smax_value = S64_MAX;
		} else {
			dst_reg->smin_value += smin_val;
			dst_reg->smax_value += smax_val;
		}
		if (dst_reg->umin_value + umin_val < umin_val ||
		    dst_reg->umax_value + umax_val < umax_val) {
			dst_reg->umin_value = 0;
			dst_reg->umax_value = U64_MAX;
		} else {
			dst_reg->umin_value += umin_val;
			dst_reg->umax_value += umax_val;
		}
3696
		dst_reg->var_off = tnum_add(dst_reg->var_off, src_reg.var_off);
3697 3698
		break;
	case BPF_SUB:
3699 3700 3701 3702 3703
		ret = sanitize_val_alu(env, insn);
		if (ret < 0) {
			verbose(env, "R%d tried to sub from different pointers or scalars\n", dst);
			return ret;
		}
3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721
		if (signed_sub_overflows(dst_reg->smin_value, smax_val) ||
		    signed_sub_overflows(dst_reg->smax_value, smin_val)) {
			/* Overflow possible, we know nothing */
			dst_reg->smin_value = S64_MIN;
			dst_reg->smax_value = S64_MAX;
		} else {
			dst_reg->smin_value -= smax_val;
			dst_reg->smax_value -= smin_val;
		}
		if (dst_reg->umin_value < umax_val) {
			/* Overflow possible, we know nothing */
			dst_reg->umin_value = 0;
			dst_reg->umax_value = U64_MAX;
		} else {
			/* Cannot overflow (as long as bounds are consistent) */
			dst_reg->umin_value -= umax_val;
			dst_reg->umax_value -= umin_val;
		}
3722
		dst_reg->var_off = tnum_sub(dst_reg->var_off, src_reg.var_off);
3723 3724
		break;
	case BPF_MUL:
3725 3726
		dst_reg->var_off = tnum_mul(dst_reg->var_off, src_reg.var_off);
		if (smin_val < 0 || dst_reg->smin_value < 0) {
3727
			/* Ain't nobody got time to multiply that sign */
3728 3729
			__mark_reg_unbounded(dst_reg);
			__update_reg_bounds(dst_reg);
3730 3731
			break;
		}
3732 3733
		/* Both values are positive, so we can work with unsigned and
		 * copy the result to signed (unless it exceeds S64_MAX).
3734
		 */
3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751
		if (umax_val > U32_MAX || dst_reg->umax_value > U32_MAX) {
			/* Potential overflow, we know nothing */
			__mark_reg_unbounded(dst_reg);
			/* (except what we can learn from the var_off) */
			__update_reg_bounds(dst_reg);
			break;
		}
		dst_reg->umin_value *= umin_val;
		dst_reg->umax_value *= umax_val;
		if (dst_reg->umax_value > S64_MAX) {
			/* Overflow possible, we know nothing */
			dst_reg->smin_value = S64_MIN;
			dst_reg->smax_value = S64_MAX;
		} else {
			dst_reg->smin_value = dst_reg->umin_value;
			dst_reg->smax_value = dst_reg->umax_value;
		}
3752 3753
		break;
	case BPF_AND:
3754
		if (src_known && dst_known) {
3755 3756
			__mark_reg_known(dst_reg, dst_reg->var_off.value &
						  src_reg.var_off.value);
3757 3758
			break;
		}
3759 3760
		/* We get our minimum from the var_off, since that's inherently
		 * bitwise.  Our maximum is the minimum of the operands' maxima.
3761
		 */
3762
		dst_reg->var_off = tnum_and(dst_reg->var_off, src_reg.var_off);
3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779
		dst_reg->umin_value = dst_reg->var_off.value;
		dst_reg->umax_value = min(dst_reg->umax_value, umax_val);
		if (dst_reg->smin_value < 0 || smin_val < 0) {
			/* Lose signed bounds when ANDing negative numbers,
			 * ain't nobody got time for that.
			 */
			dst_reg->smin_value = S64_MIN;
			dst_reg->smax_value = S64_MAX;
		} else {
			/* ANDing two positives gives a positive, so safe to
			 * cast result into s64.
			 */
			dst_reg->smin_value = dst_reg->umin_value;
			dst_reg->smax_value = dst_reg->umax_value;
		}
		/* We may learn something more from the var_off */
		__update_reg_bounds(dst_reg);
3780 3781 3782
		break;
	case BPF_OR:
		if (src_known && dst_known) {
3783 3784
			__mark_reg_known(dst_reg, dst_reg->var_off.value |
						  src_reg.var_off.value);
3785 3786
			break;
		}
3787 3788
		/* We get our maximum from the var_off, and our minimum is the
		 * maximum of the operands' minima
3789 3790
		 */
		dst_reg->var_off = tnum_or(dst_reg->var_off, src_reg.var_off);
3791 3792 3793 3794 3795 3796 3797 3798 3799
		dst_reg->umin_value = max(dst_reg->umin_value, umin_val);
		dst_reg->umax_value = dst_reg->var_off.value |
				      dst_reg->var_off.mask;
		if (dst_reg->smin_value < 0 || smin_val < 0) {
			/* Lose signed bounds when ORing negative numbers,
			 * ain't nobody got time for that.
			 */
			dst_reg->smin_value = S64_MIN;
			dst_reg->smax_value = S64_MAX;
3800
		} else {
3801 3802 3803 3804 3805
			/* ORing two positives gives a positive, so safe to
			 * cast result into s64.
			 */
			dst_reg->smin_value = dst_reg->umin_value;
			dst_reg->smax_value = dst_reg->umax_value;
3806
		}
3807 3808
		/* We may learn something more from the var_off */
		__update_reg_bounds(dst_reg);
3809 3810
		break;
	case BPF_LSH:
3811 3812 3813
		if (umax_val >= insn_bitness) {
			/* Shifts greater than 31 or 63 are undefined.
			 * This includes shifts by a negative number.
3814
			 */
3815
			mark_reg_unknown(env, regs, insn->dst_reg);
3816 3817
			break;
		}
3818 3819
		/* We lose all sign bit information (except what we can pick
		 * up from var_off)
3820
		 */
3821 3822 3823 3824 3825 3826
		dst_reg->smin_value = S64_MIN;
		dst_reg->smax_value = S64_MAX;
		/* If we might shift our top bit out, then we know nothing */
		if (dst_reg->umax_value > 1ULL << (63 - umax_val)) {
			dst_reg->umin_value = 0;
			dst_reg->umax_value = U64_MAX;
3827
		} else {
3828 3829
			dst_reg->umin_value <<= umin_val;
			dst_reg->umax_value <<= umax_val;
3830
		}
3831
		dst_reg->var_off = tnum_lshift(dst_reg->var_off, umin_val);
3832 3833
		/* We may learn something more from the var_off */
		__update_reg_bounds(dst_reg);
3834 3835
		break;
	case BPF_RSH:
3836 3837 3838
		if (umax_val >= insn_bitness) {
			/* Shifts greater than 31 or 63 are undefined.
			 * This includes shifts by a negative number.
3839
			 */
3840
			mark_reg_unknown(env, regs, insn->dst_reg);
3841 3842
			break;
		}
3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858
		/* BPF_RSH is an unsigned shift.  If the value in dst_reg might
		 * be negative, then either:
		 * 1) src_reg might be zero, so the sign bit of the result is
		 *    unknown, so we lose our signed bounds
		 * 2) it's known negative, thus the unsigned bounds capture the
		 *    signed bounds
		 * 3) the signed bounds cross zero, so they tell us nothing
		 *    about the result
		 * If the value in dst_reg is known nonnegative, then again the
		 * unsigned bounts capture the signed bounds.
		 * Thus, in all cases it suffices to blow away our signed bounds
		 * and rely on inferring new ones from the unsigned bounds and
		 * var_off of the result.
		 */
		dst_reg->smin_value = S64_MIN;
		dst_reg->smax_value = S64_MAX;
3859
		dst_reg->var_off = tnum_rshift(dst_reg->var_off, umin_val);
3860 3861 3862 3863
		dst_reg->umin_value >>= umax_val;
		dst_reg->umax_value >>= umin_val;
		/* We may learn something more from the var_off */
		__update_reg_bounds(dst_reg);
3864
		break;
3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887
	case BPF_ARSH:
		if (umax_val >= insn_bitness) {
			/* Shifts greater than 31 or 63 are undefined.
			 * This includes shifts by a negative number.
			 */
			mark_reg_unknown(env, regs, insn->dst_reg);
			break;
		}

		/* Upon reaching here, src_known is true and
		 * umax_val is equal to umin_val.
		 */
		dst_reg->smin_value >>= umin_val;
		dst_reg->smax_value >>= umin_val;
		dst_reg->var_off = tnum_arshift(dst_reg->var_off, umin_val);

		/* blow away the dst_reg umin_value/umax_value and rely on
		 * dst_reg var_off to refine the result.
		 */
		dst_reg->umin_value = 0;
		dst_reg->umax_value = U64_MAX;
		__update_reg_bounds(dst_reg);
		break;
3888
	default:
3889
		mark_reg_unknown(env, regs, insn->dst_reg);
3890 3891 3892
		break;
	}

3893 3894 3895 3896 3897
	if (BPF_CLASS(insn->code) != BPF_ALU64) {
		/* 32-bit ALU ops are (32,32)->32 */
		coerce_reg_to_size(dst_reg, 4);
	}

3898 3899
	__reg_deduce_bounds(dst_reg);
	__reg_bound_offset(dst_reg);
3900 3901 3902 3903 3904 3905 3906 3907 3908
	return 0;
}

/* Handles ALU ops other than BPF_END, BPF_NEG and BPF_MOV: computes new min/max
 * and var_off.
 */
static int adjust_reg_min_max_vals(struct bpf_verifier_env *env,
				   struct bpf_insn *insn)
{
3909 3910 3911
	struct bpf_verifier_state *vstate = env->cur_state;
	struct bpf_func_state *state = vstate->frame[vstate->curframe];
	struct bpf_reg_state *regs = state->regs, *dst_reg, *src_reg;
3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923
	struct bpf_reg_state *ptr_reg = NULL, off_reg = {0};
	u8 opcode = BPF_OP(insn->code);

	dst_reg = &regs[insn->dst_reg];
	src_reg = NULL;
	if (dst_reg->type != SCALAR_VALUE)
		ptr_reg = dst_reg;
	if (BPF_SRC(insn->code) == BPF_X) {
		src_reg = &regs[insn->src_reg];
		if (src_reg->type != SCALAR_VALUE) {
			if (dst_reg->type != SCALAR_VALUE) {
				/* Combining two pointers by any ALU op yields
3924 3925
				 * an arbitrary scalar. Disallow all math except
				 * pointer subtraction
3926
				 */
3927
				if (opcode == BPF_SUB && env->allow_ptr_leaks) {
3928 3929
					mark_reg_unknown(env, regs, insn->dst_reg);
					return 0;
3930
				}
3931 3932 3933 3934
				verbose(env, "R%d pointer %s pointer prohibited\n",
					insn->dst_reg,
					bpf_alu_string[opcode >> 4]);
				return -EACCES;
3935 3936 3937 3938 3939
			} else {
				/* scalar += pointer
				 * This is legal, but we have to reverse our
				 * src/dest handling in computing the range
				 */
3940 3941
				return adjust_ptr_min_max_vals(env, insn,
							       src_reg, dst_reg);
3942 3943 3944
			}
		} else if (ptr_reg) {
			/* pointer += scalar */
3945 3946
			return adjust_ptr_min_max_vals(env, insn,
						       dst_reg, src_reg);
3947 3948 3949 3950 3951 3952
		}
	} else {
		/* Pretend the src is a reg with a known value, since we only
		 * need to be able to read from this state.
		 */
		off_reg.type = SCALAR_VALUE;
3953
		__mark_reg_known(&off_reg, insn->imm);
3954
		src_reg = &off_reg;
3955 3956 3957
		if (ptr_reg) /* pointer += K */
			return adjust_ptr_min_max_vals(env, insn,
						       ptr_reg, src_reg);
3958 3959 3960 3961
	}

	/* Got here implies adding two SCALAR_VALUEs */
	if (WARN_ON_ONCE(ptr_reg)) {
3962
		print_verifier_state(env, state);
3963
		verbose(env, "verifier internal error: unexpected ptr_reg\n");
3964 3965 3966
		return -EINVAL;
	}
	if (WARN_ON(!src_reg)) {
3967
		print_verifier_state(env, state);
3968
		verbose(env, "verifier internal error: no src_reg\n");
3969 3970 3971
		return -EINVAL;
	}
	return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
3972 3973
}

3974
/* check validity of 32-bit and 64-bit arithmetic operations */
3975
static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
3976
{
3977
	struct bpf_reg_state *regs = cur_regs(env);
3978 3979 3980 3981 3982 3983 3984 3985
	u8 opcode = BPF_OP(insn->code);
	int err;

	if (opcode == BPF_END || opcode == BPF_NEG) {
		if (opcode == BPF_NEG) {
			if (BPF_SRC(insn->code) != 0 ||
			    insn->src_reg != BPF_REG_0 ||
			    insn->off != 0 || insn->imm != 0) {
3986
				verbose(env, "BPF_NEG uses reserved fields\n");
3987 3988 3989 3990
				return -EINVAL;
			}
		} else {
			if (insn->src_reg != BPF_REG_0 || insn->off != 0 ||
3991 3992
			    (insn->imm != 16 && insn->imm != 32 && insn->imm != 64) ||
			    BPF_CLASS(insn->code) == BPF_ALU64) {
3993
				verbose(env, "BPF_END uses reserved fields\n");
3994 3995 3996 3997 3998
				return -EINVAL;
			}
		}

		/* check src operand */
3999
		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
4000 4001 4002
		if (err)
			return err;

4003
		if (is_pointer_value(env, insn->dst_reg)) {
4004
			verbose(env, "R%d pointer arithmetic prohibited\n",
4005 4006 4007 4008
				insn->dst_reg);
			return -EACCES;
		}

4009
		/* check dest operand */
4010
		err = check_reg_arg(env, insn->dst_reg, DST_OP);
4011 4012 4013 4014 4015 4016 4017
		if (err)
			return err;

	} else if (opcode == BPF_MOV) {

		if (BPF_SRC(insn->code) == BPF_X) {
			if (insn->imm != 0 || insn->off != 0) {
4018
				verbose(env, "BPF_MOV uses reserved fields\n");
4019 4020 4021 4022
				return -EINVAL;
			}

			/* check src operand */
4023
			err = check_reg_arg(env, insn->src_reg, SRC_OP);
4024 4025 4026 4027
			if (err)
				return err;
		} else {
			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
4028
				verbose(env, "BPF_MOV uses reserved fields\n");
4029 4030 4031 4032
				return -EINVAL;
			}
		}

4033 4034
		/* check dest operand, mark as required later */
		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
4035 4036 4037 4038
		if (err)
			return err;

		if (BPF_SRC(insn->code) == BPF_X) {
4039 4040 4041
			struct bpf_reg_state *src_reg = regs + insn->src_reg;
			struct bpf_reg_state *dst_reg = regs + insn->dst_reg;

4042 4043 4044 4045
			if (BPF_CLASS(insn->code) == BPF_ALU64) {
				/* case: R1 = R2
				 * copy register state to dest reg
				 */
4046 4047
				*dst_reg = *src_reg;
				dst_reg->live |= REG_LIVE_WRITTEN;
4048
			} else {
4049
				/* R1 = (u32) R2 */
4050
				if (is_pointer_value(env, insn->src_reg)) {
4051 4052
					verbose(env,
						"R%d partial copy of pointer\n",
4053 4054
						insn->src_reg);
					return -EACCES;
4055 4056 4057 4058 4059 4060
				} else if (src_reg->type == SCALAR_VALUE) {
					*dst_reg = *src_reg;
					dst_reg->live |= REG_LIVE_WRITTEN;
				} else {
					mark_reg_unknown(env, regs,
							 insn->dst_reg);
4061
				}
4062
				coerce_reg_to_size(dst_reg, 4);
4063 4064 4065 4066 4067
			}
		} else {
			/* case: R = imm
			 * remember the value we stored into this reg
			 */
4068 4069
			/* clear any state __mark_reg_known doesn't set */
			mark_reg_unknown(env, regs, insn->dst_reg);
4070
			regs[insn->dst_reg].type = SCALAR_VALUE;
4071 4072 4073 4074 4075 4076 4077
			if (BPF_CLASS(insn->code) == BPF_ALU64) {
				__mark_reg_known(regs + insn->dst_reg,
						 insn->imm);
			} else {
				__mark_reg_known(regs + insn->dst_reg,
						 (u32)insn->imm);
			}
4078 4079 4080
		}

	} else if (opcode > BPF_END) {
4081
		verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
4082 4083 4084 4085 4086 4087
		return -EINVAL;

	} else {	/* all other ALU ops: and, sub, xor, add, ... */

		if (BPF_SRC(insn->code) == BPF_X) {
			if (insn->imm != 0 || insn->off != 0) {
4088
				verbose(env, "BPF_ALU uses reserved fields\n");
4089 4090 4091
				return -EINVAL;
			}
			/* check src1 operand */
4092
			err = check_reg_arg(env, insn->src_reg, SRC_OP);
4093 4094 4095 4096
			if (err)
				return err;
		} else {
			if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
4097
				verbose(env, "BPF_ALU uses reserved fields\n");
4098 4099 4100 4101 4102
				return -EINVAL;
			}
		}

		/* check src2 operand */
4103
		err = check_reg_arg(env, insn->dst_reg, SRC_OP);
4104 4105 4106 4107 4108
		if (err)
			return err;

		if ((opcode == BPF_MOD || opcode == BPF_DIV) &&
		    BPF_SRC(insn->code) == BPF_K && insn->imm == 0) {
4109
			verbose(env, "div by zero\n");
4110 4111 4112
			return -EINVAL;
		}

4113 4114 4115 4116 4117
		if ((opcode == BPF_LSH || opcode == BPF_RSH ||
		     opcode == BPF_ARSH) && BPF_SRC(insn->code) == BPF_K) {
			int size = BPF_CLASS(insn->code) == BPF_ALU64 ? 64 : 32;

			if (insn->imm < 0 || insn->imm >= size) {
4118
				verbose(env, "invalid shift %d\n", insn->imm);
4119 4120 4121 4122
				return -EINVAL;
			}
		}

4123
		/* check dest operand */
4124
		err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
4125 4126 4127
		if (err)
			return err;

4128
		return adjust_reg_min_max_vals(env, insn);
4129 4130 4131 4132 4133
	}

	return 0;
}

4134
static void find_good_pkt_pointers(struct bpf_verifier_state *vstate,
4135
				   struct bpf_reg_state *dst_reg,
4136
				   enum bpf_reg_type type,
4137
				   bool range_right_open)
4138
{
4139
	struct bpf_func_state *state = vstate->frame[vstate->curframe];
4140
	struct bpf_reg_state *regs = state->regs, *reg;
4141
	u16 new_range;
4142
	int i, j;
4143

4144 4145
	if (dst_reg->off < 0 ||
	    (dst_reg->off == 0 && range_right_open))
4146 4147 4148
		/* This doesn't give us any range */
		return;

4149 4150
	if (dst_reg->umax_value > MAX_PACKET_OFF ||
	    dst_reg->umax_value + dst_reg->off > MAX_PACKET_OFF)
4151 4152 4153 4154 4155
		/* Risk of overflow.  For instance, ptr + (1<<63) may be less
		 * than pkt_end, but that's because it's also less than pkt.
		 */
		return;

4156 4157 4158 4159 4160
	new_range = dst_reg->off;
	if (range_right_open)
		new_range--;

	/* Examples for register markings:
4161
	 *
4162
	 * pkt_data in dst register:
4163 4164 4165 4166 4167 4168
	 *
	 *   r2 = r3;
	 *   r2 += 8;
	 *   if (r2 > pkt_end) goto <handle exception>
	 *   <access okay>
	 *
4169 4170 4171 4172 4173
	 *   r2 = r3;
	 *   r2 += 8;
	 *   if (r2 < pkt_end) goto <access okay>
	 *   <handle exception>
	 *
4174 4175 4176 4177 4178
	 *   Where:
	 *     r2 == dst_reg, pkt_end == src_reg
	 *     r2=pkt(id=n,off=8,r=0)
	 *     r3=pkt(id=n,off=0,r=0)
	 *
4179
	 * pkt_data in src register:
4180 4181 4182 4183 4184 4185
	 *
	 *   r2 = r3;
	 *   r2 += 8;
	 *   if (pkt_end >= r2) goto <access okay>
	 *   <handle exception>
	 *
4186 4187 4188 4189 4190
	 *   r2 = r3;
	 *   r2 += 8;
	 *   if (pkt_end <= r2) goto <handle exception>
	 *   <access okay>
	 *
4191 4192 4193 4194 4195 4196
	 *   Where:
	 *     pkt_end == dst_reg, r2 == src_reg
	 *     r2=pkt(id=n,off=8,r=0)
	 *     r3=pkt(id=n,off=0,r=0)
	 *
	 * Find register r3 and mark its range as r3=pkt(id=n,off=0,r=8)
4197 4198 4199
	 * or r3=pkt(id=n,off=0,r=8-1), so that range of bytes [r3, r3 + 8)
	 * and [r3, r3 + 8-1) respectively is safe to access depending on
	 * the check.
4200
	 */
4201

4202 4203 4204 4205 4206
	/* If our ids match, then we must have the same max_value.  And we
	 * don't care about the other reg's fixed offset, since if it's too big
	 * the range won't allow anything.
	 * dst_reg->off is known < MAX_PACKET_OFF, therefore it fits in a u16.
	 */
4207
	for (i = 0; i < MAX_BPF_REG; i++)
4208
		if (regs[i].type == type && regs[i].id == dst_reg->id)
4209
			/* keep the maximum range already checked */
4210
			regs[i].range = max(regs[i].range, new_range);
4211

4212 4213
	for (j = 0; j <= vstate->curframe; j++) {
		state = vstate->frame[j];
4214 4215
		bpf_for_each_spilled_reg(i, state, reg) {
			if (!reg)
4216 4217 4218 4219
				continue;
			if (reg->type == type && reg->id == dst_reg->id)
				reg->range = max(reg->range, new_range);
		}
4220 4221 4222
	}
}

4223 4224 4225 4226 4227 4228
/* compute branch direction of the expression "if (reg opcode val) goto target;"
 * and return:
 *  1 - branch will be taken and "goto target" will be executed
 *  0 - branch will not be taken and fall-through to next insn
 * -1 - unknown. Example: "if (reg < 5)" is unknown when register value range [0,10]
 */
Jiong Wang's avatar
Jiong Wang committed
4229 4230
static int is_branch_taken(struct bpf_reg_state *reg, u64 val, u8 opcode,
			   bool is_jmp32)
4231
{
Jiong Wang's avatar
Jiong Wang committed
4232
	struct bpf_reg_state reg_lo;
4233 4234
	s64 sval;

4235 4236 4237
	if (__is_pointer_value(false, reg))
		return -1;

Jiong Wang's avatar
Jiong Wang committed
4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271
	if (is_jmp32) {
		reg_lo = *reg;
		reg = &reg_lo;
		/* For JMP32, only low 32 bits are compared, coerce_reg_to_size
		 * could truncate high bits and update umin/umax according to
		 * information of low bits.
		 */
		coerce_reg_to_size(reg, 4);
		/* smin/smax need special handling. For example, after coerce,
		 * if smin_value is 0x00000000ffffffffLL, the value is -1 when
		 * used as operand to JMP32. It is a negative number from s32's
		 * point of view, while it is a positive number when seen as
		 * s64. The smin/smax are kept as s64, therefore, when used with
		 * JMP32, they need to be transformed into s32, then sign
		 * extended back to s64.
		 *
		 * Also, smin/smax were copied from umin/umax. If umin/umax has
		 * different sign bit, then min/max relationship doesn't
		 * maintain after casting into s32, for this case, set smin/smax
		 * to safest range.
		 */
		if ((reg->umax_value ^ reg->umin_value) &
		    (1ULL << 31)) {
			reg->smin_value = S32_MIN;
			reg->smax_value = S32_MAX;
		}
		reg->smin_value = (s64)(s32)reg->smin_value;
		reg->smax_value = (s64)(s32)reg->smax_value;

		val = (u32)val;
		sval = (s64)(s32)val;
	} else {
		sval = (s64)val;
	}
4272

4273 4274 4275 4276 4277 4278 4279 4280 4281
	switch (opcode) {
	case BPF_JEQ:
		if (tnum_is_const(reg->var_off))
			return !!tnum_equals_const(reg->var_off, val);
		break;
	case BPF_JNE:
		if (tnum_is_const(reg->var_off))
			return !tnum_equals_const(reg->var_off, val);
		break;
4282 4283 4284 4285 4286 4287
	case BPF_JSET:
		if ((~reg->var_off.mask & reg->var_off.value) & val)
			return 1;
		if (!((reg->var_off.mask | reg->var_off.value) & val))
			return 0;
		break;
4288 4289 4290 4291 4292 4293 4294
	case BPF_JGT:
		if (reg->umin_value > val)
			return 1;
		else if (reg->umax_value <= val)
			return 0;
		break;
	case BPF_JSGT:
4295
		if (reg->smin_value > sval)
4296
			return 1;
4297
		else if (reg->smax_value < sval)
4298 4299 4300 4301 4302 4303 4304 4305 4306
			return 0;
		break;
	case BPF_JLT:
		if (reg->umax_value < val)
			return 1;
		else if (reg->umin_value >= val)
			return 0;
		break;
	case BPF_JSLT:
4307
		if (reg->smax_value < sval)
4308
			return 1;
4309
		else if (reg->smin_value >= sval)
4310 4311 4312 4313 4314 4315 4316 4317 4318
			return 0;
		break;
	case BPF_JGE:
		if (reg->umin_value >= val)
			return 1;
		else if (reg->umax_value < val)
			return 0;
		break;
	case BPF_JSGE:
4319
		if (reg->smin_value >= sval)
4320
			return 1;
4321
		else if (reg->smax_value < sval)
4322 4323 4324 4325 4326 4327 4328 4329 4330
			return 0;
		break;
	case BPF_JLE:
		if (reg->umax_value <= val)
			return 1;
		else if (reg->umin_value > val)
			return 0;
		break;
	case BPF_JSLE:
4331
		if (reg->smax_value <= sval)
4332
			return 1;
4333
		else if (reg->smin_value > sval)
4334 4335 4336 4337 4338 4339 4340
			return 0;
		break;
	}

	return -1;
}

Jiong Wang's avatar
Jiong Wang committed
4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363
/* Generate min value of the high 32-bit from TNUM info. */
static u64 gen_hi_min(struct tnum var)
{
	return var.value & ~0xffffffffULL;
}

/* Generate max value of the high 32-bit from TNUM info. */
static u64 gen_hi_max(struct tnum var)
{
	return (var.value | var.mask) & ~0xffffffffULL;
}

/* Return true if VAL is compared with a s64 sign extended from s32, and they
 * are with the same signedness.
 */
static bool cmp_val_with_extended_s64(s64 sval, struct bpf_reg_state *reg)
{
	return ((s32)sval >= 0 &&
		reg->smin_value >= 0 && reg->smax_value <= S32_MAX) ||
	       ((s32)sval < 0 &&
		reg->smax_value <= 0 && reg->smin_value >= S32_MIN);
}

4364 4365 4366
/* Adjusts the register min/max values in the case that the dst_reg is the
 * variable register that we are working on, and src_reg is a constant or we're
 * simply doing a BPF_K check.
4367
 * In JEQ/JNE cases we also adjust the var_off values.
4368 4369 4370
 */
static void reg_set_min_max(struct bpf_reg_state *true_reg,
			    struct bpf_reg_state *false_reg, u64 val,
Jiong Wang's avatar
Jiong Wang committed
4371
			    u8 opcode, bool is_jmp32)
4372
{
4373 4374
	s64 sval;

4375 4376 4377 4378 4379 4380 4381 4382
	/* If the dst_reg is a pointer, we can't learn anything about its
	 * variable offset from the compare (unless src_reg were a pointer into
	 * the same object, but we don't bother with that.
	 * Since false_reg and true_reg have the same type by construction, we
	 * only need to check one of them for pointerness.
	 */
	if (__is_pointer_value(false, false_reg))
		return;
4383

Jiong Wang's avatar
Jiong Wang committed
4384 4385
	val = is_jmp32 ? (u32)val : val;
	sval = is_jmp32 ? (s64)(s32)val : (s64)val;
4386

4387 4388 4389
	switch (opcode) {
	case BPF_JEQ:
	case BPF_JNE:
4390 4391 4392 4393 4394 4395 4396
	{
		struct bpf_reg_state *reg =
			opcode == BPF_JEQ ? true_reg : false_reg;

		/* For BPF_JEQ, if this is false we know nothing Jon Snow, but
		 * if it is true we know the value for sure. Likewise for
		 * BPF_JNE.
4397
		 */
Jiong Wang's avatar
Jiong Wang committed
4398 4399 4400 4401 4402 4403 4404 4405 4406
		if (is_jmp32) {
			u64 old_v = reg->var_off.value;
			u64 hi_mask = ~0xffffffffULL;

			reg->var_off.value = (old_v & hi_mask) | val;
			reg->var_off.mask &= hi_mask;
		} else {
			__mark_reg_known(reg, val);
		}
4407
		break;
4408
	}
4409 4410 4411 4412 4413 4414 4415
	case BPF_JSET:
		false_reg->var_off = tnum_and(false_reg->var_off,
					      tnum_const(~val));
		if (is_power_of_2(val))
			true_reg->var_off = tnum_or(true_reg->var_off,
						    tnum_const(val));
		break;
4416
	case BPF_JGE:
4417 4418 4419 4420 4421
	case BPF_JGT:
	{
		u64 false_umax = opcode == BPF_JGT ? val    : val - 1;
		u64 true_umin = opcode == BPF_JGT ? val + 1 : val;

Jiong Wang's avatar
Jiong Wang committed
4422 4423 4424 4425
		if (is_jmp32) {
			false_umax += gen_hi_max(false_reg->var_off);
			true_umin += gen_hi_min(true_reg->var_off);
		}
4426 4427
		false_reg->umax_value = min(false_reg->umax_value, false_umax);
		true_reg->umin_value = max(true_reg->umin_value, true_umin);
4428
		break;
4429
	}
4430
	case BPF_JSGE:
4431 4432 4433 4434 4435
	case BPF_JSGT:
	{
		s64 false_smax = opcode == BPF_JSGT ? sval    : sval - 1;
		s64 true_smin = opcode == BPF_JSGT ? sval + 1 : sval;

Jiong Wang's avatar
Jiong Wang committed
4436 4437 4438 4439 4440
		/* If the full s64 was not sign-extended from s32 then don't
		 * deduct further info.
		 */
		if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
			break;
4441 4442
		false_reg->smax_value = min(false_reg->smax_value, false_smax);
		true_reg->smin_value = max(true_reg->smin_value, true_smin);
4443
		break;
4444
	}
4445
	case BPF_JLE:
4446 4447 4448 4449 4450
	case BPF_JLT:
	{
		u64 false_umin = opcode == BPF_JLT ? val    : val + 1;
		u64 true_umax = opcode == BPF_JLT ? val - 1 : val;

Jiong Wang's avatar
Jiong Wang committed
4451 4452 4453 4454
		if (is_jmp32) {
			false_umin += gen_hi_min(false_reg->var_off);
			true_umax += gen_hi_max(true_reg->var_off);
		}
4455 4456
		false_reg->umin_value = max(false_reg->umin_value, false_umin);
		true_reg->umax_value = min(true_reg->umax_value, true_umax);
4457
		break;
4458
	}
4459
	case BPF_JSLE:
4460 4461 4462 4463 4464
	case BPF_JSLT:
	{
		s64 false_smin = opcode == BPF_JSLT ? sval    : sval + 1;
		s64 true_smax = opcode == BPF_JSLT ? sval - 1 : sval;

Jiong Wang's avatar
Jiong Wang committed
4465 4466
		if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
			break;
4467 4468
		false_reg->smin_value = max(false_reg->smin_value, false_smin);
		true_reg->smax_value = min(true_reg->smax_value, true_smax);
4469
		break;
4470
	}
4471 4472 4473 4474
	default:
		break;
	}

4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485
	__reg_deduce_bounds(false_reg);
	__reg_deduce_bounds(true_reg);
	/* We might have learned some bits from the bounds. */
	__reg_bound_offset(false_reg);
	__reg_bound_offset(true_reg);
	/* Intersecting with the old var_off might have improved our bounds
	 * slightly.  e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
	 * then new var_off is (0; 0x7f...fc) which improves our umax.
	 */
	__update_reg_bounds(false_reg);
	__update_reg_bounds(true_reg);
4486 4487
}

4488 4489
/* Same as above, but for the case that dst_reg holds a constant and src_reg is
 * the variable reg.
4490 4491 4492
 */
static void reg_set_min_max_inv(struct bpf_reg_state *true_reg,
				struct bpf_reg_state *false_reg, u64 val,
Jiong Wang's avatar
Jiong Wang committed
4493
				u8 opcode, bool is_jmp32)
4494
{
4495 4496
	s64 sval;

4497 4498
	if (__is_pointer_value(false, false_reg))
		return;
4499

Jiong Wang's avatar
Jiong Wang committed
4500 4501
	val = is_jmp32 ? (u32)val : val;
	sval = is_jmp32 ? (s64)(s32)val : (s64)val;
4502

4503 4504 4505
	switch (opcode) {
	case BPF_JEQ:
	case BPF_JNE:
4506 4507 4508 4509
	{
		struct bpf_reg_state *reg =
			opcode == BPF_JEQ ? true_reg : false_reg;

Jiong Wang's avatar
Jiong Wang committed
4510 4511 4512 4513 4514 4515 4516 4517 4518
		if (is_jmp32) {
			u64 old_v = reg->var_off.value;
			u64 hi_mask = ~0xffffffffULL;

			reg->var_off.value = (old_v & hi_mask) | val;
			reg->var_off.mask &= hi_mask;
		} else {
			__mark_reg_known(reg, val);
		}
4519
		break;
4520
	}
4521 4522 4523 4524 4525 4526 4527
	case BPF_JSET:
		false_reg->var_off = tnum_and(false_reg->var_off,
					      tnum_const(~val));
		if (is_power_of_2(val))
			true_reg->var_off = tnum_or(true_reg->var_off,
						    tnum_const(val));
		break;
4528
	case BPF_JGE:
4529 4530 4531 4532 4533
	case BPF_JGT:
	{
		u64 false_umin = opcode == BPF_JGT ? val    : val + 1;
		u64 true_umax = opcode == BPF_JGT ? val - 1 : val;

Jiong Wang's avatar
Jiong Wang committed
4534 4535 4536 4537
		if (is_jmp32) {
			false_umin += gen_hi_min(false_reg->var_off);
			true_umax += gen_hi_max(true_reg->var_off);
		}
4538 4539
		false_reg->umin_value = max(false_reg->umin_value, false_umin);
		true_reg->umax_value = min(true_reg->umax_value, true_umax);
4540
		break;
4541
	}
4542
	case BPF_JSGE:
4543 4544 4545 4546 4547
	case BPF_JSGT:
	{
		s64 false_smin = opcode == BPF_JSGT ? sval    : sval + 1;
		s64 true_smax = opcode == BPF_JSGT ? sval - 1 : sval;

Jiong Wang's avatar
Jiong Wang committed
4548 4549
		if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
			break;
4550 4551
		false_reg->smin_value = max(false_reg->smin_value, false_smin);
		true_reg->smax_value = min(true_reg->smax_value, true_smax);
4552
		break;
4553
	}
4554
	case BPF_JLE:
4555 4556 4557 4558 4559
	case BPF_JLT:
	{
		u64 false_umax = opcode == BPF_JLT ? val    : val - 1;
		u64 true_umin = opcode == BPF_JLT ? val + 1 : val;

Jiong Wang's avatar
Jiong Wang committed
4560 4561 4562 4563
		if (is_jmp32) {
			false_umax += gen_hi_max(false_reg->var_off);
			true_umin += gen_hi_min(true_reg->var_off);
		}
4564 4565
		false_reg->umax_value = min(false_reg->umax_value, false_umax);
		true_reg->umin_value = max(true_reg->umin_value, true_umin);
4566
		break;
4567
	}
4568
	case BPF_JSLE:
4569 4570 4571 4572 4573
	case BPF_JSLT:
	{
		s64 false_smax = opcode == BPF_JSLT ? sval    : sval - 1;
		s64 true_smin = opcode == BPF_JSLT ? sval + 1 : sval;

Jiong Wang's avatar
Jiong Wang committed
4574 4575
		if (is_jmp32 && !cmp_val_with_extended_s64(sval, false_reg))
			break;
4576 4577
		false_reg->smax_value = min(false_reg->smax_value, false_smax);
		true_reg->smin_value = max(true_reg->smin_value, true_smin);
4578
		break;
4579
	}
4580 4581 4582 4583
	default:
		break;
	}

4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594
	__reg_deduce_bounds(false_reg);
	__reg_deduce_bounds(true_reg);
	/* We might have learned some bits from the bounds. */
	__reg_bound_offset(false_reg);
	__reg_bound_offset(true_reg);
	/* Intersecting with the old var_off might have improved our bounds
	 * slightly.  e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
	 * then new var_off is (0; 0x7f...fc) which improves our umax.
	 */
	__update_reg_bounds(false_reg);
	__update_reg_bounds(true_reg);
4595 4596 4597 4598 4599 4600
}

/* Regs are known to be equal, so intersect their min/max/var_off */
static void __reg_combine_min_max(struct bpf_reg_state *src_reg,
				  struct bpf_reg_state *dst_reg)
{
4601 4602 4603 4604 4605 4606 4607 4608
	src_reg->umin_value = dst_reg->umin_value = max(src_reg->umin_value,
							dst_reg->umin_value);
	src_reg->umax_value = dst_reg->umax_value = min(src_reg->umax_value,
							dst_reg->umax_value);
	src_reg->smin_value = dst_reg->smin_value = max(src_reg->smin_value,
							dst_reg->smin_value);
	src_reg->smax_value = dst_reg->smax_value = min(src_reg->smax_value,
							dst_reg->smax_value);
4609 4610
	src_reg->var_off = dst_reg->var_off = tnum_intersect(src_reg->var_off,
							     dst_reg->var_off);
4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625
	/* We might have learned new bounds from the var_off. */
	__update_reg_bounds(src_reg);
	__update_reg_bounds(dst_reg);
	/* We might have learned something about the sign bit. */
	__reg_deduce_bounds(src_reg);
	__reg_deduce_bounds(dst_reg);
	/* We might have learned some bits from the bounds. */
	__reg_bound_offset(src_reg);
	__reg_bound_offset(dst_reg);
	/* Intersecting with the old var_off might have improved our bounds
	 * slightly.  e.g. if umax was 0x7f...f and var_off was (0; 0xf...fc),
	 * then new var_off is (0; 0x7f...fc) which improves our umax.
	 */
	__update_reg_bounds(src_reg);
	__update_reg_bounds(dst_reg);
4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639
}

static void reg_combine_min_max(struct bpf_reg_state *true_src,
				struct bpf_reg_state *true_dst,
				struct bpf_reg_state *false_src,
				struct bpf_reg_state *false_dst,
				u8 opcode)
{
	switch (opcode) {
	case BPF_JEQ:
		__reg_combine_min_max(true_src, true_dst);
		break;
	case BPF_JNE:
		__reg_combine_min_max(false_src, false_dst);
4640
		break;
4641
	}
4642 4643
}

4644 4645
static void mark_ptr_or_null_reg(struct bpf_func_state *state,
				 struct bpf_reg_state *reg, u32 id,
4646
				 bool is_null)
4647
{
4648
	if (reg_type_may_be_null(reg->type) && reg->id == id) {
4649 4650 4651 4652
		/* Old offset (both fixed and variable parts) should
		 * have been known-zero, because we don't allow pointer
		 * arithmetic on pointers that might be NULL.
		 */
4653 4654
		if (WARN_ON_ONCE(reg->smin_value || reg->smax_value ||
				 !tnum_equals_const(reg->var_off, 0) ||
4655
				 reg->off)) {
4656 4657
			__mark_reg_known_zero(reg);
			reg->off = 0;
4658 4659 4660
		}
		if (is_null) {
			reg->type = SCALAR_VALUE;
4661 4662 4663 4664 4665 4666 4667
		} else if (reg->type == PTR_TO_MAP_VALUE_OR_NULL) {
			if (reg->map_ptr->inner_map_meta) {
				reg->type = CONST_PTR_TO_MAP;
				reg->map_ptr = reg->map_ptr->inner_map_meta;
			} else {
				reg->type = PTR_TO_MAP_VALUE;
			}
4668 4669
		} else if (reg->type == PTR_TO_SOCKET_OR_NULL) {
			reg->type = PTR_TO_SOCKET;
4670 4671
		} else if (reg->type == PTR_TO_SOCK_COMMON_OR_NULL) {
			reg->type = PTR_TO_SOCK_COMMON;
4672 4673
		} else if (reg->type == PTR_TO_TCP_SOCK_OR_NULL) {
			reg->type = PTR_TO_TCP_SOCK;
4674
		}
4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685 4686 4687
		if (is_null) {
			/* We don't need id and ref_obj_id from this point
			 * onwards anymore, thus we should better reset it,
			 * so that state pruning has chances to take effect.
			 */
			reg->id = 0;
			reg->ref_obj_id = 0;
		} else if (!reg_may_point_to_spin_lock(reg)) {
			/* For not-NULL ptr, reg->ref_obj_id will be reset
			 * in release_reg_references().
			 *
			 * reg->id is still used by spin_lock ptr. Other
			 * than spin_lock ptr type, reg->id can be reset.
4688 4689
			 */
			reg->id = 0;
4690
		}
4691 4692 4693 4694 4695 4696
	}
}

/* The logic is similar to find_good_pkt_pointers(), both could eventually
 * be folded together at some point.
 */
4697 4698
static void mark_ptr_or_null_regs(struct bpf_verifier_state *vstate, u32 regno,
				  bool is_null)
4699
{
4700
	struct bpf_func_state *state = vstate->frame[vstate->curframe];
4701
	struct bpf_reg_state *reg, *regs = state->regs;
4702
	u32 ref_obj_id = regs[regno].ref_obj_id;
4703
	u32 id = regs[regno].id;
4704
	int i, j;
4705

4706 4707 4708 4709 4710 4711
	if (ref_obj_id && ref_obj_id == id && is_null)
		/* regs[regno] is in the " == NULL" branch.
		 * No one could have freed the reference state before
		 * doing the NULL check.
		 */
		WARN_ON_ONCE(release_reference_state(state, id));
4712

4713
	for (i = 0; i < MAX_BPF_REG; i++)
4714
		mark_ptr_or_null_reg(state, &regs[i], id, is_null);
4715

4716 4717
	for (j = 0; j <= vstate->curframe; j++) {
		state = vstate->frame[j];
4718 4719
		bpf_for_each_spilled_reg(i, state, reg) {
			if (!reg)
4720
				continue;
4721
			mark_ptr_or_null_reg(state, reg, id, is_null);
4722
		}
4723 4724 4725
	}
}

4726 4727 4728 4729 4730 4731 4732 4733 4734
static bool try_match_pkt_pointers(const struct bpf_insn *insn,
				   struct bpf_reg_state *dst_reg,
				   struct bpf_reg_state *src_reg,
				   struct bpf_verifier_state *this_branch,
				   struct bpf_verifier_state *other_branch)
{
	if (BPF_SRC(insn->code) != BPF_X)
		return false;

Jiong Wang's avatar
Jiong Wang committed
4735 4736 4737 4738
	/* Pointers are always 64-bit. */
	if (BPF_CLASS(insn->code) == BPF_JMP32)
		return false;

4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775 4776 4777 4778 4779 4780 4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819 4820 4821 4822
	switch (BPF_OP(insn->code)) {
	case BPF_JGT:
		if ((dst_reg->type == PTR_TO_PACKET &&
		     src_reg->type == PTR_TO_PACKET_END) ||
		    (dst_reg->type == PTR_TO_PACKET_META &&
		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
			/* pkt_data' > pkt_end, pkt_meta' > pkt_data */
			find_good_pkt_pointers(this_branch, dst_reg,
					       dst_reg->type, false);
		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
			    src_reg->type == PTR_TO_PACKET) ||
			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
			    src_reg->type == PTR_TO_PACKET_META)) {
			/* pkt_end > pkt_data', pkt_data > pkt_meta' */
			find_good_pkt_pointers(other_branch, src_reg,
					       src_reg->type, true);
		} else {
			return false;
		}
		break;
	case BPF_JLT:
		if ((dst_reg->type == PTR_TO_PACKET &&
		     src_reg->type == PTR_TO_PACKET_END) ||
		    (dst_reg->type == PTR_TO_PACKET_META &&
		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
			/* pkt_data' < pkt_end, pkt_meta' < pkt_data */
			find_good_pkt_pointers(other_branch, dst_reg,
					       dst_reg->type, true);
		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
			    src_reg->type == PTR_TO_PACKET) ||
			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
			    src_reg->type == PTR_TO_PACKET_META)) {
			/* pkt_end < pkt_data', pkt_data > pkt_meta' */
			find_good_pkt_pointers(this_branch, src_reg,
					       src_reg->type, false);
		} else {
			return false;
		}
		break;
	case BPF_JGE:
		if ((dst_reg->type == PTR_TO_PACKET &&
		     src_reg->type == PTR_TO_PACKET_END) ||
		    (dst_reg->type == PTR_TO_PACKET_META &&
		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
			/* pkt_data' >= pkt_end, pkt_meta' >= pkt_data */
			find_good_pkt_pointers(this_branch, dst_reg,
					       dst_reg->type, true);
		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
			    src_reg->type == PTR_TO_PACKET) ||
			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
			    src_reg->type == PTR_TO_PACKET_META)) {
			/* pkt_end >= pkt_data', pkt_data >= pkt_meta' */
			find_good_pkt_pointers(other_branch, src_reg,
					       src_reg->type, false);
		} else {
			return false;
		}
		break;
	case BPF_JLE:
		if ((dst_reg->type == PTR_TO_PACKET &&
		     src_reg->type == PTR_TO_PACKET_END) ||
		    (dst_reg->type == PTR_TO_PACKET_META &&
		     reg_is_init_pkt_pointer(src_reg, PTR_TO_PACKET))) {
			/* pkt_data' <= pkt_end, pkt_meta' <= pkt_data */
			find_good_pkt_pointers(other_branch, dst_reg,
					       dst_reg->type, false);
		} else if ((dst_reg->type == PTR_TO_PACKET_END &&
			    src_reg->type == PTR_TO_PACKET) ||
			   (reg_is_init_pkt_pointer(dst_reg, PTR_TO_PACKET) &&
			    src_reg->type == PTR_TO_PACKET_META)) {
			/* pkt_end <= pkt_data', pkt_data <= pkt_meta' */
			find_good_pkt_pointers(this_branch, src_reg,
					       src_reg->type, true);
		} else {
			return false;
		}
		break;
	default:
		return false;
	}

	return true;
}

4823
static int check_cond_jmp_op(struct bpf_verifier_env *env,
4824 4825
			     struct bpf_insn *insn, int *insn_idx)
{
4826 4827 4828 4829
	struct bpf_verifier_state *this_branch = env->cur_state;
	struct bpf_verifier_state *other_branch;
	struct bpf_reg_state *regs = this_branch->frame[this_branch->curframe]->regs;
	struct bpf_reg_state *dst_reg, *other_branch_regs;
4830
	u8 opcode = BPF_OP(insn->code);
Jiong Wang's avatar
Jiong Wang committed
4831
	bool is_jmp32;
4832 4833
	int err;

Jiong Wang's avatar
Jiong Wang committed
4834 4835 4836
	/* Only conditional jumps are expected to reach here. */
	if (opcode == BPF_JA || opcode > BPF_JSLE) {
		verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
4837 4838 4839 4840 4841
		return -EINVAL;
	}

	if (BPF_SRC(insn->code) == BPF_X) {
		if (insn->imm != 0) {
Jiong Wang's avatar
Jiong Wang committed
4842
			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
4843 4844 4845 4846
			return -EINVAL;
		}

		/* check src1 operand */
4847
		err = check_reg_arg(env, insn->src_reg, SRC_OP);
4848 4849
		if (err)
			return err;
4850 4851

		if (is_pointer_value(env, insn->src_reg)) {
4852
			verbose(env, "R%d pointer comparison prohibited\n",
4853 4854 4855
				insn->src_reg);
			return -EACCES;
		}
4856 4857
	} else {
		if (insn->src_reg != BPF_REG_0) {
Jiong Wang's avatar
Jiong Wang committed
4858
			verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
4859 4860 4861 4862 4863
			return -EINVAL;
		}
	}

	/* check src2 operand */
4864
	err = check_reg_arg(env, insn->dst_reg, SRC_OP);
4865 4866 4867
	if (err)
		return err;

4868
	dst_reg = &regs[insn->dst_reg];
Jiong Wang's avatar
Jiong Wang committed
4869
	is_jmp32 = BPF_CLASS(insn->code) == BPF_JMP32;
4870

4871
	if (BPF_SRC(insn->code) == BPF_K) {
Jiong Wang's avatar
Jiong Wang committed
4872 4873
		int pred = is_branch_taken(dst_reg, insn->imm, opcode,
					   is_jmp32);
4874 4875 4876

		if (pred == 1) {
			 /* only follow the goto, ignore fall-through */
4877 4878
			*insn_idx += insn->off;
			return 0;
4879 4880
		} else if (pred == 0) {
			/* only follow fall-through branch, since
4881 4882 4883 4884 4885 4886
			 * that's where the program will go
			 */
			return 0;
		}
	}

4887 4888
	other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx,
				  false);
4889 4890
	if (!other_branch)
		return -EFAULT;
4891
	other_branch_regs = other_branch->frame[other_branch->curframe]->regs;
4892

4893 4894
	/* detect if we are comparing against a constant value so we can adjust
	 * our min/max values for our dst register.
4895 4896 4897 4898
	 * this is only legit if both are scalars (or pointers to the same
	 * object, I suppose, but we don't support that right now), because
	 * otherwise the different base pointers mean the offsets aren't
	 * comparable.
4899 4900
	 */
	if (BPF_SRC(insn->code) == BPF_X) {
Jiong Wang's avatar
Jiong Wang committed
4901 4902 4903 4904 4905 4906 4907 4908 4909 4910
		struct bpf_reg_state *src_reg = &regs[insn->src_reg];
		struct bpf_reg_state lo_reg0 = *dst_reg;
		struct bpf_reg_state lo_reg1 = *src_reg;
		struct bpf_reg_state *src_lo, *dst_lo;

		dst_lo = &lo_reg0;
		src_lo = &lo_reg1;
		coerce_reg_to_size(dst_lo, 4);
		coerce_reg_to_size(src_lo, 4);

4911
		if (dst_reg->type == SCALAR_VALUE &&
Jiong Wang's avatar
Jiong Wang committed
4912 4913 4914
		    src_reg->type == SCALAR_VALUE) {
			if (tnum_is_const(src_reg->var_off) ||
			    (is_jmp32 && tnum_is_const(src_lo->var_off)))
4915
				reg_set_min_max(&other_branch_regs[insn->dst_reg],
Jiong Wang's avatar
Jiong Wang committed
4916 4917 4918 4919 4920 4921 4922
						dst_reg,
						is_jmp32
						? src_lo->var_off.value
						: src_reg->var_off.value,
						opcode, is_jmp32);
			else if (tnum_is_const(dst_reg->var_off) ||
				 (is_jmp32 && tnum_is_const(dst_lo->var_off)))
4923
				reg_set_min_max_inv(&other_branch_regs[insn->src_reg],
Jiong Wang's avatar
Jiong Wang committed
4924 4925 4926 4927 4928 4929 4930
						    src_reg,
						    is_jmp32
						    ? dst_lo->var_off.value
						    : dst_reg->var_off.value,
						    opcode, is_jmp32);
			else if (!is_jmp32 &&
				 (opcode == BPF_JEQ || opcode == BPF_JNE))
4931
				/* Comparing for equality, we can combine knowledge */
4932 4933
				reg_combine_min_max(&other_branch_regs[insn->src_reg],
						    &other_branch_regs[insn->dst_reg],
Jiong Wang's avatar
Jiong Wang committed
4934
						    src_reg, dst_reg, opcode);
4935 4936
		}
	} else if (dst_reg->type == SCALAR_VALUE) {
4937
		reg_set_min_max(&other_branch_regs[insn->dst_reg],
Jiong Wang's avatar
Jiong Wang committed
4938
					dst_reg, insn->imm, opcode, is_jmp32);
4939 4940
	}

Jiong Wang's avatar
Jiong Wang committed
4941 4942 4943 4944 4945
	/* detect if R == 0 where R is returned from bpf_map_lookup_elem().
	 * NOTE: these optimizations below are related with pointer comparison
	 *       which will never be JMP32.
	 */
	if (!is_jmp32 && BPF_SRC(insn->code) == BPF_K &&
4946
	    insn->imm == 0 && (opcode == BPF_JEQ || opcode == BPF_JNE) &&
4947 4948
	    reg_type_may_be_null(dst_reg->type)) {
		/* Mark all identical registers in each branch as either
4949 4950
		 * safe or unknown depending R == 0 or R != 0 conditional.
		 */
4951 4952 4953 4954
		mark_ptr_or_null_regs(this_branch, insn->dst_reg,
				      opcode == BPF_JNE);
		mark_ptr_or_null_regs(other_branch, insn->dst_reg,
				      opcode == BPF_JEQ);
4955 4956 4957
	} else if (!try_match_pkt_pointers(insn, dst_reg, &regs[insn->src_reg],
					   this_branch, other_branch) &&
		   is_pointer_value(env, insn->dst_reg)) {
4958 4959
		verbose(env, "R%d pointer comparison prohibited\n",
			insn->dst_reg);
4960
		return -EACCES;
4961
	}
4962
	if (env->log.level)
4963
		print_verifier_state(env, this_branch->frame[this_branch->curframe]);
4964 4965 4966
	return 0;
}

4967 4968 4969 4970 4971 4972 4973 4974
/* return the map pointer stored inside BPF_LD_IMM64 instruction */
static struct bpf_map *ld_imm64_to_map_ptr(struct bpf_insn *insn)
{
	u64 imm64 = ((u64) (u32) insn[0].imm) | ((u64) (u32) insn[1].imm) << 32;

	return (struct bpf_map *) (unsigned long) imm64;
}

4975
/* verify BPF_LD_IMM64 instruction */
4976
static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
4977
{
4978
	struct bpf_reg_state *regs = cur_regs(env);
4979 4980 4981
	int err;

	if (BPF_SIZE(insn->code) != BPF_DW) {
4982
		verbose(env, "invalid BPF_LD_IMM insn\n");
4983 4984 4985
		return -EINVAL;
	}
	if (insn->off != 0) {
4986
		verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
4987 4988 4989
		return -EINVAL;
	}

4990
	err = check_reg_arg(env, insn->dst_reg, DST_OP);
4991 4992 4993
	if (err)
		return err;

4994 4995 4996
	if (insn->src_reg == 0) {
		u64 imm = ((u64)(insn + 1)->imm << 32) | (u32)insn->imm;

4997
		regs[insn->dst_reg].type = SCALAR_VALUE;
4998
		__mark_reg_known(&regs[insn->dst_reg], imm);
4999
		return 0;
5000
	}
5001 5002 5003 5004 5005 5006 5007 5008 5009

	/* replace_map_fd_with_map_ptr() should have caught bad ld_imm64 */
	BUG_ON(insn->src_reg != BPF_PSEUDO_MAP_FD);

	regs[insn->dst_reg].type = CONST_PTR_TO_MAP;
	regs[insn->dst_reg].map_ptr = ld_imm64_to_map_ptr(insn);
	return 0;
}

5010 5011 5012 5013 5014
static bool may_access_skb(enum bpf_prog_type type)
{
	switch (type) {
	case BPF_PROG_TYPE_SOCKET_FILTER:
	case BPF_PROG_TYPE_SCHED_CLS:
5015
	case BPF_PROG_TYPE_SCHED_ACT:
5016 5017 5018 5019 5020 5021
		return true;
	default:
		return false;
	}
}

5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036
/* verify safety of LD_ABS|LD_IND instructions:
 * - they can only appear in the programs where ctx == skb
 * - since they are wrappers of function calls, they scratch R1-R5 registers,
 *   preserve R6-R9, and store return value into R0
 *
 * Implicit input:
 *   ctx == skb == R6 == CTX
 *
 * Explicit input:
 *   SRC == any register
 *   IMM == 32-bit immediate
 *
 * Output:
 *   R0 - 8/16/32-bit skb data converted to cpu endianness
 */
5037
static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
5038
{
5039
	struct bpf_reg_state *regs = cur_regs(env);
5040 5041 5042
	u8 mode = BPF_MODE(insn->code);
	int i, err;

5043
	if (!may_access_skb(env->prog->type)) {
5044
		verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
5045 5046 5047
		return -EINVAL;
	}

5048 5049 5050 5051 5052
	if (!env->ops->gen_ld_abs) {
		verbose(env, "bpf verifier is misconfigured\n");
		return -EINVAL;
	}

5053
	if (env->subprog_cnt > 1) {
5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064
		/* when program has LD_ABS insn JITs and interpreter assume
		 * that r1 == ctx == skb which is not the case for callees
		 * that can have arbitrary arguments. It's problematic
		 * for main prog as well since JITs would need to analyze
		 * all functions in order to make proper register save/restore
		 * decisions in the main prog. Hence disallow LD_ABS with calls
		 */
		verbose(env, "BPF_LD_[ABS|IND] instructions cannot be mixed with bpf-to-bpf calls\n");
		return -EINVAL;
	}

5065
	if (insn->dst_reg != BPF_REG_0 || insn->off != 0 ||
5066
	    BPF_SIZE(insn->code) == BPF_DW ||
5067
	    (mode == BPF_ABS && insn->src_reg != BPF_REG_0)) {
5068
		verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
5069 5070 5071 5072
		return -EINVAL;
	}

	/* check whether implicit source operand (register R6) is readable */
5073
	err = check_reg_arg(env, BPF_REG_6, SRC_OP);
5074 5075 5076
	if (err)
		return err;

5077 5078 5079 5080 5081 5082 5083 5084 5085 5086
	/* Disallow usage of BPF_LD_[ABS|IND] with reference tracking, as
	 * gen_ld_abs() may terminate the program at runtime, leading to
	 * reference leak.
	 */
	err = check_reference_leak(env);
	if (err) {
		verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
		return err;
	}

5087 5088 5089 5090 5091
	if (env->cur_state->active_spin_lock) {
		verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
		return -EINVAL;
	}

5092
	if (regs[BPF_REG_6].type != PTR_TO_CTX) {
5093 5094
		verbose(env,
			"at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
5095 5096 5097 5098 5099
		return -EINVAL;
	}

	if (mode == BPF_IND) {
		/* check explicit source operand */
5100
		err = check_reg_arg(env, insn->src_reg, SRC_OP);
5101 5102 5103 5104 5105
		if (err)
			return err;
	}

	/* reset caller saved regs to unreadable */
5106
	for (i = 0; i < CALLER_SAVED_REGS; i++) {
5107
		mark_reg_not_init(env, regs, caller_saved[i]);
5108 5109
		check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
	}
5110 5111

	/* mark destination R0 register as readable, since it contains
5112 5113
	 * the value fetched from the packet.
	 * Already marked as written above.
5114
	 */
5115
	mark_reg_unknown(env, regs, BPF_REG_0);
5116 5117 5118
	return 0;
}

5119 5120 5121 5122 5123 5124 5125 5126
static int check_return_code(struct bpf_verifier_env *env)
{
	struct bpf_reg_state *reg;
	struct tnum range = tnum_range(0, 1);

	switch (env->prog->type) {
	case BPF_PROG_TYPE_CGROUP_SKB:
	case BPF_PROG_TYPE_CGROUP_SOCK:
Andrey Ignatov's avatar
Andrey Ignatov committed
5127
	case BPF_PROG_TYPE_CGROUP_SOCK_ADDR:
5128
	case BPF_PROG_TYPE_SOCK_OPS:
5129
	case BPF_PROG_TYPE_CGROUP_DEVICE:
5130 5131 5132 5133 5134
		break;
	default:
		return 0;
	}

5135
	reg = cur_regs(env) + BPF_REG_0;
5136
	if (reg->type != SCALAR_VALUE) {
5137
		verbose(env, "At program exit the register R0 is not a known value (%s)\n",
5138 5139 5140 5141 5142
			reg_type_str[reg->type]);
		return -EINVAL;
	}

	if (!tnum_in(range, reg->var_off)) {
5143
		verbose(env, "At program exit the register R0 ");
5144 5145 5146 5147
		if (!tnum_is_unknown(reg->var_off)) {
			char tn_buf[48];

			tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
5148
			verbose(env, "has value %s", tn_buf);
5149
		} else {
5150
			verbose(env, "has unknown scalar value");
5151
		}
5152
		verbose(env, " should have been 0 or 1\n");
5153 5154 5155 5156 5157
		return -EINVAL;
	}
	return 0;
}

5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176 5177 5178 5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191 5192 5193 5194 5195 5196 5197
/* non-recursive DFS pseudo code
 * 1  procedure DFS-iterative(G,v):
 * 2      label v as discovered
 * 3      let S be a stack
 * 4      S.push(v)
 * 5      while S is not empty
 * 6            t <- S.pop()
 * 7            if t is what we're looking for:
 * 8                return t
 * 9            for all edges e in G.adjacentEdges(t) do
 * 10               if edge e is already labelled
 * 11                   continue with the next edge
 * 12               w <- G.adjacentVertex(t,e)
 * 13               if vertex w is not discovered and not explored
 * 14                   label e as tree-edge
 * 15                   label w as discovered
 * 16                   S.push(w)
 * 17                   continue at 5
 * 18               else if vertex w is discovered
 * 19                   label e as back-edge
 * 20               else
 * 21                   // vertex w is explored
 * 22                   label e as forward- or cross-edge
 * 23           label t as explored
 * 24           S.pop()
 *
 * convention:
 * 0x10 - discovered
 * 0x11 - discovered and fall-through edge labelled
 * 0x12 - discovered and fall-through and branch edges labelled
 * 0x20 - explored
 */

enum {
	DISCOVERED = 0x10,
	EXPLORED = 0x20,
	FALLTHROUGH = 1,
	BRANCH = 2,
};

5198
#define STATE_LIST_MARK ((struct bpf_verifier_state_list *) -1L)
5199

5200 5201 5202 5203 5204 5205 5206 5207 5208
static int *insn_stack;	/* stack of insns to process */
static int cur_stack;	/* current stack index */
static int *insn_state;

/* t, w, e - match pseudo-code above:
 * t - index of current instruction
 * w - next instruction
 * e - edge
 */
5209
static int push_insn(int t, int w, int e, struct bpf_verifier_env *env)
5210 5211 5212 5213 5214 5215 5216 5217
{
	if (e == FALLTHROUGH && insn_state[t] >= (DISCOVERED | FALLTHROUGH))
		return 0;

	if (e == BRANCH && insn_state[t] >= (DISCOVERED | BRANCH))
		return 0;

	if (w < 0 || w >= env->prog->len) {
5218
		verbose_linfo(env, t, "%d: ", t);
5219
		verbose(env, "jump out of range from insn %d to %d\n", t, w);
5220 5221 5222
		return -EINVAL;
	}

5223 5224 5225 5226
	if (e == BRANCH)
		/* mark branch target for state pruning */
		env->explored_states[w] = STATE_LIST_MARK;

5227 5228 5229 5230 5231 5232 5233 5234 5235
	if (insn_state[w] == 0) {
		/* tree-edge */
		insn_state[t] = DISCOVERED | e;
		insn_state[w] = DISCOVERED;
		if (cur_stack >= env->prog->len)
			return -E2BIG;
		insn_stack[cur_stack++] = w;
		return 1;
	} else if ((insn_state[w] & 0xF0) == DISCOVERED) {
5236 5237
		verbose_linfo(env, t, "%d: ", t);
		verbose_linfo(env, w, "%d: ", w);
5238
		verbose(env, "back-edge from insn %d to %d\n", t, w);
5239 5240 5241 5242 5243
		return -EINVAL;
	} else if (insn_state[w] == EXPLORED) {
		/* forward- or cross-edge */
		insn_state[t] = DISCOVERED | e;
	} else {
5244
		verbose(env, "insn state internal bug\n");
5245 5246 5247 5248 5249 5250 5251 5252
		return -EFAULT;
	}
	return 0;
}

/* non-recursive depth-first-search to detect loops in BPF program
 * loop == back-edge in directed graph
 */
5253
static int check_cfg(struct bpf_verifier_env *env)
5254 5255 5256 5257 5258 5259 5260 5261 5262 5263 5264 5265 5266 5267 5268 5269 5270 5271 5272 5273 5274 5275 5276 5277 5278
{
	struct bpf_insn *insns = env->prog->insnsi;
	int insn_cnt = env->prog->len;
	int ret = 0;
	int i, t;

	insn_state = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
	if (!insn_state)
		return -ENOMEM;

	insn_stack = kcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
	if (!insn_stack) {
		kfree(insn_state);
		return -ENOMEM;
	}

	insn_state[0] = DISCOVERED; /* mark 1st insn as discovered */
	insn_stack[0] = 0; /* 0 is the first instruction */
	cur_stack = 1;

peek_stack:
	if (cur_stack == 0)
		goto check_state;
	t = insn_stack[cur_stack - 1];

Jiong Wang's avatar
Jiong Wang committed
5279 5280
	if (BPF_CLASS(insns[t].code) == BPF_JMP ||
	    BPF_CLASS(insns[t].code) == BPF_JMP32) {
5281 5282 5283 5284 5285 5286 5287 5288 5289 5290
		u8 opcode = BPF_OP(insns[t].code);

		if (opcode == BPF_EXIT) {
			goto mark_explored;
		} else if (opcode == BPF_CALL) {
			ret = push_insn(t, t + 1, FALLTHROUGH, env);
			if (ret == 1)
				goto peek_stack;
			else if (ret < 0)
				goto err_free;
5291 5292
			if (t + 1 < insn_cnt)
				env->explored_states[t + 1] = STATE_LIST_MARK;
5293 5294 5295 5296 5297 5298 5299 5300
			if (insns[t].src_reg == BPF_PSEUDO_CALL) {
				env->explored_states[t] = STATE_LIST_MARK;
				ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env);
				if (ret == 1)
					goto peek_stack;
				else if (ret < 0)
					goto err_free;
			}
5301 5302 5303 5304 5305 5306 5307 5308 5309 5310 5311 5312
		} else if (opcode == BPF_JA) {
			if (BPF_SRC(insns[t].code) != BPF_K) {
				ret = -EINVAL;
				goto err_free;
			}
			/* unconditional jump with single edge */
			ret = push_insn(t, t + insns[t].off + 1,
					FALLTHROUGH, env);
			if (ret == 1)
				goto peek_stack;
			else if (ret < 0)
				goto err_free;
5313 5314 5315
			/* tell verifier to check for equivalent states
			 * after every call and jump
			 */
5316 5317
			if (t + 1 < insn_cnt)
				env->explored_states[t + 1] = STATE_LIST_MARK;
5318 5319
		} else {
			/* conditional jump with two edges */
5320
			env->explored_states[t] = STATE_LIST_MARK;
5321 5322 5323 5324 5325 5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346
			ret = push_insn(t, t + 1, FALLTHROUGH, env);
			if (ret == 1)
				goto peek_stack;
			else if (ret < 0)
				goto err_free;

			ret = push_insn(t, t + insns[t].off + 1, BRANCH, env);
			if (ret == 1)
				goto peek_stack;
			else if (ret < 0)
				goto err_free;
		}
	} else {
		/* all other non-branch instructions with single
		 * fall-through edge
		 */
		ret = push_insn(t, t + 1, FALLTHROUGH, env);
		if (ret == 1)
			goto peek_stack;
		else if (ret < 0)
			goto err_free;
	}

mark_explored:
	insn_state[t] = EXPLORED;
	if (cur_stack-- <= 0) {
5347
		verbose(env, "pop stack internal bug\n");
5348 5349 5350 5351 5352 5353 5354 5355
		ret = -EFAULT;
		goto err_free;
	}
	goto peek_stack;

check_state:
	for (i = 0; i < insn_cnt; i++) {
		if (insn_state[i] != EXPLORED) {
5356
			verbose(env, "unreachable insn %d\n", i);
5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368
			ret = -EINVAL;
			goto err_free;
		}
	}
	ret = 0; /* cfg looks good */

err_free:
	kfree(insn_state);
	kfree(insn_stack);
	return ret;
}

5369 5370 5371 5372
/* The minimum supported BTF func info size */
#define MIN_BPF_FUNCINFO_SIZE	8
#define MAX_FUNCINFO_REC_SIZE	252

5373 5374 5375
static int check_btf_func(struct bpf_verifier_env *env,
			  const union bpf_attr *attr,
			  union bpf_attr __user *uattr)
5376
{
5377
	u32 i, nfuncs, urec_size, min_size;
5378
	u32 krec_size = sizeof(struct bpf_func_info);
5379
	struct bpf_func_info *krecord;
5380
	const struct btf_type *type;
5381 5382
	struct bpf_prog *prog;
	const struct btf *btf;
5383
	void __user *urecord;
5384
	u32 prev_offset = 0;
5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403
	int ret = 0;

	nfuncs = attr->func_info_cnt;
	if (!nfuncs)
		return 0;

	if (nfuncs != env->subprog_cnt) {
		verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
		return -EINVAL;
	}

	urec_size = attr->func_info_rec_size;
	if (urec_size < MIN_BPF_FUNCINFO_SIZE ||
	    urec_size > MAX_FUNCINFO_REC_SIZE ||
	    urec_size % sizeof(u32)) {
		verbose(env, "invalid func info rec size %u\n", urec_size);
		return -EINVAL;
	}

5404 5405
	prog = env->prog;
	btf = prog->aux->btf;
5406 5407 5408 5409

	urecord = u64_to_user_ptr(attr->func_info);
	min_size = min_t(u32, krec_size, urec_size);

5410
	krecord = kvcalloc(nfuncs, krec_size, GFP_KERNEL | __GFP_NOWARN);
5411 5412
	if (!krecord)
		return -ENOMEM;
5413

5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424
	for (i = 0; i < nfuncs; i++) {
		ret = bpf_check_uarg_tail_zero(urecord, krec_size, urec_size);
		if (ret) {
			if (ret == -E2BIG) {
				verbose(env, "nonzero tailing record in func info");
				/* set the size kernel expects so loader can zero
				 * out the rest of the record.
				 */
				if (put_user(min_size, &uattr->func_info_rec_size))
					ret = -EFAULT;
			}
5425
			goto err_free;
5426 5427
		}

5428
		if (copy_from_user(&krecord[i], urecord, min_size)) {
5429
			ret = -EFAULT;
5430
			goto err_free;
5431 5432
		}

5433
		/* check insn_off */
5434
		if (i == 0) {
5435
			if (krecord[i].insn_off) {
5436
				verbose(env,
5437 5438
					"nonzero insn_off %u for the first func info record",
					krecord[i].insn_off);
5439
				ret = -EINVAL;
5440
				goto err_free;
5441
			}
5442
		} else if (krecord[i].insn_off <= prev_offset) {
5443 5444
			verbose(env,
				"same or smaller insn offset (%u) than previous func info record (%u)",
5445
				krecord[i].insn_off, prev_offset);
5446
			ret = -EINVAL;
5447
			goto err_free;
5448 5449
		}

5450
		if (env->subprog_info[i].start != krecord[i].insn_off) {
5451 5452
			verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
			ret = -EINVAL;
5453
			goto err_free;
5454 5455 5456
		}

		/* check type_id */
5457
		type = btf_type_by_id(btf, krecord[i].type_id);
5458 5459
		if (!type || BTF_INFO_KIND(type->info) != BTF_KIND_FUNC) {
			verbose(env, "invalid type id %d in func info",
5460
				krecord[i].type_id);
5461
			ret = -EINVAL;
5462
			goto err_free;
5463 5464
		}

5465
		prev_offset = krecord[i].insn_off;
5466 5467 5468
		urecord += urec_size;
	}

5469 5470
	prog->aux->func_info = krecord;
	prog->aux->func_info_cnt = nfuncs;
5471 5472
	return 0;

5473
err_free:
5474
	kvfree(krecord);
5475 5476 5477
	return ret;
}

5478 5479 5480 5481 5482 5483 5484 5485
static void adjust_btf_func(struct bpf_verifier_env *env)
{
	int i;

	if (!env->prog->aux->func_info)
		return;

	for (i = 0; i < env->subprog_cnt; i++)
5486
		env->prog->aux->func_info[i].insn_off = env->subprog_info[i].start;
5487 5488
}

5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499 5500 5501 5502 5503 5504 5505 5506 5507 5508 5509 5510 5511 5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535 5536 5537 5538 5539 5540 5541 5542 5543 5544 5545 5546 5547 5548 5549 5550 5551 5552 5553 5554 5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567
#define MIN_BPF_LINEINFO_SIZE	(offsetof(struct bpf_line_info, line_col) + \
		sizeof(((struct bpf_line_info *)(0))->line_col))
#define MAX_LINEINFO_REC_SIZE	MAX_FUNCINFO_REC_SIZE

static int check_btf_line(struct bpf_verifier_env *env,
			  const union bpf_attr *attr,
			  union bpf_attr __user *uattr)
{
	u32 i, s, nr_linfo, ncopy, expected_size, rec_size, prev_offset = 0;
	struct bpf_subprog_info *sub;
	struct bpf_line_info *linfo;
	struct bpf_prog *prog;
	const struct btf *btf;
	void __user *ulinfo;
	int err;

	nr_linfo = attr->line_info_cnt;
	if (!nr_linfo)
		return 0;

	rec_size = attr->line_info_rec_size;
	if (rec_size < MIN_BPF_LINEINFO_SIZE ||
	    rec_size > MAX_LINEINFO_REC_SIZE ||
	    rec_size & (sizeof(u32) - 1))
		return -EINVAL;

	/* Need to zero it in case the userspace may
	 * pass in a smaller bpf_line_info object.
	 */
	linfo = kvcalloc(nr_linfo, sizeof(struct bpf_line_info),
			 GFP_KERNEL | __GFP_NOWARN);
	if (!linfo)
		return -ENOMEM;

	prog = env->prog;
	btf = prog->aux->btf;

	s = 0;
	sub = env->subprog_info;
	ulinfo = u64_to_user_ptr(attr->line_info);
	expected_size = sizeof(struct bpf_line_info);
	ncopy = min_t(u32, expected_size, rec_size);
	for (i = 0; i < nr_linfo; i++) {
		err = bpf_check_uarg_tail_zero(ulinfo, expected_size, rec_size);
		if (err) {
			if (err == -E2BIG) {
				verbose(env, "nonzero tailing record in line_info");
				if (put_user(expected_size,
					     &uattr->line_info_rec_size))
					err = -EFAULT;
			}
			goto err_free;
		}

		if (copy_from_user(&linfo[i], ulinfo, ncopy)) {
			err = -EFAULT;
			goto err_free;
		}

		/*
		 * Check insn_off to ensure
		 * 1) strictly increasing AND
		 * 2) bounded by prog->len
		 *
		 * The linfo[0].insn_off == 0 check logically falls into
		 * the later "missing bpf_line_info for func..." case
		 * because the first linfo[0].insn_off must be the
		 * first sub also and the first sub must have
		 * subprog_info[0].start == 0.
		 */
		if ((i && linfo[i].insn_off <= prev_offset) ||
		    linfo[i].insn_off >= prog->len) {
			verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n",
				i, linfo[i].insn_off, prev_offset,
				prog->len);
			err = -EINVAL;
			goto err_free;
		}

5568 5569 5570 5571 5572 5573 5574 5575
		if (!prog->insnsi[linfo[i].insn_off].code) {
			verbose(env,
				"Invalid insn code at line_info[%u].insn_off\n",
				i);
			err = -EINVAL;
			goto err_free;
		}

5576 5577
		if (!btf_name_by_offset(btf, linfo[i].line_off) ||
		    !btf_name_by_offset(btf, linfo[i].file_name_off)) {
5578 5579 5580 5581 5582 5583 5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636 5637 5638
			verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
			err = -EINVAL;
			goto err_free;
		}

		if (s != env->subprog_cnt) {
			if (linfo[i].insn_off == sub[s].start) {
				sub[s].linfo_idx = i;
				s++;
			} else if (sub[s].start < linfo[i].insn_off) {
				verbose(env, "missing bpf_line_info for func#%u\n", s);
				err = -EINVAL;
				goto err_free;
			}
		}

		prev_offset = linfo[i].insn_off;
		ulinfo += rec_size;
	}

	if (s != env->subprog_cnt) {
		verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n",
			env->subprog_cnt - s, s);
		err = -EINVAL;
		goto err_free;
	}

	prog->aux->linfo = linfo;
	prog->aux->nr_linfo = nr_linfo;

	return 0;

err_free:
	kvfree(linfo);
	return err;
}

static int check_btf_info(struct bpf_verifier_env *env,
			  const union bpf_attr *attr,
			  union bpf_attr __user *uattr)
{
	struct btf *btf;
	int err;

	if (!attr->func_info_cnt && !attr->line_info_cnt)
		return 0;

	btf = btf_get_by_fd(attr->prog_btf_fd);
	if (IS_ERR(btf))
		return PTR_ERR(btf);
	env->prog->aux->btf = btf;

	err = check_btf_func(env, attr, uattr);
	if (err)
		return err;

	err = check_btf_line(env, attr, uattr);
	if (err)
		return err;

	return 0;
5639 5640
}

5641 5642 5643 5644
/* check %cur's range satisfies %old's */
static bool range_within(struct bpf_reg_state *old,
			 struct bpf_reg_state *cur)
{
5645 5646 5647 5648
	return old->umin_value <= cur->umin_value &&
	       old->umax_value >= cur->umax_value &&
	       old->smin_value <= cur->smin_value &&
	       old->smax_value >= cur->smax_value;
5649 5650 5651 5652 5653 5654 5655 5656 5657 5658 5659 5660 5661 5662 5663 5664 5665 5666
}

/* Maximum number of register states that can exist at once */
#define ID_MAP_SIZE	(MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE)
struct idpair {
	u32 old;
	u32 cur;
};

/* If in the old state two registers had the same id, then they need to have
 * the same id in the new state as well.  But that id could be different from
 * the old state, so we need to track the mapping from old to new ids.
 * Once we have seen that, say, a reg with old id 5 had new id 9, any subsequent
 * regs with old id 5 must also have new id 9 for the new state to be safe.  But
 * regs with a different old id could still have new id 9, we don't care about
 * that.
 * So we look through our idmap to see if this old id has been seen before.  If
 * so, we require the new id to match; otherwise, we add the id pair to the map.
5667
 */
5668
static bool check_ids(u32 old_id, u32 cur_id, struct idpair *idmap)
5669
{
5670
	unsigned int i;
5671

5672 5673 5674 5675 5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686
	for (i = 0; i < ID_MAP_SIZE; i++) {
		if (!idmap[i].old) {
			/* Reached an empty slot; haven't seen this id before */
			idmap[i].old = old_id;
			idmap[i].cur = cur_id;
			return true;
		}
		if (idmap[i].old == old_id)
			return idmap[i].cur == cur_id;
	}
	/* We ran out of idmap slots, which should be impossible */
	WARN_ON_ONCE(1);
	return false;
}

5687 5688 5689 5690 5691 5692 5693 5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766 5767 5768 5769 5770 5771 5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782
static void clean_func_state(struct bpf_verifier_env *env,
			     struct bpf_func_state *st)
{
	enum bpf_reg_liveness live;
	int i, j;

	for (i = 0; i < BPF_REG_FP; i++) {
		live = st->regs[i].live;
		/* liveness must not touch this register anymore */
		st->regs[i].live |= REG_LIVE_DONE;
		if (!(live & REG_LIVE_READ))
			/* since the register is unused, clear its state
			 * to make further comparison simpler
			 */
			__mark_reg_not_init(&st->regs[i]);
	}

	for (i = 0; i < st->allocated_stack / BPF_REG_SIZE; i++) {
		live = st->stack[i].spilled_ptr.live;
		/* liveness must not touch this stack slot anymore */
		st->stack[i].spilled_ptr.live |= REG_LIVE_DONE;
		if (!(live & REG_LIVE_READ)) {
			__mark_reg_not_init(&st->stack[i].spilled_ptr);
			for (j = 0; j < BPF_REG_SIZE; j++)
				st->stack[i].slot_type[j] = STACK_INVALID;
		}
	}
}

static void clean_verifier_state(struct bpf_verifier_env *env,
				 struct bpf_verifier_state *st)
{
	int i;

	if (st->frame[0]->regs[0].live & REG_LIVE_DONE)
		/* all regs in this state in all frames were already marked */
		return;

	for (i = 0; i <= st->curframe; i++)
		clean_func_state(env, st->frame[i]);
}

/* the parentage chains form a tree.
 * the verifier states are added to state lists at given insn and
 * pushed into state stack for future exploration.
 * when the verifier reaches bpf_exit insn some of the verifer states
 * stored in the state lists have their final liveness state already,
 * but a lot of states will get revised from liveness point of view when
 * the verifier explores other branches.
 * Example:
 * 1: r0 = 1
 * 2: if r1 == 100 goto pc+1
 * 3: r0 = 2
 * 4: exit
 * when the verifier reaches exit insn the register r0 in the state list of
 * insn 2 will be seen as !REG_LIVE_READ. Then the verifier pops the other_branch
 * of insn 2 and goes exploring further. At the insn 4 it will walk the
 * parentage chain from insn 4 into insn 2 and will mark r0 as REG_LIVE_READ.
 *
 * Since the verifier pushes the branch states as it sees them while exploring
 * the program the condition of walking the branch instruction for the second
 * time means that all states below this branch were already explored and
 * their final liveness markes are already propagated.
 * Hence when the verifier completes the search of state list in is_state_visited()
 * we can call this clean_live_states() function to mark all liveness states
 * as REG_LIVE_DONE to indicate that 'parent' pointers of 'struct bpf_reg_state'
 * will not be used.
 * This function also clears the registers and stack for states that !READ
 * to simplify state merging.
 *
 * Important note here that walking the same branch instruction in the callee
 * doesn't meant that the states are DONE. The verifier has to compare
 * the callsites
 */
static void clean_live_states(struct bpf_verifier_env *env, int insn,
			      struct bpf_verifier_state *cur)
{
	struct bpf_verifier_state_list *sl;
	int i;

	sl = env->explored_states[insn];
	if (!sl)
		return;

	while (sl != STATE_LIST_MARK) {
		if (sl->state.curframe != cur->curframe)
			goto next;
		for (i = 0; i <= cur->curframe; i++)
			if (sl->state.frame[i]->callsite != cur->frame[i]->callsite)
				goto next;
		clean_verifier_state(env, &sl->state);
next:
		sl = sl->next;
	}
}

5783
/* Returns true if (rold safe implies rcur safe) */
5784 5785
static bool regsafe(struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
		    struct idpair *idmap)
5786
{
5787 5788
	bool equal;

5789 5790 5791 5792
	if (!(rold->live & REG_LIVE_READ))
		/* explored state didn't use this */
		return true;

5793
	equal = memcmp(rold, rcur, offsetof(struct bpf_reg_state, parent)) == 0;
5794 5795 5796 5797 5798 5799 5800 5801

	if (rold->type == PTR_TO_STACK)
		/* two stack pointers are equal only if they're pointing to
		 * the same stack frame, since fp-8 in foo != fp-8 in bar
		 */
		return equal && rold->frameno == rcur->frameno;

	if (equal)
5802 5803
		return true;

5804 5805
	if (rold->type == NOT_INIT)
		/* explored state can't have used this */
5806
		return true;
5807 5808 5809 5810 5811 5812 5813 5814 5815
	if (rcur->type == NOT_INIT)
		return false;
	switch (rold->type) {
	case SCALAR_VALUE:
		if (rcur->type == SCALAR_VALUE) {
			/* new val must satisfy old val knowledge */
			return range_within(rold, rcur) &&
			       tnum_in(rold->var_off, rcur->var_off);
		} else {
5816 5817 5818 5819 5820 5821
			/* We're trying to use a pointer in place of a scalar.
			 * Even if the scalar was unbounded, this could lead to
			 * pointer leaks because scalars are allowed to leak
			 * while pointers are not. We could make this safe in
			 * special cases if root is calling us, but it's
			 * probably not worth the hassle.
5822
			 */
5823
			return false;
5824 5825
		}
	case PTR_TO_MAP_VALUE:
5826 5827
		/* If the new min/max/var_off satisfy the old ones and
		 * everything else matches, we are OK.
5828 5829 5830 5831 5832
		 * 'id' is not compared, since it's only used for maps with
		 * bpf_spin_lock inside map element and in such cases if
		 * the rest of the prog is valid for one map element then
		 * it's valid for all map elements regardless of the key
		 * used in bpf_map_lookup()
5833 5834 5835 5836
		 */
		return memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)) == 0 &&
		       range_within(rold, rcur) &&
		       tnum_in(rold->var_off, rcur->var_off);
5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850
	case PTR_TO_MAP_VALUE_OR_NULL:
		/* a PTR_TO_MAP_VALUE could be safe to use as a
		 * PTR_TO_MAP_VALUE_OR_NULL into the same map.
		 * However, if the old PTR_TO_MAP_VALUE_OR_NULL then got NULL-
		 * checked, doing so could have affected others with the same
		 * id, and we can't check for that because we lost the id when
		 * we converted to a PTR_TO_MAP_VALUE.
		 */
		if (rcur->type != PTR_TO_MAP_VALUE_OR_NULL)
			return false;
		if (memcmp(rold, rcur, offsetof(struct bpf_reg_state, id)))
			return false;
		/* Check our ids match any regs they're supposed to */
		return check_ids(rold->id, rcur->id, idmap);
5851
	case PTR_TO_PACKET_META:
5852
	case PTR_TO_PACKET:
5853
		if (rcur->type != rold->type)
5854 5855 5856 5857 5858 5859 5860 5861 5862 5863 5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876
			return false;
		/* We must have at least as much range as the old ptr
		 * did, so that any accesses which were safe before are
		 * still safe.  This is true even if old range < old off,
		 * since someone could have accessed through (ptr - k), or
		 * even done ptr -= k in a register, to get a safe access.
		 */
		if (rold->range > rcur->range)
			return false;
		/* If the offsets don't match, we can't trust our alignment;
		 * nor can we be sure that we won't fall out of range.
		 */
		if (rold->off != rcur->off)
			return false;
		/* id relations must be preserved */
		if (rold->id && !check_ids(rold->id, rcur->id, idmap))
			return false;
		/* new val must satisfy old val knowledge */
		return range_within(rold, rcur) &&
		       tnum_in(rold->var_off, rcur->var_off);
	case PTR_TO_CTX:
	case CONST_PTR_TO_MAP:
	case PTR_TO_PACKET_END:
5877
	case PTR_TO_FLOW_KEYS:
5878 5879
	case PTR_TO_SOCKET:
	case PTR_TO_SOCKET_OR_NULL:
5880 5881
	case PTR_TO_SOCK_COMMON:
	case PTR_TO_SOCK_COMMON_OR_NULL:
5882 5883
	case PTR_TO_TCP_SOCK:
	case PTR_TO_TCP_SOCK_OR_NULL:
5884 5885 5886 5887 5888 5889 5890
		/* Only valid matches are exact, which memcmp() above
		 * would have accepted
		 */
	default:
		/* Don't know what's going on, just say it's not safe */
		return false;
	}
5891

5892 5893
	/* Shouldn't get here; if we do, say it's not safe */
	WARN_ON_ONCE(1);
5894 5895 5896
	return false;
}

5897 5898
static bool stacksafe(struct bpf_func_state *old,
		      struct bpf_func_state *cur,
5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909
		      struct idpair *idmap)
{
	int i, spi;

	/* walk slots of the explored stack and ignore any additional
	 * slots in the current stack, since explored(safe) state
	 * didn't use them
	 */
	for (i = 0; i < old->allocated_stack; i++) {
		spi = i / BPF_REG_SIZE;

5910 5911
		if (!(old->stack[spi].spilled_ptr.live & REG_LIVE_READ)) {
			i += BPF_REG_SIZE - 1;
5912
			/* explored state didn't use this */
5913
			continue;
5914
		}
5915

5916 5917
		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_INVALID)
			continue;
5918 5919 5920 5921 5922 5923 5924

		/* explored stack has more populated slots than current stack
		 * and these slots were used
		 */
		if (i >= cur->allocated_stack)
			return false;

5925 5926 5927 5928 5929 5930 5931
		/* if old state was safe with misc data in the stack
		 * it will be safe with zero-initialized stack.
		 * The opposite is not true
		 */
		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_MISC &&
		    cur->stack[spi].slot_type[i % BPF_REG_SIZE] == STACK_ZERO)
			continue;
5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961
		if (old->stack[spi].slot_type[i % BPF_REG_SIZE] !=
		    cur->stack[spi].slot_type[i % BPF_REG_SIZE])
			/* Ex: old explored (safe) state has STACK_SPILL in
			 * this stack slot, but current has has STACK_MISC ->
			 * this verifier states are not equivalent,
			 * return false to continue verification of this path
			 */
			return false;
		if (i % BPF_REG_SIZE)
			continue;
		if (old->stack[spi].slot_type[0] != STACK_SPILL)
			continue;
		if (!regsafe(&old->stack[spi].spilled_ptr,
			     &cur->stack[spi].spilled_ptr,
			     idmap))
			/* when explored and current stack slot are both storing
			 * spilled registers, check that stored pointers types
			 * are the same as well.
			 * Ex: explored safe path could have stored
			 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -8}
			 * but current path has stored:
			 * (bpf_reg_state) {.type = PTR_TO_STACK, .off = -16}
			 * such verifier states are not equivalent.
			 * return false to continue verification of this path
			 */
			return false;
	}
	return true;
}

5962 5963 5964 5965 5966 5967 5968 5969
static bool refsafe(struct bpf_func_state *old, struct bpf_func_state *cur)
{
	if (old->acquired_refs != cur->acquired_refs)
		return false;
	return !memcmp(old->refs, cur->refs,
		       sizeof(*old->refs) * old->acquired_refs);
}

5970 5971 5972 5973 5974 5975 5976 5977 5978 5979 5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991 5992 5993 5994 5995
/* compare two verifier states
 *
 * all states stored in state_list are known to be valid, since
 * verifier reached 'bpf_exit' instruction through them
 *
 * this function is called when verifier exploring different branches of
 * execution popped from the state stack. If it sees an old state that has
 * more strict register state and more strict stack state then this execution
 * branch doesn't need to be explored further, since verifier already
 * concluded that more strict state leads to valid finish.
 *
 * Therefore two states are equivalent if register state is more conservative
 * and explored stack state is more conservative than the current one.
 * Example:
 *       explored                   current
 * (slot1=INV slot2=MISC) == (slot1=MISC slot2=MISC)
 * (slot1=MISC slot2=MISC) != (slot1=INV slot2=MISC)
 *
 * In other words if current stack state (one being explored) has more
 * valid slots than old one that already passed validation, it means
 * the verifier can stop exploring and conclude that current state is valid too
 *
 * Similarly with registers. If explored state has register type as invalid
 * whereas register type in current state is meaningful, it means that
 * the current state will reach 'bpf_exit' instruction safely
 */
5996 5997
static bool func_states_equal(struct bpf_func_state *old,
			      struct bpf_func_state *cur)
5998
{
5999 6000
	struct idpair *idmap;
	bool ret = false;
6001 6002
	int i;

6003 6004 6005
	idmap = kcalloc(ID_MAP_SIZE, sizeof(struct idpair), GFP_KERNEL);
	/* If we failed to allocate the idmap, just say it's not safe */
	if (!idmap)
6006
		return false;
6007 6008

	for (i = 0; i < MAX_BPF_REG; i++) {
6009
		if (!regsafe(&old->regs[i], &cur->regs[i], idmap))
6010
			goto out_free;
6011 6012
	}

6013 6014
	if (!stacksafe(old, cur, idmap))
		goto out_free;
6015 6016 6017

	if (!refsafe(old, cur))
		goto out_free;
6018 6019 6020 6021
	ret = true;
out_free:
	kfree(idmap);
	return ret;
6022 6023
}

6024 6025 6026 6027 6028 6029 6030 6031 6032
static bool states_equal(struct bpf_verifier_env *env,
			 struct bpf_verifier_state *old,
			 struct bpf_verifier_state *cur)
{
	int i;

	if (old->curframe != cur->curframe)
		return false;

6033 6034 6035 6036 6037 6038
	/* Verification state from speculative execution simulation
	 * must never prune a non-speculative execution one.
	 */
	if (old->speculative && !cur->speculative)
		return false;

6039 6040 6041
	if (old->active_spin_lock != cur->active_spin_lock)
		return false;

6042 6043 6044 6045 6046 6047 6048 6049 6050 6051 6052 6053
	/* for states to be equal callsites have to be the same
	 * and all frame states need to be equivalent
	 */
	for (i = 0; i <= old->curframe; i++) {
		if (old->frame[i]->callsite != cur->frame[i]->callsite)
			return false;
		if (!func_states_equal(old->frame[i], cur->frame[i]))
			return false;
	}
	return true;
}

6054
/* A write screens off any subsequent reads; but write marks come from the
6055 6056 6057 6058
 * straight-line code between a state and its parent.  When we arrive at an
 * equivalent state (jump target or such) we didn't arrive by the straight-line
 * code, so read marks in the state must propagate to the parent regardless
 * of the state's write marks. That's what 'parent == state->parent' comparison
6059
 * in mark_reg_read() is for.
6060
 */
6061 6062 6063
static int propagate_liveness(struct bpf_verifier_env *env,
			      const struct bpf_verifier_state *vstate,
			      struct bpf_verifier_state *vparent)
6064
{
6065 6066
	int i, frame, err = 0;
	struct bpf_func_state *state, *parent;
6067

6068 6069 6070 6071 6072
	if (vparent->curframe != vstate->curframe) {
		WARN(1, "propagate_live: parent frame %d current frame %d\n",
		     vparent->curframe, vstate->curframe);
		return -EFAULT;
	}
6073 6074 6075 6076
	/* Propagate read liveness of registers... */
	BUILD_BUG_ON(BPF_REG_FP + 1 != MAX_BPF_REG);
	/* We don't need to worry about FP liveness because it's read-only */
	for (i = 0; i < BPF_REG_FP; i++) {
6077
		if (vparent->frame[vparent->curframe]->regs[i].live & REG_LIVE_READ)
6078
			continue;
6079
		if (vstate->frame[vstate->curframe]->regs[i].live & REG_LIVE_READ) {
6080 6081
			err = mark_reg_read(env, &vstate->frame[vstate->curframe]->regs[i],
					    &vparent->frame[vstate->curframe]->regs[i]);
6082 6083
			if (err)
				return err;
6084 6085
		}
	}
6086

6087
	/* ... and stack slots */
6088 6089 6090 6091 6092 6093 6094 6095
	for (frame = 0; frame <= vstate->curframe; frame++) {
		state = vstate->frame[frame];
		parent = vparent->frame[frame];
		for (i = 0; i < state->allocated_stack / BPF_REG_SIZE &&
			    i < parent->allocated_stack / BPF_REG_SIZE; i++) {
			if (parent->stack[i].spilled_ptr.live & REG_LIVE_READ)
				continue;
			if (state->stack[i].spilled_ptr.live & REG_LIVE_READ)
6096 6097
				mark_reg_read(env, &state->stack[i].spilled_ptr,
					      &parent->stack[i].spilled_ptr);
6098 6099
		}
	}
6100
	return err;
6101 6102
}

6103
static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
6104
{
6105 6106
	struct bpf_verifier_state_list *new_sl;
	struct bpf_verifier_state_list *sl;
6107
	struct bpf_verifier_state *cur = env->cur_state, *new;
6108
	int i, j, err, states_cnt = 0;
6109 6110 6111 6112 6113 6114 6115 6116

	sl = env->explored_states[insn_idx];
	if (!sl)
		/* this 'insn_idx' instruction wasn't marked, so we will not
		 * be doing state search here
		 */
		return 0;

6117 6118
	clean_live_states(env, insn_idx, cur);

6119
	while (sl != STATE_LIST_MARK) {
6120
		if (states_equal(env, &sl->state, cur)) {
6121
			/* reached equivalent register/stack state,
6122 6123
			 * prune the search.
			 * Registers read by the continuation are read by us.
6124 6125 6126 6127 6128 6129
			 * If we have any write marks in env->cur_state, they
			 * will prevent corresponding reads in the continuation
			 * from reaching our parent (an explored_state).  Our
			 * own state will get the read marks recorded, but
			 * they'll be immediately forgotten as we're pruning
			 * this state and will pop a new one.
6130
			 */
6131 6132 6133
			err = propagate_liveness(env, &sl->state, cur);
			if (err)
				return err;
6134
			return 1;
6135
		}
6136
		sl = sl->next;
6137
		states_cnt++;
6138 6139
	}

6140 6141 6142
	if (!env->allow_ptr_leaks && states_cnt > BPF_COMPLEXITY_LIMIT_STATES)
		return 0;

6143 6144
	/* there were no equivalent states, remember current one.
	 * technically the current state is not proven to be safe yet,
6145 6146 6147 6148
	 * but it will either reach outer most bpf_exit (which means it's safe)
	 * or it will be rejected. Since there are no loops, we won't be
	 * seeing this tuple (frame[0].callsite, frame[1].callsite, .. insn_idx)
	 * again on the way to bpf_exit
6149
	 */
6150
	new_sl = kzalloc(sizeof(struct bpf_verifier_state_list), GFP_KERNEL);
6151 6152 6153 6154
	if (!new_sl)
		return -ENOMEM;

	/* add new state to the head of linked list */
6155 6156
	new = &new_sl->state;
	err = copy_verifier_state(new, cur);
6157
	if (err) {
6158
		free_verifier_state(new, false);
6159 6160 6161
		kfree(new_sl);
		return err;
	}
6162 6163
	new_sl->next = env->explored_states[insn_idx];
	env->explored_states[insn_idx] = new_sl;
6164 6165 6166 6167 6168 6169 6170 6171 6172 6173
	/* connect new state to parentage chain. Current frame needs all
	 * registers connected. Only r6 - r9 of the callers are alive (pushed
	 * to the stack implicitly by JITs) so in callers' frames connect just
	 * r6 - r9 as an optimization. Callers will have r1 - r5 connected to
	 * the state of the call instruction (with WRITTEN set), and r0 comes
	 * from callee with its full parentage chain, anyway.
	 */
	for (j = 0; j <= cur->curframe; j++)
		for (i = j < cur->curframe ? BPF_REG_6 : 0; i < BPF_REG_FP; i++)
			cur->frame[j]->regs[i].parent = &new->frame[j]->regs[i];
6174 6175 6176 6177 6178 6179
	/* clear write marks in current state: the writes we did are not writes
	 * our child did, so they don't screen off its reads from us.
	 * (There are no read marks in current state, because reads always mark
	 * their parent and current state never has children yet.  Only
	 * explored_states can get read marks.)
	 */
6180
	for (i = 0; i < BPF_REG_FP; i++)
6181 6182 6183 6184 6185
		cur->frame[cur->curframe]->regs[i].live = REG_LIVE_NONE;

	/* all stack frames are accessible from callee, clear them all */
	for (j = 0; j <= cur->curframe; j++) {
		struct bpf_func_state *frame = cur->frame[j];
6186
		struct bpf_func_state *newframe = new->frame[j];
6187

6188
		for (i = 0; i < frame->allocated_stack / BPF_REG_SIZE; i++) {
6189
			frame->stack[i].spilled_ptr.live = REG_LIVE_NONE;
6190 6191 6192
			frame->stack[i].spilled_ptr.parent =
						&newframe->stack[i].spilled_ptr;
		}
6193
	}
6194 6195 6196
	return 0;
}

6197 6198 6199 6200 6201 6202 6203
/* Return true if it's OK to have the same insn return a different type. */
static bool reg_type_mismatch_ok(enum bpf_reg_type type)
{
	switch (type) {
	case PTR_TO_CTX:
	case PTR_TO_SOCKET:
	case PTR_TO_SOCKET_OR_NULL:
6204 6205
	case PTR_TO_SOCK_COMMON:
	case PTR_TO_SOCK_COMMON_OR_NULL:
6206 6207
	case PTR_TO_TCP_SOCK:
	case PTR_TO_TCP_SOCK_OR_NULL:
6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224 6225 6226 6227 6228 6229 6230 6231
		return false;
	default:
		return true;
	}
}

/* If an instruction was previously used with particular pointer types, then we
 * need to be careful to avoid cases such as the below, where it may be ok
 * for one branch accessing the pointer, but not ok for the other branch:
 *
 * R1 = sock_ptr
 * goto X;
 * ...
 * R1 = some_other_valid_ptr;
 * goto X;
 * ...
 * R2 = *(u32 *)(R1 + 0);
 */
static bool reg_type_mismatch(enum bpf_reg_type src, enum bpf_reg_type prev)
{
	return src != prev && (!reg_type_mismatch_ok(src) ||
			       !reg_type_mismatch_ok(prev));
}

6232
static int do_check(struct bpf_verifier_env *env)
6233
{
6234
	struct bpf_verifier_state *state;
6235
	struct bpf_insn *insns = env->prog->insnsi;
6236
	struct bpf_reg_state *regs;
6237
	int insn_cnt = env->prog->len, i;
6238 6239 6240
	int insn_processed = 0;
	bool do_print_state = false;

6241 6242
	env->prev_linfo = NULL;

6243 6244 6245
	state = kzalloc(sizeof(struct bpf_verifier_state), GFP_KERNEL);
	if (!state)
		return -ENOMEM;
6246
	state->curframe = 0;
6247
	state->speculative = false;
6248 6249 6250 6251 6252 6253 6254 6255 6256 6257
	state->frame[0] = kzalloc(sizeof(struct bpf_func_state), GFP_KERNEL);
	if (!state->frame[0]) {
		kfree(state);
		return -ENOMEM;
	}
	env->cur_state = state;
	init_func_state(env, state->frame[0],
			BPF_MAIN_FUNC /* callsite */,
			0 /* frameno */,
			0 /* subprogno, zero == main subprog */);
6258

6259 6260 6261 6262 6263
	for (;;) {
		struct bpf_insn *insn;
		u8 class;
		int err;

6264
		if (env->insn_idx >= insn_cnt) {
6265
			verbose(env, "invalid insn idx %d insn_cnt %d\n",
6266
				env->insn_idx, insn_cnt);
6267 6268 6269
			return -EFAULT;
		}

6270
		insn = &insns[env->insn_idx];
6271 6272
		class = BPF_CLASS(insn->code);

6273
		if (++insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
6274 6275
			verbose(env,
				"BPF program is too large. Processed %d insn\n",
6276 6277 6278 6279
				insn_processed);
			return -E2BIG;
		}

6280
		err = is_state_visited(env, env->insn_idx);
6281 6282 6283 6284
		if (err < 0)
			return err;
		if (err == 1) {
			/* found equivalent state, can prune the search */
6285
			if (env->log.level) {
6286
				if (do_print_state)
6287 6288 6289 6290
					verbose(env, "\nfrom %d to %d%s: safe\n",
						env->prev_insn_idx, env->insn_idx,
						env->cur_state->speculative ?
						" (speculative execution)" : "");
6291
				else
6292
					verbose(env, "%d: safe\n", env->insn_idx);
6293 6294 6295 6296
			}
			goto process_bpf_exit;
		}

6297 6298 6299
		if (signal_pending(current))
			return -EAGAIN;

6300 6301 6302
		if (need_resched())
			cond_resched();

6303 6304
		if (env->log.level > 1 || (env->log.level && do_print_state)) {
			if (env->log.level > 1)
6305
				verbose(env, "%d:", env->insn_idx);
6306
			else
6307 6308 6309 6310
				verbose(env, "\nfrom %d to %d%s:",
					env->prev_insn_idx, env->insn_idx,
					env->cur_state->speculative ?
					" (speculative execution)" : "");
6311
			print_verifier_state(env, state->frame[state->curframe]);
6312 6313 6314
			do_print_state = false;
		}

6315
		if (env->log.level) {
6316 6317
			const struct bpf_insn_cbs cbs = {
				.cb_print	= verbose,
6318
				.private_data	= env,
6319 6320
			};

6321 6322
			verbose_linfo(env, env->insn_idx, "; ");
			verbose(env, "%d: ", env->insn_idx);
6323
			print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
6324 6325
		}

6326
		if (bpf_prog_is_dev_bound(env->prog->aux)) {
6327 6328
			err = bpf_prog_offload_verify_insn(env, env->insn_idx,
							   env->prev_insn_idx);
6329 6330 6331
			if (err)
				return err;
		}
6332

6333
		regs = cur_regs(env);
6334
		env->insn_aux_data[env->insn_idx].seen = true;
6335

6336
		if (class == BPF_ALU || class == BPF_ALU64) {
6337
			err = check_alu_op(env, insn);
6338 6339 6340 6341
			if (err)
				return err;

		} else if (class == BPF_LDX) {
6342
			enum bpf_reg_type *prev_src_type, src_reg_type;
6343 6344 6345

			/* check for reserved fields is already done */

6346
			/* check src operand */
6347
			err = check_reg_arg(env, insn->src_reg, SRC_OP);
6348 6349 6350
			if (err)
				return err;

6351
			err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
6352 6353 6354
			if (err)
				return err;

6355 6356
			src_reg_type = regs[insn->src_reg].type;

6357 6358 6359
			/* check that memory (src_reg + off) is readable,
			 * the state of dst_reg will be updated by this func
			 */
6360 6361 6362
			err = check_mem_access(env, env->insn_idx, insn->src_reg,
					       insn->off, BPF_SIZE(insn->code),
					       BPF_READ, insn->dst_reg, false);
6363 6364 6365
			if (err)
				return err;

6366
			prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type;
6367 6368

			if (*prev_src_type == NOT_INIT) {
6369 6370
				/* saw a valid insn
				 * dst_reg = *(u32 *)(src_reg + off)
6371
				 * save type to validate intersecting paths
6372
				 */
6373
				*prev_src_type = src_reg_type;
6374

6375
			} else if (reg_type_mismatch(src_reg_type, *prev_src_type)) {
6376 6377 6378 6379 6380 6381 6382
				/* ABuser program is trying to use the same insn
				 * dst_reg = *(u32*) (src_reg + off)
				 * with different pointer types:
				 * src_reg == ctx in one branch and
				 * src_reg == stack|map in some other branch.
				 * Reject it.
				 */
6383
				verbose(env, "same insn cannot be used with different pointers\n");
6384 6385 6386
				return -EINVAL;
			}

6387
		} else if (class == BPF_STX) {
6388
			enum bpf_reg_type *prev_dst_type, dst_reg_type;
6389

6390
			if (BPF_MODE(insn->code) == BPF_XADD) {
6391
				err = check_xadd(env, env->insn_idx, insn);
6392 6393
				if (err)
					return err;
6394
				env->insn_idx++;
6395 6396 6397 6398
				continue;
			}

			/* check src1 operand */
6399
			err = check_reg_arg(env, insn->src_reg, SRC_OP);
6400 6401 6402
			if (err)
				return err;
			/* check src2 operand */
6403
			err = check_reg_arg(env, insn->dst_reg, SRC_OP);
6404 6405 6406
			if (err)
				return err;

6407 6408
			dst_reg_type = regs[insn->dst_reg].type;

6409
			/* check that memory (dst_reg + off) is writeable */
6410 6411 6412
			err = check_mem_access(env, env->insn_idx, insn->dst_reg,
					       insn->off, BPF_SIZE(insn->code),
					       BPF_WRITE, insn->src_reg, false);
6413 6414 6415
			if (err)
				return err;

6416
			prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type;
6417 6418 6419

			if (*prev_dst_type == NOT_INIT) {
				*prev_dst_type = dst_reg_type;
6420
			} else if (reg_type_mismatch(dst_reg_type, *prev_dst_type)) {
6421
				verbose(env, "same insn cannot be used with different pointers\n");
6422 6423 6424
				return -EINVAL;
			}

6425 6426 6427
		} else if (class == BPF_ST) {
			if (BPF_MODE(insn->code) != BPF_MEM ||
			    insn->src_reg != BPF_REG_0) {
6428
				verbose(env, "BPF_ST uses reserved fields\n");
6429 6430 6431
				return -EINVAL;
			}
			/* check src operand */
6432
			err = check_reg_arg(env, insn->dst_reg, SRC_OP);
6433 6434 6435
			if (err)
				return err;

6436
			if (is_ctx_reg(env, insn->dst_reg)) {
6437
				verbose(env, "BPF_ST stores into R%d %s is not allowed\n",
6438 6439
					insn->dst_reg,
					reg_type_str[reg_state(env, insn->dst_reg)->type]);
6440 6441 6442
				return -EACCES;
			}

6443
			/* check that memory (dst_reg + off) is writeable */
6444 6445 6446
			err = check_mem_access(env, env->insn_idx, insn->dst_reg,
					       insn->off, BPF_SIZE(insn->code),
					       BPF_WRITE, -1, false);
6447 6448 6449
			if (err)
				return err;

Jiong Wang's avatar
Jiong Wang committed
6450
		} else if (class == BPF_JMP || class == BPF_JMP32) {
6451 6452 6453 6454 6455
			u8 opcode = BPF_OP(insn->code);

			if (opcode == BPF_CALL) {
				if (BPF_SRC(insn->code) != BPF_K ||
				    insn->off != 0 ||
6456 6457
				    (insn->src_reg != BPF_REG_0 &&
				     insn->src_reg != BPF_PSEUDO_CALL) ||
Jiong Wang's avatar
Jiong Wang committed
6458 6459
				    insn->dst_reg != BPF_REG_0 ||
				    class == BPF_JMP32) {
6460
					verbose(env, "BPF_CALL uses reserved fields\n");
6461 6462 6463
					return -EINVAL;
				}

6464 6465 6466 6467 6468 6469
				if (env->cur_state->active_spin_lock &&
				    (insn->src_reg == BPF_PSEUDO_CALL ||
				     insn->imm != BPF_FUNC_spin_unlock)) {
					verbose(env, "function calls are not allowed while holding a lock\n");
					return -EINVAL;
				}
6470
				if (insn->src_reg == BPF_PSEUDO_CALL)
6471
					err = check_func_call(env, insn, &env->insn_idx);
6472
				else
6473
					err = check_helper_call(env, insn->imm, env->insn_idx);
6474 6475 6476 6477 6478 6479 6480
				if (err)
					return err;

			} else if (opcode == BPF_JA) {
				if (BPF_SRC(insn->code) != BPF_K ||
				    insn->imm != 0 ||
				    insn->src_reg != BPF_REG_0 ||
Jiong Wang's avatar
Jiong Wang committed
6481 6482
				    insn->dst_reg != BPF_REG_0 ||
				    class == BPF_JMP32) {
6483
					verbose(env, "BPF_JA uses reserved fields\n");
6484 6485 6486
					return -EINVAL;
				}

6487
				env->insn_idx += insn->off + 1;
6488 6489 6490 6491 6492 6493
				continue;

			} else if (opcode == BPF_EXIT) {
				if (BPF_SRC(insn->code) != BPF_K ||
				    insn->imm != 0 ||
				    insn->src_reg != BPF_REG_0 ||
Jiong Wang's avatar
Jiong Wang committed
6494 6495
				    insn->dst_reg != BPF_REG_0 ||
				    class == BPF_JMP32) {
6496
					verbose(env, "BPF_EXIT uses reserved fields\n");
6497 6498 6499
					return -EINVAL;
				}

6500 6501 6502 6503 6504
				if (env->cur_state->active_spin_lock) {
					verbose(env, "bpf_spin_unlock is missing\n");
					return -EINVAL;
				}

6505 6506
				if (state->curframe) {
					/* exit from nested function */
6507 6508
					env->prev_insn_idx = env->insn_idx;
					err = prepare_func_exit(env, &env->insn_idx);
6509 6510 6511 6512 6513 6514
					if (err)
						return err;
					do_print_state = true;
					continue;
				}

6515 6516 6517 6518
				err = check_reference_leak(env);
				if (err)
					return err;

6519 6520 6521 6522 6523 6524
				/* eBPF calling convetion is such that R0 is used
				 * to return the value from eBPF program.
				 * Make sure that it's readable at this time
				 * of bpf_exit, which means that program wrote
				 * something into it earlier
				 */
6525
				err = check_reg_arg(env, BPF_REG_0, SRC_OP);
6526 6527 6528
				if (err)
					return err;

6529
				if (is_pointer_value(env, BPF_REG_0)) {
6530
					verbose(env, "R0 leaks addr as return value\n");
6531 6532 6533
					return -EACCES;
				}

6534 6535 6536
				err = check_return_code(env);
				if (err)
					return err;
6537
process_bpf_exit:
6538 6539
				err = pop_stack(env, &env->prev_insn_idx,
						&env->insn_idx);
6540 6541 6542
				if (err < 0) {
					if (err != -ENOENT)
						return err;
6543 6544 6545 6546 6547 6548
					break;
				} else {
					do_print_state = true;
					continue;
				}
			} else {
6549
				err = check_cond_jmp_op(env, insn, &env->insn_idx);
6550 6551 6552 6553 6554 6555 6556
				if (err)
					return err;
			}
		} else if (class == BPF_LD) {
			u8 mode = BPF_MODE(insn->code);

			if (mode == BPF_ABS || mode == BPF_IND) {
6557 6558 6559 6560
				err = check_ld_abs(env, insn);
				if (err)
					return err;

6561 6562 6563 6564 6565
			} else if (mode == BPF_IMM) {
				err = check_ld_imm(env, insn);
				if (err)
					return err;

6566 6567
				env->insn_idx++;
				env->insn_aux_data[env->insn_idx].seen = true;
6568
			} else {
6569
				verbose(env, "invalid BPF_LD mode\n");
6570 6571 6572
				return -EINVAL;
			}
		} else {
6573
			verbose(env, "unknown insn class %d\n", class);
6574 6575 6576
			return -EINVAL;
		}

6577
		env->insn_idx++;
6578 6579
	}

6580 6581
	verbose(env, "processed %d insns (limit %d), stack depth ",
		insn_processed, BPF_COMPLEXITY_LIMIT_INSNS);
6582
	for (i = 0; i < env->subprog_cnt; i++) {
6583
		u32 depth = env->subprog_info[i].stack_depth;
6584 6585

		verbose(env, "%d", depth);
6586
		if (i + 1 < env->subprog_cnt)
6587 6588 6589
			verbose(env, "+");
	}
	verbose(env, "\n");
6590
	env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
6591 6592 6593
	return 0;
}

6594 6595 6596
static int check_map_prealloc(struct bpf_map *map)
{
	return (map->map_type != BPF_MAP_TYPE_HASH &&
6597 6598
		map->map_type != BPF_MAP_TYPE_PERCPU_HASH &&
		map->map_type != BPF_MAP_TYPE_HASH_OF_MAPS) ||
6599 6600 6601
		!(map->map_flags & BPF_F_NO_PREALLOC);
}

6602 6603 6604 6605 6606 6607 6608 6609 6610 6611 6612 6613 6614
static bool is_tracing_prog_type(enum bpf_prog_type type)
{
	switch (type) {
	case BPF_PROG_TYPE_KPROBE:
	case BPF_PROG_TYPE_TRACEPOINT:
	case BPF_PROG_TYPE_PERF_EVENT:
	case BPF_PROG_TYPE_RAW_TRACEPOINT:
		return true;
	default:
		return false;
	}
}

6615 6616
static int check_map_prog_compatibility(struct bpf_verifier_env *env,
					struct bpf_map *map,
6617 6618 6619
					struct bpf_prog *prog)

{
6620 6621 6622 6623 6624 6625 6626
	/* Make sure that BPF_PROG_TYPE_PERF_EVENT programs only use
	 * preallocated hash maps, since doing memory allocation
	 * in overflow_handler can crash depending on where nmi got
	 * triggered.
	 */
	if (prog->type == BPF_PROG_TYPE_PERF_EVENT) {
		if (!check_map_prealloc(map)) {
6627
			verbose(env, "perf_event programs can only use preallocated hash map\n");
6628 6629 6630 6631
			return -EINVAL;
		}
		if (map->inner_map_meta &&
		    !check_map_prealloc(map->inner_map_meta)) {
6632
			verbose(env, "perf_event programs can only use preallocated inner hash map\n");
6633 6634
			return -EINVAL;
		}
6635
	}
6636

6637 6638 6639 6640 6641 6642 6643
	if ((is_tracing_prog_type(prog->type) ||
	     prog->type == BPF_PROG_TYPE_SOCKET_FILTER) &&
	    map_value_has_spin_lock(map)) {
		verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
		return -EINVAL;
	}

6644
	if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) &&
6645
	    !bpf_offload_prog_map_match(prog, map)) {
6646 6647 6648 6649
		verbose(env, "offload device mismatch between prog and map\n");
		return -EINVAL;
	}

6650 6651 6652
	return 0;
}

6653 6654 6655 6656 6657 6658
static bool bpf_map_is_cgroup_storage(struct bpf_map *map)
{
	return (map->map_type == BPF_MAP_TYPE_CGROUP_STORAGE ||
		map->map_type == BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE);
}

6659 6660 6661
/* look for pseudo eBPF instructions that access map FDs and
 * replace them with actual map pointers
 */
6662
static int replace_map_fd_with_map_ptr(struct bpf_verifier_env *env)
6663 6664 6665
{
	struct bpf_insn *insn = env->prog->insnsi;
	int insn_cnt = env->prog->len;
6666
	int i, j, err;
6667

6668
	err = bpf_prog_calc_tag(env->prog);
6669 6670 6671
	if (err)
		return err;

6672
	for (i = 0; i < insn_cnt; i++, insn++) {
6673
		if (BPF_CLASS(insn->code) == BPF_LDX &&
6674
		    (BPF_MODE(insn->code) != BPF_MEM || insn->imm != 0)) {
6675
			verbose(env, "BPF_LDX uses reserved fields\n");
6676 6677 6678
			return -EINVAL;
		}

6679 6680 6681
		if (BPF_CLASS(insn->code) == BPF_STX &&
		    ((BPF_MODE(insn->code) != BPF_MEM &&
		      BPF_MODE(insn->code) != BPF_XADD) || insn->imm != 0)) {
6682
			verbose(env, "BPF_STX uses reserved fields\n");
6683 6684 6685
			return -EINVAL;
		}

6686 6687 6688 6689 6690 6691 6692
		if (insn[0].code == (BPF_LD | BPF_IMM | BPF_DW)) {
			struct bpf_map *map;
			struct fd f;

			if (i == insn_cnt - 1 || insn[1].code != 0 ||
			    insn[1].dst_reg != 0 || insn[1].src_reg != 0 ||
			    insn[1].off != 0) {
6693
				verbose(env, "invalid bpf_ld_imm64 insn\n");
6694 6695 6696 6697 6698 6699 6700
				return -EINVAL;
			}

			if (insn->src_reg == 0)
				/* valid generic load 64-bit imm */
				goto next_insn;

6701 6702 6703
			if (insn[0].src_reg != BPF_PSEUDO_MAP_FD ||
			    insn[1].imm != 0) {
				verbose(env, "unrecognized bpf_ld_imm64 insn\n");
6704 6705 6706
				return -EINVAL;
			}

6707
			f = fdget(insn[0].imm);
6708
			map = __bpf_map_get(f);
6709
			if (IS_ERR(map)) {
6710
				verbose(env, "fd %d is not pointing to valid bpf_map\n",
6711
					insn[0].imm);
6712 6713 6714
				return PTR_ERR(map);
			}

6715
			err = check_map_prog_compatibility(env, map, env->prog);
6716 6717 6718 6719 6720
			if (err) {
				fdput(f);
				return err;
			}

6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737 6738 6739
			/* store map pointer inside BPF_LD_IMM64 instruction */
			insn[0].imm = (u32) (unsigned long) map;
			insn[1].imm = ((u64) (unsigned long) map) >> 32;

			/* check whether we recorded this map already */
			for (j = 0; j < env->used_map_cnt; j++)
				if (env->used_maps[j] == map) {
					fdput(f);
					goto next_insn;
				}

			if (env->used_map_cnt >= MAX_USED_MAPS) {
				fdput(f);
				return -E2BIG;
			}

			/* hold the map. If the program is rejected by verifier,
			 * the map will be released by release_maps() or it
			 * will be used by the valid program until it's unloaded
6740
			 * and all maps are released in free_used_maps()
6741
			 */
6742 6743 6744 6745 6746 6747 6748
			map = bpf_map_inc(map, false);
			if (IS_ERR(map)) {
				fdput(f);
				return PTR_ERR(map);
			}
			env->used_maps[env->used_map_cnt++] = map;

6749
			if (bpf_map_is_cgroup_storage(map) &&
6750
			    bpf_cgroup_storage_assign(env->prog, map)) {
6751
				verbose(env, "only one cgroup storage of each type is allowed\n");
6752 6753 6754 6755
				fdput(f);
				return -EBUSY;
			}

6756 6757 6758 6759
			fdput(f);
next_insn:
			insn++;
			i++;
6760 6761 6762 6763 6764 6765 6766
			continue;
		}

		/* Basic sanity check before we invest more work here. */
		if (!bpf_opcode_in_insntable(insn->code)) {
			verbose(env, "unknown opcode %02x\n", insn->code);
			return -EINVAL;
6767 6768 6769 6770 6771 6772 6773 6774 6775 6776 6777
		}
	}

	/* now all pseudo BPF_LD_IMM64 instructions load valid
	 * 'struct bpf_map *' into a register instead of user map_fd.
	 * These pointers will be used later by verifier to validate map access.
	 */
	return 0;
}

/* drop refcnt of maps used by the rejected program */
6778
static void release_maps(struct bpf_verifier_env *env)
6779
{
6780
	enum bpf_cgroup_storage_type stype;
6781 6782
	int i;

6783 6784 6785
	for_each_cgroup_storage_type(stype) {
		if (!env->prog->aux->cgroup_storage[stype])
			continue;
6786
		bpf_cgroup_storage_release(env->prog,
6787 6788
			env->prog->aux->cgroup_storage[stype]);
	}
6789

6790 6791 6792 6793 6794
	for (i = 0; i < env->used_map_cnt; i++)
		bpf_map_put(env->used_maps[i]);
}

/* convert pseudo BPF_LD_IMM64 into generic BPF_LD_IMM64 */
6795
static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
6796 6797 6798 6799 6800 6801 6802 6803 6804 6805
{
	struct bpf_insn *insn = env->prog->insnsi;
	int insn_cnt = env->prog->len;
	int i;

	for (i = 0; i < insn_cnt; i++, insn++)
		if (insn->code == (BPF_LD | BPF_IMM | BPF_DW))
			insn->src_reg = 0;
}

6806 6807 6808 6809 6810 6811 6812 6813
/* single env->prog->insni[off] instruction was replaced with the range
 * insni[off, off + cnt).  Adjust corresponding insn_aux_data by copying
 * [0, off) and [off, end) to new locations, so the patched range stays zero
 */
static int adjust_insn_aux_data(struct bpf_verifier_env *env, u32 prog_len,
				u32 off, u32 cnt)
{
	struct bpf_insn_aux_data *new_data, *old_data = env->insn_aux_data;
6814
	int i;
6815 6816 6817

	if (cnt == 1)
		return 0;
6818 6819
	new_data = vzalloc(array_size(prog_len,
				      sizeof(struct bpf_insn_aux_data)));
6820 6821 6822 6823 6824
	if (!new_data)
		return -ENOMEM;
	memcpy(new_data, old_data, sizeof(struct bpf_insn_aux_data) * off);
	memcpy(new_data + off + cnt - 1, old_data + off,
	       sizeof(struct bpf_insn_aux_data) * (prog_len - off - cnt + 1));
6825 6826
	for (i = off; i < off + cnt - 1; i++)
		new_data[i].seen = true;
6827 6828 6829 6830 6831
	env->insn_aux_data = new_data;
	vfree(old_data);
	return 0;
}

6832 6833 6834 6835 6836 6837
static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
{
	int i;

	if (len == 1)
		return;
6838 6839
	/* NOTE: fake 'exit' subprog should be updated as well. */
	for (i = 0; i <= env->subprog_cnt; i++) {
6840
		if (env->subprog_info[i].start <= off)
6841
			continue;
6842
		env->subprog_info[i].start += len - 1;
6843 6844 6845
	}
}

6846 6847 6848 6849 6850 6851 6852 6853 6854 6855
static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off,
					    const struct bpf_insn *patch, u32 len)
{
	struct bpf_prog *new_prog;

	new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
	if (!new_prog)
		return NULL;
	if (adjust_insn_aux_data(env, new_prog->len, off, len))
		return NULL;
6856
	adjust_subprog_starts(env, off, len);
6857 6858 6859
	return new_prog;
}

6860 6861 6862 6863 6864 6865 6866 6867 6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904 6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940 6941 6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985
static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env,
					      u32 off, u32 cnt)
{
	int i, j;

	/* find first prog starting at or after off (first to remove) */
	for (i = 0; i < env->subprog_cnt; i++)
		if (env->subprog_info[i].start >= off)
			break;
	/* find first prog starting at or after off + cnt (first to stay) */
	for (j = i; j < env->subprog_cnt; j++)
		if (env->subprog_info[j].start >= off + cnt)
			break;
	/* if j doesn't start exactly at off + cnt, we are just removing
	 * the front of previous prog
	 */
	if (env->subprog_info[j].start != off + cnt)
		j--;

	if (j > i) {
		struct bpf_prog_aux *aux = env->prog->aux;
		int move;

		/* move fake 'exit' subprog as well */
		move = env->subprog_cnt + 1 - j;

		memmove(env->subprog_info + i,
			env->subprog_info + j,
			sizeof(*env->subprog_info) * move);
		env->subprog_cnt -= j - i;

		/* remove func_info */
		if (aux->func_info) {
			move = aux->func_info_cnt - j;

			memmove(aux->func_info + i,
				aux->func_info + j,
				sizeof(*aux->func_info) * move);
			aux->func_info_cnt -= j - i;
			/* func_info->insn_off is set after all code rewrites,
			 * in adjust_btf_func() - no need to adjust
			 */
		}
	} else {
		/* convert i from "first prog to remove" to "first to adjust" */
		if (env->subprog_info[i].start == off)
			i++;
	}

	/* update fake 'exit' subprog as well */
	for (; i <= env->subprog_cnt; i++)
		env->subprog_info[i].start -= cnt;

	return 0;
}

static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off,
				      u32 cnt)
{
	struct bpf_prog *prog = env->prog;
	u32 i, l_off, l_cnt, nr_linfo;
	struct bpf_line_info *linfo;

	nr_linfo = prog->aux->nr_linfo;
	if (!nr_linfo)
		return 0;

	linfo = prog->aux->linfo;

	/* find first line info to remove, count lines to be removed */
	for (i = 0; i < nr_linfo; i++)
		if (linfo[i].insn_off >= off)
			break;

	l_off = i;
	l_cnt = 0;
	for (; i < nr_linfo; i++)
		if (linfo[i].insn_off < off + cnt)
			l_cnt++;
		else
			break;

	/* First live insn doesn't match first live linfo, it needs to "inherit"
	 * last removed linfo.  prog is already modified, so prog->len == off
	 * means no live instructions after (tail of the program was removed).
	 */
	if (prog->len != off && l_cnt &&
	    (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
		l_cnt--;
		linfo[--i].insn_off = off + cnt;
	}

	/* remove the line info which refer to the removed instructions */
	if (l_cnt) {
		memmove(linfo + l_off, linfo + i,
			sizeof(*linfo) * (nr_linfo - i));

		prog->aux->nr_linfo -= l_cnt;
		nr_linfo = prog->aux->nr_linfo;
	}

	/* pull all linfo[i].insn_off >= off + cnt in by cnt */
	for (i = l_off; i < nr_linfo; i++)
		linfo[i].insn_off -= cnt;

	/* fix up all subprogs (incl. 'exit') which start >= off */
	for (i = 0; i <= env->subprog_cnt; i++)
		if (env->subprog_info[i].linfo_idx > l_off) {
			/* program may have started in the removed region but
			 * may not be fully removed
			 */
			if (env->subprog_info[i].linfo_idx >= l_off + l_cnt)
				env->subprog_info[i].linfo_idx -= l_cnt;
			else
				env->subprog_info[i].linfo_idx = l_off;
		}

	return 0;
}

static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
{
	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
	unsigned int orig_prog_len = env->prog->len;
	int err;

6986 6987 6988
	if (bpf_prog_is_dev_bound(env->prog->aux))
		bpf_prog_offload_remove_insns(env, off, cnt);

6989 6990 6991 6992 6993 6994 6995 6996 6997 6998 6999 7000 7001 7002 7003 7004 7005 7006
	err = bpf_remove_insns(env->prog, off, cnt);
	if (err)
		return err;

	err = adjust_subprog_starts_after_remove(env, off, cnt);
	if (err)
		return err;

	err = bpf_adj_linfo_after_remove(env, off, cnt);
	if (err)
		return err;

	memmove(aux_data + off,	aux_data + off + cnt,
		sizeof(*aux_data) * (orig_prog_len - off - cnt));

	return 0;
}

7007 7008 7009 7010 7011 7012 7013 7014 7015 7016
/* The verifier does more data flow analysis than llvm and will not
 * explore branches that are dead at run time. Malicious programs can
 * have dead code too. Therefore replace all dead at-run-time code
 * with 'ja -1'.
 *
 * Just nops are not optimal, e.g. if they would sit at the end of the
 * program and through another bug we would manage to jump there, then
 * we'd execute beyond program memory otherwise. Returning exception
 * code also wouldn't work since we can have subprogs where the dead
 * code could be located.
7017 7018 7019 7020
 */
static void sanitize_dead_code(struct bpf_verifier_env *env)
{
	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
7021
	struct bpf_insn trap = BPF_JMP_IMM(BPF_JA, 0, 0, -1);
7022 7023 7024 7025 7026 7027 7028
	struct bpf_insn *insn = env->prog->insnsi;
	const int insn_cnt = env->prog->len;
	int i;

	for (i = 0; i < insn_cnt; i++) {
		if (aux_data[i].seen)
			continue;
7029
		memcpy(insn + i, &trap, sizeof(trap));
7030 7031 7032
	}
}

7033 7034 7035 7036
static bool insn_is_cond_jump(u8 code)
{
	u8 op;

Jiong Wang's avatar
Jiong Wang committed
7037 7038 7039
	if (BPF_CLASS(code) == BPF_JMP32)
		return true;

7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7058 7059 7060 7061 7062 7063 7064 7065
	if (BPF_CLASS(code) != BPF_JMP)
		return false;

	op = BPF_OP(code);
	return op != BPF_JA && op != BPF_EXIT && op != BPF_CALL;
}

static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
{
	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
	struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
	struct bpf_insn *insn = env->prog->insnsi;
	const int insn_cnt = env->prog->len;
	int i;

	for (i = 0; i < insn_cnt; i++, insn++) {
		if (!insn_is_cond_jump(insn->code))
			continue;

		if (!aux_data[i + 1].seen)
			ja.off = insn->off;
		else if (!aux_data[i + 1 + insn->off].seen)
			ja.off = 0;
		else
			continue;

7066 7067 7068
		if (bpf_prog_is_dev_bound(env->prog->aux))
			bpf_prog_offload_replace_insn(env, i, &ja);

7069 7070 7071 7072
		memcpy(insn, &ja, sizeof(ja));
	}
}

7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091 7092 7093 7094 7095 7096
static int opt_remove_dead_code(struct bpf_verifier_env *env)
{
	struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
	int insn_cnt = env->prog->len;
	int i, err;

	for (i = 0; i < insn_cnt; i++) {
		int j;

		j = 0;
		while (i + j < insn_cnt && !aux_data[i + j].seen)
			j++;
		if (!j)
			continue;

		err = verifier_remove_insns(env, i, j);
		if (err)
			return err;
		insn_cnt = env->prog->len;
	}

	return 0;
}

7097 7098 7099 7100 7101 7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114 7115 7116 7117
static int opt_remove_nops(struct bpf_verifier_env *env)
{
	const struct bpf_insn ja = BPF_JMP_IMM(BPF_JA, 0, 0, 0);
	struct bpf_insn *insn = env->prog->insnsi;
	int insn_cnt = env->prog->len;
	int i, err;

	for (i = 0; i < insn_cnt; i++) {
		if (memcmp(&insn[i], &ja, sizeof(ja)))
			continue;

		err = verifier_remove_insns(env, i, 1);
		if (err)
			return err;
		insn_cnt--;
		i--;
	}

	return 0;
}

7118 7119 7120 7121
/* convert load instructions that access fields of a context type into a
 * sequence of instructions that access fields of the underlying structure:
 *     struct __sk_buff    -> struct sk_buff
 *     struct bpf_sock_ops -> struct sock
7122
 */
7123
static int convert_ctx_accesses(struct bpf_verifier_env *env)
7124
{
7125
	const struct bpf_verifier_ops *ops = env->ops;
7126
	int i, cnt, size, ctx_field_size, delta = 0;
7127
	const int insn_cnt = env->prog->len;
7128
	struct bpf_insn insn_buf[16], *insn;
7129
	u32 target_size, size_default, off;
7130
	struct bpf_prog *new_prog;
7131
	enum bpf_access_type type;
7132
	bool is_narrower_load;
7133

7134 7135 7136 7137 7138
	if (ops->gen_prologue || env->seen_direct_write) {
		if (!ops->gen_prologue) {
			verbose(env, "bpf verifier is misconfigured\n");
			return -EINVAL;
		}
7139 7140 7141
		cnt = ops->gen_prologue(insn_buf, env->seen_direct_write,
					env->prog);
		if (cnt >= ARRAY_SIZE(insn_buf)) {
7142
			verbose(env, "bpf verifier is misconfigured\n");
7143 7144
			return -EINVAL;
		} else if (cnt) {
7145
			new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
7146 7147
			if (!new_prog)
				return -ENOMEM;
7148

7149
			env->prog = new_prog;
7150
			delta += cnt - 1;
7151 7152 7153
		}
	}

7154
	if (bpf_prog_is_dev_bound(env->prog->aux))
7155 7156
		return 0;

7157
	insn = env->prog->insnsi + delta;
7158

7159
	for (i = 0; i < insn_cnt; i++, insn++) {
7160 7161
		bpf_convert_ctx_access_t convert_ctx_access;

7162 7163 7164
		if (insn->code == (BPF_LDX | BPF_MEM | BPF_B) ||
		    insn->code == (BPF_LDX | BPF_MEM | BPF_H) ||
		    insn->code == (BPF_LDX | BPF_MEM | BPF_W) ||
7165
		    insn->code == (BPF_LDX | BPF_MEM | BPF_DW))
7166
			type = BPF_READ;
7167 7168 7169
		else if (insn->code == (BPF_STX | BPF_MEM | BPF_B) ||
			 insn->code == (BPF_STX | BPF_MEM | BPF_H) ||
			 insn->code == (BPF_STX | BPF_MEM | BPF_W) ||
7170
			 insn->code == (BPF_STX | BPF_MEM | BPF_DW))
7171 7172
			type = BPF_WRITE;
		else
7173 7174
			continue;

7175 7176 7177 7178 7179 7180 7181 7182 7183 7184 7185 7186 7187 7188 7189 7190 7191 7192 7193 7194 7195 7196 7197 7198 7199 7200 7201 7202
		if (type == BPF_WRITE &&
		    env->insn_aux_data[i + delta].sanitize_stack_off) {
			struct bpf_insn patch[] = {
				/* Sanitize suspicious stack slot with zero.
				 * There are no memory dependencies for this store,
				 * since it's only using frame pointer and immediate
				 * constant of zero
				 */
				BPF_ST_MEM(BPF_DW, BPF_REG_FP,
					   env->insn_aux_data[i + delta].sanitize_stack_off,
					   0),
				/* the original STX instruction will immediately
				 * overwrite the same stack slot with appropriate value
				 */
				*insn,
			};

			cnt = ARRAY_SIZE(patch);
			new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
			if (!new_prog)
				return -ENOMEM;

			delta    += cnt - 1;
			env->prog = new_prog;
			insn      = new_prog->insnsi + i + delta;
			continue;
		}

7203 7204 7205 7206 7207 7208 7209
		switch (env->insn_aux_data[i + delta].ptr_type) {
		case PTR_TO_CTX:
			if (!ops->convert_ctx_access)
				continue;
			convert_ctx_access = ops->convert_ctx_access;
			break;
		case PTR_TO_SOCKET:
7210
		case PTR_TO_SOCK_COMMON:
7211 7212
			convert_ctx_access = bpf_sock_convert_ctx_access;
			break;
7213 7214 7215
		case PTR_TO_TCP_SOCK:
			convert_ctx_access = bpf_tcp_sock_convert_ctx_access;
			break;
7216
		default:
7217
			continue;
7218
		}
7219

7220
		ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
7221
		size = BPF_LDST_BYTES(insn);
7222 7223 7224 7225 7226 7227

		/* If the read access is a narrower load of the field,
		 * convert to a 4/8-byte load, to minimum program type specific
		 * convert_ctx_access changes. If conversion is successful,
		 * we will apply proper mask to the result.
		 */
7228
		is_narrower_load = size < ctx_field_size;
7229 7230
		size_default = bpf_ctx_off_adjust_machine(ctx_field_size);
		off = insn->off;
7231
		if (is_narrower_load) {
7232 7233 7234
			u8 size_code;

			if (type == BPF_WRITE) {
7235
				verbose(env, "bpf verifier narrow ctx access misconfigured\n");
7236 7237
				return -EINVAL;
			}
7238

7239
			size_code = BPF_H;
7240 7241 7242 7243
			if (ctx_field_size == 4)
				size_code = BPF_W;
			else if (ctx_field_size == 8)
				size_code = BPF_DW;
7244

7245
			insn->off = off & ~(size_default - 1);
7246 7247
			insn->code = BPF_LDX | BPF_MEM | size_code;
		}
7248 7249

		target_size = 0;
7250 7251
		cnt = convert_ctx_access(type, insn, insn_buf, env->prog,
					 &target_size);
7252 7253
		if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf) ||
		    (ctx_field_size && !target_size)) {
7254
			verbose(env, "bpf verifier is misconfigured\n");
7255 7256
			return -EINVAL;
		}
7257 7258

		if (is_narrower_load && size < target_size) {
7259 7260 7261 7262 7263 7264 7265
			u8 shift = (off & (size_default - 1)) * 8;

			if (ctx_field_size <= 4) {
				if (shift)
					insn_buf[cnt++] = BPF_ALU32_IMM(BPF_RSH,
									insn->dst_reg,
									shift);
7266
				insn_buf[cnt++] = BPF_ALU32_IMM(BPF_AND, insn->dst_reg,
7267
								(1 << size * 8) - 1);
7268 7269 7270 7271 7272
			} else {
				if (shift)
					insn_buf[cnt++] = BPF_ALU64_IMM(BPF_RSH,
									insn->dst_reg,
									shift);
7273
				insn_buf[cnt++] = BPF_ALU64_IMM(BPF_AND, insn->dst_reg,
7274
								(1 << size * 8) - 1);
7275
			}
7276
		}
7277

7278
		new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
7279 7280 7281
		if (!new_prog)
			return -ENOMEM;

7282
		delta += cnt - 1;
7283 7284 7285

		/* keep walking new program and skip insns we just inserted */
		env->prog = new_prog;
7286
		insn      = new_prog->insnsi + i + delta;
7287 7288 7289 7290 7291
	}

	return 0;
}

7292 7293 7294 7295
static int jit_subprogs(struct bpf_verifier_env *env)
{
	struct bpf_prog *prog = env->prog, **func, *tmp;
	int i, j, subprog_start, subprog_end = 0, len, subprog;
7296
	struct bpf_insn *insn;
7297
	void *old_bpf_func;
7298
	int err;
7299

7300
	if (env->subprog_cnt <= 1)
7301 7302
		return 0;

7303
	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
7304 7305 7306
		if (insn->code != (BPF_JMP | BPF_CALL) ||
		    insn->src_reg != BPF_PSEUDO_CALL)
			continue;
7307 7308 7309 7310
		/* Upon error here we cannot fall back to interpreter but
		 * need a hard reject of the program. Thus -EFAULT is
		 * propagated in any case.
		 */
7311 7312 7313 7314 7315 7316 7317 7318 7319
		subprog = find_subprog(env, i + insn->imm + 1);
		if (subprog < 0) {
			WARN_ONCE(1, "verifier bug. No program starts at insn %d\n",
				  i + insn->imm + 1);
			return -EFAULT;
		}
		/* temporarily remember subprog id inside insn instead of
		 * aux_data, since next loop will split up all insns into funcs
		 */
7320
		insn->off = subprog;
7321 7322 7323 7324 7325 7326 7327 7328
		/* remember original imm in case JIT fails and fallback
		 * to interpreter will be needed
		 */
		env->insn_aux_data[i].call_imm = insn->imm;
		/* point imm to __bpf_call_base+1 from JITs point of view */
		insn->imm = 1;
	}

7329 7330 7331 7332 7333
	err = bpf_prog_alloc_jited_linfo(prog);
	if (err)
		goto out_undo_insn;

	err = -ENOMEM;
7334
	func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
7335
	if (!func)
7336
		goto out_undo_insn;
7337

7338
	for (i = 0; i < env->subprog_cnt; i++) {
7339
		subprog_start = subprog_end;
7340
		subprog_end = env->subprog_info[i + 1].start;
7341 7342

		len = subprog_end - subprog_start;
7343 7344 7345 7346 7347 7348
		/* BPF_PROG_RUN doesn't call subprogs directly,
		 * hence main prog stats include the runtime of subprogs.
		 * subprogs don't have IDs and not reachable via prog_get_next_id
		 * func[i]->aux->stats will never be accessed and stays NULL
		 */
		func[i] = bpf_prog_alloc_no_stats(bpf_prog_size(len), GFP_USER);
7349 7350 7351 7352
		if (!func[i])
			goto out_free;
		memcpy(func[i]->insnsi, &prog->insnsi[subprog_start],
		       len * sizeof(struct bpf_insn));
7353
		func[i]->type = prog->type;
7354
		func[i]->len = len;
7355 7356
		if (bpf_prog_calc_tag(func[i]))
			goto out_free;
7357
		func[i]->is_func = 1;
7358 7359 7360 7361 7362
		func[i]->aux->func_idx = i;
		/* the btf and func_info will be freed only at prog->aux */
		func[i]->aux->btf = prog->aux->btf;
		func[i]->aux->func_info = prog->aux->func_info;

7363 7364 7365 7366
		/* Use bpf_prog_F_tag to indicate functions in stack traces.
		 * Long term would need debug info to populate names
		 */
		func[i]->aux->name[0] = 'F';
7367
		func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
7368
		func[i]->jit_requested = 1;
7369 7370 7371 7372
		func[i]->aux->linfo = prog->aux->linfo;
		func[i]->aux->nr_linfo = prog->aux->nr_linfo;
		func[i]->aux->jited_linfo = prog->aux->jited_linfo;
		func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
7373 7374 7375 7376 7377 7378 7379 7380 7381 7382 7383
		func[i] = bpf_int_jit_compile(func[i]);
		if (!func[i]->jited) {
			err = -ENOTSUPP;
			goto out_free;
		}
		cond_resched();
	}
	/* at this point all bpf functions were successfully JITed
	 * now populate all bpf_calls with correct addresses and
	 * run last pass of JIT
	 */
7384
	for (i = 0; i < env->subprog_cnt; i++) {
7385 7386 7387 7388 7389 7390 7391 7392 7393 7394
		insn = func[i]->insnsi;
		for (j = 0; j < func[i]->len; j++, insn++) {
			if (insn->code != (BPF_JMP | BPF_CALL) ||
			    insn->src_reg != BPF_PSEUDO_CALL)
				continue;
			subprog = insn->off;
			insn->imm = (u64 (*)(u64, u64, u64, u64, u64))
				func[subprog]->bpf_func -
				__bpf_call_base;
		}
7395 7396 7397 7398 7399 7400 7401 7402 7403 7404 7405 7406 7407 7408

		/* we use the aux data to keep a list of the start addresses
		 * of the JITed images for each function in the program
		 *
		 * for some architectures, such as powerpc64, the imm field
		 * might not be large enough to hold the offset of the start
		 * address of the callee's JITed image from __bpf_call_base
		 *
		 * in such cases, we can lookup the start address of a callee
		 * by using its subprog id, available from the off field of
		 * the call instruction, as an index for this list
		 */
		func[i]->aux->func = func;
		func[i]->aux->func_cnt = env->subprog_cnt;
7409
	}
7410
	for (i = 0; i < env->subprog_cnt; i++) {
7411 7412 7413 7414
		old_bpf_func = func[i]->bpf_func;
		tmp = bpf_int_jit_compile(func[i]);
		if (tmp != func[i] || func[i]->bpf_func != old_bpf_func) {
			verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
7415
			err = -ENOTSUPP;
7416 7417 7418 7419 7420 7421 7422 7423
			goto out_free;
		}
		cond_resched();
	}

	/* finally lock prog and jit images for all functions and
	 * populate kallsysm
	 */
7424
	for (i = 0; i < env->subprog_cnt; i++) {
7425 7426 7427
		bpf_prog_lock_ro(func[i]);
		bpf_prog_kallsyms_add(func[i]);
	}
7428 7429 7430 7431 7432 7433 7434 7435 7436 7437 7438

	/* Last step: make now unused interpreter insns from main
	 * prog consistent for later dump requests, so they can
	 * later look the same as if they were interpreted only.
	 */
	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
		if (insn->code != (BPF_JMP | BPF_CALL) ||
		    insn->src_reg != BPF_PSEUDO_CALL)
			continue;
		insn->off = env->insn_aux_data[i].call_imm;
		subprog = find_subprog(env, i + insn->off + 1);
7439
		insn->imm = subprog;
7440 7441
	}

7442 7443 7444
	prog->jited = 1;
	prog->bpf_func = func[0]->bpf_func;
	prog->aux->func = func;
7445
	prog->aux->func_cnt = env->subprog_cnt;
7446
	bpf_prog_free_unused_jited_linfo(prog);
7447 7448
	return 0;
out_free:
7449
	for (i = 0; i < env->subprog_cnt; i++)
7450 7451 7452
		if (func[i])
			bpf_jit_free(func[i]);
	kfree(func);
7453
out_undo_insn:
7454 7455 7456 7457 7458 7459 7460 7461 7462
	/* cleanup main prog to be interpreted */
	prog->jit_requested = 0;
	for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
		if (insn->code != (BPF_JMP | BPF_CALL) ||
		    insn->src_reg != BPF_PSEUDO_CALL)
			continue;
		insn->off = 0;
		insn->imm = env->insn_aux_data[i].call_imm;
	}
7463
	bpf_prog_free_jited_linfo(prog);
7464 7465 7466
	return err;
}

7467 7468
static int fixup_call_args(struct bpf_verifier_env *env)
{
7469
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
7470 7471 7472
	struct bpf_prog *prog = env->prog;
	struct bpf_insn *insn = prog->insnsi;
	int i, depth;
7473
#endif
7474
	int err = 0;
7475

7476 7477
	if (env->prog->jit_requested &&
	    !bpf_prog_is_dev_bound(env->prog->aux)) {
7478 7479
		err = jit_subprogs(env);
		if (err == 0)
7480
			return 0;
7481 7482
		if (err == -EFAULT)
			return err;
7483 7484
	}
#ifndef CONFIG_BPF_JIT_ALWAYS_ON
7485 7486 7487 7488 7489 7490 7491 7492 7493
	for (i = 0; i < prog->len; i++, insn++) {
		if (insn->code != (BPF_JMP | BPF_CALL) ||
		    insn->src_reg != BPF_PSEUDO_CALL)
			continue;
		depth = get_callee_stack_depth(env, insn, i);
		if (depth < 0)
			return depth;
		bpf_patch_call_args(insn, depth);
	}
7494 7495 7496
	err = 0;
#endif
	return err;
7497 7498
}

7499
/* fixup insn->imm field of bpf_call instructions
7500
 * and inline eligible helpers as explicit sequence of BPF instructions
7501 7502 7503
 *
 * this function is called after eBPF program passed verification
 */
7504
static int fixup_bpf_calls(struct bpf_verifier_env *env)
7505
{
7506 7507
	struct bpf_prog *prog = env->prog;
	struct bpf_insn *insn = prog->insnsi;
7508
	const struct bpf_func_proto *fn;
7509
	const int insn_cnt = prog->len;
7510
	const struct bpf_map_ops *ops;
7511
	struct bpf_insn_aux_data *aux;
7512 7513 7514 7515
	struct bpf_insn insn_buf[16];
	struct bpf_prog *new_prog;
	struct bpf_map *map_ptr;
	int i, cnt, delta = 0;
7516

7517
	for (i = 0; i < insn_cnt; i++, insn++) {
7518 7519 7520
		if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
		    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
		    insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
7521
		    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548
			bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
			struct bpf_insn mask_and_div[] = {
				BPF_MOV32_REG(insn->src_reg, insn->src_reg),
				/* Rx div 0 -> 0 */
				BPF_JMP_IMM(BPF_JNE, insn->src_reg, 0, 2),
				BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
				BPF_JMP_IMM(BPF_JA, 0, 0, 1),
				*insn,
			};
			struct bpf_insn mask_and_mod[] = {
				BPF_MOV32_REG(insn->src_reg, insn->src_reg),
				/* Rx mod 0 -> Rx */
				BPF_JMP_IMM(BPF_JEQ, insn->src_reg, 0, 1),
				*insn,
			};
			struct bpf_insn *patchlet;

			if (insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
			    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
				patchlet = mask_and_div + (is64 ? 1 : 0);
				cnt = ARRAY_SIZE(mask_and_div) - (is64 ? 1 : 0);
			} else {
				patchlet = mask_and_mod + (is64 ? 1 : 0);
				cnt = ARRAY_SIZE(mask_and_mod) - (is64 ? 1 : 0);
			}

			new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
7549 7550 7551 7552 7553 7554 7555 7556 7557
			if (!new_prog)
				return -ENOMEM;

			delta    += cnt - 1;
			env->prog = prog = new_prog;
			insn      = new_prog->insnsi + i + delta;
			continue;
		}

7558 7559 7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576
		if (BPF_CLASS(insn->code) == BPF_LD &&
		    (BPF_MODE(insn->code) == BPF_ABS ||
		     BPF_MODE(insn->code) == BPF_IND)) {
			cnt = env->ops->gen_ld_abs(insn, insn_buf);
			if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
				verbose(env, "bpf verifier is misconfigured\n");
				return -EINVAL;
			}

			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
			if (!new_prog)
				return -ENOMEM;

			delta    += cnt - 1;
			env->prog = prog = new_prog;
			insn      = new_prog->insnsi + i + delta;
			continue;
		}

7577 7578 7579 7580 7581 7582 7583 7584 7585 7586
		if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
		    insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
			const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
			const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
			struct bpf_insn insn_buf[16];
			struct bpf_insn *patch = &insn_buf[0];
			bool issrc, isneg;
			u32 off_reg;

			aux = &env->insn_aux_data[i + delta];
7587 7588
			if (!aux->alu_state ||
			    aux->alu_state == BPF_ALU_NON_POINTER)
7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624 7625 7626 7627 7628
				continue;

			isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
			issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
				BPF_ALU_SANITIZE_SRC;

			off_reg = issrc ? insn->src_reg : insn->dst_reg;
			if (isneg)
				*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
			*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit - 1);
			*patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
			*patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
			*patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
			*patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
			if (issrc) {
				*patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX,
							 off_reg);
				insn->src_reg = BPF_REG_AX;
			} else {
				*patch++ = BPF_ALU64_REG(BPF_AND, off_reg,
							 BPF_REG_AX);
			}
			if (isneg)
				insn->code = insn->code == code_add ?
					     code_sub : code_add;
			*patch++ = *insn;
			if (issrc && isneg)
				*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
			cnt = patch - insn_buf;

			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
			if (!new_prog)
				return -ENOMEM;

			delta    += cnt - 1;
			env->prog = prog = new_prog;
			insn      = new_prog->insnsi + i + delta;
			continue;
		}

7629 7630
		if (insn->code != (BPF_JMP | BPF_CALL))
			continue;
7631 7632
		if (insn->src_reg == BPF_PSEUDO_CALL)
			continue;
7633

7634 7635 7636 7637
		if (insn->imm == BPF_FUNC_get_route_realm)
			prog->dst_needed = 1;
		if (insn->imm == BPF_FUNC_get_prandom_u32)
			bpf_user_rnd_init_once();
7638 7639
		if (insn->imm == BPF_FUNC_override_return)
			prog->kprobe_override = 1;
7640
		if (insn->imm == BPF_FUNC_tail_call) {
7641 7642 7643 7644 7645 7646
			/* If we tail call into other programs, we
			 * cannot make any assumptions since they can
			 * be replaced dynamically during runtime in
			 * the program array.
			 */
			prog->cb_access = 1;
7647
			env->prog->aux->stack_depth = MAX_BPF_STACK;
7648
			env->prog->aux->max_pkt_offset = MAX_PACKET_OFF;
7649

7650 7651 7652 7653
			/* mark bpf_tail_call as different opcode to avoid
			 * conditional branch in the interpeter for every normal
			 * call and to prevent accidental JITing by JIT compiler
			 * that doesn't support bpf_tail_call yet
7654
			 */
7655
			insn->imm = 0;
7656
			insn->code = BPF_JMP | BPF_TAIL_CALL;
7657

7658 7659 7660 7661
			aux = &env->insn_aux_data[i + delta];
			if (!bpf_map_ptr_unpriv(aux))
				continue;

7662 7663 7664 7665 7666 7667
			/* instead of changing every JIT dealing with tail_call
			 * emit two extra insns:
			 * if (index >= max_entries) goto out;
			 * index &= array->index_mask;
			 * to avoid out-of-bounds cpu speculation
			 */
7668
			if (bpf_map_ptr_poisoned(aux)) {
7669
				verbose(env, "tail_call abusing map_ptr\n");
7670 7671
				return -EINVAL;
			}
7672 7673

			map_ptr = BPF_MAP_PTR(aux->map_state);
7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684 7685 7686 7687 7688
			insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
						  map_ptr->max_entries, 2);
			insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
						    container_of(map_ptr,
								 struct bpf_array,
								 map)->index_mask);
			insn_buf[2] = *insn;
			cnt = 3;
			new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
			if (!new_prog)
				return -ENOMEM;

			delta    += cnt - 1;
			env->prog = prog = new_prog;
			insn      = new_prog->insnsi + i + delta;
7689 7690
			continue;
		}
7691

7692
		/* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
7693 7694
		 * and other inlining handlers are currently limited to 64 bit
		 * only.
7695
		 */
7696
		if (prog->jit_requested && BITS_PER_LONG == 64 &&
7697 7698
		    (insn->imm == BPF_FUNC_map_lookup_elem ||
		     insn->imm == BPF_FUNC_map_update_elem ||
7699 7700 7701 7702
		     insn->imm == BPF_FUNC_map_delete_elem ||
		     insn->imm == BPF_FUNC_map_push_elem   ||
		     insn->imm == BPF_FUNC_map_pop_elem    ||
		     insn->imm == BPF_FUNC_map_peek_elem)) {
7703 7704 7705 7706 7707
			aux = &env->insn_aux_data[i + delta];
			if (bpf_map_ptr_poisoned(aux))
				goto patch_call_imm;

			map_ptr = BPF_MAP_PTR(aux->map_state);
7708 7709 7710 7711 7712 7713 7714 7715
			ops = map_ptr->ops;
			if (insn->imm == BPF_FUNC_map_lookup_elem &&
			    ops->map_gen_lookup) {
				cnt = ops->map_gen_lookup(map_ptr, insn_buf);
				if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
					verbose(env, "bpf verifier is misconfigured\n");
					return -EINVAL;
				}
7716

7717 7718 7719 7720
				new_prog = bpf_patch_insn_data(env, i + delta,
							       insn_buf, cnt);
				if (!new_prog)
					return -ENOMEM;
7721

7722 7723 7724 7725 7726
				delta    += cnt - 1;
				env->prog = prog = new_prog;
				insn      = new_prog->insnsi + i + delta;
				continue;
			}
7727

7728 7729 7730 7731 7732 7733 7734
			BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
				     (void *(*)(struct bpf_map *map, void *key))NULL));
			BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
				     (int (*)(struct bpf_map *map, void *key))NULL));
			BUILD_BUG_ON(!__same_type(ops->map_update_elem,
				     (int (*)(struct bpf_map *map, void *key, void *value,
					      u64 flags))NULL));
7735 7736 7737 7738 7739 7740 7741 7742
			BUILD_BUG_ON(!__same_type(ops->map_push_elem,
				     (int (*)(struct bpf_map *map, void *value,
					      u64 flags))NULL));
			BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
				     (int (*)(struct bpf_map *map, void *value))NULL));
			BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
				     (int (*)(struct bpf_map *map, void *value))NULL));

7743 7744 7745 7746 7747 7748 7749 7750 7751 7752 7753 7754 7755
			switch (insn->imm) {
			case BPF_FUNC_map_lookup_elem:
				insn->imm = BPF_CAST_CALL(ops->map_lookup_elem) -
					    __bpf_call_base;
				continue;
			case BPF_FUNC_map_update_elem:
				insn->imm = BPF_CAST_CALL(ops->map_update_elem) -
					    __bpf_call_base;
				continue;
			case BPF_FUNC_map_delete_elem:
				insn->imm = BPF_CAST_CALL(ops->map_delete_elem) -
					    __bpf_call_base;
				continue;
7756 7757 7758 7759 7760 7761 7762 7763 7764 7765 7766 7767
			case BPF_FUNC_map_push_elem:
				insn->imm = BPF_CAST_CALL(ops->map_push_elem) -
					    __bpf_call_base;
				continue;
			case BPF_FUNC_map_pop_elem:
				insn->imm = BPF_CAST_CALL(ops->map_pop_elem) -
					    __bpf_call_base;
				continue;
			case BPF_FUNC_map_peek_elem:
				insn->imm = BPF_CAST_CALL(ops->map_peek_elem) -
					    __bpf_call_base;
				continue;
7768
			}
7769

7770
			goto patch_call_imm;
7771 7772 7773
		}

patch_call_imm:
7774
		fn = env->ops->get_func_proto(insn->imm, env->prog);
7775 7776 7777 7778
		/* all functions that have prototype and verifier allowed
		 * programs to call them, must be real in-kernel functions
		 */
		if (!fn->func) {
7779 7780
			verbose(env,
				"kernel subsystem misconfigured func %s#%d\n",
7781 7782
				func_id_name(insn->imm), insn->imm);
			return -EFAULT;
7783
		}
7784
		insn->imm = fn->func - __bpf_call_base;
7785 7786
	}

7787 7788
	return 0;
}
7789

7790
static void free_states(struct bpf_verifier_env *env)
7791
{
7792
	struct bpf_verifier_state_list *sl, *sln;
7793 7794 7795 7796 7797 7798 7799 7800 7801 7802 7803
	int i;

	if (!env->explored_states)
		return;

	for (i = 0; i < env->prog->len; i++) {
		sl = env->explored_states[i];

		if (sl)
			while (sl != STATE_LIST_MARK) {
				sln = sl->next;
7804
				free_verifier_state(&sl->state, false);
7805 7806 7807 7808 7809 7810 7811 7812
				kfree(sl);
				sl = sln;
			}
	}

	kfree(env->explored_states);
}

7813 7814
int bpf_check(struct bpf_prog **prog, union bpf_attr *attr,
	      union bpf_attr __user *uattr)
7815
{
7816
	struct bpf_verifier_env *env;
7817
	struct bpf_verifier_log *log;
7818
	int i, len, ret = -EINVAL;
7819
	bool is_priv;
7820

7821 7822 7823 7824
	/* no program is valid */
	if (ARRAY_SIZE(bpf_verifier_ops) == 0)
		return -EINVAL;

7825
	/* 'struct bpf_verifier_env' can be global, but since it's not small,
7826 7827
	 * allocate/free it every time bpf_check() is called
	 */
7828
	env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
7829 7830
	if (!env)
		return -ENOMEM;
7831
	log = &env->log;
7832

7833
	len = (*prog)->len;
7834
	env->insn_aux_data =
7835
		vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
7836 7837 7838
	ret = -ENOMEM;
	if (!env->insn_aux_data)
		goto err_free_env;
7839 7840
	for (i = 0; i < len; i++)
		env->insn_aux_data[i].orig_idx = i;
7841
	env->prog = *prog;
7842
	env->ops = bpf_verifier_ops[env->prog->type];
7843

7844 7845 7846 7847 7848 7849 7850
	/* grab the mutex to protect few globals used by verifier */
	mutex_lock(&bpf_verifier_lock);

	if (attr->log_level || attr->log_buf || attr->log_size) {
		/* user requested verbose verifier output
		 * and supplied buffer to store the verification trace
		 */
7851 7852 7853
		log->level = attr->log_level;
		log->ubuf = (char __user *) (unsigned long) attr->log_buf;
		log->len_total = attr->log_size;
7854 7855

		ret = -EINVAL;
7856 7857 7858
		/* log attributes have to be sane */
		if (log->len_total < 128 || log->len_total > UINT_MAX >> 8 ||
		    !log->level || !log->ubuf)
7859
			goto err_unlock;
7860
	}
7861 7862 7863

	env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
7864
		env->strict_alignment = true;
7865 7866
	if (attr->prog_flags & BPF_F_ANY_ALIGNMENT)
		env->strict_alignment = false;
7867

7868 7869 7870
	is_priv = capable(CAP_SYS_ADMIN);
	env->allow_ptr_leaks = is_priv;

7871 7872 7873 7874
	ret = replace_map_fd_with_map_ptr(env);
	if (ret < 0)
		goto skip_full_check;

7875
	if (bpf_prog_is_dev_bound(env->prog->aux)) {
7876
		ret = bpf_prog_offload_verifier_prep(env->prog);
7877
		if (ret)
7878
			goto skip_full_check;
7879 7880
	}

7881
	env->explored_states = kcalloc(env->prog->len,
7882
				       sizeof(struct bpf_verifier_state_list *),
7883 7884 7885 7886 7887
				       GFP_USER);
	ret = -ENOMEM;
	if (!env->explored_states)
		goto skip_full_check;

7888
	ret = check_subprogs(env);
7889 7890 7891
	if (ret < 0)
		goto skip_full_check;

7892
	ret = check_btf_info(env, attr, uattr);
7893 7894 7895
	if (ret < 0)
		goto skip_full_check;

7896 7897 7898 7899
	ret = check_cfg(env);
	if (ret < 0)
		goto skip_full_check;

7900
	ret = do_check(env);
7901 7902 7903 7904
	if (env->cur_state) {
		free_verifier_state(env->cur_state, true);
		env->cur_state = NULL;
	}
7905

7906 7907 7908
	if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux))
		ret = bpf_prog_offload_finalize(env);

7909
skip_full_check:
7910
	while (!pop_stack(env, NULL, NULL));
7911
	free_states(env);
7912

7913
	if (ret == 0)
7914
		ret = check_max_stack_depth(env);
7915

7916
	/* instruction rewrites happen after this point */
7917 7918 7919
	if (is_priv) {
		if (ret == 0)
			opt_hard_wire_dead_code_branches(env);
7920 7921
		if (ret == 0)
			ret = opt_remove_dead_code(env);
7922 7923
		if (ret == 0)
			ret = opt_remove_nops(env);
7924 7925 7926
	} else {
		if (ret == 0)
			sanitize_dead_code(env);
7927 7928
	}

7929 7930 7931 7932
	if (ret == 0)
		/* program is valid, convert *(u32*)(ctx + off) accesses */
		ret = convert_ctx_accesses(env);

7933
	if (ret == 0)
7934
		ret = fixup_bpf_calls(env);
7935

7936 7937 7938
	if (ret == 0)
		ret = fixup_call_args(env);

7939
	if (log->level && bpf_verifier_log_full(log))
7940
		ret = -ENOSPC;
7941
	if (log->level && !log->ubuf) {
7942
		ret = -EFAULT;
7943
		goto err_release_maps;
7944 7945
	}

7946 7947
	if (ret == 0 && env->used_map_cnt) {
		/* if program passed verifier, update used_maps in bpf_prog_info */
7948 7949 7950
		env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt,
							  sizeof(env->used_maps[0]),
							  GFP_KERNEL);
7951

7952
		if (!env->prog->aux->used_maps) {
7953
			ret = -ENOMEM;
7954
			goto err_release_maps;
7955 7956
		}

7957
		memcpy(env->prog->aux->used_maps, env->used_maps,
7958
		       sizeof(env->used_maps[0]) * env->used_map_cnt);
7959
		env->prog->aux->used_map_cnt = env->used_map_cnt;
7960 7961 7962 7963 7964 7965

		/* program is valid. Convert pseudo bpf_ld_imm64 into generic
		 * bpf_ld_imm64 instructions
		 */
		convert_pseudo_ld_imm64(env);
	}
7966

7967 7968 7969
	if (ret == 0)
		adjust_btf_func(env);

7970
err_release_maps:
7971
	if (!env->prog->aux->used_maps)
7972
		/* if we didn't copy map pointers into bpf_prog_info, release
7973
		 * them now. Otherwise free_used_maps() will release them.
7974 7975
		 */
		release_maps(env);
7976
	*prog = env->prog;
7977
err_unlock:
7978
	mutex_unlock(&bpf_verifier_lock);
7979 7980 7981
	vfree(env->insn_aux_data);
err_free_env:
	kfree(env);
7982 7983
	return ret;
}