ccp-dev.c 14.4 KB
Newer Older
1 2 3
/*
 * AMD Cryptographic Coprocessor (CCP) driver
 *
4
 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 6
 *
 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7
 * Author: Gary R Hook <gary.hook@amd.com>
8 9 10 11 12 13 14 15 16 17 18
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/interrupt.h>
#include <linux/spinlock.h>
19
#include <linux/spinlock_types.h>
20
#include <linux/types.h>
21 22 23 24
#include <linux/mutex.h>
#include <linux/delay.h>
#include <linux/hw_random.h>
#include <linux/cpu.h>
25
#ifdef CONFIG_X86
26
#include <asm/cpu_device_id.h>
27
#endif
28 29 30 31
#include <linux/ccp.h>

#include "ccp-dev.h"

32 33 34 35 36
struct ccp_tasklet_data {
	struct completion completion;
	struct ccp_cmd *cmd;
};

37
/* Human-readable error strings */
38
#define CCP_MAX_ERROR_CODE	64
39
static char *ccp_error_codes[] = {
40
	"",
41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
	"ILLEGAL_ENGINE",
	"ILLEGAL_KEY_ID",
	"ILLEGAL_FUNCTION_TYPE",
	"ILLEGAL_FUNCTION_MODE",
	"ILLEGAL_FUNCTION_ENCRYPT",
	"ILLEGAL_FUNCTION_SIZE",
	"Zlib_MISSING_INIT_EOM",
	"ILLEGAL_FUNCTION_RSVD",
	"ILLEGAL_BUFFER_LENGTH",
	"VLSB_FAULT",
	"ILLEGAL_MEM_ADDR",
	"ILLEGAL_MEM_SEL",
	"ILLEGAL_CONTEXT_ID",
	"ILLEGAL_KEY_ADDR",
	"0xF Reserved",
	"Zlib_ILLEGAL_MULTI_QUEUE",
	"Zlib_ILLEGAL_JOBID_CHANGE",
	"CMD_TIMEOUT",
	"IDMA0_AXI_SLVERR",
	"IDMA0_AXI_DECERR",
	"0x15 Reserved",
	"IDMA1_AXI_SLAVE_FAULT",
	"IDMA1_AIXI_DECERR",
	"0x18 Reserved",
	"ZLIBVHB_AXI_SLVERR",
	"ZLIBVHB_AXI_DECERR",
	"0x1B Reserved",
	"ZLIB_UNEXPECTED_EOM",
	"ZLIB_EXTRA_DATA",
	"ZLIB_BTYPE",
	"ZLIB_UNDEFINED_SYMBOL",
	"ZLIB_UNDEFINED_DISTANCE_S",
	"ZLIB_CODE_LENGTH_SYMBOL",
	"ZLIB _VHB_ILLEGAL_FETCH",
	"ZLIB_UNCOMPRESSED_LEN",
	"ZLIB_LIMIT_REACHED",
	"ZLIB_CHECKSUM_MISMATCH0",
	"ODMA0_AXI_SLVERR",
	"ODMA0_AXI_DECERR",
	"0x28 Reserved",
	"ODMA1_AXI_SLVERR",
	"ODMA1_AXI_DECERR",
83 84
};

85
void ccp_log_error(struct ccp_device *d, unsigned int e)
86
{
87 88 89 90 91 92 93
	if (WARN_ON(e >= CCP_MAX_ERROR_CODE))
		return;

	if (e < ARRAY_SIZE(ccp_error_codes))
		dev_err(d->dev, "CCP error %d: %s\n", e, ccp_error_codes[e]);
	else
		dev_err(d->dev, "CCP error %d: Unknown Error\n", e);
94 95
}

96 97 98 99 100 101 102 103 104 105 106 107 108 109
/* List of CCPs, CCP count, read-write access lock, and access functions
 *
 * Lock structure: get ccp_unit_lock for reading whenever we need to
 * examine the CCP list. While holding it for reading we can acquire
 * the RR lock to update the round-robin next-CCP pointer. The unit lock
 * must be acquired before the RR lock.
 *
 * If the unit-lock is acquired for writing, we have total control over
 * the list, so there's no value in getting the RR lock.
 */
static DEFINE_RWLOCK(ccp_unit_lock);
static LIST_HEAD(ccp_units);

/* Round-robin counter */
110
static DEFINE_SPINLOCK(ccp_rr_lock);
111 112
static struct ccp_device *ccp_rr;

113 114 115 116 117
/**
 * ccp_add_device - add a CCP device to the list
 *
 * @ccp: ccp_device struct pointer
 *
118 119
 * Put this CCP on the unit list, which makes it available
 * for use.
120 121
 *
 * Returns zero if a CCP device is present, -ENODEV otherwise.
122
 */
123
void ccp_add_device(struct ccp_device *ccp)
124
{
125 126 127 128 129 130 131 132 133 134
	unsigned long flags;

	write_lock_irqsave(&ccp_unit_lock, flags);
	list_add_tail(&ccp->entry, &ccp_units);
	if (!ccp_rr)
		/* We already have the list lock (we're first) so this
		 * pointer can't change on us. Set its initial value.
		 */
		ccp_rr = ccp;
	write_unlock_irqrestore(&ccp_unit_lock, flags);
135 136
}

137 138 139 140 141 142
/**
 * ccp_del_device - remove a CCP device from the list
 *
 * @ccp: ccp_device struct pointer
 *
 * Remove this unit from the list of devices. If the next device
143 144 145
 * up for use is this one, adjust the pointer. If this is the last
 * device, NULL the pointer.
 */
146
void ccp_del_device(struct ccp_device *ccp)
147
{
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167
	unsigned long flags;

	write_lock_irqsave(&ccp_unit_lock, flags);
	if (ccp_rr == ccp) {
		/* ccp_unit_lock is read/write; any read access
		 * will be suspended while we make changes to the
		 * list and RR pointer.
		 */
		if (list_is_last(&ccp_rr->entry, &ccp_units))
			ccp_rr = list_first_entry(&ccp_units, struct ccp_device,
						  entry);
		else
			ccp_rr = list_next_entry(ccp_rr, entry);
	}
	list_del(&ccp->entry);
	if (list_empty(&ccp_units))
		ccp_rr = NULL;
	write_unlock_irqrestore(&ccp_unit_lock, flags);
}

168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190


int ccp_register_rng(struct ccp_device *ccp)
{
	int ret = 0;

	dev_dbg(ccp->dev, "Registering RNG...\n");
	/* Register an RNG */
	ccp->hwrng.name = ccp->rngname;
	ccp->hwrng.read = ccp_trng_read;
	ret = hwrng_register(&ccp->hwrng);
	if (ret)
		dev_err(ccp->dev, "error registering hwrng (%d)\n", ret);

	return ret;
}

void ccp_unregister_rng(struct ccp_device *ccp)
{
	if (ccp->hwrng.name)
		hwrng_unregister(&ccp->hwrng);
}

191 192 193 194 195 196 197 198 199 200
static struct ccp_device *ccp_get_device(void)
{
	unsigned long flags;
	struct ccp_device *dp = NULL;

	/* We round-robin through the unit list.
	 * The (ccp_rr) pointer refers to the next unit to use.
	 */
	read_lock_irqsave(&ccp_unit_lock, flags);
	if (!list_empty(&ccp_units)) {
201
		spin_lock(&ccp_rr_lock);
202 203 204 205 206 207
		dp = ccp_rr;
		if (list_is_last(&ccp_rr->entry, &ccp_units))
			ccp_rr = list_first_entry(&ccp_units, struct ccp_device,
						  entry);
		else
			ccp_rr = list_next_entry(ccp_rr, entry);
208
		spin_unlock(&ccp_rr_lock);
209 210 211 212
	}
	read_unlock_irqrestore(&ccp_unit_lock, flags);

	return dp;
213 214
}

215 216 217 218 219 220 221
/**
 * ccp_present - check if a CCP device is present
 *
 * Returns zero if a CCP device is present, -ENODEV otherwise.
 */
int ccp_present(void)
{
222 223
	unsigned long flags;
	int ret;
224

225 226 227 228 229
	read_lock_irqsave(&ccp_unit_lock, flags);
	ret = list_empty(&ccp_units);
	read_unlock_irqrestore(&ccp_unit_lock, flags);

	return ret ? -ENODEV : 0;
230 231 232
}
EXPORT_SYMBOL_GPL(ccp_present);

233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
/**
 * ccp_version - get the version of the CCP device
 *
 * Returns the version from the first unit on the list;
 * otherwise a zero if no CCP device is present
 */
unsigned int ccp_version(void)
{
	struct ccp_device *dp;
	unsigned long flags;
	int ret = 0;

	read_lock_irqsave(&ccp_unit_lock, flags);
	if (!list_empty(&ccp_units)) {
		dp = list_first_entry(&ccp_units, struct ccp_device, entry);
		ret = dp->vdata->version;
	}
	read_unlock_irqrestore(&ccp_unit_lock, flags);

	return ret;
}
EXPORT_SYMBOL_GPL(ccp_version);

256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
/**
 * ccp_enqueue_cmd - queue an operation for processing by the CCP
 *
 * @cmd: ccp_cmd struct to be processed
 *
 * Queue a cmd to be processed by the CCP. If queueing the cmd
 * would exceed the defined length of the cmd queue the cmd will
 * only be queued if the CCP_CMD_MAY_BACKLOG flag is set and will
 * result in a return code of -EBUSY.
 *
 * The callback routine specified in the ccp_cmd struct will be
 * called to notify the caller of completion (if the cmd was not
 * backlogged) or advancement out of the backlog. If the cmd has
 * advanced out of the backlog the "err" value of the callback
 * will be -EINPROGRESS. Any other "err" value during callback is
 * the result of the operation.
 *
 * The cmd has been successfully queued if:
 *   the return code is -EINPROGRESS or
 *   the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set
 */
int ccp_enqueue_cmd(struct ccp_cmd *cmd)
{
279
	struct ccp_device *ccp;
280 281 282 283
	unsigned long flags;
	unsigned int i;
	int ret;

284 285 286
	/* Some commands might need to be sent to a specific device */
	ccp = cmd->ccp ? cmd->ccp : ccp_get_device();

287 288 289 290 291 292 293 294 295 296 297 298 299 300
	if (!ccp)
		return -ENODEV;

	/* Caller must supply a callback routine */
	if (!cmd->callback)
		return -EINVAL;

	cmd->ccp = ccp;

	spin_lock_irqsave(&ccp->cmd_lock, flags);

	i = ccp->cmd_q_count;

	if (ccp->cmd_count >= MAX_CMD_QLEN) {
301 302
		if (cmd->flags & CCP_CMD_MAY_BACKLOG) {
			ret = -EBUSY;
303
			list_add_tail(&cmd->entry, &ccp->backlog);
304 305 306
		} else {
			ret = -ENOSPC;
		}
307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
	} else {
		ret = -EINPROGRESS;
		ccp->cmd_count++;
		list_add_tail(&cmd->entry, &ccp->cmd);

		/* Find an idle queue */
		if (!ccp->suspending) {
			for (i = 0; i < ccp->cmd_q_count; i++) {
				if (ccp->cmd_q[i].active)
					continue;

				break;
			}
		}
	}

	spin_unlock_irqrestore(&ccp->cmd_lock, flags);

	/* If we found an idle queue, wake it up */
	if (i < ccp->cmd_q_count)
		wake_up_process(ccp->cmd_q[i].kthread);

	return ret;
}
EXPORT_SYMBOL_GPL(ccp_enqueue_cmd);

static void ccp_do_cmd_backlog(struct work_struct *work)
{
	struct ccp_cmd *cmd = container_of(work, struct ccp_cmd, work);
	struct ccp_device *ccp = cmd->ccp;
	unsigned long flags;
	unsigned int i;

	cmd->callback(cmd->data, -EINPROGRESS);

	spin_lock_irqsave(&ccp->cmd_lock, flags);

	ccp->cmd_count++;
	list_add_tail(&cmd->entry, &ccp->cmd);

	/* Find an idle queue */
	for (i = 0; i < ccp->cmd_q_count; i++) {
		if (ccp->cmd_q[i].active)
			continue;

		break;
	}

	spin_unlock_irqrestore(&ccp->cmd_lock, flags);

	/* If we found an idle queue, wake it up */
	if (i < ccp->cmd_q_count)
		wake_up_process(ccp->cmd_q[i].kthread);
}

static struct ccp_cmd *ccp_dequeue_cmd(struct ccp_cmd_queue *cmd_q)
{
	struct ccp_device *ccp = cmd_q->ccp;
	struct ccp_cmd *cmd = NULL;
	struct ccp_cmd *backlog = NULL;
	unsigned long flags;

	spin_lock_irqsave(&ccp->cmd_lock, flags);

	cmd_q->active = 0;

	if (ccp->suspending) {
		cmd_q->suspended = 1;

		spin_unlock_irqrestore(&ccp->cmd_lock, flags);
		wake_up_interruptible(&ccp->suspend_queue);

		return NULL;
	}

	if (ccp->cmd_count) {
		cmd_q->active = 1;

		cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
		list_del(&cmd->entry);

		ccp->cmd_count--;
	}

	if (!list_empty(&ccp->backlog)) {
		backlog = list_first_entry(&ccp->backlog, struct ccp_cmd,
					   entry);
		list_del(&backlog->entry);
	}

	spin_unlock_irqrestore(&ccp->cmd_lock, flags);

	if (backlog) {
		INIT_WORK(&backlog->work, ccp_do_cmd_backlog);
		schedule_work(&backlog->work);
	}

	return cmd;
}

407
static void ccp_do_cmd_complete(unsigned long data)
408
{
409 410
	struct ccp_tasklet_data *tdata = (struct ccp_tasklet_data *)data;
	struct ccp_cmd *cmd = tdata->cmd;
411 412

	cmd->callback(cmd->data, cmd->ret);
413

414
	complete(&tdata->completion);
415 416
}

417 418 419 420 421 422
/**
 * ccp_cmd_queue_thread - create a kernel thread to manage a CCP queue
 *
 * @data: thread-specific data
 */
int ccp_cmd_queue_thread(void *data)
423 424 425
{
	struct ccp_cmd_queue *cmd_q = (struct ccp_cmd_queue *)data;
	struct ccp_cmd *cmd;
426 427 428 429
	struct ccp_tasklet_data tdata;
	struct tasklet_struct tasklet;

	tasklet_init(&tasklet, ccp_do_cmd_complete, (unsigned long)&tdata);
430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446

	set_current_state(TASK_INTERRUPTIBLE);
	while (!kthread_should_stop()) {
		schedule();

		set_current_state(TASK_INTERRUPTIBLE);

		cmd = ccp_dequeue_cmd(cmd_q);
		if (!cmd)
			continue;

		__set_current_state(TASK_RUNNING);

		/* Execute the command */
		cmd->ret = ccp_run_cmd(cmd_q, cmd);

		/* Schedule the completion callback */
447 448 449 450
		tdata.cmd = cmd;
		init_completion(&tdata.completion);
		tasklet_schedule(&tasklet);
		wait_for_completion(&tdata.completion);
451 452 453 454 455 456 457 458 459 460 461 462
	}

	__set_current_state(TASK_RUNNING);

	return 0;
}

/**
 * ccp_alloc_struct - allocate and initialize the ccp_device struct
 *
 * @dev: device struct of the CCP
 */
463
struct ccp_device *ccp_alloc_struct(struct sp_device *sp)
464
{
465
	struct device *dev = sp->dev;
466 467
	struct ccp_device *ccp;

468
	ccp = devm_kzalloc(dev, sizeof(*ccp), GFP_KERNEL);
469
	if (!ccp)
470 471
		return NULL;
	ccp->dev = dev;
472 473
	ccp->sp = sp;
	ccp->axcache = sp->axcache;
474 475 476 477 478 479

	INIT_LIST_HEAD(&ccp->cmd);
	INIT_LIST_HEAD(&ccp->backlog);

	spin_lock_init(&ccp->cmd_lock);
	mutex_init(&ccp->req_mutex);
480 481 482
	mutex_init(&ccp->sb_mutex);
	ccp->sb_count = KSB_COUNT;
	ccp->sb_start = 0;
483

484 485 486 487
	/* Initialize the wait queues */
	init_waitqueue_head(&ccp->sb_queue);
	init_waitqueue_head(&ccp->suspend_queue);

488 489
	snprintf(ccp->name, MAX_CCP_NAME_LEN, "ccp-%u", sp->ord);
	snprintf(ccp->rngname, MAX_CCP_NAME_LEN, "ccp-%u-rng", sp->ord);
490

491 492 493
	return ccp;
}

494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait)
{
	struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng);
	u32 trng_value;
	int len = min_t(int, sizeof(trng_value), max);

	/* Locking is provided by the caller so we can update device
	 * hwrng-related fields safely
	 */
	trng_value = ioread32(ccp->io_regs + TRNG_OUT_REG);
	if (!trng_value) {
		/* Zero is returned if not data is available or if a
		 * bad-entropy error is present. Assume an error if
		 * we exceed TRNG_RETRIES reads of zero.
		 */
		if (ccp->hwrng_retries++ > TRNG_RETRIES)
			return -EIO;

		return 0;
	}

	/* Reset the counter and save the rng value */
	ccp->hwrng_retries = 0;
	memcpy(data, &trng_value, len);

	return len;
}

522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
#ifdef CONFIG_PM
bool ccp_queues_suspended(struct ccp_device *ccp)
{
	unsigned int suspended = 0;
	unsigned long flags;
	unsigned int i;

	spin_lock_irqsave(&ccp->cmd_lock, flags);

	for (i = 0; i < ccp->cmd_q_count; i++)
		if (ccp->cmd_q[i].suspended)
			suspended++;

	spin_unlock_irqrestore(&ccp->cmd_lock, flags);

	return ccp->cmd_q_count == suspended;
}
539

540
int ccp_dev_suspend(struct sp_device *sp, pm_message_t state)
541
{
542
	struct ccp_device *ccp = sp->ccp_data;
543 544 545
	unsigned long flags;
	unsigned int i;

546 547 548 549
	/* If there's no device there's nothing to do */
	if (!ccp)
		return 0;

550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567
	spin_lock_irqsave(&ccp->cmd_lock, flags);

	ccp->suspending = 1;

	/* Wake all the queue kthreads to prepare for suspend */
	for (i = 0; i < ccp->cmd_q_count; i++)
		wake_up_process(ccp->cmd_q[i].kthread);

	spin_unlock_irqrestore(&ccp->cmd_lock, flags);

	/* Wait for all queue kthreads to say they're done */
	while (!ccp_queues_suspended(ccp))
		wait_event_interruptible(ccp->suspend_queue,
					 ccp_queues_suspended(ccp));

	return 0;
}

568
int ccp_dev_resume(struct sp_device *sp)
569
{
570
	struct ccp_device *ccp = sp->ccp_data;
571 572 573
	unsigned long flags;
	unsigned int i;

574 575 576 577
	/* If there's no device there's nothing to do */
	if (!ccp)
		return 0;

578 579 580 581 582 583 584 585 586 587 588 589 590 591
	spin_lock_irqsave(&ccp->cmd_lock, flags);

	ccp->suspending = 0;

	/* Wake up all the kthreads */
	for (i = 0; i < ccp->cmd_q_count; i++) {
		ccp->cmd_q[i].suspended = 0;
		wake_up_process(ccp->cmd_q[i].kthread);
	}

	spin_unlock_irqrestore(&ccp->cmd_lock, flags);

	return 0;
}
592 593
#endif

594
int ccp_dev_init(struct sp_device *sp)
595
{
596 597 598
	struct device *dev = sp->dev;
	struct ccp_device *ccp;
	int ret;
599

600 601
	ret = -ENOMEM;
	ccp = ccp_alloc_struct(sp);
602
	if (!ccp)
603 604 605 606 607 608 609 610 611
		goto e_err;
	sp->ccp_data = ccp;

	ccp->vdata = (struct ccp_vdata *)sp->dev_vdata->ccp_vdata;
	if (!ccp->vdata || !ccp->vdata->version) {
		ret = -ENODEV;
		dev_err(dev, "missing driver data\n");
		goto e_err;
	}
612

613
	ccp->use_tasklet = sp->use_tasklet;
614

615 616 617
	ccp->io_regs = sp->io_map + ccp->vdata->offset;
	if (ccp->vdata->setup)
		ccp->vdata->setup(ccp);
618

619
	ret = ccp->vdata->perform->init(ccp);
620
	if (ret)
621
		goto e_err;
622

623
	dev_notice(dev, "ccp enabled\n");
624 625

	return 0;
626

627 628
e_err:
	sp->ccp_data = NULL;
629

630
	dev_notice(dev, "ccp initialization failed\n");
631

632
	return ret;
633 634
}

635
void ccp_dev_destroy(struct sp_device *sp)
636
{
637
	struct ccp_device *ccp = sp->ccp_data;
638

639 640
	if (!ccp)
		return;
641

642 643
	ccp->vdata->perform->destroy(ccp);
}