gcm.c 32.9 KB
Newer Older
Mikko Herranen's avatar
Mikko Herranen committed
1 2 3 4 5 6 7 8 9 10 11
/*
 * GCM: Galois/Counter Mode.
 *
 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 as published
 * by the Free Software Foundation.
 */

#include <crypto/gf128mul.h>
Herbert Xu's avatar
Herbert Xu committed
12
#include <crypto/internal/aead.h>
13
#include <crypto/internal/skcipher.h>
14
#include <crypto/internal/hash.h>
15
#include <crypto/null.h>
16
#include <crypto/scatterwalk.h>
17
#include <crypto/gcm.h>
18 19
#include <crypto/hash.h>
#include "internal.h"
Mikko Herranen's avatar
Mikko Herranen committed
20 21 22 23 24 25 26
#include <linux/err.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>

struct gcm_instance_ctx {
27
	struct crypto_skcipher_spawn ctr;
28
	struct crypto_ahash_spawn ghash;
Mikko Herranen's avatar
Mikko Herranen committed
29 30 31
};

struct crypto_gcm_ctx {
Herbert Xu's avatar
Herbert Xu committed
32
	struct crypto_skcipher *ctr;
33
	struct crypto_ahash *ghash;
Mikko Herranen's avatar
Mikko Herranen committed
34 35
};

Herbert Xu's avatar
Herbert Xu committed
36 37 38 39 40
struct crypto_rfc4106_ctx {
	struct crypto_aead *child;
	u8 nonce[4];
};

41 42 43 44 45 46
struct crypto_rfc4106_req_ctx {
	struct scatterlist src[3];
	struct scatterlist dst[3];
	struct aead_request subreq;
};

47 48 49 50
struct crypto_rfc4543_instance_ctx {
	struct crypto_aead_spawn aead;
};

51 52
struct crypto_rfc4543_ctx {
	struct crypto_aead *child;
53
	struct crypto_sync_skcipher *null;
54 55 56 57 58 59 60
	u8 nonce[4];
};

struct crypto_rfc4543_req_ctx {
	struct aead_request subreq;
};

Mikko Herranen's avatar
Mikko Herranen committed
61
struct crypto_gcm_ghash_ctx {
62 63
	unsigned int cryptlen;
	struct scatterlist *src;
64
	int (*complete)(struct aead_request *req, u32 flags);
Mikko Herranen's avatar
Mikko Herranen committed
65 66 67
};

struct crypto_gcm_req_priv_ctx {
68
	u8 iv[16];
Mikko Herranen's avatar
Mikko Herranen committed
69
	u8 auth_tag[16];
Herbert Xu's avatar
Herbert Xu committed
70
	u8 iauth_tag[16];
71 72 73
	struct scatterlist src[3];
	struct scatterlist dst[3];
	struct scatterlist sg;
74 75 76
	struct crypto_gcm_ghash_ctx ghash_ctx;
	union {
		struct ahash_request ahreq;
Herbert Xu's avatar
Herbert Xu committed
77
		struct skcipher_request skreq;
78
	} u;
Mikko Herranen's avatar
Mikko Herranen committed
79 80
};

81 82 83 84 85 86
static struct {
	u8 buf[16];
	struct scatterlist sg;
} *gcm_zeroes;

static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);
87

88 89 90 91 92 93 94 95
static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
	struct aead_request *req)
{
	unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));

	return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
}

Mikko Herranen's avatar
Mikko Herranen committed
96 97 98 99
static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
			     unsigned int keylen)
{
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
100
	struct crypto_ahash *ghash = ctx->ghash;
Herbert Xu's avatar
Herbert Xu committed
101
	struct crypto_skcipher *ctr = ctx->ctr;
102 103
	struct {
		be128 hash;
104
		u8 iv[16];
105

106
		struct crypto_wait wait;
107 108

		struct scatterlist sg[1];
Herbert Xu's avatar
Herbert Xu committed
109
		struct skcipher_request req;
110 111
	} *data;
	int err;
Mikko Herranen's avatar
Mikko Herranen committed
112

Herbert Xu's avatar
Herbert Xu committed
113 114 115 116 117
	crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
	crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
				       CRYPTO_TFM_REQ_MASK);
	err = crypto_skcipher_setkey(ctr, key, keylen);
	crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
118
				    CRYPTO_TFM_RES_MASK);
Mikko Herranen's avatar
Mikko Herranen committed
119
	if (err)
120
		return err;
Mikko Herranen's avatar
Mikko Herranen committed
121

Herbert Xu's avatar
Herbert Xu committed
122
	data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
123 124 125 126
		       GFP_KERNEL);
	if (!data)
		return -ENOMEM;

127
	crypto_init_wait(&data->wait);
128
	sg_init_one(data->sg, &data->hash, sizeof(data->hash));
Herbert Xu's avatar
Herbert Xu committed
129 130 131
	skcipher_request_set_tfm(&data->req, ctr);
	skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
						  CRYPTO_TFM_REQ_MAY_BACKLOG,
132 133
				      crypto_req_done,
				      &data->wait);
Herbert Xu's avatar
Herbert Xu committed
134 135 136
	skcipher_request_set_crypt(&data->req, data->sg, data->sg,
				   sizeof(data->hash), data->iv);

137 138
	err = crypto_wait_req(crypto_skcipher_encrypt(&data->req),
							&data->wait);
139

Mikko Herranen's avatar
Mikko Herranen committed
140 141 142
	if (err)
		goto out;

143 144 145 146 147 148
	crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK);
	crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) &
			       CRYPTO_TFM_REQ_MASK);
	err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128));
	crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) &
			      CRYPTO_TFM_RES_MASK);
Mikko Herranen's avatar
Mikko Herranen committed
149

150
out:
151
	kzfree(data);
Mikko Herranen's avatar
Mikko Herranen committed
152 153 154
	return err;
}

Herbert Xu's avatar
Herbert Xu committed
155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
static int crypto_gcm_setauthsize(struct crypto_aead *tfm,
				  unsigned int authsize)
{
	switch (authsize) {
	case 4:
	case 8:
	case 12:
	case 13:
	case 14:
	case 15:
	case 16:
		break;
	default:
		return -EINVAL;
	}

	return 0;
}

174
static void crypto_gcm_init_common(struct aead_request *req)
Mikko Herranen's avatar
Mikko Herranen committed
175
{
176
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
177
	__be32 counter = cpu_to_be32(1);
178
	struct scatterlist *sg;
179 180

	memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
181 182
	memcpy(pctx->iv, req->iv, GCM_AES_IV_SIZE);
	memcpy(pctx->iv + GCM_AES_IV_SIZE, &counter, 4);
183

184
	sg_init_table(pctx->src, 3);
185
	sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
186 187
	sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen);
	if (sg != pctx->src + 1)
188
		sg_chain(pctx->src, 2, sg);
189 190

	if (req->src != req->dst) {
191
		sg_init_table(pctx->dst, 3);
192
		sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
193 194
		sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen);
		if (sg != pctx->dst + 1)
195
			sg_chain(pctx->dst, 2, sg);
196
	}
197 198 199 200 201 202 203 204
}

static void crypto_gcm_init_crypt(struct aead_request *req,
				  unsigned int cryptlen)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
Herbert Xu's avatar
Herbert Xu committed
205
	struct skcipher_request *skreq = &pctx->u.skreq;
206 207 208
	struct scatterlist *dst;

	dst = req->src == req->dst ? pctx->src : pctx->dst;
Mikko Herranen's avatar
Mikko Herranen committed
209

Herbert Xu's avatar
Herbert Xu committed
210 211
	skcipher_request_set_tfm(skreq, ctx->ctr);
	skcipher_request_set_crypt(skreq, pctx->src, dst,
212
				     cryptlen + sizeof(pctx->auth_tag),
213
				     pctx->iv);
214 215 216 217 218 219 220 221 222
}

static inline unsigned int gcm_remain(unsigned int len)
{
	len &= 0xfU;
	return len ? 16 - len : 0;
}

static void gcm_hash_len_done(struct crypto_async_request *areq, int err);
Mikko Herranen's avatar
Mikko Herranen committed
223

224
static int gcm_hash_update(struct aead_request *req,
225
			   crypto_completion_t compl,
226
			   struct scatterlist *src,
227
			   unsigned int len, u32 flags)
228
{
229
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
230
	struct ahash_request *ahreq = &pctx->u.ahreq;
Mikko Herranen's avatar
Mikko Herranen committed
231

232
	ahash_request_set_callback(ahreq, flags, compl, req);
233 234 235
	ahash_request_set_crypt(ahreq, src, NULL, len);

	return crypto_ahash_update(ahreq);
Mikko Herranen's avatar
Mikko Herranen committed
236 237
}

238 239
static int gcm_hash_remain(struct aead_request *req,
			   unsigned int remain,
240
			   crypto_completion_t compl, u32 flags)
Mikko Herranen's avatar
Mikko Herranen committed
241
{
242
	return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags);
243 244
}

245
static int gcm_hash_len(struct aead_request *req, u32 flags)
246
{
247
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
248 249 250 251 252 253 254
	struct ahash_request *ahreq = &pctx->u.ahreq;
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
	u128 lengths;

	lengths.a = cpu_to_be64(req->assoclen * 8);
	lengths.b = cpu_to_be64(gctx->cryptlen * 8);
	memcpy(pctx->iauth_tag, &lengths, 16);
255 256 257 258
	sg_init_one(&pctx->sg, pctx->iauth_tag, 16);
	ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req);
	ahash_request_set_crypt(ahreq, &pctx->sg,
				pctx->iauth_tag, sizeof(lengths));
259

260
	return crypto_ahash_finup(ahreq);
261 262
}

263
static int gcm_hash_len_continue(struct aead_request *req, u32 flags)
264
{
265
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
266 267
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;

268
	return gctx->complete(req, flags);
269 270
}

271
static void gcm_hash_len_done(struct crypto_async_request *areq, int err)
272 273
{
	struct aead_request *req = areq->data;
274

275 276
	if (err)
		goto out;
277

278 279 280
	err = gcm_hash_len_continue(req, 0);
	if (err == -EINPROGRESS)
		return;
281

282 283
out:
	aead_request_complete(req, err);
284 285
}

286
static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)
287
{
288 289
	return gcm_hash_len(req, flags) ?:
	       gcm_hash_len_continue(req, flags);
290 291
}

292 293
static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq,
				       int err)
294 295
{
	struct aead_request *req = areq->data;
296

297 298 299 300 301 302 303 304 305
	if (err)
		goto out;

	err = gcm_hash_crypt_remain_continue(req, 0);
	if (err == -EINPROGRESS)
		return;

out:
	aead_request_complete(req, err);
306 307
}

308
static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)
309
{
310 311 312 313
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
	unsigned int remain;

314 315 316 317 318
	remain = gcm_remain(gctx->cryptlen);
	if (remain)
		return gcm_hash_remain(req, remain,
				       gcm_hash_crypt_remain_done, flags) ?:
		       gcm_hash_crypt_remain_continue(req, flags);
319

320
	return gcm_hash_crypt_remain_continue(req, flags);
321 322
}

323
static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err)
324 325
{
	struct aead_request *req = areq->data;
326

327 328 329 330 331 332 333 334 335
	if (err)
		goto out;

	err = gcm_hash_crypt_continue(req, 0);
	if (err == -EINPROGRESS)
		return;

out:
	aead_request_complete(req, err);
336 337
}

338
static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)
339
{
340 341 342
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;

343 344 345 346 347 348
	if (gctx->cryptlen)
		return gcm_hash_update(req, gcm_hash_crypt_done,
				       gctx->src, gctx->cryptlen, flags) ?:
		       gcm_hash_crypt_continue(req, flags);

	return gcm_hash_crypt_remain_continue(req, flags);
349 350
}

351 352
static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq,
				       int err)
353 354
{
	struct aead_request *req = areq->data;
355

356 357 358 359 360 361 362 363 364
	if (err)
		goto out;

	err = gcm_hash_assoc_remain_continue(req, 0);
	if (err == -EINPROGRESS)
		return;

out:
	aead_request_complete(req, err);
365 366
}

367
static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)
368
{
369 370
	unsigned int remain;

371 372 373 374 375
	remain = gcm_remain(req->assoclen);
	if (remain)
		return gcm_hash_remain(req, remain,
				       gcm_hash_assoc_remain_done, flags) ?:
		       gcm_hash_assoc_remain_continue(req, flags);
376

377
	return gcm_hash_assoc_remain_continue(req, flags);
378 379
}

380
static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err)
381 382
{
	struct aead_request *req = areq->data;
383

384 385 386 387 388 389 390 391 392
	if (err)
		goto out;

	err = gcm_hash_assoc_continue(req, 0);
	if (err == -EINPROGRESS)
		return;

out:
	aead_request_complete(req, err);
393 394
}

395
static int gcm_hash_init_continue(struct aead_request *req, u32 flags)
396
{
397 398 399 400
	if (req->assoclen)
		return gcm_hash_update(req, gcm_hash_assoc_done,
				       req->src, req->assoclen, flags) ?:
		       gcm_hash_assoc_continue(req, flags);
401

402
	return gcm_hash_assoc_remain_continue(req, flags);
403 404 405 406 407 408
}

static void gcm_hash_init_done(struct crypto_async_request *areq, int err)
{
	struct aead_request *req = areq->data;

409 410 411 412 413 414 415 416 417
	if (err)
		goto out;

	err = gcm_hash_init_continue(req, 0);
	if (err == -EINPROGRESS)
		return;

out:
	aead_request_complete(req, err);
418 419
}

420
static int gcm_hash(struct aead_request *req, u32 flags)
421
{
422
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
423
	struct ahash_request *ahreq = &pctx->u.ahreq;
424
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
425 426 427

	ahash_request_set_tfm(ahreq, ctx->ghash);

428 429 430
	ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req);
	return crypto_ahash_init(ahreq) ?:
	       gcm_hash_init_continue(req, flags);
431 432
}

433
static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)
434
{
435
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
436 437
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	u8 *auth_tag = pctx->auth_tag;
Mikko Herranen's avatar
Mikko Herranen committed
438

439 440 441
	crypto_xor(auth_tag, pctx->iauth_tag, 16);
	scatterwalk_map_and_copy(auth_tag, req->dst,
				 req->assoclen + req->cryptlen,
Herbert Xu's avatar
Herbert Xu committed
442
				 crypto_aead_authsize(aead), 1);
443
	return 0;
Herbert Xu's avatar
Herbert Xu committed
444 445
}

446
static int gcm_encrypt_continue(struct aead_request *req, u32 flags)
Herbert Xu's avatar
Herbert Xu committed
447
{
448
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
449
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
Herbert Xu's avatar
Herbert Xu committed
450

451 452 453
	gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst);
	gctx->cryptlen = req->cryptlen;
	gctx->complete = gcm_enc_copy_hash;
Herbert Xu's avatar
Herbert Xu committed
454

455
	return gcm_hash(req, flags);
Mikko Herranen's avatar
Mikko Herranen committed
456 457
}

458
static void gcm_encrypt_done(struct crypto_async_request *areq, int err)
459 460 461
{
	struct aead_request *req = areq->data;

462 463 464 465 466 467
	if (err)
		goto out;

	err = gcm_encrypt_continue(req, 0);
	if (err == -EINPROGRESS)
		return;
468

469
out:
470
	aead_request_complete(req, err);
471 472
}

Mikko Herranen's avatar
Mikko Herranen committed
473 474
static int crypto_gcm_encrypt(struct aead_request *req)
{
475
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
Herbert Xu's avatar
Herbert Xu committed
476
	struct skcipher_request *skreq = &pctx->u.skreq;
477
	u32 flags = aead_request_flags(req);
478

479 480
	crypto_gcm_init_common(req);
	crypto_gcm_init_crypt(req, req->cryptlen);
Herbert Xu's avatar
Herbert Xu committed
481
	skcipher_request_set_callback(skreq, flags, gcm_encrypt_done, req);
482

Herbert Xu's avatar
Herbert Xu committed
483
	return crypto_skcipher_encrypt(skreq) ?:
484
	       gcm_encrypt_continue(req, flags);
Mikko Herranen's avatar
Mikko Herranen committed
485 486
}

487
static int crypto_gcm_verify(struct aead_request *req)
488
{
489
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
490 491 492 493 494 495
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	u8 *auth_tag = pctx->auth_tag;
	u8 *iauth_tag = pctx->iauth_tag;
	unsigned int authsize = crypto_aead_authsize(aead);
	unsigned int cryptlen = req->cryptlen - authsize;

496
	crypto_xor(auth_tag, iauth_tag, 16);
497 498
	scatterwalk_map_and_copy(iauth_tag, req->src,
				 req->assoclen + cryptlen, authsize, 0);
499
	return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
500 501
}

502
static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
Mikko Herranen's avatar
Mikko Herranen committed
503
{
504 505 506
	struct aead_request *req = areq->data;

	if (!err)
507
		err = crypto_gcm_verify(req);
508 509

	aead_request_complete(req, err);
Mikko Herranen's avatar
Mikko Herranen committed
510 511
}

512
static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)
513 514
{
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
Herbert Xu's avatar
Herbert Xu committed
515
	struct skcipher_request *skreq = &pctx->u.skreq;
516 517
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;

518
	crypto_gcm_init_crypt(req, gctx->cryptlen);
Herbert Xu's avatar
Herbert Xu committed
519 520
	skcipher_request_set_callback(skreq, flags, gcm_decrypt_done, req);
	return crypto_skcipher_decrypt(skreq) ?: crypto_gcm_verify(req);
521 522
}

Mikko Herranen's avatar
Mikko Herranen committed
523 524
static int crypto_gcm_decrypt(struct aead_request *req)
{
Herbert Xu's avatar
Herbert Xu committed
525
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
526
	struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
527
	struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
Herbert Xu's avatar
Herbert Xu committed
528
	unsigned int authsize = crypto_aead_authsize(aead);
529
	unsigned int cryptlen = req->cryptlen;
530
	u32 flags = aead_request_flags(req);
Mikko Herranen's avatar
Mikko Herranen committed
531

Herbert Xu's avatar
Herbert Xu committed
532
	cryptlen -= authsize;
Mikko Herranen's avatar
Mikko Herranen committed
533

534
	crypto_gcm_init_common(req);
Mikko Herranen's avatar
Mikko Herranen committed
535

536 537 538
	gctx->src = sg_next(pctx->src);
	gctx->cryptlen = cryptlen;
	gctx->complete = gcm_dec_hash_continue;
Mikko Herranen's avatar
Mikko Herranen committed
539

540
	return gcm_hash(req, flags);
Mikko Herranen's avatar
Mikko Herranen committed
541 542
}

543
static int crypto_gcm_init_tfm(struct crypto_aead *tfm)
Mikko Herranen's avatar
Mikko Herranen committed
544
{
545 546 547
	struct aead_instance *inst = aead_alg_instance(tfm);
	struct gcm_instance_ctx *ictx = aead_instance_ctx(inst);
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
Herbert Xu's avatar
Herbert Xu committed
548
	struct crypto_skcipher *ctr;
549
	struct crypto_ahash *ghash;
Mikko Herranen's avatar
Mikko Herranen committed
550 551 552
	unsigned long align;
	int err;

553 554 555 556
	ghash = crypto_spawn_ahash(&ictx->ghash);
	if (IS_ERR(ghash))
		return PTR_ERR(ghash);

557
	ctr = crypto_spawn_skcipher(&ictx->ctr);
Mikko Herranen's avatar
Mikko Herranen committed
558 559
	err = PTR_ERR(ctr);
	if (IS_ERR(ctr))
560
		goto err_free_hash;
Mikko Herranen's avatar
Mikko Herranen committed
561 562

	ctx->ctr = ctr;
563
	ctx->ghash = ghash;
Mikko Herranen's avatar
Mikko Herranen committed
564

565
	align = crypto_aead_alignmask(tfm);
Mikko Herranen's avatar
Mikko Herranen committed
566
	align &= ~(crypto_tfm_ctx_alignment() - 1);
567
	crypto_aead_set_reqsize(tfm,
568
		align + offsetof(struct crypto_gcm_req_priv_ctx, u) +
Herbert Xu's avatar
Herbert Xu committed
569 570
		max(sizeof(struct skcipher_request) +
		    crypto_skcipher_reqsize(ctr),
571
		    sizeof(struct ahash_request) +
572
		    crypto_ahash_reqsize(ghash)));
Mikko Herranen's avatar
Mikko Herranen committed
573 574

	return 0;
575 576 577 578

err_free_hash:
	crypto_free_ahash(ghash);
	return err;
Mikko Herranen's avatar
Mikko Herranen committed
579 580
}

581
static void crypto_gcm_exit_tfm(struct crypto_aead *tfm)
Mikko Herranen's avatar
Mikko Herranen committed
582
{
583
	struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
Mikko Herranen's avatar
Mikko Herranen committed
584

585
	crypto_free_ahash(ctx->ghash);
Herbert Xu's avatar
Herbert Xu committed
586
	crypto_free_skcipher(ctx->ctr);
Mikko Herranen's avatar
Mikko Herranen committed
587 588
}

589 590 591 592 593 594 595 596 597
static void crypto_gcm_free(struct aead_instance *inst)
{
	struct gcm_instance_ctx *ctx = aead_instance_ctx(inst);

	crypto_drop_skcipher(&ctx->ctr);
	crypto_drop_ahash(&ctx->ghash);
	kfree(inst);
}

598 599 600 601 602
static int crypto_gcm_create_common(struct crypto_template *tmpl,
				    struct rtattr **tb,
				    const char *full_name,
				    const char *ctr_name,
				    const char *ghash_name)
Mikko Herranen's avatar
Mikko Herranen committed
603
{
604
	struct crypto_attr_type *algt;
605
	struct aead_instance *inst;
Herbert Xu's avatar
Herbert Xu committed
606
	struct skcipher_alg *ctr;
607
	struct crypto_alg *ghash_alg;
608
	struct hash_alg_common *ghash;
Mikko Herranen's avatar
Mikko Herranen committed
609 610 611
	struct gcm_instance_ctx *ctx;
	int err;

612 613
	algt = crypto_get_attr_type(tb);
	if (IS_ERR(algt))
614
		return PTR_ERR(algt);
Mikko Herranen's avatar
Mikko Herranen committed
615

616
	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
617
		return -EINVAL;
Mikko Herranen's avatar
Mikko Herranen committed
618

619 620
	ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type,
				    CRYPTO_ALG_TYPE_HASH,
621 622 623
				    CRYPTO_ALG_TYPE_AHASH_MASK |
				    crypto_requires_sync(algt->type,
							 algt->mask));
624
	if (IS_ERR(ghash_alg))
625 626 627
		return PTR_ERR(ghash_alg);

	ghash = __crypto_hash_alg_common(ghash_alg);
628 629

	err = -ENOMEM;
630 631
	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
	if (!inst)
632
		goto out_put_ghash;
Mikko Herranen's avatar
Mikko Herranen committed
633

634 635 636
	ctx = aead_instance_ctx(inst);
	err = crypto_init_ahash_spawn(&ctx->ghash, ghash,
				      aead_crypto_instance(inst));
637 638 639
	if (err)
		goto err_free_inst;

640 641 642 643 644
	err = -EINVAL;
	if (ghash->digestsize != 16)
		goto err_drop_ghash;

	crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst));
645 646 647
	err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
				   crypto_requires_sync(algt->type,
							algt->mask));
648
	if (err)
649
		goto err_drop_ghash;
650

Herbert Xu's avatar
Herbert Xu committed
651
	ctr = crypto_spawn_skcipher_alg(&ctx->ctr);
Mikko Herranen's avatar
Mikko Herranen committed
652

653
	/* We only support 16-byte blocks. */
654
	err = -EINVAL;
Herbert Xu's avatar
Herbert Xu committed
655
	if (crypto_skcipher_alg_ivsize(ctr) != 16)
656 657 658
		goto out_put_ctr;

	/* Not a stream cipher? */
Herbert Xu's avatar
Herbert Xu committed
659
	if (ctr->base.cra_blocksize != 1)
Mikko Herranen's avatar
Mikko Herranen committed
660 661 662
		goto out_put_ctr;

	err = -ENAMETOOLONG;
663
	if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
Herbert Xu's avatar
Herbert Xu committed
664
		     "gcm_base(%s,%s)", ctr->base.cra_driver_name,
665
		     ghash_alg->cra_driver_name) >=
666
	    CRYPTO_MAX_ALG_NAME)
667
		goto out_put_ctr;
Mikko Herranen's avatar
Mikko Herranen committed
668

669 670
	memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);

Herbert Xu's avatar
Herbert Xu committed
671 672
	inst->alg.base.cra_flags = (ghash->base.cra_flags |
				    ctr->base.cra_flags) & CRYPTO_ALG_ASYNC;
673
	inst->alg.base.cra_priority = (ghash->base.cra_priority +
Herbert Xu's avatar
Herbert Xu committed
674
				       ctr->base.cra_priority) / 2;
675 676
	inst->alg.base.cra_blocksize = 1;
	inst->alg.base.cra_alignmask = ghash->base.cra_alignmask |
Herbert Xu's avatar
Herbert Xu committed
677
				       ctr->base.cra_alignmask;
678
	inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
679
	inst->alg.ivsize = GCM_AES_IV_SIZE;
Herbert Xu's avatar
Herbert Xu committed
680
	inst->alg.chunksize = crypto_skcipher_alg_chunksize(ctr);
681 682 683 684 685 686 687 688
	inst->alg.maxauthsize = 16;
	inst->alg.init = crypto_gcm_init_tfm;
	inst->alg.exit = crypto_gcm_exit_tfm;
	inst->alg.setkey = crypto_gcm_setkey;
	inst->alg.setauthsize = crypto_gcm_setauthsize;
	inst->alg.encrypt = crypto_gcm_encrypt;
	inst->alg.decrypt = crypto_gcm_decrypt;

689 690
	inst->free = crypto_gcm_free;

691 692 693
	err = aead_register_instance(tmpl, inst);
	if (err)
		goto out_put_ctr;
Mikko Herranen's avatar
Mikko Herranen committed
694

695
out_put_ghash:
696
	crypto_mod_put(ghash_alg);
697
	return err;
698 699 700

out_put_ctr:
	crypto_drop_skcipher(&ctx->ctr);
701 702
err_drop_ghash:
	crypto_drop_ahash(&ctx->ghash);
Mikko Herranen's avatar
Mikko Herranen committed
703 704
err_free_inst:
	kfree(inst);
705
	goto out_put_ghash;
Mikko Herranen's avatar
Mikko Herranen committed
706 707
}

708
static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb)
709 710 711 712 713 714 715
{
	const char *cipher_name;
	char ctr_name[CRYPTO_MAX_ALG_NAME];
	char full_name[CRYPTO_MAX_ALG_NAME];

	cipher_name = crypto_attr_alg_name(tb[1]);
	if (IS_ERR(cipher_name))
716
		return PTR_ERR(cipher_name);
717 718 719

	if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
	    CRYPTO_MAX_ALG_NAME)
720
		return -ENAMETOOLONG;
721 722 723

	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >=
	    CRYPTO_MAX_ALG_NAME)
724
		return -ENAMETOOLONG;
725

726 727
	return crypto_gcm_create_common(tmpl, tb, full_name,
					ctr_name, "ghash");
728 729
}

Mikko Herranen's avatar
Mikko Herranen committed
730 731
static struct crypto_template crypto_gcm_tmpl = {
	.name = "gcm",
732
	.create = crypto_gcm_create,
Mikko Herranen's avatar
Mikko Herranen committed
733 734 735
	.module = THIS_MODULE,
};

736 737
static int crypto_gcm_base_create(struct crypto_template *tmpl,
				  struct rtattr **tb)
738 739
{
	const char *ctr_name;
740
	const char *ghash_name;
741 742 743 744
	char full_name[CRYPTO_MAX_ALG_NAME];

	ctr_name = crypto_attr_alg_name(tb[1]);
	if (IS_ERR(ctr_name))
745
		return PTR_ERR(ctr_name);
746

747 748
	ghash_name = crypto_attr_alg_name(tb[2]);
	if (IS_ERR(ghash_name))
749
		return PTR_ERR(ghash_name);
750 751 752

	if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)",
		     ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME)
753
		return -ENAMETOOLONG;
754

755 756
	return crypto_gcm_create_common(tmpl, tb, full_name,
					ctr_name, ghash_name);
757 758 759 760
}

static struct crypto_template crypto_gcm_base_tmpl = {
	.name = "gcm_base",
761
	.create = crypto_gcm_base_create,
762 763 764
	.module = THIS_MODULE,
};

Herbert Xu's avatar
Herbert Xu committed
765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806
static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key,
				 unsigned int keylen)
{
	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
	struct crypto_aead *child = ctx->child;
	int err;

	if (keylen < 4)
		return -EINVAL;

	keylen -= 4;
	memcpy(ctx->nonce, key + keylen, 4);

	crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
	crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
				     CRYPTO_TFM_REQ_MASK);
	err = crypto_aead_setkey(child, key, keylen);
	crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
				      CRYPTO_TFM_RES_MASK);

	return err;
}

static int crypto_rfc4106_setauthsize(struct crypto_aead *parent,
				      unsigned int authsize)
{
	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);

	switch (authsize) {
	case 8:
	case 12:
	case 16:
		break;
	default:
		return -EINVAL;
	}

	return crypto_aead_setauthsize(ctx->child, authsize);
}

static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)
{
807
	struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req);
Herbert Xu's avatar
Herbert Xu committed
808 809
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead);
810
	struct aead_request *subreq = &rctx->subreq;
Herbert Xu's avatar
Herbert Xu committed
811
	struct crypto_aead *child = ctx->child;
812
	struct scatterlist *sg;
Herbert Xu's avatar
Herbert Xu committed
813 814 815
	u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),
			   crypto_aead_alignmask(child) + 1);

816
	scatterwalk_map_and_copy(iv + GCM_AES_IV_SIZE, req->src, 0, req->assoclen - 8, 0);
817

Herbert Xu's avatar
Herbert Xu committed
818 819 820
	memcpy(iv, ctx->nonce, 4);
	memcpy(iv + 4, req->iv, 8);

821
	sg_init_table(rctx->src, 3);
822
	sg_set_buf(rctx->src, iv + GCM_AES_IV_SIZE, req->assoclen - 8);
823 824 825 826 827 828
	sg = scatterwalk_ffwd(rctx->src + 1, req->src, req->assoclen);
	if (sg != rctx->src + 1)
		sg_chain(rctx->src, 2, sg);

	if (req->src != req->dst) {
		sg_init_table(rctx->dst, 3);
829
		sg_set_buf(rctx->dst, iv + GCM_AES_IV_SIZE, req->assoclen - 8);
830 831 832 833 834
		sg = scatterwalk_ffwd(rctx->dst + 1, req->dst, req->assoclen);
		if (sg != rctx->dst + 1)
			sg_chain(rctx->dst, 2, sg);
	}

Herbert Xu's avatar
Herbert Xu committed
835 836 837
	aead_request_set_tfm(subreq, child);
	aead_request_set_callback(subreq, req->base.flags, req->base.complete,
				  req->base.data);
838 839 840 841
	aead_request_set_crypt(subreq, rctx->src,
			       req->src == req->dst ? rctx->src : rctx->dst,
			       req->cryptlen, iv);
	aead_request_set_ad(subreq, req->assoclen - 8);
Herbert Xu's avatar
Herbert Xu committed
842 843 844 845 846 847

	return subreq;
}

static int crypto_rfc4106_encrypt(struct aead_request *req)
{
848 849 850
	if (req->assoclen != 16 && req->assoclen != 20)
		return -EINVAL;

Herbert Xu's avatar
Herbert Xu committed
851 852 853 854 855 856 857
	req = crypto_rfc4106_crypt(req);

	return crypto_aead_encrypt(req);
}

static int crypto_rfc4106_decrypt(struct aead_request *req)
{
858 859 860
	if (req->assoclen != 16 && req->assoclen != 20)
		return -EINVAL;

Herbert Xu's avatar
Herbert Xu committed
861 862 863 864 865
	req = crypto_rfc4106_crypt(req);

	return crypto_aead_decrypt(req);
}

866
static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm)
Herbert Xu's avatar
Herbert Xu committed
867
{
868 869 870
	struct aead_instance *inst = aead_alg_instance(tfm);
	struct crypto_aead_spawn *spawn = aead_instance_ctx(inst);
	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
Herbert Xu's avatar
Herbert Xu committed
871 872 873 874 875 876 877 878 879 880 881
	struct crypto_aead *aead;
	unsigned long align;

	aead = crypto_spawn_aead(spawn);
	if (IS_ERR(aead))
		return PTR_ERR(aead);

	ctx->child = aead;

	align = crypto_aead_alignmask(aead);
	align &= ~(crypto_tfm_ctx_alignment() - 1);
882 883
	crypto_aead_set_reqsize(
		tfm,
884
		sizeof(struct crypto_rfc4106_req_ctx) +
885
		ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
886
		align + 24);
Herbert Xu's avatar
Herbert Xu committed
887 888 889 890

	return 0;
}

891
static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm)
Herbert Xu's avatar
Herbert Xu committed
892
{
893
	struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
Herbert Xu's avatar
Herbert Xu committed
894 895 896 897

	crypto_free_aead(ctx->child);
}

898 899 900 901 902 903
static void crypto_rfc4106_free(struct aead_instance *inst)
{
	crypto_drop_aead(aead_instance_ctx(inst));
	kfree(inst);
}

904 905
static int crypto_rfc4106_create(struct crypto_template *tmpl,
				 struct rtattr **tb)
Herbert Xu's avatar
Herbert Xu committed
906 907
{
	struct crypto_attr_type *algt;
908
	struct aead_instance *inst;
Herbert Xu's avatar
Herbert Xu committed
909
	struct crypto_aead_spawn *spawn;
910
	struct aead_alg *alg;
Herbert Xu's avatar
Herbert Xu committed
911 912 913 914 915
	const char *ccm_name;
	int err;

	algt = crypto_get_attr_type(tb);
	if (IS_ERR(algt))
916
		return PTR_ERR(algt);
Herbert Xu's avatar
Herbert Xu committed
917

918
	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
919
		return -EINVAL;
Herbert Xu's avatar
Herbert Xu committed
920 921 922

	ccm_name = crypto_attr_alg_name(tb[1]);
	if (IS_ERR(ccm_name))
923
		return PTR_ERR(ccm_name);
Herbert Xu's avatar
Herbert Xu committed
924 925 926

	inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
	if (!inst)
927
		return -ENOMEM;
Herbert Xu's avatar
Herbert Xu committed
928

929 930
	spawn = aead_instance_ctx(inst);
	crypto_set_aead_spawn(spawn, aead_crypto_instance(inst));
Herbert Xu's avatar
Herbert Xu committed
931 932 933 934 935
	err = crypto_grab_aead(spawn, ccm_name, 0,
			       crypto_requires_sync(algt->type, algt->mask));
	if (err)
		goto out_free_inst;

936
	alg = crypto_spawn_aead_alg(spawn);
Herbert Xu's avatar
Herbert Xu committed
937 938 939

	err = -EINVAL;

940
	/* Underlying IV size must be 12. */
941
	if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE)
Herbert Xu's avatar
Herbert Xu committed
942 943 944
		goto out_drop_alg;

	/* Not a stream cipher? */
945
	if (alg->base.cra_blocksize != 1)
Herbert Xu's avatar
Herbert Xu committed
946 947 948
		goto out_drop_alg;

	err = -ENAMETOOLONG;
949 950 951 952 953
	if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
		     "rfc4106(%s)", alg->base.cra_name) >=
	    CRYPTO_MAX_ALG_NAME ||
	    snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
		     "rfc4106(%s)", alg->base.cra_driver_name) >=
Herbert Xu's avatar
Herbert Xu committed
954 955 956
	    CRYPTO_MAX_ALG_NAME)
		goto out_drop_alg;

957
	inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
958 959 960
	inst->alg.base.cra_priority = alg->base.cra_priority;
	inst->alg.base.cra_blocksize = 1;
	inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
Herbert Xu's avatar
Herbert Xu committed
961

962
	inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
Herbert Xu's avatar
Herbert Xu committed
963

964
	inst->alg.ivsize = GCM_RFC4106_IV_SIZE;
Herbert Xu's avatar
Herbert Xu committed
965
	inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
966
	inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
Herbert Xu's avatar
Herbert Xu committed
967

968 969
	inst->alg.init = crypto_rfc4106_init_tfm;
	inst->alg.exit = crypto_rfc4106_exit_tfm;
Herbert Xu's avatar
Herbert Xu committed
970

971 972 973 974
	inst->alg.setkey = crypto_rfc4106_setkey;
	inst->alg.setauthsize = crypto_rfc4106_setauthsize;
	inst->alg.encrypt = crypto_rfc4106_encrypt;
	inst->alg.decrypt = crypto_rfc4106_decrypt;
Herbert Xu's avatar
Herbert Xu committed
975

976 977
	inst->free = crypto_rfc4106_free;

978 979 980
	err = aead_register_instance(tmpl, inst);
	if (err)
		goto out_drop_alg;
Herbert Xu's avatar
Herbert Xu committed
981 982

out:
983
	return err;
Herbert Xu's avatar
Herbert Xu committed
984 985 986 987 988 989 990 991 992 993

out_drop_alg:
	crypto_drop_aead(spawn);
out_free_inst:
	kfree(inst);
	goto out;
}

static struct crypto_template crypto_rfc4106_tmpl = {
	.name = "rfc4106",
994
	.create = crypto_rfc4106_create,
Herbert Xu's avatar
Herbert Xu committed
995 996 997
	.module = THIS_MODULE,
};

998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key,
				 unsigned int keylen)
{
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);
	struct crypto_aead *child = ctx->child;
	int err;

	if (keylen < 4)
		return -EINVAL;

	keylen -= 4;
	memcpy(ctx->nonce, key + keylen, 4);

	crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
	crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
				     CRYPTO_TFM_REQ_MASK);
	err = crypto_aead_setkey(child, key, keylen);
	crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
				      CRYPTO_TFM_RES_MASK);

	return err;
}

static int crypto_rfc4543_setauthsize(struct crypto_aead *parent,
				      unsigned int authsize)
{
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);

	if (authsize != 16)
		return -EINVAL;

	return crypto_aead_setauthsize(ctx->child, authsize);
}

1032
static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)
1033 1034 1035
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
1036
	struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);
1037 1038 1039 1040
	struct aead_request *subreq = &rctx->subreq;
	unsigned int authsize = crypto_aead_authsize(aead);
	u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child),
			   crypto_aead_alignmask(ctx->child) + 1);
1041 1042 1043 1044 1045 1046 1047
	int err;

	if (req->src != req->dst) {
		err = crypto_rfc4543_copy_src_to_dst(req, enc);
		if (err)
			return err;
	}
1048 1049 1050 1051 1052

	memcpy(iv, ctx->nonce, 4);
	memcpy(iv + 4, req->iv, 8);

	aead_request_set_tfm(subreq, ctx->child);
1053 1054 1055 1056 1057 1058 1059 1060
	aead_request_set_callback(subreq, req->base.flags,
				  req->base.complete, req->base.data);
	aead_request_set_crypt(subreq, req->src, req->dst,
			       enc ? 0 : authsize, iv);
	aead_request_set_ad(subreq, req->assoclen + req->cryptlen -
				    subreq->cryptlen);

	return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq);
1061 1062
}

1063 1064 1065 1066 1067
static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)
{
	struct crypto_aead *aead = crypto_aead_reqtfm(req);
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
	unsigned int authsize = crypto_aead_authsize(aead);
1068 1069
	unsigned int nbytes = req->assoclen + req->cryptlen -
			      (enc ? 0 : authsize);
1070
	SYNC_SKCIPHER_REQUEST_ON_STACK(nreq, ctx->null);
Herbert Xu's avatar
Herbert Xu committed
1071

1072
	skcipher_request_set_sync_tfm(nreq, ctx->null);
Herbert Xu's avatar
Herbert Xu committed
1073 1074
	skcipher_request_set_callback(nreq, req->base.flags, NULL, NULL);
	skcipher_request_set_crypt(nreq, req->src, req->dst, nbytes, NULL);
1075

Herbert Xu's avatar
Herbert Xu committed
1076
	return crypto_skcipher_encrypt(nreq);
1077 1078
}

1079 1080
static int crypto_rfc4543_encrypt(struct aead_request *req)
{
1081
	return crypto_rfc4543_crypt(req, true);
1082 1083 1084 1085
}

static int crypto_rfc4543_decrypt(struct aead_request *req)
{
1086
	return crypto_rfc4543_crypt(req, false);
1087 1088
}

1089
static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)
1090
{
1091 1092
	struct aead_instance *inst = aead_alg_instance(tfm);
	struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst);
1093
	struct crypto_aead_spawn *spawn = &ictx->aead;
1094
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
1095
	struct crypto_aead *aead;
1096
	struct crypto_sync_skcipher *null;
1097
	unsigned long align;
1098
	int err = 0;
1099 1100 1101 1102 1103

	aead = crypto_spawn_aead(spawn);
	if (IS_ERR(aead))
		return PTR_ERR(aead);

1104
	null = crypto_get_default_null_skcipher();
1105 1106 1107 1108
	err = PTR_ERR(null);
	if (IS_ERR(null))
		goto err_free_aead;

1109
	ctx->child = aead;
1110
	ctx->null = null;
1111 1112 1113

	align = crypto_aead_alignmask(aead);
	align &= ~(crypto_tfm_ctx_alignment() - 1);
1114 1115
	crypto_aead_set_reqsize(
		tfm,
1116 1117
		sizeof(struct crypto_rfc4543_req_ctx) +
		ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
1118
		align + GCM_AES_IV_SIZE);
1119 1120

	return 0;
1121 1122 1123 1124

err_free_aead:
	crypto_free_aead(aead);
	return err;
1125 1126
}

1127
static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm)
1128
{
1129
	struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
1130 1131

	crypto_free_aead(ctx->child);
1132
	crypto_put_default_null_skcipher();
1133 1134
}

1135 1136 1137 1138 1139 1140 1141 1142 1143
static void crypto_rfc4543_free(struct aead_instance *inst)
{
	struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst);

	crypto_drop_aead(&ctx->aead);

	kfree(inst);
}

1144 1145
static int crypto_rfc4543_create(struct crypto_template *tmpl,
				struct rtattr **tb)
1146 1147
{
	struct crypto_attr_type *algt;
1148
	struct aead_instance *inst;
1149
	struct crypto_aead_spawn *spawn;
1150
	struct aead_alg *alg;
1151
	struct crypto_rfc4543_instance_ctx *ctx;
1152 1153 1154 1155 1156
	const char *ccm_name;
	int err;

	algt = crypto_get_attr_type(tb);
	if (IS_ERR(algt))
1157
		return PTR_ERR(algt);
1158

1159
	if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
1160
		return -EINVAL;
1161 1162 1163

	ccm_name = crypto_attr_alg_name(tb[1]);
	if (IS_ERR(ccm_name))
1164
		return PTR_ERR(ccm_name);
1165

1166
	inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
1167
	if (!inst)
1168
		return -ENOMEM;
1169

1170
	ctx = aead_instance_ctx(inst);
1171
	spawn = &ctx->aead;
1172
	crypto_set_aead_spawn(spawn, aead_crypto_instance(inst));
1173 1174 1175 1176 1177
	err = crypto_grab_aead(spawn, ccm_name, 0,
			       crypto_requires_sync(algt->type, algt->mask));
	if (err)
		goto out_free_inst;

1178
	alg = crypto_spawn_aead_alg(spawn);
1179 1180 1181

	err = -EINVAL;

1182
	/* Underlying IV size must be 12. */
1183
	if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE)
1184
		goto out_drop_alg;
1185 1186

	/* Not a stream cipher? */
1187
	if (alg->base.cra_blocksize != 1)
1188
		goto out_drop_alg;
1189 1190

	err = -ENAMETOOLONG;
1191 1192 1193 1194 1195
	if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
		     "rfc4543(%s)", alg->base.cra_name) >=
	    CRYPTO_MAX_ALG_NAME ||
	    snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
		     "rfc4543(%s)", alg->base.cra_driver_name) >=
1196
	    CRYPTO_MAX_ALG_NAME)
1197
		goto out_drop_alg;
1198

1199 1200 1201 1202
	inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
	inst->alg.base.cra_priority = alg->base.cra_priority;
	inst->alg.base.cra_blocksize = 1;
	inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
1203

1204
	inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx);
1205

1206
	inst->alg.ivsize = GCM_RFC4543_IV_SIZE;
Herbert Xu's avatar
Herbert Xu committed
1207
	inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
1208
	inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
1209

1210 1211
	inst->alg.init = crypto_rfc4543_init_tfm;
	inst->alg.exit = crypto_rfc4543_exit_tfm;
1212

1213 1214 1215 1216
	inst->alg.setkey = crypto_rfc4543_setkey;
	inst->alg.setauthsize = crypto_rfc4543_setauthsize;
	inst->alg.encrypt = crypto_rfc4543_encrypt;
	inst->alg.decrypt = crypto_rfc4543_decrypt;
1217

1218 1219
	inst->free = crypto_rfc4543_free,

1220 1221 1222
	err = aead_register_instance(tmpl, inst);
	if (err)
		goto out_drop_alg;
1223 1224

out:
1225
	return err;
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235

out_drop_alg:
	crypto_drop_aead(spawn);
out_free_inst:
	kfree(inst);
	goto out;
}

static struct crypto_template crypto_rfc4543_tmpl = {
	.name = "rfc4543",
1236
	.create = crypto_rfc4543_create,
1237 1238 1239
	.module = THIS_MODULE,
};

Mikko Herranen's avatar
Mikko Herranen committed
1240 1241
static int __init crypto_gcm_module_init(void)
{
1242 1243
	int err;

1244
	gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL);
1245 1246 1247
	if (!gcm_zeroes)
		return -ENOMEM;

1248 1249
	sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf));

1250 1251 1252 1253 1254 1255 1256 1257
	err = crypto_register_template(&crypto_gcm_base_tmpl);
	if (err)
		goto out;

	err = crypto_register_template(&crypto_gcm_tmpl);
	if (err)
		goto out_undo_base;

Herbert Xu's avatar
Herbert Xu committed
1258 1259 1260 1261
	err = crypto_register_template(&crypto_rfc4106_tmpl);
	if (err)
		goto out_undo_gcm;

1262 1263 1264 1265
	err = crypto_register_template(&crypto_rfc4543_tmpl);
	if (err)
		goto out_undo_rfc4106;

1266
	return 0;
1267

1268 1269
out_undo_rfc4106:
	crypto_unregister_template(&crypto_rfc4106_tmpl);
Herbert Xu's avatar
Herbert Xu committed
1270 1271
out_undo_gcm:
	crypto_unregister_template(&crypto_gcm_tmpl);
1272 1273
out_undo_base:
	crypto_unregister_template(&crypto_gcm_base_tmpl);
1274 1275 1276
out:
	kfree(gcm_zeroes);
	return err;
Mikko Herranen's avatar
Mikko Herranen committed
1277 1278 1279 1280
}

static void __exit crypto_gcm_module_exit(void)
{
1281
	kfree(gcm_zeroes);
1282
	crypto_unregister_template(&crypto_rfc4543_tmpl);
Herbert Xu's avatar
Herbert Xu committed
1283
	crypto_unregister_template(&crypto_rfc4106_tmpl);
Mikko Herranen's avatar
Mikko Herranen committed
1284
	crypto_unregister_template(&crypto_gcm_tmpl);
1285
	crypto_unregister_template(&crypto_gcm_base_tmpl);
Mikko Herranen's avatar
Mikko Herranen committed
1286 1287 1288 1289 1290 1291 1292 1293
}

module_init(crypto_gcm_module_init);
module_exit(crypto_gcm_module_exit);

MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("Galois/Counter Mode");
MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
1294 1295 1296
MODULE_ALIAS_CRYPTO("gcm_base");
MODULE_ALIAS_CRYPTO("rfc4106");
MODULE_ALIAS_CRYPTO("rfc4543");
1297
MODULE_ALIAS_CRYPTO("gcm");