1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * GCM: Galois/Counter Mode.
4 *
5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
6 */
7
8#include <crypto/gf128mul.h>
9#include <crypto/internal/aead.h>
10#include <crypto/internal/skcipher.h>
11#include <crypto/internal/hash.h>
12#include <crypto/scatterwalk.h>
13#include <crypto/gcm.h>
14#include <crypto/hash.h>
15#include <linux/err.h>
16#include <linux/init.h>
17#include <linux/kernel.h>
18#include <linux/module.h>
19#include <linux/slab.h>
20
21struct gcm_instance_ctx {
22 struct crypto_skcipher_spawn ctr;
23 struct crypto_ahash_spawn ghash;
24};
25
26struct crypto_gcm_ctx {
27 struct crypto_skcipher *ctr;
28 struct crypto_ahash *ghash;
29};
30
31struct crypto_rfc4106_ctx {
32 struct crypto_aead *child;
33 u8 nonce[4];
34};
35
36struct crypto_rfc4106_req_ctx {
37 struct scatterlist src[3];
38 struct scatterlist dst[3];
39 struct aead_request subreq;
40};
41
42struct crypto_rfc4543_instance_ctx {
43 struct crypto_aead_spawn aead;
44};
45
46struct crypto_rfc4543_ctx {
47 struct crypto_aead *child;
48 u8 nonce[4];
49};
50
51struct crypto_rfc4543_req_ctx {
52 struct aead_request subreq;
53};
54
55struct crypto_gcm_ghash_ctx {
56 unsigned int cryptlen;
57 struct scatterlist *src;
58 int (*complete)(struct aead_request *req, u32 flags);
59};
60
61struct crypto_gcm_req_priv_ctx {
62 u8 iv[16];
63 u8 auth_tag[16];
64 u8 iauth_tag[16];
65 struct scatterlist src[3];
66 struct scatterlist dst[3];
67 struct scatterlist sg;
68 struct crypto_gcm_ghash_ctx ghash_ctx;
69 union {
70 struct ahash_request ahreq;
71 struct skcipher_request skreq;
72 } u;
73};
74
75static struct {
76 u8 buf[16];
77 struct scatterlist sg;
78} *gcm_zeroes;
79
80static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
81 struct aead_request *req)
82{
83 unsigned long align = crypto_aead_alignmask(tfm: crypto_aead_reqtfm(req));
84
85 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
86}
87
88static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
89 unsigned int keylen)
90{
91 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm: aead);
92 struct crypto_ahash *ghash = ctx->ghash;
93 struct crypto_skcipher *ctr = ctx->ctr;
94 struct {
95 be128 hash;
96 u8 iv[16];
97
98 struct crypto_wait wait;
99
100 struct scatterlist sg[1];
101 struct skcipher_request req;
102 } *data;
103 int err;
104
105 crypto_skcipher_clear_flags(tfm: ctr, CRYPTO_TFM_REQ_MASK);
106 crypto_skcipher_set_flags(tfm: ctr, flags: crypto_aead_get_flags(tfm: aead) &
107 CRYPTO_TFM_REQ_MASK);
108 err = crypto_skcipher_setkey(tfm: ctr, key, keylen);
109 if (err)
110 return err;
111
112 data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
113 GFP_KERNEL);
114 if (!data)
115 return -ENOMEM;
116
117 crypto_init_wait(wait: &data->wait);
118 sg_init_one(data->sg, &data->hash, sizeof(data->hash));
119 skcipher_request_set_tfm(req: &data->req, tfm: ctr);
120 skcipher_request_set_callback(req: &data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
121 CRYPTO_TFM_REQ_MAY_BACKLOG,
122 compl: crypto_req_done,
123 data: &data->wait);
124 skcipher_request_set_crypt(req: &data->req, src: data->sg, dst: data->sg,
125 cryptlen: sizeof(data->hash), iv: data->iv);
126
127 err = crypto_wait_req(err: crypto_skcipher_encrypt(req: &data->req),
128 wait: &data->wait);
129
130 if (err)
131 goto out;
132
133 crypto_ahash_clear_flags(tfm: ghash, CRYPTO_TFM_REQ_MASK);
134 crypto_ahash_set_flags(tfm: ghash, flags: crypto_aead_get_flags(tfm: aead) &
135 CRYPTO_TFM_REQ_MASK);
136 err = crypto_ahash_setkey(tfm: ghash, key: (u8 *)&data->hash, keylen: sizeof(be128));
137out:
138 kfree_sensitive(objp: data);
139 return err;
140}
141
142static int crypto_gcm_setauthsize(struct crypto_aead *tfm,
143 unsigned int authsize)
144{
145 return crypto_gcm_check_authsize(authsize);
146}
147
148static void crypto_gcm_init_common(struct aead_request *req)
149{
150 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
151 __be32 counter = cpu_to_be32(1);
152 struct scatterlist *sg;
153
154 memset(s: pctx->auth_tag, c: 0, n: sizeof(pctx->auth_tag));
155 memcpy(to: pctx->iv, from: req->iv, GCM_AES_IV_SIZE);
156 memcpy(to: pctx->iv + GCM_AES_IV_SIZE, from: &counter, len: 4);
157
158 sg_init_table(pctx->src, 3);
159 sg_set_buf(sg: pctx->src, buf: pctx->auth_tag, buflen: sizeof(pctx->auth_tag));
160 sg = scatterwalk_ffwd(dst: pctx->src + 1, src: req->src, len: req->assoclen);
161 if (sg != pctx->src + 1)
162 sg_chain(prv: pctx->src, prv_nents: 2, sgl: sg);
163
164 if (req->src != req->dst) {
165 sg_init_table(pctx->dst, 3);
166 sg_set_buf(sg: pctx->dst, buf: pctx->auth_tag, buflen: sizeof(pctx->auth_tag));
167 sg = scatterwalk_ffwd(dst: pctx->dst + 1, src: req->dst, len: req->assoclen);
168 if (sg != pctx->dst + 1)
169 sg_chain(prv: pctx->dst, prv_nents: 2, sgl: sg);
170 }
171}
172
173static void crypto_gcm_init_crypt(struct aead_request *req,
174 unsigned int cryptlen)
175{
176 struct crypto_aead *aead = crypto_aead_reqtfm(req);
177 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm: aead);
178 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
179 struct skcipher_request *skreq = &pctx->u.skreq;
180 struct scatterlist *dst;
181
182 dst = req->src == req->dst ? pctx->src : pctx->dst;
183
184 skcipher_request_set_tfm(req: skreq, tfm: ctx->ctr);
185 skcipher_request_set_crypt(req: skreq, src: pctx->src, dst,
186 cryptlen: cryptlen + sizeof(pctx->auth_tag),
187 iv: pctx->iv);
188}
189
190static inline unsigned int gcm_remain(unsigned int len)
191{
192 len &= 0xfU;
193 return len ? 16 - len : 0;
194}
195
196static void gcm_hash_len_done(void *data, int err);
197
198static int gcm_hash_update(struct aead_request *req,
199 crypto_completion_t compl,
200 struct scatterlist *src,
201 unsigned int len, u32 flags)
202{
203 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
204 struct ahash_request *ahreq = &pctx->u.ahreq;
205
206 ahash_request_set_callback(req: ahreq, flags, compl, data: req);
207 ahash_request_set_crypt(req: ahreq, src, NULL, nbytes: len);
208
209 return crypto_ahash_update(req: ahreq);
210}
211
212static int gcm_hash_remain(struct aead_request *req,
213 unsigned int remain,
214 crypto_completion_t compl, u32 flags)
215{
216 return gcm_hash_update(req, compl, src: &gcm_zeroes->sg, len: remain, flags);
217}
218
219static int gcm_hash_len(struct aead_request *req, u32 flags)
220{
221 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
222 struct ahash_request *ahreq = &pctx->u.ahreq;
223 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
224 be128 lengths;
225
226 lengths.a = cpu_to_be64(req->assoclen * 8);
227 lengths.b = cpu_to_be64(gctx->cryptlen * 8);
228 memcpy(to: pctx->iauth_tag, from: &lengths, len: 16);
229 sg_init_one(&pctx->sg, pctx->iauth_tag, 16);
230 ahash_request_set_callback(req: ahreq, flags, compl: gcm_hash_len_done, data: req);
231 ahash_request_set_crypt(req: ahreq, src: &pctx->sg,
232 result: pctx->iauth_tag, nbytes: sizeof(lengths));
233
234 return crypto_ahash_finup(req: ahreq);
235}
236
237static int gcm_hash_len_continue(struct aead_request *req, u32 flags)
238{
239 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
240 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
241
242 return gctx->complete(req, flags);
243}
244
245static void gcm_hash_len_done(void *data, int err)
246{
247 struct aead_request *req = data;
248
249 if (err)
250 goto out;
251
252 err = gcm_hash_len_continue(req, flags: 0);
253 if (err == -EINPROGRESS)
254 return;
255
256out:
257 aead_request_complete(req, err);
258}
259
260static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)
261{
262 return gcm_hash_len(req, flags) ?:
263 gcm_hash_len_continue(req, flags);
264}
265
266static void gcm_hash_crypt_remain_done(void *data, int err)
267{
268 struct aead_request *req = data;
269
270 if (err)
271 goto out;
272
273 err = gcm_hash_crypt_remain_continue(req, flags: 0);
274 if (err == -EINPROGRESS)
275 return;
276
277out:
278 aead_request_complete(req, err);
279}
280
281static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)
282{
283 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
284 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
285 unsigned int remain;
286
287 remain = gcm_remain(len: gctx->cryptlen);
288 if (remain)
289 return gcm_hash_remain(req, remain,
290 compl: gcm_hash_crypt_remain_done, flags) ?:
291 gcm_hash_crypt_remain_continue(req, flags);
292
293 return gcm_hash_crypt_remain_continue(req, flags);
294}
295
296static void gcm_hash_crypt_done(void *data, int err)
297{
298 struct aead_request *req = data;
299
300 if (err)
301 goto out;
302
303 err = gcm_hash_crypt_continue(req, flags: 0);
304 if (err == -EINPROGRESS)
305 return;
306
307out:
308 aead_request_complete(req, err);
309}
310
311static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)
312{
313 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
314 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
315
316 if (gctx->cryptlen)
317 return gcm_hash_update(req, compl: gcm_hash_crypt_done,
318 src: gctx->src, len: gctx->cryptlen, flags) ?:
319 gcm_hash_crypt_continue(req, flags);
320
321 return gcm_hash_crypt_remain_continue(req, flags);
322}
323
324static void gcm_hash_assoc_remain_done(void *data, int err)
325{
326 struct aead_request *req = data;
327
328 if (err)
329 goto out;
330
331 err = gcm_hash_assoc_remain_continue(req, flags: 0);
332 if (err == -EINPROGRESS)
333 return;
334
335out:
336 aead_request_complete(req, err);
337}
338
339static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)
340{
341 unsigned int remain;
342
343 remain = gcm_remain(len: req->assoclen);
344 if (remain)
345 return gcm_hash_remain(req, remain,
346 compl: gcm_hash_assoc_remain_done, flags) ?:
347 gcm_hash_assoc_remain_continue(req, flags);
348
349 return gcm_hash_assoc_remain_continue(req, flags);
350}
351
352static void gcm_hash_assoc_done(void *data, int err)
353{
354 struct aead_request *req = data;
355
356 if (err)
357 goto out;
358
359 err = gcm_hash_assoc_continue(req, flags: 0);
360 if (err == -EINPROGRESS)
361 return;
362
363out:
364 aead_request_complete(req, err);
365}
366
367static int gcm_hash_init_continue(struct aead_request *req, u32 flags)
368{
369 if (req->assoclen)
370 return gcm_hash_update(req, compl: gcm_hash_assoc_done,
371 src: req->src, len: req->assoclen, flags) ?:
372 gcm_hash_assoc_continue(req, flags);
373
374 return gcm_hash_assoc_remain_continue(req, flags);
375}
376
377static void gcm_hash_init_done(void *data, int err)
378{
379 struct aead_request *req = data;
380
381 if (err)
382 goto out;
383
384 err = gcm_hash_init_continue(req, flags: 0);
385 if (err == -EINPROGRESS)
386 return;
387
388out:
389 aead_request_complete(req, err);
390}
391
392static int gcm_hash(struct aead_request *req, u32 flags)
393{
394 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
395 struct ahash_request *ahreq = &pctx->u.ahreq;
396 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm: crypto_aead_reqtfm(req));
397
398 ahash_request_set_tfm(req: ahreq, tfm: ctx->ghash);
399
400 ahash_request_set_callback(req: ahreq, flags, compl: gcm_hash_init_done, data: req);
401 return crypto_ahash_init(req: ahreq) ?:
402 gcm_hash_init_continue(req, flags);
403}
404
405static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)
406{
407 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
408 struct crypto_aead *aead = crypto_aead_reqtfm(req);
409 u8 *auth_tag = pctx->auth_tag;
410
411 crypto_xor(dst: auth_tag, src: pctx->iauth_tag, size: 16);
412 scatterwalk_map_and_copy(buf: auth_tag, sg: req->dst,
413 start: req->assoclen + req->cryptlen,
414 nbytes: crypto_aead_authsize(tfm: aead), out: 1);
415 return 0;
416}
417
418static int gcm_encrypt_continue(struct aead_request *req, u32 flags)
419{
420 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
421 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
422
423 gctx->src = sg_next(sg: req->src == req->dst ? pctx->src : pctx->dst);
424 gctx->cryptlen = req->cryptlen;
425 gctx->complete = gcm_enc_copy_hash;
426
427 return gcm_hash(req, flags);
428}
429
430static void gcm_encrypt_done(void *data, int err)
431{
432 struct aead_request *req = data;
433
434 if (err)
435 goto out;
436
437 err = gcm_encrypt_continue(req, flags: 0);
438 if (err == -EINPROGRESS)
439 return;
440
441out:
442 aead_request_complete(req, err);
443}
444
445static int crypto_gcm_encrypt(struct aead_request *req)
446{
447 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
448 struct skcipher_request *skreq = &pctx->u.skreq;
449 u32 flags = aead_request_flags(req);
450
451 crypto_gcm_init_common(req);
452 crypto_gcm_init_crypt(req, cryptlen: req->cryptlen);
453 skcipher_request_set_callback(req: skreq, flags, compl: gcm_encrypt_done, data: req);
454
455 return crypto_skcipher_encrypt(req: skreq) ?:
456 gcm_encrypt_continue(req, flags);
457}
458
459static int crypto_gcm_verify(struct aead_request *req)
460{
461 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
462 struct crypto_aead *aead = crypto_aead_reqtfm(req);
463 u8 *auth_tag = pctx->auth_tag;
464 u8 *iauth_tag = pctx->iauth_tag;
465 unsigned int authsize = crypto_aead_authsize(tfm: aead);
466 unsigned int cryptlen = req->cryptlen - authsize;
467
468 crypto_xor(dst: auth_tag, src: iauth_tag, size: 16);
469 scatterwalk_map_and_copy(buf: iauth_tag, sg: req->src,
470 start: req->assoclen + cryptlen, nbytes: authsize, out: 0);
471 return crypto_memneq(a: iauth_tag, b: auth_tag, size: authsize) ? -EBADMSG : 0;
472}
473
474static void gcm_decrypt_done(void *data, int err)
475{
476 struct aead_request *req = data;
477
478 if (!err)
479 err = crypto_gcm_verify(req);
480
481 aead_request_complete(req, err);
482}
483
484static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)
485{
486 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
487 struct skcipher_request *skreq = &pctx->u.skreq;
488 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
489
490 crypto_gcm_init_crypt(req, cryptlen: gctx->cryptlen);
491 skcipher_request_set_callback(req: skreq, flags, compl: gcm_decrypt_done, data: req);
492 return crypto_skcipher_decrypt(req: skreq) ?: crypto_gcm_verify(req);
493}
494
495static int crypto_gcm_decrypt(struct aead_request *req)
496{
497 struct crypto_aead *aead = crypto_aead_reqtfm(req);
498 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
499 struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
500 unsigned int authsize = crypto_aead_authsize(tfm: aead);
501 unsigned int cryptlen = req->cryptlen;
502 u32 flags = aead_request_flags(req);
503
504 cryptlen -= authsize;
505
506 crypto_gcm_init_common(req);
507
508 gctx->src = sg_next(sg: pctx->src);
509 gctx->cryptlen = cryptlen;
510 gctx->complete = gcm_dec_hash_continue;
511
512 return gcm_hash(req, flags);
513}
514
515static int crypto_gcm_init_tfm(struct crypto_aead *tfm)
516{
517 struct aead_instance *inst = aead_alg_instance(aead: tfm);
518 struct gcm_instance_ctx *ictx = aead_instance_ctx(inst);
519 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
520 struct crypto_skcipher *ctr;
521 struct crypto_ahash *ghash;
522 unsigned long align;
523 int err;
524
525 ghash = crypto_spawn_ahash(spawn: &ictx->ghash);
526 if (IS_ERR(ptr: ghash))
527 return PTR_ERR(ptr: ghash);
528
529 ctr = crypto_spawn_skcipher(spawn: &ictx->ctr);
530 err = PTR_ERR(ptr: ctr);
531 if (IS_ERR(ptr: ctr))
532 goto err_free_hash;
533
534 ctx->ctr = ctr;
535 ctx->ghash = ghash;
536
537 align = crypto_aead_alignmask(tfm);
538 align &= ~(crypto_tfm_ctx_alignment() - 1);
539 crypto_aead_set_reqsize(aead: tfm,
540 reqsize: align + offsetof(struct crypto_gcm_req_priv_ctx, u) +
541 max(sizeof(struct skcipher_request) +
542 crypto_skcipher_reqsize(ctr),
543 sizeof(struct ahash_request) +
544 crypto_ahash_reqsize(ghash)));
545
546 return 0;
547
548err_free_hash:
549 crypto_free_ahash(tfm: ghash);
550 return err;
551}
552
553static void crypto_gcm_exit_tfm(struct crypto_aead *tfm)
554{
555 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
556
557 crypto_free_ahash(tfm: ctx->ghash);
558 crypto_free_skcipher(tfm: ctx->ctr);
559}
560
561static void crypto_gcm_free(struct aead_instance *inst)
562{
563 struct gcm_instance_ctx *ctx = aead_instance_ctx(inst);
564
565 crypto_drop_skcipher(spawn: &ctx->ctr);
566 crypto_drop_ahash(spawn: &ctx->ghash);
567 kfree(objp: inst);
568}
569
570static int crypto_gcm_create_common(struct crypto_template *tmpl,
571 struct rtattr **tb,
572 const char *ctr_name,
573 const char *ghash_name)
574{
575 struct skcipher_alg_common *ctr;
576 u32 mask;
577 struct aead_instance *inst;
578 struct gcm_instance_ctx *ctx;
579 struct hash_alg_common *ghash;
580 int err;
581
582 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, mask_ret: &mask);
583 if (err)
584 return err;
585
586 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
587 if (!inst)
588 return -ENOMEM;
589 ctx = aead_instance_ctx(inst);
590
591 err = crypto_grab_ahash(spawn: &ctx->ghash, inst: aead_crypto_instance(inst),
592 name: ghash_name, type: 0, mask);
593 if (err)
594 goto err_free_inst;
595 ghash = crypto_spawn_ahash_alg(spawn: &ctx->ghash);
596
597 err = -EINVAL;
598 if (strcmp(ghash->base.cra_name, "ghash") != 0 ||
599 ghash->digestsize != 16)
600 goto err_free_inst;
601
602 err = crypto_grab_skcipher(spawn: &ctx->ctr, inst: aead_crypto_instance(inst),
603 name: ctr_name, type: 0, mask);
604 if (err)
605 goto err_free_inst;
606 ctr = crypto_spawn_skcipher_alg_common(spawn: &ctx->ctr);
607
608 /* The skcipher algorithm must be CTR mode, using 16-byte blocks. */
609 err = -EINVAL;
610 if (strncmp(ctr->base.cra_name, "ctr(", 4) != 0 ||
611 ctr->ivsize != 16 || ctr->base.cra_blocksize != 1)
612 goto err_free_inst;
613
614 err = -ENAMETOOLONG;
615 if (snprintf(buf: inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
616 fmt: "gcm(%s", ctr->base.cra_name + 4) >= CRYPTO_MAX_ALG_NAME)
617 goto err_free_inst;
618
619 if (snprintf(buf: inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
620 fmt: "gcm_base(%s,%s)", ctr->base.cra_driver_name,
621 ghash->base.cra_driver_name) >=
622 CRYPTO_MAX_ALG_NAME)
623 goto err_free_inst;
624
625 inst->alg.base.cra_priority = (ghash->base.cra_priority +
626 ctr->base.cra_priority) / 2;
627 inst->alg.base.cra_blocksize = 1;
628 inst->alg.base.cra_alignmask = ctr->base.cra_alignmask;
629 inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
630 inst->alg.ivsize = GCM_AES_IV_SIZE;
631 inst->alg.chunksize = ctr->chunksize;
632 inst->alg.maxauthsize = 16;
633 inst->alg.init = crypto_gcm_init_tfm;
634 inst->alg.exit = crypto_gcm_exit_tfm;
635 inst->alg.setkey = crypto_gcm_setkey;
636 inst->alg.setauthsize = crypto_gcm_setauthsize;
637 inst->alg.encrypt = crypto_gcm_encrypt;
638 inst->alg.decrypt = crypto_gcm_decrypt;
639
640 inst->free = crypto_gcm_free;
641
642 err = aead_register_instance(tmpl, inst);
643 if (err) {
644err_free_inst:
645 crypto_gcm_free(inst);
646 }
647 return err;
648}
649
650static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb)
651{
652 const char *cipher_name;
653 char ctr_name[CRYPTO_MAX_ALG_NAME];
654
655 cipher_name = crypto_attr_alg_name(rta: tb[1]);
656 if (IS_ERR(ptr: cipher_name))
657 return PTR_ERR(ptr: cipher_name);
658
659 if (snprintf(buf: ctr_name, CRYPTO_MAX_ALG_NAME, fmt: "ctr(%s)", cipher_name) >=
660 CRYPTO_MAX_ALG_NAME)
661 return -ENAMETOOLONG;
662
663 return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name: "ghash");
664}
665
666static int crypto_gcm_base_create(struct crypto_template *tmpl,
667 struct rtattr **tb)
668{
669 const char *ctr_name;
670 const char *ghash_name;
671
672 ctr_name = crypto_attr_alg_name(rta: tb[1]);
673 if (IS_ERR(ptr: ctr_name))
674 return PTR_ERR(ptr: ctr_name);
675
676 ghash_name = crypto_attr_alg_name(rta: tb[2]);
677 if (IS_ERR(ptr: ghash_name))
678 return PTR_ERR(ptr: ghash_name);
679
680 return crypto_gcm_create_common(tmpl, tb, ctr_name, ghash_name);
681}
682
683static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key,
684 unsigned int keylen)
685{
686 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm: parent);
687 struct crypto_aead *child = ctx->child;
688
689 if (keylen < 4)
690 return -EINVAL;
691
692 keylen -= 4;
693 memcpy(to: ctx->nonce, from: key + keylen, len: 4);
694
695 crypto_aead_clear_flags(tfm: child, CRYPTO_TFM_REQ_MASK);
696 crypto_aead_set_flags(tfm: child, flags: crypto_aead_get_flags(tfm: parent) &
697 CRYPTO_TFM_REQ_MASK);
698 return crypto_aead_setkey(tfm: child, key, keylen);
699}
700
701static int crypto_rfc4106_setauthsize(struct crypto_aead *parent,
702 unsigned int authsize)
703{
704 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm: parent);
705 int err;
706
707 err = crypto_rfc4106_check_authsize(authsize);
708 if (err)
709 return err;
710
711 return crypto_aead_setauthsize(tfm: ctx->child, authsize);
712}
713
714static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)
715{
716 struct crypto_rfc4106_req_ctx *rctx = aead_request_ctx(req);
717 struct crypto_aead *aead = crypto_aead_reqtfm(req);
718 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm: aead);
719 struct aead_request *subreq = &rctx->subreq;
720 struct crypto_aead *child = ctx->child;
721 struct scatterlist *sg;
722 u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),
723 crypto_aead_alignmask(child) + 1);
724
725 scatterwalk_map_and_copy(buf: iv + GCM_AES_IV_SIZE, sg: req->src, start: 0, nbytes: req->assoclen - 8, out: 0);
726
727 memcpy(to: iv, from: ctx->nonce, len: 4);
728 memcpy(to: iv + 4, from: req->iv, len: 8);
729
730 sg_init_table(rctx->src, 3);
731 sg_set_buf(sg: rctx->src, buf: iv + GCM_AES_IV_SIZE, buflen: req->assoclen - 8);
732 sg = scatterwalk_ffwd(dst: rctx->src + 1, src: req->src, len: req->assoclen);
733 if (sg != rctx->src + 1)
734 sg_chain(prv: rctx->src, prv_nents: 2, sgl: sg);
735
736 if (req->src != req->dst) {
737 sg_init_table(rctx->dst, 3);
738 sg_set_buf(sg: rctx->dst, buf: iv + GCM_AES_IV_SIZE, buflen: req->assoclen - 8);
739 sg = scatterwalk_ffwd(dst: rctx->dst + 1, src: req->dst, len: req->assoclen);
740 if (sg != rctx->dst + 1)
741 sg_chain(prv: rctx->dst, prv_nents: 2, sgl: sg);
742 }
743
744 aead_request_set_tfm(req: subreq, tfm: child);
745 aead_request_set_callback(req: subreq, flags: req->base.flags, compl: req->base.complete,
746 data: req->base.data);
747 aead_request_set_crypt(req: subreq, src: rctx->src,
748 dst: req->src == req->dst ? rctx->src : rctx->dst,
749 cryptlen: req->cryptlen, iv);
750 aead_request_set_ad(req: subreq, assoclen: req->assoclen - 8);
751
752 return subreq;
753}
754
755static int crypto_rfc4106_encrypt(struct aead_request *req)
756{
757 int err;
758
759 err = crypto_ipsec_check_assoclen(assoclen: req->assoclen);
760 if (err)
761 return err;
762
763 req = crypto_rfc4106_crypt(req);
764
765 return crypto_aead_encrypt(req);
766}
767
768static int crypto_rfc4106_decrypt(struct aead_request *req)
769{
770 int err;
771
772 err = crypto_ipsec_check_assoclen(assoclen: req->assoclen);
773 if (err)
774 return err;
775
776 req = crypto_rfc4106_crypt(req);
777
778 return crypto_aead_decrypt(req);
779}
780
781static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm)
782{
783 struct aead_instance *inst = aead_alg_instance(aead: tfm);
784 struct crypto_aead_spawn *spawn = aead_instance_ctx(inst);
785 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
786 struct crypto_aead *aead;
787 unsigned long align;
788
789 aead = crypto_spawn_aead(spawn);
790 if (IS_ERR(ptr: aead))
791 return PTR_ERR(ptr: aead);
792
793 ctx->child = aead;
794
795 align = crypto_aead_alignmask(tfm: aead);
796 align &= ~(crypto_tfm_ctx_alignment() - 1);
797 crypto_aead_set_reqsize(
798 aead: tfm,
799 reqsize: sizeof(struct crypto_rfc4106_req_ctx) +
800 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
801 align + 24);
802
803 return 0;
804}
805
806static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm)
807{
808 struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
809
810 crypto_free_aead(tfm: ctx->child);
811}
812
813static void crypto_rfc4106_free(struct aead_instance *inst)
814{
815 crypto_drop_aead(spawn: aead_instance_ctx(inst));
816 kfree(objp: inst);
817}
818
819static int crypto_rfc4106_create(struct crypto_template *tmpl,
820 struct rtattr **tb)
821{
822 u32 mask;
823 struct aead_instance *inst;
824 struct crypto_aead_spawn *spawn;
825 struct aead_alg *alg;
826 int err;
827
828 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, mask_ret: &mask);
829 if (err)
830 return err;
831
832 inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
833 if (!inst)
834 return -ENOMEM;
835
836 spawn = aead_instance_ctx(inst);
837 err = crypto_grab_aead(spawn, inst: aead_crypto_instance(inst),
838 name: crypto_attr_alg_name(rta: tb[1]), type: 0, mask);
839 if (err)
840 goto err_free_inst;
841
842 alg = crypto_spawn_aead_alg(spawn);
843
844 err = -EINVAL;
845
846 /* Underlying IV size must be 12. */
847 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE)
848 goto err_free_inst;
849
850 /* Not a stream cipher? */
851 if (alg->base.cra_blocksize != 1)
852 goto err_free_inst;
853
854 err = -ENAMETOOLONG;
855 if (snprintf(buf: inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
856 fmt: "rfc4106(%s)", alg->base.cra_name) >=
857 CRYPTO_MAX_ALG_NAME ||
858 snprintf(buf: inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
859 fmt: "rfc4106(%s)", alg->base.cra_driver_name) >=
860 CRYPTO_MAX_ALG_NAME)
861 goto err_free_inst;
862
863 inst->alg.base.cra_priority = alg->base.cra_priority;
864 inst->alg.base.cra_blocksize = 1;
865 inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
866
867 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
868
869 inst->alg.ivsize = GCM_RFC4106_IV_SIZE;
870 inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
871 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
872
873 inst->alg.init = crypto_rfc4106_init_tfm;
874 inst->alg.exit = crypto_rfc4106_exit_tfm;
875
876 inst->alg.setkey = crypto_rfc4106_setkey;
877 inst->alg.setauthsize = crypto_rfc4106_setauthsize;
878 inst->alg.encrypt = crypto_rfc4106_encrypt;
879 inst->alg.decrypt = crypto_rfc4106_decrypt;
880
881 inst->free = crypto_rfc4106_free;
882
883 err = aead_register_instance(tmpl, inst);
884 if (err) {
885err_free_inst:
886 crypto_rfc4106_free(inst);
887 }
888 return err;
889}
890
891static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key,
892 unsigned int keylen)
893{
894 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm: parent);
895 struct crypto_aead *child = ctx->child;
896
897 if (keylen < 4)
898 return -EINVAL;
899
900 keylen -= 4;
901 memcpy(to: ctx->nonce, from: key + keylen, len: 4);
902
903 crypto_aead_clear_flags(tfm: child, CRYPTO_TFM_REQ_MASK);
904 crypto_aead_set_flags(tfm: child, flags: crypto_aead_get_flags(tfm: parent) &
905 CRYPTO_TFM_REQ_MASK);
906 return crypto_aead_setkey(tfm: child, key, keylen);
907}
908
909static int crypto_rfc4543_setauthsize(struct crypto_aead *parent,
910 unsigned int authsize)
911{
912 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm: parent);
913
914 if (authsize != 16)
915 return -EINVAL;
916
917 return crypto_aead_setauthsize(tfm: ctx->child, authsize);
918}
919
920static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)
921{
922 struct crypto_aead *aead = crypto_aead_reqtfm(req);
923 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm: aead);
924 struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);
925 struct aead_request *subreq = &rctx->subreq;
926 unsigned int authsize = crypto_aead_authsize(tfm: aead);
927 u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child),
928 crypto_aead_alignmask(ctx->child) + 1);
929
930 if (req->src != req->dst) {
931 unsigned int nbytes = req->assoclen + req->cryptlen -
932 (enc ? 0 : authsize);
933
934 memcpy_sglist(dst: req->dst, src: req->src, nbytes);
935 }
936
937 memcpy(to: iv, from: ctx->nonce, len: 4);
938 memcpy(to: iv + 4, from: req->iv, len: 8);
939
940 aead_request_set_tfm(req: subreq, tfm: ctx->child);
941 aead_request_set_callback(req: subreq, flags: req->base.flags,
942 compl: req->base.complete, data: req->base.data);
943 aead_request_set_crypt(req: subreq, src: req->src, dst: req->dst,
944 cryptlen: enc ? 0 : authsize, iv);
945 aead_request_set_ad(req: subreq, assoclen: req->assoclen + req->cryptlen -
946 subreq->cryptlen);
947
948 return enc ? crypto_aead_encrypt(req: subreq) : crypto_aead_decrypt(req: subreq);
949}
950
951static int crypto_rfc4543_encrypt(struct aead_request *req)
952{
953 return crypto_ipsec_check_assoclen(assoclen: req->assoclen) ?:
954 crypto_rfc4543_crypt(req, enc: true);
955}
956
957static int crypto_rfc4543_decrypt(struct aead_request *req)
958{
959 return crypto_ipsec_check_assoclen(assoclen: req->assoclen) ?:
960 crypto_rfc4543_crypt(req, enc: false);
961}
962
963static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)
964{
965 struct aead_instance *inst = aead_alg_instance(aead: tfm);
966 struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst);
967 struct crypto_aead_spawn *spawn = &ictx->aead;
968 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
969 struct crypto_aead *aead;
970 unsigned long align;
971
972 aead = crypto_spawn_aead(spawn);
973 if (IS_ERR(ptr: aead))
974 return PTR_ERR(ptr: aead);
975
976 ctx->child = aead;
977
978 align = crypto_aead_alignmask(tfm: aead);
979 align &= ~(crypto_tfm_ctx_alignment() - 1);
980 crypto_aead_set_reqsize(
981 aead: tfm,
982 reqsize: sizeof(struct crypto_rfc4543_req_ctx) +
983 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
984 align + GCM_AES_IV_SIZE);
985
986 return 0;
987}
988
989static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm)
990{
991 struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
992
993 crypto_free_aead(tfm: ctx->child);
994}
995
996static void crypto_rfc4543_free(struct aead_instance *inst)
997{
998 struct crypto_rfc4543_instance_ctx *ctx = aead_instance_ctx(inst);
999
1000 crypto_drop_aead(spawn: &ctx->aead);
1001
1002 kfree(objp: inst);
1003}
1004
1005static int crypto_rfc4543_create(struct crypto_template *tmpl,
1006 struct rtattr **tb)
1007{
1008 u32 mask;
1009 struct aead_instance *inst;
1010 struct aead_alg *alg;
1011 struct crypto_rfc4543_instance_ctx *ctx;
1012 int err;
1013
1014 err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AEAD, mask_ret: &mask);
1015 if (err)
1016 return err;
1017
1018 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
1019 if (!inst)
1020 return -ENOMEM;
1021
1022 ctx = aead_instance_ctx(inst);
1023 err = crypto_grab_aead(spawn: &ctx->aead, inst: aead_crypto_instance(inst),
1024 name: crypto_attr_alg_name(rta: tb[1]), type: 0, mask);
1025 if (err)
1026 goto err_free_inst;
1027
1028 alg = crypto_spawn_aead_alg(spawn: &ctx->aead);
1029
1030 err = -EINVAL;
1031
1032 /* Underlying IV size must be 12. */
1033 if (crypto_aead_alg_ivsize(alg) != GCM_AES_IV_SIZE)
1034 goto err_free_inst;
1035
1036 /* Not a stream cipher? */
1037 if (alg->base.cra_blocksize != 1)
1038 goto err_free_inst;
1039
1040 err = -ENAMETOOLONG;
1041 if (snprintf(buf: inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
1042 fmt: "rfc4543(%s)", alg->base.cra_name) >=
1043 CRYPTO_MAX_ALG_NAME ||
1044 snprintf(buf: inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1045 fmt: "rfc4543(%s)", alg->base.cra_driver_name) >=
1046 CRYPTO_MAX_ALG_NAME)
1047 goto err_free_inst;
1048
1049 inst->alg.base.cra_priority = alg->base.cra_priority;
1050 inst->alg.base.cra_blocksize = 1;
1051 inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
1052
1053 inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx);
1054
1055 inst->alg.ivsize = GCM_RFC4543_IV_SIZE;
1056 inst->alg.chunksize = crypto_aead_alg_chunksize(alg);
1057 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
1058
1059 inst->alg.init = crypto_rfc4543_init_tfm;
1060 inst->alg.exit = crypto_rfc4543_exit_tfm;
1061
1062 inst->alg.setkey = crypto_rfc4543_setkey;
1063 inst->alg.setauthsize = crypto_rfc4543_setauthsize;
1064 inst->alg.encrypt = crypto_rfc4543_encrypt;
1065 inst->alg.decrypt = crypto_rfc4543_decrypt;
1066
1067 inst->free = crypto_rfc4543_free;
1068
1069 err = aead_register_instance(tmpl, inst);
1070 if (err) {
1071err_free_inst:
1072 crypto_rfc4543_free(inst);
1073 }
1074 return err;
1075}
1076
1077static struct crypto_template crypto_gcm_tmpls[] = {
1078 {
1079 .name = "gcm_base",
1080 .create = crypto_gcm_base_create,
1081 .module = THIS_MODULE,
1082 }, {
1083 .name = "gcm",
1084 .create = crypto_gcm_create,
1085 .module = THIS_MODULE,
1086 }, {
1087 .name = "rfc4106",
1088 .create = crypto_rfc4106_create,
1089 .module = THIS_MODULE,
1090 }, {
1091 .name = "rfc4543",
1092 .create = crypto_rfc4543_create,
1093 .module = THIS_MODULE,
1094 },
1095};
1096
1097static int __init crypto_gcm_module_init(void)
1098{
1099 int err;
1100
1101 gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL);
1102 if (!gcm_zeroes)
1103 return -ENOMEM;
1104
1105 sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf));
1106
1107 err = crypto_register_templates(tmpls: crypto_gcm_tmpls,
1108 ARRAY_SIZE(crypto_gcm_tmpls));
1109 if (err)
1110 kfree(objp: gcm_zeroes);
1111
1112 return err;
1113}
1114
1115static void __exit crypto_gcm_module_exit(void)
1116{
1117 kfree(objp: gcm_zeroes);
1118 crypto_unregister_templates(tmpls: crypto_gcm_tmpls,
1119 ARRAY_SIZE(crypto_gcm_tmpls));
1120}
1121
1122module_init(crypto_gcm_module_init);
1123module_exit(crypto_gcm_module_exit);
1124
1125MODULE_LICENSE("GPL");
1126MODULE_DESCRIPTION("Galois/Counter Mode");
1127MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
1128MODULE_ALIAS_CRYPTO("gcm_base");
1129MODULE_ALIAS_CRYPTO("rfc4106");
1130MODULE_ALIAS_CRYPTO("rfc4543");
1131MODULE_ALIAS_CRYPTO("gcm");
1132