1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Cryptographic API for algorithms (i.e., low-level API).
4 *
5 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
6 */
7
8#include <crypto/algapi.h>
9#include <linux/err.h>
10#include <linux/errno.h>
11#include <linux/fips.h>
12#include <linux/init.h>
13#include <linux/kernel.h>
14#include <linux/list.h>
15#include <linux/module.h>
16#include <linux/rtnetlink.h>
17#include <linux/slab.h>
18#include <linux/string.h>
19#include <linux/workqueue.h>
20
21#include "internal.h"
22
23static LIST_HEAD(crypto_template_list);
24
25static inline void crypto_check_module_sig(struct module *mod)
26{
27 if (fips_enabled && mod && !module_sig_ok(module: mod))
28 panic(fmt: "Module %s signature verification failed in FIPS mode\n",
29 module_name(mod));
30}
31
32static int crypto_check_alg(struct crypto_alg *alg)
33{
34 crypto_check_module_sig(mod: alg->cra_module);
35
36 if (!alg->cra_name[0] || !alg->cra_driver_name[0])
37 return -EINVAL;
38
39 if (alg->cra_alignmask & (alg->cra_alignmask + 1))
40 return -EINVAL;
41
42 /* General maximums for all algs. */
43 if (alg->cra_alignmask > MAX_ALGAPI_ALIGNMASK)
44 return -EINVAL;
45
46 if (alg->cra_blocksize > MAX_ALGAPI_BLOCKSIZE)
47 return -EINVAL;
48
49 /* Lower maximums for specific alg types. */
50 if (!alg->cra_type && (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) ==
51 CRYPTO_ALG_TYPE_CIPHER) {
52 if (alg->cra_alignmask > MAX_CIPHER_ALIGNMASK)
53 return -EINVAL;
54
55 if (alg->cra_blocksize > MAX_CIPHER_BLOCKSIZE)
56 return -EINVAL;
57 }
58
59 if (alg->cra_priority < 0)
60 return -EINVAL;
61
62 refcount_set(r: &alg->cra_refcnt, n: 1);
63
64 return 0;
65}
66
67static void crypto_free_instance(struct crypto_instance *inst)
68{
69 inst->alg.cra_type->free(inst);
70}
71
72static void crypto_destroy_instance_workfn(struct work_struct *w)
73{
74 struct crypto_template *tmpl = container_of(w, struct crypto_template,
75 free_work);
76 struct crypto_instance *inst;
77 struct hlist_node *n;
78 HLIST_HEAD(list);
79
80 down_write(sem: &crypto_alg_sem);
81 hlist_for_each_entry_safe(inst, n, &tmpl->dead, list) {
82 if (refcount_read(r: &inst->alg.cra_refcnt) != -1)
83 continue;
84 hlist_del(n: &inst->list);
85 hlist_add_head(n: &inst->list, h: &list);
86 }
87 up_write(sem: &crypto_alg_sem);
88
89 hlist_for_each_entry_safe(inst, n, &list, list)
90 crypto_free_instance(inst);
91}
92
93static void crypto_destroy_instance(struct crypto_alg *alg)
94{
95 struct crypto_instance *inst = container_of(alg,
96 struct crypto_instance,
97 alg);
98 struct crypto_template *tmpl = inst->tmpl;
99
100 refcount_set(r: &alg->cra_refcnt, n: -1);
101 schedule_work(work: &tmpl->free_work);
102}
103
104/*
105 * This function adds a spawn to the list secondary_spawns which
106 * will be used at the end of crypto_remove_spawns to unregister
107 * instances, unless the spawn happens to be one that is depended
108 * on by the new algorithm (nalg in crypto_remove_spawns).
109 *
110 * This function is also responsible for resurrecting any algorithms
111 * in the dependency chain of nalg by unsetting n->dead.
112 */
113static struct list_head *crypto_more_spawns(struct crypto_alg *alg,
114 struct list_head *stack,
115 struct list_head *top,
116 struct list_head *secondary_spawns)
117{
118 struct crypto_spawn *spawn, *n;
119
120 spawn = list_first_entry_or_null(stack, struct crypto_spawn, list);
121 if (!spawn)
122 return NULL;
123
124 n = list_prev_entry(spawn, list);
125 list_move(list: &spawn->list, head: secondary_spawns);
126
127 if (list_is_last(list: &n->list, head: stack))
128 return top;
129
130 n = list_next_entry(n, list);
131 if (!spawn->dead)
132 n->dead = false;
133
134 return &n->inst->alg.cra_users;
135}
136
137static void crypto_remove_instance(struct crypto_instance *inst,
138 struct list_head *list)
139{
140 struct crypto_template *tmpl = inst->tmpl;
141
142 if (crypto_is_dead(alg: &inst->alg))
143 return;
144
145 inst->alg.cra_flags |= CRYPTO_ALG_DEAD;
146
147 if (!tmpl)
148 return;
149
150 list_del_init(entry: &inst->alg.cra_list);
151 hlist_del(n: &inst->list);
152 hlist_add_head(n: &inst->list, h: &tmpl->dead);
153
154 BUG_ON(!list_empty(&inst->alg.cra_users));
155
156 crypto_alg_put(alg: &inst->alg);
157}
158
159/*
160 * Given an algorithm alg, remove all algorithms that depend on it
161 * through spawns. If nalg is not null, then exempt any algorithms
162 * that is depended on by nalg. This is useful when nalg itself
163 * depends on alg.
164 */
165void crypto_remove_spawns(struct crypto_alg *alg, struct list_head *list,
166 struct crypto_alg *nalg)
167{
168 u32 new_type = (nalg ?: alg)->cra_flags;
169 struct crypto_spawn *spawn, *n;
170 LIST_HEAD(secondary_spawns);
171 struct list_head *spawns;
172 LIST_HEAD(stack);
173 LIST_HEAD(top);
174
175 spawns = &alg->cra_users;
176 list_for_each_entry_safe(spawn, n, spawns, list) {
177 if ((spawn->alg->cra_flags ^ new_type) & spawn->mask)
178 continue;
179
180 list_move(list: &spawn->list, head: &top);
181 }
182
183 /*
184 * Perform a depth-first walk starting from alg through
185 * the cra_users tree. The list stack records the path
186 * from alg to the current spawn.
187 */
188 spawns = &top;
189 do {
190 while (!list_empty(head: spawns)) {
191 struct crypto_instance *inst;
192
193 spawn = list_first_entry(spawns, struct crypto_spawn,
194 list);
195 inst = spawn->inst;
196
197 list_move(list: &spawn->list, head: &stack);
198 spawn->dead = !spawn->registered || &inst->alg != nalg;
199
200 if (!spawn->registered)
201 break;
202
203 BUG_ON(&inst->alg == alg);
204
205 if (&inst->alg == nalg)
206 break;
207
208 spawns = &inst->alg.cra_users;
209
210 /*
211 * Even if spawn->registered is true, the
212 * instance itself may still be unregistered.
213 * This is because it may have failed during
214 * registration. Therefore we still need to
215 * make the following test.
216 *
217 * We may encounter an unregistered instance here, since
218 * an instance's spawns are set up prior to the instance
219 * being registered. An unregistered instance will have
220 * NULL ->cra_users.next, since ->cra_users isn't
221 * properly initialized until registration. But an
222 * unregistered instance cannot have any users, so treat
223 * it the same as ->cra_users being empty.
224 */
225 if (spawns->next == NULL)
226 break;
227 }
228 } while ((spawns = crypto_more_spawns(alg, stack: &stack, top: &top,
229 secondary_spawns: &secondary_spawns)));
230
231 /*
232 * Remove all instances that are marked as dead. Also
233 * complete the resurrection of the others by moving them
234 * back to the cra_users list.
235 */
236 list_for_each_entry_safe(spawn, n, &secondary_spawns, list) {
237 if (!spawn->dead)
238 list_move(list: &spawn->list, head: &spawn->alg->cra_users);
239 else if (spawn->registered)
240 crypto_remove_instance(inst: spawn->inst, list);
241 }
242}
243EXPORT_SYMBOL_GPL(crypto_remove_spawns);
244
245static void crypto_alg_finish_registration(struct crypto_alg *alg,
246 struct list_head *algs_to_put)
247{
248 struct crypto_alg *q;
249
250 list_for_each_entry(q, &crypto_alg_list, cra_list) {
251 if (q == alg)
252 continue;
253
254 if (crypto_is_moribund(alg: q))
255 continue;
256
257 if (crypto_is_larval(alg: q))
258 continue;
259
260 if (strcmp(alg->cra_name, q->cra_name))
261 continue;
262
263 if (strcmp(alg->cra_driver_name, q->cra_driver_name) &&
264 q->cra_priority > alg->cra_priority)
265 continue;
266
267 crypto_remove_spawns(q, algs_to_put, alg);
268 }
269
270 crypto_notify(val: CRYPTO_MSG_ALG_LOADED, v: alg);
271}
272
273static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg)
274{
275 struct crypto_larval *larval;
276
277 if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) ||
278 (alg->cra_flags & CRYPTO_ALG_INTERNAL))
279 return NULL; /* No self-test needed */
280
281 larval = crypto_larval_alloc(name: alg->cra_name,
282 type: alg->cra_flags | CRYPTO_ALG_TESTED, mask: 0);
283 if (IS_ERR(ptr: larval))
284 return larval;
285
286 larval->adult = crypto_mod_get(alg);
287 if (!larval->adult) {
288 kfree(objp: larval);
289 return ERR_PTR(error: -ENOENT);
290 }
291
292 refcount_set(r: &larval->alg.cra_refcnt, n: 1);
293 memcpy(to: larval->alg.cra_driver_name, from: alg->cra_driver_name,
294 CRYPTO_MAX_ALG_NAME);
295 larval->alg.cra_priority = alg->cra_priority;
296
297 return larval;
298}
299
300static struct crypto_larval *
301__crypto_register_alg(struct crypto_alg *alg, struct list_head *algs_to_put)
302{
303 struct crypto_alg *q;
304 struct crypto_larval *larval;
305 int ret = -EAGAIN;
306
307 if (crypto_is_dead(alg))
308 goto err;
309
310 INIT_LIST_HEAD(list: &alg->cra_users);
311
312 ret = -EEXIST;
313
314 list_for_each_entry(q, &crypto_alg_list, cra_list) {
315 if (q == alg)
316 goto err;
317
318 if (crypto_is_moribund(alg: q))
319 continue;
320
321 if (crypto_is_larval(alg: q)) {
322 if (!strcmp(alg->cra_driver_name, q->cra_driver_name))
323 goto err;
324 continue;
325 }
326
327 if (!strcmp(q->cra_driver_name, alg->cra_name) ||
328 !strcmp(q->cra_driver_name, alg->cra_driver_name) ||
329 !strcmp(q->cra_name, alg->cra_driver_name))
330 goto err;
331 }
332
333 larval = crypto_alloc_test_larval(alg);
334 if (IS_ERR(ptr: larval))
335 goto out;
336
337 list_add(new: &alg->cra_list, head: &crypto_alg_list);
338
339 if (larval) {
340 /* No cheating! */
341 alg->cra_flags &= ~CRYPTO_ALG_TESTED;
342
343 list_add(new: &larval->alg.cra_list, head: &crypto_alg_list);
344 } else {
345 alg->cra_flags |= CRYPTO_ALG_TESTED;
346 crypto_alg_finish_registration(alg, algs_to_put);
347 }
348
349out:
350 return larval;
351
352err:
353 larval = ERR_PTR(error: ret);
354 goto out;
355}
356
357void crypto_alg_tested(const char *name, int err)
358{
359 struct crypto_larval *test;
360 struct crypto_alg *alg;
361 struct crypto_alg *q;
362 LIST_HEAD(list);
363
364 down_write(sem: &crypto_alg_sem);
365 list_for_each_entry(q, &crypto_alg_list, cra_list) {
366 if (crypto_is_moribund(alg: q) || !crypto_is_larval(alg: q))
367 continue;
368
369 test = (struct crypto_larval *)q;
370
371 if (!strcmp(q->cra_driver_name, name))
372 goto found;
373 }
374
375 pr_err("alg: Unexpected test result for %s: %d\n", name, err);
376 up_write(sem: &crypto_alg_sem);
377 return;
378
379found:
380 q->cra_flags |= CRYPTO_ALG_DEAD;
381 alg = test->adult;
382
383 if (crypto_is_dead(alg))
384 goto complete;
385
386 if (err == -ECANCELED)
387 alg->cra_flags |= CRYPTO_ALG_FIPS_INTERNAL;
388 else if (err)
389 goto complete;
390 else
391 alg->cra_flags &= ~CRYPTO_ALG_FIPS_INTERNAL;
392
393 alg->cra_flags |= CRYPTO_ALG_TESTED;
394
395 crypto_alg_finish_registration(alg, algs_to_put: &list);
396
397complete:
398 list_del_init(entry: &test->alg.cra_list);
399 complete_all(&test->completion);
400
401 up_write(sem: &crypto_alg_sem);
402
403 crypto_alg_put(alg: &test->alg);
404 crypto_remove_final(list: &list);
405}
406EXPORT_SYMBOL_GPL(crypto_alg_tested);
407
408void crypto_remove_final(struct list_head *list)
409{
410 struct crypto_alg *alg;
411 struct crypto_alg *n;
412
413 list_for_each_entry_safe(alg, n, list, cra_list) {
414 list_del_init(entry: &alg->cra_list);
415 crypto_alg_put(alg);
416 }
417}
418EXPORT_SYMBOL_GPL(crypto_remove_final);
419
420static void crypto_free_alg(struct crypto_alg *alg)
421{
422 unsigned int algsize = alg->cra_type->algsize;
423 u8 *p = (u8 *)alg - algsize;
424
425 crypto_destroy_alg(alg);
426 kfree(objp: p);
427}
428
429int crypto_register_alg(struct crypto_alg *alg)
430{
431 struct crypto_larval *larval;
432 bool test_started = false;
433 LIST_HEAD(algs_to_put);
434 int err;
435
436 alg->cra_flags &= ~CRYPTO_ALG_DEAD;
437 err = crypto_check_alg(alg);
438 if (err)
439 return err;
440
441 if (alg->cra_flags & CRYPTO_ALG_DUP_FIRST &&
442 !WARN_ON_ONCE(alg->cra_destroy)) {
443 unsigned int algsize = alg->cra_type->algsize;
444 u8 *p = (u8 *)alg - algsize;
445
446 p = kmemdup(p, algsize + sizeof(*alg), GFP_KERNEL);
447 if (!p)
448 return -ENOMEM;
449
450 alg = (void *)(p + algsize);
451 alg->cra_destroy = crypto_free_alg;
452 }
453
454 down_write(sem: &crypto_alg_sem);
455 larval = __crypto_register_alg(alg, algs_to_put: &algs_to_put);
456 if (!IS_ERR_OR_NULL(ptr: larval)) {
457 test_started = crypto_boot_test_finished();
458 larval->test_started = test_started;
459 }
460 up_write(sem: &crypto_alg_sem);
461
462 if (IS_ERR(ptr: larval)) {
463 crypto_alg_put(alg);
464 return PTR_ERR(ptr: larval);
465 }
466
467 if (test_started)
468 crypto_schedule_test(larval);
469 else
470 crypto_remove_final(&algs_to_put);
471
472 return 0;
473}
474EXPORT_SYMBOL_GPL(crypto_register_alg);
475
476static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list)
477{
478 if (unlikely(list_empty(&alg->cra_list)))
479 return -ENOENT;
480
481 alg->cra_flags |= CRYPTO_ALG_DEAD;
482
483 list_del_init(entry: &alg->cra_list);
484 crypto_remove_spawns(alg, list, NULL);
485
486 return 0;
487}
488
489void crypto_unregister_alg(struct crypto_alg *alg)
490{
491 int ret;
492 LIST_HEAD(list);
493
494 down_write(sem: &crypto_alg_sem);
495 ret = crypto_remove_alg(alg, list: &list);
496 up_write(sem: &crypto_alg_sem);
497
498 if (WARN(ret, "Algorithm %s is not registered", alg->cra_driver_name))
499 return;
500
501 WARN_ON(!alg->cra_destroy && refcount_read(&alg->cra_refcnt) != 1);
502
503 list_add(new: &alg->cra_list, head: &list);
504 crypto_remove_final(&list);
505}
506EXPORT_SYMBOL_GPL(crypto_unregister_alg);
507
508int crypto_register_algs(struct crypto_alg *algs, int count)
509{
510 int i, ret;
511
512 for (i = 0; i < count; i++) {
513 ret = crypto_register_alg(&algs[i]);
514 if (ret)
515 goto err;
516 }
517
518 return 0;
519
520err:
521 for (--i; i >= 0; --i)
522 crypto_unregister_alg(&algs[i]);
523
524 return ret;
525}
526EXPORT_SYMBOL_GPL(crypto_register_algs);
527
528void crypto_unregister_algs(struct crypto_alg *algs, int count)
529{
530 int i;
531
532 for (i = 0; i < count; i++)
533 crypto_unregister_alg(&algs[i]);
534}
535EXPORT_SYMBOL_GPL(crypto_unregister_algs);
536
537int crypto_register_template(struct crypto_template *tmpl)
538{
539 struct crypto_template *q;
540 int err = -EEXIST;
541
542 INIT_WORK(&tmpl->free_work, crypto_destroy_instance_workfn);
543
544 down_write(sem: &crypto_alg_sem);
545
546 crypto_check_module_sig(mod: tmpl->module);
547
548 list_for_each_entry(q, &crypto_template_list, list) {
549 if (q == tmpl)
550 goto out;
551 }
552
553 list_add(new: &tmpl->list, head: &crypto_template_list);
554 err = 0;
555out:
556 up_write(sem: &crypto_alg_sem);
557 return err;
558}
559EXPORT_SYMBOL_GPL(crypto_register_template);
560
561int crypto_register_templates(struct crypto_template *tmpls, int count)
562{
563 int i, err;
564
565 for (i = 0; i < count; i++) {
566 err = crypto_register_template(&tmpls[i]);
567 if (err)
568 goto out;
569 }
570 return 0;
571
572out:
573 for (--i; i >= 0; --i)
574 crypto_unregister_template(tmpl: &tmpls[i]);
575 return err;
576}
577EXPORT_SYMBOL_GPL(crypto_register_templates);
578
579void crypto_unregister_template(struct crypto_template *tmpl)
580{
581 struct crypto_instance *inst;
582 struct hlist_node *n;
583 struct hlist_head *list;
584 LIST_HEAD(users);
585
586 down_write(sem: &crypto_alg_sem);
587
588 BUG_ON(list_empty(&tmpl->list));
589 list_del_init(entry: &tmpl->list);
590
591 list = &tmpl->instances;
592 hlist_for_each_entry(inst, list, list) {
593 int err = crypto_remove_alg(alg: &inst->alg, list: &users);
594
595 BUG_ON(err);
596 }
597
598 up_write(sem: &crypto_alg_sem);
599
600 hlist_for_each_entry_safe(inst, n, list, list) {
601 BUG_ON(refcount_read(&inst->alg.cra_refcnt) != 1);
602 crypto_free_instance(inst);
603 }
604 crypto_remove_final(&users);
605
606 flush_work(work: &tmpl->free_work);
607}
608EXPORT_SYMBOL_GPL(crypto_unregister_template);
609
610void crypto_unregister_templates(struct crypto_template *tmpls, int count)
611{
612 int i;
613
614 for (i = count - 1; i >= 0; --i)
615 crypto_unregister_template(&tmpls[i]);
616}
617EXPORT_SYMBOL_GPL(crypto_unregister_templates);
618
619static struct crypto_template *__crypto_lookup_template(const char *name)
620{
621 struct crypto_template *q, *tmpl = NULL;
622
623 down_read(sem: &crypto_alg_sem);
624 list_for_each_entry(q, &crypto_template_list, list) {
625 if (strcmp(q->name, name))
626 continue;
627 if (unlikely(!crypto_tmpl_get(q)))
628 continue;
629
630 tmpl = q;
631 break;
632 }
633 up_read(sem: &crypto_alg_sem);
634
635 return tmpl;
636}
637
638struct crypto_template *crypto_lookup_template(const char *name)
639{
640 return try_then_request_module(__crypto_lookup_template(name),
641 "crypto-%s", name);
642}
643EXPORT_SYMBOL_GPL(crypto_lookup_template);
644
645int crypto_register_instance(struct crypto_template *tmpl,
646 struct crypto_instance *inst)
647{
648 struct crypto_larval *larval;
649 struct crypto_spawn *spawn;
650 u32 fips_internal = 0;
651 LIST_HEAD(algs_to_put);
652 int err;
653
654 err = crypto_check_alg(alg: &inst->alg);
655 if (err)
656 return err;
657
658 inst->alg.cra_module = tmpl->module;
659 inst->alg.cra_flags |= CRYPTO_ALG_INSTANCE;
660 inst->alg.cra_destroy = crypto_destroy_instance;
661
662 down_write(sem: &crypto_alg_sem);
663
664 larval = ERR_PTR(error: -EAGAIN);
665 for (spawn = inst->spawns; spawn;) {
666 struct crypto_spawn *next;
667
668 if (spawn->dead)
669 goto unlock;
670
671 next = spawn->next;
672 spawn->inst = inst;
673 spawn->registered = true;
674
675 fips_internal |= spawn->alg->cra_flags;
676
677 crypto_mod_put(alg: spawn->alg);
678
679 spawn = next;
680 }
681
682 inst->alg.cra_flags |= (fips_internal & CRYPTO_ALG_FIPS_INTERNAL);
683
684 larval = __crypto_register_alg(alg: &inst->alg, algs_to_put: &algs_to_put);
685 if (IS_ERR(ptr: larval))
686 goto unlock;
687 else if (larval)
688 larval->test_started = true;
689
690 hlist_add_head(n: &inst->list, h: &tmpl->instances);
691 inst->tmpl = tmpl;
692
693unlock:
694 up_write(sem: &crypto_alg_sem);
695
696 if (IS_ERR(ptr: larval))
697 return PTR_ERR(ptr: larval);
698
699 if (larval)
700 crypto_schedule_test(larval);
701 else
702 crypto_remove_final(&algs_to_put);
703
704 return 0;
705}
706EXPORT_SYMBOL_GPL(crypto_register_instance);
707
708void crypto_unregister_instance(struct crypto_instance *inst)
709{
710 LIST_HEAD(list);
711
712 down_write(sem: &crypto_alg_sem);
713
714 crypto_remove_spawns(&inst->alg, &list, NULL);
715 crypto_remove_instance(inst, list: &list);
716
717 up_write(sem: &crypto_alg_sem);
718
719 crypto_remove_final(&list);
720}
721EXPORT_SYMBOL_GPL(crypto_unregister_instance);
722
723int crypto_grab_spawn(struct crypto_spawn *spawn, struct crypto_instance *inst,
724 const char *name, u32 type, u32 mask)
725{
726 struct crypto_alg *alg;
727 int err = -EAGAIN;
728
729 if (WARN_ON_ONCE(inst == NULL))
730 return -EINVAL;
731
732 /* Allow the result of crypto_attr_alg_name() to be passed directly */
733 if (IS_ERR(ptr: name))
734 return PTR_ERR(ptr: name);
735
736 alg = crypto_find_alg(alg_name: name, frontend: spawn->frontend,
737 type: type | CRYPTO_ALG_FIPS_INTERNAL, mask);
738 if (IS_ERR(ptr: alg))
739 return PTR_ERR(ptr: alg);
740
741 down_write(sem: &crypto_alg_sem);
742 if (!crypto_is_moribund(alg)) {
743 list_add(new: &spawn->list, head: &alg->cra_users);
744 spawn->alg = alg;
745 spawn->mask = mask;
746 spawn->next = inst->spawns;
747 inst->spawns = spawn;
748 inst->alg.cra_flags |=
749 (alg->cra_flags & CRYPTO_ALG_INHERITED_FLAGS);
750 err = 0;
751 }
752 up_write(sem: &crypto_alg_sem);
753 if (err)
754 crypto_mod_put(alg);
755 return err;
756}
757EXPORT_SYMBOL_GPL(crypto_grab_spawn);
758
759void crypto_drop_spawn(struct crypto_spawn *spawn)
760{
761 if (!spawn->alg) /* not yet initialized? */
762 return;
763
764 down_write(sem: &crypto_alg_sem);
765 if (!spawn->dead)
766 list_del(entry: &spawn->list);
767 up_write(sem: &crypto_alg_sem);
768
769 if (!spawn->registered)
770 crypto_mod_put(alg: spawn->alg);
771}
772EXPORT_SYMBOL_GPL(crypto_drop_spawn);
773
774static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn)
775{
776 struct crypto_alg *alg = ERR_PTR(error: -EAGAIN);
777 struct crypto_alg *target;
778 bool shoot = false;
779
780 down_read(sem: &crypto_alg_sem);
781 if (!spawn->dead) {
782 alg = spawn->alg;
783 if (!crypto_mod_get(alg)) {
784 target = crypto_alg_get(alg);
785 shoot = true;
786 alg = ERR_PTR(error: -EAGAIN);
787 }
788 }
789 up_read(sem: &crypto_alg_sem);
790
791 if (shoot) {
792 crypto_shoot_alg(alg: target);
793 crypto_alg_put(alg: target);
794 }
795
796 return alg;
797}
798
799struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type,
800 u32 mask)
801{
802 struct crypto_alg *alg;
803 struct crypto_tfm *tfm;
804
805 alg = crypto_spawn_alg(spawn);
806 if (IS_ERR(ptr: alg))
807 return ERR_CAST(ptr: alg);
808
809 tfm = ERR_PTR(error: -EINVAL);
810 if (unlikely((alg->cra_flags ^ type) & mask))
811 goto out_put_alg;
812
813 tfm = __crypto_alloc_tfm(alg, type, mask);
814 if (IS_ERR(ptr: tfm))
815 goto out_put_alg;
816
817 return tfm;
818
819out_put_alg:
820 crypto_mod_put(alg);
821 return tfm;
822}
823EXPORT_SYMBOL_GPL(crypto_spawn_tfm);
824
825void *crypto_spawn_tfm2(struct crypto_spawn *spawn)
826{
827 struct crypto_alg *alg;
828 struct crypto_tfm *tfm;
829
830 alg = crypto_spawn_alg(spawn);
831 if (IS_ERR(ptr: alg))
832 return ERR_CAST(ptr: alg);
833
834 tfm = crypto_create_tfm(alg, frontend: spawn->frontend);
835 if (IS_ERR(ptr: tfm))
836 goto out_put_alg;
837
838 return tfm;
839
840out_put_alg:
841 crypto_mod_put(alg);
842 return tfm;
843}
844EXPORT_SYMBOL_GPL(crypto_spawn_tfm2);
845
846int crypto_register_notifier(struct notifier_block *nb)
847{
848 return blocking_notifier_chain_register(nh: &crypto_chain, nb);
849}
850EXPORT_SYMBOL_GPL(crypto_register_notifier);
851
852int crypto_unregister_notifier(struct notifier_block *nb)
853{
854 return blocking_notifier_chain_unregister(nh: &crypto_chain, nb);
855}
856EXPORT_SYMBOL_GPL(crypto_unregister_notifier);
857
858struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb)
859{
860 struct rtattr *rta = tb[0];
861 struct crypto_attr_type *algt;
862
863 if (!rta)
864 return ERR_PTR(error: -ENOENT);
865 if (RTA_PAYLOAD(rta) < sizeof(*algt))
866 return ERR_PTR(error: -EINVAL);
867 if (rta->rta_type != CRYPTOA_TYPE)
868 return ERR_PTR(error: -EINVAL);
869
870 algt = RTA_DATA(rta);
871
872 return algt;
873}
874EXPORT_SYMBOL_GPL(crypto_get_attr_type);
875
876/**
877 * crypto_check_attr_type() - check algorithm type and compute inherited mask
878 * @tb: the template parameters
879 * @type: the algorithm type the template would be instantiated as
880 * @mask_ret: (output) the mask that should be passed to crypto_grab_*()
881 * to restrict the flags of any inner algorithms
882 *
883 * Validate that the algorithm type the user requested is compatible with the
884 * one the template would actually be instantiated as. E.g., if the user is
885 * doing crypto_alloc_shash("cbc(aes)", ...), this would return an error because
886 * the "cbc" template creates an "skcipher" algorithm, not an "shash" algorithm.
887 *
888 * Also compute the mask to use to restrict the flags of any inner algorithms.
889 *
890 * Return: 0 on success; -errno on failure
891 */
892int crypto_check_attr_type(struct rtattr **tb, u32 type, u32 *mask_ret)
893{
894 struct crypto_attr_type *algt;
895
896 algt = crypto_get_attr_type(tb);
897 if (IS_ERR(ptr: algt))
898 return PTR_ERR(ptr: algt);
899
900 if ((algt->type ^ type) & algt->mask)
901 return -EINVAL;
902
903 *mask_ret = crypto_algt_inherited_mask(algt);
904 return 0;
905}
906EXPORT_SYMBOL_GPL(crypto_check_attr_type);
907
908const char *crypto_attr_alg_name(struct rtattr *rta)
909{
910 struct crypto_attr_alg *alga;
911
912 if (!rta)
913 return ERR_PTR(error: -ENOENT);
914 if (RTA_PAYLOAD(rta) < sizeof(*alga))
915 return ERR_PTR(error: -EINVAL);
916 if (rta->rta_type != CRYPTOA_ALG)
917 return ERR_PTR(error: -EINVAL);
918
919 alga = RTA_DATA(rta);
920 alga->name[CRYPTO_MAX_ALG_NAME - 1] = 0;
921
922 return alga->name;
923}
924EXPORT_SYMBOL_GPL(crypto_attr_alg_name);
925
926int __crypto_inst_setname(struct crypto_instance *inst, const char *name,
927 const char *driver, struct crypto_alg *alg)
928{
929 if (snprintf(buf: inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, fmt: "%s(%s)", name,
930 alg->cra_name) >= CRYPTO_MAX_ALG_NAME)
931 return -ENAMETOOLONG;
932
933 if (snprintf(buf: inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, fmt: "%s(%s)",
934 driver, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
935 return -ENAMETOOLONG;
936
937 return 0;
938}
939EXPORT_SYMBOL_GPL(__crypto_inst_setname);
940
941void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen)
942{
943 INIT_LIST_HEAD(list: &queue->list);
944 queue->backlog = &queue->list;
945 queue->qlen = 0;
946 queue->max_qlen = max_qlen;
947}
948EXPORT_SYMBOL_GPL(crypto_init_queue);
949
950int crypto_enqueue_request(struct crypto_queue *queue,
951 struct crypto_async_request *request)
952{
953 int err = -EINPROGRESS;
954
955 if (unlikely(queue->qlen >= queue->max_qlen)) {
956 if (!(request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
957 err = -ENOSPC;
958 goto out;
959 }
960 err = -EBUSY;
961 if (queue->backlog == &queue->list)
962 queue->backlog = &request->list;
963 }
964
965 queue->qlen++;
966 list_add_tail(new: &request->list, head: &queue->list);
967
968out:
969 return err;
970}
971EXPORT_SYMBOL_GPL(crypto_enqueue_request);
972
973void crypto_enqueue_request_head(struct crypto_queue *queue,
974 struct crypto_async_request *request)
975{
976 if (unlikely(queue->qlen >= queue->max_qlen))
977 queue->backlog = queue->backlog->prev;
978
979 queue->qlen++;
980 list_add(new: &request->list, head: &queue->list);
981}
982EXPORT_SYMBOL_GPL(crypto_enqueue_request_head);
983
984struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue)
985{
986 struct list_head *request;
987
988 if (unlikely(!queue->qlen))
989 return NULL;
990
991 queue->qlen--;
992
993 if (queue->backlog != &queue->list)
994 queue->backlog = queue->backlog->next;
995
996 request = queue->list.next;
997 list_del_init(entry: request);
998
999 return list_entry(request, struct crypto_async_request, list);
1000}
1001EXPORT_SYMBOL_GPL(crypto_dequeue_request);
1002
1003static inline void crypto_inc_byte(u8 *a, unsigned int size)
1004{
1005 u8 *b = (a + size);
1006 u8 c;
1007
1008 for (; size; size--) {
1009 c = *--b + 1;
1010 *b = c;
1011 if (c)
1012 break;
1013 }
1014}
1015
1016void crypto_inc(u8 *a, unsigned int size)
1017{
1018 __be32 *b = (__be32 *)(a + size);
1019 u32 c;
1020
1021 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) ||
1022 IS_ALIGNED((unsigned long)b, __alignof__(*b)))
1023 for (; size >= 4; size -= 4) {
1024 c = be32_to_cpu(*--b) + 1;
1025 *b = cpu_to_be32(c);
1026 if (likely(c))
1027 return;
1028 }
1029
1030 crypto_inc_byte(a, size);
1031}
1032EXPORT_SYMBOL_GPL(crypto_inc);
1033
1034unsigned int crypto_alg_extsize(struct crypto_alg *alg)
1035{
1036 return alg->cra_ctxsize +
1037 (alg->cra_alignmask & ~(crypto_tfm_ctx_alignment() - 1));
1038}
1039EXPORT_SYMBOL_GPL(crypto_alg_extsize);
1040
1041int crypto_type_has_alg(const char *name, const struct crypto_type *frontend,
1042 u32 type, u32 mask)
1043{
1044 int ret = 0;
1045 struct crypto_alg *alg = crypto_find_alg(alg_name: name, frontend, type, mask);
1046
1047 if (!IS_ERR(ptr: alg)) {
1048 crypto_mod_put(alg);
1049 ret = 1;
1050 }
1051
1052 return ret;
1053}
1054EXPORT_SYMBOL_GPL(crypto_type_has_alg);
1055
1056static void __init crypto_start_tests(void)
1057{
1058 if (!IS_BUILTIN(CONFIG_CRYPTO_ALGAPI))
1059 return;
1060
1061 if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS))
1062 return;
1063
1064 set_crypto_boot_test_finished();
1065
1066 for (;;) {
1067 struct crypto_larval *larval = NULL;
1068 struct crypto_alg *q;
1069
1070 down_write(sem: &crypto_alg_sem);
1071
1072 list_for_each_entry(q, &crypto_alg_list, cra_list) {
1073 struct crypto_larval *l;
1074
1075 if (!crypto_is_larval(alg: q))
1076 continue;
1077
1078 l = (void *)q;
1079
1080 if (!crypto_is_test_larval(larval: l))
1081 continue;
1082
1083 if (l->test_started)
1084 continue;
1085
1086 l->test_started = true;
1087 larval = l;
1088 break;
1089 }
1090
1091 up_write(sem: &crypto_alg_sem);
1092
1093 if (!larval)
1094 break;
1095
1096 crypto_schedule_test(larval);
1097 }
1098}
1099
1100static int __init crypto_algapi_init(void)
1101{
1102 crypto_init_proc();
1103 crypto_start_tests();
1104 return 0;
1105}
1106
1107static void __exit crypto_algapi_exit(void)
1108{
1109 crypto_exit_proc();
1110}
1111
1112/*
1113 * We run this at late_initcall so that all the built-in algorithms
1114 * have had a chance to register themselves first.
1115 */
1116late_initcall(crypto_algapi_init);
1117module_exit(crypto_algapi_exit);
1118
1119MODULE_LICENSE("GPL");
1120MODULE_DESCRIPTION("Cryptographic algorithms API");
1121MODULE_SOFTDEP("pre: cryptomgr");
1122