1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Asynchronous Compression operations
4 *
5 * Copyright (c) 2016, Intel Corporation
6 * Authors: Weigang Li <weigang.li@intel.com>
7 * Giovanni Cabiddu <giovanni.cabiddu@intel.com>
8 */
9
10#include <crypto/internal/acompress.h>
11#include <crypto/scatterwalk.h>
12#include <linux/cryptouser.h>
13#include <linux/cpumask.h>
14#include <linux/err.h>
15#include <linux/kernel.h>
16#include <linux/module.h>
17#include <linux/percpu.h>
18#include <linux/scatterlist.h>
19#include <linux/sched.h>
20#include <linux/seq_file.h>
21#include <linux/smp.h>
22#include <linux/spinlock.h>
23#include <linux/string.h>
24#include <linux/workqueue.h>
25#include <net/netlink.h>
26
27#include "compress.h"
28
29struct crypto_scomp;
30
31enum {
32 ACOMP_WALK_SLEEP = 1 << 0,
33 ACOMP_WALK_SRC_LINEAR = 1 << 1,
34 ACOMP_WALK_DST_LINEAR = 1 << 2,
35};
36
37static const struct crypto_type crypto_acomp_type;
38
39static void acomp_reqchain_done(void *data, int err);
40
41static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg)
42{
43 return container_of(alg, struct acomp_alg, calg.base);
44}
45
46static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm)
47{
48 return __crypto_acomp_alg(alg: crypto_acomp_tfm(tfm)->__crt_alg);
49}
50
51static int __maybe_unused crypto_acomp_report(
52 struct sk_buff *skb, struct crypto_alg *alg)
53{
54 struct crypto_report_acomp racomp;
55
56 memset(s: &racomp, c: 0, n: sizeof(racomp));
57
58 strscpy(racomp.type, "acomp", sizeof(racomp.type));
59
60 return nla_put(skb, attrtype: CRYPTOCFGA_REPORT_ACOMP, attrlen: sizeof(racomp), data: &racomp);
61}
62
63static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
64 __maybe_unused;
65
66static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg)
67{
68 seq_puts(m, s: "type : acomp\n");
69}
70
71static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm)
72{
73 struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
74 struct acomp_alg *alg = crypto_acomp_alg(tfm: acomp);
75
76 if (alg->exit)
77 alg->exit(acomp);
78
79 if (acomp_is_async(tfm: acomp))
80 crypto_free_acomp(tfm: crypto_acomp_fb(tfm: acomp));
81}
82
83static int crypto_acomp_init_tfm(struct crypto_tfm *tfm)
84{
85 struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
86 struct acomp_alg *alg = crypto_acomp_alg(tfm: acomp);
87 struct crypto_acomp *fb = NULL;
88 int err;
89
90 if (tfm->__crt_alg->cra_type != &crypto_acomp_type)
91 return crypto_init_scomp_ops_async(tfm);
92
93 if (acomp_is_async(tfm: acomp)) {
94 fb = crypto_alloc_acomp(alg_name: crypto_acomp_alg_name(tfm: acomp), type: 0,
95 CRYPTO_ALG_ASYNC);
96 if (IS_ERR(ptr: fb))
97 return PTR_ERR(ptr: fb);
98
99 err = -EINVAL;
100 if (crypto_acomp_reqsize(tfm: fb) > MAX_SYNC_COMP_REQSIZE)
101 goto out_free_fb;
102
103 tfm->fb = crypto_acomp_tfm(tfm: fb);
104 }
105
106 acomp->compress = alg->compress;
107 acomp->decompress = alg->decompress;
108 acomp->reqsize = alg->base.cra_reqsize;
109
110 acomp->base.exit = crypto_acomp_exit_tfm;
111
112 if (!alg->init)
113 return 0;
114
115 err = alg->init(acomp);
116 if (err)
117 goto out_free_fb;
118
119 return 0;
120
121out_free_fb:
122 crypto_free_acomp(tfm: fb);
123 return err;
124}
125
126static unsigned int crypto_acomp_extsize(struct crypto_alg *alg)
127{
128 int extsize = crypto_alg_extsize(alg);
129
130 if (alg->cra_type != &crypto_acomp_type)
131 extsize += sizeof(struct crypto_scomp *);
132
133 return extsize;
134}
135
136static const struct crypto_type crypto_acomp_type = {
137 .extsize = crypto_acomp_extsize,
138 .init_tfm = crypto_acomp_init_tfm,
139#ifdef CONFIG_PROC_FS
140 .show = crypto_acomp_show,
141#endif
142#if IS_ENABLED(CONFIG_CRYPTO_USER)
143 .report = crypto_acomp_report,
144#endif
145 .maskclear = ~CRYPTO_ALG_TYPE_MASK,
146 .maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK,
147 .type = CRYPTO_ALG_TYPE_ACOMPRESS,
148 .tfmsize = offsetof(struct crypto_acomp, base),
149 .algsize = offsetof(struct acomp_alg, base),
150};
151
152struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type,
153 u32 mask)
154{
155 return crypto_alloc_tfm(alg_name, frontend: &crypto_acomp_type, type, mask);
156}
157EXPORT_SYMBOL_GPL(crypto_alloc_acomp);
158
159struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type,
160 u32 mask, int node)
161{
162 return crypto_alloc_tfm_node(alg_name, frontend: &crypto_acomp_type, type, mask,
163 node);
164}
165EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node);
166
167static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt)
168{
169 struct acomp_req_chain *state = &req->chain;
170
171 state->compl = req->base.complete;
172 state->data = req->base.data;
173 req->base.complete = cplt;
174 req->base.data = state;
175}
176
177static void acomp_restore_req(struct acomp_req *req)
178{
179 struct acomp_req_chain *state = req->base.data;
180
181 req->base.complete = state->compl;
182 req->base.data = state->data;
183}
184
185static void acomp_reqchain_virt(struct acomp_req *req)
186{
187 struct acomp_req_chain *state = &req->chain;
188 unsigned int slen = req->slen;
189 unsigned int dlen = req->dlen;
190
191 if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT)
192 acomp_request_set_src_dma(req, src: state->src, slen);
193 if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT)
194 acomp_request_set_dst_dma(req, dst: state->dst, dlen);
195}
196
197static void acomp_virt_to_sg(struct acomp_req *req)
198{
199 struct acomp_req_chain *state = &req->chain;
200
201 state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT |
202 CRYPTO_ACOMP_REQ_DST_VIRT);
203
204 if (acomp_request_src_isvirt(req)) {
205 unsigned int slen = req->slen;
206 const u8 *svirt = req->svirt;
207
208 state->src = svirt;
209 sg_init_one(&state->ssg, svirt, slen);
210 acomp_request_set_src_sg(req, src: &state->ssg, slen);
211 }
212
213 if (acomp_request_dst_isvirt(req)) {
214 unsigned int dlen = req->dlen;
215 u8 *dvirt = req->dvirt;
216
217 state->dst = dvirt;
218 sg_init_one(&state->dsg, dvirt, dlen);
219 acomp_request_set_dst_sg(req, dst: &state->dsg, dlen);
220 }
221}
222
223static int acomp_do_nondma(struct acomp_req *req, bool comp)
224{
225 ACOMP_FBREQ_ON_STACK(fbreq, req);
226 int err;
227
228 if (comp)
229 err = crypto_acomp_compress(req: fbreq);
230 else
231 err = crypto_acomp_decompress(req: fbreq);
232
233 req->dlen = fbreq->dlen;
234 return err;
235}
236
237static int acomp_do_one_req(struct acomp_req *req, bool comp)
238{
239 if (acomp_request_isnondma(req))
240 return acomp_do_nondma(req, comp);
241
242 acomp_virt_to_sg(req);
243 return comp ? crypto_acomp_reqtfm(req)->compress(req) :
244 crypto_acomp_reqtfm(req)->decompress(req);
245}
246
247static int acomp_reqchain_finish(struct acomp_req *req, int err)
248{
249 acomp_reqchain_virt(req);
250 acomp_restore_req(req);
251 return err;
252}
253
254static void acomp_reqchain_done(void *data, int err)
255{
256 struct acomp_req *req = data;
257 crypto_completion_t compl;
258
259 compl = req->chain.compl;
260 data = req->chain.data;
261
262 if (err == -EINPROGRESS)
263 goto notify;
264
265 err = acomp_reqchain_finish(req, err);
266
267notify:
268 compl(data, err);
269}
270
271static int acomp_do_req_chain(struct acomp_req *req, bool comp)
272{
273 int err;
274
275 acomp_save_req(req, cplt: acomp_reqchain_done);
276
277 err = acomp_do_one_req(req, comp);
278 if (err == -EBUSY || err == -EINPROGRESS)
279 return err;
280
281 return acomp_reqchain_finish(req, err);
282}
283
284int crypto_acomp_compress(struct acomp_req *req)
285{
286 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
287
288 if (acomp_req_on_stack(req) && acomp_is_async(tfm))
289 return -EAGAIN;
290 if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
291 return crypto_acomp_reqtfm(req)->compress(req);
292 return acomp_do_req_chain(req, comp: true);
293}
294EXPORT_SYMBOL_GPL(crypto_acomp_compress);
295
296int crypto_acomp_decompress(struct acomp_req *req)
297{
298 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
299
300 if (acomp_req_on_stack(req) && acomp_is_async(tfm))
301 return -EAGAIN;
302 if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
303 return crypto_acomp_reqtfm(req)->decompress(req);
304 return acomp_do_req_chain(req, comp: false);
305}
306EXPORT_SYMBOL_GPL(crypto_acomp_decompress);
307
308void comp_prepare_alg(struct comp_alg_common *alg)
309{
310 struct crypto_alg *base = &alg->base;
311
312 base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
313}
314
315int crypto_register_acomp(struct acomp_alg *alg)
316{
317 struct crypto_alg *base = &alg->calg.base;
318
319 comp_prepare_alg(alg: &alg->calg);
320
321 base->cra_type = &crypto_acomp_type;
322 base->cra_flags |= CRYPTO_ALG_TYPE_ACOMPRESS;
323
324 return crypto_register_alg(alg: base);
325}
326EXPORT_SYMBOL_GPL(crypto_register_acomp);
327
328void crypto_unregister_acomp(struct acomp_alg *alg)
329{
330 crypto_unregister_alg(alg: &alg->base);
331}
332EXPORT_SYMBOL_GPL(crypto_unregister_acomp);
333
334int crypto_register_acomps(struct acomp_alg *algs, int count)
335{
336 int i, ret;
337
338 for (i = 0; i < count; i++) {
339 ret = crypto_register_acomp(&algs[i]);
340 if (ret)
341 goto err;
342 }
343
344 return 0;
345
346err:
347 for (--i; i >= 0; --i)
348 crypto_unregister_acomp(&algs[i]);
349
350 return ret;
351}
352EXPORT_SYMBOL_GPL(crypto_register_acomps);
353
354void crypto_unregister_acomps(struct acomp_alg *algs, int count)
355{
356 int i;
357
358 for (i = count - 1; i >= 0; --i)
359 crypto_unregister_acomp(&algs[i]);
360}
361EXPORT_SYMBOL_GPL(crypto_unregister_acomps);
362
363static void acomp_stream_workfn(struct work_struct *work)
364{
365 struct crypto_acomp_streams *s =
366 container_of(work, struct crypto_acomp_streams, stream_work);
367 struct crypto_acomp_stream __percpu *streams = s->streams;
368 int cpu;
369
370 for_each_cpu(cpu, &s->stream_want) {
371 struct crypto_acomp_stream *ps;
372 void *ctx;
373
374 ps = per_cpu_ptr(streams, cpu);
375 if (ps->ctx)
376 continue;
377
378 ctx = s->alloc_ctx();
379 if (IS_ERR(ptr: ctx))
380 break;
381
382 spin_lock_bh(lock: &ps->lock);
383 ps->ctx = ctx;
384 spin_unlock_bh(lock: &ps->lock);
385
386 cpumask_clear_cpu(cpu, dstp: &s->stream_want);
387 }
388}
389
390void crypto_acomp_free_streams(struct crypto_acomp_streams *s)
391{
392 struct crypto_acomp_stream __percpu *streams = s->streams;
393 void (*free_ctx)(void *);
394 int i;
395
396 s->streams = NULL;
397 if (!streams)
398 return;
399
400 cancel_work_sync(work: &s->stream_work);
401 free_ctx = s->free_ctx;
402
403 for_each_possible_cpu(i) {
404 struct crypto_acomp_stream *ps = per_cpu_ptr(streams, i);
405
406 if (!ps->ctx)
407 continue;
408
409 free_ctx(ps->ctx);
410 }
411
412 free_percpu(pdata: streams);
413}
414EXPORT_SYMBOL_GPL(crypto_acomp_free_streams);
415
416int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s)
417{
418 struct crypto_acomp_stream __percpu *streams;
419 struct crypto_acomp_stream *ps;
420 unsigned int i;
421 void *ctx;
422
423 if (s->streams)
424 return 0;
425
426 streams = alloc_percpu(struct crypto_acomp_stream);
427 if (!streams)
428 return -ENOMEM;
429
430 ctx = s->alloc_ctx();
431 if (IS_ERR(ptr: ctx)) {
432 free_percpu(pdata: streams);
433 return PTR_ERR(ptr: ctx);
434 }
435
436 i = cpumask_first(cpu_possible_mask);
437 ps = per_cpu_ptr(streams, i);
438 ps->ctx = ctx;
439
440 for_each_possible_cpu(i) {
441 ps = per_cpu_ptr(streams, i);
442 spin_lock_init(&ps->lock);
443 }
444
445 s->streams = streams;
446
447 INIT_WORK(&s->stream_work, acomp_stream_workfn);
448 return 0;
449}
450EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams);
451
452struct crypto_acomp_stream *crypto_acomp_lock_stream_bh(
453 struct crypto_acomp_streams *s) __acquires(stream)
454{
455 struct crypto_acomp_stream __percpu *streams = s->streams;
456 int cpu = raw_smp_processor_id();
457 struct crypto_acomp_stream *ps;
458
459 ps = per_cpu_ptr(streams, cpu);
460 spin_lock_bh(lock: &ps->lock);
461 if (likely(ps->ctx))
462 return ps;
463 spin_unlock(lock: &ps->lock);
464
465 cpumask_set_cpu(cpu, dstp: &s->stream_want);
466 schedule_work(work: &s->stream_work);
467
468 ps = per_cpu_ptr(streams, cpumask_first(cpu_possible_mask));
469 spin_lock(lock: &ps->lock);
470 return ps;
471}
472EXPORT_SYMBOL_GPL(crypto_acomp_lock_stream_bh);
473
474void acomp_walk_done_src(struct acomp_walk *walk, int used)
475{
476 walk->slen -= used;
477 if ((walk->flags & ACOMP_WALK_SRC_LINEAR))
478 scatterwalk_advance(walk: &walk->in, nbytes: used);
479 else
480 scatterwalk_done_src(walk: &walk->in, nbytes: used);
481
482 if ((walk->flags & ACOMP_WALK_SLEEP))
483 cond_resched();
484}
485EXPORT_SYMBOL_GPL(acomp_walk_done_src);
486
487void acomp_walk_done_dst(struct acomp_walk *walk, int used)
488{
489 walk->dlen -= used;
490 if ((walk->flags & ACOMP_WALK_DST_LINEAR))
491 scatterwalk_advance(walk: &walk->out, nbytes: used);
492 else
493 scatterwalk_done_dst(walk: &walk->out, nbytes: used);
494
495 if ((walk->flags & ACOMP_WALK_SLEEP))
496 cond_resched();
497}
498EXPORT_SYMBOL_GPL(acomp_walk_done_dst);
499
500int acomp_walk_next_src(struct acomp_walk *walk)
501{
502 unsigned int slen = walk->slen;
503 unsigned int max = UINT_MAX;
504
505 if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
506 max = PAGE_SIZE;
507 if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
508 walk->in.__addr = (void *)(((u8 *)walk->in.sg) +
509 walk->in.offset);
510 return min(slen, max);
511 }
512
513 return slen ? scatterwalk_next(walk: &walk->in, total: slen) : 0;
514}
515EXPORT_SYMBOL_GPL(acomp_walk_next_src);
516
517int acomp_walk_next_dst(struct acomp_walk *walk)
518{
519 unsigned int dlen = walk->dlen;
520 unsigned int max = UINT_MAX;
521
522 if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
523 max = PAGE_SIZE;
524 if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
525 walk->out.__addr = (void *)(((u8 *)walk->out.sg) +
526 walk->out.offset);
527 return min(dlen, max);
528 }
529
530 return dlen ? scatterwalk_next(walk: &walk->out, total: dlen) : 0;
531}
532EXPORT_SYMBOL_GPL(acomp_walk_next_dst);
533
534int acomp_walk_virt(struct acomp_walk *__restrict walk,
535 struct acomp_req *__restrict req, bool atomic)
536{
537 struct scatterlist *src = req->src;
538 struct scatterlist *dst = req->dst;
539
540 walk->slen = req->slen;
541 walk->dlen = req->dlen;
542
543 if (!walk->slen || !walk->dlen)
544 return -EINVAL;
545
546 walk->flags = 0;
547 if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic)
548 walk->flags |= ACOMP_WALK_SLEEP;
549 if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT))
550 walk->flags |= ACOMP_WALK_SRC_LINEAR;
551 if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT))
552 walk->flags |= ACOMP_WALK_DST_LINEAR;
553
554 if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
555 walk->in.sg = (void *)req->svirt;
556 walk->in.offset = 0;
557 } else
558 scatterwalk_start(walk: &walk->in, sg: src);
559 if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
560 walk->out.sg = (void *)req->dvirt;
561 walk->out.offset = 0;
562 } else
563 scatterwalk_start(walk: &walk->out, sg: dst);
564
565 return 0;
566}
567EXPORT_SYMBOL_GPL(acomp_walk_virt);
568
569struct acomp_req *acomp_request_clone(struct acomp_req *req,
570 size_t total, gfp_t gfp)
571{
572 struct acomp_req *nreq;
573
574 nreq = container_of(crypto_request_clone(&req->base, total, gfp),
575 struct acomp_req, base);
576 if (nreq == req)
577 return req;
578
579 if (req->src == &req->chain.ssg)
580 nreq->src = &nreq->chain.ssg;
581 if (req->dst == &req->chain.dsg)
582 nreq->dst = &nreq->chain.dsg;
583 return nreq;
584}
585EXPORT_SYMBOL_GPL(acomp_request_clone);
586
587MODULE_LICENSE("GPL");
588MODULE_DESCRIPTION("Asynchronous compression type");
589