1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Synchronous Compression operations
4 *
5 * Copyright 2015 LG Electronics Inc.
6 * Copyright (c) 2016, Intel Corporation
7 * Author: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
8 */
9
10#include <crypto/internal/scompress.h>
11#include <crypto/scatterwalk.h>
12#include <linux/cpumask.h>
13#include <linux/cryptouser.h>
14#include <linux/err.h>
15#include <linux/highmem.h>
16#include <linux/kernel.h>
17#include <linux/module.h>
18#include <linux/overflow.h>
19#include <linux/scatterlist.h>
20#include <linux/seq_file.h>
21#include <linux/slab.h>
22#include <linux/string.h>
23#include <linux/workqueue.h>
24#include <net/netlink.h>
25
26#include "compress.h"
27
28struct scomp_scratch {
29 spinlock_t lock;
30 union {
31 void *src;
32 unsigned long saddr;
33 };
34};
35
36static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = {
37 .lock = __SPIN_LOCK_UNLOCKED(scomp_scratch.lock),
38};
39
40static const struct crypto_type crypto_scomp_type;
41static int scomp_scratch_users;
42static DEFINE_MUTEX(scomp_lock);
43
44static cpumask_t scomp_scratch_want;
45static void scomp_scratch_workfn(struct work_struct *work);
46static DECLARE_WORK(scomp_scratch_work, scomp_scratch_workfn);
47
48static int __maybe_unused crypto_scomp_report(
49 struct sk_buff *skb, struct crypto_alg *alg)
50{
51 struct crypto_report_comp rscomp;
52
53 memset(s: &rscomp, c: 0, n: sizeof(rscomp));
54
55 strscpy(rscomp.type, "scomp", sizeof(rscomp.type));
56
57 return nla_put(skb, attrtype: CRYPTOCFGA_REPORT_COMPRESS,
58 attrlen: sizeof(rscomp), data: &rscomp);
59}
60
61static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
62 __maybe_unused;
63
64static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
65{
66 seq_puts(m, s: "type : scomp\n");
67}
68
69static void crypto_scomp_free_scratches(void)
70{
71 struct scomp_scratch *scratch;
72 int i;
73
74 for_each_possible_cpu(i) {
75 scratch = per_cpu_ptr(&scomp_scratch, i);
76
77 free_page(scratch->saddr);
78 scratch->src = NULL;
79 }
80}
81
82static int scomp_alloc_scratch(struct scomp_scratch *scratch, int cpu)
83{
84 int node = cpu_to_node(cpu);
85 struct page *page;
86
87 page = alloc_pages_node(node, GFP_KERNEL, 0);
88 if (!page)
89 return -ENOMEM;
90 spin_lock_bh(lock: &scratch->lock);
91 scratch->src = page_address(page);
92 spin_unlock_bh(lock: &scratch->lock);
93 return 0;
94}
95
96static void scomp_scratch_workfn(struct work_struct *work)
97{
98 int cpu;
99
100 for_each_cpu(cpu, &scomp_scratch_want) {
101 struct scomp_scratch *scratch;
102
103 scratch = per_cpu_ptr(&scomp_scratch, cpu);
104 if (scratch->src)
105 continue;
106 if (scomp_alloc_scratch(scratch, cpu))
107 break;
108
109 cpumask_clear_cpu(cpu, dstp: &scomp_scratch_want);
110 }
111}
112
113static int crypto_scomp_alloc_scratches(void)
114{
115 unsigned int i = cpumask_first(cpu_possible_mask);
116 struct scomp_scratch *scratch;
117
118 scratch = per_cpu_ptr(&scomp_scratch, i);
119 return scomp_alloc_scratch(scratch, cpu: i);
120}
121
122static int crypto_scomp_init_tfm(struct crypto_tfm *tfm)
123{
124 struct scomp_alg *alg = crypto_scomp_alg(tfm: __crypto_scomp_tfm(tfm));
125 int ret = 0;
126
127 mutex_lock(lock: &scomp_lock);
128 ret = crypto_acomp_alloc_streams(s: &alg->streams);
129 if (ret)
130 goto unlock;
131 if (!scomp_scratch_users++) {
132 ret = crypto_scomp_alloc_scratches();
133 if (ret)
134 scomp_scratch_users--;
135 }
136unlock:
137 mutex_unlock(lock: &scomp_lock);
138
139 return ret;
140}
141
142static struct scomp_scratch *scomp_lock_scratch(void) __acquires(scratch)
143{
144 int cpu = raw_smp_processor_id();
145 struct scomp_scratch *scratch;
146
147 scratch = per_cpu_ptr(&scomp_scratch, cpu);
148 spin_lock(lock: &scratch->lock);
149 if (likely(scratch->src))
150 return scratch;
151 spin_unlock(lock: &scratch->lock);
152
153 cpumask_set_cpu(cpu, dstp: &scomp_scratch_want);
154 schedule_work(work: &scomp_scratch_work);
155
156 scratch = per_cpu_ptr(&scomp_scratch, cpumask_first(cpu_possible_mask));
157 spin_lock(lock: &scratch->lock);
158 return scratch;
159}
160
161static inline void scomp_unlock_scratch(struct scomp_scratch *scratch)
162 __releases(scratch)
163{
164 spin_unlock(lock: &scratch->lock);
165}
166
167static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
168{
169 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
170 struct crypto_scomp **tfm_ctx = acomp_tfm_ctx(tfm);
171 bool src_isvirt = acomp_request_src_isvirt(req);
172 bool dst_isvirt = acomp_request_dst_isvirt(req);
173 struct crypto_scomp *scomp = *tfm_ctx;
174 struct crypto_acomp_stream *stream;
175 struct scomp_scratch *scratch;
176 unsigned int slen = req->slen;
177 unsigned int dlen = req->dlen;
178 struct page *spage, *dpage;
179 unsigned int n;
180 const u8 *src;
181 size_t soff;
182 size_t doff;
183 u8 *dst;
184 int ret;
185
186 if (!req->src || !slen)
187 return -EINVAL;
188
189 if (!req->dst || !dlen)
190 return -EINVAL;
191
192 if (dst_isvirt)
193 dst = req->dvirt;
194 else {
195 if (dlen <= req->dst->length) {
196 dpage = sg_page(sg: req->dst);
197 doff = req->dst->offset;
198 } else
199 return -ENOSYS;
200
201 dpage += doff / PAGE_SIZE;
202 doff = offset_in_page(doff);
203
204 n = (dlen - 1) / PAGE_SIZE;
205 n += (offset_in_page(dlen - 1) + doff) / PAGE_SIZE;
206 if (PageHighMem(page: dpage + n) &&
207 size_add(addend1: doff, addend2: dlen) > PAGE_SIZE)
208 return -ENOSYS;
209 dst = kmap_local_page(page: dpage) + doff;
210 }
211
212 if (src_isvirt)
213 src = req->svirt;
214 else {
215 src = NULL;
216 do {
217 if (slen <= req->src->length) {
218 spage = sg_page(sg: req->src);
219 soff = req->src->offset;
220 } else
221 break;
222
223 spage = spage + soff / PAGE_SIZE;
224 soff = offset_in_page(soff);
225
226 n = (slen - 1) / PAGE_SIZE;
227 n += (offset_in_page(slen - 1) + soff) / PAGE_SIZE;
228 if (PageHighMem(page: spage + n) &&
229 size_add(addend1: soff, addend2: slen) > PAGE_SIZE)
230 break;
231 src = kmap_local_page(page: spage) + soff;
232 } while (0);
233 }
234
235 stream = crypto_acomp_lock_stream_bh(s: &crypto_scomp_alg(tfm: scomp)->streams);
236
237 if (!src_isvirt && !src) {
238 const u8 *src;
239
240 scratch = scomp_lock_scratch();
241 src = scratch->src;
242 memcpy_from_sglist(buf: scratch->src, sg: req->src, start: 0, nbytes: slen);
243
244 if (dir)
245 ret = crypto_scomp_compress(tfm: scomp, src, slen,
246 dst, dlen: &dlen, ctx: stream->ctx);
247 else
248 ret = crypto_scomp_decompress(tfm: scomp, src, slen,
249 dst, dlen: &dlen, ctx: stream->ctx);
250
251 scomp_unlock_scratch(scratch);
252 } else if (dir)
253 ret = crypto_scomp_compress(tfm: scomp, src, slen,
254 dst, dlen: &dlen, ctx: stream->ctx);
255 else
256 ret = crypto_scomp_decompress(tfm: scomp, src, slen,
257 dst, dlen: &dlen, ctx: stream->ctx);
258
259 crypto_acomp_unlock_stream_bh(stream);
260
261 req->dlen = dlen;
262
263 if (!src_isvirt && src)
264 kunmap_local(src);
265 if (!dst_isvirt) {
266 kunmap_local(dst);
267 dlen += doff;
268 for (;;) {
269 flush_dcache_page(page: dpage);
270 if (dlen <= PAGE_SIZE)
271 break;
272 dlen -= PAGE_SIZE;
273 dpage++;
274 }
275 }
276
277 return ret;
278}
279
280static int scomp_acomp_compress(struct acomp_req *req)
281{
282 return scomp_acomp_comp_decomp(req, dir: 1);
283}
284
285static int scomp_acomp_decompress(struct acomp_req *req)
286{
287 return scomp_acomp_comp_decomp(req, dir: 0);
288}
289
290static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm)
291{
292 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
293
294 crypto_free_scomp(tfm: *ctx);
295
296 flush_work(work: &scomp_scratch_work);
297 mutex_lock(lock: &scomp_lock);
298 if (!--scomp_scratch_users)
299 crypto_scomp_free_scratches();
300 mutex_unlock(lock: &scomp_lock);
301}
302
303int crypto_init_scomp_ops_async(struct crypto_tfm *tfm)
304{
305 struct crypto_alg *calg = tfm->__crt_alg;
306 struct crypto_acomp *crt = __crypto_acomp_tfm(tfm);
307 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
308 struct crypto_scomp *scomp;
309
310 if (!crypto_mod_get(alg: calg))
311 return -EAGAIN;
312
313 scomp = crypto_create_tfm(alg: calg, frontend: &crypto_scomp_type);
314 if (IS_ERR(ptr: scomp)) {
315 crypto_mod_put(alg: calg);
316 return PTR_ERR(ptr: scomp);
317 }
318
319 *ctx = scomp;
320 tfm->exit = crypto_exit_scomp_ops_async;
321
322 crt->compress = scomp_acomp_compress;
323 crt->decompress = scomp_acomp_decompress;
324
325 return 0;
326}
327
328static void crypto_scomp_destroy(struct crypto_alg *alg)
329{
330 struct scomp_alg *scomp = __crypto_scomp_alg(alg);
331
332 crypto_acomp_free_streams(s: &scomp->streams);
333}
334
335static const struct crypto_type crypto_scomp_type = {
336 .extsize = crypto_alg_extsize,
337 .init_tfm = crypto_scomp_init_tfm,
338 .destroy = crypto_scomp_destroy,
339#ifdef CONFIG_PROC_FS
340 .show = crypto_scomp_show,
341#endif
342#if IS_ENABLED(CONFIG_CRYPTO_USER)
343 .report = crypto_scomp_report,
344#endif
345 .maskclear = ~CRYPTO_ALG_TYPE_MASK,
346 .maskset = CRYPTO_ALG_TYPE_MASK,
347 .type = CRYPTO_ALG_TYPE_SCOMPRESS,
348 .tfmsize = offsetof(struct crypto_scomp, base),
349 .algsize = offsetof(struct scomp_alg, base),
350};
351
352static void scomp_prepare_alg(struct scomp_alg *alg)
353{
354 struct crypto_alg *base = &alg->calg.base;
355
356 comp_prepare_alg(alg: &alg->calg);
357
358 base->cra_flags |= CRYPTO_ALG_REQ_VIRT;
359}
360
361int crypto_register_scomp(struct scomp_alg *alg)
362{
363 struct crypto_alg *base = &alg->calg.base;
364
365 scomp_prepare_alg(alg);
366
367 base->cra_type = &crypto_scomp_type;
368 base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS;
369
370 return crypto_register_alg(alg: base);
371}
372EXPORT_SYMBOL_GPL(crypto_register_scomp);
373
374void crypto_unregister_scomp(struct scomp_alg *alg)
375{
376 crypto_unregister_alg(alg: &alg->base);
377}
378EXPORT_SYMBOL_GPL(crypto_unregister_scomp);
379
380int crypto_register_scomps(struct scomp_alg *algs, int count)
381{
382 int i, ret;
383
384 for (i = 0; i < count; i++) {
385 ret = crypto_register_scomp(&algs[i]);
386 if (ret)
387 goto err;
388 }
389
390 return 0;
391
392err:
393 for (--i; i >= 0; --i)
394 crypto_unregister_scomp(&algs[i]);
395
396 return ret;
397}
398EXPORT_SYMBOL_GPL(crypto_register_scomps);
399
400void crypto_unregister_scomps(struct scomp_alg *algs, int count)
401{
402 int i;
403
404 for (i = count - 1; i >= 0; --i)
405 crypto_unregister_scomp(&algs[i]);
406}
407EXPORT_SYMBOL_GPL(crypto_unregister_scomps);
408
409MODULE_LICENSE("GPL");
410MODULE_DESCRIPTION("Synchronous compression type");
411