1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library functions
4 *
5 * Copyright (c) Jean-Luc Cooke <jlcooke@certainkey.com>
6 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
7 * Copyright (c) 2003 Kyle McMartin <kyle@debian.org>
8 * Copyright 2025 Google LLC
9 */
10
11#include <crypto/hmac.h>
12#include <crypto/sha2.h>
13#include <linux/export.h>
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/overflow.h>
17#include <linux/string.h>
18#include <linux/unaligned.h>
19#include <linux/wordpart.h>
20
21static const struct sha512_block_state sha384_iv = {
22 .h = {
23 SHA384_H0, SHA384_H1, SHA384_H2, SHA384_H3,
24 SHA384_H4, SHA384_H5, SHA384_H6, SHA384_H7,
25 },
26};
27
28static const struct sha512_block_state sha512_iv = {
29 .h = {
30 SHA512_H0, SHA512_H1, SHA512_H2, SHA512_H3,
31 SHA512_H4, SHA512_H5, SHA512_H6, SHA512_H7,
32 },
33};
34
35static const u64 sha512_K[80] = {
36 0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, 0xb5c0fbcfec4d3b2fULL,
37 0xe9b5dba58189dbbcULL, 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL,
38 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, 0xd807aa98a3030242ULL,
39 0x12835b0145706fbeULL, 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL,
40 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, 0x9bdc06a725c71235ULL,
41 0xc19bf174cf692694ULL, 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL,
42 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, 0x2de92c6f592b0275ULL,
43 0x4a7484aa6ea6e483ULL, 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL,
44 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, 0xb00327c898fb213fULL,
45 0xbf597fc7beef0ee4ULL, 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL,
46 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, 0x27b70a8546d22ffcULL,
47 0x2e1b21385c26c926ULL, 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL,
48 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, 0x81c2c92e47edaee6ULL,
49 0x92722c851482353bULL, 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL,
50 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, 0xd192e819d6ef5218ULL,
51 0xd69906245565a910ULL, 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL,
52 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, 0x2748774cdf8eeb99ULL,
53 0x34b0bcb5e19b48a8ULL, 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL,
54 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, 0x748f82ee5defb2fcULL,
55 0x78a5636f43172f60ULL, 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL,
56 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, 0xbef9a3f7b2c67915ULL,
57 0xc67178f2e372532bULL, 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL,
58 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, 0x06f067aa72176fbaULL,
59 0x0a637dc5a2c898a6ULL, 0x113f9804bef90daeULL, 0x1b710b35131c471bULL,
60 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, 0x3c9ebe0a15c9bebcULL,
61 0x431d67c49c100d4cULL, 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL,
62 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL,
63};
64
65#define Ch(x, y, z) ((z) ^ ((x) & ((y) ^ (z))))
66#define Maj(x, y, z) (((x) & (y)) | ((z) & ((x) | (y))))
67#define e0(x) (ror64((x), 28) ^ ror64((x), 34) ^ ror64((x), 39))
68#define e1(x) (ror64((x), 14) ^ ror64((x), 18) ^ ror64((x), 41))
69#define s0(x) (ror64((x), 1) ^ ror64((x), 8) ^ ((x) >> 7))
70#define s1(x) (ror64((x), 19) ^ ror64((x), 61) ^ ((x) >> 6))
71
72static void sha512_block_generic(struct sha512_block_state *state,
73 const u8 *data)
74{
75 u64 a = state->h[0];
76 u64 b = state->h[1];
77 u64 c = state->h[2];
78 u64 d = state->h[3];
79 u64 e = state->h[4];
80 u64 f = state->h[5];
81 u64 g = state->h[6];
82 u64 h = state->h[7];
83 u64 t1, t2;
84 u64 W[16];
85
86 for (int j = 0; j < 16; j++)
87 W[j] = get_unaligned_be64(p: data + j * sizeof(u64));
88
89 for (int i = 0; i < 80; i += 8) {
90 if ((i & 15) == 0 && i != 0) {
91 for (int j = 0; j < 16; j++) {
92 W[j & 15] += s1(W[(j - 2) & 15]) +
93 W[(j - 7) & 15] +
94 s0(W[(j - 15) & 15]);
95 }
96 }
97 t1 = h + e1(e) + Ch(e, f, g) + sha512_K[i] + W[(i & 15)];
98 t2 = e0(a) + Maj(a, b, c); d += t1; h = t1 + t2;
99 t1 = g + e1(d) + Ch(d, e, f) + sha512_K[i+1] + W[(i & 15) + 1];
100 t2 = e0(h) + Maj(h, a, b); c += t1; g = t1 + t2;
101 t1 = f + e1(c) + Ch(c, d, e) + sha512_K[i+2] + W[(i & 15) + 2];
102 t2 = e0(g) + Maj(g, h, a); b += t1; f = t1 + t2;
103 t1 = e + e1(b) + Ch(b, c, d) + sha512_K[i+3] + W[(i & 15) + 3];
104 t2 = e0(f) + Maj(f, g, h); a += t1; e = t1 + t2;
105 t1 = d + e1(a) + Ch(a, b, c) + sha512_K[i+4] + W[(i & 15) + 4];
106 t2 = e0(e) + Maj(e, f, g); h += t1; d = t1 + t2;
107 t1 = c + e1(h) + Ch(h, a, b) + sha512_K[i+5] + W[(i & 15) + 5];
108 t2 = e0(d) + Maj(d, e, f); g += t1; c = t1 + t2;
109 t1 = b + e1(g) + Ch(g, h, a) + sha512_K[i+6] + W[(i & 15) + 6];
110 t2 = e0(c) + Maj(c, d, e); f += t1; b = t1 + t2;
111 t1 = a + e1(f) + Ch(f, g, h) + sha512_K[i+7] + W[(i & 15) + 7];
112 t2 = e0(b) + Maj(b, c, d); e += t1; a = t1 + t2;
113 }
114
115 state->h[0] += a;
116 state->h[1] += b;
117 state->h[2] += c;
118 state->h[3] += d;
119 state->h[4] += e;
120 state->h[5] += f;
121 state->h[6] += g;
122 state->h[7] += h;
123}
124
125static void __maybe_unused
126sha512_blocks_generic(struct sha512_block_state *state,
127 const u8 *data, size_t nblocks)
128{
129 do {
130 sha512_block_generic(state, data);
131 data += SHA512_BLOCK_SIZE;
132 } while (--nblocks);
133}
134
135#ifdef CONFIG_CRYPTO_LIB_SHA512_ARCH
136#include "sha512.h" /* $(SRCARCH)/sha512.h */
137#else
138#define sha512_blocks sha512_blocks_generic
139#endif
140
141static void __sha512_init(struct __sha512_ctx *ctx,
142 const struct sha512_block_state *iv,
143 u64 initial_bytecount)
144{
145 ctx->state = *iv;
146 ctx->bytecount_lo = initial_bytecount;
147 ctx->bytecount_hi = 0;
148}
149
150void sha384_init(struct sha384_ctx *ctx)
151{
152 __sha512_init(ctx: &ctx->ctx, iv: &sha384_iv, initial_bytecount: 0);
153}
154EXPORT_SYMBOL_GPL(sha384_init);
155
156void sha512_init(struct sha512_ctx *ctx)
157{
158 __sha512_init(ctx: &ctx->ctx, iv: &sha512_iv, initial_bytecount: 0);
159}
160EXPORT_SYMBOL_GPL(sha512_init);
161
162void __sha512_update(struct __sha512_ctx *ctx, const u8 *data, size_t len)
163{
164 size_t partial = ctx->bytecount_lo % SHA512_BLOCK_SIZE;
165
166 if (check_add_overflow(ctx->bytecount_lo, len, &ctx->bytecount_lo))
167 ctx->bytecount_hi++;
168
169 if (partial + len >= SHA512_BLOCK_SIZE) {
170 size_t nblocks;
171
172 if (partial) {
173 size_t l = SHA512_BLOCK_SIZE - partial;
174
175 memcpy(to: &ctx->buf[partial], from: data, len: l);
176 data += l;
177 len -= l;
178
179 sha512_blocks(state: &ctx->state, data: ctx->buf, nblocks: 1);
180 }
181
182 nblocks = len / SHA512_BLOCK_SIZE;
183 len %= SHA512_BLOCK_SIZE;
184
185 if (nblocks) {
186 sha512_blocks(state: &ctx->state, data, nblocks);
187 data += nblocks * SHA512_BLOCK_SIZE;
188 }
189 partial = 0;
190 }
191 if (len)
192 memcpy(to: &ctx->buf[partial], from: data, len);
193}
194EXPORT_SYMBOL_GPL(__sha512_update);
195
196static void __sha512_final(struct __sha512_ctx *ctx,
197 u8 *out, size_t digest_size)
198{
199 u64 bitcount_hi = (ctx->bytecount_hi << 3) | (ctx->bytecount_lo >> 61);
200 u64 bitcount_lo = ctx->bytecount_lo << 3;
201 size_t partial = ctx->bytecount_lo % SHA512_BLOCK_SIZE;
202
203 ctx->buf[partial++] = 0x80;
204 if (partial > SHA512_BLOCK_SIZE - 16) {
205 memset(s: &ctx->buf[partial], c: 0, SHA512_BLOCK_SIZE - partial);
206 sha512_blocks(state: &ctx->state, data: ctx->buf, nblocks: 1);
207 partial = 0;
208 }
209 memset(s: &ctx->buf[partial], c: 0, SHA512_BLOCK_SIZE - 16 - partial);
210 *(__be64 *)&ctx->buf[SHA512_BLOCK_SIZE - 16] = cpu_to_be64(bitcount_hi);
211 *(__be64 *)&ctx->buf[SHA512_BLOCK_SIZE - 8] = cpu_to_be64(bitcount_lo);
212 sha512_blocks(state: &ctx->state, data: ctx->buf, nblocks: 1);
213
214 for (size_t i = 0; i < digest_size; i += 8)
215 put_unaligned_be64(val: ctx->state.h[i / 8], p: out + i);
216}
217
218void sha384_final(struct sha384_ctx *ctx, u8 out[SHA384_DIGEST_SIZE])
219{
220 __sha512_final(ctx: &ctx->ctx, out, SHA384_DIGEST_SIZE);
221 memzero_explicit(s: ctx, count: sizeof(*ctx));
222}
223EXPORT_SYMBOL_GPL(sha384_final);
224
225void sha512_final(struct sha512_ctx *ctx, u8 out[SHA512_DIGEST_SIZE])
226{
227 __sha512_final(ctx: &ctx->ctx, out, SHA512_DIGEST_SIZE);
228 memzero_explicit(s: ctx, count: sizeof(*ctx));
229}
230EXPORT_SYMBOL_GPL(sha512_final);
231
232void sha384(const u8 *data, size_t len, u8 out[SHA384_DIGEST_SIZE])
233{
234 struct sha384_ctx ctx;
235
236 sha384_init(&ctx);
237 sha384_update(ctx: &ctx, data, len);
238 sha384_final(&ctx, out);
239}
240EXPORT_SYMBOL_GPL(sha384);
241
242void sha512(const u8 *data, size_t len, u8 out[SHA512_DIGEST_SIZE])
243{
244 struct sha512_ctx ctx;
245
246 sha512_init(&ctx);
247 sha512_update(ctx: &ctx, data, len);
248 sha512_final(&ctx, out);
249}
250EXPORT_SYMBOL_GPL(sha512);
251
252static void __hmac_sha512_preparekey(struct sha512_block_state *istate,
253 struct sha512_block_state *ostate,
254 const u8 *raw_key, size_t raw_key_len,
255 const struct sha512_block_state *iv)
256{
257 union {
258 u8 b[SHA512_BLOCK_SIZE];
259 unsigned long w[SHA512_BLOCK_SIZE / sizeof(unsigned long)];
260 } derived_key = { 0 };
261
262 if (unlikely(raw_key_len > SHA512_BLOCK_SIZE)) {
263 if (iv == &sha384_iv)
264 sha384(raw_key, raw_key_len, derived_key.b);
265 else
266 sha512(raw_key, raw_key_len, derived_key.b);
267 } else {
268 memcpy(to: derived_key.b, from: raw_key, len: raw_key_len);
269 }
270
271 for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++)
272 derived_key.w[i] ^= REPEAT_BYTE(HMAC_IPAD_VALUE);
273 *istate = *iv;
274 sha512_blocks(state: istate, data: derived_key.b, nblocks: 1);
275
276 for (size_t i = 0; i < ARRAY_SIZE(derived_key.w); i++)
277 derived_key.w[i] ^= REPEAT_BYTE(HMAC_OPAD_VALUE ^
278 HMAC_IPAD_VALUE);
279 *ostate = *iv;
280 sha512_blocks(state: ostate, data: derived_key.b, nblocks: 1);
281
282 memzero_explicit(s: &derived_key, count: sizeof(derived_key));
283}
284
285void hmac_sha384_preparekey(struct hmac_sha384_key *key,
286 const u8 *raw_key, size_t raw_key_len)
287{
288 __hmac_sha512_preparekey(istate: &key->key.istate, ostate: &key->key.ostate,
289 raw_key, raw_key_len, iv: &sha384_iv);
290}
291EXPORT_SYMBOL_GPL(hmac_sha384_preparekey);
292
293void hmac_sha512_preparekey(struct hmac_sha512_key *key,
294 const u8 *raw_key, size_t raw_key_len)
295{
296 __hmac_sha512_preparekey(istate: &key->key.istate, ostate: &key->key.ostate,
297 raw_key, raw_key_len, iv: &sha512_iv);
298}
299EXPORT_SYMBOL_GPL(hmac_sha512_preparekey);
300
301void __hmac_sha512_init(struct __hmac_sha512_ctx *ctx,
302 const struct __hmac_sha512_key *key)
303{
304 __sha512_init(ctx: &ctx->sha_ctx, iv: &key->istate, SHA512_BLOCK_SIZE);
305 ctx->ostate = key->ostate;
306}
307EXPORT_SYMBOL_GPL(__hmac_sha512_init);
308
309void hmac_sha384_init_usingrawkey(struct hmac_sha384_ctx *ctx,
310 const u8 *raw_key, size_t raw_key_len)
311{
312 __hmac_sha512_preparekey(istate: &ctx->ctx.sha_ctx.state, ostate: &ctx->ctx.ostate,
313 raw_key, raw_key_len, iv: &sha384_iv);
314 ctx->ctx.sha_ctx.bytecount_lo = SHA512_BLOCK_SIZE;
315 ctx->ctx.sha_ctx.bytecount_hi = 0;
316}
317EXPORT_SYMBOL_GPL(hmac_sha384_init_usingrawkey);
318
319void hmac_sha512_init_usingrawkey(struct hmac_sha512_ctx *ctx,
320 const u8 *raw_key, size_t raw_key_len)
321{
322 __hmac_sha512_preparekey(istate: &ctx->ctx.sha_ctx.state, ostate: &ctx->ctx.ostate,
323 raw_key, raw_key_len, iv: &sha512_iv);
324 ctx->ctx.sha_ctx.bytecount_lo = SHA512_BLOCK_SIZE;
325 ctx->ctx.sha_ctx.bytecount_hi = 0;
326}
327EXPORT_SYMBOL_GPL(hmac_sha512_init_usingrawkey);
328
329static void __hmac_sha512_final(struct __hmac_sha512_ctx *ctx,
330 u8 *out, size_t digest_size)
331{
332 /* Generate the padded input for the outer hash in ctx->sha_ctx.buf. */
333 __sha512_final(ctx: &ctx->sha_ctx, out: ctx->sha_ctx.buf, digest_size);
334 memset(s: &ctx->sha_ctx.buf[digest_size], c: 0,
335 SHA512_BLOCK_SIZE - digest_size);
336 ctx->sha_ctx.buf[digest_size] = 0x80;
337 *(__be32 *)&ctx->sha_ctx.buf[SHA512_BLOCK_SIZE - 4] =
338 cpu_to_be32(8 * (SHA512_BLOCK_SIZE + digest_size));
339
340 /* Compute the outer hash, which gives the HMAC value. */
341 sha512_blocks(state: &ctx->ostate, data: ctx->sha_ctx.buf, nblocks: 1);
342 for (size_t i = 0; i < digest_size; i += 8)
343 put_unaligned_be64(val: ctx->ostate.h[i / 8], p: out + i);
344
345 memzero_explicit(s: ctx, count: sizeof(*ctx));
346}
347
348void hmac_sha384_final(struct hmac_sha384_ctx *ctx,
349 u8 out[SHA384_DIGEST_SIZE])
350{
351 __hmac_sha512_final(ctx: &ctx->ctx, out, SHA384_DIGEST_SIZE);
352}
353EXPORT_SYMBOL_GPL(hmac_sha384_final);
354
355void hmac_sha512_final(struct hmac_sha512_ctx *ctx,
356 u8 out[SHA512_DIGEST_SIZE])
357{
358 __hmac_sha512_final(ctx: &ctx->ctx, out, SHA512_DIGEST_SIZE);
359}
360EXPORT_SYMBOL_GPL(hmac_sha512_final);
361
362void hmac_sha384(const struct hmac_sha384_key *key,
363 const u8 *data, size_t data_len, u8 out[SHA384_DIGEST_SIZE])
364{
365 struct hmac_sha384_ctx ctx;
366
367 hmac_sha384_init(ctx: &ctx, key);
368 hmac_sha384_update(ctx: &ctx, data, data_len);
369 hmac_sha384_final(&ctx, out);
370}
371EXPORT_SYMBOL_GPL(hmac_sha384);
372
373void hmac_sha512(const struct hmac_sha512_key *key,
374 const u8 *data, size_t data_len, u8 out[SHA512_DIGEST_SIZE])
375{
376 struct hmac_sha512_ctx ctx;
377
378 hmac_sha512_init(ctx: &ctx, key);
379 hmac_sha512_update(ctx: &ctx, data, data_len);
380 hmac_sha512_final(&ctx, out);
381}
382EXPORT_SYMBOL_GPL(hmac_sha512);
383
384void hmac_sha384_usingrawkey(const u8 *raw_key, size_t raw_key_len,
385 const u8 *data, size_t data_len,
386 u8 out[SHA384_DIGEST_SIZE])
387{
388 struct hmac_sha384_ctx ctx;
389
390 hmac_sha384_init_usingrawkey(&ctx, raw_key, raw_key_len);
391 hmac_sha384_update(ctx: &ctx, data, data_len);
392 hmac_sha384_final(&ctx, out);
393}
394EXPORT_SYMBOL_GPL(hmac_sha384_usingrawkey);
395
396void hmac_sha512_usingrawkey(const u8 *raw_key, size_t raw_key_len,
397 const u8 *data, size_t data_len,
398 u8 out[SHA512_DIGEST_SIZE])
399{
400 struct hmac_sha512_ctx ctx;
401
402 hmac_sha512_init_usingrawkey(&ctx, raw_key, raw_key_len);
403 hmac_sha512_update(ctx: &ctx, data, data_len);
404 hmac_sha512_final(&ctx, out);
405}
406EXPORT_SYMBOL_GPL(hmac_sha512_usingrawkey);
407
408#ifdef sha512_mod_init_arch
409static int __init sha512_mod_init(void)
410{
411 sha512_mod_init_arch();
412 return 0;
413}
414subsys_initcall(sha512_mod_init);
415
416static void __exit sha512_mod_exit(void)
417{
418}
419module_exit(sha512_mod_exit);
420#endif
421
422MODULE_DESCRIPTION("SHA-384, SHA-512, HMAC-SHA384, and HMAC-SHA512 library functions");
423MODULE_LICENSE("GPL");
424