1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * SHA-1 optimized for x86_64
4 *
5 * Copyright 2025 Google LLC
6 */
7#include <asm/fpu/api.h>
8#include <linux/static_call.h>
9
10DEFINE_STATIC_CALL(sha1_blocks_x86, sha1_blocks_generic);
11
12#define DEFINE_X86_SHA1_FN(c_fn, asm_fn) \
13 asmlinkage void asm_fn(struct sha1_block_state *state, \
14 const u8 *data, size_t nblocks); \
15 static void c_fn(struct sha1_block_state *state, \
16 const u8 *data, size_t nblocks) \
17 { \
18 if (likely(irq_fpu_usable())) { \
19 kernel_fpu_begin(); \
20 asm_fn(state, data, nblocks); \
21 kernel_fpu_end(); \
22 } else { \
23 sha1_blocks_generic(state, data, nblocks); \
24 } \
25 }
26
27DEFINE_X86_SHA1_FN(sha1_blocks_ssse3, sha1_transform_ssse3);
28DEFINE_X86_SHA1_FN(sha1_blocks_avx, sha1_transform_avx);
29DEFINE_X86_SHA1_FN(sha1_blocks_ni, sha1_ni_transform);
30
31#define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */
32
33asmlinkage void sha1_transform_avx2(struct sha1_block_state *state,
34 const u8 *data, size_t nblocks);
35static void sha1_blocks_avx2(struct sha1_block_state *state,
36 const u8 *data, size_t nblocks)
37{
38 if (likely(irq_fpu_usable())) {
39 kernel_fpu_begin();
40 /* Select the optimal transform based on the number of blocks */
41 if (nblocks >= SHA1_AVX2_BLOCK_OPTSIZE)
42 sha1_transform_avx2(state, data, nblocks);
43 else
44 sha1_transform_avx(state, data, nblocks);
45 kernel_fpu_end();
46 } else {
47 sha1_blocks_generic(state, data, nblocks);
48 }
49}
50
51static void sha1_blocks(struct sha1_block_state *state,
52 const u8 *data, size_t nblocks)
53{
54 static_call(sha1_blocks_x86)(state, data, nblocks);
55}
56
57#define sha1_mod_init_arch sha1_mod_init_arch
58static void sha1_mod_init_arch(void)
59{
60 if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
61 static_call_update(sha1_blocks_x86, sha1_blocks_ni);
62 } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
63 NULL) &&
64 boot_cpu_has(X86_FEATURE_AVX)) {
65 if (boot_cpu_has(X86_FEATURE_AVX2) &&
66 boot_cpu_has(X86_FEATURE_BMI1) &&
67 boot_cpu_has(X86_FEATURE_BMI2))
68 static_call_update(sha1_blocks_x86, sha1_blocks_avx2);
69 else
70 static_call_update(sha1_blocks_x86, sha1_blocks_avx);
71 } else if (boot_cpu_has(X86_FEATURE_SSSE3)) {
72 static_call_update(sha1_blocks_x86, sha1_blocks_ssse3);
73 }
74}
75