1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Calculate a CRC T10-DIF with vpmsum acceleration
4 *
5 * Copyright 2017, Daniel Axtens, IBM Corporation.
6 * [based on crc32c-vpmsum_glue.c]
7 */
8
9 #include <linux/crc-t10dif.h>
10 #include <crypto/internal/simd.h>
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/string.h>
14 #include <linux/kernel.h>
15 #include <linux/cpufeature.h>
16 #include <asm/simd.h>
17 #include <asm/switch_to.h>
18
19 #define VMX_ALIGN 16
20 #define VMX_ALIGN_MASK (VMX_ALIGN-1)
21
22 #define VECTOR_BREAKPOINT 64
23
24 static DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
25
26 u32 __crct10dif_vpmsum(u32 crc, unsigned char const *p, size_t len);
27
crc_t10dif_arch(u16 crci,const u8 * p,size_t len)28 u16 crc_t10dif_arch(u16 crci, const u8 *p, size_t len)
29 {
30 unsigned int prealign;
31 unsigned int tail;
32 u32 crc = crci;
33
34 if (len < (VECTOR_BREAKPOINT + VMX_ALIGN) ||
35 !static_branch_likely(&have_vec_crypto) || !crypto_simd_usable())
36 return crc_t10dif_generic(crc, p, len);
37
38 if ((unsigned long)p & VMX_ALIGN_MASK) {
39 prealign = VMX_ALIGN - ((unsigned long)p & VMX_ALIGN_MASK);
40 crc = crc_t10dif_generic(crc, p, prealign);
41 len -= prealign;
42 p += prealign;
43 }
44
45 if (len & ~VMX_ALIGN_MASK) {
46 crc <<= 16;
47 preempt_disable();
48 pagefault_disable();
49 enable_kernel_altivec();
50 crc = __crct10dif_vpmsum(crc, p, len & ~VMX_ALIGN_MASK);
51 disable_kernel_altivec();
52 pagefault_enable();
53 preempt_enable();
54 crc >>= 16;
55 }
56
57 tail = len & VMX_ALIGN_MASK;
58 if (tail) {
59 p += len & ~VMX_ALIGN_MASK;
60 crc = crc_t10dif_generic(crc, p, tail);
61 }
62
63 return crc & 0xffff;
64 }
65 EXPORT_SYMBOL(crc_t10dif_arch);
66
crc_t10dif_powerpc_init(void)67 static int __init crc_t10dif_powerpc_init(void)
68 {
69 if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
70 (cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
71 static_branch_enable(&have_vec_crypto);
72 return 0;
73 }
74 arch_initcall(crc_t10dif_powerpc_init);
75
crc_t10dif_powerpc_exit(void)76 static void __exit crc_t10dif_powerpc_exit(void)
77 {
78 }
79 module_exit(crc_t10dif_powerpc_exit);
80
crc_t10dif_is_optimized(void)81 bool crc_t10dif_is_optimized(void)
82 {
83 return static_key_enabled(&have_vec_crypto);
84 }
85 EXPORT_SYMBOL(crc_t10dif_is_optimized);
86
87 MODULE_AUTHOR("Daniel Axtens <[email protected]>");
88 MODULE_DESCRIPTION("CRCT10DIF using vector polynomial multiply-sum instructions");
89 MODULE_LICENSE("GPL");
90