1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * This file is derived from memcpy_32.c in the Linux kernel.
4 */
5
6 #include <string.h>
7 #include <stdbool.h>
8 #include <asan.h>
9
memmove(void * dest,const void * src,size_t n)10 void *memmove(void *dest, const void *src, size_t n)
11 {
12 int d0, d1, d2, d3, d4, d5;
13 char *ret = dest;
14
15 #if (ENV_SEPARATE_ROMSTAGE && CONFIG(ASAN_IN_ROMSTAGE)) || \
16 (ENV_RAMSTAGE && CONFIG(ASAN_IN_RAMSTAGE))
17 check_memory_region((unsigned long)src, n, false, _RET_IP_);
18 check_memory_region((unsigned long)dest, n, true, _RET_IP_);
19 #endif
20
21 __asm__ __volatile__(
22 /* Handle more 16bytes in loop */
23 "cmp $0x10, %0\n\t"
24 "jb 1f\n\t"
25
26 /* Decide forward/backward copy mode */
27 "cmp %2, %1\n\t"
28 "jb 2f\n\t"
29
30 /*
31 * movs instruction have many startup latency
32 * so we handle small size by general register.
33 */
34 "cmp $680, %0\n\t"
35 "jb 3f\n\t"
36 /*
37 * movs instruction is only good for aligned case.
38 */
39 "mov %1, %3\n\t"
40 "xor %2, %3\n\t"
41 "and $0xff, %3\n\t"
42 "jz 4f\n\t"
43 "3:\n\t"
44 "sub $0x10, %0\n\t"
45
46 /*
47 * We gobble 16byts forward in each loop.
48 */
49 "3:\n\t"
50 "sub $0x10, %0\n\t"
51 "mov 0*4(%1), %3\n\t"
52 "mov 1*4(%1), %4\n\t"
53 "mov %3, 0*4(%2)\n\t"
54 "mov %4, 1*4(%2)\n\t"
55 "mov 2*4(%1), %3\n\t"
56 "mov 3*4(%1), %4\n\t"
57 "mov %3, 2*4(%2)\n\t"
58 "mov %4, 3*4(%2)\n\t"
59 "lea 0x10(%1), %1\n\t"
60 "lea 0x10(%2), %2\n\t"
61 "jae 3b\n\t"
62 "add $0x10, %0\n\t"
63 "jmp 1f\n\t"
64
65 /*
66 * Handle data forward by movs.
67 */
68 ".p2align 4\n\t"
69 "4:\n\t"
70 "mov -4(%1, %0), %3\n\t"
71 "lea -4(%2, %0), %4\n\t"
72 "shr $2, %0\n\t"
73 "rep movsl\n\t"
74 "mov %3, (%4)\n\t"
75 "jmp 11f\n\t"
76 /*
77 * Handle data backward by movs.
78 */
79 ".p2align 4\n\t"
80 "6:\n\t"
81 "mov (%1), %3\n\t"
82 "mov %2, %4\n\t"
83 "lea -4(%1, %0), %1\n\t"
84 "lea -4(%2, %0), %2\n\t"
85 "shr $2, %0\n\t"
86 "std\n\t"
87 "rep movsl\n\t"
88 "mov %3,(%4)\n\t"
89 "cld\n\t"
90 "jmp 11f\n\t"
91
92 /*
93 * Start to prepare for backward copy.
94 */
95 ".p2align 4\n\t"
96 "2:\n\t"
97 "cmp $680, %0\n\t"
98 "jb 5f\n\t"
99 "mov %1, %3\n\t"
100 "xor %2, %3\n\t"
101 "and $0xff, %3\n\t"
102 "jz 6b\n\t"
103
104 /*
105 * Calculate copy position to tail.
106 */
107 "5:\n\t"
108 "add %0, %1\n\t"
109 "add %0, %2\n\t"
110 "sub $0x10, %0\n\t"
111
112 /*
113 * We gobble 16byts backward in each loop.
114 */
115 "7:\n\t"
116 "sub $0x10, %0\n\t"
117
118 "mov -1*4(%1), %3\n\t"
119 "mov -2*4(%1), %4\n\t"
120 "mov %3, -1*4(%2)\n\t"
121 "mov %4, -2*4(%2)\n\t"
122 "mov -3*4(%1), %3\n\t"
123 "mov -4*4(%1), %4\n\t"
124 "mov %3, -3*4(%2)\n\t"
125 "mov %4, -4*4(%2)\n\t"
126 "lea -0x10(%1), %1\n\t"
127 "lea -0x10(%2), %2\n\t"
128 "jae 7b\n\t"
129 /*
130 * Calculate copy position to head.
131 */
132 "add $0x10, %0\n\t"
133 "sub %0, %1\n\t"
134 "sub %0, %2\n\t"
135
136 /*
137 * Move data from 8 bytes to 15 bytes.
138 */
139 ".p2align 4\n\t"
140 "1:\n\t"
141 "cmp $8, %0\n\t"
142 "jb 8f\n\t"
143 "mov 0*4(%1), %3\n\t"
144 "mov 1*4(%1), %4\n\t"
145 "mov -2*4(%1, %0), %5\n\t"
146 "mov -1*4(%1, %0), %1\n\t"
147
148 "mov %3, 0*4(%2)\n\t"
149 "mov %4, 1*4(%2)\n\t"
150 "mov %5, -2*4(%2, %0)\n\t"
151 "mov %1, -1*4(%2, %0)\n\t"
152 "jmp 11f\n\t"
153
154 /*
155 * Move data from 4 bytes to 7 bytes.
156 */
157 ".p2align 4\n\t"
158 "8:\n\t"
159 "cmp $4, %0\n\t"
160 "jb 9f\n\t"
161 "mov 0*4(%1), %3\n\t"
162 "mov -1*4(%1, %0), %4\n\t"
163 "mov %3, 0*4(%2)\n\t"
164 "mov %4, -1*4(%2, %0)\n\t"
165 "jmp 11f\n\t"
166
167 /*
168 * Move data from 2 bytes to 3 bytes.
169 */
170 ".p2align 4\n\t"
171 "9:\n\t"
172 "cmp $2, %0\n\t"
173 "jb 10f\n\t"
174 "movw 0*2(%1), %%dx\n\t"
175 "movw -1*2(%1, %0), %%bx\n\t"
176 "movw %%dx, 0*2(%2)\n\t"
177 "movw %%bx, -1*2(%2, %0)\n\t"
178 "jmp 11f\n\t"
179
180 /*
181 * Move data for 1 byte.
182 */
183 ".p2align 4\n\t"
184 "10:\n\t"
185 "cmp $1, %0\n\t"
186 "jb 11f\n\t"
187 "movb (%1), %%cl\n\t"
188 "movb %%cl, (%2)\n\t"
189 ".p2align 4\n\t"
190 "11:"
191 : "=&c" (d0), "=&S" (d1), "=&D" (d2),
192 "=r" (d3), "=r" (d4), "=r"(d5)
193 : "0" (n),
194 "1" (src),
195 "2" (dest)
196 : "memory");
197
198 return ret;
199 }
200