15
15
#include <asm/asm-offsets.h>
16
16
#include <asm/thread_info.h>
17
17
#include <asm/cpufeature.h>
18
#include <asm/alternative-asm.h>
19
.macro ALTERNATIVE_JUMP feature,orig,alt
21
* By placing feature2 after feature1 in altinstructions section, we logically
23
* If CPU has feature2, jmp to alt2 is used
24
* else if CPU has feature1, jmp to alt1 is used
25
* else jmp to orig is used.
27
.macro ALTERNATIVE_JUMP feature1,feature2,orig,alt1,alt2
21
29
.byte 0xe9 /* 32bit jump */
22
30
.long \orig-1f /* by default jump to orig */
24
32
.section .altinstr_replacement,"ax"
25
33
2: .byte 0xe9 /* near jump with 32bit immediate */
26
.long \alt-1b /* offset */ /* or alternatively to alt */
34
.long \alt1-1b /* offset */ /* or alternatively to alt1 */
35
3: .byte 0xe9 /* near jump with 32bit immediate */
36
.long \alt2-1b /* offset */ /* or alternatively to alt2 */
28
39
.section .altinstructions,"a"
32
.word \feature /* when feature is set */
40
altinstruction_entry 0b,2b,\feature1,5,5
41
altinstruction_entry 0b,3b,\feature2,5,5
74
81
cmpq TI_addr_limit(%rax),%rcx
76
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
83
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
84
copy_user_generic_unrolled,copy_user_generic_string, \
85
copy_user_enhanced_fast_string
78
87
ENDPROC(_copy_to_user)
87
96
cmpq TI_addr_limit(%rax),%rcx
89
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
98
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
99
copy_user_generic_unrolled,copy_user_generic_string, \
100
copy_user_enhanced_fast_string
91
102
ENDPROC(_copy_from_user)
257
268
ENDPROC(copy_user_generic_string)
271
* Some CPUs are adding enhanced REP MOVSB/STOSB instructions.
272
* It's recommended to use enhanced REP MOVSB/STOSB if it's enabled.
280
* eax uncopied bytes or 0 if successful.
282
ENTRY(copy_user_enhanced_fast_string)
293
12: movl %ecx,%edx /* ecx is zerorest also */
294
jmp copy_user_handle_tail
297
.section __ex_table,"a"
302
ENDPROC(copy_user_enhanced_fast_string)