b7de4d37eb
Copy from Base:System/glibc based on submit request 27940 from user dirkmueller OBS-URL: https://build.opensuse.org/request/show/27940 OBS-URL: https://build.opensuse.org/package/show/openSUSE:Factory/glibc?expand=0&rev=14
4435 lines
100 KiB
Diff
4435 lines
100 KiB
Diff
Index: sysdeps/x86_64/dl-machine.h
|
|
===================================================================
|
|
--- sysdeps/x86_64/dl-machine.h.orig
|
|
+++ sysdeps/x86_64/dl-machine.h
|
|
@@ -226,6 +226,40 @@ dl_platform_init (void)
|
|
if (GLRO(dl_platform) != NULL && *GLRO(dl_platform) == '\0')
|
|
/* Avoid an empty string which would disturb us. */
|
|
GLRO(dl_platform) = NULL;
|
|
+
|
|
+ long int t1, t2;
|
|
+ t1 = 0;
|
|
+ t2 = 0;
|
|
+
|
|
+ asm (
|
|
+ "mov $0x80000000, %%eax # get highest level of support\n\t"
|
|
+ "cpuid\n\t"
|
|
+ "cmp $0x80000006, %%eax # check for support of cache info\n\t"
|
|
+ "jb 1f\n\t"
|
|
+ "mov $0x80000005, %%eax # get L1 info\n\t"
|
|
+ "cpuid\n\t"
|
|
+ "shr $24, %%ecx\n\t"
|
|
+ "shl $10, %%ecx\n\t"
|
|
+ "mov %%rcx, %0\n\t"
|
|
+ "mov $0x80000006, %%eax # get L2 info\n\t"
|
|
+ "cpuid\n\t"
|
|
+ "shr $16, %%ecx\n\t"
|
|
+ "shl $10, %%ecx\n\t"
|
|
+ "mov %%rcx, %1\n\t"
|
|
+ "1:\n\t"
|
|
+ :"=r" (t1), "=r" (t2) :: "%rbx", "%rax", "%rcx", "%rdx"
|
|
+ );
|
|
+
|
|
+ if (t1)
|
|
+ {
|
|
+ GLRO(dl_cache1size) = t1;
|
|
+ GLRO(dl_cache1sizehalf) = t1 / 2;
|
|
+ }
|
|
+ if (t2)
|
|
+ {
|
|
+ GLRO(dl_cache2size) = t2;
|
|
+ GLRO(dl_cache2sizehalf) = t2 / 2;
|
|
+ }
|
|
}
|
|
|
|
static inline Elf64_Addr
|
|
Index: sysdeps/x86_64/Makefile
|
|
===================================================================
|
|
--- sysdeps/x86_64/Makefile.orig
|
|
+++ sysdeps/x86_64/Makefile
|
|
@@ -4,7 +4,8 @@ long-double-fcts = yes
|
|
ifeq ($(subdir),csu)
|
|
sysdep_routines += hp-timing
|
|
elide-routines.os += hp-timing
|
|
-gen-as-const-headers += link-defines.sym
|
|
+# get offset to rtld_global._dl_*
|
|
+gen-as-const-headers += link-defines.sym rtld-global-offsets.sym
|
|
endif
|
|
|
|
ifeq ($(subdir),gmon)
|
|
Index: sysdeps/x86_64/strcpy.S
|
|
===================================================================
|
|
--- sysdeps/x86_64/strcpy.S.orig
|
|
+++ sysdeps/x86_64/strcpy.S
|
|
@@ -1,159 +1,833 @@
|
|
-/* strcpy/stpcpy implementation for x86-64.
|
|
- Copyright (C) 2002 Free Software Foundation, Inc.
|
|
- This file is part of the GNU C Library.
|
|
- Contributed by Andreas Jaeger <aj@suse.de>, 2002.
|
|
-
|
|
- The GNU C Library is free software; you can redistribute it and/or
|
|
- modify it under the terms of the GNU Lesser General Public
|
|
- License as published by the Free Software Foundation; either
|
|
- version 2.1 of the License, or (at your option) any later version.
|
|
-
|
|
- The GNU C Library is distributed in the hope that it will be useful,
|
|
- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
- Lesser General Public License for more details.
|
|
-
|
|
- You should have received a copy of the GNU Lesser General Public
|
|
- License along with the GNU C Library; if not, write to the Free
|
|
- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
|
- 02111-1307 USA. */
|
|
-
|
|
-#include <sysdep.h>
|
|
-#include "asm-syntax.h"
|
|
-#include "bp-sym.h"
|
|
-#include "bp-asm.h"
|
|
+# $Header: /K8_Projects/Glibc/amd64strcpy.S 7 2/12/04 19:06 Emenezes $
|
|
|
|
-#ifndef USE_AS_STPCPY
|
|
+# (c) 2002 Advanced Micro Devices, Inc.
|
|
+# YOUR USE OF THIS CODE IS SUBJECT TO THE TERMS
|
|
+# AND CONDITIONS OF THE GNU LESSER GENERAL PUBLIC
|
|
+# LICENSE FOUND IN THE "README" FILE THAT IS
|
|
+# INCLUDED WITH THIS FILE
|
|
+
|
|
+#include "sysdep.h"
|
|
+#include <rtld-global-offsets.h>
|
|
+
|
|
+ /* XXX: strncpy is broken, just use this for strcpy for now. */
|
|
+#ifdef PIC
|
|
+ .globl _rtld_local_ro
|
|
+ .hidden _rtld_local_ro
|
|
+ .set _rtld_local_ro,_rtld_global_ro
|
|
+#endif
|
|
+#ifndef STRCPY
|
|
# define STRCPY strcpy
|
|
#endif
|
|
+#define LABEL(s) L(strcpy##s)
|
|
+
|
|
+ .text
|
|
+
|
|
+ENTRY (STRCPY) # (char *, const char *)
|
|
+
|
|
+#ifdef USE_AS_STRNCPY // (char *, const char *, size_t)
|
|
+ test %rdx, %rdx # (char *, const char *, size_t)
|
|
+ mov %rdx, %r11
|
|
+ jz LABEL(exitn) # early exit
|
|
+#endif
|
|
+
|
|
+ xor %edx, %edx
|
|
+
|
|
+LABEL(aligntry):
|
|
+ mov %rsi, %r8 # align by source
|
|
+ and $7, %r8
|
|
+ jz LABEL(alignafter)
|
|
+
|
|
+LABEL(align): # 8-byte align
|
|
+ sub $8, %r8
|
|
|
|
- .text
|
|
-ENTRY (BP_SYM (STRCPY))
|
|
- movq %rsi, %rcx /* Source register. */
|
|
- andl $7, %ecx /* mask alignment bits */
|
|
- movq %rdi, %rdx /* Duplicate destination pointer. */
|
|
-
|
|
- jz 5f /* aligned => start loop */
|
|
-
|
|
- neg %ecx /* We need to align to 8 bytes. */
|
|
- addl $8,%ecx
|
|
- /* Search the first bytes directly. */
|
|
-0:
|
|
- movb (%rsi), %al /* Fetch a byte */
|
|
- testb %al, %al /* Is it NUL? */
|
|
- movb %al, (%rdx) /* Store it */
|
|
- jz 4f /* If it was NUL, done! */
|
|
- incq %rsi
|
|
- incq %rdx
|
|
- decl %ecx
|
|
- jnz 0b
|
|
-
|
|
-5:
|
|
- movq $0xfefefefefefefeff,%r8
|
|
-
|
|
- /* Now the sources is aligned. Unfortunatly we cannot force
|
|
- to have both source and destination aligned, so ignore the
|
|
- alignment of the destination. */
|
|
.p2align 4
|
|
-1:
|
|
- /* 1st unroll. */
|
|
- movq (%rsi), %rax /* Read double word (8 bytes). */
|
|
- addq $8, %rsi /* Adjust pointer for next word. */
|
|
- movq %rax, %r9 /* Save a copy for NUL finding. */
|
|
- addq %r8, %r9 /* add the magic value to the word. We get
|
|
- carry bits reported for each byte which
|
|
- is *not* 0 */
|
|
- jnc 3f /* highest byte is NUL => return pointer */
|
|
- xorq %rax, %r9 /* (word+magic)^word */
|
|
- orq %r8, %r9 /* set all non-carry bits */
|
|
- incq %r9 /* add 1: if one carry bit was *not* set
|
|
- the addition will not result in 0. */
|
|
-
|
|
- jnz 3f /* found NUL => return pointer */
|
|
-
|
|
- movq %rax, (%rdx) /* Write value to destination. */
|
|
- addq $8, %rdx /* Adjust pointer. */
|
|
-
|
|
- /* 2nd unroll. */
|
|
- movq (%rsi), %rax /* Read double word (8 bytes). */
|
|
- addq $8, %rsi /* Adjust pointer for next word. */
|
|
- movq %rax, %r9 /* Save a copy for NUL finding. */
|
|
- addq %r8, %r9 /* add the magic value to the word. We get
|
|
- carry bits reported for each byte which
|
|
- is *not* 0 */
|
|
- jnc 3f /* highest byte is NUL => return pointer */
|
|
- xorq %rax, %r9 /* (word+magic)^word */
|
|
- orq %r8, %r9 /* set all non-carry bits */
|
|
- incq %r9 /* add 1: if one carry bit was *not* set
|
|
- the addition will not result in 0. */
|
|
-
|
|
- jnz 3f /* found NUL => return pointer */
|
|
-
|
|
- movq %rax, (%rdx) /* Write value to destination. */
|
|
- addq $8, %rdx /* Adjust pointer. */
|
|
-
|
|
- /* 3rd unroll. */
|
|
- movq (%rsi), %rax /* Read double word (8 bytes). */
|
|
- addq $8, %rsi /* Adjust pointer for next word. */
|
|
- movq %rax, %r9 /* Save a copy for NUL finding. */
|
|
- addq %r8, %r9 /* add the magic value to the word. We get
|
|
- carry bits reported for each byte which
|
|
- is *not* 0 */
|
|
- jnc 3f /* highest byte is NUL => return pointer */
|
|
- xorq %rax, %r9 /* (word+magic)^word */
|
|
- orq %r8, %r9 /* set all non-carry bits */
|
|
- incq %r9 /* add 1: if one carry bit was *not* set
|
|
- the addition will not result in 0. */
|
|
-
|
|
- jnz 3f /* found NUL => return pointer */
|
|
-
|
|
- movq %rax, (%rdx) /* Write value to destination. */
|
|
- addq $8, %rdx /* Adjust pointer. */
|
|
-
|
|
- /* 4th unroll. */
|
|
- movq (%rsi), %rax /* Read double word (8 bytes). */
|
|
- addq $8, %rsi /* Adjust pointer for next word. */
|
|
- movq %rax, %r9 /* Save a copy for NUL finding. */
|
|
- addq %r8, %r9 /* add the magic value to the word. We get
|
|
- carry bits reported for each byte which
|
|
- is *not* 0 */
|
|
- jnc 3f /* highest byte is NUL => return pointer */
|
|
- xorq %rax, %r9 /* (word+magic)^word */
|
|
- orq %r8, %r9 /* set all non-carry bits */
|
|
- incq %r9 /* add 1: if one carry bit was *not* set
|
|
- the addition will not result in 0. */
|
|
-
|
|
- jnz 3f /* found NUL => return pointer */
|
|
-
|
|
- movq %rax, (%rdx) /* Write value to destination. */
|
|
- addq $8, %rdx /* Adjust pointer. */
|
|
- jmp 1b /* Next iteration. */
|
|
|
|
- /* Do the last few bytes. %rax contains the value to write.
|
|
- The loop is unrolled twice. */
|
|
+LABEL(alignloop):
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ dec %r11
|
|
+ jl LABEL(exitn)
|
|
+#endif
|
|
+
|
|
+ mov (%rsi, %rdx), %al # check if same character
|
|
+ test %al, %al # check if character a NUL
|
|
+ mov %al, (%rdi, %rdx)
|
|
+ jz LABEL(exit)
|
|
+
|
|
+ inc %edx
|
|
+ inc %r8
|
|
+ jnz LABEL(alignloop)
|
|
+
|
|
.p2align 4
|
|
+
|
|
+LABEL(alignafter):
|
|
+
|
|
+LABEL(8try):
|
|
+ mov $0xfefefefefefefeff, %rcx
|
|
+
|
|
+LABEL(8): # 8-byte
|
|
+ mov (%rsi, %rdx), %rax
|
|
+
|
|
+LABEL(8loop):
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+LABEL(8after):
|
|
+
|
|
+LABEL(64try):
|
|
+#ifdef PIC
|
|
+ mov _rtld_local_ro@GOTPCREL(%rip), %r8
|
|
+ mov RTLD_GLOBAL_DL_CACHE1SIZEHALF(%r8), %r9
|
|
+#else
|
|
+ mov _dl_cache1sizehalf, %r9
|
|
+#endif
|
|
+
|
|
+
|
|
+LABEL(64): # 64-byte
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+LABEL(64loop):
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ cmp %r9, %rdx
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ lea 8 (%rdx), %rdx
|
|
+
|
|
+ jbe LABEL(64loop)
|
|
+
|
|
+LABEL(64after):
|
|
+
|
|
+LABEL(pretry):
|
|
+#ifdef PIC
|
|
+ mov _rtld_local_ro@GOTPCREL(%rip), %r8
|
|
+ mov RTLD_GLOBAL_DL_CACHE2SIZEHALF(%r8), %r9
|
|
+#else
|
|
+ mov _dl_cache2sizehalf, %r9
|
|
+#endif
|
|
+
|
|
+LABEL(pre): # 64-byte prefetch
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+LABEL(preloop):
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %edx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
+
|
|
+ cmp %r9, %rdx
|
|
+
|
|
+ mov %rax, (%rdi, %rdx)
|
|
+ prefetcht0 512 + 8 (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ prefetcht0 512 + 8 (%rsi, %rdx)
|
|
+ lea 8 (%rdx), %rdx
|
|
+
|
|
+ jb LABEL(preloop)
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+LABEL(preafter):
|
|
+
|
|
+LABEL(NTtry):
|
|
+ sfence
|
|
+
|
|
+LABEL(NT): # 64-byte NT
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+LABEL(NTloop):
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(NTtail)
|
|
+
|
|
+ movnti %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %rdx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(NTtail)
|
|
+
|
|
+ movnti %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %rdx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(NTtail)
|
|
+
|
|
+ movnti %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %rdx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(NTtail)
|
|
+
|
|
+ movnti %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %rdx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(NTtail)
|
|
+
|
|
+ movnti %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %rdx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(NTtail)
|
|
+
|
|
+ movnti %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %rdx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(NTtail)
|
|
+
|
|
+ movnti %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ add $8, %rdx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ sub $8, %r11
|
|
+ jl LABEL(tail)
|
|
+#endif
|
|
+
|
|
+ mov %rcx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
+
|
|
+ xor %rax, %r8
|
|
+ or %rcx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(NTtail)
|
|
+
|
|
+ movnti %rax, (%rdi, %rdx)
|
|
+ mov 8 (%rsi, %rdx), %rax
|
|
+ prefetchnta 768 + 8 (%rsi, %rdx)
|
|
+ add $8, %rdx
|
|
+
|
|
+ jmp LABEL(NTloop)
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+LABEL(NTtail):
|
|
+ sfence
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+LABEL(NTafter):
|
|
+
|
|
+LABEL(tailtry):
|
|
+
|
|
+LABEL(tail): # 1-byte tail
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ add $8, %r11
|
|
+#endif
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+LABEL(tailloop):
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ dec %r11
|
|
+ jl LABEL(exitn)
|
|
+#endif
|
|
+
|
|
+ test %al, %al
|
|
+ mov %al, (%rdi, %rdx)
|
|
+ jz LABEL(exit)
|
|
+
|
|
+ inc %rdx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ dec %r11
|
|
+ jl LABEL(exitn)
|
|
+
|
|
+ mov %ah, %al
|
|
+#endif
|
|
+
|
|
+ test %ah, %ah
|
|
+ mov %ah, (%rdi, %rdx)
|
|
+ jz LABEL(exit)
|
|
+
|
|
+ inc %rdx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ dec %r11
|
|
+ jl LABEL(exitn)
|
|
+#endif
|
|
+
|
|
+ shr $16, %rax
|
|
+
|
|
+ test %al, %al
|
|
+ mov %al, (%rdi, %rdx)
|
|
+ jz LABEL(exit)
|
|
+
|
|
+ inc %rdx
|
|
+
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ dec %r11
|
|
+ jl LABEL(exitn)
|
|
+
|
|
+ mov %ah, %al
|
|
+#endif
|
|
+
|
|
+ test %ah, %ah
|
|
+ mov %ah, (%rdi, %rdx)
|
|
+ jz LABEL(exit)
|
|
+
|
|
+ shr $16, %rax
|
|
+ inc %rdx
|
|
+
|
|
+ jmp LABEL(tailloop)
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+LABEL(tailafter):
|
|
+
|
|
+LABEL(exit):
|
|
+#ifdef USE_AS_STRNCPY
|
|
+ test %r11, %r11
|
|
+ mov %r11, %rcx
|
|
+
|
|
+#ifdef USE_AS_STPCPY
|
|
+ lea (%rdi, %rdx), %r8
|
|
+#else
|
|
+ mov %rdi, %r8
|
|
+#endif
|
|
+
|
|
+ jz 2f
|
|
+
|
|
+ xor %eax, %eax # bzero () would do too, but usually there are only a handfull of bytes left
|
|
+ shr $3, %rcx
|
|
+ lea 1 (%rdi, %rdx), %rdi
|
|
+ jz 1f
|
|
+
|
|
+ rep stosq
|
|
+
|
|
+1:
|
|
+ mov %r11d, %ecx
|
|
+ and $7, %ecx
|
|
+ jz 2f
|
|
+
|
|
+ .p2align 4,, 3
|
|
+
|
|
3:
|
|
- /* Note that stpcpy needs to return with the value of the NUL
|
|
- byte. */
|
|
- movb %al, (%rdx) /* 1st byte. */
|
|
- testb %al, %al /* Is it NUL. */
|
|
- jz 4f /* yes, finish. */
|
|
- incq %rdx /* Increment destination. */
|
|
- movb %ah, (%rdx) /* 2nd byte. */
|
|
- testb %ah, %ah /* Is it NUL?. */
|
|
- jz 4f /* yes, finish. */
|
|
- incq %rdx /* Increment destination. */
|
|
- shrq $16, %rax /* Shift... */
|
|
- jmp 3b /* and look at next two bytes in %rax. */
|
|
+ dec %ecx
|
|
+ mov %al, (%rdi, %rcx)
|
|
+ jnz 3b
|
|
+
|
|
+ .p2align 4,, 3
|
|
+
|
|
+2:
|
|
+ mov %r8, %rax
|
|
+ ret
|
|
+
|
|
+#endif
|
|
+
|
|
+ .p2align 4
|
|
|
|
-4:
|
|
+LABEL(exitn):
|
|
#ifdef USE_AS_STPCPY
|
|
- movq %rdx, %rax /* Destination is return value. */
|
|
+ lea (%rdi, %rdx), %rax
|
|
#else
|
|
- movq %rdi, %rax /* Source is return value. */
|
|
+ mov %rdi, %rax
|
|
#endif
|
|
- retq
|
|
-END (BP_SYM (STRCPY))
|
|
-#ifndef USE_AS_STPCPY
|
|
-libc_hidden_builtin_def (strcpy)
|
|
+
|
|
+ ret
|
|
+
|
|
+END (STRCPY)
|
|
+#if !defined USE_AS_STPCPY && !defined USE_AS_STRNCPY
|
|
+libc_hidden_builtin_def (STRCPY)
|
|
#endif
|
|
Index: sysdeps/unix/sysv/linux/x86_64/dl-procinfo.c
|
|
===================================================================
|
|
--- sysdeps/unix/sysv/linux/x86_64/dl-procinfo.c.orig
|
|
+++ sysdeps/unix/sysv/linux/x86_64/dl-procinfo.c
|
|
@@ -1,5 +1,5 @@
|
|
#ifdef IS_IN_ldconfig
|
|
# include <sysdeps/i386/dl-procinfo.c>
|
|
#else
|
|
-# include <sysdeps/generic/dl-procinfo.c>
|
|
+# include <sysdeps/x86_64/dl-procinfo.c>
|
|
#endif
|
|
Index: sysdeps/x86_64/dl-procinfo.c
|
|
===================================================================
|
|
--- /dev/null
|
|
+++ sysdeps/x86_64/dl-procinfo.c
|
|
@@ -0,0 +1,108 @@
|
|
+/* Data for x86-64 version of processor capability information.
|
|
+ Copyright (C) 2004 Free Software Foundation, Inc.
|
|
+ This file is part of the GNU C Library.
|
|
+ Contributed by Andreas Jaeger <aj@suse.de>, 2004.
|
|
+
|
|
+ The GNU C Library is free software; you can redistribute it and/or
|
|
+ modify it under the terms of the GNU Lesser General Public
|
|
+ License as published by the Free Software Foundation; either
|
|
+ version 2.1 of the License, or (at your option) any later version.
|
|
+
|
|
+ The GNU C Library is distributed in the hope that it will be useful,
|
|
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
+ Lesser General Public License for more details.
|
|
+
|
|
+ You should have received a copy of the GNU Lesser General Public
|
|
+ License along with the GNU C Library; if not, write to the Free
|
|
+ Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
|
+ 02111-1307 USA. */
|
|
+
|
|
+/* This information must be kept in sync with the _DL_HWCAP_COUNT and
|
|
+ _DL_PLATFORM_COUNT definitions in procinfo.h.
|
|
+
|
|
+ If anything should be added here check whether the size of each string
|
|
+ is still ok with the given array size.
|
|
+
|
|
+ All the #ifdefs in the definitions ar equite irritating but
|
|
+ necessary if we want to avoid duplicating the information. There
|
|
+ are three different modes:
|
|
+
|
|
+ - PROCINFO_DECL is defined. This means we are only interested in
|
|
+ declarations.
|
|
+
|
|
+ - PROCINFO_DECL is not defined:
|
|
+
|
|
+ + if SHARED is defined the file is included in an array
|
|
+ initializer. The .element = { ... } syntax is needed.
|
|
+
|
|
+ + if SHARED is not defined a normal array initialization is
|
|
+ needed.
|
|
+ */
|
|
+
|
|
+#ifndef PROCINFO_CLASS
|
|
+#define PROCINFO_CLASS
|
|
+#endif
|
|
+
|
|
+ /* _dl_cache1size: size of L1 cache */
|
|
+#if !defined PROCINFO_DECL && defined SHARED
|
|
+ ._dl_cache1size
|
|
+#else
|
|
+PROCINFO_CLASS long int _dl_cache1size
|
|
+#endif
|
|
+#ifndef PROCINFO_DECL
|
|
+= 1024 * 64
|
|
+#endif
|
|
+#if !defined SHARED || defined PROCINFO_DECL
|
|
+;
|
|
+#else
|
|
+,
|
|
+#endif
|
|
+
|
|
+ /* _dl_cache1sizehalf: 1/2 size of L1 cache */
|
|
+#if !defined PROCINFO_DECL && defined SHARED
|
|
+ ._dl_cache1sizehalf
|
|
+#else
|
|
+PROCINFO_CLASS long int _dl_cache1sizehalf
|
|
+#endif
|
|
+#ifndef PROCINFO_DECL
|
|
+= 1024 * 64 / 2
|
|
+#endif
|
|
+#if !defined SHARED || defined PROCINFO_DECL
|
|
+;
|
|
+#else
|
|
+,
|
|
+#endif
|
|
+
|
|
+ /* _dl_cache2size: size of L2 cache */
|
|
+#if !defined PROCINFO_DECL && defined SHARED
|
|
+ ._dl_cache2size
|
|
+#else
|
|
+PROCINFO_CLASS long int _dl_cache2size
|
|
+#endif
|
|
+#ifndef PROCINFO_DECL
|
|
+= 1024 * 1024
|
|
+#endif
|
|
+#if !defined SHARED || defined PROCINFO_DECL
|
|
+;
|
|
+#else
|
|
+,
|
|
+#endif
|
|
+
|
|
+ /* _dl_cache2size: 1/2 size of L2 cache */
|
|
+#if !defined PROCINFO_DECL && defined SHARED
|
|
+ ._dl_cache2sizehalf
|
|
+#else
|
|
+PROCINFO_CLASS long int _dl_cache2sizehalf
|
|
+#endif
|
|
+#ifndef PROCINFO_DECL
|
|
+= 1024 * 1024 / 2
|
|
+#endif
|
|
+#if !defined SHARED || defined PROCINFO_DECL
|
|
+;
|
|
+#else
|
|
+,
|
|
+#endif
|
|
+
|
|
+#undef PROCINFO_DECL
|
|
+#undef PROCINFO_CLASS
|
|
Index: sysdeps/x86_64/elf/rtld-global-offsets.sym
|
|
===================================================================
|
|
--- /dev/null
|
|
+++ sysdeps/x86_64/elf/rtld-global-offsets.sym
|
|
@@ -0,0 +1,10 @@
|
|
+#define SHARED 1
|
|
+
|
|
+#include <ldsodefs.h>
|
|
+
|
|
+#define rtdl_global_offsetof(mem) offsetof (struct rtld_global_ro, mem)
|
|
+
|
|
+RTLD_GLOBAL_DL_CACHE1SIZE rtdl_global_offsetof (_dl_cache1size)
|
|
+RTLD_GLOBAL_DL_CACHE1SIZEHALF rtdl_global_offsetof (_dl_cache1sizehalf)
|
|
+RTLD_GLOBAL_DL_CACHE2SIZE rtdl_global_offsetof (_dl_cache2size)
|
|
+RTLD_GLOBAL_DL_CACHE2SIZEHALF rtdl_global_offsetof (_dl_cache2sizehalf)
|
|
Index: sysdeps/x86_64/memcmp.S
|
|
===================================================================
|
|
--- sysdeps/x86_64/memcmp.S.orig
|
|
+++ sysdeps/x86_64/memcmp.S
|
|
@@ -1,358 +1,442 @@
|
|
-/* memcmp with SSE2
|
|
- Copyright (C) 2009 Free Software Foundation, Inc.
|
|
- Contributed by Intel Corporation.
|
|
- This file is part of the GNU C Library.
|
|
-
|
|
- The GNU C Library is free software; you can redistribute it and/or
|
|
- modify it under the terms of the GNU Lesser General Public
|
|
- License as published by the Free Software Foundation; either
|
|
- version 2.1 of the License, or (at your option) any later version.
|
|
-
|
|
- The GNU C Library is distributed in the hope that it will be useful,
|
|
- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
- Lesser General Public License for more details.
|
|
-
|
|
- You should have received a copy of the GNU Lesser General Public
|
|
- License along with the GNU C Library; if not, write to the Free
|
|
- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
|
- 02111-1307 USA. */
|
|
-
|
|
-#include <sysdep.h>
|
|
-
|
|
- .text
|
|
-ENTRY (memcmp)
|
|
- test %rdx, %rdx
|
|
- jz L(finz)
|
|
- cmpq $1, %rdx
|
|
- jle L(finr1b)
|
|
- subq %rdi, %rsi
|
|
- movq %rdx, %r10
|
|
- cmpq $32, %r10
|
|
- jge L(gt32)
|
|
- /* Handle small chunks and last block of less than 32 bytes. */
|
|
-L(small):
|
|
- testq $1, %r10
|
|
- jz L(s2b)
|
|
- movzbl (%rdi), %eax
|
|
- movzbl (%rdi, %rsi), %edx
|
|
- subq $1, %r10
|
|
- je L(finz1)
|
|
- addq $1, %rdi
|
|
- subl %edx, %eax
|
|
- jnz L(exit)
|
|
-L(s2b):
|
|
- testq $2, %r10
|
|
- jz L(s4b)
|
|
- movzwl (%rdi), %eax
|
|
- movzwl (%rdi, %rsi), %edx
|
|
- subq $2, %r10
|
|
- je L(fin2_7)
|
|
- addq $2, %rdi
|
|
- cmpl %edx, %eax
|
|
- jnz L(fin2_7)
|
|
-L(s4b):
|
|
- testq $4, %r10
|
|
- jz L(s8b)
|
|
- movl (%rdi), %eax
|
|
- movl (%rdi, %rsi), %edx
|
|
- subq $4, %r10
|
|
- je L(fin2_7)
|
|
- addq $4, %rdi
|
|
- cmpl %edx, %eax
|
|
- jnz L(fin2_7)
|
|
-L(s8b):
|
|
- testq $8, %r10
|
|
- jz L(s16b)
|
|
- movq (%rdi), %rax
|
|
- movq (%rdi, %rsi), %rdx
|
|
- subq $8, %r10
|
|
- je L(fin2_7)
|
|
- addq $8, %rdi
|
|
- cmpq %rdx, %rax
|
|
- jnz L(fin2_7)
|
|
-L(s16b):
|
|
- movdqu (%rdi), %xmm1
|
|
- movdqu (%rdi, %rsi), %xmm0
|
|
- pcmpeqb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- xorl %eax, %eax
|
|
- subl $0xffff, %edx
|
|
- jz L(finz)
|
|
- bsfl %edx, %ecx
|
|
- leaq (%rdi, %rcx), %rcx
|
|
- movzbl (%rcx), %eax
|
|
- movzbl (%rsi, %rcx), %edx
|
|
- jmp L(finz1)
|
|
-
|
|
- .p2align 4,, 4
|
|
-L(finr1b):
|
|
- movzbl (%rdi), %eax
|
|
- movzbl (%rsi), %edx
|
|
-L(finz1):
|
|
- subl %edx, %eax
|
|
-L(exit):
|
|
- ret
|
|
-
|
|
- .p2align 4,, 4
|
|
-L(fin2_7):
|
|
- cmpq %rdx, %rax
|
|
- jz L(finz)
|
|
- movq %rax, %r11
|
|
- subq %rdx, %r11
|
|
- bsfq %r11, %rcx
|
|
- sarq $3, %rcx
|
|
- salq $3, %rcx
|
|
- sarq %cl, %rax
|
|
- movzbl %al, %eax
|
|
- sarq %cl, %rdx
|
|
- movzbl %dl, %edx
|
|
- subl %edx, %eax
|
|
- ret
|
|
-
|
|
- .p2align 4,, 4
|
|
-L(finz):
|
|
- xorl %eax, %eax
|
|
- ret
|
|
-
|
|
- /* For blocks bigger than 32 bytes
|
|
- 1. Advance one of the addr pointer to be 16B aligned.
|
|
- 2. Treat the case of both addr pointers aligned to 16B
|
|
- separately to avoid movdqu.
|
|
- 3. Handle any blocks of greater than 64 consecutive bytes with
|
|
- unrolling to reduce branches.
|
|
- 4. At least one addr pointer is 16B aligned, use memory version
|
|
- of pcmbeqb.
|
|
- */
|
|
- .p2align 4,, 4
|
|
-L(gt32):
|
|
- movq %rdx, %r11
|
|
- addq %rdi, %r11
|
|
- movq %rdi, %r8
|
|
-
|
|
- andq $15, %r8
|
|
- jz L(16am)
|
|
- /* Both pointers may be misaligned. */
|
|
- movdqu (%rdi), %xmm1
|
|
- movdqu (%rdi, %rsi), %xmm0
|
|
- pcmpeqb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- neg %r8
|
|
- leaq 16(%rdi, %r8), %rdi
|
|
-L(16am):
|
|
- /* Handle two 16B aligned pointers separately. */
|
|
- testq $15, %rsi
|
|
- jz L(ATR)
|
|
- testq $16, %rdi
|
|
- jz L(A32)
|
|
- movdqu (%rdi, %rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-L(A32):
|
|
- movq %r11, %r10
|
|
- andq $-32, %r10
|
|
- cmpq %r10, %rdi
|
|
- jge L(mt16)
|
|
- /* Pre-unroll to be ready for unrolled 64B loop. */
|
|
- testq $32, %rdi
|
|
- jz L(A64)
|
|
- movdqu (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- movdqu (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
-L(A64):
|
|
- movq %r11, %r10
|
|
- andq $-64, %r10
|
|
- cmpq %r10, %rdi
|
|
- jge L(mt32)
|
|
-
|
|
-L(A64main):
|
|
- movdqu (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- movdqu (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- movdqu (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- movdqu (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- cmpq %rdi, %r10
|
|
- jne L(A64main)
|
|
-
|
|
-L(mt32):
|
|
- movq %r11, %r10
|
|
- andq $-32, %r10
|
|
- cmpq %r10, %rdi
|
|
- jge L(mt16)
|
|
-
|
|
-L(A32main):
|
|
- movdqu (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- movdqu (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- cmpq %rdi, %r10
|
|
- jne L(A32main)
|
|
-L(mt16):
|
|
- subq %rdi, %r11
|
|
- je L(finz)
|
|
- movq %r11, %r10
|
|
- jmp L(small)
|
|
-
|
|
- .p2align 4,, 4
|
|
-L(neq):
|
|
- bsfl %edx, %ecx
|
|
- movzbl (%rdi, %rcx), %eax
|
|
- addq %rdi, %rsi
|
|
- movzbl (%rsi,%rcx), %edx
|
|
- jmp L(finz1)
|
|
-
|
|
- .p2align 4,, 4
|
|
-L(ATR):
|
|
- movq %r11, %r10
|
|
- andq $-32, %r10
|
|
- cmpq %r10, %rdi
|
|
- jge L(mt16)
|
|
- testq $16, %rdi
|
|
- jz L(ATR32)
|
|
-
|
|
- movdqa (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
- cmpq %rdi, %r10
|
|
- je L(mt16)
|
|
-
|
|
-L(ATR32):
|
|
- movq %r11, %r10
|
|
- andq $-64, %r10
|
|
- testq $32, %rdi
|
|
- jz L(ATR64)
|
|
-
|
|
- movdqa (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- movdqa (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
-L(ATR64):
|
|
- cmpq %rdi, %r10
|
|
- je L(mt32)
|
|
-
|
|
-L(ATR64main):
|
|
- movdqa (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- movdqa (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- movdqa (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- movdqa (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
- cmpq %rdi, %r10
|
|
- jne L(ATR64main)
|
|
-
|
|
- movq %r11, %r10
|
|
- andq $-32, %r10
|
|
- cmpq %r10, %rdi
|
|
- jge L(mt16)
|
|
-
|
|
-L(ATR32res):
|
|
- movdqa (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- movdqa (%rdi,%rsi), %xmm0
|
|
- pcmpeqb (%rdi), %xmm0
|
|
- pmovmskb %xmm0, %edx
|
|
- subl $0xffff, %edx
|
|
- jnz L(neq)
|
|
- addq $16, %rdi
|
|
-
|
|
- cmpq %r10, %rdi
|
|
- jne L(ATR32res)
|
|
-
|
|
- subq %rdi, %r11
|
|
- je L(finz)
|
|
- movq %r11, %r10
|
|
- jmp L(small)
|
|
- /* Align to 16byte to improve instruction fetch. */
|
|
- .p2align 4,, 4
|
|
-END(memcmp)
|
|
+# $Header: /K8_Projects/Glibc/amd64memcmp.S 4 10/06/03 10:57 Emenezes $
|
|
+
|
|
+# (c) 2002 Advanced Micro Devices, Inc.
|
|
+# YOUR USE OF THIS CODE IS SUBJECT TO THE TERMS
|
|
+# AND CONDITIONS OF THE GNU LESSER GENERAL PUBLIC
|
|
+# LICENSE FOUND IN THE "README" FILE THAT IS
|
|
+# INCLUDED WITH THIS FILE
|
|
+
|
|
+#include "sysdep.h"
|
|
+#include <rtld-global-offsets.h>
|
|
+
|
|
+#ifdef PIC
|
|
+ .globl _rtld_local_ro
|
|
+ .hidden _rtld_local_ro
|
|
+ .set _rtld_local_ro,_rtld_global_ro
|
|
+#endif
|
|
+
|
|
+ .text
|
|
+
|
|
+ENTRY (memcmp) # (const void *, const void*, size_t)
|
|
+
|
|
+L(memcmptry1):
|
|
+ cmp $8, %rdx
|
|
+ jae L(memcmp1after)
|
|
+
|
|
+L(memcmp1): # 1-byte
|
|
+ test %rdx, %rdx
|
|
+ mov $0, %eax
|
|
+ jz L(memcmpexit)
|
|
+
|
|
+L(memcmp1loop):
|
|
+ movzbl (%rdi), %eax
|
|
+ movzbl (%rsi), %ecx
|
|
+ sub %ecx, %eax
|
|
+ jnz L(memcmpexit)
|
|
+
|
|
+ dec %rdx
|
|
+
|
|
+ lea 1 (%rdi), %rdi
|
|
+ lea 1 (%rsi), %rsi
|
|
+
|
|
+ jnz L(memcmp1loop)
|
|
+
|
|
+L(memcmpexit):
|
|
+ rep
|
|
+ ret
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+L(memcmp1after):
|
|
+
|
|
+L(memcmp8try):
|
|
+ cmp $32, %rdx
|
|
+ jae L(memcmp8after)
|
|
+
|
|
+L(memcmp8): # 8-byte
|
|
+ mov %edx, %ecx
|
|
+ shr $3, %ecx
|
|
+ jz L(memcmp1)
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+L(memcmp8loop):
|
|
+ mov (%rsi), %rax
|
|
+ cmp (%rdi), %rax
|
|
+ jne L(memcmp1)
|
|
+
|
|
+ sub $8, %rdx
|
|
+ dec %ecx
|
|
+
|
|
+ lea 8 (%rsi), %rsi
|
|
+ lea 8 (%rdi), %rdi
|
|
+
|
|
+ jnz L(memcmp8loop)
|
|
+
|
|
+L(memcmp8skip):
|
|
+ and $7, %edx
|
|
+ jnz L(memcmp1)
|
|
+
|
|
+ xor %eax, %eax
|
|
+ ret
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+L(memcmp8after):
|
|
+
|
|
+L(memcmp32try):
|
|
+ cmp $2048, %rdx
|
|
+ ja L(memcmp32after)
|
|
+
|
|
+L(memcmp32): # 32-byte
|
|
+ mov %edx, %ecx
|
|
+ shr $5, %ecx
|
|
+ jz L(memcmp8)
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+L(memcmp32loop):
|
|
+ mov (%rsi), %rax
|
|
+ mov 8 (%rsi), %r8
|
|
+ mov 16 (%rsi), %r9
|
|
+ mov 24 (%rsi), %r10
|
|
+ sub (%rdi), %rax
|
|
+ sub 8 (%rdi), %r8
|
|
+ sub 16 (%rdi), %r9
|
|
+ sub 24 (%rdi), %r10
|
|
+
|
|
+ or %rax, %r8
|
|
+ or %r9, %r10
|
|
+ or %r8, %r10
|
|
+ jnz L(memcmp8)
|
|
+
|
|
+ sub $32, %rdx
|
|
+ dec %ecx
|
|
+
|
|
+ lea 32 (%rsi), %rsi
|
|
+ lea 32 (%rdi), %rdi
|
|
+
|
|
+ jnz L(memcmp32loop)
|
|
+
|
|
+L(memcmp32skip):
|
|
+ and $31, %edx
|
|
+ jnz L(memcmp8)
|
|
+
|
|
+ xor %eax, %eax
|
|
+ ret
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+L(memcmp32after):
|
|
+
|
|
+#ifdef PIC
|
|
+ mov _rtld_local_ro@GOTPCREL(%rip), %r8
|
|
+ mov RTLD_GLOBAL_DL_CACHE1SIZEHALF(%r8), %r9
|
|
+#else
|
|
+ mov _dl_cache1sizehalf, %r9
|
|
+#endif
|
|
+ prefetcht0 (%r9)
|
|
+
|
|
+
|
|
+.alignsrctry:
|
|
+ mov %esi, %r8d # align by source
|
|
+
|
|
+ and $7, %r8d
|
|
+ jz .alignsrcafter # not unaligned
|
|
+
|
|
+.alignsrc: # align
|
|
+ lea -8 (%r8, %rdx), %rdx
|
|
+ sub $8, %r8d
|
|
+
|
|
+# .p2align 4
|
|
+
|
|
+.alignsrcloop:
|
|
+ movzbl (%rdi), %eax
|
|
+ movzbl (%rsi), %ecx
|
|
+ sub %ecx, %eax
|
|
+ jnz L(memcmpexit)
|
|
+
|
|
+ inc %r8d
|
|
+
|
|
+ lea 1 (%rdi), %rdi
|
|
+ lea 1 (%rsi), %rsi
|
|
+
|
|
+ jnz .alignsrcloop
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+.alignsrcafter:
|
|
+
|
|
+
|
|
+L(memcmp64try):
|
|
+#ifdef PIC
|
|
+ mov _rtld_local_ro@GOTPCREL(%rip), %r8
|
|
+ mov RTLD_GLOBAL_DL_CACHE1SIZEHALF(%r8), %rcx
|
|
+#else
|
|
+ mov _dl_cache1sizehalf, %rcx
|
|
+#endif
|
|
+ cmp %rdx, %rcx
|
|
+ cmova %rdx, %rcx
|
|
+
|
|
+L(memcmp64): # 64-byte
|
|
+ shr $6, %rcx
|
|
+ jz L(memcmp32)
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+L(memcmp64loop):
|
|
+ mov (%rsi), %rax
|
|
+ mov 8 (%rsi), %r8
|
|
+ sub (%rdi), %rax
|
|
+ sub 8 (%rdi), %r8
|
|
+ or %r8, %rax
|
|
+
|
|
+ mov 16 (%rsi), %r9
|
|
+ mov 24 (%rsi), %r10
|
|
+ sub 16 (%rdi), %r9
|
|
+ sub 24 (%rdi), %r10
|
|
+ or %r10, %r9
|
|
+
|
|
+ or %r9, %rax
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ mov 32 (%rsi), %rax
|
|
+ mov 40 (%rsi), %r8
|
|
+ sub 32 (%rdi), %rax
|
|
+ sub 40 (%rdi), %r8
|
|
+ or %r8, %rax
|
|
+
|
|
+ mov 48 (%rsi), %r9
|
|
+ mov 56 (%rsi), %r10
|
|
+ sub 48 (%rdi), %r9
|
|
+ sub 56 (%rdi), %r10
|
|
+ or %r10, %r9
|
|
+
|
|
+ or %r9, %rax
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ lea 64 (%rsi), %rsi
|
|
+ lea 64 (%rdi), %rdi
|
|
+
|
|
+ sub $64, %rdx
|
|
+ dec %rcx
|
|
+ jnz L(memcmp64loop)
|
|
+
|
|
+# .p2align 4
|
|
+
|
|
+L(memcmp64skip):
|
|
+ cmp $2048, %rdx
|
|
+ ja L(memcmp64after)
|
|
+
|
|
+ test %edx, %edx
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ xor %eax, %eax
|
|
+ ret
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+L(memcmp64after):
|
|
+
|
|
+L(memcmppretry):
|
|
+
|
|
+L(memcmppre): # 64-byte prefetching
|
|
+#ifdef PIC
|
|
+ mov _rtld_local_ro@GOTPCREL(%rip), %r8
|
|
+ mov RTLD_GLOBAL_DL_CACHE2SIZEHALF(%r8), %rcx
|
|
+#else
|
|
+ mov _dl_cache2sizehalf, %rcx
|
|
+#endif
|
|
+ cmp %rdx, %rcx
|
|
+ cmova %rdx, %rcx
|
|
+
|
|
+ shr $6, %rcx
|
|
+ jz L(memcmppreskip)
|
|
+
|
|
+ prefetcht0 512 (%rsi)
|
|
+ prefetcht0 512 (%rdi)
|
|
+
|
|
+ mov (%rsi), %rax
|
|
+ mov 8 (%rsi), %r9
|
|
+ mov 16 (%rsi), %r10
|
|
+ mov 24 (%rsi), %r11
|
|
+ sub (%rdi), %rax
|
|
+ sub 8 (%rdi), %r9
|
|
+ sub 16 (%rdi), %r10
|
|
+ sub 24 (%rdi), %r11
|
|
+
|
|
+ or %r9, %rax
|
|
+ or %r11, %r10
|
|
+ or %r10, %rax
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ mov 32 (%rsi), %rax
|
|
+ mov 40 (%rsi), %r9
|
|
+ mov 48 (%rsi), %r10
|
|
+ mov 56 (%rsi), %r11
|
|
+ sub 32 (%rdi), %rax
|
|
+ sub 40 (%rdi), %r9
|
|
+ sub 48 (%rdi), %r10
|
|
+ sub 56 (%rdi), %r11
|
|
+
|
|
+ or %r9, %rax
|
|
+ or %r11, %r10
|
|
+ or %r10, %rax
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ lea 64 (%rsi), %rsi
|
|
+ lea 64 (%rdi), %rdi
|
|
+
|
|
+ sub $64, %rdx
|
|
+ dec %rcx
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+L(memcmppreloop):
|
|
+ prefetcht0 512 (%rsi)
|
|
+ prefetcht0 512 (%rdi)
|
|
+
|
|
+ mov (%rsi), %rax
|
|
+ mov 8 (%rsi), %r9
|
|
+ mov 16 (%rsi), %r10
|
|
+ mov 24 (%rsi), %r11
|
|
+ sub (%rdi), %rax
|
|
+ sub 8 (%rdi), %r9
|
|
+ sub 16 (%rdi), %r10
|
|
+ sub 24 (%rdi), %r11
|
|
+
|
|
+ or %r9, %rax
|
|
+ or %r11, %r10
|
|
+ or %r10, %rax
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ mov 32 (%rsi), %rax
|
|
+ mov 40 (%rsi), %r9
|
|
+ mov 48 (%rsi), %r10
|
|
+ mov 56 (%rsi), %r11
|
|
+ sub 32 (%rdi), %rax
|
|
+ sub 40 (%rdi), %r9
|
|
+ sub 48 (%rdi), %r10
|
|
+ sub 56 (%rdi), %r11
|
|
+
|
|
+ or %r9, %rax
|
|
+ or %r11, %r10
|
|
+ or %r10, %rax
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ lea 64 (%rsi), %rsi
|
|
+ lea 64 (%rdi), %rdi
|
|
+
|
|
+ sub $64, %rdx
|
|
+ dec %rcx
|
|
+ jnz L(memcmppreloop)
|
|
+
|
|
+# .p2align 4
|
|
+
|
|
+L(memcmppreskip):
|
|
+ cmp $2048, %rdx
|
|
+ ja L(memcmppreafter)
|
|
+
|
|
+ test %edx, %edx
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ xor %eax, %eax
|
|
+ ret
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+L(memcmppreafter):
|
|
+
|
|
+L(memcmp128try):
|
|
+
|
|
+L(memcmp128): # 128-byte
|
|
+ mov %rdx, %rcx
|
|
+ shr $7, %rcx
|
|
+ jz L(memcmp128skip)
|
|
+
|
|
+ .p2align 4
|
|
+
|
|
+L(memcmp128loop):
|
|
+ prefetcht0 512 (%rsi)
|
|
+ prefetcht0 512 (%rdi)
|
|
+
|
|
+ mov (%rsi), %rax
|
|
+ mov 8 (%rsi), %r8
|
|
+ sub (%rdi), %rax
|
|
+ sub 8 (%rdi), %r8
|
|
+ mov 16 (%rsi), %r9
|
|
+ mov 24 (%rsi), %r10
|
|
+ sub 16 (%rdi), %r9
|
|
+ sub 24 (%rdi), %r10
|
|
+
|
|
+ or %r8, %rax
|
|
+ or %r9, %r10
|
|
+ or %r10, %rax
|
|
+
|
|
+ mov 32 (%rsi), %r8
|
|
+ mov 40 (%rsi), %r9
|
|
+ sub 32 (%rdi), %r8
|
|
+ sub 40 (%rdi), %r9
|
|
+ mov 48 (%rsi), %r10
|
|
+ mov 56 (%rsi), %r11
|
|
+ sub 48 (%rdi), %r10
|
|
+ sub 56 (%rdi), %r11
|
|
+
|
|
+ or %r9, %r8
|
|
+ or %r11, %r10
|
|
+ or %r10, %r8
|
|
+
|
|
+ or %r8, %rax
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ prefetcht0 576 (%rsi)
|
|
+ prefetcht0 576 (%rdi)
|
|
+
|
|
+ mov 64 (%rsi), %rax
|
|
+ mov 72 (%rsi), %r8
|
|
+ sub 64 (%rdi), %rax
|
|
+ sub 72 (%rdi), %r8
|
|
+ mov 80 (%rsi), %r9
|
|
+ mov 88 (%rsi), %r10
|
|
+ sub 80 (%rdi), %r9
|
|
+ sub 88 (%rdi), %r10
|
|
+
|
|
+ or %r8, %rax
|
|
+ or %r9, %r10
|
|
+ or %r10, %rax
|
|
+
|
|
+ mov 96 (%rsi), %r8
|
|
+ mov 104 (%rsi), %r9
|
|
+ sub 96 (%rdi), %r8
|
|
+ sub 104 (%rdi), %r9
|
|
+ mov 112 (%rsi), %r10
|
|
+ mov 120 (%rsi), %r11
|
|
+ sub 112 (%rdi), %r10
|
|
+ sub 120 (%rdi), %r11
|
|
+
|
|
+ or %r9, %r8
|
|
+ or %r11, %r10
|
|
+ or %r10, %r8
|
|
+
|
|
+ or %r8, %rax
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ sub $128, %rdx
|
|
+ dec %rcx
|
|
+
|
|
+ lea 128 (%rsi), %rsi
|
|
+ lea 128 (%rdi), %rdi
|
|
+
|
|
+ jnz L(memcmp128loop)
|
|
+
|
|
+L(memcmp128skip):
|
|
+ and $127, %edx
|
|
+ jnz L(memcmp32)
|
|
+
|
|
+ xor %eax, %eax
|
|
+ ret
|
|
+
|
|
+END (memcmp)
|
|
|
|
#undef bcmp
|
|
weak_alias (memcmp, bcmp)
|
|
Index: sysdeps/x86_64/strcmp.S
|
|
===================================================================
|
|
--- sysdeps/x86_64/strcmp.S.orig
|
|
+++ sysdeps/x86_64/strcmp.S
|
|
@@ -1,2108 +1,490 @@
|
|
-/* Highly optimized version for x86-64.
|
|
- Copyright (C) 1999, 2000, 2002, 2003, 2005, 2009
|
|
- Free Software Foundation, Inc.
|
|
- This file is part of the GNU C Library.
|
|
- Based on i686 version contributed by Ulrich Drepper
|
|
- <drepper@cygnus.com>, 1999.
|
|
- Updated with SSE2 support contributed by Intel Corporation.
|
|
-
|
|
- The GNU C Library is free software; you can redistribute it and/or
|
|
- modify it under the terms of the GNU Lesser General Public
|
|
- License as published by the Free Software Foundation; either
|
|
- version 2.1 of the License, or (at your option) any later version.
|
|
-
|
|
- The GNU C Library is distributed in the hope that it will be useful,
|
|
- but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
- Lesser General Public License for more details.
|
|
-
|
|
- You should have received a copy of the GNU Lesser General Public
|
|
- License along with the GNU C Library; if not, write to the Free
|
|
- Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
|
|
- 02111-1307 USA. */
|
|
|
|
#include <sysdep.h>
|
|
#include "asm-syntax.h"
|
|
#include "bp-sym.h"
|
|
#include "bp-asm.h"
|
|
|
|
-#undef UPDATE_STRNCMP_COUNTER
|
|
-
|
|
#ifndef LABEL
|
|
#define LABEL(l) L(l)
|
|
#endif
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
-/* Since the counter, %r11, is unsigned, we branch to strcmp_exitz
|
|
- if the new counter > the old one or is 0. */
|
|
-# define UPDATE_STRNCMP_COUNTER \
|
|
- /* calculate left number to compare */ \
|
|
- lea -16(%rcx, %r11), %r9; \
|
|
- cmp %r9, %r11; \
|
|
- jb LABEL(strcmp_exitz); \
|
|
- test %r9, %r9; \
|
|
- je LABEL(strcmp_exitz); \
|
|
- mov %r9, %r11
|
|
-
|
|
-#else
|
|
-# define UPDATE_STRNCMP_COUNTER
|
|
+#ifndef USE_AS_STRNCMP
|
|
# ifndef STRCMP
|
|
# define STRCMP strcmp
|
|
# endif
|
|
#endif
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
.text
|
|
-#else
|
|
- .section .text.ssse3,"ax",@progbits
|
|
-#endif
|
|
|
|
-ENTRY (BP_SYM (STRCMP))
|
|
-#ifdef NOT_IN_libc
|
|
-/* Simple version since we can't use SSE registers in ld.so. */
|
|
-L(oop): movb (%rdi), %al
|
|
- cmpb (%rsi), %al
|
|
- jne L(neq)
|
|
- incq %rdi
|
|
- incq %rsi
|
|
- testb %al, %al
|
|
- jnz L(oop)
|
|
-
|
|
- xorl %eax, %eax
|
|
- ret
|
|
-
|
|
-L(neq): movl $1, %eax
|
|
- movl $-1, %ecx
|
|
- cmovbl %ecx, %eax
|
|
- ret
|
|
-END (BP_SYM (STRCMP))
|
|
-#else /* NOT_IN_libc */
|
|
-/*
|
|
- * This implementation uses SSE to compare up to 16 bytes at a time.
|
|
- */
|
|
-#ifdef USE_AS_STRNCMP
|
|
- test %rdx, %rdx
|
|
- je LABEL(strcmp_exitz)
|
|
- cmp $1, %rdx
|
|
- je LABEL(Byte0)
|
|
- mov %rdx, %r11
|
|
-#endif
|
|
- mov %esi, %ecx
|
|
- mov %edi, %eax
|
|
-/* Use 64bit AND here to avoid long NOP padding. */
|
|
- and $0x3f, %rcx /* rsi alignment in cache line */
|
|
- and $0x3f, %rax /* rdi alignment in cache line */
|
|
- cmp $0x30, %ecx
|
|
- ja LABEL(crosscache) /* rsi: 16-byte load will cross cache line */
|
|
- cmp $0x30, %eax
|
|
- ja LABEL(crosscache) /* rdi: 16-byte load will cross cache line */
|
|
- movlpd (%rdi), %xmm1
|
|
- movlpd (%rsi), %xmm2
|
|
- movhpd 8(%rdi), %xmm1
|
|
- movhpd 8(%rsi), %xmm2
|
|
- pxor %xmm0, %xmm0 /* clear %xmm0 for null char checks */
|
|
- pcmpeqb %xmm1, %xmm0 /* Any null chars? */
|
|
- pcmpeqb %xmm2, %xmm1 /* compare first 16 bytes for equality */
|
|
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx /* if first 16 bytes are same, edx == 0xffff */
|
|
- jnz LABEL(less16bytes) /* If not, find different value or null char */
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz) /* finish comparision */
|
|
-#endif
|
|
- add $16, %rsi /* prepare to search next 16 bytes */
|
|
- add $16, %rdi /* prepare to search next 16 bytes */
|
|
+ENTRY (STRCMP) # (const char *, const char *)
|
|
|
|
- /*
|
|
- * Determine source and destination string offsets from 16-byte alignment.
|
|
- * Use relative offset difference between the two to determine which case
|
|
- * below to use.
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(crosscache):
|
|
- and $0xfffffffffffffff0, %rsi /* force %rsi is 16 byte aligned */
|
|
- and $0xfffffffffffffff0, %rdi /* force %rdi is 16 byte aligned */
|
|
- mov $0xffff, %edx /* for equivalent offset */
|
|
- xor %r8d, %r8d
|
|
- and $0xf, %ecx /* offset of rsi */
|
|
- and $0xf, %eax /* offset of rdi */
|
|
- cmp %eax, %ecx
|
|
- je LABEL(ashr_0) /* rsi and rdi relative offset same */
|
|
- ja LABEL(bigger)
|
|
- mov %edx, %r8d /* r8d is offset flag for exit tail */
|
|
- xchg %ecx, %eax
|
|
- xchg %rsi, %rdi
|
|
-LABEL(bigger):
|
|
- lea 15(%rax), %r9
|
|
- sub %rcx, %r9
|
|
- lea LABEL(unaligned_table)(%rip), %r10
|
|
- movslq (%r10, %r9,4), %r9
|
|
- lea (%r10, %r9), %r10
|
|
- jmp *%r10 /* jump to corresponding case */
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_0
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(0~15) n(0~15) 15(15+ n-n) ashr_0
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_0):
|
|
-
|
|
- movdqa (%rsi), %xmm1
|
|
- pxor %xmm0, %xmm0 /* clear %xmm0 for null char check */
|
|
- pcmpeqb %xmm1, %xmm0 /* Any null chars? */
|
|
- pcmpeqb (%rdi), %xmm1 /* compare 16 bytes for equality */
|
|
- psubb %xmm0, %xmm1 /* packed sub of comparison results*/
|
|
- pmovmskb %xmm1, %r9d
|
|
- shr %cl, %edx /* adjust 0xffff for offset */
|
|
- shr %cl, %r9d /* adjust for 16-byte offset */
|
|
- sub %r9d, %edx
|
|
- /*
|
|
- * edx must be the same with r9d if in left byte (16-rcx) is equal to
|
|
- * the start from (16-rax) and no null char was seen.
|
|
- */
|
|
- jne LABEL(less32bytes) /* mismatch or null char */
|
|
- UPDATE_STRNCMP_COUNTER
|
|
- mov $16, %rcx
|
|
- mov $16, %r9
|
|
- pxor %xmm0, %xmm0 /* clear xmm0, may have changed above */
|
|
-
|
|
- /*
|
|
- * Now both strings are aligned at 16-byte boundary. Loop over strings
|
|
- * checking 32-bytes per iteration.
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_0):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit) /* mismatch or null char seen */
|
|
+ xor %ecx, %ecx
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
- add $16, %rcx
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
- add $16, %rcx
|
|
- jmp LABEL(loop_ashr_0)
|
|
+#ifdef USE_AS_STRNCMP // (const char *, const char *, size_t)
|
|
+ mov %r14, -8 (%rsp)
|
|
+ mov %rdx, %r14
|
|
|
|
-/*
|
|
- * The following cases will be handled by ashr_1
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(15) n -15 0(15 +(n-15) - n) ashr_1
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_1):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0 /* Any null chars? */
|
|
- pslldq $15, %xmm2 /* shift first string to align with second */
|
|
- pcmpeqb %xmm1, %xmm2 /* compare 16 bytes for equality */
|
|
- psubb %xmm0, %xmm2 /* packed sub of comparison results*/
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx /* adjust 0xffff for offset */
|
|
- shr %cl, %r9d /* adjust for 16-byte offset */
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes) /* mismatch or null char seen */
|
|
- movdqa (%rdi), %xmm3
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads*/
|
|
- mov $1, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 1(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_1):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_1) /* cross page boundary */
|
|
-
|
|
-LABEL(gobble_ashr_1):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4 /* store for next cycle */
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $1, %xmm3
|
|
- pslldq $15, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $1, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
-
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ test %rdx, %rdx
|
|
+ mov %edx, %eax
|
|
+ jz LABEL(exitz)
|
|
#endif
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_1) /* cross page boundary */
|
|
+LABEL(aligntry):
|
|
+ mov %rsi, %r8 # align by "source"
|
|
+ and $8 - 1, %r8 # between 0 and 8 characters compared
|
|
+ jz LABEL(alignafter)
|
|
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4 /* store for next cycle */
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $1, %xmm3
|
|
- pslldq $15, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $1, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+LABEL(align):
|
|
+ sub $8, %r8
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_1)
|
|
-
|
|
- /*
|
|
- * Nibble avoids loads across page boundary. This is to avoid a potential
|
|
- * access into unmapped memory.
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_1):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char*/
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xfffe, %edx
|
|
- jnz LABEL(ashr_1_exittail) /* find null char*/
|
|
+ .p2align 4
|
|
+
|
|
+LABEL(alignloop):
|
|
+ mov (%rsi, %rcx), %al
|
|
+ mov (%rdi, %rcx), %dl
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- cmp $14, %r11
|
|
- jbe LABEL(ashr_1_exittail)
|
|
+ dec %r14
|
|
+ jl LABEL(exitafter)
|
|
#endif
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10 /* substract 4K from %r10 */
|
|
- jmp LABEL(gobble_ashr_1)
|
|
-
|
|
- /*
|
|
- * Once find null char, determine if there is a string mismatch
|
|
- * before the null char.
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_1_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $1, %xmm0
|
|
- psrldq $1, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_2
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(14~15) n -14 1(15 +(n-14) - n) ashr_2
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_2):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $14, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $2, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 2(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_2):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_2)
|
|
-
|
|
-LABEL(gobble_ashr_2):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $2, %xmm3
|
|
- pslldq $14, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $2, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ cmp %dl, %al # check if same character
|
|
+ jne LABEL(exitafter)
|
|
+ test %al, %al # check if character a NUL
|
|
+ jz LABEL(exitafter)
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ inc %ecx
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ inc %r8
|
|
+ jnz LABEL(alignloop)
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_2) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $2, %xmm3
|
|
- pslldq $14, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $2, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ .p2align 4
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+LABEL(alignafter):
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_2)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_2):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xfffc, %edx
|
|
- jnz LABEL(ashr_2_exittail)
|
|
+ mov %r15, -32 (%rsp)
|
|
+ mov %rbp, -24 (%rsp)
|
|
+ mov %rbx, -16 (%rsp)
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- cmp $13, %r11
|
|
- jbe LABEL(ashr_2_exittail)
|
|
-#endif
|
|
+LABEL(pagealigntry): # page align by "destination"
|
|
+ mov $4096, %r15d # page size is 4096
|
|
+ lea (%rdi, %rcx), %ebp
|
|
+ and $4095, %ebp # page mask
|
|
+ sub %r15d, %ebp
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_2)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_2_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $2, %xmm0
|
|
- psrldq $2, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_3
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(13~15) n -13 2(15 +(n-13) - n) ashr_3
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_3):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $13, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $3, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 3(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_3):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_3)
|
|
-
|
|
-LABEL(gobble_ashr_3):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $3, %xmm3
|
|
- pslldq $13, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $3, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+LABEL(64): # 64-byte
|
|
+ mov $0xfefefefefefefeff, %rbx # magic number
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ .p2align 4
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+LABEL(64loop):
|
|
+ add $64, %ebp # check if "destination" crosses a page unevenly
|
|
+ jle LABEL(64gobble)
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_3) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $3, %xmm3
|
|
- pslldq $13, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $3, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ sub %r15d, %ebp
|
|
+ lea 64 (%rcx), %r8
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ .p2align 4
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_3)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_3):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xfff8, %edx
|
|
- jnz LABEL(ashr_3_exittail)
|
|
+LABEL(64nibble):
|
|
+ mov (%rsi, %rcx), %al
|
|
+ mov (%rdi, %rcx), %dl
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- cmp $12, %r11
|
|
- jbe LABEL(ashr_3_exittail)
|
|
+ dec %r14
|
|
+ jl .exit
|
|
#endif
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_3)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_3_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $3, %xmm0
|
|
- psrldq $3, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_4
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(12~15) n -12 3(15 +(n-12) - n) ashr_4
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_4):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $12, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $4, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 4(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_4):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_4)
|
|
-
|
|
-LABEL(gobble_ashr_4):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $4, %xmm3
|
|
- pslldq $12, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $4, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ cmp %dl, %al # check if same character
|
|
+ jne .exit
|
|
+ test %al, %al # check if character a NUL
|
|
+ jz .exit
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ inc %ecx
|
|
+
|
|
+ cmp %ecx, %r8d
|
|
+ ja LABEL(64nibble)
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ .p2align 4
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_4) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $4, %xmm3
|
|
- pslldq $12, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $4, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+LABEL(64gobble):
|
|
+ mov (%rsi, %rcx), %rax
|
|
+ mov (%rdi, %rcx), %rdx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ sub $8, %r14
|
|
+ jl LABEL(tail)
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_4)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_4):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xfff0, %edx
|
|
- jnz LABEL(ashr_4_exittail)
|
|
+ mov %rbx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- cmp $11, %r11
|
|
- jbe LABEL(ashr_4_exittail)
|
|
-#endif
|
|
+ mov %rbx, %r9
|
|
+ add %rdx, %r9
|
|
+ sbb %r11, %r11
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_4)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_4_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $4, %xmm0
|
|
- psrldq $4, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_5
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(11~15) n - 11 4(15 +(n-11) - n) ashr_5
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_5):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $11, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $5, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 5(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_5):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_5)
|
|
-
|
|
-LABEL(gobble_ashr_5):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $5, %xmm3
|
|
- pslldq $11, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $5, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ xor %rax, %r8
|
|
+ or %rbx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ xor %rdx, %r9
|
|
+ or %rbx, %r9
|
|
+ sub %r11, %r9
|
|
+ jnz LABEL(tail)
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %rdx, %rax
|
|
+ jne LABEL(tail)
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_5) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $5, %xmm3
|
|
- pslldq $11, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $5, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ mov 8 (%rsi, %rcx), %rax
|
|
+ mov 8 (%rdi, %rcx), %rdx
|
|
+ add $8, %ecx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ sub $8, %r14
|
|
+ jl LABEL(tail)
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_5)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_5):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xffe0, %edx
|
|
- jnz LABEL(ashr_5_exittail)
|
|
+ mov %rbx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- cmp $10, %r11
|
|
- jbe LABEL(ashr_5_exittail)
|
|
-#endif
|
|
+ mov %rbx, %r9
|
|
+ add %rdx, %r9
|
|
+ sbb %r11, %r11
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_5)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_5_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $5, %xmm0
|
|
- psrldq $5, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_6
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(10~15) n - 10 5(15 +(n-10) - n) ashr_6
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_6):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $10, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $6, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 6(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_6):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_6)
|
|
-
|
|
-LABEL(gobble_ashr_6):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $6, %xmm3
|
|
- pslldq $10, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $6, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ xor %rax, %r8
|
|
+ or %rbx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ xor %rdx, %r9
|
|
+ or %rbx, %r9
|
|
+ sub %r11, %r9
|
|
+ jnz LABEL(tail)
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %rdx, %rax
|
|
+ jne LABEL(tail)
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_6) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $6, %xmm3
|
|
- pslldq $10, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $6, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ mov 8 (%rsi, %rcx), %rax
|
|
+ mov 8 (%rdi, %rcx), %rdx
|
|
+ add $8, %ecx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ sub $8, %r14
|
|
+ jl LABEL(tail)
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_6)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_6):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xffc0, %edx
|
|
- jnz LABEL(ashr_6_exittail)
|
|
+ mov %rbx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- cmp $9, %r11
|
|
- jbe LABEL(ashr_6_exittail)
|
|
-#endif
|
|
+ mov %rbx, %r9
|
|
+ add %rdx, %r9
|
|
+ sbb %r11, %r11
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_6)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_6_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $6, %xmm0
|
|
- psrldq $6, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_7
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(9~15) n - 9 6(15 +(n - 9) - n) ashr_7
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_7):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $9, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $7, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 7(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_7):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_7)
|
|
-
|
|
-LABEL(gobble_ashr_7):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $7, %xmm3
|
|
- pslldq $9, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $7, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ xor %rax, %r8
|
|
+ or %rbx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ xor %rdx, %r9
|
|
+ or %rbx, %r9
|
|
+ sub %r11, %r9
|
|
+ jnz LABEL(tail)
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %rdx, %rax
|
|
+ jne LABEL(tail)
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_7) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $7, %xmm3
|
|
- pslldq $9, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $7, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ mov 8 (%rsi, %rcx), %rax
|
|
+ mov 8 (%rdi, %rcx), %rdx
|
|
+ add $8, %ecx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ sub $8, %r14
|
|
+ jl LABEL(tail)
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_7)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_7):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xff80, %edx
|
|
- jnz LABEL(ashr_7_exittail)
|
|
+ mov %rbx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- cmp $8, %r11
|
|
- jbe LABEL(ashr_7_exittail)
|
|
-#endif
|
|
+ mov %rbx, %r9
|
|
+ add %rdx, %r9
|
|
+ sbb %r11, %r11
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_7)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_7_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $7, %xmm0
|
|
- psrldq $7, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_8
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(8~15) n - 8 7(15 +(n - 8) - n) ashr_8
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_8):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $8, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $8, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 8(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_8):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_8)
|
|
-
|
|
-LABEL(gobble_ashr_8):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $8, %xmm3
|
|
- pslldq $8, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $8, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ xor %rax, %r8
|
|
+ or %rbx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ xor %rdx, %r9
|
|
+ or %rbx, %r9
|
|
+ sub %r11, %r9
|
|
+ jnz LABEL(tail)
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %rdx, %rax
|
|
+ jne LABEL(tail)
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_8) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $8, %xmm3
|
|
- pslldq $8, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $8, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ mov 8 (%rsi, %rcx), %rax
|
|
+ mov 8 (%rdi, %rcx), %rdx
|
|
+ add $8, %ecx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ sub $8, %r14
|
|
+ jl LABEL(tail)
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_8)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_8):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xff00, %edx
|
|
- jnz LABEL(ashr_8_exittail)
|
|
+ mov %rbx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- cmp $7, %r11
|
|
- jbe LABEL(ashr_8_exittail)
|
|
-#endif
|
|
+ mov %rbx, %r9
|
|
+ add %rdx, %r9
|
|
+ sbb %r11, %r11
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_8)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_8_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $8, %xmm0
|
|
- psrldq $8, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_9
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(7~15) n - 7 8(15 +(n - 7) - n) ashr_9
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_9):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $7, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $9, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 9(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_9):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_9)
|
|
-
|
|
-LABEL(gobble_ashr_9):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $9, %xmm3
|
|
- pslldq $7, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $9, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ xor %rax, %r8
|
|
+ or %rbx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ xor %rdx, %r9
|
|
+ or %rbx, %r9
|
|
+ sub %r11, %r9
|
|
+ jnz LABEL(tail)
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %rdx, %rax
|
|
+ jne LABEL(tail)
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_9) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $9, %xmm3
|
|
- pslldq $7, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $9, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ mov 8 (%rsi, %rcx), %rax
|
|
+ mov 8 (%rdi, %rcx), %rdx
|
|
+ add $8, %ecx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ sub $8, %r14
|
|
+ jl LABEL(tail)
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3 /* store for next cycle */
|
|
- jmp LABEL(loop_ashr_9)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_9):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xfe00, %edx
|
|
- jnz LABEL(ashr_9_exittail)
|
|
+ mov %rbx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- cmp $6, %r11
|
|
- jbe LABEL(ashr_9_exittail)
|
|
-#endif
|
|
+ mov %rbx, %r9
|
|
+ add %rdx, %r9
|
|
+ sbb %r11, %r11
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_9)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_9_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $9, %xmm0
|
|
- psrldq $9, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_10
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(6~15) n - 6 9(15 +(n - 6) - n) ashr_10
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_10):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $6, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $10, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 10(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_10):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_10)
|
|
-
|
|
-LABEL(gobble_ashr_10):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $10, %xmm3
|
|
- pslldq $6, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $10, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ xor %rax, %r8
|
|
+ or %rbx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ xor %rdx, %r9
|
|
+ or %rbx, %r9
|
|
+ sub %r11, %r9
|
|
+ jnz LABEL(tail)
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %rdx, %rax
|
|
+ jne LABEL(tail)
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_10) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $10, %xmm3
|
|
- pslldq $6, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $10, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ mov 8 (%rsi, %rcx), %rax
|
|
+ mov 8 (%rdi, %rcx), %rdx
|
|
+ add $8, %ecx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ sub $8, %r14
|
|
+ jl LABEL(tail)
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_10)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_10):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xfc00, %edx
|
|
- jnz LABEL(ashr_10_exittail)
|
|
+ mov %rbx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- cmp $5, %r11
|
|
- jbe LABEL(ashr_10_exittail)
|
|
-#endif
|
|
+ mov %rbx, %r9
|
|
+ add %rdx, %r9
|
|
+ sbb %r11, %r11
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_10)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_10_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $10, %xmm0
|
|
- psrldq $10, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_11
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(5~15) n - 5 10(15 +(n - 5) - n) ashr_11
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_11):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $5, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $11, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 11(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_11):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_11)
|
|
-
|
|
-LABEL(gobble_ashr_11):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $11, %xmm3
|
|
- pslldq $5, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $11, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ xor %rax, %r8
|
|
+ or %rbx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ xor %rdx, %r9
|
|
+ or %rbx, %r9
|
|
+ sub %r11, %r9
|
|
+ jnz LABEL(tail)
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %rdx, %rax
|
|
+ jne LABEL(tail)
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_11) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $11, %xmm3
|
|
- pslldq $5, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $11, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ mov 8 (%rsi, %rcx), %rax
|
|
+ mov 8 (%rdi, %rcx), %rdx
|
|
+ add $8, %ecx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ sub $8, %r14
|
|
+ jl LABEL(tail)
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_11)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_11):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xf800, %edx
|
|
- jnz LABEL(ashr_11_exittail)
|
|
+ mov %rbx, %r8
|
|
+ add %rax, %r8
|
|
+ sbb %r10, %r10
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- cmp $4, %r11
|
|
- jbe LABEL(ashr_11_exittail)
|
|
-#endif
|
|
+ mov %rbx, %r9
|
|
+ add %rdx, %r9
|
|
+ sbb %r11, %r11
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_11)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_11_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $11, %xmm0
|
|
- psrldq $11, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_12
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(4~15) n - 4 11(15 +(n - 4) - n) ashr_12
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_12):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $4, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $12, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 12(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_12):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_12)
|
|
-
|
|
-LABEL(gobble_ashr_12):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $12, %xmm3
|
|
- pslldq $4, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $12, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ xor %rax, %r8
|
|
+ or %rbx, %r8
|
|
+ sub %r10, %r8
|
|
+ jnz LABEL(tail)
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ xor %rdx, %r9
|
|
+ or %rbx, %r9
|
|
+ sub %r11, %r9
|
|
+ jnz LABEL(tail)
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %rdx, %rax
|
|
+ jne LABEL(tail)
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_12) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $12, %xmm3
|
|
- pslldq $4, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $12, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ add $8, %ecx
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+ jmp LABEL(64loop)
|
|
+
|
|
+LABEL(64after):
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_12)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_12):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xf000, %edx
|
|
- jnz LABEL(ashr_12_exittail)
|
|
+LABEL(tailtry):
|
|
+# mov (%rsi, %rcx), %rax
|
|
+# mov (%rdi, %rcx), %rdx
|
|
+# add $8, %rcx
|
|
|
|
+LABEL(tail): # byte tail
|
|
#ifdef USE_AS_STRNCMP
|
|
- cmp $3, %r11
|
|
- jbe LABEL(ashr_12_exittail)
|
|
+ add $7, %r14
|
|
#endif
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_12)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_12_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $12, %xmm0
|
|
- psrldq $12, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_13
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(3~15) n - 3 12(15 +(n - 3) - n) ashr_13
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_13):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $3, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $13, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 13(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_13):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_13)
|
|
-
|
|
-LABEL(gobble_ashr_13):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $13, %xmm3
|
|
- pslldq $3, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $13, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ cmp %dl, %al # check if same character
|
|
+ jne .exit
|
|
+ test %al, %al # check if character a NUL
|
|
+ jz .exit
|
|
+
|
|
+ shr $8, %rax
|
|
+ shr $8, %rdx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ dec %r14
|
|
+ jl .exit
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %dl, %al
|
|
+ jne .exit
|
|
+ test %al, %al
|
|
+ jz .exit
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_13) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $13, %xmm3
|
|
- pslldq $3, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $13, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ shr $8, %rax
|
|
+ shr $8, %rdx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ dec %r14
|
|
+ jl .exit
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_13)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_13):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xe000, %edx
|
|
- jnz LABEL(ashr_13_exittail)
|
|
+ cmp %dl, %al
|
|
+ jne .exit
|
|
+ test %al, %al
|
|
+ jz .exit
|
|
+
|
|
+ shr $8, %rax
|
|
+ shr $8, %rdx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- cmp $2, %r11
|
|
- jbe LABEL(ashr_13_exittail)
|
|
+ dec %r14
|
|
+ jl .exit
|
|
#endif
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_13)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_13_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $13, %xmm0
|
|
- psrldq $13, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_14
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(2~15) n - 2 13(15 +(n - 2) - n) ashr_14
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_14):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $2, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $14, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 14(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_14):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_14)
|
|
-
|
|
-LABEL(gobble_ashr_14):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $14, %xmm3
|
|
- pslldq $2, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $14, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ cmp %dl, %al
|
|
+ jne .exit
|
|
+ test %al, %al
|
|
+ jz .exit
|
|
+
|
|
+ shr $8, %rax
|
|
+ shr $8, %rdx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ dec %r14
|
|
+ jl .exit
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %dl, %al
|
|
+ jne .exit
|
|
+ test %al, %al
|
|
+ jz .exit
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_14) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $14, %xmm3
|
|
- pslldq $2, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $14, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ shr $8, %eax
|
|
+ shr $8, %edx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ dec %r14
|
|
+ jl .exit
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_14)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_14):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0xc000, %edx
|
|
- jnz LABEL(ashr_14_exittail)
|
|
+ cmp %dl, %al
|
|
+ jne .exit
|
|
+ test %al, %al
|
|
+ jz .exit
|
|
+
|
|
+ shr $8, %eax
|
|
+ shr $8, %edx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- cmp $1, %r11
|
|
- jbe LABEL(ashr_14_exittail)
|
|
+ dec %r14
|
|
+ jl .exit
|
|
#endif
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_14)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_14_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $14, %xmm0
|
|
- psrldq $14, %xmm3
|
|
- jmp LABEL(aftertail)
|
|
-
|
|
-/*
|
|
- * The following cases will be handled by ashr_15
|
|
- * rcx(offset of rsi) rax(offset of rdi) relative offset corresponding case
|
|
- * n(1~15) n - 1 14(15 +(n - 1) - n) ashr_15
|
|
- */
|
|
- .p2align 4
|
|
-LABEL(ashr_15):
|
|
- pxor %xmm0, %xmm0
|
|
- movdqa (%rdi), %xmm2
|
|
- movdqa (%rsi), %xmm1
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pslldq $1, %xmm2
|
|
- pcmpeqb %xmm1, %xmm2
|
|
- psubb %xmm0, %xmm2
|
|
- pmovmskb %xmm2, %r9d
|
|
- shr %cl, %edx
|
|
- shr %cl, %r9d
|
|
- sub %r9d, %edx
|
|
- jnz LABEL(less32bytes)
|
|
-
|
|
- movdqa (%rdi), %xmm3
|
|
-
|
|
- UPDATE_STRNCMP_COUNTER
|
|
-
|
|
- pxor %xmm0, %xmm0
|
|
- mov $16, %rcx /* index for loads */
|
|
- mov $15, %r9d /* byte position left over from less32bytes case */
|
|
- /*
|
|
- * Setup %r10 value allows us to detect crossing a page boundary.
|
|
- * When %r10 goes positive we have crossed a page boundary and
|
|
- * need to do a nibble.
|
|
- */
|
|
- lea 15(%rdi), %r10
|
|
- and $0xfff, %r10 /* offset into 4K page */
|
|
-
|
|
- sub $0x1000, %r10 /* subtract 4K pagesize */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(loop_ashr_15):
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_15)
|
|
-
|
|
-LABEL(gobble_ashr_15):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $15, %xmm3
|
|
- pslldq $1, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $15, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ cmp %dl, %al
|
|
+ jne .exit
|
|
+ test %al, %al
|
|
+ jz .exit
|
|
+
|
|
+ shr $8, %eax
|
|
+ shr $8, %edx
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+ dec %r14
|
|
+ jl .exit
|
|
#endif
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
+ cmp %dl, %al
|
|
+ jne .exit
|
|
+# test %al, %al
|
|
+# jz .exit
|
|
|
|
- add $16, %r10
|
|
- jg LABEL(nibble_ashr_15) /* cross page boundary */
|
|
-
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- movdqa (%rdi, %rcx), %xmm2
|
|
- movdqa %xmm2, %xmm4
|
|
-
|
|
-#ifndef USE_SSSE3
|
|
- psrldq $15, %xmm3
|
|
- pslldq $1, %xmm2
|
|
- por %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#else
|
|
- palignr $15, %xmm3, %xmm2 /* merge into one 16byte value */
|
|
-#endif
|
|
-
|
|
- pcmpeqb %xmm1, %xmm0
|
|
- pcmpeqb %xmm2, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- sub $0xffff, %edx
|
|
- jnz LABEL(exit)
|
|
+ .p2align 4,, 15
|
|
|
|
-#ifdef USE_AS_STRNCMP
|
|
- sub $16, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
-#endif
|
|
+LABEL(tailafter):
|
|
|
|
- add $16, %rcx
|
|
- movdqa %xmm4, %xmm3
|
|
- jmp LABEL(loop_ashr_15)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(nibble_ashr_15):
|
|
- pcmpeqb %xmm3, %xmm0 /* check nibble for null char */
|
|
- pmovmskb %xmm0, %edx
|
|
- test $0x8000, %edx
|
|
- jnz LABEL(ashr_15_exittail)
|
|
+.exit:
|
|
+ mov -32 (%rsp), %r15
|
|
+ mov -24 (%rsp), %rbp
|
|
+ mov -16 (%rsp), %rbx
|
|
|
|
+ .p2align 4,, 3
|
|
+
|
|
+LABEL(exitafter):
|
|
#ifdef USE_AS_STRNCMP
|
|
- test %r11, %r11
|
|
- je LABEL(ashr_15_exittail)
|
|
+ test %r14, %r14
|
|
+ cmovl %edx, %eax
|
|
#endif
|
|
|
|
- pxor %xmm0, %xmm0
|
|
- sub $0x1000, %r10
|
|
- jmp LABEL(gobble_ashr_15)
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ashr_15_exittail):
|
|
- movdqa (%rsi, %rcx), %xmm1
|
|
- psrldq $15, %xmm3
|
|
- psrldq $15, %xmm0
|
|
-
|
|
- .p2align 4
|
|
-LABEL(aftertail):
|
|
- pcmpeqb %xmm3, %xmm1
|
|
- psubb %xmm0, %xmm1
|
|
- pmovmskb %xmm1, %edx
|
|
- not %edx
|
|
-
|
|
- .p2align 4
|
|
-LABEL(exit):
|
|
- lea -16(%r9, %rcx), %rax /* locate the exact offset for rdi */
|
|
-LABEL(less32bytes):
|
|
- lea (%rdi, %rax), %rdi /* locate the exact address for first operand(rdi) */
|
|
- lea (%rsi, %rcx), %rsi /* locate the exact address for second operand(rsi) */
|
|
- test %r8d, %r8d
|
|
- jz LABEL(ret)
|
|
- xchg %rsi, %rdi /* recover original order according to flag(%r8d) */
|
|
-
|
|
- .p2align 4
|
|
-LABEL(ret):
|
|
-LABEL(less16bytes):
|
|
- bsf %rdx, %rdx /* find and store bit index in %rdx */
|
|
+ movzx %al, %eax
|
|
+ movzx %dl, %edx
|
|
+ sub %eax, %edx
|
|
+ xchg %edx, %eax
|
|
|
|
#ifdef USE_AS_STRNCMP
|
|
- sub %rdx, %r11
|
|
- jbe LABEL(strcmp_exitz)
|
|
+LABEL(exitz):
|
|
+ mov -8 (%rsp), %r14
|
|
#endif
|
|
- movzbl (%rsi, %rdx), %ecx
|
|
- movzbl (%rdi, %rdx), %eax
|
|
-
|
|
- sub %ecx, %eax
|
|
- ret
|
|
+ ret
|
|
|
|
-LABEL(strcmp_exitz):
|
|
- xor %eax, %eax
|
|
- ret
|
|
-
|
|
- .p2align 4
|
|
-LABEL(Byte0):
|
|
- movzx (%rsi), %ecx
|
|
- movzx (%rdi), %eax
|
|
-
|
|
- sub %ecx, %eax
|
|
- ret
|
|
-END (BP_SYM (STRCMP))
|
|
-
|
|
- .section .rodata,"a",@progbits
|
|
- .p2align 3
|
|
-LABEL(unaligned_table):
|
|
- .int LABEL(ashr_1) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_2) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_3) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_4) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_5) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_6) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_7) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_8) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_9) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_10) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_11) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_12) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_13) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_14) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_15) - LABEL(unaligned_table)
|
|
- .int LABEL(ashr_0) - LABEL(unaligned_table)
|
|
-#endif /* NOT_IN_libc */
|
|
+END (strcmp)
|
|
libc_hidden_builtin_def (STRCMP)
|
|
Index: sysdeps/x86_64/memcpy.S
|
|
===================================================================
|
|
--- sysdeps/x86_64/memcpy.S.orig
|
|
+++ sysdeps/x86_64/memcpy.S
|
|
@@ -39,7 +39,7 @@
|
|
|
|
.text
|
|
|
|
-#if defined PIC && !defined NOT_IN_libc
|
|
+#if defined PIC && !defined NOT_IN_libc && !defined USE_AS_BCOPY
|
|
ENTRY (__memcpy_chk)
|
|
|
|
cmpq %rdx, %rcx
|