* init.c (__pthread_initialize_minimal_internal): Check whether

	private futexes are available.
	* allocatestack.c (allocate_stack): Copy private_futex field from
	current thread into the new stack.
	* sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Use private
	futexes if they are available.
	* sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S: Likewise
	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Adjust so that change
	in libc-lowlevellock.S allow using private futexes.
	* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Define
	FUTEX_PRIVATE_FLAG.
	* sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise.
	* sysdeps/unix/sysv/linux/x86_64/pthread_once.S: Use private futexes
	if they are available.
	* sysdeps/unix/sysv/linux/i386/pthread_once.S: Likewise.
	* sysdeps/x86_64/tcb-offsets.sym: Add PRIVATE_FUTEX.
	* sysdeps/i386/tcb-offsets.sym: Likewise.
	* sysdeps/x86_64/tls.h (tcbhead_t): Add private_futex field.
	* sysdeps/i386/tls.h (tcbhead_t): Likewise.

15 files changed, 192 insertions, 31 deletions

diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index 2d4f50b73b..8198f00e0a 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,26 @@
+2007-05-23  Ulrich Drepper  <drepper@redhat.com>
+
+	* init.c (__pthread_initialize_minimal_internal): Check whether
+	private futexes are available.
+	* allocatestack.c (allocate_stack): Copy private_futex field from
+	current thread into the new stack.
+	* sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S: Use private
+	futexes if they are available.
+	* sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S: Likewise
+	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.S: Adjust so that change
+	in libc-lowlevellock.S allow using private futexes.
+	* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/lowlevellock.h: Define
+	FUTEX_PRIVATE_FLAG.
+	* sysdeps/unix/sysv/linux/i386/lowlevellock.h: Likewise.
+	* sysdeps/unix/sysv/linux/x86_64/pthread_once.S: Use private futexes
+	if they are available.
+	* sysdeps/unix/sysv/linux/i386/pthread_once.S: Likewise.
+	* sysdeps/x86_64/tcb-offsets.sym: Add PRIVATE_FUTEX.
+	* sysdeps/i386/tcb-offsets.sym: Likewise.
+	* sysdeps/x86_64/tls.h (tcbhead_t): Add private_futex field.
+	* sysdeps/i386/tls.h (tcbhead_t): Likewise.
+
 2007-05-21  Ulrich Drepper  <drepper@redhat.com>
 
 	* sysdeps/pthread/pthread-functions.h (struct pthread_functions):
diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c
index e556dbac08..5dac000d6e 100644
--- a/nptl/allocatestack.c
+++ b/nptl/allocatestack.c
@@ -376,6 +376,12 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
       __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 #endif
 
+#ifndef __ASSUME_PRIVATE_FUTEX
+      /* The thread must know when private futexes are supported.  */
+      pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
+						header.private_futex);
+#endif
+
 #ifdef NEED_DL_SYSINFO
       /* Copy the sysinfo value from the parent.  */
       THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
@@ -510,6 +516,12 @@ allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
 	  __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
 #endif
 
+#ifndef __ASSUME_PRIVATE_FUTEX
+	  /* The thread must know when private futexes are supported.  */
+	  pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
+						    header.private_futex);
+#endif
+
 #ifdef NEED_DL_SYSINFO
 	  /* Copy the sysinfo value from the parent.  */
 	  THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
diff --git a/nptl/init.c b/nptl/init.c
index f67467aa5a..ff81087d0c 100644
--- a/nptl/init.c
+++ b/nptl/init.c
@@ -276,6 +276,18 @@ __pthread_initialize_minimal_internal (void)
 #endif
     set_robust_list_not_avail ();
 
+#ifndef __ASSUME_PRIVATE_FUTEX
+  /* Private futexes are always used (at least internally) so that
+     doing the test once this early is beneficial.  */
+  {
+    int word;
+    res = INTERNAL_SYSCALL (futex, err, 3, &word,
+			    FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1);
+    if (!INTERNAL_SYSCALL_ERROR_P (res, err))
+      THREAD_SETMEM (pd, header.private_futex, FUTEX_PRIVATE_FLAG);
+  }
+#endif
+
   /* Set initial thread's stack block from 0 up to __libc_stack_end.
      It will be bigger than it actually is, but for unwind.c/pt-longjmp.c
      purposes this is good enough.  */
diff --git a/nptl/sysdeps/i386/tcb-offsets.sym b/nptl/sysdeps/i386/tcb-offsets.sym
index 7c8d9a5ca5..69f9deb368 100644
--- a/nptl/sysdeps/i386/tcb-offsets.sym
+++ b/nptl/sysdeps/i386/tcb-offsets.sym
@@ -12,3 +12,6 @@ CLEANUP			offsetof (struct pthread, cleanup)
 CLEANUP_PREV		offsetof (struct _pthread_cleanup_buffer, __prev)
 MUTEX_FUTEX		offsetof (pthread_mutex_t, __data.__lock)
 POINTER_GUARD		offsetof (tcbhead_t, pointer_guard)
+#ifndef __ASSUME_PRIVATE_FUTEX
+PRIVATE_FUTEX		offsetof (tcbhead_t, private_futex)
+#endif
diff --git a/nptl/sysdeps/i386/tls.h b/nptl/sysdeps/i386/tls.h
index d9044f3fde..3bdf0f2b04 100644
--- a/nptl/sysdeps/i386/tls.h
+++ b/nptl/sysdeps/i386/tls.h
@@ -28,6 +28,7 @@
 # include <stdlib.h>
 # include <list.h>
 # include <sysdep.h>
+# include <kernel-features.h>
 
 
 /* Type for the dtv.  */
@@ -53,6 +54,9 @@ typedef struct
   uintptr_t stack_guard;
   uintptr_t pointer_guard;
   int gscope_flag;
+#ifndef __ASSUME_PRIVATE_FUTEX
+  int private_futex;
+#endif
 } tcbhead_t;
 
 # define TLS_MULTIPLE_THREADS_IN_TCB 1
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S
index 88885b735d..f246711489 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/libc-lowlevellock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -17,6 +17,8 @@
    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
    02111-1307 USA.  */
 
+#include <kernel-features.h>
+
 /* In libc.so we do not unconditionally use the lock prefix.  Only if
    the application is using threads.  */
 #ifndef UP
@@ -27,4 +29,17 @@
 0:
 #endif
 
+/* All locks in libc are private.  Use the kernel feature if possible.  */
+#define FUTEX_PRIVATE_FLAG	128
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define FUTEX_WAIT		(0 | FUTEX_PRIVATE_FLAG)
+# define FUTEX_WAKE		(1 | FUTEX_PRIVATE_FLAG)
+#else
+# define LOAD_FUTEX_WAIT(reg) \
+	movl	%gs:PRIVATE_FUTEX, reg
+# define LOAD_FUTEX_WAKE(reg) \
+	movl	%gs:PRIVATE_FUTEX, reg ; \
+	orl	$FUTEX_WAKE, reg
+#endif
+
 #include "lowlevellock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
index e2da5b04cf..1c47b37bb0 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/lowlevellock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -32,8 +32,22 @@
 
 #define SYS_gettimeofday	__NR_gettimeofday
 #define SYS_futex		240
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
+#ifndef FUTEX_WAIT
+# define FUTEX_WAIT		0
+# define FUTEX_WAKE		1
+#endif
+
+#ifndef LOAD_FUTEX_WAIT
+# if FUTEX_WAIT == 0
+#  define LOAD_FUTEX_WAIT(reg) \
+	xorl	reg, reg
+# else
+#  define LOAD_FUTEX_WAIT(reg) \
+	movl	$FUTEX_WAIT, reg
+# endif
+# define LOAD_FUTEX_WAKE(reg) \
+	movl	$FUTEX_WAKE, reg
+#endif
 
 
 	.globl	__lll_mutex_lock_wait
@@ -55,7 +69,7 @@ __lll_mutex_lock_wait:
 	movl	$2, %edx
 	movl	%ecx, %ebx
 	xorl	%esi, %esi	/* No timeout.  */
-	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
+	LOAD_FUTEX_WAIT (%ecx)
 
 	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
 	jne 2f
@@ -151,7 +165,7 @@ __lll_mutex_timedlock_wait:
 
 	/* Futex call.  */
 	movl	%esp, %esi
-	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
+	LOAD_FUTEX_WAIT (%ecx)
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 	movl	%eax, %ecx
@@ -252,7 +266,7 @@ __lll_mutex_unlock_wake:
 
 	movl	%eax, %ebx
 	movl	$0, (%eax)
-	movl	$FUTEX_WAKE, %ecx
+	LOAD_FUTEX_WAKE (%ecx)
 	movl	$1, %edx	/* Wake one thread.  */
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
@@ -314,6 +328,8 @@ __lll_timedwait_tid:
 	jz	4f
 
 	movl	%esp, %esi
+	/* XXX The kernel so far uses global futex for the wakeup at
+	   all times.  */
 	xorl	%ecx, %ecx	/* movl $FUTEX_WAIT, %ecx */
 	movl	%ebp, %ebx
 	movl	$SYS_futex, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
index 21de09fe91..b89d5dd7d1 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -38,6 +38,7 @@
 #define FUTEX_LOCK_PI		6
 #define FUTEX_UNLOCK_PI		7
 #define FUTEX_TRYLOCK_PI	8
+#define FUTEX_PRIVATE_FLAG	128
 
 
 /* Initializer for compatibility lock.  */
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S
index 312933c5e4..8ff0dad27f 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/pthread_once.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -19,6 +19,8 @@
 
 #include <unwindbuf.h>
 #include <sysdep.h>
+#include <kernel-features.h>
+
 
 #ifndef UP
 # define LOCK lock
@@ -26,8 +28,10 @@
 # define LOCK
 #endif
 
-#define SYS_futex	240
-#define FUTEX_WAKE	1
+#define SYS_futex		240
+#define FUTEX_WAIT		0
+#define FUTEX_WAKE		1
+#define FUTEX_PRIVATE_FLAG	128
 
 	.comm	__fork_generation, 4, 4
 
@@ -90,7 +94,16 @@ __pthread_once:
 	jnz	3f	/* Different for generation -> run initializer.  */
 
 	/* Somebody else got here first.  Wait.  */
-	movl	%esi, %ecx		/* movl $FUTEX_WAIT, %ecx */
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %ecx
+#else
+# if FUTEX_WAIT == 0
+	movl	%gs:PRIVATE_FUTEX, %ecx
+# else
+	movl	$FUTEX_WAIT, %ecx
+	orl	%gs:PRIVATE_FUTEX, %ecx
+# endif
+#endif
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 	jmp	6b
@@ -131,7 +144,12 @@ __pthread_once:
 
 	/* Wake up all other threads.  */
 	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
+#else
 	movl	$FUTEX_WAKE, %ecx
+	orl	%gs:PRIVATE_FUTEX, %ecx
+#endif
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 
@@ -152,7 +170,12 @@ __pthread_once:
 	movl	$0, (%ebx)
 
 	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %ecx
+#else
 	movl	$FUTEX_WAKE, %ecx
+	orl	%gs:PRIVATE_FUTEX, %ecx
+#endif
 	movl	$SYS_futex, %eax
 	ENTER_KERNEL
 
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S
index 3621efa4fc..21bf58cbbd 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/libc-lowlevellock.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -17,6 +17,8 @@
    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
    02111-1307 USA.  */
 
+#include <kernel-features.h>
+
 /* In libc.so we do not unconditionally use the lock prefix.  Only if
    the application is using threads.  */
 #ifndef UP
@@ -27,4 +29,17 @@
 0:
 #endif
 
+/* All locks in libc are private.  Use the kernel feature if possible.  */
+#define FUTEX_PRIVATE_FLAG	128
+#ifdef __ASSUME_PRIVATE_FUTEX
+# define FUTEX_WAIT		(0 | FUTEX_PRIVATE_FLAG)
+# define FUTEX_WAKE		(1 | FUTEX_PRIVATE_FLAG)
+#else
+# define LOAD_FUTEX_WAIT(reg) \
+	movl	%fs:PRIVATE_FUTEX, reg
+# define LOAD_FUTEX_WAKE(reg) \
+	movl	%fs:PRIVATE_FUTEX, reg ; \
+	orl	$FUTEX_WAKE, reg
+#endif
+
 #include "lowlevellock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
index 6724ded762..de70b1852c 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.S
@@ -31,8 +31,23 @@
 #endif
 
 #define SYS_futex		202
-#define FUTEX_WAIT		0
-#define FUTEX_WAKE		1
+#ifndef FUTEX_WAIT
+# define FUTEX_WAIT		0
+# define FUTEX_WAKE		1
+#endif
+
+#ifndef LOAD_FUTEX_WAIT
+# if FUTEX_WAIT == 0
+#  define LOAD_FUTEX_WAIT(reg) \
+	xorl	reg, reg
+# else
+#  define LOAD_FUTEX_WAIT(reg) \
+	movl	$FUTEX_WAIT, reg
+# endif
+# define LOAD_FUTEX_WAKE(reg) \
+	movl	$FUTEX_WAKE, reg
+#endif
+
 
 /* For the calculation see asm/vsyscall.h.  */
 #define VSYSCALL_ADDR_vgettimeofday	0xffffffffff600000
@@ -52,11 +67,7 @@ __lll_mutex_lock_wait:
 	cfi_offset(%rdx, -24)
 	xorq	%r10, %r10	/* No timeout.  */
 	movl	$2, %edx
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
-#else
-	movl	$FUTEX_WAIT, %esi
-#endif
+	LOAD_FUTEX_WAIT (%esi)
 
 	cmpl	%edx, %eax	/* NB:	 %edx == 2 */
 	jne	2f
@@ -151,11 +162,7 @@ __lll_mutex_timedlock_wait:
 	je	8f
 
 	movq	%rsp, %r10
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
-#else
-	movl	$FUTEX_WAIT, %esi
-#endif
+	LOAD_FUTEX_WAIT (%esi)
 	movq	%r12, %rdi
 	movl	$SYS_futex, %eax
 	syscall
@@ -247,7 +254,7 @@ __lll_mutex_unlock_wake:
 	cfi_offset(%rdx, -24)
 
 	movl	$0, (%rdi)
-	movl	$FUTEX_WAKE, %esi
+	LOAD_FUTEX_WAKE (%esi)
 	movl	$1, %edx	/* Wake one thread.  */
 	movl	$SYS_futex, %eax
 	syscall
@@ -311,6 +318,8 @@ __lll_timedwait_tid:
 	jz	4f
 
 	movq	%rsp, %r10
+	/* XXX The kernel so far uses global futex for the wakeup at
+	   all times.  */
 #if FUTEX_WAIT == 0
 	xorl	%esi, %esi
 #else
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
index bb988f3b2b..fbe48b37aa 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h
@@ -39,6 +39,7 @@
 #define FUTEX_LOCK_PI		6
 #define FUTEX_UNLOCK_PI		7
 #define FUTEX_TRYLOCK_PI	8
+#define FUTEX_PRIVATE_FLAG	128
 
 
 /* Initializer for compatibility lock.  */
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
index 9db5516913..7740c599d1 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_once.S
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2005 Free Software Foundation, Inc.
+/* Copyright (C) 2002, 2003, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
 
@@ -17,14 +17,19 @@
    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
    02111-1307 USA.  */
 
+#include <kernel-features.h>
+#include <tcb-offsets.h>
+
 #ifndef UP
 # define LOCK lock
 #else
 # define LOCK
 #endif
 
-#define SYS_futex	202
-#define FUTEX_WAKE	1
+#define SYS_futex		202
+#define FUTEX_WAIT		0
+#define FUTEX_WAKE		1
+#define FUTEX_PRIVATE_FLAG	128
 
 	.comm	__fork_generation, 4, 4
 
@@ -74,10 +79,15 @@ __pthread_once:
 	jnz	3f	/* Different for generation -> run initializer.  */
 
 	/* Somebody else got here first.  Wait.  */
-#if FUTEX_WAIT == 0
-	xorl	%esi, %esi
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAIT|FUTEX_PRIVATE_FLAG, %esi
 #else
+# if FUTEX_WAIT == 0
+	movl	%fs:PRIVATE_FUTEX, %esi
+# else
 	movl	$FUTEX_WAIT, %esi
+	orl	%fs:PRIVATE_FUTEX, %esi
+# endif
 #endif
 	movl	$SYS_futex, %eax
 	syscall
@@ -106,7 +116,12 @@ __pthread_once:
 
 	/* Wake up all other threads.  */
 	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
+#else
 	movl	$FUTEX_WAKE, %esi
+	orl	%fs:PRIVATE_FUTEX, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 
@@ -133,7 +148,12 @@ clear_once_control:
 	movl	$0, (%rdi)
 
 	movl	$0x7fffffff, %edx
+#ifdef __ASSUME_PRIVATE_FUTEX
+	movl	$FUTEX_WAKE|FUTEX_PRIVATE_FLAG, %esi
+#else
 	movl	$FUTEX_WAKE, %esi
+	orl	%fs:PRIVATE_FUTEX, %esi
+#endif
 	movl	$SYS_futex, %eax
 	syscall
 
diff --git a/nptl/sysdeps/x86_64/tcb-offsets.sym b/nptl/sysdeps/x86_64/tcb-offsets.sym
index 21274ecab9..1c70c6bde7 100644
--- a/nptl/sysdeps/x86_64/tcb-offsets.sym
+++ b/nptl/sysdeps/x86_64/tcb-offsets.sym
@@ -12,3 +12,6 @@ MUTEX_FUTEX		offsetof (pthread_mutex_t, __data.__lock)
 MULTIPLE_THREADS_OFFSET	offsetof (tcbhead_t, multiple_threads)
 POINTER_GUARD		offsetof (tcbhead_t, pointer_guard)
 VGETCPU_CACHE_OFFSET	offsetof (tcbhead_t, vgetcpu_cache)
+#ifndef __ASSUME_PRIVATE_FUTEX
+PRIVATE_FUTEX		offsetof (tcbhead_t, private_futex)
+#endif
diff --git a/nptl/sysdeps/x86_64/tls.h b/nptl/sysdeps/x86_64/tls.h
index 3a69e0428a..5ddaafadad 100644
--- a/nptl/sysdeps/x86_64/tls.h
+++ b/nptl/sysdeps/x86_64/tls.h
@@ -27,6 +27,7 @@
 # include <stdint.h>
 # include <stdlib.h>
 # include <sysdep.h>
+# include <kernel-features.h>
 
 
 /* Type for the dtv.  */
@@ -53,6 +54,9 @@ typedef struct
   uintptr_t stack_guard;
   uintptr_t pointer_guard;
   unsigned long int vgetcpu_cache[2];
+#ifndef __ASSUME_PRIVATE_FUTEX
+  int private_futex;
+#endif
 } tcbhead_t;
 
 #else /* __ASSEMBLER__ */