From c02f903a4f39902d429d3ec0d602c8753d846a68 Mon Sep 17 00:00:00 2001 From: Pedro Gonnet Date: Fri, 20 Dec 2024 01:06:54 -0800 Subject: [PATCH] Use the `c11` built-in atomic functions directly. 2nd try, fixed subtle difference in `pthreadpool_decrement_fetch_acquire_release_size_t` this time around. PiperOrigin-RevId: 708224758 diff --git a/BUILD.bazel b/BUILD.bazel index adea02a..bea3a4a 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -65,7 +65,7 @@ cc_library( "//conditions:default": [], }), copts = [ - "-std=gnu11", + "-std=c11", ] + select({ ":optimized_build": ["-O2"], "//conditions:default": [], diff --git a/CMakeLists.txt b/CMakeLists.txt index f06aada..efff8cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,16 @@ IF(CCACHE_BINARY) ENDIF() ENDIF() +# ---[ Language options. +SET(CMAKE_C_STANDARD 11) +SET(CMAKE_C_EXTENSIONS NO) +SET(CMAKE_CXX_STANDARD 11) +SET(CMAKE_CXX_EXTENSIONS NO) +IF(CMAKE_C_COMPILER_ID STREQUAL "MSVC") + STRING(APPEND CMAKE_C_FLAGS " /experimental:c11atomics") + STRING(APPEND CMAKE_CXX_FLAGS " /experimental:c11atomics") +ENDIF() + # ---[ Options. SET(PTHREADPOOL_LIBRARY_TYPE "default" CACHE STRING "Type of library (shared, static, or default) to build") SET_PROPERTY(CACHE PTHREADPOOL_LIBRARY_TYPE PROPERTY STRINGS default static shared) @@ -60,12 +70,6 @@ IF(PTHREADPOOL_BUILD_TESTS) ENABLE_TESTING() ENDIF() -MACRO(PTHREADPOOL_TARGET_ENABLE_CXX11 target) - SET_TARGET_PROPERTIES(${target} PROPERTIES - CXX_STANDARD 11 - CXX_EXTENSIONS NO) -ENDMACRO() - # ---[ Download deps IF(NOT DEFINED FXDIV_SOURCE_DIR) MESSAGE(STATUS "Downloading FXdiv to ${CMAKE_BINARY_DIR}/FXdiv-source (define FXDIV_SOURCE_DIR to avoid it)") @@ -159,9 +163,6 @@ ELSE() TARGET_COMPILE_DEFINITIONS(pthreadpool PRIVATE PTHREADPOOL_USE_FASTPATH=0) ENDIF() -SET_TARGET_PROPERTIES(pthreadpool PROPERTIES - C_STANDARD 11 - C_EXTENSIONS NO) TARGET_LINK_LIBRARIES(pthreadpool PUBLIC pthreadpool_interface) TARGET_INCLUDE_DIRECTORIES(pthreadpool PRIVATE src) IF(NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten") @@ -204,16 +205,10 @@ IF(PTHREADPOOL_BUILD_TESTS) ENDIF() ADD_EXECUTABLE(pthreadpool-test test/pthreadpool.cc) - SET_TARGET_PROPERTIES(pthreadpool-test PROPERTIES - CXX_STANDARD 11 - CXX_EXTENSIONS NO) TARGET_LINK_LIBRARIES(pthreadpool-test pthreadpool gtest gtest_main) ADD_TEST(pthreadpool pthreadpool-test) ADD_EXECUTABLE(pthreadpool-cxx-test test/pthreadpool-cxx.cc) - SET_TARGET_PROPERTIES(pthreadpool-cxx-test PROPERTIES - CXX_STANDARD 11 - CXX_EXTENSIONS NO) TARGET_LINK_LIBRARIES(pthreadpool-cxx-test pthreadpool gtest gtest_main) ADD_TEST(pthreadpool-cxx pthreadpool-cxx-test) ENDIF() @@ -228,14 +223,8 @@ IF(PTHREADPOOL_BUILD_BENCHMARKS) ENDIF() ADD_EXECUTABLE(latency-bench bench/latency.cc) - SET_TARGET_PROPERTIES(latency-bench PROPERTIES - CXX_STANDARD 11 - CXX_EXTENSIONS NO) TARGET_LINK_LIBRARIES(latency-bench pthreadpool benchmark) ADD_EXECUTABLE(throughput-bench bench/throughput.cc) - SET_TARGET_PROPERTIES(throughput-bench PROPERTIES - CXX_STANDARD 11 - CXX_EXTENSIONS NO) TARGET_LINK_LIBRARIES(throughput-bench pthreadpool benchmark) ENDIF() diff --git a/src/pthreads.c b/src/pthreads.c index 505fa0d..6f7829f 100644 --- a/src/pthreads.c +++ b/src/pthreads.c @@ -7,6 +7,9 @@ // This source code is licensed under the BSD-style license found in the // LICENSE file in the root directory of this source tree. +// Needed for syscall. +#define _GNU_SOURCE + /* Standard C headers */ #include #include @@ -32,19 +35,22 @@ /* Old Android NDKs do not define SYS_futex and FUTEX_PRIVATE_FLAG */ #ifndef SYS_futex #define SYS_futex __NR_futex -#endif +#endif // SYS_futex + #ifndef FUTEX_PRIVATE_FLAG #define FUTEX_PRIVATE_FLAG 128 -#endif +#endif // FUTEX_PRIVATE_FLAG + #elif defined(__EMSCRIPTEN__) /* math.h for INFINITY constant */ #include #include + #else #error \ "Platform-specific implementation of futex_wait and futex_wake_all required" -#endif -#endif +#endif // defined(__linux__) +#endif // PTHREADPOOL_USE_FUTEX /* Windows-specific headers */ #ifdef _WIN32 diff --git a/src/threadpool-atomics.h b/src/threadpool-atomics.h index de84057..6b8e71c 100644 --- a/src/threadpool-atomics.h +++ b/src/threadpool-atomics.h @@ -11,6 +11,7 @@ #define __PTHREADPOOL_SRC_THREADPOOL_ATOMICS_H_ /* Standard C headers */ +#include #include #include #include @@ -44,110 +45,9 @@ /* Configuration header */ #include "threadpool-common.h" -#if defined(__wasm__) && defined(__clang__) -/* - * Clang for WebAssembly target lacks stdatomic.h header, - * even though it supports the necessary low-level intrinsics. - * Thus, we implement pthreadpool atomic functions on top of - * low-level Clang-specific interfaces for this target. - */ - -typedef _Atomic(uint32_t) pthreadpool_atomic_uint32_t; -typedef _Atomic(size_t) pthreadpool_atomic_size_t; -typedef _Atomic(void*) pthreadpool_atomic_void_p; - -static inline uint32_t pthreadpool_load_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address) { - return __c11_atomic_load(address, __ATOMIC_RELAXED); -} - -static inline size_t pthreadpool_load_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return __c11_atomic_load(address, __ATOMIC_RELAXED); -} - -static inline void* pthreadpool_load_relaxed_void_p( - pthreadpool_atomic_void_p* address) { - return __c11_atomic_load(address, __ATOMIC_RELAXED); -} - -static inline uint32_t pthreadpool_load_acquire_uint32_t( - pthreadpool_atomic_uint32_t* address) { - return __c11_atomic_load(address, __ATOMIC_ACQUIRE); -} - -static inline size_t pthreadpool_load_acquire_size_t( - pthreadpool_atomic_size_t* address) { - return __c11_atomic_load(address, __ATOMIC_ACQUIRE); -} - -static inline void pthreadpool_store_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - __c11_atomic_store(address, value, __ATOMIC_RELAXED); -} - -static inline void pthreadpool_store_relaxed_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - __c11_atomic_store(address, value, __ATOMIC_RELAXED); -} - -static inline void pthreadpool_store_relaxed_void_p( - pthreadpool_atomic_void_p* address, void* value) { - __c11_atomic_store(address, value, __ATOMIC_RELAXED); -} - -static inline void pthreadpool_store_release_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - __c11_atomic_store(address, value, __ATOMIC_RELEASE); -} - -static inline void pthreadpool_store_release_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - __c11_atomic_store(address, value, __ATOMIC_RELEASE); -} - -static inline size_t pthreadpool_decrement_fetch_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return __c11_atomic_fetch_sub(address, 1, __ATOMIC_RELAXED) - 1; -} - -static inline size_t pthreadpool_decrement_fetch_release_size_t( - pthreadpool_atomic_size_t* address) { - return __c11_atomic_fetch_sub(address, 1, __ATOMIC_RELEASE) - 1; -} - -static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t( - pthreadpool_atomic_size_t* address) { - return __c11_atomic_fetch_sub(address, 1, __ATOMIC_ACQ_REL) - 1; -} - -static inline bool pthreadpool_try_decrement_relaxed_size_t( - pthreadpool_atomic_size_t* value) { - size_t actual_value = __c11_atomic_load(value, __ATOMIC_RELAXED); - while (actual_value != 0) { - if (__c11_atomic_compare_exchange_weak(value, &actual_value, - actual_value - 1, __ATOMIC_RELAXED, - __ATOMIC_RELAXED)) { - return true; - } - } - return false; -} - -static inline void pthreadpool_fence_acquire() { - __c11_atomic_thread_fence(__ATOMIC_ACQUIRE); -} - -static inline void pthreadpool_fence_release() { - __c11_atomic_thread_fence(__ATOMIC_RELEASE); -} -#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ - !defined(__STDC_NO_ATOMICS__) -#include - -typedef _Atomic(uint32_t) pthreadpool_atomic_uint32_t; -typedef _Atomic(size_t) pthreadpool_atomic_size_t; -typedef _Atomic(void*) pthreadpool_atomic_void_p; +typedef atomic_uint_fast32_t pthreadpool_atomic_uint32_t; +typedef atomic_size_t pthreadpool_atomic_size_t; +typedef atomic_uintptr_t pthreadpool_atomic_void_p; static inline uint32_t pthreadpool_load_relaxed_uint32_t( pthreadpool_atomic_uint32_t* address) { @@ -161,7 +61,7 @@ static inline size_t pthreadpool_load_relaxed_size_t( static inline void* pthreadpool_load_relaxed_void_p( pthreadpool_atomic_void_p* address) { - return atomic_load_explicit(address, memory_order_relaxed); + return (void*)atomic_load_explicit(address, memory_order_relaxed); } static inline uint32_t pthreadpool_load_acquire_uint32_t( @@ -186,7 +86,7 @@ static inline void pthreadpool_store_relaxed_size_t( static inline void pthreadpool_store_relaxed_void_p( pthreadpool_atomic_void_p* address, void* value) { - atomic_store_explicit(address, value, memory_order_relaxed); + atomic_store_explicit(address, (uintptr_t)value, memory_order_relaxed); } static inline void pthreadpool_store_release_uint32_t( @@ -216,18 +116,7 @@ static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t( static inline bool pthreadpool_try_decrement_relaxed_size_t( pthreadpool_atomic_size_t* value) { -#if defined(__clang__) && (defined(__arm__) || defined(__aarch64__)) - size_t actual_value; - do { - actual_value = __builtin_arm_ldrex((const volatile size_t*)value); - if (actual_value == 0) { - __builtin_arm_clrex(); - return false; - } - } while (__builtin_arm_strex(actual_value - 1, (volatile size_t*)value) != 0); - return true; -#else - size_t actual_value = pthreadpool_load_relaxed_size_t(value); + size_t actual_value = atomic_load_explicit(value, memory_order_acquire); while (actual_value != 0) { if (atomic_compare_exchange_weak_explicit( value, &actual_value, actual_value - 1, memory_order_relaxed, @@ -236,7 +125,6 @@ static inline bool pthreadpool_try_decrement_relaxed_size_t( } } return false; -#endif } static inline void pthreadpool_fence_acquire() { @@ -246,503 +134,6 @@ static inline void pthreadpool_fence_acquire() { static inline void pthreadpool_fence_release() { atomic_thread_fence(memory_order_release); } -#elif defined(__GNUC__) -typedef uint32_t volatile pthreadpool_atomic_uint32_t; -typedef size_t volatile pthreadpool_atomic_size_t; -typedef void* volatile pthreadpool_atomic_void_p; - -static inline uint32_t pthreadpool_load_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address) { - return *address; -} - -static inline size_t pthreadpool_load_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return *address; -} - -static inline void* pthreadpool_load_relaxed_void_p( - pthreadpool_atomic_void_p* address) { - return *address; -} - -static inline uint32_t pthreadpool_load_acquire_uint32_t( - pthreadpool_atomic_uint32_t* address) { - return *address; -} - -static inline size_t pthreadpool_load_acquire_size_t( - pthreadpool_atomic_size_t* address) { - return *address; -} - -static inline void pthreadpool_store_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - *address = value; -} - -static inline void pthreadpool_store_relaxed_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - *address = value; -} - -static inline void pthreadpool_store_relaxed_void_p( - pthreadpool_atomic_void_p* address, void* value) { - *address = value; -} - -static inline void pthreadpool_store_release_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - *address = value; -} - -static inline void pthreadpool_store_release_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - *address = value; -} - -static inline size_t pthreadpool_decrement_fetch_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return __sync_sub_and_fetch(address, 1); -} - -static inline size_t pthreadpool_decrement_fetch_release_size_t( - pthreadpool_atomic_size_t* address) { - return __sync_sub_and_fetch(address, 1); -} - -static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t( - pthreadpool_atomic_size_t* address) { - return __sync_sub_and_fetch(address, 1); -} - -static inline bool pthreadpool_try_decrement_relaxed_size_t( - pthreadpool_atomic_size_t* value) { - size_t actual_value = *value; - while (actual_value != 0) { - const size_t new_value = actual_value - 1; - const size_t expected_value = actual_value; - actual_value = - __sync_val_compare_and_swap(value, expected_value, new_value); - if (actual_value == expected_value) { - return true; - } - } - return false; -} - -static inline void pthreadpool_fence_acquire() { __sync_synchronize(); } - -static inline void pthreadpool_fence_release() { __sync_synchronize(); } -#elif defined(_MSC_VER) && defined(_M_ARM) -typedef volatile uint32_t pthreadpool_atomic_uint32_t; -typedef volatile size_t pthreadpool_atomic_size_t; -typedef void* volatile pthreadpool_atomic_void_p; - -static inline uint32_t pthreadpool_load_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address) { - return (uint32_t)__iso_volatile_load32((const volatile __int32*)address); -} - -static inline size_t pthreadpool_load_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)__iso_volatile_load32((const volatile __int32*)address); -} - -static inline void* pthreadpool_load_relaxed_void_p( - pthreadpool_atomic_void_p* address) { - return (void*)__iso_volatile_load32((const volatile __int32*)address); -} - -static inline uint32_t pthreadpool_load_acquire_uint32_t( - pthreadpool_atomic_uint32_t* address) { - const uint32_t value = - (uint32_t)__iso_volatile_load32((const volatile __int32*)address); - __dmb(_ARM_BARRIER_ISH); - _ReadBarrier(); - return value; -} - -static inline size_t pthreadpool_load_acquire_size_t( - pthreadpool_atomic_size_t* address) { - const size_t value = - (size_t)__iso_volatile_load32((const volatile __int32*)address); - __dmb(_ARM_BARRIER_ISH); - _ReadBarrier(); - return value; -} - -static inline void pthreadpool_store_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - __iso_volatile_store32((volatile __int32*)address, (__int32)value); -} - -static inline void pthreadpool_store_relaxed_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - __iso_volatile_store32((volatile __int32*)address, (__int32)value); -} - -static inline void pthreadpool_store_relaxed_void_p( - pthreadpool_atomic_void_p* address, void* value) { - __iso_volatile_store32((volatile __int32*)address, (__int32)value); -} - -static inline void pthreadpool_store_release_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - _WriteBarrier(); - __dmb(_ARM_BARRIER_ISH); - __iso_volatile_store32((volatile __int32*)address, (__int32)value); -} - -static inline void pthreadpool_store_release_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - _WriteBarrier(); - __dmb(_ARM_BARRIER_ISH); - __iso_volatile_store32((volatile __int32*)address, (__int32)value); -} - -static inline size_t pthreadpool_decrement_fetch_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement_nf((volatile long*)address); -} - -static inline size_t pthreadpool_decrement_fetch_release_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement_rel((volatile long*)address); -} - -static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement((volatile long*)address); -} - -static inline bool pthreadpool_try_decrement_relaxed_size_t( - pthreadpool_atomic_size_t* value) { - size_t actual_value = - (size_t)__iso_volatile_load32((const volatile __int32*)value); - while (actual_value != 0) { - const size_t new_value = actual_value - 1; - const size_t expected_value = actual_value; - actual_value = _InterlockedCompareExchange_nf( - (volatile long*)value, (long)new_value, (long)expected_value); - if (actual_value == expected_value) { - return true; - } - } - return false; -} - -static inline void pthreadpool_fence_acquire() { - __dmb(_ARM_BARRIER_ISH); - _ReadBarrier(); -} - -static inline void pthreadpool_fence_release() { - _WriteBarrier(); - __dmb(_ARM_BARRIER_ISH); -} -#elif defined(_MSC_VER) && defined(_M_ARM64) -typedef volatile uint32_t pthreadpool_atomic_uint32_t; -typedef volatile size_t pthreadpool_atomic_size_t; -typedef void* volatile pthreadpool_atomic_void_p; - -static inline uint32_t pthreadpool_load_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address) { - return (uint32_t)__iso_volatile_load32((const volatile __int32*)address); -} - -static inline size_t pthreadpool_load_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)__iso_volatile_load64((const volatile __int64*)address); -} - -static inline void* pthreadpool_load_relaxed_void_p( - pthreadpool_atomic_void_p* address) { - return (void*)__iso_volatile_load64((const volatile __int64*)address); -} - -static inline uint32_t pthreadpool_load_acquire_uint32_t( - pthreadpool_atomic_uint32_t* address) { - return (uint32_t)__ldar32((volatile unsigned __int32*)address); -} - -static inline size_t pthreadpool_load_acquire_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)__ldar64((volatile unsigned __int64*)address); -} - -static inline void pthreadpool_store_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - __iso_volatile_store32((volatile __int32*)address, (__int32)value); -} - -static inline void pthreadpool_store_relaxed_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - __iso_volatile_store64((volatile __int64*)address, (__int64)value); -} - -static inline void pthreadpool_store_relaxed_void_p( - pthreadpool_atomic_void_p* address, void* value) { - __iso_volatile_store64((volatile __int64*)address, (__int64)value); -} - -static inline void pthreadpool_store_release_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - _WriteBarrier(); - __stlr32((unsigned __int32 volatile*)address, (unsigned __int32)value); -} - -static inline void pthreadpool_store_release_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - _WriteBarrier(); - __stlr64((unsigned __int64 volatile*)address, (unsigned __int64)value); -} - -static inline size_t pthreadpool_decrement_fetch_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement64_nf((volatile __int64*)address); -} - -static inline size_t pthreadpool_decrement_fetch_release_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement64_rel((volatile __int64*)address); -} - -static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement64((volatile __int64*)address); -} - -static inline bool pthreadpool_try_decrement_relaxed_size_t( - pthreadpool_atomic_size_t* value) { - size_t actual_value = - (size_t)__iso_volatile_load64((const volatile __int64*)value); - while (actual_value != 0) { - const size_t new_value = actual_value - 1; - const size_t expected_value = actual_value; - actual_value = _InterlockedCompareExchange64_nf( - (volatile __int64*)value, (__int64)new_value, (__int64)expected_value); - if (actual_value == expected_value) { - return true; - } - } - return false; -} - -static inline void pthreadpool_fence_acquire() { - __dmb(_ARM64_BARRIER_ISHLD); - _ReadBarrier(); -} - -static inline void pthreadpool_fence_release() { - _WriteBarrier(); - __dmb(_ARM64_BARRIER_ISH); -} -#elif defined(_MSC_VER) && defined(_M_IX86) -typedef volatile uint32_t pthreadpool_atomic_uint32_t; -typedef volatile size_t pthreadpool_atomic_size_t; -typedef void* volatile pthreadpool_atomic_void_p; - -static inline uint32_t pthreadpool_load_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address) { - return *address; -} - -static inline size_t pthreadpool_load_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return *address; -} - -static inline void* pthreadpool_load_relaxed_void_p( - pthreadpool_atomic_void_p* address) { - return *address; -} - -static inline uint32_t pthreadpool_load_acquire_uint32_t( - pthreadpool_atomic_uint32_t* address) { - /* x86 loads always have acquire semantics; use only a compiler barrier */ - const uint32_t value = *address; - _ReadBarrier(); - return value; -} - -static inline size_t pthreadpool_load_acquire_size_t( - pthreadpool_atomic_size_t* address) { - /* x86 loads always have acquire semantics; use only a compiler barrier */ - const size_t value = *address; - _ReadBarrier(); - return value; -} - -static inline void pthreadpool_store_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - *address = value; -} - -static inline void pthreadpool_store_relaxed_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - *address = value; -} - -static inline void pthreadpool_store_relaxed_void_p( - pthreadpool_atomic_void_p* address, void* value) { - *address = value; -} - -static inline void pthreadpool_store_release_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - /* x86 stores always have release semantics; use only a compiler barrier */ - _WriteBarrier(); - *address = value; -} - -static inline void pthreadpool_store_release_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - /* x86 stores always have release semantics; use only a compiler barrier */ - _WriteBarrier(); - *address = value; -} - -static inline size_t pthreadpool_decrement_fetch_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement((volatile long*)address); -} - -static inline size_t pthreadpool_decrement_fetch_release_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement((volatile long*)address); -} - -static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement((volatile long*)address); -} - -static inline bool pthreadpool_try_decrement_relaxed_size_t( - pthreadpool_atomic_size_t* value) { - size_t actual_value = *value; - while (actual_value != 0) { - const size_t new_value = actual_value - 1; - const size_t expected_value = actual_value; - actual_value = _InterlockedCompareExchange( - (volatile long*)value, (long)new_value, (long)expected_value); - if (actual_value == expected_value) { - return true; - } - } - return false; -} - -static inline void pthreadpool_fence_acquire() { _mm_lfence(); } - -static inline void pthreadpool_fence_release() { _mm_sfence(); } -#elif defined(_MSC_VER) && defined(_M_X64) -typedef volatile uint32_t pthreadpool_atomic_uint32_t; -typedef volatile size_t pthreadpool_atomic_size_t; -typedef void* volatile pthreadpool_atomic_void_p; - -static inline uint32_t pthreadpool_load_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address) { - return *address; -} - -static inline size_t pthreadpool_load_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return *address; -} - -static inline void* pthreadpool_load_relaxed_void_p( - pthreadpool_atomic_void_p* address) { - return *address; -} - -static inline uint32_t pthreadpool_load_acquire_uint32_t( - pthreadpool_atomic_uint32_t* address) { - /* x86-64 loads always have acquire semantics; use only a compiler barrier */ - const uint32_t value = *address; - _ReadBarrier(); - return value; -} - -static inline size_t pthreadpool_load_acquire_size_t( - pthreadpool_atomic_size_t* address) { - /* x86-64 loads always have acquire semantics; use only a compiler barrier */ - const size_t value = *address; - _ReadBarrier(); - return value; -} - -static inline void pthreadpool_store_relaxed_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - *address = value; -} - -static inline void pthreadpool_store_relaxed_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - *address = value; -} - -static inline void pthreadpool_store_relaxed_void_p( - pthreadpool_atomic_void_p* address, void* value) { - *address = value; -} - -static inline void pthreadpool_store_release_uint32_t( - pthreadpool_atomic_uint32_t* address, uint32_t value) { - /* x86-64 stores always have release semantics; use only a compiler barrier */ - _WriteBarrier(); - *address = value; -} - -static inline void pthreadpool_store_release_size_t( - pthreadpool_atomic_size_t* address, size_t value) { - /* x86-64 stores always have release semantics; use only a compiler barrier */ - _WriteBarrier(); - *address = value; -} - -static inline size_t pthreadpool_decrement_fetch_relaxed_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement64((volatile __int64*)address); -} - -static inline size_t pthreadpool_decrement_fetch_release_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement64((volatile __int64*)address); -} - -static inline size_t pthreadpool_decrement_fetch_acquire_release_size_t( - pthreadpool_atomic_size_t* address) { - return (size_t)_InterlockedDecrement64((volatile __int64*)address); -} - -static inline bool pthreadpool_try_decrement_relaxed_size_t( - pthreadpool_atomic_size_t* value) { - size_t actual_value = *value; - while (actual_value != 0) { - const size_t new_value = actual_value - 1; - const size_t expected_value = actual_value; - actual_value = _InterlockedCompareExchange64( - (volatile __int64*)value, (__int64)new_value, (__int64)expected_value); - if (actual_value == expected_value) { - return true; - } - } - return false; -} - -static inline void pthreadpool_fence_acquire() { - _mm_lfence(); - _ReadBarrier(); -} - -static inline void pthreadpool_fence_release() { - _WriteBarrier(); - _mm_sfence(); -} -#else -#error "Platform-specific implementation of threadpool-atomics.h required" -#endif static inline void pthreadpool_yield(uint32_t step) { if (step < PTHREADPOOL_SPIN_PAUSE_ITERATIONS) {