#include "common/threads.h" #include "common/assert.h" #include #include #include // IWYU pragma: keep #include // IWYU pragma: keep #include #if KYTY_PLATFORM == KYTY_PLATFORM_WINDOWS && KYTY_COMPILER == KYTY_COMPILER_CLANG #define KYTY_WIN_CS #endif // macOS has no clock_nanosleep. #if KYTY_PLATFORM != KYTY_PLATFORM_WINDOWS && !defined(__APPLE__) #define KYTY_POSIX_HIGH_RES_SLEEP #include #endif #include #include #include #ifdef KYTY_WIN_CS #include // IWYU pragma: keep // IWYU pragma: no_include constexpr DWORD KYTY_CS_SPIN_COUNT = 4000; #ifndef CREATE_WAITABLE_TIMER_HIGH_RESOLUTION #define CREATE_WAITABLE_TIMER_HIGH_RESOLUTION 0x00000002 #endif static void SleepHighResolution100ns(uint64_t units_100ns) { if (units_100ns == 0) { return; } thread_local HANDLE timer = CreateWaitableTimerExW( nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); if (timer == nullptr) { thread_local HANDLE fallback_timer = CreateWaitableTimerW(nullptr, TRUE, nullptr); timer = fallback_timer; } if (timer != nullptr) { LARGE_INTEGER due_time {}; due_time.QuadPart = -static_cast(units_100ns); if (SetWaitableTimerEx(timer, &due_time, 0, nullptr, nullptr, nullptr, 0) != 0) { WaitForSingleObject(timer, INFINITE); return; } } std::this_thread::sleep_for(std::chrono::nanoseconds(units_100ns * 100)); } // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include // IWYU pragma: no_include <__mutex_base> // IWYU pragma: no_include <__threading_support> // IWYU pragma: no_include // IWYU pragma: no_include using InitializeConditionVariable_func_t = /*WINBASEAPI*/ VOID WINAPI (*)(PCONDITION_VARIABLE); using WakeConditionVariable_func_t = /*WINBASEAPI*/ VOID WINAPI (*)(PCONDITION_VARIABLE); using WakeAllConditionVariable_func_t = /*WINBASEAPI*/ VOID WINAPI (*)(PCONDITION_VARIABLE); using SleepConditionVariableCS_func_t = /*WINBASEAPI*/ BOOL WINAPI (*)(PCONDITION_VARIABLE, PCRITICAL_SECTION, DWORD); static InitializeConditionVariable_func_t ResolveInitializeConditionVariable() { if (HMODULE h = GetModuleHandle("KernelBase"); h != nullptr) // @suppress("Invalid arguments") { return reinterpret_cast( GetProcAddress(h, "InitializeConditionVariable")); } return nullptr; } static WakeConditionVariable_func_t ResolveWakeConditionVariable() { if (HMODULE h = GetModuleHandle("KernelBase"); h != nullptr) // @suppress("Invalid arguments") { return reinterpret_cast( GetProcAddress(h, "WakeConditionVariable")); } return nullptr; } static WakeAllConditionVariable_func_t ResolveWakeAllConditionVariable() { if (HMODULE h = GetModuleHandle("KernelBase"); h != nullptr) // @suppress("Invalid arguments") { return reinterpret_cast( GetProcAddress(h, "WakeAllConditionVariable")); } return nullptr; } static SleepConditionVariableCS_func_t ResolveSleepConditionVariableCS() { if (HMODULE h = GetModuleHandle("KernelBase"); h != nullptr) // @suppress("Invalid arguments") { return reinterpret_cast( GetProcAddress(h, "SleepConditionVariableCS")); } return nullptr; } #endif #ifdef KYTY_POSIX_HIGH_RES_SLEEP // An absolute deadline preserves the requested sleep across signal interruptions. static void SleepHighResolutionNanos(uint64_t nanos) { if (nanos == 0) { return; } constexpr uint64_t NANOS_PER_SEC = 1000000000; timespec deadline {}; if (clock_gettime(CLOCK_MONOTONIC, &deadline) != 0) { std::this_thread::sleep_for(std::chrono::nanoseconds(nanos)); return; } auto target_nsec = static_cast(deadline.tv_nsec) + nanos; deadline.tv_sec += static_cast(target_nsec / NANOS_PER_SEC); deadline.tv_nsec = static_cast(target_nsec % NANOS_PER_SEC); while (clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &deadline, nullptr) == EINTR) { } } #endif namespace Common { using thread_id_t = std::thread::id; struct MutexPrivate { #ifdef KYTY_WIN_CS MutexPrivate() { InitializeCriticalSectionAndSpinCount(&m_cs, KYTY_CS_SPIN_COUNT); } ~MutexPrivate() { DeleteCriticalSection(&m_cs); } KYTY_CLASS_NO_COPY(MutexPrivate); CRITICAL_SECTION m_cs {}; #else std::recursive_mutex m_mutex; #endif }; struct CondVarPrivate { #ifdef KYTY_WIN_CS CondVarPrivate() { static auto func = ResolveInitializeConditionVariable(); EXIT_NOT_IMPLEMENTED(func == nullptr); func(&m_cv); } ~CondVarPrivate() = default; KYTY_CLASS_NO_COPY(CondVarPrivate); CONDITION_VARIABLE m_cv {}; #else std::condition_variable_any m_cv; #endif }; static wait_poll_func_t g_cond_wait_poll_callback = nullptr; struct ThreadPrivate { ThreadPrivate(thread_func_t f, void* a): func(f), arg(a), m_thread(&Run, this) {} static void Run(ThreadPrivate* t) { t->unique_id = Thread::GetThreadIdUnique(); t->started = true; t->func(t->arg); } thread_func_t func; void* arg; std::atomic_bool finished = false; std::atomic_bool auto_delete = false; std::atomic_bool started = false; int unique_id = 0; std::thread m_thread; }; static thread_id_t g_main_thread; static int g_main_thread_int; static std::atomic g_thread_counter = 0; void InitializeThreads() { g_main_thread = std::this_thread::get_id(); g_main_thread_int = Thread::GetThreadIdUnique(); } Thread::Thread(thread_func_t func, void* arg) : m_thread(std::make_unique(func, arg)) { while (!m_thread->started) { Common::Thread::SleepMicro(1000); } } Thread::~Thread() { EXIT_IF(!m_thread->finished && !m_thread->auto_delete); m_thread.reset(); } void Thread::Join() { EXIT_IF(m_thread->finished || m_thread->auto_delete); m_thread->m_thread.join(); m_thread->finished = true; } void Thread::Detach() { EXIT_IF(m_thread->finished || m_thread->auto_delete); m_thread->auto_delete = true; m_thread->m_thread.detach(); } void Thread::SleepMicro(uint32_t micros) { #ifdef KYTY_WIN_CS SleepHighResolution100ns(static_cast(micros) * 10); #elif defined(KYTY_POSIX_HIGH_RES_SLEEP) SleepHighResolutionNanos(static_cast(micros) * 1000); #else std::this_thread::sleep_for(std::chrono::microseconds(micros)); #endif } void Thread::SleepNano(uint64_t nanos) { #ifdef KYTY_WIN_CS SleepHighResolution100ns((nanos + 99) / 100); #elif defined(KYTY_POSIX_HIGH_RES_SLEEP) SleepHighResolutionNanos(nanos); #else std::this_thread::sleep_for(std::chrono::nanoseconds(nanos)); #endif } bool Thread::IsMainThread() { return g_main_thread == std::this_thread::get_id(); } std::string Thread::GetId() const { std::stringstream ss; ss << m_thread->m_thread.get_id(); return ss.str(); } int Thread::GetUniqueId() const { return m_thread->unique_id; } std::string Thread::GetThreadId() { std::stringstream ss; ss << std::this_thread::get_id(); return ss.str(); } Mutex::Mutex(): m_mutex(std::make_unique()) {} Mutex::~Mutex() { m_mutex.reset(); } void Mutex::Lock() { #ifdef KYTY_WIN_CS EnterCriticalSection(&m_mutex->m_cs); #else m_mutex->m_mutex.lock(); #endif } void Mutex::Unlock() { #ifdef KYTY_WIN_CS LeaveCriticalSection(&m_mutex->m_cs); #else m_mutex->m_mutex.unlock(); #endif } bool Mutex::TryLock() { #ifdef KYTY_WIN_CS return (TryEnterCriticalSection(&m_mutex->m_cs) != 0); #else return m_mutex->m_mutex.try_lock(); #endif } CondVar::CondVar(): m_cond_var(std::make_unique()) {} CondVar::~CondVar() { m_cond_var.reset(); } void CondVar::Wait(Mutex* mutex) { #ifndef KYTY_WIN_CS std::unique_lock cpp_lock(mutex->m_mutex->m_mutex, std::adopt_lock_t()); #endif auto poll_callback = [&] { auto* callback = g_cond_wait_poll_callback; if (callback == nullptr) { return; } #if defined(KYTY_WIN_CS) LeaveCriticalSection(&mutex->m_mutex->m_cs); callback(); EnterCriticalSection(&mutex->m_mutex->m_cs); #else cpp_lock.unlock(); callback(); cpp_lock.lock(); #endif }; #ifdef KYTY_WIN_CS static auto func = ResolveSleepConditionVariableCS(); EXIT_NOT_IMPLEMENTED(func == nullptr); if (g_cond_wait_poll_callback == nullptr) { func(&m_cond_var->m_cv, &mutex->m_mutex->m_cs, INFINITE); } else { if (func(&m_cond_var->m_cv, &mutex->m_mutex->m_cs, 10) == 0 && GetLastError() == ERROR_TIMEOUT) { poll_callback(); } } #else if (g_cond_wait_poll_callback == nullptr) { m_cond_var->m_cv.wait(cpp_lock); } else { if (m_cond_var->m_cv.wait_for(cpp_lock, std::chrono::microseconds(10000)) == std::cv_status::timeout) { poll_callback(); } } cpp_lock.release(); #endif } void CondVar::SetWaitPollCallback(wait_poll_func_t callback) { g_cond_wait_poll_callback = callback; } bool CondVar::WaitFor(Mutex* mutex, uint32_t micros) { bool ok = false; #ifndef KYTY_WIN_CS std::unique_lock cpp_lock(mutex->m_mutex->m_mutex, std::adopt_lock_t()); #endif #ifdef KYTY_WIN_CS static auto func = ResolveSleepConditionVariableCS(); EXIT_NOT_IMPLEMENTED(func == nullptr); ok = !(func(&m_cond_var->m_cv, &mutex->m_mutex->m_cs, (micros < 1000 ? 1 : micros / 1000)) == 0 && GetLastError() == ERROR_TIMEOUT); #else ok = (m_cond_var->m_cv.wait_for(cpp_lock, std::chrono::microseconds(micros)) == std::cv_status::no_timeout); cpp_lock.release(); #endif return ok; } void CondVar::Signal() { #ifdef KYTY_WIN_CS static auto func = ResolveWakeConditionVariable(); EXIT_NOT_IMPLEMENTED(func == nullptr); func(&m_cond_var->m_cv); #else m_cond_var->m_cv.notify_one(); #endif } void CondVar::SignalAll() { #ifdef KYTY_WIN_CS static auto func = ResolveWakeAllConditionVariable(); EXIT_NOT_IMPLEMENTED(func == nullptr); func(&m_cond_var->m_cv); #else m_cond_var->m_cv.notify_all(); #endif } int Thread::GetThreadIdUnique() { static thread_local int tid = ++g_thread_counter; return tid; } } // namespace Common