diff options
Diffstat (limited to 'lib/mlibc/options/posix/generic/pthread-stubs.cpp')
| -rw-r--r-- | lib/mlibc/options/posix/generic/pthread-stubs.cpp | 1426 |
1 files changed, 0 insertions, 1426 deletions
diff --git a/lib/mlibc/options/posix/generic/pthread-stubs.cpp b/lib/mlibc/options/posix/generic/pthread-stubs.cpp deleted file mode 100644 index 9720bc2..0000000 --- a/lib/mlibc/options/posix/generic/pthread-stubs.cpp +++ /dev/null @@ -1,1426 +0,0 @@ - -#include <stddef.h> -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <pthread.h> -#include <unistd.h> -#include <errno.h> -#include <inttypes.h> - -#include <bits/ensure.h> -#include <frg/allocation.hpp> -#include <frg/array.hpp> -#include <mlibc/allocator.hpp> -#include <mlibc/debug.hpp> -#include <mlibc/posix-sysdeps.hpp> -#include <mlibc/thread.hpp> -#include <mlibc/tcb.hpp> -#include <mlibc/tid.hpp> -#include <mlibc/threads.hpp> - -static bool enableTrace = false; - -struct ScopeTrace { - ScopeTrace(const char *file, int line, const char *function) - : _file(file), _line(line), _function(function) { - if(!enableTrace) - return; - mlibc::infoLogger() << "trace: Enter scope " - << _file << ":" << _line << " (in function " - << _function << ")" << frg::endlog; - } - - ~ScopeTrace() { - if(!enableTrace) - return; - mlibc::infoLogger() << "trace: Exit scope" << frg::endlog; - } - -private: - const char *_file; - int _line; - const char *_function; -}; - -#define SCOPE_TRACE() ScopeTrace(__FILE__, __LINE__, __FUNCTION__) - -static constexpr unsigned int mutexRecursive = 1; -static constexpr unsigned int mutexErrorCheck = 2; - -// TODO: either use uint32_t or determine the bit based on sizeof(int). -static constexpr unsigned int mutex_owner_mask = (static_cast<uint32_t>(1) << 30) - 1; -static constexpr unsigned int mutex_waiters_bit = static_cast<uint32_t>(1) << 31; - -// Only valid for the internal __mlibc_m mutex of wrlocks. -static constexpr unsigned int mutex_excl_bit = static_cast<uint32_t>(1) << 30; - -static constexpr unsigned int rc_count_mask = (static_cast<uint32_t>(1) << 31) - 1; -static constexpr unsigned int rc_waiters_bit = static_cast<uint32_t>(1) << 31; - -static constexpr size_t default_stacksize = 0x200000; -static constexpr size_t default_guardsize = 4096; - -// ---------------------------------------------------------------------------- -// pthread_attr and pthread functions. -// ---------------------------------------------------------------------------- - -// pthread_attr functions. -int pthread_attr_init(pthread_attr_t *attr) { - *attr = pthread_attr_t{}; - attr->__mlibc_stacksize = default_stacksize; - attr->__mlibc_guardsize = default_guardsize; - attr->__mlibc_detachstate = PTHREAD_CREATE_JOINABLE; - return 0; -} - -int pthread_attr_destroy(pthread_attr_t *) { - return 0; -} - -int pthread_attr_getdetachstate(const pthread_attr_t *attr, int *detachstate) { - *detachstate = attr->__mlibc_detachstate; - return 0; -} -int pthread_attr_setdetachstate(pthread_attr_t *attr, int detachstate) { - if (detachstate != PTHREAD_CREATE_DETACHED && - detachstate != PTHREAD_CREATE_JOINABLE) - return EINVAL; - - attr->__mlibc_detachstate = detachstate; - return 0; -} - -int pthread_attr_getstacksize(const pthread_attr_t *__restrict attr, size_t *__restrict stacksize) { - *stacksize = attr->__mlibc_stacksize; - return 0; -} - -int pthread_attr_setstacksize(pthread_attr_t *attr, size_t stacksize) { - if (stacksize < PTHREAD_STACK_MIN) - return EINVAL; - attr->__mlibc_stacksize = stacksize; - return 0; -} - -int pthread_attr_getstackaddr(const pthread_attr_t *attr, void **stackaddr) { - *stackaddr = attr->__mlibc_stackaddr; - return 0; -} -int pthread_attr_setstackaddr(pthread_attr_t *attr, void *stackaddr) { - attr->__mlibc_stackaddr = stackaddr; - return 0; -} - -int pthread_attr_getstack(const pthread_attr_t *attr, void **stackaddr, size_t *stacksize) { - *stackaddr = attr->__mlibc_stackaddr; - *stacksize = attr->__mlibc_stacksize; - return 0; -} -int pthread_attr_setstack(pthread_attr_t *attr, void *stackaddr, size_t stacksize) { - if (stacksize < PTHREAD_STACK_MIN) - return EINVAL; - attr->__mlibc_stacksize = stacksize; - attr->__mlibc_stackaddr = stackaddr; - return 0; -} - -int pthread_attr_getguardsize(const pthread_attr_t *__restrict attr, size_t *__restrict guardsize) { - *guardsize = attr->__mlibc_guardsize; - return 0; -} -int pthread_attr_setguardsize(pthread_attr_t *attr, size_t guardsize) { - attr->__mlibc_guardsize = guardsize; - return 0; -} - -int pthread_attr_getscope(const pthread_attr_t *attr, int *scope) { - *scope = attr->__mlibc_scope; - return 0; -} -int pthread_attr_setscope(pthread_attr_t *attr, int scope) { - if (scope != PTHREAD_SCOPE_SYSTEM && - scope != PTHREAD_SCOPE_PROCESS) - return EINVAL; - if (scope == PTHREAD_SCOPE_PROCESS) - return ENOTSUP; - attr->__mlibc_scope = scope; - return 0; -} - -int pthread_attr_getinheritsched(const pthread_attr_t *attr, int *inheritsched) { - *inheritsched = attr->__mlibc_inheritsched; - return 0; -} -int pthread_attr_setinheritsched(pthread_attr_t *attr, int inheritsched) { - if (inheritsched != PTHREAD_INHERIT_SCHED && - inheritsched != PTHREAD_EXPLICIT_SCHED) - return EINVAL; - attr->__mlibc_inheritsched = inheritsched; - return 0; -} - -int pthread_attr_getschedparam(const pthread_attr_t *__restrict attr, struct sched_param *__restrict schedparam) { - *schedparam = attr->__mlibc_schedparam; - return 0; -} -int pthread_attr_setschedparam(pthread_attr_t *__restrict attr, const struct sched_param *__restrict schedparam) { - // TODO: this is supposed to return EINVAL for when the schedparam doesn't make sense - // for the given schedpolicy. - attr->__mlibc_schedparam = *schedparam; - return 0; -} - -int pthread_attr_getschedpolicy(const pthread_attr_t *__restrict attr, int *__restrict policy) { - *policy = attr->__mlibc_schedpolicy; - return 0; -} -int pthread_attr_setschedpolicy(pthread_attr_t *__restrict attr, int policy) { - if (policy != SCHED_FIFO && policy != SCHED_RR && - policy != SCHED_OTHER) - return EINVAL; - attr->__mlibc_schedpolicy = policy; - return 0; -} - -#if __MLIBC_LINUX_OPTION -int pthread_attr_getaffinity_np(const pthread_attr_t *__restrict attr, - size_t cpusetsize, cpu_set_t *__restrict cpusetp) { - if (!attr) - return EINVAL; - - if (!attr->__mlibc_cpuset) { - memset(cpusetp, -1, cpusetsize); - return 0; - } - - for (size_t cnt = cpusetsize; cnt < attr->__mlibc_cpusetsize; cnt++) - if (reinterpret_cast<char*>(attr->__mlibc_cpuset)[cnt] != '\0') - return ERANGE; - - auto p = memcpy(cpusetp, attr->__mlibc_cpuset, - std::min(cpusetsize, attr->__mlibc_cpusetsize)); - if (cpusetsize > attr->__mlibc_cpusetsize) - memset(p, '\0', cpusetsize - attr->__mlibc_cpusetsize); - - return 0; -} - -int pthread_attr_setaffinity_np(pthread_attr_t *__restrict attr, - size_t cpusetsize, const cpu_set_t *__restrict cpusetp) { - if (!attr) - return EINVAL; - - if (!cpusetp || !cpusetsize) { - attr->__mlibc_cpuset = NULL; - attr->__mlibc_cpusetsize = 0; - return 0; - } - - if (attr->__mlibc_cpusetsize != cpusetsize) { - auto newp = realloc(attr->__mlibc_cpuset, cpusetsize); - if (!newp) - return ENOMEM; - - attr->__mlibc_cpuset = static_cast<cpu_set_t*>(newp); - attr->__mlibc_cpusetsize = cpusetsize; - } - - memcpy(attr->__mlibc_cpuset, cpusetp, cpusetsize); - return 0; -} - -int pthread_attr_getsigmask_np(const pthread_attr_t *__restrict attr, - sigset_t *__restrict sigmask) { - if (!attr) - return EINVAL; - - if (!attr->__mlibc_sigmaskset) { - sigemptyset(sigmask); - return PTHREAD_ATTR_NO_SIGMASK_NP; - } - - *sigmask = attr->__mlibc_sigmask; - - return 0; -} -int pthread_attr_setsigmask_np(pthread_attr_t *__restrict attr, - const sigset_t *__restrict sigmask) { - if (!attr) - return EINVAL; - - if (!sigmask) { - attr->__mlibc_sigmaskset = 0; - return 0; - } - - attr->__mlibc_sigmask = *sigmask; - attr->__mlibc_sigmaskset = 1; - - // Filter out internally used signals. - sigdelset(&attr->__mlibc_sigmask, SIGCANCEL); - - return 0; -} - -namespace { - void get_own_stackinfo(void **stack_addr, size_t *stack_size) { - auto fp = fopen("/proc/self/maps", "r"); - if (!fp) { - mlibc::infoLogger() << "mlibc pthreads: /proc/self/maps does not exist! Producing incorrect" - " stack results!" << frg::endlog; - return; - } - - char line[256]; - auto sp = mlibc::get_sp(); - while (fgets(line, 256, fp)) { - uintptr_t from, to; - if(sscanf(line, "%" SCNxPTR "-%" SCNxPTR, &from, &to) != 2) - continue; - if (sp < to && sp > from) { - // We need to return the lowest byte of the stack. - *stack_addr = reinterpret_cast<void*>(from); - *stack_size = to - from; - fclose(fp); - return; - } - } - - fclose(fp); - } -} - -int pthread_getattr_np(pthread_t thread, pthread_attr_t *attr) { - auto tcb = reinterpret_cast<Tcb*>(thread); - *attr = pthread_attr_t{}; - - if (!tcb->stackAddr || !tcb->stackSize) { - get_own_stackinfo(&attr->__mlibc_stackaddr, &attr->__mlibc_stacksize); - } else { - attr->__mlibc_stacksize = tcb->stackSize; - attr->__mlibc_stackaddr = tcb->stackAddr; - } - - attr->__mlibc_guardsize = tcb->guardSize; - attr->__mlibc_detachstate = tcb->isJoinable ? PTHREAD_CREATE_JOINABLE : PTHREAD_CREATE_DETACHED; - mlibc::infoLogger() << "pthread_getattr_np(): Implementation is incomplete!" << frg::endlog; - return 0; -} - -int pthread_getaffinity_np(pthread_t thread, size_t cpusetsize, cpu_set_t *mask) { - MLIBC_CHECK_OR_ENOSYS(mlibc::sys_getthreadaffinity, ENOSYS); - return mlibc::sys_getthreadaffinity(reinterpret_cast<Tcb*>(thread)->tid, cpusetsize, mask); -} - -int pthread_setaffinity_np(pthread_t thread, size_t cpusetsize, const cpu_set_t *mask) { - MLIBC_CHECK_OR_ENOSYS(mlibc::sys_setthreadaffinity, ENOSYS); - return mlibc::sys_setthreadaffinity(reinterpret_cast<Tcb*>(thread)->tid, cpusetsize, mask); -} -#endif // __MLIBC_LINUX_OPTION - -extern "C" Tcb *__rtdl_allocateTcb(); - -// pthread functions. -int pthread_create(pthread_t *__restrict thread, const pthread_attr_t *__restrict attrp, - void *(*entry) (void *), void *__restrict user_arg) { - return mlibc::thread_create(thread, attrp, reinterpret_cast<void *>(entry), user_arg, false); -} - -pthread_t pthread_self(void) { - return reinterpret_cast<pthread_t>(mlibc::get_current_tcb()); -} - -int pthread_equal(pthread_t t1, pthread_t t2) { - if(t1 == t2) - return 1; - return 0; -} - -namespace { - struct key_global_info { - bool in_use; - - void (*dtor)(void *); - uint64_t generation; - }; - - constinit frg::array< - key_global_info, - PTHREAD_KEYS_MAX - > key_globals_{}; - - FutexLock key_mutex_; -} - -namespace mlibc { - __attribute__ ((__noreturn__)) void do_exit() { - sys_thread_exit(); - __builtin_unreachable(); - } -} - -__attribute__ ((__noreturn__)) void pthread_exit(void *ret_val) { - auto self = mlibc::get_current_tcb(); - - if (__atomic_load_n(&self->cancelBits, __ATOMIC_RELAXED) & tcbExitingBit) - mlibc::do_exit(); - - __atomic_fetch_or(&self->cancelBits, tcbExitingBit, __ATOMIC_RELAXED); - - auto hand = self->cleanupEnd; - while (hand) { - auto old = hand; - hand->func(hand->arg); - hand = hand->prev; - frg::destruct(getAllocator(), old); - } - - for (size_t j = 0; j < PTHREAD_DESTRUCTOR_ITERATIONS; j++) { - for (size_t i = 0; i < PTHREAD_KEYS_MAX; i++) { - if (auto v = pthread_getspecific(i)) { - key_mutex_.lock(); - auto dtor = key_globals_[i].dtor; - key_mutex_.unlock(); - - if (dtor) { - dtor(v); - (*self->localKeys)[i].value = nullptr; - } - } - } - } - - self->returnValue.voidPtr = ret_val; - __atomic_store_n(&self->didExit, 1, __ATOMIC_RELEASE); - mlibc::sys_futex_wake(&self->didExit); - - // TODO: clean up thread resources when we are detached. - - // TODO: do exit(0) when we're the only thread instead - mlibc::do_exit(); -} - -int pthread_join(pthread_t thread, void **ret) { - return mlibc::thread_join(thread, ret); -} - -int pthread_detach(pthread_t thread) { - auto tcb = reinterpret_cast<Tcb*>(thread); - if (!__atomic_load_n(&tcb->isJoinable, __ATOMIC_RELAXED)) - return EINVAL; - - int expected = 1; - if(!__atomic_compare_exchange_n(&tcb->isJoinable, &expected, 0, false, __ATOMIC_RELEASE, - __ATOMIC_RELAXED)) - return EINVAL; - - return 0; -} - -void pthread_cleanup_push(void (*func) (void *), void *arg) { - auto self = mlibc::get_current_tcb(); - - auto hand = frg::construct<Tcb::CleanupHandler>(getAllocator()); - hand->func = func; - hand->arg = arg; - hand->next = nullptr; - hand->prev = self->cleanupEnd; - - if (self->cleanupEnd) - self->cleanupEnd->next = hand; - - self->cleanupEnd = hand; - - if (!self->cleanupBegin) - self->cleanupBegin = self->cleanupEnd; -} - -void pthread_cleanup_pop(int execute) { - auto self = mlibc::get_current_tcb(); - - auto hand = self->cleanupEnd; - - if (self->cleanupEnd) - self->cleanupEnd = self->cleanupEnd->prev; - if (self->cleanupEnd) - self->cleanupEnd->next = nullptr; - - if (execute) - hand->func(hand->arg); - - frg::destruct(getAllocator(), hand); -} - -int pthread_setname_np(pthread_t thread, const char *name) { - auto tcb = reinterpret_cast<Tcb*>(thread); - - auto sysdep = MLIBC_CHECK_OR_ENOSYS(mlibc::sys_thread_setname, ENOSYS); - if(int e = sysdep(tcb, name); e) { - return e; - } - - return 0; -} - -int pthread_getname_np(pthread_t thread, char *name, size_t size) { - auto tcb = reinterpret_cast<Tcb*>(thread); - - auto sysdep = MLIBC_CHECK_OR_ENOSYS(mlibc::sys_thread_getname, ENOSYS); - if(int e = sysdep(tcb, name, size); e) { - return e; - } - - return 0; -} - -int pthread_setschedparam(pthread_t thread, int policy, const struct sched_param *param) { - auto tcb = reinterpret_cast<Tcb*>(thread); - - MLIBC_CHECK_OR_ENOSYS(mlibc::sys_setschedparam, ENOSYS); - if(int e = mlibc::sys_setschedparam(tcb, policy, param); e) { - return e; - } - - return 0; -} - -int pthread_getschedparam(pthread_t thread, int *policy, struct sched_param *param) { - auto tcb = reinterpret_cast<Tcb*>(thread); - - MLIBC_CHECK_OR_ENOSYS(mlibc::sys_getschedparam, ENOSYS); - if(int e = mlibc::sys_getschedparam(tcb, policy, param); e) { - return e; - } - - return 0; -} - -//pthread cancel functions - -extern "C" void __mlibc_do_cancel() { - //TODO(geert): for now the same as pthread_exit() - pthread_exit(PTHREAD_CANCELED); -} - -namespace { - - void sigcancel_handler(int signal, siginfo_t *info, void *ucontext) { - ucontext_t *uctx = static_cast<ucontext_t*>(ucontext); - // The function could be called from other signals, or from another - // process, in which case we should do nothing. - if (signal != SIGCANCEL || info->si_pid != getpid() || - info->si_code != SI_TKILL) - return; - - auto tcb = reinterpret_cast<Tcb*>(mlibc::get_current_tcb()); - int old_value = tcb->cancelBits; - - /* - * When a thread is marked with deferred cancellation and performs a blocking syscall, - * the spec mandates that the syscall can get interrupted before it has caused any side - * effects (e.g. before a read() has read any bytes from disk). If the syscall has - * already caused side effects it should return its partial work, and set the program - * counter just after the syscall. If the syscall hasn't caused any side effects, it - * should fail with EINTR and set the program counter to the syscall instruction. - * - * cancellable_syscall: - * test whether_a_cancel_is_queued - * je cancel - * syscall - * end_cancellable_syscall - * - * The mlibc::sys_before_cancellable_syscall sysdep should return 1 when the - * program counter is between the 'canellable_syscall' and 'end_cancellable_syscall' label. - */ - if (!(old_value & tcbCancelAsyncBit) && - mlibc::sys_before_cancellable_syscall && !mlibc::sys_before_cancellable_syscall(uctx)) - return; - - int bitmask = tcbCancelTriggerBit | tcbCancelingBit; - while (1) { - int new_value = old_value | bitmask; - - // Check if we are already cancelled or exiting - if (old_value == new_value || old_value & tcbExitingBit) - return; - - int current_value = old_value; - if (__atomic_compare_exchange_n(&tcb->cancelBits, ¤t_value, - new_value, true,__ATOMIC_RELAXED, __ATOMIC_RELAXED)) { - tcb->returnValue.voidPtr = PTHREAD_CANCELED; - - // Perform cancellation - __mlibc_do_cancel(); - - break; - } - - old_value = current_value; - } - } -} - -namespace mlibc { -namespace { - -struct PthreadSignalInstaller { - PthreadSignalInstaller() { - struct sigaction sa; - sa.sa_sigaction = sigcancel_handler; - sa.sa_flags = SA_SIGINFO; - auto e = ENOSYS; - if(sys_sigaction) - e = sys_sigaction(SIGCANCEL, &sa, NULL); - // Opt-out of cancellation support. - if(e == ENOSYS) - return; - __ensure(!e); - } -}; - -PthreadSignalInstaller pthread_signal_installer; - -} // anonymous namespace -} // namespace mlibc - -int pthread_setcanceltype(int type, int *oldtype) { - if (type != PTHREAD_CANCEL_DEFERRED && type != PTHREAD_CANCEL_ASYNCHRONOUS) - return EINVAL; - - auto self = reinterpret_cast<Tcb *>(mlibc::get_current_tcb()); - int old_value = self->cancelBits; - while (1) { - int new_value = old_value & ~tcbCancelAsyncBit; - if (type == PTHREAD_CANCEL_ASYNCHRONOUS) - new_value |= tcbCancelAsyncBit; - - if (oldtype) - *oldtype = ((old_value & tcbCancelAsyncBit) - ? PTHREAD_CANCEL_ASYNCHRONOUS - : PTHREAD_CANCEL_DEFERRED); - - // Avoid unecessary atomic op. - if (old_value == new_value) - break; - - int current_value = old_value; - if (__atomic_compare_exchange_n(&self->cancelBits, ¤t_value, - new_value, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { - - if (mlibc::tcb_async_cancelled(new_value)) - __mlibc_do_cancel(); - - break; - } - - old_value = current_value; - } - - return 0; -} -int pthread_setcancelstate(int state, int *oldstate) { - if (state != PTHREAD_CANCEL_ENABLE && state != PTHREAD_CANCEL_DISABLE) - return EINVAL; - - auto self = reinterpret_cast<Tcb *>(mlibc::get_current_tcb()); - int old_value = self->cancelBits; - while (1) { - int new_value = old_value & ~tcbCancelEnableBit; - if (state == PTHREAD_CANCEL_ENABLE) - new_value |= tcbCancelEnableBit; - - if (oldstate) - *oldstate = ((old_value & tcbCancelEnableBit) - ? PTHREAD_CANCEL_ENABLE - : PTHREAD_CANCEL_DISABLE); - - // Avoid unecessary atomic op. - if (old_value == new_value) - break; - - int current_value = old_value; - if (__atomic_compare_exchange_n(&self->cancelBits, ¤t_value, - new_value, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { - - if (mlibc::tcb_async_cancelled(new_value)) - __mlibc_do_cancel(); - - sigset_t set = {}; - sigaddset(&set, SIGCANCEL); - if (new_value & PTHREAD_CANCEL_ENABLE) - sigprocmask(SIG_UNBLOCK, &set, NULL); - else - sigprocmask(SIG_BLOCK, &set, NULL); - break; - } - - old_value = current_value; - } - - return 0; -} -void pthread_testcancel(void) { - auto self = reinterpret_cast<Tcb *>(mlibc::get_current_tcb()); - int value = self->cancelBits; - if ((value & tcbCancelEnableBit) && (value & tcbCancelTriggerBit)) { - __mlibc_do_cancel(); - __builtin_unreachable(); - } -} -int pthread_cancel(pthread_t thread) { - if (!mlibc::sys_tgkill) { - MLIBC_MISSING_SYSDEP(); - return ENOSYS; - } - - auto tcb = reinterpret_cast<Tcb *>(thread); - // Check if the TCB is valid, somewhat.. - if (tcb->selfPointer != tcb) - return ESRCH; - - int old_value = __atomic_load_n(&tcb->cancelBits, __ATOMIC_RELAXED); - while (1) { - int bitmask = tcbCancelTriggerBit; - - int new_value = old_value | bitmask; - if (old_value == new_value) - break; - - int current_value = old_value; - if (__atomic_compare_exchange_n(&tcb->cancelBits, ¤t_value, - new_value, true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) { - if (mlibc::tcb_cancel_enabled(new_value)) { - pid_t pid = getpid(); - - int res = mlibc::sys_tgkill(pid, tcb->tid, SIGCANCEL); - - current_value = __atomic_load_n(&tcb->cancelBits, __ATOMIC_RELAXED); - - // If we can't find the thread anymore, it's possible that it exited between - // us setting the cancel trigger bit, and us sending the signal. Check the - // cancelBits for tcbExitingBit to confirm that. - // XXX(qookie): This will be an use-after-free once we start freeing TCBs on - // exit. Perhaps the TCB should be refcounted. - if (!(res == ESRCH && (current_value & tcbExitingBit))) - return res; - } - - break; - } - - old_value = current_value; - } - - return 0; -} - -int pthread_atfork(void (*prepare) (void), void (*parent) (void), void (*child) (void)) { - auto self = mlibc::get_current_tcb(); - - auto hand = frg::construct<Tcb::AtforkHandler>(getAllocator()); - if (!hand) - return -1; - - hand->prepare = prepare; - hand->parent = parent; - hand->child = child; - hand->next = nullptr; - hand->prev = self->atforkEnd; - - if (self->atforkEnd) - self->atforkEnd->next = hand; - - self->atforkEnd = hand; - - if (!self->atforkBegin) - self->atforkBegin = self->atforkEnd; - - return 0; -} - -// ---------------------------------------------------------------------------- -// pthread_key functions. -// ---------------------------------------------------------------------------- - -int pthread_key_create(pthread_key_t *out, void (*destructor)(void *)) { - SCOPE_TRACE(); - - auto g = frg::guard(&key_mutex_); - - pthread_key_t key = PTHREAD_KEYS_MAX; - for (size_t i = 0; i < PTHREAD_KEYS_MAX; i++) { - if (!key_globals_[i].in_use) { - key = i; - break; - } - } - - if (key == PTHREAD_KEYS_MAX) - return EAGAIN; - - key_globals_[key].in_use = true; - key_globals_[key].dtor = destructor; - - *out = key; - - return 0; -} - -int pthread_key_delete(pthread_key_t key) { - SCOPE_TRACE(); - - auto g = frg::guard(&key_mutex_); - - if (key >= PTHREAD_KEYS_MAX || !key_globals_[key].in_use) - return EINVAL; - - key_globals_[key].in_use = false; - key_globals_[key].dtor = nullptr; - key_globals_[key].generation++; - - return 0; -} - -void *pthread_getspecific(pthread_key_t key) { - SCOPE_TRACE(); - - auto self = mlibc::get_current_tcb(); - auto g = frg::guard(&key_mutex_); - - if (key >= PTHREAD_KEYS_MAX || !key_globals_[key].in_use) - return nullptr; - - if (key_globals_[key].generation > (*self->localKeys)[key].generation) { - (*self->localKeys)[key].value = nullptr; - (*self->localKeys)[key].generation = key_globals_[key].generation; - } - - return (*self->localKeys)[key].value; -} - -int pthread_setspecific(pthread_key_t key, const void *value) { - SCOPE_TRACE(); - - auto self = mlibc::get_current_tcb(); - auto g = frg::guard(&key_mutex_); - - if (key >= PTHREAD_KEYS_MAX || !key_globals_[key].in_use) - return EINVAL; - - (*self->localKeys)[key].value = const_cast<void *>(value); - (*self->localKeys)[key].generation = key_globals_[key].generation; - - return 0; -} - -// ---------------------------------------------------------------------------- -// pthread_once functions. -// ---------------------------------------------------------------------------- - -static constexpr unsigned int onceComplete = 1; -static constexpr unsigned int onceLocked = 2; - -int pthread_once(pthread_once_t *once, void (*function) (void)) { - SCOPE_TRACE(); - - auto expected = __atomic_load_n(&once->__mlibc_done, __ATOMIC_ACQUIRE); - - // fast path: the function was already run. - while(!(expected & onceComplete)) { - if(!expected) { - // try to acquire the mutex. - if(!__atomic_compare_exchange_n(&once->__mlibc_done, - &expected, onceLocked, false, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE)) - continue; - - function(); - - // unlock the mutex. - __atomic_exchange_n(&once->__mlibc_done, onceComplete, __ATOMIC_RELEASE); - if(int e = mlibc::sys_futex_wake((int *)&once->__mlibc_done); e) - __ensure(!"sys_futex_wake() failed"); - return 0; - }else{ - // a different thread is currently running the initializer. - __ensure(expected == onceLocked); - if(int e = mlibc::sys_futex_wait((int *)&once->__mlibc_done, onceLocked, nullptr); e) - __ensure(!"sys_futex_wait() failed"); - expected = __atomic_load_n(&once->__mlibc_done, __ATOMIC_ACQUIRE); - } - } - - return 0; -} - -// ---------------------------------------------------------------------------- -// pthread_mutexattr and pthread_mutex functions. -// ---------------------------------------------------------------------------- - -// pthread_mutexattr functions -int pthread_mutexattr_init(pthread_mutexattr_t *attr) { - SCOPE_TRACE(); - return mlibc::thread_mutexattr_init(attr); -} - -int pthread_mutexattr_destroy(pthread_mutexattr_t *attr) { - SCOPE_TRACE(); - return mlibc::thread_mutexattr_destroy(attr); -} - -int pthread_mutexattr_gettype(const pthread_mutexattr_t *__restrict attr, int *__restrict type) { - return mlibc::thread_mutexattr_gettype(attr, type); -} - -int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) { - return mlibc::thread_mutexattr_settype(attr, type); -} - -int pthread_mutexattr_getrobust(const pthread_mutexattr_t *__restrict attr, - int *__restrict robust) { - *robust = attr->__mlibc_robust; - return 0; -} -int pthread_mutexattr_setrobust(pthread_mutexattr_t *attr, int robust) { - if (robust != PTHREAD_MUTEX_STALLED && robust != PTHREAD_MUTEX_ROBUST) - return EINVAL; - - attr->__mlibc_robust = robust; - return 0; -} - -int pthread_mutexattr_getpshared(const pthread_mutexattr_t *attr, int *pshared) { - *pshared = attr->__mlibc_pshared; - return 0; -} -int pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared) { - if (pshared != PTHREAD_PROCESS_PRIVATE && pshared != PTHREAD_PROCESS_SHARED) - return EINVAL; - - attr->__mlibc_pshared = pshared; - return 0; -} - -int pthread_mutexattr_getprotocol(const pthread_mutexattr_t *__restrict attr, - int *__restrict protocol) { - *protocol = attr->__mlibc_protocol; - return 0; -} - -int pthread_mutexattr_setprotocol(pthread_mutexattr_t *attr, int protocol) { - if (protocol != PTHREAD_PRIO_NONE && protocol != PTHREAD_PRIO_INHERIT - && protocol != PTHREAD_PRIO_PROTECT) - return EINVAL; - - attr->__mlibc_protocol = protocol; - return 0; -} - -int pthread_mutexattr_getprioceiling(const pthread_mutexattr_t *__restrict attr, - int *__restrict prioceiling) { - (void)attr; - (void)prioceiling; - return EINVAL; -} - -int pthread_mutexattr_setprioceiling(pthread_mutexattr_t *attr, int prioceiling) { - (void)attr; - (void)prioceiling; - return EINVAL; -} - -// pthread_mutex functions -int pthread_mutex_init(pthread_mutex_t *__restrict mutex, - const pthread_mutexattr_t *__restrict attr) { - SCOPE_TRACE(); - - return mlibc::thread_mutex_init(mutex, attr); -} - -int pthread_mutex_destroy(pthread_mutex_t *mutex) { - return mlibc::thread_mutex_destroy(mutex); -} - -int pthread_mutex_lock(pthread_mutex_t *mutex) { - SCOPE_TRACE(); - - return mlibc::thread_mutex_lock(mutex); -} - -int pthread_mutex_trylock(pthread_mutex_t *mutex) { - SCOPE_TRACE(); - - unsigned int this_tid = mlibc::this_tid(); - unsigned int expected = __atomic_load_n(&mutex->__mlibc_state, __ATOMIC_RELAXED); - if(!expected) { - // Try to take the mutex here. - if(__atomic_compare_exchange_n(&mutex->__mlibc_state, - &expected, this_tid, false, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE)) { - __ensure(!mutex->__mlibc_recursion); - mutex->__mlibc_recursion = 1; - return 0; - } - } else { - // If this (recursive) mutex is already owned by us, increment the recursion level. - if((expected & mutex_owner_mask) == this_tid) { - if(!(mutex->__mlibc_flags & mutexRecursive)) { - return EBUSY; - } - ++mutex->__mlibc_recursion; - return 0; - } - } - - return EBUSY; -} - -int pthread_mutex_timedlock(pthread_mutex_t *__restrict, - const struct timespec *__restrict) { - __ensure(!"Not implemented"); - __builtin_unreachable(); -} - -int pthread_mutex_unlock(pthread_mutex_t *mutex) { - SCOPE_TRACE(); - - return mlibc::thread_mutex_unlock(mutex); -} - -int pthread_mutex_consistent(pthread_mutex_t *) { - __ensure(!"Not implemented"); - __builtin_unreachable(); -} - -// ---------------------------------------------------------------------------- -// pthread_condattr and pthread_cond functions. -// ---------------------------------------------------------------------------- - -int pthread_condattr_init(pthread_condattr_t *attr) { - attr->__mlibc_pshared = PTHREAD_PROCESS_PRIVATE; - attr->__mlibc_clock = CLOCK_REALTIME; - return 0; -} - -int pthread_condattr_destroy(pthread_condattr_t *attr) { - memset(attr, 0, sizeof(*attr)); - return 0; -} - -int pthread_condattr_getclock(const pthread_condattr_t *__restrict attr, - clockid_t *__restrict clock) { - *clock = attr->__mlibc_clock; - return 0; -} - -int pthread_condattr_setclock(pthread_condattr_t *attr, clockid_t clock) { - if (clock != CLOCK_REALTIME && clock != CLOCK_MONOTONIC - && clock != CLOCK_MONOTONIC_RAW && clock != CLOCK_REALTIME_COARSE - && clock != CLOCK_MONOTONIC_COARSE && clock != CLOCK_BOOTTIME) - return EINVAL; - - attr->__mlibc_clock = clock; - return 0; -} - -int pthread_condattr_getpshared(const pthread_condattr_t *__restrict attr, - int *__restrict pshared) { - *pshared = attr->__mlibc_pshared; - return 0; -} - -int pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared) { - if (pshared != PTHREAD_PROCESS_PRIVATE && pshared != PTHREAD_PROCESS_SHARED) - return EINVAL; - - attr->__mlibc_pshared = pshared; - return 0; -} - -int pthread_cond_init(pthread_cond_t *__restrict cond, const pthread_condattr_t *__restrict attr) { - SCOPE_TRACE(); - - return mlibc::thread_cond_init(cond, attr); -} - -int pthread_cond_destroy(pthread_cond_t *cond) { - SCOPE_TRACE(); - - return mlibc::thread_cond_destroy(cond); -} - -int pthread_cond_wait(pthread_cond_t *__restrict cond, pthread_mutex_t *__restrict mutex) { - return pthread_cond_timedwait(cond, mutex, nullptr); -} - -int pthread_cond_timedwait(pthread_cond_t *__restrict cond, pthread_mutex_t *__restrict mutex, - const struct timespec *__restrict abstime) { - return mlibc::thread_cond_timedwait(cond, mutex, abstime); -} - -int pthread_cond_signal(pthread_cond_t *cond) { - SCOPE_TRACE(); - - return pthread_cond_broadcast(cond); -} - -int pthread_cond_broadcast(pthread_cond_t *cond) { - SCOPE_TRACE(); - - return mlibc::thread_cond_broadcast(cond); -} - -// ---------------------------------------------------------------------------- -// pthread_barrierattr and pthread_barrier functions. -// ---------------------------------------------------------------------------- - -int pthread_barrierattr_init(pthread_barrierattr_t *attr) { - attr->__mlibc_pshared = PTHREAD_PROCESS_PRIVATE; - return 0; -} - -int pthread_barrierattr_getpshared(const pthread_barrierattr_t *__restrict attr, - int *__restrict pshared) { - *pshared = attr->__mlibc_pshared; - return 0; -} - -int pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared) { - if (pshared != PTHREAD_PROCESS_SHARED && pshared != PTHREAD_PROCESS_PRIVATE) - return EINVAL; - - attr->__mlibc_pshared = pshared; - return 0; -} - -int pthread_barrierattr_destroy(pthread_barrierattr_t *) { - return 0; -} - -int pthread_barrier_init(pthread_barrier_t *__restrict barrier, - const pthread_barrierattr_t *__restrict attr, unsigned count) { - if (count == 0) - return EINVAL; - - barrier->__mlibc_waiting = 0; - barrier->__mlibc_inside = 0; - barrier->__mlibc_seq = 0; - barrier->__mlibc_count = count; - - // Since we don't implement these yet, set a flag to error later. - auto pshared = attr ? attr->__mlibc_pshared : PTHREAD_PROCESS_PRIVATE; - barrier->__mlibc_flags = pshared; - - return 0; -} - -int pthread_barrier_destroy(pthread_barrier_t *barrier) { - // Wait until there are no threads still using the barrier. - unsigned inside = 0; - do { - unsigned expected = __atomic_load_n(&barrier->__mlibc_inside, __ATOMIC_RELAXED); - if (expected == 0) - break; - - int e = mlibc::sys_futex_wait((int *)&barrier->__mlibc_inside, expected, nullptr); - if (e != 0 && e != EAGAIN && e != EINTR) - mlibc::panicLogger() << "mlibc: sys_futex_wait() returned error " << e << frg::endlog; - } while (inside > 0); - - memset(barrier, 0, sizeof *barrier); - return 0; -} - -int pthread_barrier_wait(pthread_barrier_t *barrier) { - if (barrier->__mlibc_flags != 0) { - mlibc::panicLogger() << "mlibc: pthread_barrier_t flags were non-zero" - << frg::endlog; - } - - // inside is incremented on entry and decremented on exit. - // This is used to synchronise with pthread_barrier_destroy, to ensure that a thread doesn't pass - // the barrier and immediately destroy its state while other threads still rely on it. - - __atomic_fetch_add(&barrier->__mlibc_inside, 1, __ATOMIC_ACQUIRE); - - auto leave = [&](){ - unsigned inside = __atomic_sub_fetch(&barrier->__mlibc_inside, 1, __ATOMIC_RELEASE); - if (inside == 0) - mlibc::sys_futex_wake((int *)&barrier->__mlibc_inside); - }; - - unsigned seq = __atomic_load_n(&barrier->__mlibc_seq, __ATOMIC_ACQUIRE); - - while (true) { - unsigned expected = __atomic_load_n(&barrier->__mlibc_waiting, __ATOMIC_RELAXED); - bool swapped = __atomic_compare_exchange_n(&barrier->__mlibc_waiting, &expected, expected + 1, false, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE); - - if (swapped) { - if (expected + 1 == barrier->__mlibc_count) { - // We were the last thread to hit the barrier. Reset waiters and wake the others. - __atomic_fetch_add(&barrier->__mlibc_seq, 1, __ATOMIC_ACQUIRE); - __atomic_store_n(&barrier->__mlibc_waiting, 0, __ATOMIC_RELEASE); - - mlibc::sys_futex_wake((int *)&barrier->__mlibc_seq); - - leave(); - return PTHREAD_BARRIER_SERIAL_THREAD; - } - - while (true) { - int e = mlibc::sys_futex_wait((int *)&barrier->__mlibc_seq, seq, nullptr); - if (e != 0 && e != EAGAIN && e != EINTR) - mlibc::panicLogger() << "mlibc: sys_futex_wait() returned error " << e << frg::endlog; - - unsigned newSeq = __atomic_load_n(&barrier->__mlibc_seq, __ATOMIC_ACQUIRE); - if (newSeq > seq) { - leave(); - return 0; - } - } - } - } -} - -// ---------------------------------------------------------------------------- -// pthread_rwlock functions. -// ---------------------------------------------------------------------------- - -namespace { - void rwlock_m_lock(pthread_rwlock_t *rw, bool excl) { - unsigned int m_expected = __atomic_load_n(&rw->__mlibc_m, __ATOMIC_RELAXED); - while(true) { - if(m_expected) { - __ensure(m_expected & mutex_owner_mask); - - // Try to set the waiters bit. - if(!(m_expected & mutex_waiters_bit)) { - unsigned int desired = m_expected | mutex_waiters_bit; - if(!__atomic_compare_exchange_n(&rw->__mlibc_m, - &m_expected, desired, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) - continue; - } - - // Wait on the futex. - mlibc::sys_futex_wait((int *)&rw->__mlibc_m, m_expected | mutex_waiters_bit, nullptr); - - // Opportunistically try to take the lock after we wake up. - m_expected = 0; - }else{ - // Try to lock the mutex. - unsigned int desired = 1; - if(excl) - desired |= mutex_excl_bit; - if(__atomic_compare_exchange_n(&rw->__mlibc_m, - &m_expected, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) - break; - } - } - } - - int rwlock_m_trylock(pthread_rwlock_t *rw, bool excl) { - unsigned int m_expected = __atomic_load_n(&rw->__mlibc_m, __ATOMIC_RELAXED); - if(!m_expected) { - // Try to lock the mutex. - unsigned int desired = 1; - if(excl) - desired |= mutex_excl_bit; - if(__atomic_compare_exchange_n(&rw->__mlibc_m, - &m_expected, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) - return 0; - } - - __ensure(m_expected & mutex_owner_mask); - - // POSIX says that this function should never block but also that - // readers should not be blocked by readers. We implement this by returning EAGAIN - // (and not EBUSY) if a reader would block a reader. - if(!excl && !(m_expected & mutex_excl_bit)) - return EAGAIN; - - return EBUSY; - } - - void rwlock_m_unlock(pthread_rwlock_t *rw) { - auto m = __atomic_exchange_n(&rw->__mlibc_m, 0, __ATOMIC_RELEASE); - if(m & mutex_waiters_bit) - mlibc::sys_futex_wake((int *)&rw->__mlibc_m); - } -} - -int pthread_rwlockattr_init(pthread_rwlockattr_t *attr) { - attr->__mlibc_pshared = PTHREAD_PROCESS_PRIVATE; - return 0; -} - -int pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *__restrict attr, - int *__restrict pshared) { - *pshared = attr->__mlibc_pshared; - return 0; -} - -int pthread_rwlockattr_setpshared(pthread_rwlockattr_t *attr, int pshared) { - if (pshared != PTHREAD_PROCESS_SHARED && pshared != PTHREAD_PROCESS_PRIVATE) - return EINVAL; - - attr->__mlibc_pshared = pshared; - return 0; -} - -int pthread_rwlockattr_destroy(pthread_rwlockattr_t *) { - return 0; -} - -int pthread_rwlock_init(pthread_rwlock_t *__restrict rw, const pthread_rwlockattr_t *__restrict attr) { - SCOPE_TRACE(); - rw->__mlibc_m = 0; - rw->__mlibc_rc = 0; - - // Since we don't implement this yet, set a flag to error later. - auto pshared = attr ? attr->__mlibc_pshared : PTHREAD_PROCESS_PRIVATE; - rw->__mlibc_flags = pshared; - return 0; -} - -int pthread_rwlock_destroy(pthread_rwlock_t *rw) { - __ensure(!rw->__mlibc_m); - __ensure(!rw->__mlibc_rc); - return 0; -} - -int pthread_rwlock_trywrlock(pthread_rwlock_t *rw) { - SCOPE_TRACE(); - - if (rw->__mlibc_flags != 0) { - mlibc::panicLogger() << "mlibc: pthread_rwlock_t flags were non-zero" - << frg::endlog; - } - - // Take the __mlibc_m mutex. - // Will be released in pthread_rwlock_unlock(). - if(int e = rwlock_m_trylock(rw, true)) - return e; - - // Check that there are no readers. - unsigned int rc_expected = __atomic_load_n(&rw->__mlibc_rc, __ATOMIC_ACQUIRE); - if(rc_expected) { - rwlock_m_unlock(rw); - return EBUSY; - } - - return 0; -} - -int pthread_rwlock_wrlock(pthread_rwlock_t *rw) { - SCOPE_TRACE(); - - if (rw->__mlibc_flags != 0) { - mlibc::panicLogger() << "mlibc: pthread_rwlock_t flags were non-zero" - << frg::endlog; - } - - // Take the __mlibc_m mutex. - // Will be released in pthread_rwlock_unlock(). - rwlock_m_lock(rw, true); - - // Now wait until there are no more readers. - unsigned int rc_expected = __atomic_load_n(&rw->__mlibc_rc, __ATOMIC_ACQUIRE); - while(true) { - if(!rc_expected) - break; - - __ensure(rc_expected & rc_count_mask); - - // Try to set the waiters bit. - if(!(rc_expected & rc_waiters_bit)) { - unsigned int desired = rc_expected | rc_count_mask; - if(!__atomic_compare_exchange_n(&rw->__mlibc_rc, - &rc_expected, desired, false, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE)) - continue; - } - - // Wait on the futex. - mlibc::sys_futex_wait((int *)&rw->__mlibc_rc, rc_expected | rc_waiters_bit, nullptr); - - // Re-check the reader counter. - rc_expected = __atomic_load_n(&rw->__mlibc_rc, __ATOMIC_ACQUIRE); - } - - return 0; -} - -int pthread_rwlock_tryrdlock(pthread_rwlock_t *rw) { - SCOPE_TRACE(); - - if (rw->__mlibc_flags != 0) { - mlibc::panicLogger() << "mlibc: pthread_rwlock_t flags were non-zero" - << frg::endlog; - } - - // Increment the reader count while holding the __mlibc_m mutex. - if(int e = rwlock_m_trylock(rw, false); e) - return e; - __atomic_fetch_add(&rw->__mlibc_rc, 1, __ATOMIC_ACQUIRE); - rwlock_m_unlock(rw); - - return 0; -} - -int pthread_rwlock_rdlock(pthread_rwlock_t *rw) { - SCOPE_TRACE(); - - if (rw->__mlibc_flags != 0) { - mlibc::panicLogger() << "mlibc: pthread_rwlock_t flags were non-zero" - << frg::endlog; - } - - // Increment the reader count while holding the __mlibc_m mutex. - rwlock_m_lock(rw, false); - __atomic_fetch_add(&rw->__mlibc_rc, 1, __ATOMIC_ACQUIRE); - rwlock_m_unlock(rw); - - return 0; -} - -int pthread_rwlock_unlock(pthread_rwlock_t *rw) { - SCOPE_TRACE(); - - unsigned int rc_expected = __atomic_load_n(&rw->__mlibc_rc, __ATOMIC_RELAXED); - if(!rc_expected) { - // We are doing a write-unlock. - rwlock_m_unlock(rw); - return 0; - }else{ - // We are doing a read-unlock. - while(true) { - unsigned int count = rc_expected & rc_count_mask; - __ensure(count); - - // Try to decrement the count. - if(count == 1 && (rc_expected & rc_waiters_bit)) { - unsigned int desired = 0; - if(!__atomic_compare_exchange_n(&rw->__mlibc_rc, - &rc_expected, desired, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) - continue; - - // Wake the futex. - mlibc::sys_futex_wake((int *)&rw->__mlibc_rc); - break; - }else{ - unsigned int desired = (rc_expected & ~rc_count_mask) | (count - 1); - if(!__atomic_compare_exchange_n(&rw->__mlibc_rc, - &rc_expected, desired, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED)) - continue; - break; - } - } - - return 0; - } -} - -int pthread_getcpuclockid(pthread_t, clockid_t *) { - mlibc::infoLogger() << "mlibc: pthread_getcpuclockid() always returns ENOENT" - << frg::endlog; - return ENOENT; -} |