Introduce pthread_condattr_setclock()

This is one of the few POSIX APIs that was missing. It lets you choose a
monotonic clock for your condition variables. This might improve perf on
some platforms. It might also grant more flexibility with NTP configs. I
know Qt is one project that believes it needs this. To introduce this, I
needed to change some the *NSYNC APIs, to support passing a clock param.
There's also new benchmarks, demonstrating Cosmopolitan's supremacy over
many libc implementations when it comes to mutex performance. Cygwin has
an alarmingly bad pthread_mutex_t implementation. It is so bad that they
would have been significantly better off if they'd used naive spinlocks.

libc/calls/clock_gettime.c

@@ -24,6 +24,7 @@
 #include "libc/intrin/describeflags.h"
 #include "libc/intrin/strace.h"
 #include "libc/runtime/syslib.internal.h"
+#include "libc/sysv/consts/clock.h"
 
 #ifdef __aarch64__
 #define CGT_VDSO __vdsosym("LINUX_2.6.39", "__kernel_clock_gettime")
@@ -58,14 +59,43 @@ static int __clock_gettime_init(int clockid, struct timespec *ts) {
   return cgt(clockid, ts);
 }
 
+static int clock_gettime_impl(int clock, struct timespec *ts) {
+  int rc;
+  if (!IsLinux())
+    return __clock_gettime(clock, ts);
+TryAgain:
+
+  // Ensure fallback for old Linux sticks.
+  if (clock == 4 /* CLOCK_MONOTONIC_RAW */)
+    clock = CLOCK_MONOTONIC_RAW;
+
+  // Call appropriate implementation.
+  rc = __clock_gettime(clock, ts);
+
+  // CLOCK_MONOTONIC_RAW is Linux 2.6.28+ so not available on RHEL5
+  if (rc == -EINVAL && clock == 4 /* CLOCK_MONOTONIC_RAW */) {
+    CLOCK_MONOTONIC_RAW = CLOCK_MONOTONIC;
+    CLOCK_MONOTONIC_RAW_APPROX = CLOCK_MONOTONIC;
+    goto TryAgain;
+  }
+
+  return rc;
+}
+
 /**
  * Returns nanosecond time.
  *
  * @param clock supports the following values across OSes:
  *    - `CLOCK_REALTIME`
  *    - `CLOCK_MONOTONIC`
+ *    - `CLOCK_MONOTONIC_RAW`
+ *    - `CLOCK_MONOTONIC_RAW_APPROX`
+ *    - `CLOCK_REALTIME_FAST`
  *    - `CLOCK_REALTIME_COARSE`
+ *    - `CLOCK_REALTIME_PRECISE`
+ *    - `CLOCK_MONOTONIC_FAST`
  *    - `CLOCK_MONOTONIC_COARSE`
+ *    - `CLOCK_MONOTONIC_PRECISE`
  *    - `CLOCK_THREAD_CPUTIME_ID`
  *    - `CLOCK_PROCESS_CPUTIME_ID`
  * @param ts is where the result is stored (or null to do clock check)
@@ -80,7 +110,7 @@ static int __clock_gettime_init(int clockid, struct timespec *ts) {
  */
 int clock_gettime(int clock, struct timespec *ts) {
   // threads on win32 stacks call this so we can't asan check *ts
-  int rc = __clock_gettime(clock, ts);
+  int rc = clock_gettime_impl(clock, ts);
   if (rc) {
     errno = -rc;
     rc = -1;

libc/calls/clock_nanosleep.c

@@ -19,6 +19,7 @@
 #include "libc/calls/struct/timespec.h"
 #include "libc/dce.h"
 #include "libc/errno.h"
+#include "libc/sysv/consts/clock.h"
 #include "libc/sysv/consts/timer.h"
 
 /**
@@ -79,18 +80,32 @@
 errno_t clock_nanosleep(int clock, int flags,        //
                         const struct timespec *req,  //
                         struct timespec *rem) {
-  if (IsMetal()) {
+  if (IsMetal())
     return ENOSYS;
-  }
   if (clock == 127 ||              //
       (flags & ~TIMER_ABSTIME) ||  //
       req->tv_sec < 0 ||           //
-      !(0 <= req->tv_nsec && req->tv_nsec <= 999999999)) {
+      !(0 <= req->tv_nsec && req->tv_nsec <= 999999999))
     return EINVAL;
+  int rc;
+  errno_t err, old = errno;
+
+TryAgain:
+  // Ensure fallback for old Linux sticks.
+  if (IsLinux() && clock == 4 /* CLOCK_MONOTONIC_RAW */)
+    clock = CLOCK_MONOTONIC_RAW;
+
+  rc = sys_clock_nanosleep(clock, flags, req, rem);
+
+  // CLOCK_MONOTONIC_RAW is Linux 2.6.28+ so not available on RHEL5
+  if (IsLinux() && rc && errno == EINVAL &&
+      clock == 4 /* CLOCK_MONOTONIC_RAW */) {
+    CLOCK_MONOTONIC_RAW = CLOCK_MONOTONIC;
+    CLOCK_MONOTONIC_RAW_APPROX = CLOCK_MONOTONIC;
+    goto TryAgain;
   }
-  errno_t old = errno;
-  int rc = sys_clock_nanosleep(clock, flags, req, rem);
-  errno_t err = !rc ? 0 : errno;
+
+  err = !rc ? 0 : errno;
   errno = old;
   return err;
 }

libc/intrin/maps.h

@@ -53,6 +53,7 @@ void *__maps_pickaddr(size_t);
 void __maps_add(struct Map *);
 void __maps_free(struct Map *);
 void __maps_insert(struct Map *);
+bool __maps_track(char *, size_t);
 struct Map *__maps_alloc(void);
 struct Map *__maps_floor(const char *);
 void __maps_stack(char *, int, int, size_t, int, intptr_t);

libc/intrin/mmap.c

@@ -305,6 +305,28 @@ void __maps_insert(struct Map *map) {
   __maps_check();
 }
 
+static void __maps_track_insert(struct Map *map, char *addr, size_t size,
+                                uintptr_t map_handle) {
+  map->addr = addr;
+  map->size = size;
+  map->prot = PROT_READ | PROT_WRITE;
+  map->flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NOFORK;
+  map->hand = map_handle;
+  __maps_lock();
+  __maps_insert(map);
+  __maps_unlock();
+}
+
+bool __maps_track(char *addr, size_t size) {
+  struct Map *map;
+  do {
+    if (!(map = __maps_alloc()))
+      return false;
+  } while (map == MAPS_RETRY);
+  __maps_track_insert(map, addr, size, -1);
+  return true;
+}
+
 struct Map *__maps_alloc(void) {
   struct Map *map;
   uintptr_t tip = atomic_load_explicit(&__maps.freed, memory_order_relaxed);
@@ -321,14 +343,7 @@ struct Map *__maps_alloc(void) {
   if (sys.addr == MAP_FAILED)
     return 0;
   map = sys.addr;
-  map->addr = sys.addr;
-  map->size = gransz;
-  map->prot = PROT_READ | PROT_WRITE;
-  map->flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NOFORK;
-  map->hand = sys.maphandle;
-  __maps_lock();
-  __maps_insert(map);
-  __maps_unlock();
+  __maps_track_insert(map, sys.addr, gransz, sys.maphandle);
   for (int i = 1; i < gransz / sizeof(struct Map); ++i)
     __maps_free(map + i);
   return MAPS_RETRY;

libc/intrin/pthread_mutex_lock.c

@@ -57,12 +57,12 @@ static void pthread_mutex_lock_drepper(atomic_int *futex, char pshare) {
   LOCKTRACE("acquiring pthread_mutex_lock_drepper(%t)...", futex);
   if (word == 1)
     word = atomic_exchange_explicit(futex, 2, memory_order_acquire);
+  BLOCK_CANCELATION;
   while (word > 0) {
-    BLOCK_CANCELATION;
-    _weaken(nsync_futex_wait_)(futex, 2, pshare, 0);
-    ALLOW_CANCELATION;
+    _weaken(nsync_futex_wait_)(futex, 2, pshare, 0, 0);
     word = atomic_exchange_explicit(futex, 2, memory_order_acquire);
   }
+  ALLOW_CANCELATION;
 }
 
 static errno_t pthread_mutex_lock_recursive(pthread_mutex_t *mutex,

libc/intrin/sys_umtx_timedwait_uint.c

@@ -23,7 +23,7 @@
 int sys_umtx_timedwait_uint_cp(atomic_int *, int, int, size_t,
                                struct _umtx_time *) asm("sys_futex_cp");
 
-int sys_umtx_timedwait_uint(atomic_int *p, int expect, bool pshare,
+int sys_umtx_timedwait_uint(atomic_int *p, int expect, bool pshare, int clock,
                             const struct timespec *abstime) {
   int op;
   size_t size;
@@ -32,7 +32,7 @@ int sys_umtx_timedwait_uint(atomic_int *p, int expect, bool pshare,
     tm_p = 0;
     size = 0;
   } else {
-    timo._clockid = CLOCK_REALTIME;
+    timo._clockid = clock;
     timo._flags = UMTX_ABSTIME;
     timo._timeout = *abstime;
     tm_p = &timo;

libc/proc/proc.c

@@ -27,6 +27,7 @@
 #include "libc/errno.h"
 #include "libc/fmt/wintime.internal.h"
 #include "libc/intrin/dll.h"
+#include "libc/intrin/maps.h"
 #include "libc/intrin/strace.h"
 #include "libc/intrin/weaken.h"
 #include "libc/mem/leaks.h"
@@ -60,6 +61,8 @@
  * @fileoverview Windows Subprocess Management.
  */
 
+#define STACK_SIZE 65536
+
 struct Procs __proc;
 
 static textwindows void __proc_stats(int64_t h, struct rusage *ru) {
@@ -130,7 +133,11 @@ textwindows int __proc_harvest(struct Proc *pr, bool iswait4) {
 
 static textwindows dontinstrument uint32_t __proc_worker(void *arg) {
   struct CosmoTib tls;
+  char *sp = __builtin_frame_address(0);
   __bootstrap_tls(&tls, __builtin_frame_address(0));
+  __maps_track(
+      (char *)(((uintptr_t)sp + __pagesize - 1) & -__pagesize) - STACK_SIZE,
+      STACK_SIZE);
   for (;;) {
 
     // assemble a group of processes to wait on. if more than 64
@@ -238,7 +245,7 @@ static textwindows dontinstrument uint32_t __proc_worker(void *arg) {
 static textwindows void __proc_setup(void) {
   __proc.onbirth = CreateEvent(0, 0, 0, 0);     // auto reset
   __proc.haszombies = CreateEvent(0, 1, 0, 0);  // manual reset
-  __proc.thread = CreateThread(0, 65536, __proc_worker, 0,
+  __proc.thread = CreateThread(0, STACK_SIZE, __proc_worker, 0,
                                kNtStackSizeParamIsAReservation, 0);
 }
 

libc/sysv/consts.sh

@@ -577,10 +577,11 @@ syscon	clock	CLOCK_REALTIME_PRECISE			0			0			0			0			9			0			0			0			#
 syscon	clock	CLOCK_REALTIME_FAST			0			0			0			0			10			0			0			0			#
 syscon	clock	CLOCK_REALTIME_COARSE			5			5			0			0			10			0			0			2			# Linux 2.6.32+; bsd consensus; not available on RHEL5
 syscon	clock	CLOCK_MONOTONIC				1			1			6			6			4			3			3			1			# XNU/NT faked; could move backwards if NTP introduces negative leap second
+syscon	clock	CLOCK_MONOTONIC_RAW			4			4			4			4			4			3			3			1			# actually monotonic; not subject to NTP adjustments; Linux 2.6.28+; XNU/NT/FreeBSD/OpenBSD faked; not available on RHEL5 (will fallback to CLOCK_MONOTONIC)
+syscon	clock	CLOCK_MONOTONIC_RAW_APPROX		4			4			5			5			4			3			3			1			# goes faster on xnu, otherwise faked
 syscon	clock	CLOCK_MONOTONIC_PRECISE			1			1			6			6			11			3			3			1			#
 syscon	clock	CLOCK_MONOTONIC_FAST			1			1			6			6			12			3			3			1			#
 syscon	clock	CLOCK_MONOTONIC_COARSE			6			6			5			5			12			3			3			1			# Linux 2.6.32+; bsd consensus; not available on RHEL5
-syscon	clock	CLOCK_MONOTONIC_RAW			4			4			4			4			127			127			127			127			# actually monotonic; not subject to NTP adjustments; Linux 2.6.28+; XNU/NT/FreeBSD/OpenBSD faked; not available on RHEL5
 syscon	clock	CLOCK_PROCESS_CPUTIME_ID		2			2			12			12			15			2			0x40000000		4			# NetBSD lets you bitwise a PID into clockid_t
 syscon	clock	CLOCK_THREAD_CPUTIME_ID			3			3			16			16			14			4			0x20000000		5			#
 syscon	clock	CLOCK_PROF				127			127			127			127			2			127			2			127			#

libc/sysv/consts/CLOCK_MONOTONIC_RAW.S

@@ -1,2 +1,2 @@
 #include "libc/sysv/consts/syscon.internal.h"
-.syscon clock,CLOCK_MONOTONIC_RAW,4,4,4,4,127,127,127,127
+.syscon clock,CLOCK_MONOTONIC_RAW,4,4,4,4,4,3,3,1

libc/sysv/consts/CLOCK_MONOTONIC_RAW_APPROX.S

@@ -0,0 +1,2 @@
+#include "libc/sysv/consts/syscon.internal.h"
+.syscon clock,CLOCK_MONOTONIC_RAW_APPROX,4,4,5,5,4,3,3,1

libc/sysv/consts/clock.h

@@ -8,7 +8,8 @@ extern const int CLOCK_MONOTONIC;
 extern const int CLOCK_MONOTONIC_COARSE;
 extern const int CLOCK_MONOTONIC_FAST;
 extern const int CLOCK_MONOTONIC_PRECISE;
-extern const int CLOCK_MONOTONIC_RAW;
+extern int CLOCK_MONOTONIC_RAW;
+extern int CLOCK_MONOTONIC_RAW_APPROX;
 extern const int CLOCK_PROCESS_CPUTIME_ID;
 extern const int CLOCK_PROF;
 extern const int CLOCK_REALTIME_ALARM;
@@ -24,9 +25,19 @@ extern const int CLOCK_UPTIME_PRECISE;
 
 COSMOPOLITAN_C_END_
 
-#define CLOCK_REALTIME           0
-#define CLOCK_MONOTONIC          CLOCK_MONOTONIC
-#define CLOCK_PROCESS_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID
+#define CLOCK_REALTIME         0
+#define CLOCK_REALTIME_FAST    CLOCK_REALTIME_FAST
+#define CLOCK_REALTIME_PRECISE CLOCK_REALTIME_PRECISE
+#define CLOCK_REALTIME_COARSE  CLOCK_REALTIME_COARSE
+
+#define CLOCK_MONOTONIC            CLOCK_MONOTONIC
+#define CLOCK_MONOTONIC_RAW        CLOCK_MONOTONIC_RAW
+#define CLOCK_MONOTONIC_RAW_APPROX CLOCK_MONOTONIC_RAW_APPROX
+#define CLOCK_MONOTONIC_FAST       CLOCK_MONOTONIC_FAST
+#define CLOCK_MONOTONIC_PRECISE    CLOCK_MONOTONIC_PRECISE
+#define CLOCK_MONOTONIC_COARSE     CLOCK_MONOTONIC_COARSE
+
 #define CLOCK_THREAD_CPUTIME_ID  CLOCK_THREAD_CPUTIME_ID
+#define CLOCK_PROCESS_CPUTIME_ID CLOCK_PROCESS_CPUTIME_ID
 
 #endif /* COSMOPOLITAN_LIBC_SYSV_CONSTS_CLOCK_H_ */

libc/thread/freebsd.internal.h

@@ -47,7 +47,8 @@ struct _umtx_time {
   uint32_t _clockid;
 };
 
-int sys_umtx_timedwait_uint(_Atomic(int) *, int, bool, const struct timespec *);
+int sys_umtx_timedwait_uint(_Atomic(int) *, int, bool, int,
+                            const struct timespec *);
 
 COSMOPOLITAN_C_END_
 #endif /* COSMOPOLITAN_LIBC_THREAD_FREEBSD_INTERNAL_H_ */

libc/thread/itimer.c

@@ -24,10 +24,12 @@
 #include "libc/calls/struct/sigset.internal.h"
 #include "libc/calls/struct/timeval.h"
 #include "libc/cosmo.h"
+#include "libc/intrin/maps.h"
 #include "libc/intrin/strace.h"
 #include "libc/nt/enum/processcreationflags.h"
 #include "libc/nt/thread.h"
 #include "libc/str/str.h"
+#include "libc/sysv/consts/clock.h"
 #include "libc/sysv/consts/sicode.h"
 #include "libc/sysv/consts/sig.h"
 #include "libc/sysv/errfuns.h"
@@ -36,11 +38,17 @@
 #include "third_party/nsync/mu.h"
 #ifdef __x86_64__
 
+#define STACK_SIZE 65536
+
 struct IntervalTimer __itimer;
 
 static textwindows dontinstrument uint32_t __itimer_worker(void *arg) {
   struct CosmoTib tls;
-  __bootstrap_tls(&tls, __builtin_frame_address(0));
+  char *sp = __builtin_frame_address(0);
+  __bootstrap_tls(&tls, sp);
+  __maps_track(
+      (char *)(((uintptr_t)sp + __pagesize - 1) & -__pagesize) - STACK_SIZE,
+      STACK_SIZE);
   for (;;) {
     bool dosignal = false;
     struct timeval now, waituntil;
@@ -66,19 +74,18 @@ static textwindows dontinstrument uint32_t __itimer_worker(void *arg) {
       }
     }
     nsync_mu_unlock(&__itimer.lock);
-    if (dosignal) {
+    if (dosignal)
       __sig_generate(SIGALRM, SI_TIMER);
-    }
     nsync_mu_lock(&__itimer.lock);
-    nsync_cv_wait_with_deadline(&__itimer.cond, &__itimer.lock,
+    nsync_cv_wait_with_deadline(&__itimer.cond, &__itimer.lock, CLOCK_REALTIME,
                                 timeval_totimespec(waituntil), 0);
     nsync_mu_unlock(&__itimer.lock);
   }
   return 0;
 }
 
 static textwindows void __itimer_setup(void) {
-  __itimer.thread = CreateThread(0, 65536, __itimer_worker, 0,
+  __itimer.thread = CreateThread(0, STACK_SIZE, __itimer_worker, 0,
                                  kNtStackSizeParamIsAReservation, 0);
 }
 

libc/thread/pthread_barrier_wait.c

@@ -61,7 +61,7 @@ errno_t pthread_barrier_wait(pthread_barrier_t *barrier) {
   // wait for everyone else to arrive at barrier
   BLOCK_CANCELATION;
   while ((n = atomic_load_explicit(&barrier->_waiters, memory_order_acquire)))
-    nsync_futex_wait_(&barrier->_waiters, n, barrier->_pshared, 0);
+    nsync_futex_wait_(&barrier->_waiters, n, barrier->_pshared, 0, 0);
   ALLOW_CANCELATION;
 
   return 0;

libc/thread/pthread_cond_init.c

@@ -16,18 +16,21 @@
 │ TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR             │
 │ PERFORMANCE OF THIS SOFTWARE.                                                │
 ╚─────────────────────────────────────────────────────────────────────────────*/
+#include "libc/sysv/consts/clock.h"
 #include "libc/thread/thread.h"
 
 /**
- * Initializes condition.
+ * Initializes condition variable.
  *
  * @param attr may be null
  * @return 0 on success, or error number on failure
  */
 errno_t pthread_cond_init(pthread_cond_t *cond,
                           const pthread_condattr_t *attr) {
   *cond = (pthread_cond_t){0};
-  if (attr)
-    cond->_pshared = *attr;
+  if (attr) {
+    cond->_pshared = attr->_pshared;
+    cond->_clock = attr->_clock;
+  }
   return 0;
 }