_AtomicsShims.h

//===----------------------------------------------------------------------===//
//
// This source file is part of the Swift Atomics open source project
//
// Copyright (c) 2020 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//

#ifndef SWIFTATOMIC_HEADER_INCLUDED
#define SWIFTATOMIC_HEADER_INCLUDED 1

// Swift-importable shims for C atomics.
//
// Swift cannot import C's atomic types or any operations over them, so we need
// to meticulously wrap all of them in tiny importable types and functions.
//
// This file defines an atomic storage representation and 56 atomic operations
// for each of the 10 standard integer types in the Standard Library, as well as
// Bool and a double-word integer type. To prevent us from having to manually
// write/maintain a thousand or so functions, we use the C preprocessor to stamp
// these out.
//
// Supporting double-wide integers is tricky, because neither Swift nor C has
// standard integer representations for these on platforms where `Int` has a bit
// width of 64. Standard C can model 128-bit atomics through `_Atomic(struct
// pair)` where `pair` is a struct of two `intptr_t`s, but current Swift
// compilers (as of version 5.3) get confused by atomic structs
// (https://reviews.llvm.org/D86218). To work around that, we need to use the
// nonstandard `__uint128_t` type. The Swift compiler does seem to be able to
// deal with `_Atomic(__uint128_t)`, but it refuses to directly import
// `__uint128_t`, so that too needs to be wrapped in a dummy struct (which we
// call `_sa_dword`). We want to stamp out 128-bit atomics from the same code as
// regular atomics, so this requires all atomic operations to distinguish
// between the "exported" C type and the corresponding "internal"
// representation, and to explicitly convert between them as needed. This is
// done using the `SWIFTATOMIC_ENCODE_<swiftType>` and
// `SWIFTATOMIC_DECODE_<swiftType>` family of macros.
//
// In the case of compare-and-exchange, the conversion of the `expected`
// argument needs to go through a temporary stack variable that would result in
// slightly worse codegen for the regular single-word case, so we distinguish
// between `SIMPLE` and `COMPLEX` cmpxchg variants. The single variant assumes
// that the internal representation is identical to the exported type, and does
// away with the comparisons.
//
// FIXME: Upgrading from the preprocessor to gyb would perhaps make things more
// readable here.
//
// FIXME: Eliminate the encoding/decoding mechanism once the package requires a
// compiler that includes https://reviews.llvm.org/D86218.


#include <stdbool.h>
#include <stdint.h>
#include <stdatomic.h>
#include <assert.h>

#define SWIFTATOMIC_INLINE static inline __attribute__((__always_inline__))
#define SWIFTATOMIC_SWIFT_NAME(name) __attribute__((swift_name(#name)))

// Atomic fences
#define SWIFTATOMIC_THREAD_FENCE_FN(order)                              \
  SWIFTATOMIC_INLINE void _sa_thread_fence_##order(void)                \
  {                                                                     \
    atomic_thread_fence(memory_order_##order);                          \
  }

SWIFTATOMIC_THREAD_FENCE_FN(acquire)
SWIFTATOMIC_THREAD_FENCE_FN(release)
SWIFTATOMIC_THREAD_FENCE_FN(acq_rel)
SWIFTATOMIC_THREAD_FENCE_FN(seq_cst)

// Definition of an atomic storage type.
#define SWIFTATOMIC_STORAGE_TYPE(swiftType, cType, storageType)         \
  typedef struct {                                                      \
    _Atomic(storageType) value;                                         \
  } _sa_##swiftType                                                     \
  SWIFTATOMIC_SWIFT_NAME(_Atomic##swiftType##Storage);

// Storage value initializer
#define SWIFTATOMIC_PREPARE_FN(swiftType, cType, storageType)           \
  SWIFTATOMIC_INLINE                                                    \
  _sa_##swiftType _sa_prepare_##swiftType(cType value)                  \
  {                                                                     \
    _sa_##swiftType storage = { SWIFTATOMIC_ENCODE_##swiftType(value) }; \
    assert(atomic_is_lock_free(&storage.value));                        \
    return storage;                                                     \
  }

// Storage value disposal function
#define SWIFTATOMIC_DISPOSE_FN(swiftType, cType, storageType)           \
  SWIFTATOMIC_INLINE                                                    \
  cType _sa_dispose_##swiftType(_sa_##swiftType storage)                \
  {                                                                     \
    return SWIFTATOMIC_DECODE_##swiftType(storage.value);               \
  }

// Atomic load
#define SWIFTATOMIC_LOAD_FN(swiftType, cType, storageType, order)       \
  SWIFTATOMIC_INLINE                                                    \
  cType _sa_load_##order##_##swiftType(                                 \
    _sa_##swiftType *ptr)                                               \
  {                                                                     \
    return SWIFTATOMIC_DECODE_##swiftType(                              \
      atomic_load_explicit(&ptr->value,                                 \
                           memory_order_##order));                      \
  }

// Atomic store
#define SWIFTATOMIC_STORE_FN(swiftType, cType, storageType, order)      \
  SWIFTATOMIC_INLINE                                                    \
  void _sa_store_##order##_##swiftType(                                 \
    _sa_##swiftType *ptr,                                               \
    cType desired)                                                      \
  {                                                                     \
    atomic_store_explicit(&ptr->value,                                  \
                          SWIFTATOMIC_ENCODE_##swiftType(desired),      \
                          memory_order_##order);                        \
  }

// Atomic exchange
#define SWIFTATOMIC_EXCHANGE_FN(swiftType, cType, storageType, order)   \
  SWIFTATOMIC_INLINE                                                    \
  cType _sa_exchange_##order##_##swiftType(                             \
    _sa_##swiftType *ptr,                                               \
    cType desired)                                                      \
  {                                                                     \
    return SWIFTATOMIC_DECODE_##swiftType(                              \
      atomic_exchange_explicit(                                         \
        &ptr->value,                                                    \
        SWIFTATOMIC_ENCODE_##swiftType(desired),                        \
        memory_order_##order));                                         \
  }

// Atomic compare/exchange
#define SWIFTATOMIC_CMPXCHG_FN_SIMPLE(_kind, swiftType, cType, storageType, succ, fail) \
  SWIFTATOMIC_INLINE                                                    \
  bool                                                                  \
  _sa_cmpxchg_##_kind##_##succ##_##fail##_##swiftType(                  \
    _sa_##swiftType *ptr,                                               \
    cType *expected,                                                    \
    cType desired)                                                      \
  {                                                                     \
    return atomic_compare_exchange_##_kind##_explicit(                  \
      &ptr->value,                                                      \
      expected,                                                         \
      desired,                                                          \
      memory_order_##succ,                                              \
      memory_order_##fail);                                             \
  }

#define SWIFTATOMIC_CMPXCHG_FN_COMPLEX(_kind, swiftType, cType, storageType, succ, fail) \
  SWIFTATOMIC_INLINE                                                    \
  bool                                                                  \
  _sa_cmpxchg_##_kind##_##succ##_##fail##_##swiftType(                  \
    _sa_##swiftType *ptr,                                               \
    cType *expected,                                                    \
    cType desired)                                                      \
  {                                                                     \
    storageType _expected = SWIFTATOMIC_ENCODE_##swiftType(*expected);  \
    bool result = atomic_compare_exchange_##_kind##_explicit(           \
        &ptr->value,                                                    \
        &_expected,                                                     \
        SWIFTATOMIC_ENCODE_##swiftType(desired),                        \
        memory_order_##succ,                                            \
        memory_order_##fail);                                           \
    *expected = SWIFTATOMIC_DECODE_##swiftType(_expected);              \
    return result;                                                      \
  }

// Atomic integer operations
#define SWIFTATOMIC_INTEGER_FN(op, swiftType, cType, storageType, order) \
  SWIFTATOMIC_INLINE                                                    \
  cType _sa_fetch_##op##_##order##_##swiftType(                         \
    _sa_##swiftType *ptr,                                               \
    cType operand)                                                      \
  {                                                                     \
    return SWIFTATOMIC_DECODE_##swiftType(                              \
      atomic_fetch_##op##_explicit(                                     \
        &ptr->value,                                                    \
        SWIFTATOMIC_ENCODE_##swiftType(operand),                        \
        memory_order_##order));                                         \
    }

// Functions for each supported operation + memory ordering combination
#define SWIFTATOMIC_STORE_FNS(swiftType, cType, storageType)           \
  SWIFTATOMIC_STORE_FN(swiftType, cType, storageType, relaxed)         \
  SWIFTATOMIC_STORE_FN(swiftType, cType, storageType, release)         \
  SWIFTATOMIC_STORE_FN(swiftType, cType, storageType, seq_cst)

#define SWIFTATOMIC_LOAD_FNS(swiftType, cType, storageType)            \
  SWIFTATOMIC_LOAD_FN(swiftType, cType, storageType, relaxed)          \
  SWIFTATOMIC_LOAD_FN(swiftType, cType, storageType, acquire)          \
  SWIFTATOMIC_LOAD_FN(swiftType, cType, storageType, seq_cst)

#define SWIFTATOMIC_EXCHANGE_FNS(swiftType, cType, storageType)        \
  SWIFTATOMIC_EXCHANGE_FN(swiftType, cType, storageType, relaxed)      \
  SWIFTATOMIC_EXCHANGE_FN(swiftType, cType, storageType, acquire)      \
  SWIFTATOMIC_EXCHANGE_FN(swiftType, cType, storageType, release)      \
  SWIFTATOMIC_EXCHANGE_FN(swiftType, cType, storageType, acq_rel)      \
  SWIFTATOMIC_EXCHANGE_FN(swiftType, cType, storageType, seq_cst)

#define SWIFTATOMIC_CMPXCHG_FNS(variant, kind, swiftType, cType, storageType) \
  SWIFTATOMIC_CMPXCHG_FN_##variant(kind, swiftType, cType, storageType, relaxed, relaxed) \
  SWIFTATOMIC_CMPXCHG_FN_##variant(kind, swiftType, cType, storageType, acquire, relaxed) \
  SWIFTATOMIC_CMPXCHG_FN_##variant(kind, swiftType, cType, storageType, release, relaxed) \
  SWIFTATOMIC_CMPXCHG_FN_##variant(kind, swiftType, cType, storageType, acq_rel, relaxed) \
  SWIFTATOMIC_CMPXCHG_FN_##variant(kind, swiftType, cType, storageType, seq_cst, relaxed) \
  SWIFTATOMIC_CMPXCHG_FN_##variant(kind, swiftType, cType, storageType, acquire, acquire) \
  SWIFTATOMIC_CMPXCHG_FN_##variant(kind, swiftType, cType, storageType, acq_rel, acquire) \
  SWIFTATOMIC_CMPXCHG_FN_##variant(kind, swiftType, cType, storageType, seq_cst, acquire) \
  SWIFTATOMIC_CMPXCHG_FN_##variant(kind, swiftType, cType, storageType, seq_cst, seq_cst)

#define SWIFTATOMIC_INTEGER_FNS(op, swiftType, cType, storageType)     \
  SWIFTATOMIC_INTEGER_FN(op, swiftType, cType, storageType, relaxed)   \
  SWIFTATOMIC_INTEGER_FN(op, swiftType, cType, storageType, acquire)   \
  SWIFTATOMIC_INTEGER_FN(op, swiftType, cType, storageType, release)   \
  SWIFTATOMIC_INTEGER_FN(op, swiftType, cType, storageType, acq_rel)   \
  SWIFTATOMIC_INTEGER_FN(op, swiftType, cType, storageType, seq_cst)

#define SWIFTATOMIC_DEFINE_TYPE(variant, swiftType, cType, storageType) \
  SWIFTATOMIC_STORAGE_TYPE(swiftType, cType, storageType)              \
  SWIFTATOMIC_PREPARE_FN(swiftType, cType, storageType)                \
  SWIFTATOMIC_DISPOSE_FN(swiftType, cType, storageType)                \
  SWIFTATOMIC_LOAD_FNS(swiftType, cType, storageType)                  \
  SWIFTATOMIC_STORE_FNS(swiftType, cType, storageType)                 \
  SWIFTATOMIC_EXCHANGE_FNS(swiftType, cType, storageType)              \
  SWIFTATOMIC_CMPXCHG_FNS(variant, strong, swiftType, cType, storageType) \
  SWIFTATOMIC_CMPXCHG_FNS(variant, weak, swiftType, cType, storageType)

#define SWIFTATOMIC_DEFINE_INTEGER_TYPE(variant, swiftType, cType, storageType) \
  SWIFTATOMIC_DEFINE_TYPE(variant, swiftType, cType, storageType)      \
  SWIFTATOMIC_INTEGER_FNS(add, swiftType, cType, storageType)          \
  SWIFTATOMIC_INTEGER_FNS(sub, swiftType, cType, storageType)          \
  SWIFTATOMIC_INTEGER_FNS(or, swiftType, cType, storageType)           \
  SWIFTATOMIC_INTEGER_FNS(xor, swiftType, cType, storageType)          \
  SWIFTATOMIC_INTEGER_FNS(and, swiftType, cType, storageType)

// All known integer types
#define SWIFTATOMIC_ENCODE_Int(value) (value)
#define SWIFTATOMIC_DECODE_Int(value) (value)
SWIFTATOMIC_DEFINE_INTEGER_TYPE(SIMPLE, Int, intptr_t, intptr_t)

#define SWIFTATOMIC_ENCODE_Int8(value) (value)
#define SWIFTATOMIC_DECODE_Int8(value) (value)
SWIFTATOMIC_DEFINE_INTEGER_TYPE(SIMPLE, Int8, int8_t, int8_t)

#define SWIFTATOMIC_ENCODE_Int16(value) (value)
#define SWIFTATOMIC_DECODE_Int16(value) (value)
SWIFTATOMIC_DEFINE_INTEGER_TYPE(SIMPLE, Int16, int16_t, int16_t)

#define SWIFTATOMIC_ENCODE_Int32(value) (value)
#define SWIFTATOMIC_DECODE_Int32(value) (value)
SWIFTATOMIC_DEFINE_INTEGER_TYPE(SIMPLE, Int32, int32_t, int32_t)

#define SWIFTATOMIC_ENCODE_Int64(value) (value)
#define SWIFTATOMIC_DECODE_Int64(value) (value)
SWIFTATOMIC_DEFINE_INTEGER_TYPE(SIMPLE, Int64, int64_t, int64_t)

#define SWIFTATOMIC_ENCODE_UInt(value) (value)
#define SWIFTATOMIC_DECODE_UInt(value) (value)
SWIFTATOMIC_DEFINE_INTEGER_TYPE(SIMPLE, UInt, uintptr_t, uintptr_t)

#define SWIFTATOMIC_ENCODE_UInt8(value) (value)
#define SWIFTATOMIC_DECODE_UInt8(value) (value)
SWIFTATOMIC_DEFINE_INTEGER_TYPE(SIMPLE, UInt8, uint8_t, uint8_t)

#define SWIFTATOMIC_ENCODE_UInt16(value) (value)
#define SWIFTATOMIC_DECODE_UInt16(value) (value)
SWIFTATOMIC_DEFINE_INTEGER_TYPE(SIMPLE, UInt16, uint16_t, uint16_t)

#define SWIFTATOMIC_ENCODE_UInt32(value) (value)
#define SWIFTATOMIC_DECODE_UInt32(value) (value)
SWIFTATOMIC_DEFINE_INTEGER_TYPE(SIMPLE, UInt32, uint32_t, uint32_t)

#define SWIFTATOMIC_ENCODE_UInt64(value) (value)
#define SWIFTATOMIC_DECODE_UInt64(value) (value)
SWIFTATOMIC_DEFINE_INTEGER_TYPE(SIMPLE, UInt64, uint64_t, uint64_t)

// Atomic boolean
#define SWIFTATOMIC_ENCODE_Bool(value) (value)
#define SWIFTATOMIC_DECODE_Bool(value) (value)
SWIFTATOMIC_DEFINE_TYPE(SIMPLE, Bool, bool, bool)
SWIFTATOMIC_INTEGER_FNS(or, Bool, bool, bool)
SWIFTATOMIC_INTEGER_FNS(xor, Bool, bool, bool)
SWIFTATOMIC_INTEGER_FNS(and, Bool, bool, bool)

// Double wide atomics (__uint128_t on 64-bit platforms, uint64_t elsewhere)
#if __INTPTR_WIDTH__ == 64
#if !defined(__SIZEOF_INT128__)
#error "Double wide atomics need __uint128_t"
#endif
typedef struct {
  __uint128_t value;
} _sa_dword SWIFTATOMIC_SWIFT_NAME(DoubleWord);

SWIFTATOMIC_INLINE
_sa_dword _sa_dword_create(uintptr_t high, uintptr_t low) {
  _sa_dword result = { ((__uint128_t)high << 64) | (__uint128_t)low };
  return result;
}

SWIFTATOMIC_INLINE
uintptr_t _sa_dword_extract_high(_sa_dword value) {
  return (uintptr_t)(value.value >> 64);
}

SWIFTATOMIC_INLINE
uintptr_t _sa_dword_extract_low(_sa_dword value) {
  return (uintptr_t)value.value;
}

SWIFTATOMIC_INLINE
_sa_dword _sa_decode_dword(__uint128_t value) {
  _sa_dword result = { value };
  return result;
}

#define SWIFTATOMIC_ENCODE_DoubleWord(_value) (_value).value
#define SWIFTATOMIC_DECODE_DoubleWord(value) _sa_decode_dword(value)
SWIFTATOMIC_DEFINE_TYPE(COMPLEX, DoubleWord, _sa_dword, __uint128_t)

#elif __INTPTR_WIDTH__ == 32

typedef struct {
  uint64_t value;
} _sa_dword SWIFTATOMIC_SWIFT_NAME(DoubleWord);

SWIFTATOMIC_INLINE
_sa_dword _sa_dword_create(uintptr_t high, uintptr_t low) {
  _sa_dword result = { ((uint64_t)high << 32) | (uint64_t)low };
  return result;
}

SWIFTATOMIC_INLINE
uintptr_t _sa_dword_extract_high(_sa_dword value) {
  return (uintptr_t)(value.value >> 32);
}

SWIFTATOMIC_INLINE
uintptr_t _sa_dword_extract_low(_sa_dword value) {
  return (uintptr_t)value.value;
}

SWIFTATOMIC_INLINE
_sa_dword _sa_decode_dword(uint64_t value) {
  _sa_dword result = { value };
  return result;
}

#define SWIFTATOMIC_ENCODE_DoubleWord(_value) (_value).value
#define SWIFTATOMIC_DECODE_DoubleWord(value) _sa_decode_dword(value)
SWIFTATOMIC_DEFINE_TYPE(COMPLEX, DoubleWord, _sa_dword, uint64_t)

#else
#error "Unsupported intptr_t bit width"
#endif // __INTPTR_WIDTH

extern void _sa_retain_n(void *object, uint32_t n);
extern void _sa_release_n(void *object, uint32_t n);

#endif //SWIFTATOMIC_HEADER_INCLUDED