Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make RID_Owner lock-free for fetching. #97465

Merged
merged 1 commit into from
Oct 4, 2024
Merged

Make RID_Owner lock-free for fetching. #97465

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 68 additions & 71 deletions core/templates/rid_owner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
#define RID_OWNER_H

#include "core/os/memory.h"
#include "core/os/spin_lock.h"
#include "core/os/mutex.h"
#include "core/string/print_string.h"
#include "core/templates/hash_set.h"
#include "core/templates/list.h"
Expand Down Expand Up @@ -69,42 +69,54 @@ class RID_AllocBase {

template <typename T, bool THREAD_SAFE = false>
class RID_Alloc : public RID_AllocBase {
T **chunks = nullptr;
struct Chunk {
T data;
uint32_t validator;
};
Chunk **chunks = nullptr;
uint32_t **free_list_chunks = nullptr;
uint32_t **validator_chunks = nullptr;

uint32_t elements_in_chunk;
uint32_t max_alloc = 0;
uint32_t alloc_count = 0;
uint32_t chunk_limit = 0;

const char *description = nullptr;

mutable SpinLock spin_lock;
mutable Mutex mutex;

_FORCE_INLINE_ RID _allocate_rid() {
if constexpr (THREAD_SAFE) {
spin_lock.lock();
mutex.lock();
}

if (alloc_count == max_alloc) {
//allocate a new chunk
uint32_t chunk_count = alloc_count == 0 ? 0 : (max_alloc / elements_in_chunk);
if (THREAD_SAFE && chunk_count == chunk_limit) {
mutex.unlock();
if (description != nullptr) {
ERR_FAIL_V_MSG(RID(), vformat("Element limit for RID of type '%s' reached.", String(description)));
} else {
ERR_FAIL_V_MSG(RID(), "Element limit reached.");
}
}

//grow chunks
chunks = (T **)memrealloc(chunks, sizeof(T *) * (chunk_count + 1));
chunks[chunk_count] = (T *)memalloc(sizeof(T) * elements_in_chunk); //but don't initialize

//grow validators
validator_chunks = (uint32_t **)memrealloc(validator_chunks, sizeof(uint32_t *) * (chunk_count + 1));
validator_chunks[chunk_count] = (uint32_t *)memalloc(sizeof(uint32_t) * elements_in_chunk);
if constexpr (!THREAD_SAFE) {
chunks = (Chunk **)memrealloc(chunks, sizeof(Chunk *) * (chunk_count + 1));
}
chunks[chunk_count] = (Chunk *)memalloc(sizeof(Chunk) * elements_in_chunk); //but don't initialize
//grow free lists
free_list_chunks = (uint32_t **)memrealloc(free_list_chunks, sizeof(uint32_t *) * (chunk_count + 1));
if constexpr (!THREAD_SAFE) {
free_list_chunks = (uint32_t **)memrealloc(free_list_chunks, sizeof(uint32_t *) * (chunk_count + 1));
}
free_list_chunks[chunk_count] = (uint32_t *)memalloc(sizeof(uint32_t) * elements_in_chunk);

//initialize
for (uint32_t i = 0; i < elements_in_chunk; i++) {
// Don't initialize chunk.
validator_chunks[chunk_count][i] = 0xFFFFFFFF;
chunks[chunk_count][i].validator = 0xFFFFFFFF;
free_list_chunks[chunk_count][i] = alloc_count + i;
}

Expand All @@ -122,14 +134,13 @@ class RID_Alloc : public RID_AllocBase {
id <<= 32;
id |= free_index;

validator_chunks[free_chunk][free_element] = validator;

validator_chunks[free_chunk][free_element] |= 0x80000000; //mark uninitialized bit
chunks[free_chunk][free_element].validator = validator;
chunks[free_chunk][free_element].validator |= 0x80000000; //mark uninitialized bit

alloc_count++;

if constexpr (THREAD_SAFE) {
spin_lock.unlock();
mutex.unlock();
}

return _make_from_id(id);
Expand All @@ -156,16 +167,10 @@ class RID_Alloc : public RID_AllocBase {
if (p_rid == RID()) {
return nullptr;
}
if constexpr (THREAD_SAFE) {
spin_lock.lock();
}

uint64_t id = p_rid.get_id();
uint32_t idx = uint32_t(id & 0xFFFFFFFF);
if (unlikely(idx >= max_alloc)) {
if constexpr (THREAD_SAFE) {
spin_lock.unlock();
}
return nullptr;
}

Expand All @@ -174,38 +179,26 @@ class RID_Alloc : public RID_AllocBase {

uint32_t validator = uint32_t(id >> 32);

Chunk &c = chunks[idx_chunk][idx_element];
if (unlikely(p_initialize)) {
if (unlikely(!(validator_chunks[idx_chunk][idx_element] & 0x80000000))) {
if constexpr (THREAD_SAFE) {
spin_lock.unlock();
}
if (unlikely(!(c.validator & 0x80000000))) {
ERR_FAIL_V_MSG(nullptr, "Initializing already initialized RID");
}

if (unlikely((validator_chunks[idx_chunk][idx_element] & 0x7FFFFFFF) != validator)) {
if constexpr (THREAD_SAFE) {
spin_lock.unlock();
}
if (unlikely((c.validator & 0x7FFFFFFF) != validator)) {
ERR_FAIL_V_MSG(nullptr, "Attempting to initialize the wrong RID");
}

validator_chunks[idx_chunk][idx_element] &= 0x7FFFFFFF; //initialized
c.validator &= 0x7FFFFFFF; //initialized

} else if (unlikely(validator_chunks[idx_chunk][idx_element] != validator)) {
if constexpr (THREAD_SAFE) {
spin_lock.unlock();
}
if ((validator_chunks[idx_chunk][idx_element] & 0x80000000) && validator_chunks[idx_chunk][idx_element] != 0xFFFFFFFF) {
} else if (unlikely(c.validator != validator)) {
if ((c.validator & 0x80000000) && c.validator != 0xFFFFFFFF) {
ERR_FAIL_V_MSG(nullptr, "Attempting to use an uninitialized RID");
}
return nullptr;
}

T *ptr = &chunks[idx_chunk][idx_element];

if constexpr (THREAD_SAFE) {
spin_lock.unlock();
}
T *ptr = &c.data;

return ptr;
}
Expand All @@ -222,14 +215,14 @@ class RID_Alloc : public RID_AllocBase {

_FORCE_INLINE_ bool owns(const RID &p_rid) const {
if constexpr (THREAD_SAFE) {
spin_lock.lock();
mutex.lock();
}

uint64_t id = p_rid.get_id();
uint32_t idx = uint32_t(id & 0xFFFFFFFF);
if (unlikely(idx >= max_alloc)) {
if constexpr (THREAD_SAFE) {
spin_lock.unlock();
mutex.unlock();
}
return false;
}
Expand All @@ -239,25 +232,25 @@ class RID_Alloc : public RID_AllocBase {

uint32_t validator = uint32_t(id >> 32);

bool owned = (validator != 0x7FFFFFFF) && (validator_chunks[idx_chunk][idx_element] & 0x7FFFFFFF) == validator;
bool owned = (validator != 0x7FFFFFFF) && (chunks[idx_chunk][idx_element].validator & 0x7FFFFFFF) == validator;

if constexpr (THREAD_SAFE) {
spin_lock.unlock();
mutex.unlock();
}

return owned;
}

_FORCE_INLINE_ void free(const RID &p_rid) {
if constexpr (THREAD_SAFE) {
spin_lock.lock();
mutex.lock();
}

uint64_t id = p_rid.get_id();
uint32_t idx = uint32_t(id & 0xFFFFFFFF);
if (unlikely(idx >= max_alloc)) {
if constexpr (THREAD_SAFE) {
spin_lock.unlock();
mutex.unlock();
}
ERR_FAIL();
}
Expand All @@ -266,26 +259,26 @@ class RID_Alloc : public RID_AllocBase {
uint32_t idx_element = idx % elements_in_chunk;

uint32_t validator = uint32_t(id >> 32);
if (unlikely(validator_chunks[idx_chunk][idx_element] & 0x80000000)) {
if (unlikely(chunks[idx_chunk][idx_element].validator & 0x80000000)) {
if constexpr (THREAD_SAFE) {
spin_lock.unlock();
mutex.unlock();
}
ERR_FAIL_MSG("Attempted to free an uninitialized or invalid RID.");
} else if (unlikely(validator_chunks[idx_chunk][idx_element] != validator)) {
ERR_FAIL_MSG("Attempted to free an uninitialized or invalid RID");
} else if (unlikely(chunks[idx_chunk][idx_element].validator != validator)) {
if constexpr (THREAD_SAFE) {
spin_lock.unlock();
mutex.unlock();
}
ERR_FAIL();
}

chunks[idx_chunk][idx_element].~T();
validator_chunks[idx_chunk][idx_element] = 0xFFFFFFFF; // go invalid
chunks[idx_chunk][idx_element].data.~T();
chunks[idx_chunk][idx_element].validator = 0xFFFFFFFF; // go invalid

alloc_count--;
free_list_chunks[alloc_count / elements_in_chunk][alloc_count % elements_in_chunk] = idx;

if constexpr (THREAD_SAFE) {
spin_lock.unlock();
mutex.unlock();
}
}

Expand All @@ -294,43 +287,49 @@ class RID_Alloc : public RID_AllocBase {
}
void get_owned_list(List<RID> *p_owned) const {
if constexpr (THREAD_SAFE) {
spin_lock.lock();
mutex.lock();
}
for (size_t i = 0; i < max_alloc; i++) {
uint64_t validator = validator_chunks[i / elements_in_chunk][i % elements_in_chunk];
uint64_t validator = chunks[i / elements_in_chunk][i % elements_in_chunk].validator;
if (validator != 0xFFFFFFFF) {
p_owned->push_back(_make_from_id((validator << 32) | i));
}
}
if constexpr (THREAD_SAFE) {
spin_lock.unlock();
mutex.unlock();
}
}

//used for fast iteration in the elements or RIDs
void fill_owned_buffer(RID *p_rid_buffer) const {
if constexpr (THREAD_SAFE) {
spin_lock.lock();
mutex.lock();
}
uint32_t idx = 0;
for (size_t i = 0; i < max_alloc; i++) {
uint64_t validator = validator_chunks[i / elements_in_chunk][i % elements_in_chunk];
uint64_t validator = chunks[i / elements_in_chunk][i % elements_in_chunk].validator;
if (validator != 0xFFFFFFFF) {
p_rid_buffer[idx] = _make_from_id((validator << 32) | i);
idx++;
}
}

if constexpr (THREAD_SAFE) {
spin_lock.unlock();
mutex.unlock();
}
}

void set_description(const char *p_descrption) {
description = p_descrption;
}

RID_Alloc(uint32_t p_target_chunk_byte_size = 65536) {
RID_Alloc(uint32_t p_target_chunk_byte_size = 65536, uint32_t p_maximum_number_of_elements = 262144) {
elements_in_chunk = sizeof(T) > p_target_chunk_byte_size ? 1 : (p_target_chunk_byte_size / sizeof(T));
if constexpr (THREAD_SAFE) {
chunk_limit = (p_maximum_number_of_elements / elements_in_chunk) + 1;
chunks = (Chunk **)memalloc(sizeof(Chunk *) * chunk_limit);
free_list_chunks = (uint32_t **)memalloc(sizeof(uint32_t *) * chunk_limit);
}
}

~RID_Alloc() {
Expand All @@ -339,27 +338,25 @@ class RID_Alloc : public RID_AllocBase {
alloc_count, description ? description : typeid(T).name()));

for (size_t i = 0; i < max_alloc; i++) {
uint64_t validator = validator_chunks[i / elements_in_chunk][i % elements_in_chunk];
uint64_t validator = chunks[i / elements_in_chunk][i % elements_in_chunk].validator;
if (validator & 0x80000000) {
continue; //uninitialized
}
if (validator != 0xFFFFFFFF) {
chunks[i / elements_in_chunk][i % elements_in_chunk].~T();
chunks[i / elements_in_chunk][i % elements_in_chunk].data.~T();
}
}
}

uint32_t chunk_count = max_alloc / elements_in_chunk;
for (uint32_t i = 0; i < chunk_count; i++) {
memfree(chunks[i]);
memfree(validator_chunks[i]);
memfree(free_list_chunks[i]);
}

if (chunks) {
memfree(chunks);
memfree(free_list_chunks);
memfree(validator_chunks);
}
}
};
Expand Down Expand Up @@ -419,8 +416,8 @@ class RID_PtrOwner {
alloc.set_description(p_descrption);
}

RID_PtrOwner(uint32_t p_target_chunk_byte_size = 65536) :
alloc(p_target_chunk_byte_size) {}
RID_PtrOwner(uint32_t p_target_chunk_byte_size = 65536, uint32_t p_maximum_number_of_elements = 262144) :
alloc(p_target_chunk_byte_size, p_maximum_number_of_elements) {}
};

template <typename T, bool THREAD_SAFE = false>
Expand Down Expand Up @@ -473,8 +470,8 @@ class RID_Owner {
void set_description(const char *p_descrption) {
alloc.set_description(p_descrption);
}
RID_Owner(uint32_t p_target_chunk_byte_size = 65536) :
alloc(p_target_chunk_byte_size) {}
RID_Owner(uint32_t p_target_chunk_byte_size = 65536, uint32_t p_maximum_number_of_elements = 262144) :
alloc(p_target_chunk_byte_size, p_maximum_number_of_elements) {}
};

#endif // RID_OWNER_H
Loading