Skip to content

Commit

Permalink
Add type-specialized vectorized hash table container (dotnet#100386)
Browse files Browse the repository at this point in the history
This PR adds a type-specialized, vectorized (using 128-bit SIMD) hash table container, and migrates one part of the mono runtime to use it instead of GHashTable. It also adds a basic test suite and basic benchmark suite. Vectorization is not enabled for it in the WASM build yet because we need to make changes to the build there. It is also not vectorized for ARM MSVC.
  • Loading branch information
kg authored and matouskozak committed Apr 30, 2024
1 parent 04f20fd commit cec506f
Show file tree
Hide file tree
Showing 30 changed files with 3,394 additions and 73 deletions.
7 changes: 6 additions & 1 deletion src/mono/mono/metadata/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ else()
set(metadata_platform_sources ${metadata_unix_sources})
endif()

set(imported_native_sources
../../../native/containers/dn-simdhash.c
../../../native/containers/dn-simdhash-string-ptr.c
../../../native/containers/dn-simdhash-u32-ptr.c)

set(metadata_common_sources
appdomain.c
domain.c
Expand Down Expand Up @@ -195,7 +200,7 @@ elseif(MONO_GC STREQUAL "boehm")
set(metadata_compile_definitions "HAVE_BOEHM_GC")
endif()

set(metadata_sources "${metadata_platform_sources};${metadata_common_sources};${metadata_gc_dependent_sources};${metadata_gc_sources};${ilgen_sources}")
set(metadata_sources "${metadata_platform_sources};${metadata_common_sources};${metadata_gc_dependent_sources};${metadata_gc_sources};${ilgen_sources};${imported_native_sources}")

if(HOST_WIN32 AND NOT DISABLE_SHARED_LIBS)
add_library(metadata_objects_shared OBJECT ${metadata_sources})
Expand Down
81 changes: 39 additions & 42 deletions src/mono/mono/metadata/class.c
Original file line number Diff line number Diff line change
Expand Up @@ -3037,18 +3037,22 @@ mono_image_init_name_cache (MonoImage *image)
const char *name;
const char *nspace;
guint32 visib, nspace_index;
GHashTable *name_cache2, *nspace_table, *the_name_cache;
dn_simdhash_u32_ptr_t *name_cache2;
dn_simdhash_string_ptr_t *nspace_table, *the_name_cache;

if (image->name_cache)
return;

the_name_cache = g_hash_table_new (g_str_hash, g_str_equal);
// TODO: Figure out a good initial capacity for this table by doing a scan,
// or just pre-reserve a reasonable amount of space based on how many nspaces
// an image typically has
the_name_cache = dn_simdhash_string_ptr_new (0, NULL);

if (image_is_dynamic (image)) {
mono_image_lock (image);
if (image->name_cache) {
/* Somebody initialized it before us */
g_hash_table_destroy (the_name_cache);
dn_simdhash_free (the_name_cache);
} else {
mono_atomic_store_release (&image->name_cache, the_name_cache);
}
Expand All @@ -3057,7 +3061,7 @@ mono_image_init_name_cache (MonoImage *image)
}

/* Temporary hash table to avoid lookups in the nspace_table */
name_cache2 = g_hash_table_new (NULL, NULL);
name_cache2 = dn_simdhash_u32_ptr_new (0, NULL);

/* FIXME: metadata-update */
int rows = table_info_get_rows (t);
Expand All @@ -3074,14 +3078,13 @@ mono_image_init_name_cache (MonoImage *image)
nspace = mono_metadata_string_heap (image, cols [MONO_TYPEDEF_NAMESPACE]);

nspace_index = cols [MONO_TYPEDEF_NAMESPACE];
nspace_table = (GHashTable *)g_hash_table_lookup (name_cache2, GUINT_TO_POINTER (nspace_index));
if (!nspace_table) {
nspace_table = g_hash_table_new (g_str_hash, g_str_equal);
g_hash_table_insert (the_name_cache, (char*)nspace, nspace_table);
g_hash_table_insert (name_cache2, GUINT_TO_POINTER (nspace_index),
nspace_table);
if (!dn_simdhash_u32_ptr_try_get_value (name_cache2, nspace_index, (void **)&nspace_table)) {
// FIXME: Compute an appropriate capacity for this table to avoid growing it
nspace_table = dn_simdhash_string_ptr_new (0, NULL);
dn_simdhash_string_ptr_try_add (the_name_cache, nspace, nspace_table);
dn_simdhash_u32_ptr_try_add (name_cache2, nspace_index, nspace_table);
}
g_hash_table_insert (nspace_table, (char *) name, GUINT_TO_POINTER (i));
dn_simdhash_string_ptr_try_add (nspace_table, name, GUINT_TO_POINTER (i));
}

/* Load type names from EXPORTEDTYPES table */
Expand All @@ -3102,23 +3105,22 @@ mono_image_init_name_cache (MonoImage *image)
nspace = mono_metadata_string_heap (image, exptype_cols [MONO_EXP_TYPE_NAMESPACE]);

nspace_index = exptype_cols [MONO_EXP_TYPE_NAMESPACE];
nspace_table = (GHashTable *)g_hash_table_lookup (name_cache2, GUINT_TO_POINTER (nspace_index));
if (!nspace_table) {
nspace_table = g_hash_table_new (g_str_hash, g_str_equal);
g_hash_table_insert (the_name_cache, (char*)nspace, nspace_table);
g_hash_table_insert (name_cache2, GUINT_TO_POINTER (nspace_index),
nspace_table);
if (!dn_simdhash_u32_ptr_try_get_value (name_cache2, nspace_index, (void **)&nspace_table)) {
// FIXME: Compute an appropriate capacity for this table to avoid growing it
nspace_table = dn_simdhash_string_ptr_new (0, NULL);
dn_simdhash_string_ptr_try_add (the_name_cache, nspace, nspace_table);
dn_simdhash_u32_ptr_try_add (name_cache2, nspace_index, nspace_table);
}
g_hash_table_insert (nspace_table, (char *) name, GUINT_TO_POINTER (mono_metadata_make_token (MONO_TABLE_EXPORTEDTYPE, i + 1)));
dn_simdhash_string_ptr_try_add (nspace_table, name, GUINT_TO_POINTER (mono_metadata_make_token (MONO_TABLE_EXPORTEDTYPE, i + 1)));
}
}

g_hash_table_destroy (name_cache2);
dn_simdhash_free (name_cache2);

mono_image_lock (image);
if (image->name_cache) {
/* Somebody initialized it before us */
g_hash_table_destroy (the_name_cache);
dn_simdhash_free (the_name_cache);
} else {
mono_atomic_store_release (&image->name_cache, the_name_cache);
}
Expand All @@ -3133,23 +3135,19 @@ void
mono_image_add_to_name_cache (MonoImage *image, const char *nspace,
const char *name, guint32 index)
{
GHashTable *nspace_table;
GHashTable *name_cache;
guint32 old_index;
dn_simdhash_string_ptr_t *nspace_table, *name_cache;

mono_image_init_name_cache (image);
mono_image_lock (image);

name_cache = image->name_cache;
if (!(nspace_table = (GHashTable *)g_hash_table_lookup (name_cache, nspace))) {
nspace_table = g_hash_table_new (g_str_hash, g_str_equal);
g_hash_table_insert (name_cache, (char *)nspace, (char *)nspace_table);
if (!dn_simdhash_string_ptr_try_get_value (name_cache, nspace, (void **)&nspace_table)) {
nspace_table = dn_simdhash_string_ptr_new (0, NULL);
dn_simdhash_string_ptr_try_add (name_cache, nspace, nspace_table);
}

if ((old_index = GPOINTER_TO_UINT (g_hash_table_lookup (nspace_table, (char*) name))))
g_error ("overrwritting old token %x on image %s for type %s::%s", old_index, image->name, nspace, name);

g_hash_table_insert (nspace_table, (char *) name, GUINT_TO_POINTER (index));
if (!dn_simdhash_string_ptr_try_add (nspace_table, name, GUINT_TO_POINTER (index)))
g_error ("overrwritting old token ? on image %s for type %s::%s", image->name, nspace, name);

mono_image_unlock (image);
}
Expand All @@ -3160,9 +3158,8 @@ typedef struct {
} FindAllUserData;

static void
find_all_nocase (gpointer key, gpointer value, gpointer user_data)
find_all_nocase (const char *name, gpointer value, gpointer user_data)
{
char *name = (char*)key;
FindAllUserData *data = (FindAllUserData*)user_data;
if (mono_utf8_strcasecmp (name, (char*)data->key) == 0)
data->values = g_slist_prepend (data->values, value);
Expand All @@ -3174,9 +3171,8 @@ typedef struct {
} FindUserData;

static void
find_nocase (gpointer key, gpointer value, gpointer user_data)
find_nocase (const char *name, gpointer value, gpointer user_data)
{
char *name = (char*)key;
FindUserData *data = (FindUserData*)user_data;

if (!data->value && (mono_utf8_strcasecmp (name, (char*)data->key) == 0))
Expand Down Expand Up @@ -3303,7 +3299,7 @@ search_modules (MonoImage *image, const char *name_space, const char *name, gboo
static MonoClass *
mono_class_from_name_checked_aux (MonoImage *image, const char* name_space, const char *name, GHashTable* visited_images, gboolean case_sensitive, MonoError *error)
{
GHashTable *nspace_table = NULL;
dn_simdhash_string_ptr_t *nspace_table = NULL;
MonoImage *loaded_image = NULL;
guint32 token = 0;
MonoClass *klass;
Expand Down Expand Up @@ -3350,23 +3346,24 @@ mono_class_from_name_checked_aux (MonoImage *image, const char* name_space, cons
mono_image_lock (image);

if (case_sensitive) {
nspace_table = (GHashTable *)g_hash_table_lookup (image->name_cache, name_space);

if (nspace_table)
token = GPOINTER_TO_UINT (g_hash_table_lookup (nspace_table, name));
if (dn_simdhash_string_ptr_try_get_value (image->name_cache, name_space, (void **)&nspace_table)) {
void * temp;
if (dn_simdhash_string_ptr_try_get_value (nspace_table, name, &temp))
token = GPOINTER_TO_UINT(temp);
}
} else {
FindAllUserData all_user_data = { name_space, NULL };
FindUserData user_data = { name, NULL };
GSList *values;

// We're forced to check all matching namespaces, not just the first one found,
// because our desired type could be in any of the ones that match case-insensitively.
g_hash_table_foreach (image->name_cache, find_all_nocase, &all_user_data);
dn_simdhash_string_ptr_foreach (image->name_cache, find_all_nocase, &all_user_data);

values = all_user_data.values;
while (values && !user_data.value) {
nspace_table = (GHashTable*)values->data;
g_hash_table_foreach (nspace_table, find_nocase, &user_data);
nspace_table = (dn_simdhash_string_ptr_t *)values->data;
dn_simdhash_string_ptr_foreach (nspace_table, find_nocase, &user_data);
values = values->next;
}

Expand Down
10 changes: 8 additions & 2 deletions src/mono/mono/metadata/image.c
Original file line number Diff line number Diff line change
Expand Up @@ -1969,6 +1969,12 @@ free_hash_table (gpointer key, gpointer val, gpointer user_data)
g_hash_table_destroy ((GHashTable*)val);
}

static void
free_simdhash_table (const char *key, gpointer val, gpointer user_data)
{
dn_simdhash_free ((dn_simdhash_t*)val);
}

/*
static void
free_mr_signatures (gpointer key, gpointer val, gpointer user_data)
Expand Down Expand Up @@ -2128,8 +2134,8 @@ mono_image_close_except_pools (MonoImage *image)
if (image->ptr_cache)
g_hash_table_destroy (image->ptr_cache);
if (image->name_cache) {
g_hash_table_foreach (image->name_cache, free_hash_table, NULL);
g_hash_table_destroy (image->name_cache);
dn_simdhash_string_ptr_foreach (image->name_cache, free_simdhash_table, NULL);
dn_simdhash_free (image->name_cache);
}

free_hash (image->icall_wrapper_cache);
Expand Down
4 changes: 3 additions & 1 deletion src/mono/mono/metadata/metadata-internals.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
#include <mono/utils/mono-error.h>
#include "mono/utils/mono-conc-hashtable.h"
#include "mono/utils/refcount.h"
// for dn_simdhash_string_ptr_t and dn_simdhash_u32_ptr_t
#include "../native/containers/dn-simdhash-specializations.h"

struct _MonoType {
union {
Expand Down Expand Up @@ -438,7 +440,7 @@ struct _MonoImage {
/*
* Indexes namespaces to hash tables that map class name to typedef token.
*/
GHashTable *name_cache; /*protected by the image lock*/
dn_simdhash_string_ptr_t *name_cache; /*protected by the image lock*/

/*
* Indexed by MonoClass
Expand Down
49 changes: 29 additions & 20 deletions src/mono/mono/metadata/metadata.c
Original file line number Diff line number Diff line change
Expand Up @@ -997,10 +997,10 @@ mono_metadata_table_bounds_check_slow (MonoImage *image, int table_index, int to
if (G_LIKELY (GINT_TO_UINT32(token_index) <= table_info_get_rows (&image->tables [table_index])))
return FALSE;

if (G_LIKELY (!image->has_updates))
return TRUE;
if (G_LIKELY (!image->has_updates))
return TRUE;

return mono_metadata_update_table_bounds_check (image, table_index, token_index);
return mono_metadata_update_table_bounds_check (image, table_index, token_index);
}

void
Expand Down Expand Up @@ -1094,7 +1094,7 @@ get_blob_heap (MonoImage *image)
static gboolean
mono_delta_heap_lookup (MonoImage *base_image, MetadataHeapGetterFunc get_heap, guint32 orig_index, MonoImage **image_out, guint32 *index_out)
{
return mono_metadata_update_delta_heap_lookup (base_image, get_heap, orig_index, image_out, index_out);
return mono_metadata_update_delta_heap_lookup (base_image, get_heap, orig_index, image_out, index_out);
}

/**
Expand Down Expand Up @@ -6451,12 +6451,12 @@ mono_metadata_events_from_typedef (MonoImage *meta, guint32 index, guint *end_id
}

start = mono_metadata_decode_row_col (tdef, loc.result, MONO_EVENT_MAP_EVENTLIST);
/*
* metadata-update: note this next line needs block needs to look at the number of rows in
* EventMap and Event of the base image. Updates will add rows for new properties,
* but they won't be contiguous. if we set end to the number of rows in the updated
* Property table, the range will include properties from some other class
*/
/*
* metadata-update: note this next line needs block needs to look at the number of rows in
* EventMap and Event of the base image. Updates will add rows for new properties,
* but they won't be contiguous. if we set end to the number of rows in the updated
* Property table, the range will include properties from some other class
*/
if (loc.result + 1 < table_info_get_rows (tdef)) {
end = mono_metadata_decode_row_col (tdef, loc.result + 1, MONO_EVENT_MAP_EVENTLIST) - 1;
} else {
Expand Down Expand Up @@ -6569,12 +6569,12 @@ mono_metadata_properties_from_typedef (MonoImage *meta, guint32 index, guint *en
}

start = mono_metadata_decode_row_col (tdef, loc.result, MONO_PROPERTY_MAP_PROPERTY_LIST);
/*
* metadata-update: note this next line needs block needs to look at the number of rows in
* PropertyMap and Property of the base image. Updates will add rows for new properties,
* but they won't be contiguous. if we set end to the number of rows in the updated
* Property table, the range will include properties from some other class
*/
/*
* metadata-update: note this next line needs block needs to look at the number of rows in
* PropertyMap and Property of the base image. Updates will add rows for new properties,
* but they won't be contiguous. if we set end to the number of rows in the updated
* Property table, the range will include properties from some other class
*/
if (loc.result + 1 < table_info_get_rows (&meta->tables [MONO_TABLE_PROPERTYMAP])) {
end = mono_metadata_decode_row_col (tdef, loc.result + 1, MONO_PROPERTY_MAP_PROPERTY_LIST) - 1;
} else {
Expand Down Expand Up @@ -7088,10 +7088,10 @@ mono_metadata_get_marshal_info (MonoImage *meta, guint32 idx, gboolean is_field)

gboolean found = tdef->base && mono_binary_search (&loc, tdef->base, table_info_get_rows (tdef), tdef->row_size, table_locator);

if (G_UNLIKELY (meta->has_updates)) {
if (!found && !mono_metadata_update_metadata_linear_search (meta, tdef, &loc, table_locator))
return NULL;
}
if (G_UNLIKELY (meta->has_updates)) {
if (!found && !mono_metadata_update_metadata_linear_search (meta, tdef, &loc, table_locator))
return NULL;
}

return mono_metadata_blob_heap (meta, mono_metadata_decode_row_col (tdef, loc.result, MONO_FIELD_MARSHAL_NATIVE_TYPE));
}
Expand Down Expand Up @@ -8055,3 +8055,12 @@ mono_metadata_get_method_params (MonoImage *image, uint32_t method_idx, uint32_t

return param_index;
}

// Required by dn_simdhash
void
dn_simdhash_assert_fail (const char *file, int line, const char *condition);

void
dn_simdhash_assert_fail (const char *file, int line, const char *condition) {
mono_assertion_message (file, line, condition);
}
10 changes: 5 additions & 5 deletions src/mono/mono/utils/atomic.h
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ mono_atomic_cas_i64 (volatile gint64 *dest, gint64 exch, gint64 comp)
(void)atomic_compare_exchange_strong ((volatile atomic_llong *)dest, (long long*)&comp, exch);
return comp;
#else
#error gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC
#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
#endif
}

Expand Down Expand Up @@ -188,7 +188,7 @@ mono_atomic_xchg_i64 (volatile gint64 *dest, gint64 exch)
g_static_assert (sizeof (atomic_llong) == sizeof (*dest) && ATOMIC_LLONG_LOCK_FREE == 2);
return atomic_exchange ((volatile atomic_llong *)dest, exch);
#else
#error gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC
#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
#endif
}

Expand Down Expand Up @@ -216,7 +216,7 @@ mono_atomic_fetch_add_i64 (volatile gint64 *dest, gint64 add)
g_static_assert (sizeof (atomic_llong) == sizeof (*dest) && ATOMIC_LLONG_LOCK_FREE == 2);
return atomic_fetch_add ((volatile atomic_llong *)dest, add);
#else
#error gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC
#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
#endif
}

Expand Down Expand Up @@ -250,7 +250,7 @@ mono_atomic_load_i64 (volatile gint64 *src)
g_static_assert (sizeof (atomic_llong) == sizeof (*src) && ATOMIC_LLONG_LOCK_FREE == 2);
return atomic_load ((volatile atomic_llong *)src);
#else
#error gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC
#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
#endif
}

Expand Down Expand Up @@ -292,7 +292,7 @@ mono_atomic_store_i64 (volatile gint64 *dst, gint64 val)
g_static_assert (sizeof (atomic_llong) == sizeof (*dst) && ATOMIC_LLONG_LOCK_FREE == 2);
atomic_store ((volatile atomic_llong *)dst, val);
#else
#error gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC
#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
#endif
}

Expand Down
Loading

0 comments on commit cec506f

Please sign in to comment.