Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Mono] Initial metadata update support #45612

Merged
merged 31 commits into from
Jan 11, 2021
Merged
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
ecf447d
Initial metadata-update prototype
lambdageek Nov 2, 2020
631525a
Add metadata-update.{c,h} to CMakeLists.txt
lambdageek Nov 2, 2020
d1dc08c
fix icall-def-netcore typo
lambdageek Nov 2, 2020
8b9992f
Add icall to corelib
lambdageek Nov 2, 2020
971b970
Add console and browser metadata update samples
lambdageek Nov 2, 2020
3b93069
Add README for mbr samples
lambdageek Nov 18, 2020
f5b6a73
[build] Add initial runtime support for MonoMetadataUpdate property
lambdageek Nov 18, 2020
135efa5
[runtime] ifdef out metadata updates if not enabled
lambdageek Nov 6, 2020
486b572
[wasm] set log mask to metadata-update
lambdageek Dec 3, 2020
0e37891
[mbr] Add InjectUpdate fn to sample
lambdageek Dec 3, 2020
d76c153
[metadata-update] don't merge heaps
lambdageek Dec 3, 2020
0de44c5
Don't make entrypoint public yet
lambdageek Nov 6, 2020
05e8c36
Add LoadMetadataUpdate to linker descriptor
lambdageek Dec 6, 2020
91080db
[wasm] add default Makefile variable value
lambdageek Dec 6, 2020
2f69e35
fix mono/mono CI
lambdageek Dec 6, 2020
e847fd8
remove mono/mono/tests/enc
lambdageek Dec 7, 2020
122e8bd
[metadata-update] Add per-thread exposed generation
lambdageek Dec 9, 2020
d98a6fa
[mbr] Fixup console sample
lambdageek Dec 15, 2020
a2a252e
[metadata-update] delete unused method
lambdageek Dec 15, 2020
44e165f
[mbr] Use 2 threads in console sample
lambdageek Dec 15, 2020
687516f
[metadata-update] Respect exposed generation in MethdDef RVA lookups
lambdageek Dec 15, 2020
a7c10c2
[interp] Expose latest metadata update before transforming methods
lambdageek Dec 15, 2020
e629fb6
[mbr] Update samples after rebase
lambdageek Dec 16, 2020
a0ab40e
[metadata-update] Don't fail after the first unsupported edit
lambdageek Dec 16, 2020
a402aa1
[metadata_update] Keep track of logical table sizes for deltas
lambdageek Nov 2, 2020
6429372
[metadata-update] Use a GList for MonoImage:delta_image
lambdageek Dec 18, 2020
d3fc78e
[metadata-update] add effective table lookup debug output
lambdageek Dec 18, 2020
dce3d2c
Address review feedback
lambdageek Jan 7, 2021
945b060
[interp] Save top interp frame at MINT_SAFEPOINT to ThreadContext
lambdageek Jan 9, 2021
d40eb92
[mbr] Extend console sample
lambdageek Jan 9, 2021
04e153c
[interp] Check mono_polling_required at safepoint
lambdageek Jan 9, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/mono/cmake/config.h.in
Original file line number Diff line number Diff line change
@@ -1768,8 +1768,12 @@
/* Enable runtime checks of mempool references between metadata images (must set env var MONO_CHECK_MODE=metadata) */
#cmakedefine ENABLE_CHECKED_BUILD_METADATA 1

/* Enable runtime support for metadata updates */
#cmakedefine ENABLE_METADATA_UPDATE 1

#if defined(ENABLE_LLVM) && defined(HOST_WIN32) && defined(TARGET_WIN32) && (!defined(TARGET_AMD64) || !defined(_MSC_VER))
#error LLVM for host=Windows and target=Windows is only supported on x64 MSVC build.
#endif

#endif

1 change: 1 addition & 0 deletions src/mono/cmake/options.cmake
Original file line number Diff line number Diff line change
@@ -65,6 +65,7 @@ option (ENABLE_CHECKED_BUILD_PRIVATE_TYPES "Enable compile time checking that ge
option (ENABLE_CHECKED_BUILD_GC "Enable runtime GC Safe / Unsafe mode assertion checks (must set env var MONO_CHECK_MODE=gc)")
option (ENABLE_CHECKED_BUILD_THREAD "Enable runtime history of per-thread coop state transitions (must set env var MONO_CHECK_MODE=thread)")
option (ENABLE_CHECKED_BUILD_METADATA "Enable runtime checks of mempool references between metadata images (must set env var MONO_CHECK_MODE=metadata)")
option (ENABLE_METADATA_UPDATE "Enable runtime support for metadata updates")

set (GC_SUSPEND "default" CACHE STRING "GC suspend method (default, preemptive, coop, hybrid)")
set (CHECKED_BUILD "" CACHE STRING "Set ENABLE_CHECKED_BUILD_ options at once. Comma-separated list of lowercase ENABLE_CHECKED_BUILD_ options ie. 'gc,threads,private_types' etc.")
10 changes: 10 additions & 0 deletions src/mono/mono.proj
Original file line number Diff line number Diff line change
@@ -8,6 +8,7 @@
- MonoAOTEnableLLVM - enable LLVM for an AOT-only Mono
- MonoAOTLLVMDir - [optional] the directory where LLVM is located, for an AOT-only Mono
- MonoVerboseBuild - enable verbose build
- MonoMetadataUpdate - enable experimental method body replacement code
-->

<PropertyGroup>
@@ -302,6 +303,12 @@
<_MonoCXXFLAGS Include="-Wl,--build-id=sha1" />
</ItemGroup>

<!-- Experimental features -->
<ItemGroup Condition="'$(MonoMetadataUpdate)' == 'true'">
<!-- FIXME: Add sanity check that the interpreter is enabled -->
<_MonoCMakeArgs Include="-DENABLE_METADATA_UPDATE=1" />
</ItemGroup>

<PropertyGroup>
<_MonoCFLAGSOption>-DCMAKE_C_FLAGS="@(_MonoCPPFLAGS, ' ') @(_MonoCFLAGS, ' ')"</_MonoCFLAGSOption>
<_MonoCXXFLAGSOption>-DCMAKE_CXX_FLAGS="@(_MonoCPPFLAGS, ' ') @(_MonoCXXFLAGS, ' ')"</_MonoCXXFLAGSOption>
@@ -403,6 +410,9 @@
<MonoAOTCMakeArgs Condition="'$(MonoAotOffsetsFile)' != ''" Include="-DAOT_OFFSETS_FILE=&quot;$(MonoAotOffsetsFile)&quot;" />
<MonoAOTCMakeArgs Condition="'$(MonoAOTEnableLLVM)' == 'true'" Include="-DLLVM_PREFIX=$(MonoAOTLLVMDir)" />
<MonoAOTCMakeArgs Include="$(_MonoAOTCXXFLAGSOption)" />
<!-- Experimental features -->
<!-- FIXME: Add sanity check that the interpreter is enabled -->
<MonoAOTCMakeArgs Condition="'$(MonoMetadataUpdate)' == 'true'" Include="-DENABLE_METADATA_UPDATE=1" />

<!-- Select generator platform for VS generator -->
<MonoAOTCMakeArgs Condition="'$(OS)' == 'Windows_NT' and '$(_MonoUseNinja)' != 'true' and '$(Platform)' == 'x64'" Include="-A x64" />
13 changes: 13 additions & 0 deletions src/mono/mono/cil/tables.def
Original file line number Diff line number Diff line change
@@ -44,3 +44,16 @@ TABLEDEF(MONO_TABLE_GENERICPARAM, "GenericParam") /* 0x2a */
TABLEDEF(MONO_TABLE_METHODSPEC, "MethodSpec")
TABLEDEF(MONO_TABLE_GENERICPARAMCONSTRAINT, "GenericParamConstraint")

TABLEDEF(MONO_TABLE_UNUSED8, "Unused8")
TABLEDEF(MONO_TABLE_UNUSED9, "Unused9")
TABLEDEF(MONO_TABLE_UNUSED10, "Unused10")

/* Portable PDB tables */
TABLEDEF(MONO_TABLE_DOCUMENT, "Document")
TABLEDEF(MONO_TABLE_METHODBODY, "Methodbody")
TABLEDEF(MONO_TABLE_LOCALSCOPE, "LocalScope")
TABLEDEF(MONO_TABLE_LOCALVARIABLE, "LocalVariable")
TABLEDEF(MONO_TABLE_LOCALCONSTANT, "LocalConstant")
TABLEDEF(MONO_TABLE_IMPORTSCOPE, "ImportScope")
TABLEDEF(MONO_TABLE_STATEMACHINEMETHOD, "StateMachineMethod")
TABLEDEF(MONO_TABLE_CUSTOMDEBUGINFORMATION, "CustomDebugInformation")
2 changes: 2 additions & 0 deletions src/mono/mono/metadata/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -128,6 +128,8 @@ set(metadata_common_sources
metadata.c
metadata-verify.c
metadata-internals.h
metadata-update.h
metadata-update.c
method-builder.h
method-builder-internals.h
method-builder.c
2 changes: 2 additions & 0 deletions src/mono/mono/metadata/Makefile.am
Original file line number Diff line number Diff line change
@@ -311,6 +311,8 @@ common_sources = \
metadata.c \
metadata-verify.c \
metadata-internals.h \
metadata-update.h \
metadata-update.c \
method-builder.h \
method-builder-internals.h \
method-builder.c \
2 changes: 1 addition & 1 deletion src/mono/mono/metadata/class.c
Original file line number Diff line number Diff line change
@@ -197,7 +197,7 @@ mono_class_from_typeref_checked (MonoImage *image, guint32 type_token, MonoError
break;
}

if (idx > image->tables [MONO_TABLE_ASSEMBLYREF].rows) {
if (mono_metadata_table_bounds_check (image, MONO_TABLE_ASSEMBLYREF, idx)) {
mono_error_set_bad_image (error, image, "Image with invalid assemblyref token %08x.", idx);
return NULL;
}
7 changes: 6 additions & 1 deletion src/mono/mono/metadata/domain.c
Original file line number Diff line number Diff line change
@@ -919,7 +919,12 @@ mono_cleanup (void)
void
mono_close_exe_image (void)
{
if (exe_image)
gboolean do_close = exe_image != NULL;
#ifdef ENABLE_METADATA_UPDATE
/* FIXME: shutdown hack. We mess something up and try to double-close/free it. */
do_close = do_close && !exe_image->delta_image;
#endif
if (do_close)
mono_image_close (exe_image);
}

4 changes: 4 additions & 0 deletions src/mono/mono/metadata/icall-decl.h
Original file line number Diff line number Diff line change
@@ -308,4 +308,8 @@ ICALL_EXPORT void ves_icall_System_Threading_LowLevelLifoSemaphore_ReleaseIn
ICALL_EXPORT void ves_icall_System_Runtime_Intrinsics_X86_X86Base___cpuidex (int abcd[4], int function_id, int subfunction_id);
#endif

#if defined(ENABLE_NETCORE) && defined(ENABLE_METADATA_UPDATE)
ICALL_EXPORT void ves_icall_Mono_Runtime_LoadMetadataUpdate (MonoAssembly *assm, gconstpointer dmeta_bytes, int32_t dmeta_len, gconstpointer dil_bytes, int32_t dil_len);
#endif

#endif // __MONO_METADATA_ICALL_DECL_H__
5 changes: 5 additions & 0 deletions src/mono/mono/metadata/icall-def-netcore.h
Original file line number Diff line number Diff line change
@@ -326,6 +326,11 @@ HANDLES_REUSE_WRAPPER(MPROP_3, "get_metadata_token", ves_icall_reflection_get_to
HANDLES(MPROP_4, "get_property_info", ves_icall_RuntimePropertyInfo_get_property_info, void, 3, (MonoReflectionProperty, MonoPropertyInfo_ref, PInfo))
HANDLES(MPROP_5, "internal_from_handle_type", ves_icall_System_Reflection_RuntimePropertyInfo_internal_from_handle_type, MonoReflectionProperty, 2, (MonoProperty_ptr, MonoType_ptr))

#ifdef ENABLE_METADATA_UPDATE
ICALL_TYPE(RUNF, "System.Runtime.CompilerServices.RuntimeFeature", RUNF_1)
NOHANDLES(ICALL(RUNF_1, "LoadMetadataUpdate_internal", ves_icall_Mono_Runtime_LoadMetadataUpdate))
#endif

ICALL_TYPE(RUNH, "System.Runtime.CompilerServices.RuntimeHelpers", RUNH_1)
HANDLES(RUNH_1, "GetObjectValue", ves_icall_System_Runtime_CompilerServices_RuntimeHelpers_GetObjectValue, MonoObject, 1, (MonoObject))
HANDLES(RUNH_2, "GetUninitializedObjectInternal", ves_icall_System_Runtime_CompilerServices_RuntimeHelpers_GetUninitializedObjectInternal, MonoObject, 1, (MonoType_ptr))
18 changes: 18 additions & 0 deletions src/mono/mono/metadata/icall.c
Original file line number Diff line number Diff line change
@@ -6777,6 +6777,24 @@ ves_icall_Mono_Runtime_DumpStateTotal (guint64 *portable_hash, guint64 *unportab
return result;
}

#if defined (ENABLE_NETCORE) && defined (ENABLE_METADATA_UPDATE)
void
ves_icall_Mono_Runtime_LoadMetadataUpdate (MonoAssembly *assm,
gconstpointer dmeta_bytes, int32_t dmeta_len,
lambdageek marked this conversation as resolved.
Show resolved Hide resolved
gconstpointer dil_bytes, int32_t dil_len)
{
ERROR_DECL (error);
g_assert (assm);
g_assert (dmeta_len >= 0);
MonoImage *image_base = assm->image;
g_assert (image_base);

MonoDomain *domain = mono_domain_get ();
mono_image_load_enc_delta (domain, image_base, dmeta_bytes, dmeta_len, dil_bytes, dil_len, error);
mono_error_set_pending_exception (error);
}
#endif

MonoBoolean
ves_icall_System_Reflection_AssemblyName_ParseAssemblyName (const char *name, MonoAssemblyName *aname, MonoBoolean *is_version_defined_arg, MonoBoolean *is_token_defined_arg)
{
80 changes: 77 additions & 3 deletions src/mono/mono/metadata/image.c
Original file line number Diff line number Diff line change
@@ -24,6 +24,7 @@
#include "tabledefs.h"
#include "tokentype.h"
#include "metadata-internals.h"
#include "metadata-update.h"
#include "profiler-private.h"
#include "loader.h"
#include "marshal.h"
@@ -45,6 +46,7 @@
#include <mono/metadata/verify.h>
#include <mono/metadata/image-internals.h>
#include <mono/metadata/loaded-images-internals.h>
#include <mono/metadata/metadata-update.h>
#include <mono/metadata/w32process-internals.h>
#include <mono/metadata/debug-internals.h>
#include <mono/metadata/mono-private-unstable.h>
@@ -598,7 +600,7 @@ load_metadata_ptrs (MonoImage *image, MonoCLIImageInfo *iinfo)
}

/*
* Load representation of logical metadata tables, from the "#~" stream
* Load representation of logical metadata tables, from the "#~" or "#-" stream
*/
static gboolean
load_tables (MonoImage *image)
@@ -613,6 +615,15 @@ load_tables (MonoImage *image)
image->idx_string_wide = ((heap_sizes & 0x01) == 1);
image->idx_guid_wide = ((heap_sizes & 0x02) == 2);
image->idx_blob_wide = ((heap_sizes & 0x04) == 4);

#ifdef ENABLE_METADATA_UPDATE
if (G_UNLIKELY (image->minimal_delta)) {
/* sanity check */
g_assert (image->idx_string_wide);
g_assert (image->idx_guid_wide);
g_assert (image->idx_blob_wide);
}
#endif

valid_mask = read64 (heap_tables + 8);
rows = (const guint32 *) (heap_tables + 24);
@@ -1462,6 +1473,26 @@ mono_is_problematic_image (MonoImage *image)
return FALSE;
}

#ifdef ENABLE_METADATA_UPDATE
static void
dump_encmap (MonoImage *image)
{
MonoTableInfo *encmap = &image->tables [MONO_TABLE_ENCMAP];
if (!encmap || !encmap->rows)
return;

if (mono_trace_is_traced (G_LOG_LEVEL_DEBUG, MONO_TRACE_METADATA_UPDATE)) {
mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_METADATA_UPDATE, "ENCMAP for %s", image->filename);
for (int i = 0; i < encmap->rows; ++i) {
guint32 cols [MONO_ENCMAP_SIZE];
mono_metadata_decode_row (encmap, i, cols, MONO_ENCMAP_SIZE);
int token = cols [MONO_ENCMAP_TOKEN];
mono_trace (G_LOG_LEVEL_DEBUG, MONO_TRACE_METADATA_UPDATE, "\t0x%08x: 0x%08x table: %s", i+1, token, mono_meta_table_name (mono_metadata_token_table (token)));
}
}
}
#endif

static MonoImage *
do_mono_image_load (MonoImage *image, MonoImageOpenStatus *status,
gboolean care_about_cli, gboolean care_about_pecoff)
@@ -1526,6 +1557,10 @@ do_mono_image_load (MonoImage *image, MonoImageOpenStatus *status,
if (image->loader == &pe_loader && !image->metadata_only && !mono_verifier_verify_table_data (image, error))
goto invalid_image;

#ifdef ENABLE_METADATA_UPDATE
dump_encmap (image);
#endif

mono_image_load_names (image);

mono_image_load_time_date_stamp (image);
@@ -2513,6 +2548,20 @@ mono_image_close_except_pools_all (MonoImage**images, int image_count)
}
}

#ifdef ENABLE_METADATA_UPDATE
static void
mono_image_close_except_pools_all_list (GList *images)
{
for (GList *ptr = images; ptr; ptr = ptr->next) {
MonoImage *image = (MonoImage *)ptr->data;
if (image) {
if (!mono_image_close_except_pools (image))
ptr->data = NULL;
}
}
}
#endif

/*
* Returns whether mono_image_close_finish() must be called as well.
* We must unload images in two steps because clearing the domain in
@@ -2552,6 +2601,10 @@ mono_image_close_except_pools (MonoImage *image)

mono_metadata_clean_for_image (image);

#ifdef ENABLE_METADATA_UPDATE
mono_metadata_update_cleanup_on_close (image);
#endif

/*
* The caches inside a MonoImage might refer to metadata which is stored in referenced
* assemblies, so we can't release these references in mono_assembly_close () since the
@@ -2679,6 +2732,11 @@ mono_image_close_except_pools (MonoImage *image)
mono_image_close_except_pools_all (image->modules, image->module_count);
g_free (image->modules_loaded);

#ifdef ENABLE_METADATA_UPDATE
if (image->delta_image)
mono_image_close_except_pools_all_list (image->delta_image);
#endif

mono_os_mutex_destroy (&image->szarray_cache_lock);
mono_os_mutex_destroy (&image->lock);

@@ -2705,6 +2763,20 @@ mono_image_close_all (MonoImage**images, int image_count)
g_free (images);
}

#ifdef ENABLE_METADATA_UPDATE
static void
mono_image_close_all_list (GList *images)
{
for (GList *ptr = images; ptr; ptr = ptr->next) {
MonoImage *image = (MonoImage *)ptr->data;
if (image)
mono_image_close_finish (image);
}

g_list_free (images);
}
#endif

void
mono_image_close_finish (MonoImage *image)
{
@@ -2723,6 +2795,10 @@ mono_image_close_finish (MonoImage *image)
mono_image_close_all (image->files, image->file_count);
mono_image_close_all (image->modules, image->module_count);

#ifdef ENABLE_METADATA_UPDATE
mono_image_close_all_list (image->delta_image);
#endif

#ifndef DISABLE_PERFCOUNTERS
/* FIXME: use an explicit subtraction method as soon as it's available */
mono_atomic_fetch_add_i32 (&mono_perfcounters->loader_bytes, -1 * mono_mempool_get_allocated (image->mempool));
@@ -3425,5 +3501,3 @@ mono_image_append_class_to_reflection_info_set (MonoClass *klass)
mono_image_unlock (image);
}



Loading