From 1440466e7119624851df5b0eecac17092360601f Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Wed, 28 Jun 2023 21:25:37 +0400 Subject: [PATCH 01/21] MEMORY64 HEAP proxy objects --- emcc.py | 3 +- src/library.js | 4 +- src/library_fs.js | 6 +- src/library_memfs.js | 6 +- src/library_strings.js | 13 ++-- src/library_syscall.js | 4 +- src/library_webgpu.js | 6 +- src/preamble.js | 142 ++++++++++++++++++++++++++++++++++++++- src/runtime_safe_heap.js | 8 +-- 9 files changed, 166 insertions(+), 26 deletions(-) diff --git a/emcc.py b/emcc.py index 1134bce7f9c9c..00d32a7b3be0b 100755 --- a/emcc.py +++ b/emcc.py @@ -3716,7 +3716,8 @@ def phase_binaryen(target, options, wasm_target): # >=2GB heap support requires pointers in JS to be unsigned. rather than # require all pointers to be unsigned by default, which increases code size # a little, keep them signed, and just unsign them here if we need that. - if settings.CAN_ADDRESS_2GB: + # for MEMORY64 heap proxies are created that handle unsigning internally + if settings.CAN_ADDRESS_2GB and not settings.MEMORY64: with ToolchainProfiler.profile_block('use_unsigned_pointers_in_js'): final_js = building.use_unsigned_pointers_in_js(final_js) diff --git a/src/library.js b/src/library.js index 57b729c55196b..eb88cf4326a07 100644 --- a/src/library.js +++ b/src/library.js @@ -419,7 +419,7 @@ mergeInto(LibraryManager.library, { ? (dest, src, num) => HEAPU8.copyWithin(dest, src, src + num) : (dest, src, num) => HEAPU8.set(HEAPU8.subarray(src, src+num), dest)`, #else - emscripten_memcpy_big: (dest, src, num) => HEAPU8.copyWithin(dest, src, src + num), + emscripten_memcpy_big: (dest, src, num) => HEAPU8.copyWithin(fixPointer(dest), fixPointer(src), fixPointer(src) + fixPointer(num)), #endif #endif @@ -3090,7 +3090,7 @@ mergeInto(LibraryManager.library, { // Used by wasm-emscripten-finalize to implement STACK_OVERFLOW_CHECK __handle_stack_overflow__deps: ['emscripten_stack_get_base', 'emscripten_stack_get_end', '$ptrToString'], __handle_stack_overflow: (requested) => { - requested = requested >>> 0; + requested = fixPointer(requested); var base = _emscripten_stack_get_base(); var end = _emscripten_stack_get_end(); abort(`stack overflow (Attempt to set SP to ${ptrToString(requested)}` + diff --git a/src/library_fs.js b/src/library_fs.js index da4a1e79bbb2b..7b81089567750 100644 --- a/src/library_fs.js +++ b/src/library_fs.js @@ -1133,7 +1133,7 @@ FS.staticInit();` + }, read: (stream, buffer, offset, length, position) => { #if CAN_ADDRESS_2GB - offset >>>= 0; + offset = fixPointer(offset); #endif if (length < 0 || position < 0) { throw new FS.ErrnoError({{{ cDefs.EINVAL }}}); @@ -1167,7 +1167,7 @@ FS.staticInit();` + }, write: (stream, buffer, offset, length, position, canOwn) => { #if CAN_ADDRESS_2GB - offset >>>= 0; + offset = fixPointer(offset); #endif if (length < 0 || position < 0) { throw new FS.ErrnoError({{{ cDefs.EINVAL }}}); @@ -1243,7 +1243,7 @@ FS.staticInit();` + }, msync: (stream, buffer, offset, length, mmapFlags) => { #if CAN_ADDRESS_2GB - offset >>>= 0; + offset = fixPointer(offset); #endif if (!stream.stream_ops.msync) { return 0; diff --git a/src/library_memfs.js b/src/library_memfs.js index fc01deae05180..259f6c5999c80 100644 --- a/src/library_memfs.js +++ b/src/library_memfs.js @@ -106,7 +106,7 @@ mergeInto(LibraryManager.library, { // Never shrinks the storage. expandFileStorage: function(node, newCapacity) { #if CAN_ADDRESS_2GB - newCapacity >>>= 0; + newCapacity = fixPointer(newCapacity); #endif var prevCapacity = node.contents ? node.contents.length : 0; if (prevCapacity >= newCapacity) return; // No need to expand, the storage was already large enough. @@ -124,7 +124,7 @@ mergeInto(LibraryManager.library, { // Performs an exact resize of the backing file storage to the given size, if the size is not exactly this, the storage is fully reallocated. resizeFileStorage: function(node, newSize) { #if CAN_ADDRESS_2GB - newSize >>>= 0; + newSize = fixPointer(newSize); #endif if (node.usedBytes == newSize) return; if (newSize == 0) { @@ -361,7 +361,7 @@ mergeInto(LibraryManager.library, { throw new FS.ErrnoError({{{ cDefs.ENOMEM }}}); } #if CAN_ADDRESS_2GB - ptr >>>= 0; + ptr = fixPointer(ptr); #endif HEAP8.set(contents, ptr); } diff --git a/src/library_strings.js b/src/library_strings.js index 6110ba8bb171f..8b2346136cca4 100644 --- a/src/library_strings.js +++ b/src/library_strings.js @@ -28,7 +28,7 @@ mergeInto(LibraryManager.library, { #endif $UTF8ArrayToString: (heapOrArray, idx, maxBytesToRead) => { #if CAN_ADDRESS_2GB - idx >>>= 0; + idx = fixPointer(idx); #endif var endIdx = idx + maxBytesToRead; #if TEXTDECODER @@ -119,7 +119,7 @@ mergeInto(LibraryManager.library, { assert(typeof ptr == 'number'); #endif #if CAN_ADDRESS_2GB - ptr >>>= 0; + ptr = fixPointer(ptr); #endif #if TEXTDECODER == 2 if (!ptr) return ''; @@ -155,7 +155,7 @@ mergeInto(LibraryManager.library, { */ $stringToUTF8Array: (str, heap, outIdx, maxBytesToWrite) => { #if CAN_ADDRESS_2GB - outIdx >>>= 0; + outIdx = fixPointer(outIdx); #endif #if ASSERTIONS assert(typeof str === 'string'); @@ -263,7 +263,7 @@ mergeInto(LibraryManager.library, { // object. $AsciiToString: (ptr) => { #if CAN_ADDRESS_2GB - ptr >>>= 0; + ptr = fixPointer(ptr); #endif var str = ''; while (1) { @@ -277,6 +277,9 @@ mergeInto(LibraryManager.library, { // address 'outPtr', null-terminated and encoded in ASCII form. The copy will // require at most str.length+1 bytes of space in the HEAP. $stringToAscii: (str, buffer) => { +#if CAN_ADDRESS_2GB + buffer = fixPointer(buffer); +#endif for (var i = 0; i < str.length; ++i) { #if ASSERTIONS assert(str.charCodeAt(i) === (str.charCodeAt(i) & 0xff)); @@ -430,7 +433,7 @@ mergeInto(LibraryManager.library, { // Returns the number of bytes written, EXCLUDING the null terminator. $stringToUTF32: (str, outPtr, maxBytesToWrite) => { #if CAN_ADDRESS_2GB - outPtr >>>= 0; + outPtr = fixPointer(outPtr); #endif #if ASSERTIONS assert(outPtr % 4 == 0, 'Pointer passed to stringToUTF32 must be aligned to four bytes!'); diff --git a/src/library_syscall.js b/src/library_syscall.js index 40ab08e493021..55c9ed9931faa 100644 --- a/src/library_syscall.js +++ b/src/library_syscall.js @@ -81,7 +81,7 @@ var SyscallsLibrary = { return 0; } #if CAN_ADDRESS_2GB - addr >>>= 0; + addr = fixPointer(addr); #endif var buffer = HEAPU8.slice(addr, addr + len); FS.msync(stream, buffer, offset, len, flags); @@ -143,7 +143,7 @@ var SyscallsLibrary = { var ptr = res.ptr; {{{ makeSetValue('allocated', 0, 'res.allocated', 'i32') }}}; #if CAN_ADDRESS_2GB - ptr >>>= 0; + ptr = fixPointer(ptr); #endif {{{ makeSetValue('addr', 0, 'ptr', '*') }}}; return 0; diff --git a/src/library_webgpu.js b/src/library_webgpu.js index c84e5a6ea5934..6f77a9dd302f8 100644 --- a/src/library_webgpu.js +++ b/src/library_webgpu.js @@ -1841,7 +1841,7 @@ var LibraryWebGPU = { if (size === 0) warnOnce('getMappedRange size=0 no longer means WGPU_WHOLE_MAP_SIZE'); - size = size >>> 0; + size = fixPointer(size); if (size === {{{ gpu.WHOLE_MAP_SIZE }}}) size = undefined; var mapped; @@ -1875,7 +1875,7 @@ var LibraryWebGPU = { if (size === 0) warnOnce('getMappedRange size=0 no longer means WGPU_WHOLE_MAP_SIZE'); - size = size >>> 0; + size = fixPointer(size); if (size === {{{ gpu.WHOLE_MAP_SIZE }}}) size = undefined; if (bufferWrapper.mapMode !== {{{ gpu.MapMode.Write }}}) { @@ -1916,7 +1916,7 @@ var LibraryWebGPU = { bufferWrapper.onUnmap = []; var buffer = bufferWrapper.object; - size = size >>> 0; + size = fixPointer(size); if (size === {{{ gpu.WHOLE_MAP_SIZE }}}) size = undefined; // `callback` takes (WGPUBufferMapAsyncStatus status, void * userdata) diff --git a/src/preamble.js b/src/preamble.js index f386199301b6c..c89dd7d28e9ad 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -134,11 +134,55 @@ var HEAP, var HEAP_DATA_VIEW; #endif +function fixPointer (ptr) { + if (typeof ptr === 'number' && ptr < 0) { + return ptr >>> 0; + } + if (typeof ptr === 'bigint') { + return Number(ptr); + } + return ptr; +} + +const getHeapBlock = (type, offset, length) => { + const maxSize = Math.min(wasmMemory.buffer.byteLength, 4 * 1024 * 1024 * 1024 - 8) + + const heap = wasmMemory.buffer + switch (type) { + case 'i1': + case 'i8': + return new Int8Array(heap, offset, length || maxSize / Int8Array.BYTES_PER_ELEMENT); + case 'u1': + case 'u8': + return new Uint8Array(heap, offset, length || maxSize / Uint8Array.BYTES_PER_ELEMENT); + case 'i16': + return new Int16Array(heap, offset, length || maxSize / Int16Array.BYTES_PER_ELEMENT); + case 'u16': + return new Uint16Array(heap, offset, length || maxSize / Uint16Array.BYTES_PER_ELEMENT); + case 'i32': + return new Int32Array(heap, offset, length || maxSize / Int32Array.BYTES_PER_ELEMENT); + case 'u32': + return new Uint32Array(heap, offset, length || maxSize / Uint32Array.BYTES_PER_ELEMENT); + case 'f32': + return new Float32Array(heap, offset, length || maxSize / Float32Array.BYTES_PER_ELEMENT); + case 'f64': + return new Float64Array(heap, offset, length || maxSize / Float64Array.BYTES_PER_ELEMENT); + case 'i64': + return new BigInt64Array(heap, offset, length || maxSize / BigInt64Array.BYTES_PER_ELEMENT); + case '*': + case 'u64': + return new BigUint64Array(heap, offset, length || maxSize / BigUint64Array.BYTES_PER_ELEMENT); + default: + throw new Error('Invalid type'); + } +} + function updateMemoryViews() { var b = wasmMemory.buffer; #if SUPPORT_BIG_ENDIAN Module['HEAP_DATA_VIEW'] = HEAP_DATA_VIEW = new DataView(b); #endif +#if !WASM_BIGINT Module['HEAP8'] = HEAP8 = new Int8Array(b); Module['HEAP16'] = HEAP16 = new Int16Array(b); Module['HEAP32'] = HEAP32 = new Int32Array(b); @@ -147,9 +191,101 @@ function updateMemoryViews() { Module['HEAPU32'] = HEAPU32 = new Uint32Array(b); Module['HEAPF32'] = HEAPF32 = new Float32Array(b); Module['HEAPF64'] = HEAPF64 = new Float64Array(b); -#if WASM_BIGINT - Module['HEAP64'] = HEAP64 = new BigInt64Array(b); - Module['HEAPU64'] = HEAPU64 = new BigUint64Array(b); +#else + var maxArraySize = Math.min(b.byteLength, 4 * 1024 * 1024 * 1024 - 8); + var proxyHandler = (type, hb) => ({ + heapBlock: hb, + copyWithin: function (target, start, end) { + target = fixPointer(target) + start = fixPointer(start) + end = fixPointer(end) + const bpe = hb.BYTES_PER_ELEMENT + if (target * bpe >= maxArraySize || start * bpe >= maxArraySize || (end && end * bpe >= maxArraySize)) { + var len = end - start + var targetArray = getHeapBlock(type, target * bpe, len) + var sourceArray = getHeapBlock(type, start * bpe, len) + targetArray.set(sourceArray) + return hb + } else { + return hb.copyWithin(target, start, end) + } + }, + setOverridden: function (array, offset) { + offset = fixPointer(offset) + var offsetReal = (offset || 0) * hb.BYTES_PER_ELEMENT + if (offsetReal >= maxArraySize || array.byteLength + offsetReal >= maxArraySize) { + var targetArray = getHeapBlock(type, offsetReal, array.length) + targetArray.set(array) + } else { + hb.set(array, offset) + } + }, + subarray: function (start, end) { + start = fixPointer(start) + end = fixPointer(end) + var startReal = (start || 0) * hb.BYTES_PER_ELEMENT + var endReal = end ? end * hb.BYTES_PER_ELEMENT : wasmMemory.byteLength + if (startReal >= maxArraySize || endReal >= maxArraySize) { + return getHeapBlock(type, startReal, endReal - startReal) + } else { + return hb.subarray(start, end) + } + }, + get: function (target, property) { + if (typeof property === 'number' || typeof property === 'bigint') { + var memoryOffset = fixPointer(property) * target.BYTES_PER_ELEMENT + if (memoryOffset >= maxArraySize) { + var heap = getHeapBlock(type, memoryOffset, 1); + return heap[0]; + } else { + return this.heapBlock[property] + } + } + + if (property === 'copyWithin') { + return this.copyWithin + } + + if (property === 'set') { + return this.setOverridden + } + + if (property === 'subarray') { + return this.subarray + } + + return this.heapBlock[property] + }, + set: function (target, property, value) { + if (typeof property === 'number' || typeof property === 'bigint') { + var memoryOffset = fixPointer(property) * target.BYTES_PER_ELEMENT + if (memoryOffset >= maxArraySize) { + var heap = getHeapBlock(type, memoryOffset, 1); + heap[0] = value; + return true; + } + } + + this.heapBlock[property] = value + return true; + }, + }) + function createMemoryProxy (type) + { + const block = getHeapBlock(type, 0) + return new Proxy(block, proxyHandler(type, block)); + } + Module["HEAP8"] = HEAP8 = createMemoryProxy('i8') + Module["HEAP16"] = HEAP16 = createMemoryProxy('i16') + Module["HEAP32"] = HEAP32 = createMemoryProxy('i32') + Module["HEAPU8"] = HEAPU8 = createMemoryProxy('u8') + Module["HEAPU16"] = HEAPU16 = createMemoryProxy('u16') + Module["HEAPU32"] = HEAPU32 = createMemoryProxy('u32') + Module["HEAPF32"] = HEAPF32 = createMemoryProxy('f32') + Module["HEAPF64"] = HEAPF64 = createMemoryProxy('f64') + Module["HEAP64"] = HEAP64 = createMemoryProxy('i64') + Module["HEAPU64"] = HEAPU64 = createMemoryProxy('u64') + Module["MEMORY"] = b; #endif } diff --git a/src/runtime_safe_heap.js b/src/runtime_safe_heap.js index dbb91468b2121..2b1193e3fbdbd 100644 --- a/src/runtime_safe_heap.js +++ b/src/runtime_safe_heap.js @@ -26,7 +26,7 @@ var SAFE_HEAP_COUNTER = 0; /** @param {number|boolean=} isFloat */ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { #if CAN_ADDRESS_2GB - dest >>>= 0; + dest = fixPointer(ptr); #endif #if SAFE_HEAP_LOG out('SAFE_HEAP store: ' + [dest, value, bytes, isFloat, SAFE_HEAP_COUNTER++]); @@ -42,7 +42,7 @@ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { #else if (runtimeInitialized) { #endif - var brk = _sbrk() >>> 0; + var brk = fixPointer(_sbrk()); if (dest + bytes > brk) abort(`segmentation fault, exceeded the top of the available dynamic heap when storing ${bytes} bytes to address ${dest}. DYNAMICTOP=${brk}`); assert(brk >= _emscripten_stack_get_base(), `brk >= _emscripten_stack_get_base() (brk=${brk}, _emscripten_stack_get_base()=${_emscripten_stack_get_base()})`); // sbrk-managed memory must be above the stack assert(brk <= wasmMemory.buffer.byteLength, `brk <= wasmMemory.buffer.byteLength (brk=${brk}, wasmMemory.buffer.byteLength=${wasmMemory.buffer.byteLength})`); @@ -57,7 +57,7 @@ function SAFE_HEAP_STORE_D(dest, value, bytes) { /** @param {number|boolean=} isFloat */ function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { #if CAN_ADDRESS_2GB - dest >>>= 0; + dest = fixPointer(ptr); #endif if (dest <= 0) abort(`segmentation fault loading ${bytes} bytes from address ${dest}`); #if SAFE_HEAP == 1 @@ -70,7 +70,7 @@ function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { #else if (runtimeInitialized) { #endif - var brk = _sbrk() >>> 0; + var brk = fixPointer(_sbrk()); if (dest + bytes > brk) abort(`segmentation fault, exceeded the top of the available dynamic heap when loading ${bytes} bytes from address ${dest}. DYNAMICTOP=${brk}`); assert(brk >= _emscripten_stack_get_base(), `brk >= _emscripten_stack_get_base() (brk=${brk}, _emscripten_stack_get_base()=${_emscripten_stack_get_base()})`); // sbrk-managed memory must be above the stack assert(brk <= wasmMemory.buffer.byteLength, `brk <= wasmMemory.buffer.byteLength (brk=${brk}, wasmMemory.buffer.byteLength=${wasmMemory.buffer.byteLength})`); From dfdcf9b9c3307736ad04e4c4fd751f6485c159eb Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Thu, 29 Jun 2023 22:02:16 +0400 Subject: [PATCH 02/21] Fixed tests, conditional proxy creation --- src/preamble.js | 128 ++++++++++++------ src/preamble_minimal.js | 7 + .../metadce/test_metadce_hello_O1.jssize | 2 +- .../metadce/test_metadce_minimal_64.jssize | 2 +- 4 files changed, 94 insertions(+), 45 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index c89dd7d28e9ad..e623f813960cc 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -138,14 +138,20 @@ function fixPointer (ptr) { if (typeof ptr === 'number' && ptr < 0) { return ptr >>> 0; } - if (typeof ptr === 'bigint') { - return Number(ptr); - } return ptr; } +#if MEMORY64 +/** + * @param {string} type - Heap type + * @param {number} [offset] - Heap offset + * @param {number} [length] - typed array length + */ const getHeapBlock = (type, offset, length) => { const maxSize = Math.min(wasmMemory.buffer.byteLength, 4 * 1024 * 1024 * 1024 - 8) + if (!offset) { + offset = 0 + } const heap = wasmMemory.buffer switch (type) { @@ -176,62 +182,75 @@ const getHeapBlock = (type, offset, length) => { throw new Error('Invalid type'); } } +#endif function updateMemoryViews() { var b = wasmMemory.buffer; #if SUPPORT_BIG_ENDIAN Module['HEAP_DATA_VIEW'] = HEAP_DATA_VIEW = new DataView(b); #endif -#if !WASM_BIGINT - Module['HEAP8'] = HEAP8 = new Int8Array(b); - Module['HEAP16'] = HEAP16 = new Int16Array(b); - Module['HEAP32'] = HEAP32 = new Int32Array(b); - Module['HEAPU8'] = HEAPU8 = new Uint8Array(b); - Module['HEAPU16'] = HEAPU16 = new Uint16Array(b); - Module['HEAPU32'] = HEAPU32 = new Uint32Array(b); - Module['HEAPF32'] = HEAPF32 = new Float32Array(b); - Module['HEAPF64'] = HEAPF64 = new Float64Array(b); -#else +#if MEMORY64 var maxArraySize = Math.min(b.byteLength, 4 * 1024 * 1024 * 1024 - 8); - var proxyHandler = (type, hb) => ({ - heapBlock: hb, - copyWithin: function (target, start, end) { + var proxyHandler = (type, heapBlock) => ({ + heapBlock, + copyWithin (target, start, end) { target = fixPointer(target) start = fixPointer(start) end = fixPointer(end) - const bpe = hb.BYTES_PER_ELEMENT + const bpe = heapBlock.BYTES_PER_ELEMENT if (target * bpe >= maxArraySize || start * bpe >= maxArraySize || (end && end * bpe >= maxArraySize)) { var len = end - start var targetArray = getHeapBlock(type, target * bpe, len) var sourceArray = getHeapBlock(type, start * bpe, len) targetArray.set(sourceArray) - return hb + return heapBlock } else { - return hb.copyWithin(target, start, end) + return heapBlock.copyWithin(target, start, end) } }, - setOverridden: function (array, offset) { + setOverridden (array, offset) { offset = fixPointer(offset) - var offsetReal = (offset || 0) * hb.BYTES_PER_ELEMENT + var offsetReal = (offset || 0) * heapBlock.BYTES_PER_ELEMENT if (offsetReal >= maxArraySize || array.byteLength + offsetReal >= maxArraySize) { var targetArray = getHeapBlock(type, offsetReal, array.length) targetArray.set(array) } else { - hb.set(array, offset) + heapBlock.set(array, offset) } }, - subarray: function (start, end) { + subarray (start, end) { start = fixPointer(start) end = fixPointer(end) - var startReal = (start || 0) * hb.BYTES_PER_ELEMENT - var endReal = end ? end * hb.BYTES_PER_ELEMENT : wasmMemory.byteLength + var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT + var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength if (startReal >= maxArraySize || endReal >= maxArraySize) { return getHeapBlock(type, startReal, endReal - startReal) } else { - return hb.subarray(start, end) + return heapBlock.subarray(start, end) + } + }, + fill (value, start, end) { + var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT + var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength + if (startReal >= maxArraySize || endReal >= maxArraySize) { + const hb = getHeapBlock(type, startReal, endReal - startReal) + hb.fill(0, 0) + return heapBlock + } else { + return heapBlock.fill(value, start, end) + } + }, + slice (start, end) { + var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT + var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength + if (startReal >= maxArraySize || endReal >= maxArraySize) { + const hb = getHeapBlock(type, startReal, endReal - startReal) + return hb.slice(start, end) + } else { + return heapBlock.slice(start, end) } }, - get: function (target, property) { + get (target, property) { if (typeof property === 'number' || typeof property === 'bigint') { var memoryOffset = fixPointer(property) * target.BYTES_PER_ELEMENT if (memoryOffset >= maxArraySize) { @@ -254,9 +273,17 @@ function updateMemoryViews() { return this.subarray } - return this.heapBlock[property] + if (property === 'fill') { + return this.fill + } + + if (property === 'slice') { + return this.slice + } + + return heapBlock[property] }, - set: function (target, property, value) { + set (target, property, value) { if (typeof property === 'number' || typeof property === 'bigint') { var memoryOffset = fixPointer(property) * target.BYTES_PER_ELEMENT if (memoryOffset >= maxArraySize) { @@ -266,26 +293,41 @@ function updateMemoryViews() { } } - this.heapBlock[property] = value + heapBlock[property] = value return true; }, }) - function createMemoryProxy (type) - { + function createMemoryProxy (type) { const block = getHeapBlock(type, 0) return new Proxy(block, proxyHandler(type, block)); } - Module["HEAP8"] = HEAP8 = createMemoryProxy('i8') - Module["HEAP16"] = HEAP16 = createMemoryProxy('i16') - Module["HEAP32"] = HEAP32 = createMemoryProxy('i32') - Module["HEAPU8"] = HEAPU8 = createMemoryProxy('u8') - Module["HEAPU16"] = HEAPU16 = createMemoryProxy('u16') - Module["HEAPU32"] = HEAPU32 = createMemoryProxy('u32') - Module["HEAPF32"] = HEAPF32 = createMemoryProxy('f32') - Module["HEAPF64"] = HEAPF64 = createMemoryProxy('f64') - Module["HEAP64"] = HEAP64 = createMemoryProxy('i64') - Module["HEAPU64"] = HEAPU64 = createMemoryProxy('u64') - Module["MEMORY"] = b; + if (b.byteLength > maxArraySize) { + Module["HEAP8"] = HEAP8 = createMemoryProxy('i8') + Module["HEAPU8"] = HEAPU8 = createMemoryProxy('u8') + } else { + Module['HEAP8'] = HEAP8 = new Int8Array(b); + Module['HEAPU8'] = HEAPU8 = new Uint8Array(b); + } + if (b.byteLength > maxArraySize * 2) { + Module["HEAP16"] = HEAP16 = createMemoryProxy('i16') + Module["HEAPU16"] = HEAPU16 = createMemoryProxy('u16') + } else { + Module['HEAP16'] = HEAP16 = new Int16Array(b); + Module['HEAPU16'] = HEAPU16 = new Uint16Array(b); + } +#else + Module['HEAP8'] = HEAP8 = new Int8Array(b); + Module['HEAP16'] = HEAP16 = new Int16Array(b); + Module['HEAPU8'] = HEAPU8 = new Uint8Array(b); + Module['HEAPU16'] = HEAPU16 = new Uint16Array(b); +#endif + Module['HEAP32'] = HEAP32 = new Int32Array(b); + Module['HEAPU32'] = HEAPU32 = new Uint32Array(b); + Module['HEAPF32'] = HEAPF32 = new Float32Array(b); + Module['HEAPF64'] = HEAPF64 = new Float64Array(b); +#if WASM_BIGINT + Module['HEAP64'] = HEAP64 = new BigInt64Array(b); + Module['HEAPU64'] = HEAPU64 = new BigUint64Array(b); #endif } diff --git a/src/preamble_minimal.js b/src/preamble_minimal.js index a4a34988dd423..97cd956c7fe62 100644 --- a/src/preamble_minimal.js +++ b/src/preamble_minimal.js @@ -69,6 +69,13 @@ var HEAP8, HEAP16, HEAP32, HEAPU8, HEAPU16, HEAPU32, HEAPF32, HEAPF64, #endif wasmMemory, wasmTable; +function fixPointer (ptr) { + if (typeof ptr === 'number' && ptr < 0) { + return ptr >>> 0; + } + return ptr; +} + function updateMemoryViews() { var b = wasmMemory.buffer; #if ASSERTIONS && SHARED_MEMORY diff --git a/test/other/metadce/test_metadce_hello_O1.jssize b/test/other/metadce/test_metadce_hello_O1.jssize index 897da478cb93f..87c230a814026 100644 --- a/test/other/metadce/test_metadce_hello_O1.jssize +++ b/test/other/metadce/test_metadce_hello_O1.jssize @@ -1 +1 @@ -8218 +8601 diff --git a/test/other/metadce/test_metadce_minimal_64.jssize b/test/other/metadce/test_metadce_minimal_64.jssize index fd031f90e3ec7..725bb9700b2e6 100644 --- a/test/other/metadce/test_metadce_minimal_64.jssize +++ b/test/other/metadce/test_metadce_minimal_64.jssize @@ -1 +1 @@ -3905 +6203 From 799dc73a040f7aac09410aa1926c08b5b1affd1d Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Sun, 2 Jul 2023 17:34:56 +0400 Subject: [PATCH 03/21] Removed fixPtr function --- src/library.js | 2 +- src/library_strings.js | 18 ------------------ src/preamble_minimal.js | 7 ------- src/runtime_safe_heap.js | 6 ------ 4 files changed, 1 insertion(+), 32 deletions(-) diff --git a/src/library.js b/src/library.js index 60624f96650eb..410866991fcfb 100644 --- a/src/library.js +++ b/src/library.js @@ -422,7 +422,7 @@ mergeInto(LibraryManager.library, { ? (dest, src, num) => HEAPU8.copyWithin(dest, src, src + num) : (dest, src, num) => HEAPU8.set(HEAPU8.subarray(src, src+num), dest)`, #else - emscripten_memcpy_big: (dest, src, num) => HEAPU8.copyWithin(fixPointer(dest), fixPointer(src), fixPointer(src) + fixPointer(num)), + emscripten_memcpy_big: (dest, src, num) => HEAPU8.copyWithin(dest, src, src + num), #endif #endif diff --git a/src/library_strings.js b/src/library_strings.js index 8b2346136cca4..1ad54072971f7 100644 --- a/src/library_strings.js +++ b/src/library_strings.js @@ -27,9 +27,6 @@ mergeInto(LibraryManager.library, { $UTF8ArrayToString__deps: ['$UTF8Decoder'], #endif $UTF8ArrayToString: (heapOrArray, idx, maxBytesToRead) => { -#if CAN_ADDRESS_2GB - idx = fixPointer(idx); -#endif var endIdx = idx + maxBytesToRead; #if TEXTDECODER var endPtr = idx; @@ -118,9 +115,6 @@ mergeInto(LibraryManager.library, { #if ASSERTIONS assert(typeof ptr == 'number'); #endif -#if CAN_ADDRESS_2GB - ptr = fixPointer(ptr); -#endif #if TEXTDECODER == 2 if (!ptr) return ''; var maxPtr = ptr + maxBytesToRead; @@ -154,9 +148,6 @@ mergeInto(LibraryManager.library, { * @return {number} The number of bytes written, EXCLUDING the null terminator. */ $stringToUTF8Array: (str, heap, outIdx, maxBytesToWrite) => { -#if CAN_ADDRESS_2GB - outIdx = fixPointer(outIdx); -#endif #if ASSERTIONS assert(typeof str === 'string'); #endif @@ -262,9 +253,6 @@ mergeInto(LibraryManager.library, { // emscripten HEAP, returns a copy of that string as a Javascript String // object. $AsciiToString: (ptr) => { -#if CAN_ADDRESS_2GB - ptr = fixPointer(ptr); -#endif var str = ''; while (1) { var ch = {{{ makeGetValue('ptr++', 0, 'u8') }}}; @@ -277,9 +265,6 @@ mergeInto(LibraryManager.library, { // address 'outPtr', null-terminated and encoded in ASCII form. The copy will // require at most str.length+1 bytes of space in the HEAP. $stringToAscii: (str, buffer) => { -#if CAN_ADDRESS_2GB - buffer = fixPointer(buffer); -#endif for (var i = 0; i < str.length; ++i) { #if ASSERTIONS assert(str.charCodeAt(i) === (str.charCodeAt(i) & 0xff)); @@ -432,9 +417,6 @@ mergeInto(LibraryManager.library, { // output, not even the null terminator. // Returns the number of bytes written, EXCLUDING the null terminator. $stringToUTF32: (str, outPtr, maxBytesToWrite) => { -#if CAN_ADDRESS_2GB - outPtr = fixPointer(outPtr); -#endif #if ASSERTIONS assert(outPtr % 4 == 0, 'Pointer passed to stringToUTF32 must be aligned to four bytes!'); #endif diff --git a/src/preamble_minimal.js b/src/preamble_minimal.js index 97cd956c7fe62..a4a34988dd423 100644 --- a/src/preamble_minimal.js +++ b/src/preamble_minimal.js @@ -69,13 +69,6 @@ var HEAP8, HEAP16, HEAP32, HEAPU8, HEAPU16, HEAPU32, HEAPF32, HEAPF64, #endif wasmMemory, wasmTable; -function fixPointer (ptr) { - if (typeof ptr === 'number' && ptr < 0) { - return ptr >>> 0; - } - return ptr; -} - function updateMemoryViews() { var b = wasmMemory.buffer; #if ASSERTIONS && SHARED_MEMORY diff --git a/src/runtime_safe_heap.js b/src/runtime_safe_heap.js index ff971c5342913..c92e685b91b3b 100644 --- a/src/runtime_safe_heap.js +++ b/src/runtime_safe_heap.js @@ -25,9 +25,6 @@ var SAFE_HEAP_COUNTER = 0; /** @param {number|boolean=} isFloat */ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { -#if CAN_ADDRESS_2GB - dest = fixPointer(ptr); -#endif #if SAFE_HEAP_LOG out('SAFE_HEAP store: ' + [dest, value, bytes, isFloat, SAFE_HEAP_COUNTER++]); #endif @@ -56,9 +53,6 @@ function SAFE_HEAP_STORE_D(dest, value, bytes) { /** @param {number|boolean=} isFloat */ function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { -#if CAN_ADDRESS_2GB - dest = fixPointer(ptr); -#endif if (dest <= 0) abort(`segmentation fault loading ${bytes} bytes from address ${dest}`); #if SAFE_HEAP == 1 if (dest % bytes !== 0) abort(`alignment error loading from address ${dest}, which was expected to be aligned to a multiple of ${bytes}`); From 38927accdc70c279c30e426b74b34b0e222bd020 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Thu, 6 Jul 2023 02:08:13 +0400 Subject: [PATCH 04/21] Code style fixes, removed fixPtr --- src/library_strings.js | 15 +++++ src/preamble.js | 121 ++++++++++++++++++--------------------- src/runtime_safe_heap.js | 6 ++ 3 files changed, 76 insertions(+), 66 deletions(-) diff --git a/src/library_strings.js b/src/library_strings.js index 1ad54072971f7..6110ba8bb171f 100644 --- a/src/library_strings.js +++ b/src/library_strings.js @@ -27,6 +27,9 @@ mergeInto(LibraryManager.library, { $UTF8ArrayToString__deps: ['$UTF8Decoder'], #endif $UTF8ArrayToString: (heapOrArray, idx, maxBytesToRead) => { +#if CAN_ADDRESS_2GB + idx >>>= 0; +#endif var endIdx = idx + maxBytesToRead; #if TEXTDECODER var endPtr = idx; @@ -115,6 +118,9 @@ mergeInto(LibraryManager.library, { #if ASSERTIONS assert(typeof ptr == 'number'); #endif +#if CAN_ADDRESS_2GB + ptr >>>= 0; +#endif #if TEXTDECODER == 2 if (!ptr) return ''; var maxPtr = ptr + maxBytesToRead; @@ -148,6 +154,9 @@ mergeInto(LibraryManager.library, { * @return {number} The number of bytes written, EXCLUDING the null terminator. */ $stringToUTF8Array: (str, heap, outIdx, maxBytesToWrite) => { +#if CAN_ADDRESS_2GB + outIdx >>>= 0; +#endif #if ASSERTIONS assert(typeof str === 'string'); #endif @@ -253,6 +262,9 @@ mergeInto(LibraryManager.library, { // emscripten HEAP, returns a copy of that string as a Javascript String // object. $AsciiToString: (ptr) => { +#if CAN_ADDRESS_2GB + ptr >>>= 0; +#endif var str = ''; while (1) { var ch = {{{ makeGetValue('ptr++', 0, 'u8') }}}; @@ -417,6 +429,9 @@ mergeInto(LibraryManager.library, { // output, not even the null terminator. // Returns the number of bytes written, EXCLUDING the null terminator. $stringToUTF32: (str, outPtr, maxBytesToWrite) => { +#if CAN_ADDRESS_2GB + outPtr >>>= 0; +#endif #if ASSERTIONS assert(outPtr % 4 == 0, 'Pointer passed to stringToUTF32 must be aligned to four bytes!'); #endif diff --git a/src/preamble.js b/src/preamble.js index daab962b3b473..48e4293bf7cb8 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -134,69 +134,59 @@ var HEAP, var HEAP_DATA_VIEW; #endif -function fixPointer (ptr) { - if (typeof ptr === 'number' && ptr < 0) { - return ptr >>> 0; - } - return ptr; -} - -#if MEMORY64 -/** - * @param {string} type - Heap type - * @param {number} [offset] - Heap offset - * @param {number} [length] - typed array length - */ -const getHeapBlock = (type, offset, length) => { - const maxSize = Math.min(wasmMemory.buffer.byteLength, 4 * 1024 * 1024 * 1024 - 8) - if (!offset) { - offset = 0 - } - - const heap = wasmMemory.buffer - switch (type) { - case 'i1': - case 'i8': - return new Int8Array(heap, offset, length || maxSize / Int8Array.BYTES_PER_ELEMENT); - case 'u1': - case 'u8': - return new Uint8Array(heap, offset, length || maxSize / Uint8Array.BYTES_PER_ELEMENT); - case 'i16': - return new Int16Array(heap, offset, length || maxSize / Int16Array.BYTES_PER_ELEMENT); - case 'u16': - return new Uint16Array(heap, offset, length || maxSize / Uint16Array.BYTES_PER_ELEMENT); - case 'i32': - return new Int32Array(heap, offset, length || maxSize / Int32Array.BYTES_PER_ELEMENT); - case 'u32': - return new Uint32Array(heap, offset, length || maxSize / Uint32Array.BYTES_PER_ELEMENT); - case 'f32': - return new Float32Array(heap, offset, length || maxSize / Float32Array.BYTES_PER_ELEMENT); - case 'f64': - return new Float64Array(heap, offset, length || maxSize / Float64Array.BYTES_PER_ELEMENT); - case 'i64': - return new BigInt64Array(heap, offset, length || maxSize / BigInt64Array.BYTES_PER_ELEMENT); - case '*': - case 'u64': - return new BigUint64Array(heap, offset, length || maxSize / BigUint64Array.BYTES_PER_ELEMENT); - default: - throw new Error('Invalid type'); - } -} -#endif - function updateMemoryViews() { var b = wasmMemory.buffer; #if SUPPORT_BIG_ENDIAN Module['HEAP_DATA_VIEW'] = HEAP_DATA_VIEW = new DataView(b); #endif -#if MEMORY64 +#if MEMORY64 && MAXIMUM_MEMORY > FOUR_GB + // Chrome does not allow TypedArrays with more than 4294967296 elements + // We'll create proxy objects for HEAP(U)8 when memory is > 4gb and for HEAP(U)16 when > 8gb + // https://bugs.chromium.org/p/v8/issues/detail?id=4153 var maxArraySize = Math.min(b.byteLength, 4 * 1024 * 1024 * 1024 - 8); + /** + * @param {string} type - Heap type + * @param {number} [offset] - Heap offset + * @param {number} [length] - typed array length + */ + const getHeapBlock = (type, offset, length) => { + if (!offset) { + offset = 0 + } + + const heap = wasmMemory.buffer + switch (type) { + case 'i1': + case 'i8': + return new Int8Array(heap, offset, length || maxArraySize / Int8Array.BYTES_PER_ELEMENT); + case 'u1': + case 'u8': + return new Uint8Array(heap, offset, length || maxArraySize / Uint8Array.BYTES_PER_ELEMENT); + case 'i16': + return new Int16Array(heap, offset, length || maxArraySize / Int16Array.BYTES_PER_ELEMENT); + case 'u16': + return new Uint16Array(heap, offset, length || maxArraySize / Uint16Array.BYTES_PER_ELEMENT); + case 'i32': + return new Int32Array(heap, offset, length || maxArraySize / Int32Array.BYTES_PER_ELEMENT); + case 'u32': + return new Uint32Array(heap, offset, length || maxArraySize / Uint32Array.BYTES_PER_ELEMENT); + case 'f32': + return new Float32Array(heap, offset, length || maxArraySize / Float32Array.BYTES_PER_ELEMENT); + case 'f64': + return new Float64Array(heap, offset, length || maxArraySize / Float64Array.BYTES_PER_ELEMENT); + case 'i64': + return new BigInt64Array(heap, offset, length || maxArraySize / BigInt64Array.BYTES_PER_ELEMENT); + case '*': + case 'u64': + return new BigUint64Array(heap, offset, length || maxArraySize / BigUint64Array.BYTES_PER_ELEMENT); + default: + throw new Error('Invalid type'); + } + } + var proxyHandler = (type, heapBlock) => ({ heapBlock, copyWithin (target, start, end) { - target = fixPointer(target) - start = fixPointer(start) - end = fixPointer(end) const bpe = heapBlock.BYTES_PER_ELEMENT if (target * bpe >= maxArraySize || start * bpe >= maxArraySize || (end && end * bpe >= maxArraySize)) { var len = end - start @@ -208,8 +198,7 @@ function updateMemoryViews() { return heapBlock.copyWithin(target, start, end) } }, - setOverridden (array, offset) { - offset = fixPointer(offset) + setOverridden(array, offset) { var offsetReal = (offset || 0) * heapBlock.BYTES_PER_ELEMENT if (offsetReal >= maxArraySize || array.byteLength + offsetReal >= maxArraySize) { var targetArray = getHeapBlock(type, offsetReal, array.length) @@ -218,9 +207,7 @@ function updateMemoryViews() { heapBlock.set(array, offset) } }, - subarray (start, end) { - start = fixPointer(start) - end = fixPointer(end) + subarray(start, end) { var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength if (startReal >= maxArraySize || endReal >= maxArraySize) { @@ -229,7 +216,7 @@ function updateMemoryViews() { return heapBlock.subarray(start, end) } }, - fill (value, start, end) { + fill(value, start, end) { var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength if (startReal >= maxArraySize || endReal >= maxArraySize) { @@ -240,7 +227,7 @@ function updateMemoryViews() { return heapBlock.fill(value, start, end) } }, - slice (start, end) { + slice(start, end) { var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength if (startReal >= maxArraySize || endReal >= maxArraySize) { @@ -250,9 +237,9 @@ function updateMemoryViews() { return heapBlock.slice(start, end) } }, - get (target, property) { + get(target, property) { if (typeof property === 'number' || typeof property === 'bigint') { - var memoryOffset = fixPointer(property) * target.BYTES_PER_ELEMENT + var memoryOffset = property * target.BYTES_PER_ELEMENT if (memoryOffset >= maxArraySize) { var heap = getHeapBlock(type, memoryOffset, 1); return heap[0]; @@ -283,9 +270,9 @@ function updateMemoryViews() { return heapBlock[property] }, - set (target, property, value) { + set(target, property, value) { if (typeof property === 'number' || typeof property === 'bigint') { - var memoryOffset = fixPointer(property) * target.BYTES_PER_ELEMENT + var memoryOffset = property * target.BYTES_PER_ELEMENT if (memoryOffset >= maxArraySize) { var heap = getHeapBlock(type, memoryOffset, 1); heap[0] = value; @@ -297,10 +284,12 @@ function updateMemoryViews() { return true; }, }) - function createMemoryProxy (type) { + + function createMemoryProxy(type) { const block = getHeapBlock(type, 0) return new Proxy(block, proxyHandler(type, block)); } + if (b.byteLength > maxArraySize) { Module["HEAP8"] = HEAP8 = createMemoryProxy('i8') Module["HEAPU8"] = HEAPU8 = createMemoryProxy('u8') diff --git a/src/runtime_safe_heap.js b/src/runtime_safe_heap.js index c92e685b91b3b..a02e142e0c732 100644 --- a/src/runtime_safe_heap.js +++ b/src/runtime_safe_heap.js @@ -25,6 +25,9 @@ var SAFE_HEAP_COUNTER = 0; /** @param {number|boolean=} isFloat */ function SAFE_HEAP_STORE(dest, value, bytes, isFloat) { +#if CAN_ADDRESS_2GB + dest >>>= 0; +#endif #if SAFE_HEAP_LOG out('SAFE_HEAP store: ' + [dest, value, bytes, isFloat, SAFE_HEAP_COUNTER++]); #endif @@ -53,6 +56,9 @@ function SAFE_HEAP_STORE_D(dest, value, bytes) { /** @param {number|boolean=} isFloat */ function SAFE_HEAP_LOAD(dest, bytes, unsigned, isFloat) { +#if CAN_ADDRESS_2GB + dest >>>= 0; +#endif if (dest <= 0) abort(`segmentation fault loading ${bytes} bytes from address ${dest}`); #if SAFE_HEAP == 1 if (dest % bytes !== 0) abort(`alignment error loading from address ${dest}, which was expected to be aligned to a multiple of ${bytes}`); From 9aafa76bc4127dab8312fb53b432b2477fde2d7d Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Thu, 6 Jul 2023 02:26:20 +0400 Subject: [PATCH 05/21] Reverted emcc.py --- emcc.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/emcc.py b/emcc.py index 186d69dfcbc23..5e347e303d72f 100755 --- a/emcc.py +++ b/emcc.py @@ -3728,8 +3728,7 @@ def phase_binaryen(target, options, wasm_target): # >=2GB heap support requires pointers in JS to be unsigned. rather than # require all pointers to be unsigned by default, which increases code size # a little, keep them signed, and just unsign them here if we need that. - # for MEMORY64 heap proxies are created that handle unsigning internally - if settings.CAN_ADDRESS_2GB and not settings.MEMORY64: + if settings.CAN_ADDRESS_2GB: with ToolchainProfiler.profile_block('use_unsigned_pointers_in_js'): final_js = building.use_unsigned_pointers_in_js(final_js) From e9991c3a11912876684d9a22e95c1f591ad7b607 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Thu, 6 Jul 2023 22:18:31 +0400 Subject: [PATCH 06/21] Test fixes --- src/preamble.js | 165 +---------------- src/runtime_view_proxy.js | 170 ++++++++++++++++++ .../metadce/test_metadce_minimal_64.jssize | 2 +- test/test_browser.py | 3 +- test/test_other.py | 1 - 5 files changed, 173 insertions(+), 168 deletions(-) create mode 100644 src/runtime_view_proxy.js diff --git a/src/preamble.js b/src/preamble.js index 48e4293bf7cb8..d52ef78e34583 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -140,170 +140,7 @@ function updateMemoryViews() { Module['HEAP_DATA_VIEW'] = HEAP_DATA_VIEW = new DataView(b); #endif #if MEMORY64 && MAXIMUM_MEMORY > FOUR_GB - // Chrome does not allow TypedArrays with more than 4294967296 elements - // We'll create proxy objects for HEAP(U)8 when memory is > 4gb and for HEAP(U)16 when > 8gb - // https://bugs.chromium.org/p/v8/issues/detail?id=4153 - var maxArraySize = Math.min(b.byteLength, 4 * 1024 * 1024 * 1024 - 8); - /** - * @param {string} type - Heap type - * @param {number} [offset] - Heap offset - * @param {number} [length] - typed array length - */ - const getHeapBlock = (type, offset, length) => { - if (!offset) { - offset = 0 - } - - const heap = wasmMemory.buffer - switch (type) { - case 'i1': - case 'i8': - return new Int8Array(heap, offset, length || maxArraySize / Int8Array.BYTES_PER_ELEMENT); - case 'u1': - case 'u8': - return new Uint8Array(heap, offset, length || maxArraySize / Uint8Array.BYTES_PER_ELEMENT); - case 'i16': - return new Int16Array(heap, offset, length || maxArraySize / Int16Array.BYTES_PER_ELEMENT); - case 'u16': - return new Uint16Array(heap, offset, length || maxArraySize / Uint16Array.BYTES_PER_ELEMENT); - case 'i32': - return new Int32Array(heap, offset, length || maxArraySize / Int32Array.BYTES_PER_ELEMENT); - case 'u32': - return new Uint32Array(heap, offset, length || maxArraySize / Uint32Array.BYTES_PER_ELEMENT); - case 'f32': - return new Float32Array(heap, offset, length || maxArraySize / Float32Array.BYTES_PER_ELEMENT); - case 'f64': - return new Float64Array(heap, offset, length || maxArraySize / Float64Array.BYTES_PER_ELEMENT); - case 'i64': - return new BigInt64Array(heap, offset, length || maxArraySize / BigInt64Array.BYTES_PER_ELEMENT); - case '*': - case 'u64': - return new BigUint64Array(heap, offset, length || maxArraySize / BigUint64Array.BYTES_PER_ELEMENT); - default: - throw new Error('Invalid type'); - } - } - - var proxyHandler = (type, heapBlock) => ({ - heapBlock, - copyWithin (target, start, end) { - const bpe = heapBlock.BYTES_PER_ELEMENT - if (target * bpe >= maxArraySize || start * bpe >= maxArraySize || (end && end * bpe >= maxArraySize)) { - var len = end - start - var targetArray = getHeapBlock(type, target * bpe, len) - var sourceArray = getHeapBlock(type, start * bpe, len) - targetArray.set(sourceArray) - return heapBlock - } else { - return heapBlock.copyWithin(target, start, end) - } - }, - setOverridden(array, offset) { - var offsetReal = (offset || 0) * heapBlock.BYTES_PER_ELEMENT - if (offsetReal >= maxArraySize || array.byteLength + offsetReal >= maxArraySize) { - var targetArray = getHeapBlock(type, offsetReal, array.length) - targetArray.set(array) - } else { - heapBlock.set(array, offset) - } - }, - subarray(start, end) { - var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT - var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength - if (startReal >= maxArraySize || endReal >= maxArraySize) { - return getHeapBlock(type, startReal, endReal - startReal) - } else { - return heapBlock.subarray(start, end) - } - }, - fill(value, start, end) { - var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT - var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength - if (startReal >= maxArraySize || endReal >= maxArraySize) { - const hb = getHeapBlock(type, startReal, endReal - startReal) - hb.fill(0, 0) - return heapBlock - } else { - return heapBlock.fill(value, start, end) - } - }, - slice(start, end) { - var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT - var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength - if (startReal >= maxArraySize || endReal >= maxArraySize) { - const hb = getHeapBlock(type, startReal, endReal - startReal) - return hb.slice(start, end) - } else { - return heapBlock.slice(start, end) - } - }, - get(target, property) { - if (typeof property === 'number' || typeof property === 'bigint') { - var memoryOffset = property * target.BYTES_PER_ELEMENT - if (memoryOffset >= maxArraySize) { - var heap = getHeapBlock(type, memoryOffset, 1); - return heap[0]; - } else { - return this.heapBlock[property] - } - } - - if (property === 'copyWithin') { - return this.copyWithin - } - - if (property === 'set') { - return this.setOverridden - } - - if (property === 'subarray') { - return this.subarray - } - - if (property === 'fill') { - return this.fill - } - - if (property === 'slice') { - return this.slice - } - - return heapBlock[property] - }, - set(target, property, value) { - if (typeof property === 'number' || typeof property === 'bigint') { - var memoryOffset = property * target.BYTES_PER_ELEMENT - if (memoryOffset >= maxArraySize) { - var heap = getHeapBlock(type, memoryOffset, 1); - heap[0] = value; - return true; - } - } - - heapBlock[property] = value - return true; - }, - }) - - function createMemoryProxy(type) { - const block = getHeapBlock(type, 0) - return new Proxy(block, proxyHandler(type, block)); - } - - if (b.byteLength > maxArraySize) { - Module["HEAP8"] = HEAP8 = createMemoryProxy('i8') - Module["HEAPU8"] = HEAPU8 = createMemoryProxy('u8') - } else { - Module['HEAP8'] = HEAP8 = new Int8Array(b); - Module['HEAPU8'] = HEAPU8 = new Uint8Array(b); - } - if (b.byteLength > maxArraySize * 2) { - Module["HEAP16"] = HEAP16 = createMemoryProxy('i16') - Module["HEAPU16"] = HEAPU16 = createMemoryProxy('u16') - } else { - Module['HEAP16'] = HEAP16 = new Int16Array(b); - Module['HEAPU16'] = HEAPU16 = new Uint16Array(b); - } +#include "runtime_view_proxy.js" #else Module['HEAP8'] = HEAP8 = new Int8Array(b); Module['HEAP16'] = HEAP16 = new Int16Array(b); diff --git a/src/runtime_view_proxy.js b/src/runtime_view_proxy.js new file mode 100644 index 0000000000000..48f57cb144791 --- /dev/null +++ b/src/runtime_view_proxy.js @@ -0,0 +1,170 @@ +/** + * @license + * Copyright 2023 The Emscripten Authors + * SPDX-License-Identifier: MIT + */ + +// Chrome does not allow TypedArrays with more than 4294967296 elements +// We'll create proxy objects for HEAP(U)8 when memory is > 4gb and for HEAP(U)16 when > 8gb +// https://bugs.chromium.org/p/v8/issues/detail?id=4153 +var maxArraySize = Math.min(b.byteLength, 4 * 1024 * 1024 * 1024 - 8); +/** + * @param {string} type - Heap type + * @param {number} [offset] - Heap offset + * @param {number} [length] - typed array length + */ +const getHeapBlock = (type, offset, length) => { + if (!offset) { + offset = 0 + } + + const heap = wasmMemory.buffer + switch (type) { + case 'i1': + case 'i8': + return new Int8Array(heap, offset, length || maxArraySize / Int8Array.BYTES_PER_ELEMENT); + case 'u1': + case 'u8': + return new Uint8Array(heap, offset, length || maxArraySize / Uint8Array.BYTES_PER_ELEMENT); + case 'i16': + return new Int16Array(heap, offset, length || maxArraySize / Int16Array.BYTES_PER_ELEMENT); + case 'u16': + return new Uint16Array(heap, offset, length || maxArraySize / Uint16Array.BYTES_PER_ELEMENT); + case 'i32': + return new Int32Array(heap, offset, length || maxArraySize / Int32Array.BYTES_PER_ELEMENT); + case 'u32': + return new Uint32Array(heap, offset, length || maxArraySize / Uint32Array.BYTES_PER_ELEMENT); + case 'f32': + return new Float32Array(heap, offset, length || maxArraySize / Float32Array.BYTES_PER_ELEMENT); + case 'f64': + return new Float64Array(heap, offset, length || maxArraySize / Float64Array.BYTES_PER_ELEMENT); + case 'i64': + return new BigInt64Array(heap, offset, length || maxArraySize / BigInt64Array.BYTES_PER_ELEMENT); + case '*': + case 'u64': + return new BigUint64Array(heap, offset, length || maxArraySize / BigUint64Array.BYTES_PER_ELEMENT); + default: + throw new Error('Invalid type'); + } +} + +var proxyHandler = (type, heapBlock) => ({ + heapBlock, + copyWithin (target, start, end) { + const bpe = heapBlock.BYTES_PER_ELEMENT + if (target * bpe >= maxArraySize || start * bpe >= maxArraySize || (end && end * bpe >= maxArraySize)) { + var len = end - start + var targetArray = getHeapBlock(type, target * bpe, len) + var sourceArray = getHeapBlock(type, start * bpe, len) + targetArray.set(sourceArray) + return heapBlock + } else { + return heapBlock.copyWithin(target, start, end) + } + }, + setOverridden(array, offset) { + var offsetReal = (offset || 0) * heapBlock.BYTES_PER_ELEMENT + if (offsetReal >= maxArraySize || array.byteLength + offsetReal >= maxArraySize) { + var targetArray = getHeapBlock(type, offsetReal, array.length) + targetArray.set(array) + } else { + heapBlock.set(array, offset) + } + }, + subarray(start, end) { + var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT + var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength + if (startReal >= maxArraySize || endReal >= maxArraySize) { + return getHeapBlock(type, startReal, endReal - startReal) + } else { + return heapBlock.subarray(start, end) + } + }, + fill(value, start, end) { + var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT + var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength + if (startReal >= maxArraySize || endReal >= maxArraySize) { + const hb = getHeapBlock(type, startReal, endReal - startReal) + hb.fill(0, 0) + return heapBlock + } else { + return heapBlock.fill(value, start, end) + } + }, + slice(start, end) { + var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT + var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength + if (startReal >= maxArraySize || endReal >= maxArraySize) { + const hb = getHeapBlock(type, startReal, endReal - startReal) + return hb.slice(start, end) + } else { + return heapBlock.slice(start, end) + } + }, + get(target, property) { + if (typeof property === 'number' || typeof property === 'bigint') { + var memoryOffset = property * target.BYTES_PER_ELEMENT + if (memoryOffset >= maxArraySize) { + var heap = getHeapBlock(type, memoryOffset, 1); + return heap[0]; + } else { + return this.heapBlock[property] + } + } + + if (property === 'copyWithin') { + return this.copyWithin + } + + if (property === 'set') { + return this.setOverridden + } + + if (property === 'subarray') { + return this.subarray + } + + if (property === 'fill') { + return this.fill + } + + if (property === 'slice') { + return this.slice + } + + return heapBlock[property] + }, + set(target, property, value) { + if (typeof property === 'number' || typeof property === 'bigint') { + var memoryOffset = property * target.BYTES_PER_ELEMENT + if (memoryOffset >= maxArraySize) { + var heap = getHeapBlock(type, memoryOffset, 1); + heap[0] = value; + return true; + } + } + + heapBlock[property] = value + return true; + }, +}) + +function createMemoryProxy(type) { + const block = getHeapBlock(type, 0) + return new Proxy(block, proxyHandler(type, block)); +} + +if (b.byteLength > maxArraySize) { + Module["HEAP8"] = HEAP8 = createMemoryProxy('i8') + Module["HEAPU8"] = HEAPU8 = createMemoryProxy('u8') +} else { + Module['HEAP8'] = HEAP8 = new Int8Array(b); + Module['HEAPU8'] = HEAPU8 = new Uint8Array(b); +} +if (b.byteLength > maxArraySize * 2) { + Module["HEAP16"] = HEAP16 = createMemoryProxy('i16') + Module["HEAPU16"] = HEAPU16 = createMemoryProxy('u16') +} else { + Module['HEAP16'] = HEAP16 = new Int16Array(b); + Module['HEAPU16'] = HEAPU16 = new Uint16Array(b); +} diff --git a/test/other/metadce/test_metadce_minimal_64.jssize b/test/other/metadce/test_metadce_minimal_64.jssize index 725bb9700b2e6..e72c6b3f4abd0 100644 --- a/test/other/metadce/test_metadce_minimal_64.jssize +++ b/test/other/metadce/test_metadce_minimal_64.jssize @@ -1 +1 @@ -6203 +3844 diff --git a/test/test_browser.py b/test/test_browser.py index ae6e869eeb43e..5750afbdf6dae 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -5500,8 +5500,7 @@ def test_zzz_zzz_2gb_fail(self): self.do_run_in_out_file_test('browser', 'test_2GB_fail.cpp') @no_firefox('no 4GB support yet') - # @also_with_wasm64 Blocked on https://bugs.chromium.org/p/v8/issues/detail?id=4153 - @requires_v8 + @also_with_wasm64 def test_zzz_zzz_4gb_fail(self): # TODO Convert to an actual browser test when it reaches stable. # For now, keep this in browser as this suite runs serially, which diff --git a/test/test_other.py b/test/test_other.py index f17a99f2e9c64..a9fc19336110a 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -6264,7 +6264,6 @@ def test_failing_growth_2gb(self): def test_failing_growth_wasm64(self): # For now we skip this test because failure to create the TypedArray views # causes weird unrecoverable failures. - self.skipTest('https://bugs.chromium.org/p/v8/issues/detail?id=4153') self.require_wasm64() create_file('test.c', r''' #include From e88122594857a570048c88f6f20be68b4eb76309 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Thu, 6 Jul 2023 22:44:56 +0400 Subject: [PATCH 07/21] Removed arrow function --- src/runtime_view_proxy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime_view_proxy.js b/src/runtime_view_proxy.js index 48f57cb144791..76041b349e292 100644 --- a/src/runtime_view_proxy.js +++ b/src/runtime_view_proxy.js @@ -13,7 +13,7 @@ var maxArraySize = Math.min(b.byteLength, 4 * 1024 * 1024 * 1024 - 8); * @param {number} [offset] - Heap offset * @param {number} [length] - typed array length */ -const getHeapBlock = (type, offset, length) => { +function getHeapBlock(type, offset, length) { if (!offset) { offset = 0 } From d1a4c7014c65d3681bb68b0d52f1e00e9c927762 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Fri, 7 Jul 2023 04:09:23 +0400 Subject: [PATCH 08/21] Memory proxy optimizations + simple test --- src/runtime_view_proxy.js | 181 +++++++++++++++++++++----------------- test/test_other.py | 72 +++++++++++++++ 2 files changed, 171 insertions(+), 82 deletions(-) diff --git a/src/runtime_view_proxy.js b/src/runtime_view_proxy.js index 76041b349e292..78be5d2e51783 100644 --- a/src/runtime_view_proxy.js +++ b/src/runtime_view_proxy.js @@ -19,139 +19,156 @@ function getHeapBlock(type, offset, length) { } const heap = wasmMemory.buffer + + // we should always limit the length to maxArraySize + function createTypedArray (arrayType, offset, length) { + const bpe = arrayType.BYTES_PER_ELEMENT; + return new arrayType(heap, offset, length || Math.min(heap.byteLength - offset * bpe, maxArraySize / bpe)); + } + switch (type) { case 'i1': case 'i8': - return new Int8Array(heap, offset, length || maxArraySize / Int8Array.BYTES_PER_ELEMENT); + return createTypedArray(Int8Array, offset, length); case 'u1': case 'u8': - return new Uint8Array(heap, offset, length || maxArraySize / Uint8Array.BYTES_PER_ELEMENT); + return createTypedArray(Uint8Array, offset, length); case 'i16': - return new Int16Array(heap, offset, length || maxArraySize / Int16Array.BYTES_PER_ELEMENT); + return createTypedArray(Int16Array, offset, length); case 'u16': - return new Uint16Array(heap, offset, length || maxArraySize / Uint16Array.BYTES_PER_ELEMENT); + return createTypedArray(Uint16Array, offset, length); case 'i32': - return new Int32Array(heap, offset, length || maxArraySize / Int32Array.BYTES_PER_ELEMENT); + return createTypedArray(Int32Array, offset, length); case 'u32': - return new Uint32Array(heap, offset, length || maxArraySize / Uint32Array.BYTES_PER_ELEMENT); + return createTypedArray(Uint32Array, offset, length); case 'f32': - return new Float32Array(heap, offset, length || maxArraySize / Float32Array.BYTES_PER_ELEMENT); + return createTypedArray(Float32Array, offset, length); case 'f64': - return new Float64Array(heap, offset, length || maxArraySize / Float64Array.BYTES_PER_ELEMENT); + return createTypedArray(Float64Array, offset, length); case 'i64': - return new BigInt64Array(heap, offset, length || maxArraySize / BigInt64Array.BYTES_PER_ELEMENT); + return createTypedArray(BigInt64Array, offset, length); case '*': case 'u64': - return new BigUint64Array(heap, offset, length || maxArraySize / BigUint64Array.BYTES_PER_ELEMENT); + return createTypedArray(BigUint64Array, offset, length); default: throw new Error('Invalid type'); } } -var proxyHandler = (type, heapBlock) => ({ - heapBlock, - copyWithin (target, start, end) { - const bpe = heapBlock.BYTES_PER_ELEMENT +function createProxyHandler (type, heapBlocks) { + const firstHeapBlock = heapBlocks[0] + const bpe = firstHeapBlock.BYTES_PER_ELEMENT + + function getRealStartAndEnd(start, end) { + const startReal = (start || 0) * bpe + const endReal = end ? end * bpe : wasmMemory.byteLength + return [startReal, endReal] + } + + function copyWithin (target, start, end) { if (target * bpe >= maxArraySize || start * bpe >= maxArraySize || (end && end * bpe >= maxArraySize)) { - var len = end - start - var targetArray = getHeapBlock(type, target * bpe, len) - var sourceArray = getHeapBlock(type, start * bpe, len) + const len = end - start + const targetArray = getHeapBlock(type, target * bpe, len) + const sourceArray = getHeapBlock(type, start * bpe, len) targetArray.set(sourceArray) - return heapBlock + return heapBlocks[0] } else { - return heapBlock.copyWithin(target, start, end) + return heapBlocks[0].copyWithin(target, start, end) } - }, - setOverridden(array, offset) { - var offsetReal = (offset || 0) * heapBlock.BYTES_PER_ELEMENT + } + + function setOverridden(array, offset) { + const offsetReal = (offset || 0) * bpe if (offsetReal >= maxArraySize || array.byteLength + offsetReal >= maxArraySize) { - var targetArray = getHeapBlock(type, offsetReal, array.length) + const targetArray = getHeapBlock(type, offsetReal, array.length) targetArray.set(array) } else { - heapBlock.set(array, offset) + firstHeapBlock.set(array, offset) } - }, - subarray(start, end) { - var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT - var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength + } + + function subarray(start, end) { + const [startReal, endReal] = getRealStartAndEnd(start, end) if (startReal >= maxArraySize || endReal >= maxArraySize) { return getHeapBlock(type, startReal, endReal - startReal) } else { - return heapBlock.subarray(start, end) + return firstHeapBlock.subarray(start, end) } - }, - fill(value, start, end) { - var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT - var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength + } + + function fill(value, start, end) { + const [startReal, endReal] = getRealStartAndEnd(start, end) if (startReal >= maxArraySize || endReal >= maxArraySize) { const hb = getHeapBlock(type, startReal, endReal - startReal) - hb.fill(0, 0) - return heapBlock + hb.fill(value, 0, end - start) + return firstHeapBlock } else { - return heapBlock.fill(value, start, end) + return firstHeapBlock.fill(value, start, end) } - }, - slice(start, end) { - var startReal = (start || 0) * heapBlock.BYTES_PER_ELEMENT - var endReal = end ? end * heapBlock.BYTES_PER_ELEMENT : wasmMemory.byteLength + } + function slice(start, end) { + const [startReal, endReal] = getRealStartAndEnd(start, end) if (startReal >= maxArraySize || endReal >= maxArraySize) { const hb = getHeapBlock(type, startReal, endReal - startReal) return hb.slice(start, end) } else { - return heapBlock.slice(start, end) + return firstHeapBlock.slice(start, end) } - }, - get(target, property) { - if (typeof property === 'number' || typeof property === 'bigint') { - var memoryOffset = property * target.BYTES_PER_ELEMENT - if (memoryOffset >= maxArraySize) { - var heap = getHeapBlock(type, memoryOffset, 1); - return heap[0]; - } else { - return this.heapBlock[property] + } + + return { + get(target, property) { + if (parseInt(property, 10) == property) { + const memoryOffset = property * bpe + const blockNumber = Math.floor(memoryOffset / maxArraySize) + return heapBlocks[blockNumber][property - blockNumber * maxArraySize] } - } - if (property === 'copyWithin') { - return this.copyWithin - } + if (property === 'copyWithin') { + return copyWithin + } - if (property === 'set') { - return this.setOverridden - } + if (property === 'set') { + return setOverridden + } - if (property === 'subarray') { - return this.subarray - } + if (property === 'subarray') { + return subarray + } - if (property === 'fill') { - return this.fill - } + if (property === 'fill') { + return fill + } - if (property === 'slice') { - return this.slice - } + if (property === 'slice') { + return slice + } - return heapBlock[property] - }, - set(target, property, value) { - if (typeof property === 'number' || typeof property === 'bigint') { - var memoryOffset = property * target.BYTES_PER_ELEMENT - if (memoryOffset >= maxArraySize) { - var heap = getHeapBlock(type, memoryOffset, 1); - heap[0] = value; - return true; + return firstHeapBlock[property] + }, + set(target, property, value) { + if (parseInt(property, 10) == property) { + const memoryOffset = property * bpe + const blockNumber = Math.floor(memoryOffset / maxArraySize) + heapBlocks[blockNumber][property - blockNumber * maxArraySize] = value + return true } - } - heapBlock[property] = value - return true; - }, -}) + firstHeapBlock[property] = value + return true; + }, + } +} function createMemoryProxy(type) { - const block = getHeapBlock(type, 0) - return new Proxy(block, proxyHandler(type, block)); + const heapBlocks = [ + getHeapBlock(type, 0), + ]; + const numberOfBlocks = Math.ceil(b.byteLength / maxArraySize) + for (let i = 1; i < numberOfBlocks; i++) { + heapBlocks.push(getHeapBlock(type, i * maxArraySize)) + } + return new Proxy(heapBlocks[0], createProxyHandler(type, heapBlocks)); } if (b.byteLength > maxArraySize) { diff --git a/test/test_other.py b/test/test_other.py index 55ff9072a64bd..50153fdb6510c 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -13554,3 +13554,75 @@ def test_proxy_to_worker(self): @also_with_standalone_wasm() def test_console_out(self): self.do_other_test('test_console_out.c') + + def test_define_modularize(self): + self.run_process([EMCC, test_file('hello_world.c'), '-sMODULARIZE', '-sASSERTIONS=0']) + src = 'var module = 0; ' + read_file('a.out.js') + create_file('a.out.js', src) + self.assertContained("define([], () => Module);", src) + + create_file('run_module.js', 'var m; (global.define = (deps, factory) => { m = factory(); }).amd = true; require("./a.out.js"); m();') + output = self.run_js('run_module.js') + self.assertContained('hello, world!\n', output) + + self.run_process([EMCC, test_file('hello_world.c'), '-sMODULARIZE', '-sEXPORT_NAME="NotModule"', '-sASSERTIONS=0']) + src = 'var module = 0; ' + read_file('a.out.js') + create_file('a.out.js', src) + self.assertContained("define([], () => NotModule);", src) + + output = self.run_js('run_module.js') + self.assertContained('hello, world!\n', output) + + @requires_wasm64 + def test_memory64_proxies(self): + create_file('post.js', r''' + addOnPostRun(() => { + // check >4gb alloc + const bigChunk = _malloc(4 * 1024 * 1024 * 1024 + 100); + assert(bigChunk > 0); + + const littleChunk = _malloc(100); + HEAP8[littleChunk] = 2; + assert(HEAP8[littleChunk] === 2); + + // .subarray + const subarray = HEAP8.subarray(littleChunk, littleChunk + 100); + assert(subarray[0] === 2); + + // check .fill + HEAP8.fill(3, littleChunk, littleChunk + 99); + assert(subarray[0] === 3); + assert(subarray[98] === 3); + assert(subarray[99] === 0); + assert(HEAP8[littleChunk] === 3); + assert(HEAP8[littleChunk + 98] === 3); + assert(HEAP8[littleChunk + 99] === 0); + + // check .set + const filler = new Uint8Array(10); + filler[0] = 4; + filler[9] = 4; + HEAP8.set(filler, littleChunk, 10); + assert(subarray[0] === 4); + assert(subarray[9] === 4); + assert(HEAP8[littleChunk] === 4); + + // .copyWithin + HEAP8.copyWithin(bigChunk, littleChunk, littleChunk + 100); + assert(HEAP8[bigChunk] === 4); + + // .slice + const slice = HEAP8.slice(bigChunk, bigChunk + 100); + slice[0] = 5; + assert(HEAP8[bigChunk] === 4); + }); + ''') + self.run_process([EMCC, test_file('hello_world.c'), + '-sMEMORY64=1', + '-sINITIAL_MEMORY=5gb', + '-sMAXIMUM_MEMORY=5gb', + '-sALLOW_MEMORY_GROWTH', + '-sEXPORTED_FUNCTIONS=_malloc,_main', + '-Wno-experimental', + '--extern-post-js', 'post.js']) + self.run_js('a.out.js') From c21a0c2c77cd6d0285f3e895b43c8fdd3c22e2ca Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Fri, 7 Jul 2023 04:18:11 +0400 Subject: [PATCH 09/21] Duplicate code removed --- test/test_other.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/test_other.py b/test/test_other.py index 50153fdb6510c..af3a82c109b2b 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -5961,24 +5961,6 @@ def test_pthread_print_override_modularize(self): self.assertNotContained('hello, world!', output) self.assertContained('hello, earth!', output) - def test_define_modularize(self): - self.run_process([EMCC, test_file('hello_world.c'), '-sMODULARIZE', '-sASSERTIONS=0']) - src = 'var module = 0; ' + read_file('a.out.js') - create_file('a.out.js', src) - self.assertContained("define([], () => Module);", src) - - create_file('run_module.js', 'var m; (global.define = (deps, factory) => { m = factory(); }).amd = true; require("./a.out.js"); m();') - output = self.run_js('run_module.js') - self.assertContained('hello, world!\n', output) - - self.run_process([EMCC, test_file('hello_world.c'), '-sMODULARIZE', '-sEXPORT_NAME="NotModule"', '-sASSERTIONS=0']) - src = 'var module = 0; ' + read_file('a.out.js') - create_file('a.out.js', src) - self.assertContained("define([], () => NotModule);", src) - - output = self.run_js('run_module.js') - self.assertContained('hello, world!\n', output) - def test_EXPORT_NAME_with_html(self): err = self.expect_fail([EMCC, test_file('hello_world.c'), '-o', 'a.html', '-sEXPORT_NAME=Other']) self.assertContained('Customizing EXPORT_NAME requires that the HTML be customized to use that name', err) From 9ac2cfa62627fae4c3af556636e050559206868d Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Fri, 7 Jul 2023 04:22:03 +0400 Subject: [PATCH 10/21] Duplicate code removed --- test/test_other.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/test/test_other.py b/test/test_other.py index af3a82c109b2b..7471669047652 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -5961,6 +5961,24 @@ def test_pthread_print_override_modularize(self): self.assertNotContained('hello, world!', output) self.assertContained('hello, earth!', output) + def test_define_modularize(self): + self.run_process([EMCC, test_file('hello_world.c'), '-sMODULARIZE', '-sASSERTIONS=0']) + src = 'var module = 0; ' + read_file('a.out.js') + create_file('a.out.js', src) + self.assertContained("define([], () => Module);", src) + + create_file('run_module.js', 'var m; (global.define = (deps, factory) => { m = factory(); }).amd = true; require("./a.out.js"); m();') + output = self.run_js('run_module.js') + self.assertContained('hello, world!\n', output) + + self.run_process([EMCC, test_file('hello_world.c'), '-sMODULARIZE', '-sEXPORT_NAME="NotModule"', '-sASSERTIONS=0']) + src = 'var module = 0; ' + read_file('a.out.js') + create_file('a.out.js', src) + self.assertContained("define([], () => NotModule);", src) + + output = self.run_js('run_module.js') + self.assertContained('hello, world!\n', output) + def test_EXPORT_NAME_with_html(self): err = self.expect_fail([EMCC, test_file('hello_world.c'), '-o', 'a.html', '-sEXPORT_NAME=Other']) self.assertContained('Customizing EXPORT_NAME requires that the HTML be customized to use that name', err) @@ -13537,24 +13555,6 @@ def test_proxy_to_worker(self): def test_console_out(self): self.do_other_test('test_console_out.c') - def test_define_modularize(self): - self.run_process([EMCC, test_file('hello_world.c'), '-sMODULARIZE', '-sASSERTIONS=0']) - src = 'var module = 0; ' + read_file('a.out.js') - create_file('a.out.js', src) - self.assertContained("define([], () => Module);", src) - - create_file('run_module.js', 'var m; (global.define = (deps, factory) => { m = factory(); }).amd = true; require("./a.out.js"); m();') - output = self.run_js('run_module.js') - self.assertContained('hello, world!\n', output) - - self.run_process([EMCC, test_file('hello_world.c'), '-sMODULARIZE', '-sEXPORT_NAME="NotModule"', '-sASSERTIONS=0']) - src = 'var module = 0; ' + read_file('a.out.js') - create_file('a.out.js', src) - self.assertContained("define([], () => NotModule);", src) - - output = self.run_js('run_module.js') - self.assertContained('hello, world!\n', output) - @requires_wasm64 def test_memory64_proxies(self): create_file('post.js', r''' From 1bab6b12fb66bfee352710b5ec4c9f41a4115c69 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Fri, 7 Jul 2023 05:35:38 +0400 Subject: [PATCH 11/21] Test fixes --- test/test_other.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_other.py b/test/test_other.py index 7471669047652..7b0d1bee2147b 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -13600,11 +13600,11 @@ def test_memory64_proxies(self): }); ''') self.run_process([EMCC, test_file('hello_world.c'), - '-sMEMORY64=1', - '-sINITIAL_MEMORY=5gb', - '-sMAXIMUM_MEMORY=5gb', - '-sALLOW_MEMORY_GROWTH', - '-sEXPORTED_FUNCTIONS=_malloc,_main', - '-Wno-experimental', - '--extern-post-js', 'post.js']) + '-sMEMORY64=1', + '-sINITIAL_MEMORY=5gb', + '-sMAXIMUM_MEMORY=5gb', + '-sALLOW_MEMORY_GROWTH', + '-sEXPORTED_FUNCTIONS=_malloc,_main', + '-Wno-experimental', + '--extern-post-js', 'post.js']) self.run_js('a.out.js') From cb9414961aaaab287e00ffc7763f38db84089e74 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Fri, 7 Jul 2023 06:05:01 +0400 Subject: [PATCH 12/21] Updated memory proxy functions --- src/runtime_view_proxy.js | 83 +++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/src/runtime_view_proxy.js b/src/runtime_view_proxy.js index 78be5d2e51783..aa8c4f3f44ed2 100644 --- a/src/runtime_view_proxy.js +++ b/src/runtime_view_proxy.js @@ -7,7 +7,7 @@ // Chrome does not allow TypedArrays with more than 4294967296 elements // We'll create proxy objects for HEAP(U)8 when memory is > 4gb and for HEAP(U)16 when > 8gb // https://bugs.chromium.org/p/v8/issues/detail?id=4153 -var maxArraySize = Math.min(b.byteLength, 4 * 1024 * 1024 * 1024 - 8); +var maxArraySize = Math.min(b.byteLength, 4 * 1024 * 1024 * 1024 - 2); /** * @param {string} type - Heap type * @param {number} [offset] - Heap offset @@ -18,12 +18,12 @@ function getHeapBlock(type, offset, length) { offset = 0 } - const heap = wasmMemory.buffer + let heap = wasmMemory.buffer // we should always limit the length to maxArraySize - function createTypedArray (arrayType, offset, length) { - const bpe = arrayType.BYTES_PER_ELEMENT; - return new arrayType(heap, offset, length || Math.min(heap.byteLength - offset * bpe, maxArraySize / bpe)); + function createTypedArray(arrayType, offset, length) { + let bpe = arrayType.BYTES_PER_ELEMENT; + return new arrayType(heap, offset, length || Math.min((heap.byteLength - offset) / bpe, maxArraySize)); } switch (type) { @@ -37,39 +37,26 @@ function getHeapBlock(type, offset, length) { return createTypedArray(Int16Array, offset, length); case 'u16': return createTypedArray(Uint16Array, offset, length); - case 'i32': - return createTypedArray(Int32Array, offset, length); - case 'u32': - return createTypedArray(Uint32Array, offset, length); - case 'f32': - return createTypedArray(Float32Array, offset, length); - case 'f64': - return createTypedArray(Float64Array, offset, length); - case 'i64': - return createTypedArray(BigInt64Array, offset, length); - case '*': - case 'u64': - return createTypedArray(BigUint64Array, offset, length); default: throw new Error('Invalid type'); } } -function createProxyHandler (type, heapBlocks) { - const firstHeapBlock = heapBlocks[0] - const bpe = firstHeapBlock.BYTES_PER_ELEMENT +function createProxyHandler(type, heapBlocks) { + let firstHeapBlock = heapBlocks[0] + let bpe = firstHeapBlock.BYTES_PER_ELEMENT function getRealStartAndEnd(start, end) { - const startReal = (start || 0) * bpe - const endReal = end ? end * bpe : wasmMemory.byteLength + let startReal = (start || 0) * bpe + let endReal = end ? end * bpe : wasmMemory.byteLength return [startReal, endReal] } - function copyWithin (target, start, end) { + function copyWithin(target, start, end) { if (target * bpe >= maxArraySize || start * bpe >= maxArraySize || (end && end * bpe >= maxArraySize)) { - const len = end - start - const targetArray = getHeapBlock(type, target * bpe, len) - const sourceArray = getHeapBlock(type, start * bpe, len) + let len = end - start + let targetArray = getHeapBlock(type, target * bpe, len) + let sourceArray = getHeapBlock(type, start * bpe, len) targetArray.set(sourceArray) return heapBlocks[0] } else { @@ -78,9 +65,9 @@ function createProxyHandler (type, heapBlocks) { } function setOverridden(array, offset) { - const offsetReal = (offset || 0) * bpe + let offsetReal = (offset || 0) * bpe if (offsetReal >= maxArraySize || array.byteLength + offsetReal >= maxArraySize) { - const targetArray = getHeapBlock(type, offsetReal, array.length) + let targetArray = getHeapBlock(type, offsetReal, array.length) targetArray.set(array) } else { firstHeapBlock.set(array, offset) @@ -88,7 +75,7 @@ function createProxyHandler (type, heapBlocks) { } function subarray(start, end) { - const [startReal, endReal] = getRealStartAndEnd(start, end) + let [startReal, endReal] = getRealStartAndEnd(start, end) if (startReal >= maxArraySize || endReal >= maxArraySize) { return getHeapBlock(type, startReal, endReal - startReal) } else { @@ -97,9 +84,9 @@ function createProxyHandler (type, heapBlocks) { } function fill(value, start, end) { - const [startReal, endReal] = getRealStartAndEnd(start, end) + let [startReal, endReal] = getRealStartAndEnd(start, end) if (startReal >= maxArraySize || endReal >= maxArraySize) { - const hb = getHeapBlock(type, startReal, endReal - startReal) + let hb = getHeapBlock(type, startReal, endReal - startReal) hb.fill(value, 0, end - start) return firstHeapBlock } else { @@ -107,9 +94,9 @@ function createProxyHandler (type, heapBlocks) { } } function slice(start, end) { - const [startReal, endReal] = getRealStartAndEnd(start, end) + let [startReal, endReal] = getRealStartAndEnd(start, end) if (startReal >= maxArraySize || endReal >= maxArraySize) { - const hb = getHeapBlock(type, startReal, endReal - startReal) + let hb = getHeapBlock(type, startReal, endReal - startReal) return hb.slice(start, end) } else { return firstHeapBlock.slice(start, end) @@ -119,8 +106,8 @@ function createProxyHandler (type, heapBlocks) { return { get(target, property) { if (parseInt(property, 10) == property) { - const memoryOffset = property * bpe - const blockNumber = Math.floor(memoryOffset / maxArraySize) + let memoryOffset = property * bpe + let blockNumber = Math.floor(memoryOffset / maxArraySize) return heapBlocks[blockNumber][property - blockNumber * maxArraySize] } @@ -148,8 +135,8 @@ function createProxyHandler (type, heapBlocks) { }, set(target, property, value) { if (parseInt(property, 10) == property) { - const memoryOffset = property * bpe - const blockNumber = Math.floor(memoryOffset / maxArraySize) + let memoryOffset = property * bpe + let blockNumber = Math.floor(memoryOffset / maxArraySize) heapBlocks[blockNumber][property - blockNumber * maxArraySize] = value return true } @@ -161,26 +148,26 @@ function createProxyHandler (type, heapBlocks) { } function createMemoryProxy(type) { - const heapBlocks = [ - getHeapBlock(type, 0), - ]; - const numberOfBlocks = Math.ceil(b.byteLength / maxArraySize) - for (let i = 1; i < numberOfBlocks; i++) { - heapBlocks.push(getHeapBlock(type, i * maxArraySize)) + let heapBlocks = []; + let bpe = type === 'i16' || type === 'u16' ? 2 : 1 + let numberOfBlocks = Math.ceil(b.byteLength / maxArraySize / bpe) + for (let i = 0; i < numberOfBlocks; i++) { + heapBlocks.push(getHeapBlock(type, i * maxArraySize * bpe)) } + console.log(heapBlocks, type) return new Proxy(heapBlocks[0], createProxyHandler(type, heapBlocks)); } if (b.byteLength > maxArraySize) { - Module["HEAP8"] = HEAP8 = createMemoryProxy('i8') - Module["HEAPU8"] = HEAPU8 = createMemoryProxy('u8') + Module['HEAP8'] = HEAP8 = createMemoryProxy('i8') + Module['HEAPU8'] = HEAPU8 = createMemoryProxy('u8') } else { Module['HEAP8'] = HEAP8 = new Int8Array(b); Module['HEAPU8'] = HEAPU8 = new Uint8Array(b); } if (b.byteLength > maxArraySize * 2) { - Module["HEAP16"] = HEAP16 = createMemoryProxy('i16') - Module["HEAPU16"] = HEAPU16 = createMemoryProxy('u16') + Module['HEAP16'] = HEAP16 = createMemoryProxy('i16') + Module['HEAPU16'] = HEAPU16 = createMemoryProxy('u16') } else { Module['HEAP16'] = HEAP16 = new Int16Array(b); Module['HEAPU16'] = HEAPU16 = new Uint16Array(b); From 09555b7b65d8f649bba30de042f1368307cd55de Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Sun, 9 Jul 2023 14:09:37 +0400 Subject: [PATCH 13/21] Test fixes --- .circleci/config.yml | 2 +- test/test_browser.py | 1 + test/test_other.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 849b319e1e560..720b04fea242e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -587,7 +587,7 @@ jobs: - install-node-canary - run-tests: title: "wasm64" - test_targets: "wasm64 wasm64l.test_bigswitch" + test_targets: "wasm64 wasm64l.test_bigswitch other.test_memory64_proxies" - upload-test-results test-jsc: executor: linux-python diff --git a/test/test_browser.py b/test/test_browser.py index 5750afbdf6dae..09d1e010892ad 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -5501,6 +5501,7 @@ def test_zzz_zzz_2gb_fail(self): @no_firefox('no 4GB support yet') @also_with_wasm64 + @requires_v8 def test_zzz_zzz_4gb_fail(self): # TODO Convert to an actual browser test when it reaches stable. # For now, keep this in browser as this suite runs serially, which diff --git a/test/test_other.py b/test/test_other.py index 7b0d1bee2147b..0594636541228 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -32,7 +32,7 @@ from common import RunnerCore, path_from_root, is_slow_test, ensure_dir, disabled, make_executable from common import env_modify, no_mac, no_windows, only_windows, requires_native_clang, with_env_modify from common import create_file, parameterized, NON_ZERO, node_pthreads, TEST_ROOT, test_file -from common import compiler_for, EMBUILDER, requires_v8, requires_node, requires_wasm64 +from common import compiler_for, EMBUILDER, requires_v8, requires_node, requires_wasm64, requires_node_canary from common import requires_wasm_eh, crossplatform, with_both_sjlj, also_with_standalone_wasm from common import also_with_minimal_runtime, also_with_wasm_bigint, also_with_wasm64 from common import EMTEST_BUILD_VERBOSE, PYTHON @@ -13556,6 +13556,7 @@ def test_console_out(self): self.do_other_test('test_console_out.c') @requires_wasm64 + @requires_node_canary def test_memory64_proxies(self): create_file('post.js', r''' addOnPostRun(() => { From df688167ca5e7cba5611529338161f33b55f92e6 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Sun, 9 Jul 2023 18:02:23 +0400 Subject: [PATCH 14/21] Test fixes --- .circleci/config.yml | 4 +++- test/test_other.py | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 720b04fea242e..ef8ac4a3fa3a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -587,7 +587,7 @@ jobs: - install-node-canary - run-tests: title: "wasm64" - test_targets: "wasm64 wasm64l.test_bigswitch other.test_memory64_proxies" + test_targets: "wasm64 wasm64l.test_bigswitch other.test_memory64_proxies other.test_failing_growth_wasm64" - upload-test-results test-jsc: executor: linux-python @@ -691,6 +691,8 @@ jobs: - upload-test-results test-other: executor: bionic + environment: + EMTEST_SKIP_NODE_CANARY: "1" steps: - run: apt-get install ninja-build scons - run-tests-linux: diff --git a/test/test_other.py b/test/test_other.py index 0594636541228..478d000c02d74 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -6261,9 +6261,8 @@ def test_failing_growth_2gb(self): self.run_process([EMCC, '-O1', 'test.c', '-sALLOW_MEMORY_GROWTH']) self.assertContained('done', self.run_js('a.out.js')) + @requires_wasm64 def test_failing_growth_wasm64(self): - # For now we skip this test because failure to create the TypedArray views - # causes weird unrecoverable failures. self.require_wasm64() create_file('test.c', r''' #include From 87aeb171390e7422badeaa9341f3fd52be1e78a1 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Tue, 11 Jul 2023 02:29:29 +0400 Subject: [PATCH 15/21] Test fixes --- .circleci/config.yml | 6 +++++- test/test_other.py | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ef8ac4a3fa3a8..25f0c3ec3a0cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -587,7 +587,11 @@ jobs: - install-node-canary - run-tests: title: "wasm64" - test_targets: "wasm64 wasm64l.test_bigswitch other.test_memory64_proxies other.test_failing_growth_wasm64" + test_targets: " + wasm64 + wasm64l.test_bigswitch + other.test_memory64_proxies + other.test_failing_growth_wasm64" - upload-test-results test-jsc: executor: linux-python diff --git a/test/test_other.py b/test/test_other.py index 478d000c02d74..52e008e83acc6 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -6262,6 +6262,7 @@ def test_failing_growth_2gb(self): self.assertContained('done', self.run_js('a.out.js')) @requires_wasm64 + @requires_node_canary def test_failing_growth_wasm64(self): self.require_wasm64() create_file('test.c', r''' From c0d9a762f82f152b29546c50bf93b7f6fa0a5d1a Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Tue, 11 Jul 2023 04:29:23 +0400 Subject: [PATCH 16/21] Run 4gb memory tests in the browser instead of d8 --- test/test_browser.py | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/test/test_browser.py b/test/test_browser.py index 09d1e010892ad..50cc8a407facd 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -23,7 +23,7 @@ from common import BrowserCore, RunnerCore, path_from_root, has_browser, EMTEST_BROWSER, Reporting from common import create_file, parameterized, ensure_dir, disabled, test_file, WEBIDL_BINDER -from common import read_file, requires_v8, also_with_minimal_runtime, also_with_wasm64, EMRUN +from common import read_file, also_with_minimal_runtime, also_with_wasm64, EMRUN from tools import shared from tools import ports from tools import utils @@ -5396,14 +5396,7 @@ def test_wasm_worker_proxied_function(self): self.btest(test_file('wasm_worker/proxied_function.c'), expected='0', args=['--js-library', test_file('wasm_worker/proxied_function.js'), '-sWASM_WORKERS', '-sASSERTIONS=0']) @no_firefox('no 4GB support yet') - @requires_v8 def test_zzz_zzz_4gb(self): - # TODO Convert to an actual browser test when it reaches stable. - # For now, keep this in browser as this suite runs serially, which - # means we don't compete for memory with anything else (and run it - # at the very very end, to reduce the risk of it OOM-killing the - # browser). - # test that we can allocate in the 2-4GB range, if we enable growth and # set the max appropriately self.emcc_args += ['-O2', '-sALLOW_MEMORY_GROWTH', '-sMAXIMUM_MEMORY=4GB'] @@ -5486,14 +5479,7 @@ def test_zzz_zzz_emmalloc_memgrowth(self, *args): self.btest(test_file('browser/emmalloc_memgrowth.cpp'), expected='0', args=['-sMALLOC=emmalloc', '-sALLOW_MEMORY_GROWTH=1', '-sABORTING_MALLOC=0', '-sASSERTIONS=2', '-sMINIMAL_RUNTIME=1', '-sMAXIMUM_MEMORY=4GB']) @no_firefox('no 4GB support yet') - @requires_v8 def test_zzz_zzz_2gb_fail(self): - # TODO Convert to an actual browser test when it reaches stable. - # For now, keep this in browser as this suite runs serially, which - # means we don't compete for memory with anything else (and run it - # at the very very end, to reduce the risk of it OOM-killing the - # browser). - # test that growth doesn't go beyond 2GB without the max being set for that, # and that we can catch an allocation failure exception for that self.emcc_args += ['-O2', '-sALLOW_MEMORY_GROWTH', '-sMAXIMUM_MEMORY=2GB'] @@ -5501,14 +5487,7 @@ def test_zzz_zzz_2gb_fail(self): @no_firefox('no 4GB support yet') @also_with_wasm64 - @requires_v8 def test_zzz_zzz_4gb_fail(self): - # TODO Convert to an actual browser test when it reaches stable. - # For now, keep this in browser as this suite runs serially, which - # means we don't compete for memory with anything else (and run it - # at the very very end, to reduce the risk of it OOM-killing the - # browser). - # test that we properly report an allocation error that would overflow over # 4GB. if self.get_setting('MEMORY64'): From 34b85dc9068ddc09151c20844447690f133def71 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Tue, 11 Jul 2023 22:24:57 +0400 Subject: [PATCH 17/21] Disabled test_zzz_zzz_4gb_fail_wasm64 --- .circleci/config.yml | 2 +- test/test_browser.py | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 25f0c3ec3a0cb..7c57ab4c689d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -360,7 +360,7 @@ commands: export EM_FROZEN_CACHE="" test/runner emrun # skip test_zzz_zzz_4gb_fail as it OOMs on the current bot - test/runner posixtest_browser.test_pthread_create_1_1 browser skip:browser.test_zzz_zzz_4gb_fail + test/runner posixtest_browser.test_pthread_create_1_1 browser skip:browser.test_zzz_zzz_4gb_fail skip:browser.test_zzz_zzz_4gb_fail_wasm64 - upload-test-results test-sockets-chrome: description: "Runs emscripten sockets tests under chrome" diff --git a/test/test_browser.py b/test/test_browser.py index 50cc8a407facd..75f65b18ec259 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -23,7 +23,7 @@ from common import BrowserCore, RunnerCore, path_from_root, has_browser, EMTEST_BROWSER, Reporting from common import create_file, parameterized, ensure_dir, disabled, test_file, WEBIDL_BINDER -from common import read_file, also_with_minimal_runtime, also_with_wasm64, EMRUN +from common import read_file, requires_v8, also_with_minimal_runtime, also_with_wasm64, EMRUN from tools import shared from tools import ports from tools import utils @@ -5479,7 +5479,14 @@ def test_zzz_zzz_emmalloc_memgrowth(self, *args): self.btest(test_file('browser/emmalloc_memgrowth.cpp'), expected='0', args=['-sMALLOC=emmalloc', '-sALLOW_MEMORY_GROWTH=1', '-sABORTING_MALLOC=0', '-sASSERTIONS=2', '-sMINIMAL_RUNTIME=1', '-sMAXIMUM_MEMORY=4GB']) @no_firefox('no 4GB support yet') + @requires_v8 def test_zzz_zzz_2gb_fail(self): + # TODO Convert to an actual browser test when it reaches stable. + # For now, keep this in browser as this suite runs serially, which + # means we don't compete for memory with anything else (and run it + # at the very very end, to reduce the risk of it OOM-killing the + # browser). + # test that growth doesn't go beyond 2GB without the max being set for that, # and that we can catch an allocation failure exception for that self.emcc_args += ['-O2', '-sALLOW_MEMORY_GROWTH', '-sMAXIMUM_MEMORY=2GB'] @@ -5487,7 +5494,14 @@ def test_zzz_zzz_2gb_fail(self): @no_firefox('no 4GB support yet') @also_with_wasm64 + @requires_v8 def test_zzz_zzz_4gb_fail(self): + # TODO Convert to an actual browser test when it reaches stable. + # For now, keep this in browser as this suite runs serially, which + # means we don't compete for memory with anything else (and run it + # at the very very end, to reduce the risk of it OOM-killing the + # browser). + # test that we properly report an allocation error that would overflow over # 4GB. if self.get_setting('MEMORY64'): From ed8bccf89644b1a87224540aaba576455bebed87 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Tue, 18 Jul 2023 18:16:55 +0400 Subject: [PATCH 18/21] Removed console.log --- src/runtime_view_proxy.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/runtime_view_proxy.js b/src/runtime_view_proxy.js index aa8c4f3f44ed2..0f97f27ae1801 100644 --- a/src/runtime_view_proxy.js +++ b/src/runtime_view_proxy.js @@ -154,7 +154,6 @@ function createMemoryProxy(type) { for (let i = 0; i < numberOfBlocks; i++) { heapBlocks.push(getHeapBlock(type, i * maxArraySize * bpe)) } - console.log(heapBlocks, type) return new Proxy(heapBlocks[0], createProxyHandler(type, heapBlocks)); } From a75816683826f2591e31433a56cedf388eecf007 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Tue, 18 Jul 2023 22:12:46 +0400 Subject: [PATCH 19/21] Upstream test fixes --- test/code_size/hello_webgl2_wasm.json | 8 ++++---- test/code_size/hello_webgl2_wasm2js.json | 8 ++++---- test/code_size/hello_webgl_wasm.json | 8 ++++---- test/code_size/hello_webgl_wasm2js.json | 8 ++++---- test/code_size/random_printf_wasm.json | 8 ++++---- test/code_size/random_printf_wasm2js.json | 8 ++++---- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/test/code_size/hello_webgl2_wasm.json b/test/code_size/hello_webgl2_wasm.json index 78c1b2fe82b4c..a729d84421e02 100644 --- a/test/code_size/hello_webgl2_wasm.json +++ b/test/code_size/hello_webgl2_wasm.json @@ -3,8 +3,8 @@ "a.html.gz": 379, "a.js": 4933, "a.js.gz": 2433, - "a.wasm": 10482, - "a.wasm.gz": 6707, - "total": 15984, - "total_gz": 9519 + "a.wasm": 10492, + "a.wasm.gz": 6710, + "total": 15994, + "total_gz": 9522 } diff --git a/test/code_size/hello_webgl2_wasm2js.json b/test/code_size/hello_webgl2_wasm2js.json index 8cf5f3d806a49..62d185bd87230 100644 --- a/test/code_size/hello_webgl2_wasm2js.json +++ b/test/code_size/hello_webgl2_wasm2js.json @@ -1,10 +1,10 @@ { "a.html": 567, "a.html.gz": 379, - "a.js": 18222, - "a.js.gz": 8063, + "a.js": 18229, + "a.js.gz": 8065, "a.mem": 3171, "a.mem.gz": 2713, - "total": 21960, - "total_gz": 11155 + "total": 21967, + "total_gz": 11157 } diff --git a/test/code_size/hello_webgl_wasm.json b/test/code_size/hello_webgl_wasm.json index 12848e31f1f28..179d418ea288d 100644 --- a/test/code_size/hello_webgl_wasm.json +++ b/test/code_size/hello_webgl_wasm.json @@ -3,8 +3,8 @@ "a.html.gz": 379, "a.js": 4412, "a.js.gz": 2258, - "a.wasm": 10482, - "a.wasm.gz": 6707, - "total": 15463, - "total_gz": 9344 + "a.wasm": 10492, + "a.wasm.gz": 6710, + "total": 15473, + "total_gz": 9347 } diff --git a/test/code_size/hello_webgl_wasm2js.json b/test/code_size/hello_webgl_wasm2js.json index e12f83e6fdc9a..38032a6193762 100644 --- a/test/code_size/hello_webgl_wasm2js.json +++ b/test/code_size/hello_webgl_wasm2js.json @@ -1,10 +1,10 @@ { "a.html": 567, "a.html.gz": 379, - "a.js": 17693, - "a.js.gz": 7878, + "a.js": 17700, + "a.js.gz": 7881, "a.mem": 3171, "a.mem.gz": 2713, - "total": 21431, - "total_gz": 10970 + "total": 21438, + "total_gz": 10973 } diff --git a/test/code_size/random_printf_wasm.json b/test/code_size/random_printf_wasm.json index 8c293a8b8b617..6796a3d1db439 100644 --- a/test/code_size/random_printf_wasm.json +++ b/test/code_size/random_printf_wasm.json @@ -1,6 +1,6 @@ { - "a.html": 12793, - "a.html.gz": 6942, - "total": 12793, - "total_gz": 6942 + "a.html": 12797, + "a.html.gz": 6980, + "total": 12797, + "total_gz": 6980 } diff --git a/test/code_size/random_printf_wasm2js.json b/test/code_size/random_printf_wasm2js.json index 0414ed3307096..9d945e43c3c08 100644 --- a/test/code_size/random_printf_wasm2js.json +++ b/test/code_size/random_printf_wasm2js.json @@ -1,6 +1,6 @@ { - "a.html": 17380, - "a.html.gz": 7542, - "total": 17380, - "total_gz": 7542 + "a.html": 17407, + "a.html.gz": 7557, + "total": 17407, + "total_gz": 7557 } From 21d06c9d820c5144c95e2ed4cc76b2f60518a299 Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Wed, 26 Jul 2023 19:21:41 +0400 Subject: [PATCH 20/21] Test fixes --- test/other/test_memory64_proxies.js | 40 ++++++++++++++++++++++++ test/test_browser.py | 7 +++++ test/test_other.py | 47 +---------------------------- 3 files changed, 48 insertions(+), 46 deletions(-) create mode 100644 test/other/test_memory64_proxies.js diff --git a/test/other/test_memory64_proxies.js b/test/other/test_memory64_proxies.js new file mode 100644 index 0000000000000..748d0d2abd588 --- /dev/null +++ b/test/other/test_memory64_proxies.js @@ -0,0 +1,40 @@ +addOnPostRun(() => { + // check >4gb alloc + const bigChunk = _malloc(4 * 1024 * 1024 * 1024 + 100); + assert(bigChunk > 0); + + const littleChunk = _malloc(100); + HEAP8[littleChunk] = 2; + assert(HEAP8[littleChunk] === 2); + + // .subarray + const subarray = HEAP8.subarray(littleChunk, littleChunk + 100); + assert(subarray[0] === 2); + + // check .fill + HEAP8.fill(3, littleChunk, littleChunk + 99); + assert(subarray[0] === 3); + assert(subarray[98] === 3); + assert(subarray[99] === 0); + assert(HEAP8[littleChunk] === 3); + assert(HEAP8[littleChunk + 98] === 3); + assert(HEAP8[littleChunk + 99] === 0); + + // check .set + const filler = new Uint8Array(10); + filler[0] = 4; + filler[9] = 4; + HEAP8.set(filler, littleChunk, 10); + assert(subarray[0] === 4); + assert(subarray[9] === 4); + assert(HEAP8[littleChunk] === 4); + + // .copyWithin + HEAP8.copyWithin(bigChunk, littleChunk, littleChunk + 100); + assert(HEAP8[bigChunk] === 4); + + // .slice + const slice = HEAP8.slice(bigChunk, bigChunk + 100); + slice[0] = 5; + assert(HEAP8[bigChunk] === 4); +}); diff --git a/test/test_browser.py b/test/test_browser.py index 3d140902d5743..c73689fcd3d6c 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -5371,7 +5371,14 @@ def test_wasm_worker_proxied_function(self): self.btest(test_file('wasm_worker/proxied_function.c'), expected='0', args=['--js-library', test_file('wasm_worker/proxied_function.js'), '-sWASM_WORKERS', '-sASSERTIONS=0']) @no_firefox('no 4GB support yet') + @requires_v8 def test_zzz_zzz_4gb(self): + # TODO Convert to an actual browser test when it reaches stable. + # For now, keep this in browser as this suite runs serially, which + # means we don't compete for memory with anything else (and run it + # at the very very end, to reduce the risk of it OOM-killing the + # browser). + # test that we can allocate in the 2-4GB range, if we enable growth and # set the max appropriately self.emcc_args += ['-O2', '-sALLOW_MEMORY_GROWTH', '-sMAXIMUM_MEMORY=4GB'] diff --git a/test/test_other.py b/test/test_other.py index 91e21c715c26f..bb3603b2e60b4 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -13595,56 +13595,12 @@ def test_explicit_target(self): err = self.expect_fail([EMCC, test_file('hello_world.c'), '--target=arm64']) self.assertContained('emcc: error: unsupported target: arm64 (emcc only supports wasm64-unknown-emscripten and wasm32-unknown-emscripten', err) - def test_quick_exit(self): self.do_other_test('test_quick_exit.c') - @requires_wasm64 @requires_node_canary def test_memory64_proxies(self): - create_file('post.js', r''' - addOnPostRun(() => { - // check >4gb alloc - const bigChunk = _malloc(4 * 1024 * 1024 * 1024 + 100); - assert(bigChunk > 0); - - const littleChunk = _malloc(100); - HEAP8[littleChunk] = 2; - assert(HEAP8[littleChunk] === 2); - - // .subarray - const subarray = HEAP8.subarray(littleChunk, littleChunk + 100); - assert(subarray[0] === 2); - - // check .fill - HEAP8.fill(3, littleChunk, littleChunk + 99); - assert(subarray[0] === 3); - assert(subarray[98] === 3); - assert(subarray[99] === 0); - assert(HEAP8[littleChunk] === 3); - assert(HEAP8[littleChunk + 98] === 3); - assert(HEAP8[littleChunk + 99] === 0); - - // check .set - const filler = new Uint8Array(10); - filler[0] = 4; - filler[9] = 4; - HEAP8.set(filler, littleChunk, 10); - assert(subarray[0] === 4); - assert(subarray[9] === 4); - assert(HEAP8[littleChunk] === 4); - - // .copyWithin - HEAP8.copyWithin(bigChunk, littleChunk, littleChunk + 100); - assert(HEAP8[bigChunk] === 4); - - // .slice - const slice = HEAP8.slice(bigChunk, bigChunk + 100); - slice[0] = 5; - assert(HEAP8[bigChunk] === 4); - }); - ''') self.run_process([EMCC, test_file('hello_world.c'), '-sMEMORY64=1', '-sINITIAL_MEMORY=5gb', @@ -13652,6 +13608,5 @@ def test_memory64_proxies(self): '-sALLOW_MEMORY_GROWTH', '-sEXPORTED_FUNCTIONS=_malloc,_main', '-Wno-experimental', - '--extern-post-js', 'post.js']) + '--extern-post-js', test_file('other/test_memory64_proxies.js')]) self.run_js('a.out.js') - From 35f06f0295eb329d5822b9a6bf0bac687976fa7d Mon Sep 17 00:00:00 2001 From: Arthur Islamov Date: Wed, 26 Jul 2023 20:38:06 +0400 Subject: [PATCH 21/21] Reverted test size changes --- test/code_size/hello_webgl2_wasm.json | 8 ++++---- test/code_size/hello_webgl2_wasm2js.json | 8 ++++---- test/code_size/hello_webgl_wasm.json | 8 ++++---- test/code_size/hello_webgl_wasm2js.json | 8 ++++---- test/code_size/random_printf_wasm.json | 8 ++++---- test/code_size/random_printf_wasm2js.json | 8 ++++---- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/test/code_size/hello_webgl2_wasm.json b/test/code_size/hello_webgl2_wasm.json index a729d84421e02..78c1b2fe82b4c 100644 --- a/test/code_size/hello_webgl2_wasm.json +++ b/test/code_size/hello_webgl2_wasm.json @@ -3,8 +3,8 @@ "a.html.gz": 379, "a.js": 4933, "a.js.gz": 2433, - "a.wasm": 10492, - "a.wasm.gz": 6710, - "total": 15994, - "total_gz": 9522 + "a.wasm": 10482, + "a.wasm.gz": 6707, + "total": 15984, + "total_gz": 9519 } diff --git a/test/code_size/hello_webgl2_wasm2js.json b/test/code_size/hello_webgl2_wasm2js.json index 62d185bd87230..8cf5f3d806a49 100644 --- a/test/code_size/hello_webgl2_wasm2js.json +++ b/test/code_size/hello_webgl2_wasm2js.json @@ -1,10 +1,10 @@ { "a.html": 567, "a.html.gz": 379, - "a.js": 18229, - "a.js.gz": 8065, + "a.js": 18222, + "a.js.gz": 8063, "a.mem": 3171, "a.mem.gz": 2713, - "total": 21967, - "total_gz": 11157 + "total": 21960, + "total_gz": 11155 } diff --git a/test/code_size/hello_webgl_wasm.json b/test/code_size/hello_webgl_wasm.json index 179d418ea288d..12848e31f1f28 100644 --- a/test/code_size/hello_webgl_wasm.json +++ b/test/code_size/hello_webgl_wasm.json @@ -3,8 +3,8 @@ "a.html.gz": 379, "a.js": 4412, "a.js.gz": 2258, - "a.wasm": 10492, - "a.wasm.gz": 6710, - "total": 15473, - "total_gz": 9347 + "a.wasm": 10482, + "a.wasm.gz": 6707, + "total": 15463, + "total_gz": 9344 } diff --git a/test/code_size/hello_webgl_wasm2js.json b/test/code_size/hello_webgl_wasm2js.json index 38032a6193762..e12f83e6fdc9a 100644 --- a/test/code_size/hello_webgl_wasm2js.json +++ b/test/code_size/hello_webgl_wasm2js.json @@ -1,10 +1,10 @@ { "a.html": 567, "a.html.gz": 379, - "a.js": 17700, - "a.js.gz": 7881, + "a.js": 17693, + "a.js.gz": 7878, "a.mem": 3171, "a.mem.gz": 2713, - "total": 21438, - "total_gz": 10973 + "total": 21431, + "total_gz": 10970 } diff --git a/test/code_size/random_printf_wasm.json b/test/code_size/random_printf_wasm.json index 6796a3d1db439..8c293a8b8b617 100644 --- a/test/code_size/random_printf_wasm.json +++ b/test/code_size/random_printf_wasm.json @@ -1,6 +1,6 @@ { - "a.html": 12797, - "a.html.gz": 6980, - "total": 12797, - "total_gz": 6980 + "a.html": 12793, + "a.html.gz": 6942, + "total": 12793, + "total_gz": 6942 } diff --git a/test/code_size/random_printf_wasm2js.json b/test/code_size/random_printf_wasm2js.json index 9d945e43c3c08..0414ed3307096 100644 --- a/test/code_size/random_printf_wasm2js.json +++ b/test/code_size/random_printf_wasm2js.json @@ -1,6 +1,6 @@ { - "a.html": 17407, - "a.html.gz": 7557, - "total": 17407, - "total_gz": 7557 + "a.html": 17380, + "a.html.gz": 7542, + "total": 17380, + "total_gz": 7542 }