diff --git a/.circleci/config.yml b/.circleci/config.yml index 2500e05b55439..8e6f5f9b27b87 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -360,7 +360,7 @@ commands: export EM_FROZEN_CACHE="" test/runner emrun # skip test_zzz_zzz_4gb_fail as it OOMs on the current bot - test/runner posixtest_browser.test_pthread_create_1_1 browser skip:browser.test_zzz_zzz_4gb_fail + test/runner posixtest_browser.test_pthread_create_1_1 browser skip:browser.test_zzz_zzz_4gb_fail skip:browser.test_zzz_zzz_4gb_fail_wasm64 - upload-test-results test-sockets-chrome: description: "Runs emscripten sockets tests under chrome" @@ -587,7 +587,11 @@ jobs: - install-node-canary - run-tests: title: "wasm64" - test_targets: "wasm64 wasm64l.test_bigswitch" + test_targets: " + wasm64 + wasm64l.test_bigswitch + other.test_memory64_proxies + other.test_failing_growth_wasm64" - upload-test-results test-jsc: executor: linux-python @@ -694,6 +698,8 @@ jobs: - upload-test-results test-other: executor: bionic + environment: + EMTEST_SKIP_NODE_CANARY: "1" steps: - run: apt-get install ninja-build scons - run-tests-linux: diff --git a/src/preamble.js b/src/preamble.js index 719a82973497c..9ef852e109c52 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -140,11 +140,15 @@ function updateMemoryViews() { #if SUPPORT_BIG_ENDIAN Module['HEAP_DATA_VIEW'] = HEAP_DATA_VIEW = new DataView(b); #endif +#if MEMORY64 && MAXIMUM_MEMORY > FOUR_GB +#include "runtime_view_proxy.js" +#else Module['HEAP8'] = HEAP8 = new Int8Array(b); Module['HEAP16'] = HEAP16 = new Int16Array(b); - Module['HEAP32'] = HEAP32 = new Int32Array(b); Module['HEAPU8'] = HEAPU8 = new Uint8Array(b); Module['HEAPU16'] = HEAPU16 = new Uint16Array(b); +#endif + Module['HEAP32'] = HEAP32 = new Int32Array(b); Module['HEAPU32'] = HEAPU32 = new Uint32Array(b); Module['HEAPF32'] = HEAPF32 = new Float32Array(b); Module['HEAPF64'] = HEAPF64 = new Float64Array(b); diff --git a/src/runtime_view_proxy.js b/src/runtime_view_proxy.js new file mode 100644 index 0000000000000..0f97f27ae1801 --- /dev/null +++ b/src/runtime_view_proxy.js @@ -0,0 +1,173 @@ +/** + * @license + * Copyright 2023 The Emscripten Authors + * SPDX-License-Identifier: MIT + */ + +// Chrome does not allow TypedArrays with more than 4294967296 elements +// We'll create proxy objects for HEAP(U)8 when memory is > 4gb and for HEAP(U)16 when > 8gb +// https://bugs.chromium.org/p/v8/issues/detail?id=4153 +var maxArraySize = Math.min(b.byteLength, 4 * 1024 * 1024 * 1024 - 2); +/** + * @param {string} type - Heap type + * @param {number} [offset] - Heap offset + * @param {number} [length] - typed array length + */ +function getHeapBlock(type, offset, length) { + if (!offset) { + offset = 0 + } + + let heap = wasmMemory.buffer + + // we should always limit the length to maxArraySize + function createTypedArray(arrayType, offset, length) { + let bpe = arrayType.BYTES_PER_ELEMENT; + return new arrayType(heap, offset, length || Math.min((heap.byteLength - offset) / bpe, maxArraySize)); + } + + switch (type) { + case 'i1': + case 'i8': + return createTypedArray(Int8Array, offset, length); + case 'u1': + case 'u8': + return createTypedArray(Uint8Array, offset, length); + case 'i16': + return createTypedArray(Int16Array, offset, length); + case 'u16': + return createTypedArray(Uint16Array, offset, length); + default: + throw new Error('Invalid type'); + } +} + +function createProxyHandler(type, heapBlocks) { + let firstHeapBlock = heapBlocks[0] + let bpe = firstHeapBlock.BYTES_PER_ELEMENT + + function getRealStartAndEnd(start, end) { + let startReal = (start || 0) * bpe + let endReal = end ? end * bpe : wasmMemory.byteLength + return [startReal, endReal] + } + + function copyWithin(target, start, end) { + if (target * bpe >= maxArraySize || start * bpe >= maxArraySize || (end && end * bpe >= maxArraySize)) { + let len = end - start + let targetArray = getHeapBlock(type, target * bpe, len) + let sourceArray = getHeapBlock(type, start * bpe, len) + targetArray.set(sourceArray) + return heapBlocks[0] + } else { + return heapBlocks[0].copyWithin(target, start, end) + } + } + + function setOverridden(array, offset) { + let offsetReal = (offset || 0) * bpe + if (offsetReal >= maxArraySize || array.byteLength + offsetReal >= maxArraySize) { + let targetArray = getHeapBlock(type, offsetReal, array.length) + targetArray.set(array) + } else { + firstHeapBlock.set(array, offset) + } + } + + function subarray(start, end) { + let [startReal, endReal] = getRealStartAndEnd(start, end) + if (startReal >= maxArraySize || endReal >= maxArraySize) { + return getHeapBlock(type, startReal, endReal - startReal) + } else { + return firstHeapBlock.subarray(start, end) + } + } + + function fill(value, start, end) { + let [startReal, endReal] = getRealStartAndEnd(start, end) + if (startReal >= maxArraySize || endReal >= maxArraySize) { + let hb = getHeapBlock(type, startReal, endReal - startReal) + hb.fill(value, 0, end - start) + return firstHeapBlock + } else { + return firstHeapBlock.fill(value, start, end) + } + } + function slice(start, end) { + let [startReal, endReal] = getRealStartAndEnd(start, end) + if (startReal >= maxArraySize || endReal >= maxArraySize) { + let hb = getHeapBlock(type, startReal, endReal - startReal) + return hb.slice(start, end) + } else { + return firstHeapBlock.slice(start, end) + } + } + + return { + get(target, property) { + if (parseInt(property, 10) == property) { + let memoryOffset = property * bpe + let blockNumber = Math.floor(memoryOffset / maxArraySize) + return heapBlocks[blockNumber][property - blockNumber * maxArraySize] + } + + if (property === 'copyWithin') { + return copyWithin + } + + if (property === 'set') { + return setOverridden + } + + if (property === 'subarray') { + return subarray + } + + if (property === 'fill') { + return fill + } + + if (property === 'slice') { + return slice + } + + return firstHeapBlock[property] + }, + set(target, property, value) { + if (parseInt(property, 10) == property) { + let memoryOffset = property * bpe + let blockNumber = Math.floor(memoryOffset / maxArraySize) + heapBlocks[blockNumber][property - blockNumber * maxArraySize] = value + return true + } + + firstHeapBlock[property] = value + return true; + }, + } +} + +function createMemoryProxy(type) { + let heapBlocks = []; + let bpe = type === 'i16' || type === 'u16' ? 2 : 1 + let numberOfBlocks = Math.ceil(b.byteLength / maxArraySize / bpe) + for (let i = 0; i < numberOfBlocks; i++) { + heapBlocks.push(getHeapBlock(type, i * maxArraySize * bpe)) + } + return new Proxy(heapBlocks[0], createProxyHandler(type, heapBlocks)); +} + +if (b.byteLength > maxArraySize) { + Module['HEAP8'] = HEAP8 = createMemoryProxy('i8') + Module['HEAPU8'] = HEAPU8 = createMemoryProxy('u8') +} else { + Module['HEAP8'] = HEAP8 = new Int8Array(b); + Module['HEAPU8'] = HEAPU8 = new Uint8Array(b); +} +if (b.byteLength > maxArraySize * 2) { + Module['HEAP16'] = HEAP16 = createMemoryProxy('i16') + Module['HEAPU16'] = HEAPU16 = createMemoryProxy('u16') +} else { + Module['HEAP16'] = HEAP16 = new Int16Array(b); + Module['HEAPU16'] = HEAPU16 = new Uint16Array(b); +} diff --git a/test/other/test_memory64_proxies.js b/test/other/test_memory64_proxies.js new file mode 100644 index 0000000000000..748d0d2abd588 --- /dev/null +++ b/test/other/test_memory64_proxies.js @@ -0,0 +1,40 @@ +addOnPostRun(() => { + // check >4gb alloc + const bigChunk = _malloc(4 * 1024 * 1024 * 1024 + 100); + assert(bigChunk > 0); + + const littleChunk = _malloc(100); + HEAP8[littleChunk] = 2; + assert(HEAP8[littleChunk] === 2); + + // .subarray + const subarray = HEAP8.subarray(littleChunk, littleChunk + 100); + assert(subarray[0] === 2); + + // check .fill + HEAP8.fill(3, littleChunk, littleChunk + 99); + assert(subarray[0] === 3); + assert(subarray[98] === 3); + assert(subarray[99] === 0); + assert(HEAP8[littleChunk] === 3); + assert(HEAP8[littleChunk + 98] === 3); + assert(HEAP8[littleChunk + 99] === 0); + + // check .set + const filler = new Uint8Array(10); + filler[0] = 4; + filler[9] = 4; + HEAP8.set(filler, littleChunk, 10); + assert(subarray[0] === 4); + assert(subarray[9] === 4); + assert(HEAP8[littleChunk] === 4); + + // .copyWithin + HEAP8.copyWithin(bigChunk, littleChunk, littleChunk + 100); + assert(HEAP8[bigChunk] === 4); + + // .slice + const slice = HEAP8.slice(bigChunk, bigChunk + 100); + slice[0] = 5; + assert(HEAP8[bigChunk] === 4); +}); diff --git a/test/test_browser.py b/test/test_browser.py index ac284ae98420f..c73689fcd3d6c 100644 --- a/test/test_browser.py +++ b/test/test_browser.py @@ -5475,7 +5475,7 @@ def test_zzz_zzz_2gb_fail(self): self.do_run_in_out_file_test('browser', 'test_2GB_fail.cpp') @no_firefox('no 4GB support yet') - # @also_with_wasm64 Blocked on https://bugs.chromium.org/p/v8/issues/detail?id=4153 + @also_with_wasm64 @requires_v8 def test_zzz_zzz_4gb_fail(self): # TODO Convert to an actual browser test when it reaches stable. diff --git a/test/test_other.py b/test/test_other.py index ee358d05eb382..bb3603b2e60b4 100644 --- a/test/test_other.py +++ b/test/test_other.py @@ -32,7 +32,7 @@ from common import RunnerCore, path_from_root, is_slow_test, ensure_dir, disabled, make_executable from common import env_modify, no_mac, no_windows, only_windows, requires_native_clang, with_env_modify from common import create_file, parameterized, NON_ZERO, node_pthreads, TEST_ROOT, test_file -from common import compiler_for, EMBUILDER, requires_v8, requires_node, requires_wasm64 +from common import compiler_for, EMBUILDER, requires_v8, requires_node, requires_wasm64, requires_node_canary from common import requires_wasm_eh, crossplatform, with_both_sjlj, also_with_standalone_wasm from common import also_with_minimal_runtime, also_with_wasm_bigint, also_with_wasm64 from common import EMTEST_BUILD_VERBOSE, PYTHON @@ -6304,10 +6304,9 @@ def test_failing_growth_2gb(self): self.run_process([EMCC, '-O1', 'test.c', '-sALLOW_MEMORY_GROWTH']) self.assertContained('done', self.run_js('a.out.js')) + @requires_wasm64 + @requires_node_canary def test_failing_growth_wasm64(self): - # For now we skip this test because failure to create the TypedArray views - # causes weird unrecoverable failures. - self.skipTest('https://bugs.chromium.org/p/v8/issues/detail?id=4153') self.require_wasm64() create_file('test.c', r''' #include @@ -13598,3 +13597,16 @@ def test_explicit_target(self): def test_quick_exit(self): self.do_other_test('test_quick_exit.c') + + @requires_wasm64 + @requires_node_canary + def test_memory64_proxies(self): + self.run_process([EMCC, test_file('hello_world.c'), + '-sMEMORY64=1', + '-sINITIAL_MEMORY=5gb', + '-sMAXIMUM_MEMORY=5gb', + '-sALLOW_MEMORY_GROWTH', + '-sEXPORTED_FUNCTIONS=_malloc,_main', + '-Wno-experimental', + '--extern-post-js', test_file('other/test_memory64_proxies.js')]) + self.run_js('a.out.js')