Skip to content

Commit

Permalink
[test] Simplify test_openjpeg. NFC (#21401)
Browse files Browse the repository at this point in the history 
This change simplifies the test in 2 ways:

1. Remove the "extra testing" part of the test that tries to run the
   test again if `ALLOW_MEMORY_GROWTH` is set.  The only test modes that
   have `ALLOW_MEMORY_GROWTH` are the santizier modes and it seems odd
   for an individual test to decide to disable this.
2. Remove the alternate `is_sanitizing` method of running the test.
   I believe this was only needed becuase of (1), and so can be safely
   removed.

By removing this complexity this test now has less nesting and is more
directly readable and more similar to other tests.
  • Loading branch information
@sbc100
sbc100 authored Feb 23, 2024
1 parent 0059aae commit 6b93444
Showing 1 changed file with 90 additions and 111 deletions.
201 changes: 90 additions & 111 deletions test/test_core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import random
import re
import shutil
import sys
import time
import unittest
from pathlib import Path
Expand Down Expand Up @@ -6807,117 +6806,97 @@ def test_poppler(self):
@no_wasm64('MEMORY64 does not yet support SJLJ')
@is_slow_test
def test_openjpeg(self):
def do_test_openjpeg():
def line_splitter(data):
out = ''
counter = 0

for ch in data:
out += ch
if ch == ' ' and counter > 60:
out += '\n'
counter = 0
else:
counter += 1

return out

# remove -g, so we have one test without it by default
self.emcc_args = [x for x in self.emcc_args if x != '-g']

original_j2k = test_file('openjpeg/syntensity_lobby_s.j2k')
image_bytes = list(bytearray(read_binary(original_j2k)))
create_file('pre.js', """
Module.preRun = () => FS.createDataFile('/', 'image.j2k', %s, true, false, false);
Module.postRun = () => {
out('Data: ' + JSON.stringify(Array.from(FS.readFile('image.raw'))));
};
""" % line_splitter(str(image_bytes)))

# ensure libpng is built so that openjpeg's configure step can detect it.
# If we don't do this then we don't know what the state of the cache will be
# and this test would different non-deterministic results based on, for example,
# what other tests had previously run.
builder_cmd = [EMBUILDER, 'build', 'libpng']
if self.get_setting('MEMORY64'):
builder_cmd.append('--wasm64')
self.emcc_args.append('-Wno-pointer-to-int-cast')
self.run_process(builder_cmd)
lib = self.get_library('third_party/openjpeg',
[Path('codec/CMakeFiles/j2k_to_image.dir/index.c.o'),
Path('codec/CMakeFiles/j2k_to_image.dir/convert.c.o'),
Path('codec/CMakeFiles/j2k_to_image.dir/__/common/color.c.o'),
Path('bin/libopenjpeg.a')],
configure=['cmake', '.'],
# configure_args=['--enable-tiff=no', '--enable-jp3d=no', '--enable-png=no'],
make_args=[]) # no -j 2, since parallel builds can fail

# We use doubles in JS, so we get slightly different values than native code. So we
# check our output by comparing the average pixel difference
def image_compare(output):
# Get the image generated by JS, from the JSON.stringify'd array
m = re.search(r'\[[\d, -]*\]', output)
self.assertIsNotNone(m, 'Failed to find proper image output in: ' + output)
# Evaluate the output as a python array
js_data = eval(m.group(0))

js_data = [x if x >= 0 else 256 + x for x in js_data] # Our output may be signed, so unsign it

# Get the correct output
true_data = bytearray(read_binary(test_file('openjpeg/syntensity_lobby_s.raw')))

# Compare them
self.assertEqual(len(js_data), len(true_data))
num = len(js_data)
diff_total = js_total = true_total = 0
for i in range(num):
js_total += js_data[i]
true_total += true_data[i]
diff_total += abs(js_data[i] - true_data[i])
js_mean = js_total / float(num)
true_mean = true_total / float(num)
diff_mean = diff_total / float(num)

image_mean = 83.265
# print '[image stats:', js_mean, image_mean, true_mean, diff_mean, num, ']'
assert abs(js_mean - image_mean) < 0.01, [js_mean, image_mean]
assert abs(true_mean - image_mean) < 0.01, [true_mean, image_mean]
assert diff_mean < 0.01, diff_mean

return output

self.emcc_args += ['--minify=0'] # to compare the versions
self.emcc_args += ['--pre-js', 'pre.js']

def do_test():
self.do_runf('third_party/openjpeg/codec/j2k_to_image.c',
'Successfully generated', # The real test for valid output is in image_compare
args='-i image.j2k -o image.raw'.split(),
emcc_args=['-sUSE_LIBPNG'],
libraries=lib,
includes=[test_file('third_party/openjpeg/libopenjpeg'),
test_file('third_party/openjpeg/codec'),
test_file('third_party/openjpeg/common'),
Path(self.get_build_dir(), 'third_party/openjpeg')],
output_nicerizer=image_compare)

do_test()

# extra testing
if self.get_setting('ALLOW_MEMORY_GROWTH') == 1:
print('no memory growth', file=sys.stderr)
self.set_setting('ALLOW_MEMORY_GROWTH', 0)
do_test()
def line_splitter(data):
out = ''
counter = 0

for ch in data:
out += ch
if ch == ' ' and counter > 60:
out += '\n'
counter = 0
else:
counter += 1

if is_sanitizing(self.emcc_args):
# In ASan mode we need a large initial memory (or else wasm-ld fails).
# The OpenJPEG CMake will build several executables (which we need parts
# of in our testing, see above), so we must enable the flag for them all.
with env_modify({'EMCC_CFLAGS': '-sINITIAL_MEMORY=300MB'}):
self.emcc_args.append('-Wno-unused-command-line-argument')
do_test_openjpeg()
else:
do_test_openjpeg()
return out

# remove -g, so we have one test without it by default
self.emcc_args = [x for x in self.emcc_args if x != '-g']

original_j2k = test_file('openjpeg/syntensity_lobby_s.j2k')
image_bytes = list(bytearray(read_binary(original_j2k)))
create_file('pre.js', """
Module.preRun = () => FS.createDataFile('/', 'image.j2k', %s, true, false, false);
Module.postRun = () => {
out('Data: ' + JSON.stringify(Array.from(FS.readFile('image.raw'))));
};
""" % line_splitter(str(image_bytes)))

# ensure libpng is built so that openjpeg's configure step can detect it.
# If we don't do this then we don't know what the state of the cache will be
# and this test would different non-deterministic results based on, for example,
# what other tests had previously run.
builder_cmd = [EMBUILDER, 'build', 'libpng']
if self.get_setting('MEMORY64'):
builder_cmd.append('--wasm64')
self.emcc_args.append('-Wno-pointer-to-int-cast')
self.run_process(builder_cmd)
lib = self.get_library('third_party/openjpeg',
[Path('codec/CMakeFiles/j2k_to_image.dir/index.c.o'),
Path('codec/CMakeFiles/j2k_to_image.dir/convert.c.o'),
Path('codec/CMakeFiles/j2k_to_image.dir/__/common/color.c.o'),
Path('bin/libopenjpeg.a')],
configure=['cmake', '.'],
# configure_args=['--enable-tiff=no', '--enable-jp3d=no', '--enable-png=no'],
make_args=[]) # no -j 2, since parallel builds can fail

# We use doubles in JS, so we get slightly different values than native code. So we
# check our output by comparing the average pixel difference
def image_compare(output):
# Get the image generated by JS, from the JSON.stringify'd array
m = re.search(r'\[[\d, -]*\]', output)

self.assertIsNotNone(m, 'Failed to find proper image output in: ' + output)
# Evaluate the output as a python array
js_data = eval(m.group(0))

js_data = [x if x >= 0 else 256 + x for x in js_data] # Our output may be signed, so unsign it
# Get the correct output
true_data = bytearray(read_binary(test_file('openjpeg/syntensity_lobby_s.raw')))

# Compare them
self.assertEqual(len(js_data), len(true_data))
num = len(js_data)
diff_total = js_total = true_total = 0
for i in range(num):
js_total += js_data[i]
true_total += true_data[i]
diff_total += abs(js_data[i] - true_data[i])
js_mean = js_total / float(num)
true_mean = true_total / float(num)
diff_mean = diff_total / float(num)

image_mean = 83.265
# print '[image stats:', js_mean, image_mean, true_mean, diff_mean, num, ']'
assert abs(js_mean - image_mean) < 0.01, [js_mean, image_mean]
assert abs(true_mean - image_mean) < 0.01, [true_mean, image_mean]
assert diff_mean < 0.01, diff_mean

return output

self.emcc_args += ['--minify=0'] # to compare the versions
self.emcc_args += ['--pre-js', 'pre.js']

self.do_runf('third_party/openjpeg/codec/j2k_to_image.c',
'Successfully generated', # The real test for valid output is in image_compare
args='-i image.j2k -o image.raw'.split(),
emcc_args=['-sUSE_LIBPNG'],
libraries=lib,
includes=[test_file('third_party/openjpeg/libopenjpeg'),
test_file('third_party/openjpeg/codec'),
test_file('third_party/openjpeg/common'),
Path(self.get_build_dir(), 'third_party/openjpeg')],
output_nicerizer=image_compare)

@also_with_standalone_wasm(impure=True)
@no_asan('autodebug logging interferes with asan')
Expand Down

0 comments on commit 6b93444

Please sign in to comment.