Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[CI] Re-enable versions-test #3371

Merged
merged 1 commit into from
Dec 16, 2022
Merged

[CI] Re-enable versions-test #3371

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 7 additions & 12 deletions .github/workflows/dev-short-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -431,18 +431,13 @@ jobs:
cc -Wall -Wextra -Wpedantic -Werror -o simple examples/simple_compression.c $(pkg-config --cflags --libs libzstd)
./simple LICENSE


# This test currently fails on Github Actions specifically.
# Possible reason : TTY emulation.
# Note that the same test works fine locally and on travisCI.
# This will have to be fixed before transferring the test to GA.
# versions-compatibility:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# - name: Versions Compatibility Test
# run: |
# make -C tests versionsTest
versions-compatibility:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Versions Compatibility Test
run: |
make -C tests versionsTest


# For reference : icc tests
Expand Down
102 changes: 41 additions & 61 deletions tests/test-zstd-versions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
repo_url = 'https://github.com/facebook/zstd.git'
tmp_dir_name = 'tests/versionsTest'
make_cmd = 'make'
make_args = ['-j','CFLAGS=-O1']
make_args = ['-j','CFLAGS=-O0']
git_cmd = 'git'
test_dat_src = 'README.md'
test_dat = 'test_dat'
Expand Down Expand Up @@ -86,41 +86,46 @@ def create_dict(tag, dict_source_path):
if result == 0:
print(dict_name + ' created')
else:
print('ERROR: creating of ' + dict_name + ' failed')
raise RuntimeError('ERROR: creating of ' + dict_name + ' failed')
else:
print(dict_name + ' already exists')


def zstd(tag, args, input_file, output_file):
"""
Zstd compress input_file to output_file.
Need this helper because 0.5.0 is broken when stdout is not a TTY.
Throws an exception if the command returns non-zero.
"""
with open(input_file, "rb") as i:
with open(output_file, "wb") as o:
cmd = ['./zstd.' + tag] + args
print("Running: '{}', input={}, output={}" .format(
' '.join(cmd), input_file, output_file
))
subprocess.check_call(cmd, stdin=i, stdout=o)


def dict_compress_sample(tag, sample):
dict_name = 'dict.' + tag
DEVNULL = open(os.devnull, 'wb')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-1'], sample, sample + '_01_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-3'], sample, sample + '_03_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-5'], sample, sample + '_05_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-9'], sample, sample + '_09_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-15'], sample, sample + '_15_64_' + tag + '_dictio.zst')
zstd(tag, ['-D', dict_name, '-18'], sample, sample + '_18_64_' + tag + '_dictio.zst')
# zstdFiles = glob.glob("*.zst*")
# print(zstdFiles)
print(tag + " : dict compression completed")


def compress_sample(tag, sample):
DEVNULL = open(os.devnull, 'wb')
if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst')
if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
zstd(tag, ['-1'], sample, sample + '_01_64_' + tag + '_nodict.zst')
zstd(tag, ['-3'], sample, sample + '_03_64_' + tag + '_nodict.zst')
zstd(tag, ['-5'], sample, sample + '_05_64_' + tag + '_nodict.zst')
zstd(tag, ['-9'], sample, sample + '_09_64_' + tag + '_nodict.zst')
zstd(tag, ['-15'], sample, sample + '_15_64_' + tag + '_nodict.zst')
zstd(tag, ['-18'], sample, sample + '_18_64_' + tag + '_nodict.zst')
# zstdFiles = glob.glob("*.zst*")
# print(zstdFiles)
print(tag + " : compression completed")
Expand Down Expand Up @@ -150,23 +155,13 @@ def decompress_zst(tag):
dec_error = 0
list_zst = sorted(glob.glob('*_nodict.zst'))
for file_zst in list_zst:
print(file_zst, end=' ')
print(tag, end=' ')
print(file_zst + ' ' + tag)
file_dec = file_zst + '_d64_' + tag + '.dec'
if tag <= 'v0.5.0':
params = ['./zstd.' + tag, '-df', file_zst, file_dec]
zstd(tag, ['-d'], file_zst, file_dec)
if not filecmp.cmp(file_dec, test_dat):
raise RuntimeError('Decompression failed: tag={} file={}'.format(tag, file_zst))
else:
params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
if execute(params) == 0:
if not filecmp.cmp(file_dec, test_dat):
print('ERR !! ')
dec_error = 1
else:
print('OK ')
else:
print('command does not work')
dec_error = 1
return dec_error
print('OK ')


def decompress_dict(tag):
Expand All @@ -181,22 +176,13 @@ def decompress_dict(tag):
if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
continue
dict_name = 'dict.' + dict_tag
print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
print(file_zst + ' ' + tag + ' dict=' + dict_tag)
file_dec = file_zst + '_d64_' + tag + '.dec'
if tag <= 'v0.5.0':
params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
zstd(tag, ['-D', dict_name, '-d'], file_zst, file_dec)
if not filecmp.cmp(file_dec, test_dat):
raise RuntimeError('Decompression failed: tag={} file={}'.format(tag, file_zst))
else:
params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
if execute(params) == 0:
if not filecmp.cmp(file_dec, test_dat):
print('ERR !! ')
dec_error = 1
else:
print('OK ')
else:
print('command does not work')
dec_error = 1
return dec_error
print('OK ')


if __name__ == '__main__':
Expand Down Expand Up @@ -267,25 +253,19 @@ def decompress_dict(tag):
print('Compress test.dat by all released zstd')
print('-----------------------------------------------')

error_code = 0
for tag in tags:
print(tag)
if tag >= 'v0.5.0':
create_dict(tag, dict_source_path)
dict_compress_sample(tag, test_dat)
remove_duplicates()
error_code += decompress_dict(tag)
decompress_dict(tag)
compress_sample(tag, test_dat)
remove_duplicates()
error_code += decompress_zst(tag)
decompress_zst(tag)

print('')
print('Enumerate different compressed files')
zstds = sorted(glob.glob('*.zst'))
for zstd in zstds:
print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd))

if error_code != 0:
print('====== ERROR !!! =======')

sys.exit(error_code)