[CI] Hook cli-tests up to CI

Add cli-tests to `make test`. This adds a `python3` dependency to `make
test`, but not `make check`. We could make this dependency optional by
skipping the tests if `python3` is not present.

tests/Makefile

@@ -297,7 +297,7 @@ check: shortest
 fuzztest: test-fuzzer test-zstream test-decodecorpus
 
 .PHONY: test
-test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
+test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus test-cli-tests
 ifeq ($(QEMU_SYS),)
 test: test-pool
 endif
@@ -322,6 +322,12 @@ test-zstd test-zstd32 test-zstd-nolegacy: datagen
 	file $(ZSTD)
 	EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)
 
+test-cli-tests: ZSTD = $(PRGDIR)/zstd
+test-cli-tests: zstd datagen
+	file $(ZSTD)
+	./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen
+
+
 test-fullbench: fullbench datagen
 	$(QEMU_SYS) ./fullbench -i1
 	$(QEMU_SYS) ./fullbench -i1 -P0

tests/cli-tests/basic/help.sh

@@ -1,4 +1,7 @@
-#!/bin/sh -e
+#!/bin/sh
+
+set -e
+
 println "+ zstd -h"
 zstd -h
 println "+ zstd -H"

tests/cli-tests/basic/version.sh

@@ -1,3 +1,6 @@
-#!/bin/sh -e
+#!/bin/sh
+
+set -e
+
 zstd -V
 zstd --version

tests/cli-tests/bin/cmp_size

@@ -1,7 +1,5 @@
 #!/bin/sh
 
-# Small utility to
-
 set -e
 
 usage()

tests/cli-tests/bin/println

@@ -1,2 +1,2 @@
-#!/bin/env sh
+#!/bin/sh
 printf '%b\n' "${*}"

tests/cli-tests/common/format.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 zstd_supports_format()
 {

tests/cli-tests/common/mtime.sh

@@ -1,4 +1,4 @@
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 MTIME="stat -c %Y"
 case "$UNAME" in

tests/cli-tests/common/permissions.sh

@@ -1,4 +1,4 @@
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 GET_PERMS="stat -c %a"
 case "$UNAME" in

tests/cli-tests/compression/basic.sh

@@ -1,7 +1,9 @@
-#!/bin/sh -e
+#!/bin/sh
 
-# Uncomment the set -x line for debugging
-# set -x
+set -e
+
+# Uncomment the set -v line for debugging
+# set -v
 
 # Test compression flags and check that they work
 zstd file                 ; zstd -t file.zst

tests/cli-tests/compression/format.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source "$COMMON/format.sh"
+. "$COMMON/format.sh"
 
 set -e
 

tests/cli-tests/compression/levels.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 set -e
-set -x
+set -v
 
 datagen > file
 
@@ -12,7 +12,7 @@ zstd -1 file -o file-1.zst
 zstd -19 file -o file-19.zst
 zstd -22 --ultra file -o file-22.zst
 
-zstd -t file-{f10,f1,1,19,22}.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
 
 cmp_size -ne file-19.zst file-22.zst
 cmp_size -lt file-19.zst file-1.zst
@@ -41,10 +41,10 @@ ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
 ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
 ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
 
-cmp file-f10{,-env}.zst || die "Environment variable failed to set level"
-cmp file-1{,-env}.zst   || die "Environment variable failed to set level"
-cmp file-19{,-env}.zst  || die "Environment variable failed to set level"
-cmp file-99{,-env}.zst  || die "Environment variable failed to set level"
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
 
 # Test invalid environment clevel is the default level
 zstd -f file
@@ -60,5 +60,5 @@ ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
 ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
 ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
 
-cmp file-1{,-env}.zst  || die "Environment variable not overridden"
-cmp file-f1{,-env}.zst || die "Environment variable not overridden"
+cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"

tests/cli-tests/compression/levels.sh.stderr.exact

@@ -1,75 +1,71 @@
-+ datagen
-+ zstd --fast=10 file -o file-f10.zst
-+ zstd --fast=1 file -o file-f1.zst
-+ zstd -1 file -o file-1.zst
-+ zstd -19 file -o file-19.zst
-+ zstd -22 --ultra file -o file-22.zst
-+ zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
-+ cmp_size -ne file-19.zst file-22.zst
-+ cmp_size -lt file-19.zst file-1.zst
-+ cmp_size -lt file-1.zst file-f1.zst
-+ cmp_size -lt file-f1.zst file-f10.zst
-+ zstd --fast file -f
-+ cmp file.zst file-f1.zst
-+ zstd -0 file -o file-0.zst
-+ zstd -f file
-+ cmp file.zst file-0.zst
-+ zstd -99 file -o file-99.zst
+
+datagen > file
+
+# Compress with various levels and ensure that their sizes are ordered
+zstd --fast=10 file -o file-f10.zst
+zstd --fast=1 file -o file-f1.zst
+zstd -1 file -o file-1.zst
+zstd -19 file -o file-19.zst
+zstd -22 --ultra file -o file-22.zst
+
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
+
+cmp_size -ne file-19.zst file-22.zst
+cmp_size -lt file-19.zst file-1.zst
+cmp_size -lt file-1.zst file-f1.zst
+cmp_size -lt file-f1.zst file-f10.zst
+
+# Test default levels
+zstd --fast file -f
+cmp file.zst file-f1.zst || die "--fast is not level -1"
+
+zstd -0 file -o file-0.zst
+zstd -f file
+cmp file.zst file-0.zst || die "Level 0 is not the default level"
+
+# Test level clamping
+zstd -99 file -o file-99.zst
 Warning : compression level higher than max, reduced to 19 
-+ cmp file-19.zst file-99.zst
-+ zstd --fast=200000 file -c
-+ zstd -t
-+ zstd -5000000000 -f file
+cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
+zstd --fast=200000 file -c | zstd -t
+
+zstd -5000000000 -f file       && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
-+ :
-+ zstd --fast=5000000000 -f file
+zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
-+ :
-+ ZSTD_CLEVEL=-10
-+ zstd file -o file-f10-env.zst
-+ ZSTD_CLEVEL=1
-+ zstd file -o file-1-env.zst
-+ ZSTD_CLEVEL=+19
-+ zstd file -o file-19-env.zst
-+ ZSTD_CLEVEL=+99
-+ zstd file -o file-99-env.zst
+
+# Test setting a level through the environment variable
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
 Warning : compression level higher than max, reduced to 19 
-+ cmp file-f10.zst file-f10-env.zst
-+ cmp file-1.zst file-1-env.zst
-+ cmp file-19.zst file-19-env.zst
-+ cmp file-99.zst file-99-env.zst
-+ zstd -f file
-+ ZSTD_CLEVEL=-
-+ zstd -f file -o file-env.zst
+
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
+
+# Test invalid environment clevel is the default level
+zstd -f file
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=+
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=-a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=+a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=3a7
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst       ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=5000000000
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=10
-+ zstd -f file -1 -o file-1-env.zst
-+ ZSTD_CLEVEL=10
-+ zstd -f file --fast=1 -o file-f1-env.zst
-+ cmp file-1.zst file-1-env.zst
-+ cmp file-f1.zst file-f1-env.zst
+
+# Test environment clevel is overridden by command line
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
+
+cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"

tests/cli-tests/dict-builder/no-inputs → tests/cli-tests/dict-builder/no-inputs.sh

@@ -1,3 +1,3 @@
 #!/bin/sh
-set -x
+set -v
 zstd --train

tests/cli-tests/dict-builder/no-inputs.stderr.exact → tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact

@@ -1,4 +1,4 @@
-+ zstd --train
+zstd --train
 !  Warning : nb of samples too low for proper processing ! 
 !  Please provide _one file per sample_. 
 !  Alternatively, split files into fixed-size blocks representative of samples, with -B# 

tests/cli-tests/dictionaries/dictionary-mismatch.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 set -e
 
@@ -22,7 +22,7 @@ if [ false ]; then
 	datagen -g1000 -s0 > file0
 fi
 
-set -x
+set -v
 zstd files/0 -D dicts/0
 zstd -t files/0.zst -D dicts/0
 zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:

tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact

@@ -1,8 +1,6 @@
-+ zstd files/0 -D dicts/0
-+ zstd -t files/0.zst -D dicts/0
-+ zstd -t files/0.zst -D dicts/1
+zstd files/0 -D dicts/0
+zstd -t files/0.zst -D dicts/0
+zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
 files/0.zst : Decoding error (36) : Dictionary mismatch 
-+ :
-+ zstd -t files/0.zst
+zstd -t files/0.zst            && die "Must fail" ||:
 files/0.zst : Decoding error (36) : Dictionary mismatch 
-+ :

tests/cli-tests/dictionaries/setup_once

@@ -2,7 +2,7 @@
 
 set -e
 
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 
 mkdir files/ dicts/