From 943ef41fb5a120f1e9ed77c65ead0df767dc1598 Mon Sep 17 00:00:00 2001
From: Nick Terrell <terrelln@fb.com>
Date: Mon, 24 Jan 2022 13:52:08 -0800
Subject: [PATCH] [CI] Hook cli-tests up to CI

Add cli-tests to `make test`. This adds a `python3` dependency to `make
test`, but not `make check`. We could make this dependency optional by
skipping the tests if `python3` is not present.
---
 tests/Makefile                                |   8 +-
 tests/cli-tests/basic/help.sh                 |   5 +-
 tests/cli-tests/basic/version.sh              |   5 +-
 tests/cli-tests/bin/cmp_size                  |   2 -
 tests/cli-tests/bin/println                   |   2 +-
 tests/cli-tests/common/format.sh              |   2 +-
 tests/cli-tests/common/mtime.sh               |   2 +-
 tests/cli-tests/common/permissions.sh         |   2 +-
 tests/cli-tests/compression/basic.sh          |   8 +-
 tests/cli-tests/compression/format.sh         |   2 +-
 tests/cli-tests/compression/levels.sh         |  16 +--
 .../compression/levels.sh.stderr.exact        | 124 +++++++++---------
 .../dict-builder/{no-inputs => no-inputs.sh}  |   2 +-
 .../{no-inputs.exit => no-inputs.sh.exit}     |   0
 ...stderr.exact => no-inputs.sh.stderr.exact} |   2 +-
 .../dictionaries/dictionary-mismatch.sh       |   4 +-
 .../dictionary-mismatch.sh.stderr.exact       |  10 +-
 tests/cli-tests/dictionaries/setup_once       |   2 +-
 18 files changed, 102 insertions(+), 96 deletions(-)
 rename tests/cli-tests/dict-builder/{no-inputs => no-inputs.sh} (76%)
 rename tests/cli-tests/dict-builder/{no-inputs.exit => no-inputs.sh.exit} (100%)
 rename tests/cli-tests/dict-builder/{no-inputs.stderr.exact => no-inputs.sh.stderr.exact} (93%)

diff --git a/tests/Makefile b/tests/Makefile
index 132fa7a0818..cb77b0160a8 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -297,7 +297,7 @@ check: shortest
 fuzztest: test-fuzzer test-zstream test-decodecorpus
 
 .PHONY: test
-test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus
+test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus test-cli-tests
 ifeq ($(QEMU_SYS),)
 test: test-pool
 endif
@@ -322,6 +322,12 @@ test-zstd test-zstd32 test-zstd-nolegacy: datagen
 	file $(ZSTD)
 	EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST)
 
+test-cli-tests: ZSTD = $(PRGDIR)/zstd
+test-cli-tests: zstd datagen
+	file $(ZSTD)
+	./cli-tests/run.py --exec-prefix="$(QEMU_SYS)" --zstd="$(ZSTD)" --datagen=./datagen
+
+
 test-fullbench: fullbench datagen
 	$(QEMU_SYS) ./fullbench -i1
 	$(QEMU_SYS) ./fullbench -i1 -P0
diff --git a/tests/cli-tests/basic/help.sh b/tests/cli-tests/basic/help.sh
index c683b6a6a72..927c3ffbcc5 100755
--- a/tests/cli-tests/basic/help.sh
+++ b/tests/cli-tests/basic/help.sh
@@ -1,4 +1,7 @@
-#!/bin/sh -e
+#!/bin/sh
+
+set -e
+
 println "+ zstd -h"
 zstd -h
 println "+ zstd -H"
diff --git a/tests/cli-tests/basic/version.sh b/tests/cli-tests/basic/version.sh
index d50de0f3cac..f75eaa84fb4 100755
--- a/tests/cli-tests/basic/version.sh
+++ b/tests/cli-tests/basic/version.sh
@@ -1,3 +1,6 @@
-#!/bin/sh -e
+#!/bin/sh
+
+set -e
+
 zstd -V
 zstd --version
diff --git a/tests/cli-tests/bin/cmp_size b/tests/cli-tests/bin/cmp_size
index 5afa1c590ad..8e4bef88eab 100755
--- a/tests/cli-tests/bin/cmp_size
+++ b/tests/cli-tests/bin/cmp_size
@@ -1,7 +1,5 @@
 #!/bin/sh
 
-# Small utility to
-
 set -e
 
 usage()
diff --git a/tests/cli-tests/bin/println b/tests/cli-tests/bin/println
index 1da2460471b..494eb18c850 100755
--- a/tests/cli-tests/bin/println
+++ b/tests/cli-tests/bin/println
@@ -1,2 +1,2 @@
-#!/bin/env sh
+#!/bin/sh
 printf '%b\n' "${*}"
diff --git a/tests/cli-tests/common/format.sh b/tests/cli-tests/common/format.sh
index 20ff0f05672..e574e973075 100644
--- a/tests/cli-tests/common/format.sh
+++ b/tests/cli-tests/common/format.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 zstd_supports_format()
 {
diff --git a/tests/cli-tests/common/mtime.sh b/tests/cli-tests/common/mtime.sh
index 7ce931a96c0..344074d398c 100644
--- a/tests/cli-tests/common/mtime.sh
+++ b/tests/cli-tests/common/mtime.sh
@@ -1,4 +1,4 @@
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 MTIME="stat -c %Y"
 case "$UNAME" in
diff --git a/tests/cli-tests/common/permissions.sh b/tests/cli-tests/common/permissions.sh
index b1f6ea3ba70..6bce1f0b387 100644
--- a/tests/cli-tests/common/permissions.sh
+++ b/tests/cli-tests/common/permissions.sh
@@ -1,4 +1,4 @@
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 GET_PERMS="stat -c %a"
 case "$UNAME" in
diff --git a/tests/cli-tests/compression/basic.sh b/tests/cli-tests/compression/basic.sh
index 6f0f87932fb..8b63e40760c 100755
--- a/tests/cli-tests/compression/basic.sh
+++ b/tests/cli-tests/compression/basic.sh
@@ -1,7 +1,9 @@
-#!/bin/sh -e
+#!/bin/sh
 
-# Uncomment the set -x line for debugging
-# set -x
+set -e
+
+# Uncomment the set -v line for debugging
+# set -v
 
 # Test compression flags and check that they work
 zstd file                 ; zstd -t file.zst
diff --git a/tests/cli-tests/compression/format.sh b/tests/cli-tests/compression/format.sh
index 86fb400809c..192fa2cf29f 100755
--- a/tests/cli-tests/compression/format.sh
+++ b/tests/cli-tests/compression/format.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source "$COMMON/format.sh"
+. "$COMMON/format.sh"
 
 set -e
 
diff --git a/tests/cli-tests/compression/levels.sh b/tests/cli-tests/compression/levels.sh
index 6bd0aca0078..4837790ce5d 100755
--- a/tests/cli-tests/compression/levels.sh
+++ b/tests/cli-tests/compression/levels.sh
@@ -1,7 +1,7 @@
 #!/bin/sh
 
 set -e
-set -x
+set -v
 
 datagen > file
 
@@ -12,7 +12,7 @@ zstd -1 file -o file-1.zst
 zstd -19 file -o file-19.zst
 zstd -22 --ultra file -o file-22.zst
 
-zstd -t file-{f10,f1,1,19,22}.zst
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
 
 cmp_size -ne file-19.zst file-22.zst
 cmp_size -lt file-19.zst file-1.zst
@@ -41,10 +41,10 @@ ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
 ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
 ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
 
-cmp file-f10{,-env}.zst || die "Environment variable failed to set level"
-cmp file-1{,-env}.zst   || die "Environment variable failed to set level"
-cmp file-19{,-env}.zst  || die "Environment variable failed to set level"
-cmp file-99{,-env}.zst  || die "Environment variable failed to set level"
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
 
 # Test invalid environment clevel is the default level
 zstd -f file
@@ -60,5 +60,5 @@ ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
 ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
 ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
 
-cmp file-1{,-env}.zst  || die "Environment variable not overridden"
-cmp file-f1{,-env}.zst || die "Environment variable not overridden"
+cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
diff --git a/tests/cli-tests/compression/levels.sh.stderr.exact b/tests/cli-tests/compression/levels.sh.stderr.exact
index c0b7066fcba..cb00433e6e1 100644
--- a/tests/cli-tests/compression/levels.sh.stderr.exact
+++ b/tests/cli-tests/compression/levels.sh.stderr.exact
@@ -1,75 +1,71 @@
-+ datagen
-+ zstd --fast=10 file -o file-f10.zst
-+ zstd --fast=1 file -o file-f1.zst
-+ zstd -1 file -o file-1.zst
-+ zstd -19 file -o file-19.zst
-+ zstd -22 --ultra file -o file-22.zst
-+ zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
-+ cmp_size -ne file-19.zst file-22.zst
-+ cmp_size -lt file-19.zst file-1.zst
-+ cmp_size -lt file-1.zst file-f1.zst
-+ cmp_size -lt file-f1.zst file-f10.zst
-+ zstd --fast file -f
-+ cmp file.zst file-f1.zst
-+ zstd -0 file -o file-0.zst
-+ zstd -f file
-+ cmp file.zst file-0.zst
-+ zstd -99 file -o file-99.zst
+
+datagen > file
+
+# Compress with various levels and ensure that their sizes are ordered
+zstd --fast=10 file -o file-f10.zst
+zstd --fast=1 file -o file-f1.zst
+zstd -1 file -o file-1.zst
+zstd -19 file -o file-19.zst
+zstd -22 --ultra file -o file-22.zst
+
+zstd -t file-f10.zst file-f1.zst file-1.zst file-19.zst file-22.zst
+
+cmp_size -ne file-19.zst file-22.zst
+cmp_size -lt file-19.zst file-1.zst
+cmp_size -lt file-1.zst file-f1.zst
+cmp_size -lt file-f1.zst file-f10.zst
+
+# Test default levels
+zstd --fast file -f
+cmp file.zst file-f1.zst || die "--fast is not level -1"
+
+zstd -0 file -o file-0.zst
+zstd -f file
+cmp file.zst file-0.zst || die "Level 0 is not the default level"
+
+# Test level clamping
+zstd -99 file -o file-99.zst
 Warning : compression level higher than max, reduced to 19 
-+ cmp file-19.zst file-99.zst
-+ zstd --fast=200000 file -c
-+ zstd -t
-+ zstd -5000000000 -f file
+cmp file-19.zst file-99.zst || die "Level 99 is clamped to 19"
+zstd --fast=200000 file -c | zstd -t
+
+zstd -5000000000 -f file       && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
-+ :
-+ zstd --fast=5000000000 -f file
+zstd --fast=5000000000 -f file && die "Level too large, must fail" ||:
 error: numeric value overflows 32-bit unsigned int 
-+ :
-+ ZSTD_CLEVEL=-10
-+ zstd file -o file-f10-env.zst
-+ ZSTD_CLEVEL=1
-+ zstd file -o file-1-env.zst
-+ ZSTD_CLEVEL=+19
-+ zstd file -o file-19-env.zst
-+ ZSTD_CLEVEL=+99
-+ zstd file -o file-99-env.zst
+
+# Test setting a level through the environment variable
+ZSTD_CLEVEL=-10 zstd file -o file-f10-env.zst
+ZSTD_CLEVEL=1 zstd file -o file-1-env.zst
+ZSTD_CLEVEL=+19 zstd file -o file-19-env.zst
+ZSTD_CLEVEL=+99 zstd file -o file-99-env.zst
 Warning : compression level higher than max, reduced to 19 
-+ cmp file-f10.zst file-f10-env.zst
-+ cmp file-1.zst file-1-env.zst
-+ cmp file-19.zst file-19-env.zst
-+ cmp file-99.zst file-99-env.zst
-+ zstd -f file
-+ ZSTD_CLEVEL=-
-+ zstd -f file -o file-env.zst
+
+cmp file-f10.zst file-f10-env.zst || die "Environment variable failed to set level"
+cmp file-1.zst file-1-env.zst || die "Environment variable failed to set level"
+cmp file-19.zst file-19-env.zst || die "Environment variable failed to set level"
+cmp file-99.zst file-99-env.zst || die "Environment variable failed to set level"
+
+# Test invalid environment clevel is the default level
+zstd -f file
+ZSTD_CLEVEL=- zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=+
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=+ zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=a zstd -f file -o file-env.zst         ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=-a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=-a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=-a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=+a
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=+a zstd -f file -o file-env.zst        ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=+a: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=3a7
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=3a7 zstd -f file -o file-env.zst       ; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=3a7: not a valid integer value 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=5000000000
-+ zstd -f file -o file-env.zst
+ZSTD_CLEVEL=5000000000 zstd -f file -o file-env.zst; cmp file.zst file-env.zst
 Ignore environment variable setting ZSTD_CLEVEL=5000000000: numeric value too large 
-+ cmp file.zst file-env.zst
-+ ZSTD_CLEVEL=10
-+ zstd -f file -1 -o file-1-env.zst
-+ ZSTD_CLEVEL=10
-+ zstd -f file --fast=1 -o file-f1-env.zst
-+ cmp file-1.zst file-1-env.zst
-+ cmp file-f1.zst file-f1-env.zst
+
+# Test environment clevel is overridden by command line
+ZSTD_CLEVEL=10 zstd -f file -1 -o file-1-env.zst
+ZSTD_CLEVEL=10 zstd -f file --fast=1 -o file-f1-env.zst
+
+cmp file-1.zst file-1-env.zst  || die "Environment variable not overridden"
+cmp file-f1.zst file-f1-env.zst || die "Environment variable not overridden"
diff --git a/tests/cli-tests/dict-builder/no-inputs b/tests/cli-tests/dict-builder/no-inputs.sh
similarity index 76%
rename from tests/cli-tests/dict-builder/no-inputs
rename to tests/cli-tests/dict-builder/no-inputs.sh
index d37fdce5a20..416b83742f2 100755
--- a/tests/cli-tests/dict-builder/no-inputs
+++ b/tests/cli-tests/dict-builder/no-inputs.sh
@@ -1,3 +1,3 @@
 #!/bin/sh
-set -x
+set -v
 zstd --train
diff --git a/tests/cli-tests/dict-builder/no-inputs.exit b/tests/cli-tests/dict-builder/no-inputs.sh.exit
similarity index 100%
rename from tests/cli-tests/dict-builder/no-inputs.exit
rename to tests/cli-tests/dict-builder/no-inputs.sh.exit
diff --git a/tests/cli-tests/dict-builder/no-inputs.stderr.exact b/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
similarity index 93%
rename from tests/cli-tests/dict-builder/no-inputs.stderr.exact
rename to tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
index b3b5599d5ae..d7b3ea020f7 100644
--- a/tests/cli-tests/dict-builder/no-inputs.stderr.exact
+++ b/tests/cli-tests/dict-builder/no-inputs.sh.stderr.exact
@@ -1,4 +1,4 @@
-+ zstd --train
+zstd --train
 !  Warning : nb of samples too low for proper processing ! 
 !  Please provide _one file per sample_. 
 !  Alternatively, split files into fixed-size blocks representative of samples, with -B# 
diff --git a/tests/cli-tests/dictionaries/dictionary-mismatch.sh b/tests/cli-tests/dictionaries/dictionary-mismatch.sh
index 2b5d5a0d227..8264ccca5a2 100755
--- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh
+++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 set -e
 
@@ -22,7 +22,7 @@ if [ false ]; then
 	datagen -g1000 -s0 > file0
 fi
 
-set -x
+set -v
 zstd files/0 -D dicts/0
 zstd -t files/0.zst -D dicts/0
 zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
diff --git a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
index 399a3207d26..0afea722e98 100644
--- a/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
+++ b/tests/cli-tests/dictionaries/dictionary-mismatch.sh.stderr.exact
@@ -1,8 +1,6 @@
-+ zstd files/0 -D dicts/0
-+ zstd -t files/0.zst -D dicts/0
-+ zstd -t files/0.zst -D dicts/1
+zstd files/0 -D dicts/0
+zstd -t files/0.zst -D dicts/0
+zstd -t files/0.zst -D dicts/1 && die "Must fail" ||:
 files/0.zst : Decoding error (36) : Dictionary mismatch 
-+ :
-+ zstd -t files/0.zst
+zstd -t files/0.zst            && die "Must fail" ||:
 files/0.zst : Decoding error (36) : Dictionary mismatch 
-+ :
diff --git a/tests/cli-tests/dictionaries/setup_once b/tests/cli-tests/dictionaries/setup_once
index 6316df165fa..1241c578214 100755
--- a/tests/cli-tests/dictionaries/setup_once
+++ b/tests/cli-tests/dictionaries/setup_once
@@ -2,7 +2,7 @@
 
 set -e
 
-source "$COMMON/platform.sh"
+. "$COMMON/platform.sh"
 
 
 mkdir files/ dicts/