Skip to content

Commit

Permalink
Merge pull request kubernetes-csi#11 from pohly/verify-shellcheck
Browse files Browse the repository at this point in the history
verify shellcheck
  • Loading branch information
k8s-ci-robot authored Apr 2, 2019
2 parents 3b6af7b + 6c7ba1b commit 2069a0a
Show file tree
Hide file tree
Showing 4 changed files with 321 additions and 0 deletions.
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,15 @@ Cheat sheet:
- `git subtree add --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - add release tools to a repo which does not have them yet (only once)
- `git subtree pull --prefix=release-tools https://github.com/kubernetes-csi/csi-release-tools.git master` - update local copy to latest upstream (whenever upstream changes)
- edit, `git commit`, `git subtree push --prefix=release-tools git@github.com:<user>/csi-release-tools.git <my-new-or-existing-branch>` - push to a new branch before submitting a PR

verify-shellcheck.sh
--------------------

The [verify-shellcheck.sh](./verify-shellcheck.sh) script in this repo
is a stripped down copy of the [corresponding
script](https://github.com/kubernetes/kubernetes/blob/release-1.14/hack/verify-shellcheck.sh)
in the Kubernetes repository. It can be used to check for certain
errors shell scripts, like missing quotation marks. The default
`test-shellcheck` target in [build.make](./build.make) only checks the
scripts in this directory. Components can add more directories to
`TEST_SHELLCHECK_DIRS` to check also other scripts.
15 changes: 15 additions & 0 deletions build.make
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,18 @@ test: test-subtree
test-subtree:
@ echo; echo "### $@:"
./release-tools/verify-subtree.sh release-tools

# Components can extend the set of directories which must pass shellcheck.
# The default is to check only the release-tools directory itself.
TEST_SHELLCHECK_DIRS=release-tools
.PHONY: test-shellcheck
test: test-shellcheck
test-shellcheck:
@ echo; echo "### $@:"
@ ret=0; \
for dir in $(abspath $(TEST_SHELLCHECK_DIRS)); do \
echo; \
echo "$$dir:"; \
./release-tools/verify-shellcheck.sh "$$dir" || ret=1; \
done; \
return $$ret
148 changes: 148 additions & 0 deletions util.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
#!/usr/bin/env bash

# Copyright 2014 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

function kube::util::sourced_variable {
# Call this function to tell shellcheck that a variable is supposed to
# be used from other calling context. This helps quiet an "unused
# variable" warning from shellcheck and also document your code.
true
}

kube::util::sortable_date() {
date "+%Y%m%d-%H%M%S"
}

# arguments: target, item1, item2, item3, ...
# returns 0 if target is in the given items, 1 otherwise.
kube::util::array_contains() {
local search="$1"
local element
shift
for element; do
if [[ "${element}" == "${search}" ]]; then
return 0
fi
done
return 1
}

# Example: kube::util::trap_add 'echo "in trap DEBUG"' DEBUG
# See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal
kube::util::trap_add() {
local trap_add_cmd
trap_add_cmd=$1
shift

for trap_add_name in "$@"; do
local existing_cmd
local new_cmd

# Grab the currently defined trap commands for this trap
existing_cmd=$(trap -p "${trap_add_name}" | awk -F"'" '{print $2}')

if [[ -z "${existing_cmd}" ]]; then
new_cmd="${trap_add_cmd}"
else
new_cmd="${trap_add_cmd};${existing_cmd}"
fi

# Assign the test. Disable the shellcheck warning telling that trap
# commands should be single quoted to avoid evaluating them at this
# point instead evaluating them at run time. The logic of adding new
# commands to a single trap requires them to be evaluated right away.
# shellcheck disable=SC2064
trap "${new_cmd}" "${trap_add_name}"
done
}

kube::util::download_file() {
local -r url=$1
local -r destination_file=$2

rm "${destination_file}" 2&> /dev/null || true

for i in $(seq 5)
do
if ! curl -fsSL --retry 3 --keepalive-time 2 "${url}" -o "${destination_file}"; then
echo "Downloading ${url} failed. $((5-i)) retries left."
sleep 1
else
echo "Downloading ${url} succeed"
return 0
fi
done
return 1
}

# Wait for background jobs to finish. Return with
# an error status if any of the jobs failed.
kube::util::wait-for-jobs() {
local fail=0
local job
for job in $(jobs -p); do
wait "${job}" || fail=$((fail + 1))
done
return ${fail}
}

# kube::util::join <delim> <list...>
# Concatenates the list elements with the delimiter passed as first parameter
#
# Ex: kube::util::join , a b c
# -> a,b,c
function kube::util::join {
local IFS="$1"
shift
echo "$*"
}

# kube::util::check-file-in-alphabetical-order <file>
# Check that the file is in alphabetical order
#
function kube::util::check-file-in-alphabetical-order {
local failure_file="$1"
if ! diff -u "${failure_file}" <(LC_ALL=C sort "${failure_file}"); then
{
echo
echo "${failure_file} is not in alphabetical order. Please sort it:"
echo
echo " LC_ALL=C sort -o ${failure_file} ${failure_file}"
echo
} >&2
false
fi
}

# Some useful colors.
if [[ -z "${color_start-}" ]]; then
declare -r color_start="\033["
declare -r color_red="${color_start}0;31m"
declare -r color_yellow="${color_start}0;33m"
declare -r color_green="${color_start}0;32m"
declare -r color_blue="${color_start}1;34m"
declare -r color_cyan="${color_start}1;36m"
declare -r color_norm="${color_start}0m"

kube::util::sourced_variable "${color_start}"
kube::util::sourced_variable "${color_red}"
kube::util::sourced_variable "${color_yellow}"
kube::util::sourced_variable "${color_green}"
kube::util::sourced_variable "${color_blue}"
kube::util::sourced_variable "${color_cyan}"
kube::util::sourced_variable "${color_norm}"
fi

# ex: ts=2 sw=2 et filetype=sh
146 changes: 146 additions & 0 deletions verify-shellcheck.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
#!/usr/bin/env bash

# Copyright 2018 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -o errexit
set -o nounset
set -o pipefail

# The csi-release-tools directory.
TOOLS="$(dirname "${BASH_SOURCE[0]}")"
. "${TOOLS}/util.sh"

# Directory to check. Default is the parent of the tools themselves.
ROOT="${1:-${TOOLS}/..}"

# required version for this script, if not installed on the host we will
# use the official docker image instead. keep this in sync with SHELLCHECK_IMAGE
SHELLCHECK_VERSION="0.6.0"
# upstream shellcheck latest stable image as of January 10th, 2019
SHELLCHECK_IMAGE="koalaman/shellcheck-alpine:v0.6.0@sha256:7d4d712a2686da99d37580b4e2f45eb658b74e4b01caf67c1099adc294b96b52"

# fixed name for the shellcheck docker container so we can reliably clean it up
SHELLCHECK_CONTAINER="k8s-shellcheck"

# disabled lints
disabled=(
# this lint disallows non-constant source, which we use extensively without
# any known bugs
1090
# this lint prefers command -v to which, they are not the same
2230
)
# comma separate for passing to shellcheck
join_by() {
local IFS="$1";
shift;
echo "$*";
}
SHELLCHECK_DISABLED="$(join_by , "${disabled[@]}")"
readonly SHELLCHECK_DISABLED

# creates the shellcheck container for later use
create_container () {
# TODO(bentheelder): this is a performance hack, we create the container with
# a sleep MAX_INT32 so that it is effectively paused.
# We then repeatedly exec to it to run each shellcheck, and later rm it when
# we're done.
# This is incredibly much faster than creating a container for each shellcheck
# call ...
docker run --name "${SHELLCHECK_CONTAINER}" -d --rm -v "${ROOT}:${ROOT}" -w "${ROOT}" --entrypoint="sleep" "${SHELLCHECK_IMAGE}" 2147483647
}
# removes the shellcheck container
remove_container () {
docker rm -f "${SHELLCHECK_CONTAINER}" &> /dev/null || true
}

# ensure we're linting the source tree
cd "${ROOT}"

# find all shell scripts excluding ./_*, ./.git/*, ./vendor*,
# and anything git-ignored
all_shell_scripts=()
while IFS=$'\n' read -r script;
do git check-ignore -q "$script" || all_shell_scripts+=("$script");
done < <(find . -name "*.sh" \
-not \( \
-path ./_\* -o \
-path ./.git\* -o \
-path ./vendor\* \
\))

# detect if the host machine has the required shellcheck version installed
# if so, we will use that instead.
HAVE_SHELLCHECK=false
if which shellcheck &>/dev/null; then
detected_version="$(shellcheck --version | grep 'version: .*')"
if [[ "${detected_version}" = "version: ${SHELLCHECK_VERSION}" ]]; then
HAVE_SHELLCHECK=true
fi
fi

# tell the user which we've selected and possibly set up the container
if ${HAVE_SHELLCHECK}; then
echo "Using host shellcheck ${SHELLCHECK_VERSION} binary."
else
echo "Using shellcheck ${SHELLCHECK_VERSION} docker image."
# remove any previous container, ensure we will attempt to cleanup on exit,
# and create the container
remove_container
kube::util::trap_add 'remove_container' EXIT
if ! output="$(create_container 2>&1)"; then
{
echo "Failed to create shellcheck container with output: "
echo ""
echo "${output}"
} >&2
exit 1
fi
fi

# lint each script, tracking failures
errors=()
for f in "${all_shell_scripts[@]}"; do
set +o errexit
if ${HAVE_SHELLCHECK}; then
failedLint=$(shellcheck --exclude="${SHELLCHECK_DISABLED}" "${f}")
else
failedLint=$(docker exec -t ${SHELLCHECK_CONTAINER} \
shellcheck --exclude="${SHELLCHECK_DISABLED}" "${f}")
fi
set -o errexit
if [[ -n "${failedLint}" ]]; then
errors+=( "${failedLint}" )
fi
done

# Check to be sure all the packages that should pass lint are.
if [ ${#errors[@]} -eq 0 ]; then
echo 'Congratulations! All shell files are passing lint.'
else
{
echo "Errors from shellcheck:"
for err in "${errors[@]}"; do
echo "$err"
done
echo
echo 'Please review the above warnings. You can test via "./hack/verify-shellcheck"'
echo 'If the above warnings do not make sense, you can exempt them from shellcheck'
echo 'checking by adding the "shellcheck disable" directive'
echo '(https://github.com/koalaman/shellcheck/wiki/Directive#disable).'
echo
} >&2
false
fi

0 comments on commit 2069a0a

Please sign in to comment.