Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ci: fix license lint #3279

Merged
merged 5 commits into from
Feb 22, 2024
Merged

ci: fix license lint #3279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e02cb4c
Upgrade controller-runtime to 0.17.2, remove fork
maxsmythe Feb 21, 2024
8357202
ci: fix license lint
sozercan Feb 22, 2024
9c76a62
Merge branch 'master' into fix-license-lint
sozercan Feb 22, 2024
420b164
fix cncf exception url
sozercan Feb 22, 2024
905a726
add script path to ci
sozercan Feb 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/license-lint.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@ on:
- "go.mod"
- "go.sum"
- "vendor/**"
- "third_party/k8s.io/kubernetes/hack/verify-licenses.sh"
pull_request:
paths:
- "go.mod"
- "go.sum"
- "vendor/**"
- "third_party/k8s.io/kubernetes/hack/verify-licenses.sh"

permissions:
contents: read
Expand Down
4 changes: 2 additions & 2 deletions third_party/k8s.io/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# k8s.io/kubernetes

Forked from k8s.io/kubernetes@124fd62ad253f8362d78d5710d8d363aa1b376df
Forked from k8s.io/kubernetes@139c6daf302442aa19fed6cba8732a1b5c0eb1e0

This is a light fork to use scripts from kubernetes/kubernetes for gatekeeper's use cases.

The original code can be found at https://github.com/kubernetes/kubernetes/tree/124fd62ad253f8362d78d5710d8d363aa1b376df .
The original code can be found at https://github.com/kubernetes/kubernetes/tree/139c6daf302442aa19fed6cba8732a1b5c0eb1e0.
172 changes: 70 additions & 102 deletions third_party/k8s.io/kubernetes/hack/verify-licenses.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,27 @@

# Usage: `hack/verify-licenses.sh`.


set -o errexit
set -o nounset
set -o pipefail

KUBE_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t kubernetes.XXXXXX)
ARTIFACTS="${ARTIFACTS:-${PWD}/.tmp/licenses}"
mkdir -p "$ARTIFACTS/logs/"

KUBE_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t kubernetes.XXXXXX)

# Creating a new repository tree
# Creating a new repository tree
# Deleting vendor directory to make go-licenses fetch license URLs from go-packages source repository
git worktree add -f "${KUBE_TEMP}"/tmp_test_licenses/gatekeeper HEAD >/dev/null 2>&1 || true
cd "${KUBE_TEMP}"/tmp_test_licenses/gatekeeper && rm -rf vendor


# Explicitly opt into go modules, even though we're inside a GOPATH directory
export GO111MODULE=on

function http_code() {
curl -I -s -o /dev/null -w "%{http_code}" "$1"
}

allowed_licenses=()
packages_flagged=()
packages_url_missing=()
exit_code=0
Expand All @@ -45,117 +47,83 @@ go install github.com/google/go-licenses@latest

# Fetching CNCF Approved List Of Licenses
# Refer: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md
curl -s 'https://spdx.org/licenses/licenses.json' -o "${KUBE_TEMP}"/licenses.json

number_of_licenses=$(jq '.licenses | length' "${KUBE_TEMP}"/licenses.json)
loop_index_length=$(( number_of_licenses - 1 ))

curl -s 'https://spdx.org/licenses/licenses.json' -o "${ARTIFACTS}"/licenses.json

echo '[INFO] Fetching current list of CNCF approved licenses...'
for index in $(seq 0 $loop_index_length);
do
licenseID=$(jq ".licenses[$index] .licenseId" "${KUBE_TEMP}"/licenses.json)
if [[ $(jq ".licenses[$index] .isDeprecatedLicenseId" "${KUBE_TEMP}"/licenses.json) == false ]]
then
allowed_licenses+=("${licenseID}")
fi
done

jq -r '.licenses[] | select(.isDeprecatedLicenseId==false) .licenseId' "${ARTIFACTS}"/licenses.json | sort | uniq >"${ARTIFACTS}"/licenses.txt

# Scanning go-packages under the project & verifying against the CNCF approved list of licenses
echo '[INFO] Starting license scan on go-packages...'
go-licenses report ./... --include_tests >> "${KUBE_TEMP}"/licenses.csv

echo -e 'PACKAGE_NAME LICENSE_NAME LICENSE_URL\n' >> "${KUBE_TEMP}"/approved_licenses.dump
while IFS=, read -r GO_PACKAGE LICENSE_URL LICENSE_NAME
do
FORMATTED_LICENSE_URL=
if [[ " ${allowed_licenses[*]} " == *"${LICENSE_NAME}"* ]];
then
if [[ "${LICENSE_URL}" == 'Unknown' ]];
then
if [[ "${GO_PACKAGE}" != k8s.io/* ]];
then
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump
packages_url_missing+=("${GO_PACKAGE}")
else
LICENSE_URL='https://github.com/kubernetes/kubernetes/blob/master/LICENSE'
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump
fi
elif curl -Is "${LICENSE_URL}" | head -1 | grep -q 404;
then
# For gatekeeper, the script won't find the constraint frameworks's license atm.
if [[ "${GO_PACKAGE}" == github.com/open-policy-agent/frameworks/* ]];
then
LICENSE_URL='https://github.com/open-policy-agent/frameworks/blob/master/LICENSE'
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump
continue
fi

# Check whether the License URL is incorrectly formed
# TODO: Remove this workaround check once PR https://github.com/google/go-licenses/pull/110 is merged
IFS='/' read -r -a split_license_url <<< ${LICENSE_URL}
for part_of_url in "${split_license_url[@]}"
do
if [[ ${part_of_url} == '' ]]
then
continue
elif [[ ${part_of_url} == 'https:' ]]
then
FORMATTED_LICENSE_URL+='https://'
else
if [[ ${part_of_url} =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]
then
FORMATTED_LICENSE_URL+="${part_of_url}/${split_license_url[-1]}"
break
else
FORMATTED_LICENSE_URL+="${part_of_url}/"
fi
fi
done
if curl -Is "${FORMATTED_LICENSE_URL}" | head -1 | grep -q 404;
then
packages_url_missing+=("${GO_PACKAGE}")
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump
else
echo "${GO_PACKAGE} ${LICENSE_NAME} ${FORMATTED_LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump
fi
else
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump
fi
else
# Not all packages at this point should go to the not approved dump.
# there are a few exceptions approved by CNCF as per: https://github.com/cncf/foundation/tree/main/license-exceptions
# Currently gatekeeper uses just one of those so we are not going to do a general solution.

if [[ "${GO_PACKAGE}" == "github.com/rcrowley/go-metrics" ]] && [[ "${LICENSE_NAME}" == "BSD-2-Clause-FreeBSD" ]];
then
# as per https://github.com/cncf/foundation/blob/main/license-exceptions/cncf-exceptions-2019-11-01.json#L723-L726
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/approved_licenses.dump
go-licenses report ./... >>"${ARTIFACTS}"/licenses.csv 2>"${ARTIFACTS}"/logs/go-licenses.log

echo -e 'PACKAGE_NAME LICENSE_NAME LICENSE_URL\n' >>"${ARTIFACTS}"/approved_licenses.dump
while IFS=, read -r GO_PACKAGE LICENSE_URL LICENSE_NAME; do
if [[ "${GO_PACKAGE}" == "github.com/rcrowley/go-metrics" ]] && [[ "${LICENSE_NAME}" == "BSD-2-Clause-FreeBSD" ]]; then
# as per https://github.com/cncf/foundation/blob/86cab8e303153005492c07d2304e8c8808d896be/license-exceptions/cncf-exceptions-2019-11-01.spdx#L1594-L1603
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >>"${ARTIFACTS}"/approved_licenses.dump
continue
fi

if ! grep -q "^${LICENSE_NAME}$" "${ARTIFACTS}"/licenses.txt; then
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >>"${ARTIFACTS}"/notapproved_licenses.dump
packages_flagged+=("${GO_PACKAGE}")
continue
fi

if [[ "${LICENSE_URL}" == 'Unknown' ]]; then
if [[ "${GO_PACKAGE}" != k8s.io/* ]]; then
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >>"${ARTIFACTS}"/approved_licenses_with_missing_urls.dump
packages_url_missing+=("${GO_PACKAGE}")
else
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >> "${KUBE_TEMP}"/notapproved_licenses.dump
packages_flagged+=("${GO_PACKAGE}")
LICENSE_URL='https://github.com/kubernetes/kubernetes/blob/master/LICENSE'
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >>"${ARTIFACTS}"/approved_licenses.dump
fi
fi
done < "${KUBE_TEMP}"/licenses.csv
awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${KUBE_TEMP}"/approved_licenses.dump

continue
fi

if [[ "$(http_code "${LICENSE_URL}")" != 404 ]]; then
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >>"${ARTIFACTS}"/approved_licenses.dump
continue
fi

# The URL 404'ed. Try parent-paths.

#echo -e "DBG: err 404 ${LICENSE_URL}"
dir="$(dirname "${LICENSE_URL}")"
file="$(basename "${LICENSE_URL}")"

while [[ "${dir}" != "." ]]; do
dir="$(dirname "${dir}")"
#echo "DBG: try ${dir}/${file}"
if [[ "$(http_code "${dir}/${file}")" != 404 ]]; then
#echo "DBG: it worked"
echo "${GO_PACKAGE} ${LICENSE_NAME} ${dir}/${file}" >>"${ARTIFACTS}"/approved_licenses.dump
break
fi
#echo "DBG: still 404"
done
if [[ "${dir}" == "." ]]; then
#echo "DBG: failed to find a license"
packages_url_missing+=("${GO_PACKAGE}")
echo "${GO_PACKAGE} ${LICENSE_NAME} ${LICENSE_URL}" >>"${ARTIFACTS}"/approved_licenses_with_missing_urls.dump
fi
done <"${ARTIFACTS}"/licenses.csv
awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${ARTIFACTS}"/approved_licenses.dump

if [[ ${#packages_url_missing[@]} -gt 0 ]]; then
echo -e '\n[ERROR] The following go-packages in the project have unknown or unreachable license URL:'
awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${KUBE_TEMP}"/approved_licenses_with_missing_urls.dump
exit_code=1
echo -e '\n[ERROR] The following go-packages in the project have unknown or unreachable license URL:'
awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${ARTIFACTS}"/approved_licenses_with_missing_urls.dump
exit_code=1
fi


if [[ ${#packages_flagged[@]} -gt 0 ]]; then
echo "[ERROR] The following go-packages in the project are using non-CNCF approved licenses. Please refer to the CNCF's approved licence list for further information: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md"
awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${KUBE_TEMP}"/notapproved_licenses.dump
exit_code=1
echo -e "\n[ERROR] The following go-packages in the project are using non-CNCF approved licenses. Please refer to the CNCF's approved licence list for further information: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md"
awk '{ printf "%-100s : %-20s : %s\n", $1, $2, $3 }' "${ARTIFACTS}"/notapproved_licenses.dump
exit_code=1
elif [[ "${exit_code}" -eq 1 ]]; then
echo "[ERROR] Project is using go-packages with unknown or unreachable license URLs. Please refer to the CNCF's approved licence list for further information: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md"
echo -e "\n[ERROR] Project is using go-packages with unknown or unreachable license URLs. Please refer to the CNCF's approved licence list for further information: https://github.com/cncf/foundation/blob/main/allowed-third-party-license-policy.md"
else
echo "[SUCCESS] Scan complete! All go-packages under the project are using current CNCF approved licenses!"
echo -e "\n[SUCCESS] Scan complete! All go-packages under the project are using current CNCF approved licenses!"
fi

exit "${exit_code}"
Loading