diff --git a/.github/workflows/pre-commit-reuse.yml b/.github/workflows/pre-commit-reuse.yml index c8375e31a4b6..e8e417b2eb8c 100644 --- a/.github/workflows/pre-commit-reuse.yml +++ b/.github/workflows/pre-commit-reuse.yml @@ -84,8 +84,8 @@ jobs: if: | always() && steps.check-coverage-xml-exists.outputs.files_exists == 'true' && - steps.check-pytest-junit-exists.outputs.files_exists == false && - github.event.pull_request.head.repo.fork == false + steps.check-pytest-junit-exists.outputs.files_exists == 'true' && + ! github.event.pull_request.head.repo.fork uses: MishaKav/pytest-coverage-comment@main with: pytest-xml-coverage-path: coverage_report/coverage.xml diff --git a/.gitlab/ci/.gitlab-ci.bucket-upload.yml b/.gitlab/ci/.gitlab-ci.bucket-upload.yml index fae4a61c2cf4..ce7a3c1a65e6 100644 --- a/.gitlab/ci/.gitlab-ci.bucket-upload.yml +++ b/.gitlab/ci/.gitlab-ci.bucket-upload.yml @@ -71,6 +71,14 @@ run-unittests-and-lint-upload-flow: - .run-unittests-and-lint - .bucket-upload-rule +run-pre-commit-upload-flow: + cache: + policy: push + allow_failure: true + extends: + - .run-pre-commit + - .bucket-upload-rule + jobs-done-check-upload-flow: extends: - .jobs-done-check diff --git a/.gitlab/ci/.gitlab-ci.global.yml b/.gitlab/ci/.gitlab-ci.global.yml index 266b0e550153..040e6658611a 100644 --- a/.gitlab/ci/.gitlab-ci.global.yml +++ b/.gitlab/ci/.gitlab-ci.global.yml @@ -7,7 +7,7 @@ prefix: dev-content paths: - $PIP_CACHE_DIR - - /root/.cache/pre-commit + - $PRE_COMMIT_HOME - .venv/ - node_modules/ - .npm/ @@ -289,7 +289,6 @@ - *build_parameters - *gitlab_ci_build_parameters - *checkout-upload-commit-content-nightly - .default-after-script: - source .gitlab/helper_functions.sh - *setup-network-certs @@ -508,23 +507,30 @@ - gcloud auth configure-docker ${DOCKER_IO_DOMAIN} >> "${ARTIFACTS_FOLDER}/logs/configure_docker_with_registry.log" 2>&1 - section_end "Revoking GCP Auth and Configure Docker" # temp solution not to spam the build - - PRE_COMMIT_SUCCESS=0 - SHOULD_LINT_ALL=$(./Tests/scripts/should_lint_all.sh) + - PRE_COMMIT_SUCCESS=0 - | - if [[ -n "${SHOULD_LINT_ALL}" ]]; then - echo "Pre Commit all files" - # if we need to pre-commit all anyway we need the graph, and it's better (resource-wise) to create it here. - demisto-sdk graph update - demisto-sdk pre-commit -a --mode=nightly || PRE_COMMIT_SUCCESS=1 + if [[ -n $BUCKET_UPLOAD && $TEST_UPLOAD == "true" ]]; then + echo "Skipping validations when uploading to a test bucket." else - echo "Pre Commit only changed files" - demisto-sdk pre-commit --mode=ci || PRE_COMMIT_SUCCESS=1 + if [[ -n $BUCKET_UPLOAD ]]; then + demisto-sdk pre-commit -g --prev-version $LAST_UPLOAD_COMMIT --mode=nightly || PRE_COMMIT_SUCCESS=1 + else + if [[ -n "${SHOULD_LINT_ALL}" ]]; then + echo "Pre Commit all files" + # if we need to pre-commit all anyway we need the graph, and it's better (resource-wise) to create it here. + demisto-sdk graph update + demisto-sdk pre-commit -a --mode=nightly || PRE_COMMIT_SUCCESS=1 + else + echo "Pre Commit only changed files" + demisto-sdk pre-commit --mode=ci || PRE_COMMIT_SUCCESS=1 + fi + fi fi - echo "PRE_COMMIT_SUCCESS=$PRE_COMMIT_SUCCESS" - job-done - - + - exit $PRE_COMMIT_SUCCESS .run-validations: stage: unittests-and-validations diff --git a/.gitlab/ci/.gitlab-ci.variables.yml b/.gitlab/ci/.gitlab-ci.variables.yml index fa2d674e46a2..c8ceed3ba0c6 100644 --- a/.gitlab/ci/.gitlab-ci.variables.yml +++ b/.gitlab/ci/.gitlab-ci.variables.yml @@ -25,6 +25,7 @@ variables: BASH_ENV: "${CI_PROJECT_DIR}/artifacts/bash_env" PYTHONPATH: "${CI_PROJECT_DIR}" PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" + PRE_COMMIT_HOME: "${CI_PROJECT_DIR}/.cache/pre-commit" FEATURE_BRANCH_NAME: "v4.5.0" IFRA_ENV_TYPE: "Content-Env" GET_SOURCES_ATTEMPTS: 3 # see https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts diff --git a/.pre-commit-config_template.yaml b/.pre-commit-config_template.yaml index 2ddf7ff7b879..82b3b54ee988 100644 --- a/.pre-commit-config_template.yaml +++ b/.pre-commit-config_template.yaml @@ -81,6 +81,8 @@ repos: exclude: _test\.py|\.vulture_whitelist\.py|test_data|tests_data|TestData require_serial: true language: system + # skip nightly since we don't care about warnings in nightly + skip:nightly: true - id: pylint-in-docker name: pylint-in-docker @@ -117,7 +119,7 @@ repos: language: docker files: Packs\/.*_test\.py$ # The exclude list includes the list of all integration/scripts which require network for testing - exclude: test_data|Packs/ApiModules/Scripts/NGINXApiModule/NGINXApiModule_test.py|Packs/Base/Scripts/CommonServerPython/CommonServerPython_test.py|Packs/Base/Scripts/SanePdfReport/SanePdfReport_test.py|Packs/Base/Scripts/CheckDockerImageAvailable/CheckDockerImageAvailable_test.py|Packs/CommonScripts/Scripts/Ping/Ping_test.py|Packs/CommonScripts/Scripts/GetDockerImageLatestTag/GetDockerImageLatestTag_test.py|Packs/CommonScripts/Scripts/GetDuplicatesMlv2/GetDuplicatesMlv2_test.py|Packs/CommonScripts/Scripts/DockerHardeningCheck/DockerHardeningCheck_test.py|Packs/rasterize/Integrations/rasterize/rasterize_test.py|Packs/Slack/Integrations/Slack/Slack_test.py|Packs/SymantecDLP/Integrations/SymantecDLP/SymantecDLP_test.py|Packs/WebFileRepository/Integrations/WebFileRepository/WebFileRepository_test.py|Packs/Whois/Integrations/Whois/Whois_test.py + exclude: test_data|Packs/Base/Scripts/CommonServerPython/CommonServerPython_test.py|Packs/Base/Scripts/SanePdfReport/SanePdfReport_test.py|Packs/Base/Scripts/CheckDockerImageAvailable/CheckDockerImageAvailable_test.py|Packs/CommonScripts/Scripts/Ping/Ping_test.py|Packs/CommonScripts/Scripts/GetDockerImageLatestTag/GetDockerImageLatestTag_test.py|Packs/CommonScripts/Scripts/GetDuplicatesMlv2/GetDuplicatesMlv2_test.py|Packs/CommonScripts/Scripts/DockerHardeningCheck/DockerHardeningCheck_test.py|Packs/rasterize/Integrations/rasterize/rasterize_test.py|Packs/Slack/Integrations/Slack/Slack_test.py|Packs/SymantecDLP/Integrations/SymantecDLP/SymantecDLP_test.py|Packs/WebFileRepository/Integrations/WebFileRepository/WebFileRepository_test.py|Packs/Whois/Integrations/Whois/Whois_test.py env: COVERAGE_FILE: /src/.pre-commit/coverage/.coverage args: @@ -150,7 +152,7 @@ repos: docker_image: from-yml entry: coverage language: docker - files: Packs/ApiModules/Scripts/NGINXApiModule/NGINXApiModule_test.py|Packs/Base/Scripts/CommonServerPython/CommonServerPython_test.py|Packs/Base/Scripts/SanePdfReport/SanePdfReport_test.py|Packs/Base/Scripts/CheckDockerImageAvailable/CheckDockerImageAvailable_test.py|Packs/CommonScripts/Scripts/Ping/Ping_test.py|Packs/CommonScripts/Scripts/GetDockerImageLatestTag/GetDockerImageLatestTag_test.py|Packs/CommonScripts/Scripts/GetDuplicatesMlv2/GetDuplicatesMlv2_test.py|Packs/CommonScripts/Scripts/DockerHardeningCheck/DockerHardeningCheck_test.py|Packs/rasterize/Integrations/rasterize/rasterize_test.py|Packs/Slack/Integrations/Slack/Slack_test.py|Packs/SymantecDLP/Integrations/SymantecDLP/SymantecDLP_test.py|Packs/WebFileRepository/Integrations/WebFileRepository/WebFileRepository_test.py|Packs/Whois/Integrations/Whois/Whois_test.py + files: Packs/Base/Scripts/CommonServerPython/CommonServerPython_test.py|Packs/Base/Scripts/SanePdfReport/SanePdfReport_test.py|Packs/Base/Scripts/CheckDockerImageAvailable/CheckDockerImageAvailable_test.py|Packs/CommonScripts/Scripts/Ping/Ping_test.py|Packs/CommonScripts/Scripts/GetDockerImageLatestTag/GetDockerImageLatestTag_test.py|Packs/CommonScripts/Scripts/GetDuplicatesMlv2/GetDuplicatesMlv2_test.py|Packs/CommonScripts/Scripts/DockerHardeningCheck/DockerHardeningCheck_test.py|Packs/rasterize/Integrations/rasterize/rasterize_test.py|Packs/Slack/Integrations/Slack/Slack_test.py|Packs/SymantecDLP/Integrations/SymantecDLP/SymantecDLP_test.py|Packs/WebFileRepository/Integrations/WebFileRepository/WebFileRepository_test.py|Packs/Whois/Integrations/Whois/Whois_test.py env: COVERAGE_FILE: /src/.pre-commit/coverage/.coverage args: @@ -186,6 +188,7 @@ repos: name: validate-content-paths entry: validate-content-path language: system + require_serial: true args: - validate - --skip-depth-one-file diff --git a/Packs/Base/Scripts/SanePdfReport/SanePdfReport_test.py b/Packs/Base/Scripts/SanePdfReport/SanePdfReport_test.py index 0ac1feaf0df8..d29b3c904daa 100644 --- a/Packs/Base/Scripts/SanePdfReport/SanePdfReport_test.py +++ b/Packs/Base/Scripts/SanePdfReport/SanePdfReport_test.py @@ -48,9 +48,6 @@ def test_sane_pdf_report(mocker): assert demisto.results.call_args[0][0]['HumanReadable'] == 'Successfully generated pdf' assert demisto.results.call_args[0][0]['Contents'] - zombies, output = find_zombie_processes() - assert len(zombies) == 0 - def test_markdown_image_server(mocker, capfd): with capfd.disabled(): diff --git a/Packs/CommonScripts/Scripts/ConvertFile/ConvertFile_test.py b/Packs/CommonScripts/Scripts/ConvertFile/ConvertFile_test.py index a7fdfa1ca89c..88bebe57e1fb 100644 --- a/Packs/CommonScripts/Scripts/ConvertFile/ConvertFile_test.py +++ b/Packs/CommonScripts/Scripts/ConvertFile/ConvertFile_test.py @@ -1,3 +1,4 @@ +import subprocess from ConvertFile import main, find_zombie_processes import demistomock as demisto from CommonServerPython import entryTypes @@ -5,7 +6,6 @@ import pytest import glob import os -import subprocess RETURN_ERROR_TARGET = 'ConvertFile.return_error' @@ -84,7 +84,6 @@ def test_convert_pdf_to_html(mocker): # check no defunct processed zombies, output = find_zombie_processes() assert not zombies - assert 'defunct' not in output def test_convert_failure(mocker): diff --git a/Packs/CommonScripts/Scripts/DockerHardeningCheck/DockerHardeningCheck_test.py b/Packs/CommonScripts/Scripts/DockerHardeningCheck/DockerHardeningCheck_test.py index 25290fb213bc..a506d101a565 100644 --- a/Packs/CommonScripts/Scripts/DockerHardeningCheck/DockerHardeningCheck_test.py +++ b/Packs/CommonScripts/Scripts/DockerHardeningCheck/DockerHardeningCheck_test.py @@ -1,4 +1,4 @@ -from DockerHardeningCheck import (check_memory, mem_size_to_bytes, check_pids, check_fd_limits, check_cpus, +from DockerHardeningCheck import (check_cpus, check_memory, mem_size_to_bytes, check_pids, check_fd_limits, get_default_gateway, check_network, CLOUD_METADATA_URL) import pytest import os @@ -27,7 +27,7 @@ def test_fd_limits(): def test_check_cpus(): - if os.getenv("CI") == "true": + if os.getenv("CI"): pytest.skip("skipping as in CI we run with a single CPU") return assert check_cpus(1) # during unit tests we should fail diff --git a/Tests/scripts/lock_cloud_machines.py b/Tests/scripts/lock_cloud_machines.py index 8562e65c91ac..248020db0266 100644 --- a/Tests/scripts/lock_cloud_machines.py +++ b/Tests/scripts/lock_cloud_machines.py @@ -5,7 +5,7 @@ from time import sleep import random import requests -from google.cloud import storage # noqa +from google.cloud import storage # type: ignore[attr-defined] import argparse from Utils.github_workflow_scripts.utils import get_env_var from slack_sdk import WebClient as SlackWebClient diff --git a/Tests/test_content.py b/Tests/test_content.py index ac2a12356589..61a20545d1af 100644 --- a/Tests/test_content.py +++ b/Tests/test_content.py @@ -15,7 +15,7 @@ import requests import urllib3 from google.api_core.exceptions import PreconditionFailed -from google.cloud import storage +from google.cloud import storage # type: ignore[attr-defined] from Tests.test_dependencies import get_used_integrations from demisto_sdk.commands.common.constants import FILTER_CONF from demisto_sdk.commands.test_content.ParallelLoggingManager import ParallelLoggingManager diff --git a/Utils/merge_content_new_zip.py b/Utils/merge_content_new_zip.py index 4ed8399f8e5e..da9060422035 100755 --- a/Utils/merge_content_new_zip.py +++ b/Utils/merge_content_new_zip.py @@ -1,6 +1,6 @@ import zipfile as z import os -from google.cloud import storage +from google.cloud import storage # type: ignore[attr-defined] import argparse import shutil