From 85cebc42c633d19d9ccfcf242eb6ba7aa1b687a8 Mon Sep 17 00:00:00 2001 From: SeongjunJo Date: Mon, 13 May 2024 01:27:07 +0900 Subject: [PATCH] Supports for excluding paths from analysis Signed-off-by: SeongjunJo --- README.md | 14 ++++++++-- requirements.txt | 8 +++--- src/fosslight_scanner/_help.py | 1 + src/fosslight_scanner/cli.py | 3 ++- src/fosslight_scanner/common.py | 4 +-- src/fosslight_scanner/fosslight_scanner.py | 30 +++++++++++++--------- tox.ini | 3 +++ 7 files changed, 42 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 5d526f1..b709ab9 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ Options: * Compare mode result file: supports excel, json, yaml, html -o Output directory or file -c Number of processes to analyze source + -e Path to exclude from analysis (ex, -e {dir} {file}) -r Keep raw data -t Hide the progress bar -v Print FOSSLight Scanner version @@ -96,12 +97,21 @@ Options: $ fosslight all -p /home/source_path -d "-a 'source /test/Projects/venv/bin/activate' -d 'deactivate'" ``` -### Ex 2. Download Link and analyze +### Ex 2. Local Source Analysis with Path to Exclude +``` +$ fosslight all -p /home/source_path -e temp_dir src/temp.py +``` + +### Ex 3. Download Link and analyze ``` $ fosslight all -o test_result_wget -w "https://github.com/LGE-OSS/example.git" ``` +If you want to analyze private repository, set your github token like below. +``` +$ fosslight all -w "https://my_github_token@github.com/Foo/private_repo +``` -### Ex 3. Compare the BOM of two FOSSLight reports with yaml or excel format and check the oss status (change/add/delete) +### Ex 4. Compare the BOM of two FOSSLight reports with yaml or excel format and check the oss status (change/add/delete) ``` $ fosslight compare -p FOSSLight_before_proj.yaml FOSSLight_after_proj.yaml -f excel ``` diff --git a/requirements.txt b/requirements.txt index 0392d94..372c7ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,8 @@ openpyxl progress pyyaml beautifulsoup4 -fosslight_util>=1.4.44 -fosslight_source>=1.7.7 -fosslight_dependency>=3.15.0 -fosslight_binary>=4.1.29 +fosslight_util>=1.4.45 +fosslight_source>=1.7.8 +fosslight_dependency>=3.15.1 +fosslight_binary>=4.1.30 fosslight_prechecker>=3.0.27 \ No newline at end of file diff --git a/src/fosslight_scanner/_help.py b/src/fosslight_scanner/_help.py index e829252..f144cb8 100644 --- a/src/fosslight_scanner/_help.py +++ b/src/fosslight_scanner/_help.py @@ -27,6 +27,7 @@ -w \t\t Link to be analyzed can be downloaded by wget or git clone -f \t\t FOSSLight Report file format (excel, yaml) * Compare mode result file: supports excel, json, yaml, html + -e \t\t Path to exclude from analysis (ex, -e {dir} {file}) -o \t\t Output directory or file -c \t\t Number of processes to analyze source -r\t\t\t Keep raw data diff --git a/src/fosslight_scanner/cli.py b/src/fosslight_scanner/cli.py index eb7b914..e37a9d0 100644 --- a/src/fosslight_scanner/cli.py +++ b/src/fosslight_scanner/cli.py @@ -25,6 +25,7 @@ def main(): parser.add_argument('--timer', '-t', help='Hide the progress bar', action='store_true', dest='timer', default=False) parser.add_argument('--version', '-v', help='Print version', action='store_true', dest='version', default=False) parser.add_argument('--help', '-h', help='Print help message', action='store_true', dest='help') + parser.add_argument('--exclude', '-e', help='Path to exclude from analysis', dest='exclude_path', nargs='*', default=[]) parser.add_argument('--no_correction', help='No correction with sbom-info.yaml', action='store_true', required=False, default=False) parser.add_argument('--correct_fpath', help='Path to the sbom-info.yaml', @@ -45,7 +46,7 @@ def main(): args.mode = ['all'] run_main(args.mode, args.path, args.dep_argument, args.output, args.format, args.link, args.db_url, args.timer, args.raw, args.core, - not args.no_correction, args.correct_fpath, args.ui) + not args.no_correction, args.correct_fpath, args.ui, args.exclude_path) if __name__ == "__main__": diff --git a/src/fosslight_scanner/common.py b/src/fosslight_scanner/common.py index 8f4bfcb..ba1b432 100755 --- a/src/fosslight_scanner/common.py +++ b/src/fosslight_scanner/common.py @@ -63,7 +63,7 @@ def run_analysis(path_to_run, params, func, str_run_start, output, exe_path): return return_value -def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args): +def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args, **kwargs): # return_idx == -1 : Raw return value itself logger.info(f"## Start to run {str_run_start}") success = True @@ -71,7 +71,7 @@ def call_analysis_api(path_to_run, str_run_start, return_idx, func, *args): try: if path_to_run != "": logger.info(f"|--- Path to analyze : {path_to_run}") - result = func(*args) + result = func(*args, **kwargs) else: logger.info("Analyzing path is missing...") except SystemExit: diff --git a/src/fosslight_scanner/fosslight_scanner.py b/src/fosslight_scanner/fosslight_scanner.py index 7d13a8a..b10ff9b 100755 --- a/src/fosslight_scanner/fosslight_scanner.py +++ b/src/fosslight_scanner/fosslight_scanner.py @@ -48,7 +48,7 @@ SCANNER_MODE = ["all", "compare", "reuse", "prechecker", "binary", "bin", "src", "source", "dependency", "dep"] -def run_dependency(path_to_analyze, output_file_with_path, params=""): +def run_dependency(path_to_analyze, output_file_with_path, params="", path_to_exclude=[]): result_list = [] package_manager = "" @@ -90,7 +90,7 @@ def run_dependency(path_to_analyze, output_file_with_path, params=""): output_file_with_path, pip_activate_cmd, pip_deactivate_cmd, output_custom_dir, app_name, - github_token) + github_token, path_to_exclude=path_to_exclude) if success: result_list = result.get('SRC_FL_Dependency') except Exception as ex: @@ -106,7 +106,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False, remove_src_data=True, result_log={}, output_file="", output_extension="", num_cores=-1, db_url="", default_oss_name="", default_oss_version="", url="", - correct_mode=True, correct_fpath="", ui_mode=False): + correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[]): final_excel_dir = output_path success = True temp_output_fiiles = [] @@ -136,8 +136,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False, output_prechecker = os.path.join(_output_dir, output_files["PRECHECKER"]) success, result = call_analysis_api(src_path, "Prechecker Lint", -1, prechecker_lint, - abs_path, False, - output_prechecker) + abs_path, False, output_prechecker, + exclude_path=path_to_exclude) success_file, copied_file = copy_file(output_prechecker, output_path) if success_file: temp_output_fiiles.append(copied_file) @@ -150,12 +150,15 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False, -1, source_analysis, abs_path, src_output, - False, num_cores, False) + False, num_cores, False, + path_to_exclude=path_to_exclude) else: # Run fosslight_source by using docker image src_output = os.path.join("output", output_files["SRC"]) output_rel_path = os.path.relpath(abs_path, os.getcwd()) command = shlex.quote(f"docker run -it -v {_output_dir}:/app/output " f"fosslight -p {output_rel_path} -o {src_output}") + if path_to_exclude: + command += f" -e {' '.join(path_to_exclude)}" command_result = subprocess.run(command, stdout=subprocess.PIPE, text=True) logger.info(f"Source Analysis Result:{command_result.stdout}") @@ -168,7 +171,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False, abs_path, os.path.join(_output_dir, output_files["BIN"]), "", db_url, False, - correct_mode, correct_fpath) + correct_mode, correct_fpath, + path_to_exclude=path_to_exclude) if success: output_binary_txt_raw = f"{output_files['BIN'].split('.')[0]}.txt" success_file, copied_file = copy_file(os.path.join(_output_dir, output_binary_txt_raw), @@ -177,7 +181,7 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False, temp_output_fiiles.append(copied_file) if run_dep: - run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]), dep_arguments) + run_dependency(src_path, os.path.join(_output_dir, output_files["DEP"]), dep_arguments, path_to_exclude) else: return @@ -191,7 +195,8 @@ def run_scanner(src_path, dep_arguments, output_path, keep_raw_data=False, merge_files = [output_files["SRC"], output_files["BIN"], output_files["DEP"]] cover = CoverItem(tool_name=PKG_NAME, start_time=_start_time, - input_path=abs_path) + input_path=abs_path, + exclude_path=path_to_exclude) cover.comment = merge_cover_comment(_output_dir, merge_files) if output_extension == ".xlsx": @@ -307,8 +312,9 @@ def init(output_path="", make_outdir=True): return os.path.isdir(_output_dir), output_root_dir, result_log -def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format, url_to_analyze, db_url, - hide_progressbar=False, keep_raw_data=False, num_cores=-1, correct_mode=True, correct_fpath="", ui_mode=False): +def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format, url_to_analyze, + db_url, hide_progressbar=False, keep_raw_data=False, num_cores=-1, + correct_mode=True, correct_fpath="", ui_mode=False, path_to_exclude=[]): global _executed_path, _start_time output_file = "" @@ -417,7 +423,7 @@ def run_main(mode_list, path_arg, dep_arguments, output_file_or_dir, file_format remove_downloaded_source, {}, output_file, output_extension, num_cores, db_url, default_oss_name, default_oss_version, url_to_analyze, - correct_mode, correct_fpath, ui_mode) + correct_mode, correct_fpath, ui_mode, path_to_exclude) else: logger.error("No mode has been selected for analysis.") try: diff --git a/tox.ini b/tox.ini index bcfb490..18f83f9 100644 --- a/tox.ini +++ b/tox.ini @@ -25,9 +25,11 @@ norecursedirs = test_result_* [testenv:test_run] commands = rm -rf test_result_local_path + rm -rf test_result_exclude_path rm -rf test_result_wget fosslight -o test_result_local_path/test.xlsx -p tests -r fosslight binary source -o test_result_multi_mode/test.xlsx -p tests -r + fosslight -o test_result_exclude_path/test.xlsx -p tests -e test sample_license.txt fosslight dependency -o test_result_wget -w "https://github.com/LGE-OSS/example.git" ls test_result_wget @@ -39,5 +41,6 @@ commands = fosslight -h fosslight all -o test_result_local_path/test.xlsx -p tests -r fosslight binary dependency -o test_result_multi_mode/test.xlsx -p tests -r + fosslight -o test_result_exclude_path/test.xlsx -p tests -e test sample_license.txt fosslight source -o test_result_wget -w "https://github.com/LGE-OSS/example.git" pytest -v --flake8