Improve rule loader, rules list, and CLI help

CHANGELOG.md

@@ -18,6 +18,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - The vendored copy of the Vectorscan regular expression library included in the internal `vectorscan-sys` crate has been removed ([#151](https://github.com/praetorian-inc/noseyparker/pull/151) from @seqre).
   Instead, a copy of the Vectorscan 5.4.11 source tarball is included in this repository, and is extracted and patched during the build phase.
 - SARIF reporting format is now listed as experimental.
+- In the `scan` and `rules` command, the command-line option to load additional rules and rulesets from files has been renamed from `--rules` to `--rules-path`.
+  The old `--rules` option is still supported as an alias, but this is deprecated and will be removed in the v0.19 release.
+- The `rules list` command now includes additional fields when using JSON format ([#161](https://github.com/praetorian-inc/noseyparker/pull/161)).
 
 
 ## [v0.17.0](https://github.com/praetorian-inc/noseyparker/releases/v0.17.0) (2024-03-05)

crates/noseyparker-cli/src/args.rs

@@ -250,6 +250,10 @@ pub struct GlobalArgs {
     #[arg(global=true, long, default_value_t=Mode::Auto, value_name="MODE")]
     pub progress: Mode,
 
+    /// Ignore validation of TLS certificates
+    #[arg(long)]
+    pub ignore_certs: bool,
+
     #[command(flatten)]
     pub advanced: AdvancedArgs,
 }
@@ -363,13 +367,10 @@ pub struct GitHubReposListArgs {
 
     #[command(flatten)]
     pub output_args: OutputArgs<GitHubOutputFormat>,
-
-    /// Ignore validation of TLS certificates
-    #[arg(long)]
-    pub ignore_certs: bool,
 }
 
 #[derive(Args, Debug, Clone)]
+#[command(next_help_heading = "Input Specifier Options")]
 pub struct GitHubRepoSpecifiers {
     /// Select repositories belonging to the specified user
     ///
@@ -557,10 +558,6 @@ pub struct ScanArgs {
         help_heading="Data Collection Options",
     )]
     pub copy_blobs: CopyBlobsMode,
-
-    /// Ignore validation of TLS certificates
-    #[arg(long)]
-    pub ignore_certs: bool,
 }
 
 #[derive(Args, Debug)]
@@ -572,8 +569,10 @@ pub struct RuleSpecifierArgs {
     /// Directories are recursively walked and all discovered YAML files of rules and rulesets will be loaded.
     ///
     /// This option can be repeated.
-    #[arg(long, value_name = "PATH", value_hint = ValueHint::AnyPath)]
-    pub rules: Vec<PathBuf>,
+
+    // FIXME: remove deprecated `rules` alias in v0.19
+    #[arg(long, value_name = "PATH", value_hint = ValueHint::AnyPath, alias="rules")]
+    pub rules_path: Vec<PathBuf>,
 
     /// Enable the ruleset with the specified ID
     ///
@@ -588,6 +587,10 @@ pub struct RuleSpecifierArgs {
     /// If you want to use a custom ruleset in addition to the default ruleset, specify this option twice, e.g., `--ruleset default --ruleset CUSTOM_ID`.
     #[arg(long, value_name = "ID", default_values_t=["default".to_string()])]
     pub ruleset: Vec<String>,
+
+    /// Control whether built-in rules and rulesets are loaded.
+    #[arg(long, default_value_t=true, action=ArgAction::Set, value_name="BOOL")]
+    pub load_builtins: bool,
 }
 
 /// The mode to use for cloning a Git repository

crates/noseyparker-cli/src/cmd_github.rs

@@ -14,7 +14,7 @@ pub fn run(global_args: &GlobalArgs, args: &GitHubArgs) -> Result<()> {
     }
 }
 
-fn list_repos(_global_args: &GlobalArgs, args: &GitHubReposListArgs, api_url: Url) -> Result<()> {
+fn list_repos(global_args: &GlobalArgs, args: &GitHubReposListArgs, api_url: Url) -> Result<()> {
     if args.repo_specifiers.is_empty() {
         bail!("No repositories specified");
     }
@@ -26,7 +26,7 @@ fn list_repos(_global_args: &GlobalArgs, args: &GitHubReposListArgs, api_url: Ur
             all_organizations: args.repo_specifiers.all_organizations,
         },
         api_url,
-        args.ignore_certs,
+        global_args.ignore_certs,
         None,
     )
     .context("Failed to enumerate GitHub repositories")?;

crates/noseyparker-cli/src/cmd_rules/cmd_rules_list.rs

@@ -1,5 +1,5 @@
 use anyhow::{Context, Result};
-use noseyparker_rules::{Rule, RulesetSyntax};
+use noseyparker_rules::{Rule, RuleSyntax, RulesetSyntax};
 use serde::Serialize;
 use tracing::debug_span;
 
@@ -79,14 +79,18 @@ struct Entries<'r> {
 #[derive(Serialize)]
 struct RuleEntry<'r> {
     id: &'r str,
+    structural_id: &'r str,
     name: &'r str,
+    syntax: &'r RuleSyntax,
 }
 
 impl<'r> RuleEntry<'r> {
     pub fn new(rule: &'r Rule) -> Self {
         Self {
             id: rule.id(),
             name: rule.name(),
+            structural_id: rule.structural_id(),
+            syntax: rule.syntax(),
         }
     }
 }

crates/noseyparker-cli/src/cmd_scan.rs

@@ -115,7 +115,7 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()>
             for repo_string in github::enumerate_repo_urls(
                 &repo_specifiers,
                 api_url,
-                args.ignore_certs,
+                global_args.ignore_certs,
                 Some(&mut progress),
             )
             .context("Failed to enumerate GitHub repositories")?
@@ -160,7 +160,7 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()>
             args::GitCloneMode::Mirror => CloneMode::Mirror,
             args::GitCloneMode::Bare => CloneMode::Bare,
         };
-        let git = Git::new(args.ignore_certs);
+        let git = Git::new(global_args.ignore_certs);
 
         let mut progress =
             Progress::new_bar(repo_urls.len() as u64, "Fetching Git repos", progress_enabled);

crates/noseyparker-cli/src/rule_loader.rs

@@ -26,6 +26,11 @@ impl RuleLoader {
         }
     }
 
+    pub fn load_builtins(mut self, load_builtins: bool) -> Self {
+        self.load_builtins = load_builtins;
+        self
+    }
+
     /// Add additional file or directory paths to load rules and rulesets from.
     pub fn additional_rule_load_paths<P: AsRef<Path>, I: IntoIterator<Item = P>>(
         mut self,
@@ -85,7 +90,8 @@ impl RuleLoader {
 
     pub fn from_rule_specifiers(specs: &RuleSpecifierArgs) -> Self {
         Self::new()
-            .additional_rule_load_paths(specs.rules.as_slice())
+            .load_builtins(specs.load_builtins)
+            .additional_rule_load_paths(specs.rules_path.as_slice())
             .enable_ruleset_ids(specs.ruleset.iter())
     }
 }

crates/noseyparker-cli/tests/help/snapshots/test_noseyparker__help__help-2.snap

@@ -55,6 +55,9 @@ Global Options:
           [default: auto]
           [possible values: auto, never, always]
 
+      --ignore-certs
+          Ignore validation of TLS certificates
+
 Advanced Global Options:
       --rlimit-nofile <LIMIT>
           Set the rlimit for number of open files to LIMIT
@@ -80,4 +83,3 @@ Advanced Global Options:
 
           [default: true]
           [possible values: true, false]
-

crates/noseyparker-cli/tests/help/snapshots/test_noseyparker__help__help_scan-2.snap

@@ -55,7 +55,7 @@ Options:
           Print help (see a summary with '-h')
 
 Rule Selection Options:
-      --rules <PATH>
+      --rules-path <PATH>
           Load additional rules and rulesets from the specified file or directory
 
           The paths can be either files or directories. Directories are recursively walked and all
@@ -79,6 +79,12 @@ Rule Selection Options:
 
           [default: default]
 
+      --load-builtins <BOOL>
+          Control whether built-in rules and rulesets are loaded
+
+          [default: true]
+          [possible values: true, false]
+
 Input Specifier Options:
   [INPUT]...
           Scan the specified file, directory, or local Git repository
@@ -185,9 +191,6 @@ Metadata Collection Options:
             blob is first seen
           - minimal:    Only the Git repository in which a blob is seen
 
-      --ignore-certs
-          Ignore validation of TLS certificates
-
 Data Collection Options:
       --snippet-length <BYTES>
           Include up to the specified number of bytes before and after each match
@@ -271,4 +274,3 @@ Advanced Global Options:
 
           [default: true]
           [possible values: true, false]
-

crates/noseyparker-cli/tests/help/snapshots/test_noseyparker__help__help_scan_short-2.snap

@@ -12,8 +12,11 @@ Options:
   -h, --help              Print help (see more with '--help')
 
 Rule Selection Options:
-      --rules <PATH>  Load additional rules and rulesets from the specified file or directory
-      --ruleset <ID>  Enable the ruleset with the specified ID [default: default]
+      --rules-path <PATH>     Load additional rules and rulesets from the specified file or
+                              directory
+      --ruleset <ID>          Enable the ruleset with the specified ID [default: default]
+      --load-builtins <BOOL>  Control whether built-in rules and rulesets are loaded [default: true]
+                              [possible values: true, false]
 
 Input Specifier Options:
   [INPUT]...                    Scan the specified file, directory, or local Git repository
@@ -40,7 +43,6 @@ Metadata Collection Options:
                                     matching] [possible values: all, matching, none]
       --git-blob-provenance <MODE>  Specify which Git commit provenance metadata will be collected
                                     [default: first-seen] [possible values: first-seen, minimal]
-      --ignore-certs                Ignore validation of TLS certificates
 
 Data Collection Options:
       --snippet-length <BYTES>  Include up to the specified number of bytes before and after each
@@ -55,4 +57,3 @@ Global Options:
                          never, always]
       --progress <MODE>  Enable or disable progress bars [default: auto] [possible values: auto,
                          never, always]
-

crates/noseyparker-cli/tests/help/snapshots/test_noseyparker__help__help_short-2.snap

@@ -28,4 +28,4 @@ Global Options:
                          never, always]
       --progress <MODE>  Enable or disable progress bars [default: auto] [possible values: auto,
                          never, always]
-
+      --ignore-certs     Ignore validation of TLS certificates

crates/noseyparker-cli/tests/help/snapshots/test_noseyparker__help__no_args-3.snap

@@ -28,4 +28,4 @@ Global Options:
                          never, always]
       --progress <MODE>  Enable or disable progress bars [default: auto] [possible values: auto,
                          never, always]
-
+      --ignore-certs     Ignore validation of TLS certificates

crates/noseyparker-cli/tests/rules/mod.rs

@@ -26,3 +26,9 @@ fn rules_list_jsonl() {
 fn rules_check_builtins() {
     assert_cmd_snapshot!(noseyparker_success!("rules", "check", "--warnings-as-errors"));
 }
+
+/// Check that the `rules list --builtins false` option works as expected
+#[test]
+fn rules_list_no_builtins() {
+    assert_cmd_snapshot!(noseyparker_success!("rules", "list", "--load-builtins=false"));
+}