Add support for custom GitHub API urls (for GHES support) (#53)

This adds a new `--github-api-url URL` parameter (aliased as `--api-url`) to the `scan` and `github` commands, allowing GHES instances to be accessed in addition to the default GitHub API.

Co-authored-by: Brad Larsen <bradford.larsen@praetorian.com>

CHANGELOG.md

@@ -10,6 +10,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Additions
 - Running `noseyparker --version` now emits many compile-time details about the build, which can be useful for troubleshooting ([#48](https://github.com/praetorian-inc/noseyparker/issues/48)).
 
+- The `github` and `scan` commands now support accessing GitHub Enterprise Server instances using the new `--github-api-url URL` parameter ([#53](https://github.com/praetorian-inc/noseyparker/pull/53)).
+  Thank you @AdnaneKhan!
+
 ### Changes
 - Existing rules were modified to reduce both false positives and false negatives:
 
@@ -115,7 +118,7 @@ docker pull ghcr.io/praetorian-inc/noseyparker:v0.12.0
   - PyPI Upload Token
 
 - The `report` command now offers rudimentary SARIF support ([#4](https://github.com/praetorian-inc/noseyparker/issues/4)).
-  Thanks you @Coruscant11!
+  Thank you @Coruscant11!
 
 ### Changes
 - Several default rules have been revised to improve performance of the matching engine and to produce fewer false positives.

crates/noseyparker-cli/Cargo.toml

@@ -56,6 +56,7 @@ serde-sarif = "0.3.6"
 tracing = "0.1"
 tracing-log = "0.1"
 tracing-subscriber = { version = "0.3", features = ["tracing-log", "ansi"] }
+url = "2.3"
 vectorscan = { path = "../vectorscan" }
 
 [dev-dependencies]

crates/noseyparker-cli/src/bin/noseyparker/args.rs

@@ -1,6 +1,7 @@
 use anyhow::{Context, Result};
 use clap::{crate_description, crate_version, ArgAction, Args, Parser, Subcommand, ValueEnum};
 use std::path::PathBuf;
+use url::Url;
 
 use noseyparker::git_url::GitUrl;
 
@@ -211,6 +212,18 @@ impl std::fmt::Display for Mode {
 pub struct GitHubArgs {
     #[command(subcommand)]
     pub command: GitHubCommand,
+
+    /// Use the given URL for GitHub API access
+    ///
+    /// If accessing a GitHub Enterprise Server instance, this value should be the entire base URL
+    /// include the `api/v3` portion, e.g., `https://github.example.com/api/v3`.
+    #[arg(
+        long,
+        value_name = "URL",
+        default_value_t = Url::parse("https://api.github.com").expect("default API url should parse"),
+        visible_alias="api-url"
+    )]
+    pub github_api_url: Url,
 }
 
 #[derive(Subcommand, Debug)]
@@ -359,14 +372,28 @@ pub struct ScanInputArgs {
     #[arg(long, value_name = "NAME", display_order = 20)]
     pub github_user: Vec<String>,
 
-    /// Name of a  GitHub organization to enumerate and scan
+    /// Name of a GitHub organization to enumerate and scan
     #[arg(
         long,
         visible_alias = "github-org",
         value_name = "NAME",
         display_order = 20
     )]
     pub github_organization: Vec<String>,
+
+    /// Use the given URL for GitHub API access
+    ///
+    /// If accessing a GitHub Enterprise Server instance, this value should be the entire base URL
+    /// include the `api/v3` portion, e.g., `https://github.example.com/api/v3`.
+    #[arg(
+        long,
+        visible_alias = "api-url",
+        value_name = "URL",
+        default_value_t = Url::parse("https://api.github.com").expect("default API url should parse"),
+        display_order = 30
+    )]
+    pub github_api_url: Url,
+
 }
 
 /// This struct represents options to control content discovery.

crates/noseyparker-cli/src/bin/noseyparker/cmd_github.rs

@@ -1,23 +1,24 @@
 use anyhow::{bail, Context, Result};
+use url::Url;
 
 use crate::args::{GitHubArgs, GitHubReposListArgs, GlobalArgs, Reportable};
 use noseyparker::github;
 
 pub fn run(global_args: &GlobalArgs, args: &GitHubArgs) -> Result<()> {
     use crate::args::{GitHubCommand::*, GitHubReposCommand::*};
     match &args.command {
-        Repos(List(args)) => list_repos(global_args, args),
+        Repos(List(args_list)) => list_repos(global_args, args_list, args.github_api_url.clone()),
     }
 }
 
-fn list_repos(_global_args: &GlobalArgs, args: &GitHubReposListArgs) -> Result<()> {
+fn list_repos(_global_args: &GlobalArgs, args: &GitHubReposListArgs, api_url: Url) -> Result<()> {
     if args.repo_specifiers.is_empty() {
         bail!("No repositories specified");
     }
     let repo_urls = github::enumerate_repo_urls(&github::RepoSpecifiers {
         user: args.repo_specifiers.user.clone(),
         organization: args.repo_specifiers.organization.clone(),
-    }, None)
+    }, api_url, None)
     .context("Failed to enumerate GitHub repositories")?;
     RepoReporter(repo_urls).report(&args.output_args)
 }

crates/noseyparker-cli/src/bin/noseyparker/cmd_scan.rs

@@ -80,7 +80,8 @@ pub fn run(global_args: &args::GlobalArgs, args: &args::ScanArgs) -> Result<()>
                 progress_enabled,
             );
             let mut num_found: u64 = 0;
-            for repo_string in github::enumerate_repo_urls(&repo_specifiers, Some(&mut progress))
+            let api_url = args.input_args.github_api_url.clone();
+            for repo_string in github::enumerate_repo_urls(&repo_specifiers, api_url, Some(&mut progress))
                 .context("Failed to enumerate GitHub repositories")?
             {
                 match GitUrl::from_str(&repo_string) {

crates/noseyparker-cli/tests/help/snapshots/test_noseyparker__help__help_github-2.snap

@@ -1,5 +1,5 @@
 ---
-source: tests/test_noseyparker_help.rs
+source: crates/noseyparker-cli/tests/help/mod.rs
 expression: stdout
 ---
 Interact with GitHub
@@ -17,6 +17,15 @@ Commands:
           Print this message or the help of the given subcommand(s)
 
 Options:
+      --github-api-url <URL>
+          Use the given URL for GitHub API access
+
+          If accessing a GitHub Enterprise Server instance, this value should be the entire base URL
+          include the `api/v3` portion, e.g., `https://github.example.com/api/v3`.
+
+          [default: https://api.github.com/]
+          [aliases: api-url]
+
   -h, --help
           Print help (see a summary with '-h')
 

crates/noseyparker-cli/tests/help/snapshots/test_noseyparker__help__help_github_short-2.snap

@@ -1,5 +1,5 @@
 ---
-source: tests/test_noseyparker_help.rs
+source: crates/noseyparker-cli/tests/help/mod.rs
 expression: stdout
 ---
 Interact with GitHub
@@ -11,7 +11,9 @@ Commands:
   help   Print this message or the help of the given subcommand(s)
 
 Options:
-  -h, --help  Print help (see more with '--help')
+      --github-api-url <URL>  Use the given URL for GitHub API access [default:
+                              https://api.github.com/] [aliases: api-url]
+  -h, --help                  Print help (see more with '--help')
 
 Global Options:
   -v, --verbose...       Enable verbose output

crates/noseyparker-cli/tests/help/snapshots/test_noseyparker__help__help_scan-2.snap

@@ -1,5 +1,5 @@
 ---
-source: tests/test_noseyparker_help.rs
+source: crates/noseyparker-cli/tests/help/mod.rs
 expression: stdout
 ---
 Scan content for secrets
@@ -72,13 +72,22 @@ Input Specifier Options:
           supported.
 
       --github-organization <NAME>
-          Name of a  GitHub organization to enumerate and scan
+          Name of a GitHub organization to enumerate and scan
 
           [aliases: github-org]
 
       --github-user <NAME>
           Name of a GitHub user to enumerate and scan
 
+      --github-api-url <URL>
+          Use the given URL for GitHub API access
+
+          If accessing a GitHub Enterprise Server instance, this value should be the entire base URL
+          include the `api/v3` portion, e.g., `https://github.example.com/api/v3`.
+
+          [default: https://api.github.com/]
+          [aliases: api-url]
+
 Content Discovery Options:
       --max-file-size <MEGABYTES>
           Do not scan files larger than the specified size

crates/noseyparker-cli/tests/help/snapshots/test_noseyparker__help__help_scan_short-2.snap

@@ -1,5 +1,5 @@
 ---
-source: tests/test_noseyparker_help.rs
+source: crates/noseyparker-cli/tests/help/mod.rs
 expression: stdout
 ---
 Scan content for secrets
@@ -15,9 +15,11 @@ Options:
 Input Specifier Options:
   [INPUT]...                    Path to a file, directory, or local Git repository to scan
       --git-url <URL>               URL of a Git repository to clone and scan
-      --github-organization <NAME>  Name of a  GitHub organization to enumerate and scan [aliases:
+      --github-organization <NAME>  Name of a GitHub organization to enumerate and scan [aliases:
                                     github-org]
       --github-user <NAME>          Name of a GitHub user to enumerate and scan
+      --github-api-url <URL>        Use the given URL for GitHub API access [default:
+                                    https://api.github.com/] [aliases: api-url]
 
 Content Discovery Options:
       --max-file-size <MEGABYTES>  Do not scan files larger than the specified size [default: 100]

crates/noseyparker/src/github.rs

@@ -1,3 +1,5 @@
+use url::Url;
+
 mod auth;
 mod client;
 mod client_builder;
@@ -19,11 +21,17 @@ use crate::progress::Progress;
 ///
 /// This is a high-level wrapper for enumerating GitHub repositories that handles the details of
 /// creating an async runtime and a GitHub REST API client.
-pub fn enumerate_repo_urls(repo_specifiers: &RepoSpecifiers, progress: Option<&mut Progress>) -> anyhow::Result<Vec<String>> {
+pub fn enumerate_repo_urls(
+    repo_specifiers: &RepoSpecifiers,
+    github_url: Url,
+    progress: Option<&mut Progress>,
+) -> anyhow::Result<Vec<String>> {
     use anyhow::{bail, Context};
     use tracing::{debug, warn};
 
     let client = ClientBuilder::new()
+        .base_url(github_url)
+        .context("Failed to set base URL")?
         .personal_access_token_from_env()
         .context("Failed to load access token from environment")?
         .build()
@@ -45,7 +53,9 @@ pub fn enumerate_repo_urls(repo_specifiers: &RepoSpecifiers, progress: Option<&m
         debug!("GitHub rate limits: {:?}", rate_limit.rate);
 
         let repo_enumerator = RepoEnumerator::new(&client);
-        let repo_urls = repo_enumerator.enumerate_repo_urls(repo_specifiers, progress).await?;
+        let repo_urls = repo_enumerator
+            .enumerate_repo_urls(repo_specifiers, progress)
+            .await?;
         Ok(repo_urls) // ::<Vec<String>, Error>(repo_urls)
     });