Add ability to scan Git repo URLs and GitHub users/orgs (#27)

* `scan` now supports a new `--git-url URL` option, allowing a Git repository
  to be scanned without manually cloning it. Fixes #8 and #14.
* `scan` now supports GitHub enumeration, allowing the repositories
  of a GitHub user or organization to be scanned in bulk
* Dockerfile now uses release profile instead of dev profile, and uses the `--locked` option
* The `reqwest` dependency now uses its `native-tls-vendored` feature
  to avoid a runtime dependency on openssl on linux

.github/workflows/ci.yml

@@ -47,12 +47,13 @@ jobs:
       if: matrix.os == 'ubuntu-22.04'
       run: |
         sudo apt-get install -y --no-install-recommends \
-          libhyperscan-dev
+          libhyperscan-dev \
+          pkg-config
 
     - name: Install system dependencies (macOS)
       if: matrix.os == 'macos-12'
       run: |
-        brew install hyperscan
+        brew install hyperscan pkg-config
 
     - name: Install Rust toolchain
       id: install-rust-toolchain
@@ -71,13 +72,18 @@ jobs:
         key: ${{ runner.os }}-${{ steps.install-rust-toolchain.cachekey }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('**/Cargo.lock') }}-tests
 
     - name: Build
-      run: cargo build --verbose --locked
+      run: cargo build --locked --verbose
 
     - name: Run tests
-      run: cargo test --verbose --locked
+      env:
+        # We use the GitHub Actions automatic token when running tests, to avoid
+        # spurious failures from rate limiting when testing Nosey Parker's github
+        # enumeration capabilities.
+        NP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: cargo test --locked --verbose
 
     - name: Check rules
-      run: cargo run -- rules check data/default/rules --warnings-as-errors
+      run: cargo run --locked -- rules check data/default/rules --warnings-as-errors
 
   docs:
     name: Docs
@@ -109,10 +115,10 @@ jobs:
     - name: Check documentation
       env:
         RUSTDOCFLAGS: -D warnings
-      run: cargo doc --no-deps --document-private-items
+      run: cargo doc --locked --no-deps --document-private-items
 
   release_build:
-    name: Release Build
+    name: Tests (release build; ubuntu-20.04.stable)
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v3
@@ -139,13 +145,18 @@ jobs:
         key: ${{ runner.os }}-${{ steps.install-rust-toolchain.cachekey }}-${{ hashFiles('**/Cargo.toml') }}-${{ hashFiles('**/Cargo.lock') }}-release_build
 
     - name: Build
-      run: cargo build --release --verbose --locked
+      run: cargo build --locked --release --verbose
 
     - name: Run tests
-      run: cargo test --release --verbose --locked
+      env:
+        # We use the GitHub Actions automatic token when running tests, to avoid
+        # spurious failures from rate limiting when testing Nosey Parker's github
+        # enumeration capabilities.
+        NP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: cargo test --locked --release --verbose
 
     - name: Check rules
-      run: cargo run --release -- rules check data/default/rules --warnings-as-errors
+      run: cargo run --locked --release -- rules check data/default/rules --warnings-as-errors
 
     - name: Upload release binary
       uses: actions/upload-artifact@v3

.github/workflows/docker-image.yml

@@ -9,6 +9,7 @@ on:
 
 jobs:
   build:
+    name: Build, Test, and (Optionally) Publish
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v3

.github/workflows/rust-clippy.yml

@@ -7,7 +7,7 @@
 # More details at https://github.com/rust-lang/rust-clippy
 # and https://rust-lang.github.io/rust-clippy/
 
-name: rust-clippy analyze
+name: Clippy
 
 on:
   push:
@@ -20,7 +20,7 @@ on:
 
 jobs:
   rust-clippy-analyze:
-    name: Run rust-clippy analyzing
+    name: Clippy Analysis
     runs-on: ubuntu-latest
     permissions:
       contents: read

CHANGELOG.md

@@ -8,9 +8,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ## Unreleased
 
 ### Additions
-- Nosey Parker now has rudimentary support for enumerating repositories from GitHub users and organizations.
+- The `scan` command can now be given Git https URLs, GitHub usernames, and GitHub organization names as inputs, and will enumerate, clone, and scan as appropriate ([#14](https://github.com/praetorian-inc/noseyparker/issues/14)).
+
+- Nosey Parker now has rudimentary support for enumerating repositories from GitHub users and organizations ([#15](https://github.com/praetorian-inc/noseyparker/issues/15)).
   The new `github repos list` command uses the GitHub REST API to enumerate repositories belonging to one or more users or organizations.
-  An optional GitHub Personal Access Token can be provided via the `GITHUB_TOKEN` environment variable.
+  An optional GitHub Personal Access Token can be provided via the `NP_GITHUB_TOKEN` environment variable.
 
 - Nosey Parker now has an optional `rule_profiling` crate feature that causes performance-related statistics to be collected and reported when scanning.
   This feature imposes some performance cost and is only useful to rule authors, and so is disabled by default.

Cargo.toml

@@ -57,7 +57,7 @@ pretty_assertions = "1.3"
 prettytable-rs = "0.10"
 rayon = "1.5"
 regex = "1.7"
-reqwest = { version = "0.11", features = ["json"] }
+reqwest = { version = "0.11", features = ["json", "native-tls-vendored"] }
 rlimit = "0.9.0"
 rusqlite = { version = "0.28", features = ["bundled", "backup"] }
 secrecy = "0.8.0"
@@ -73,7 +73,7 @@ url = "2.3"
 walkdir = "2.3"
 
 [dev-dependencies]
-assert_cmd = "2.0"
+assert_cmd = { version = "2.0", features = ["color-auto"] }
 assert_fs = "1.0"
 criterion = "0.4"
 escargot = "0.5"

Dockerfile

@@ -5,7 +5,7 @@ ARG VECTORSCAN_SHA=71fae7ee8d63e1513a6df762cdb5d5f02a9120a2422cf1f31d57747c2b8d3
 ################################################################################
 # Base stage
 ################################################################################
-FROM rust:$RUST_VER AS base
+FROM rust:$RUST_VER AS base_builder
 
 ARG VECTORSCAN_VER
 ARG VECTORSCAN_SHA
@@ -37,7 +37,13 @@ RUN apt-get update &&\
 ################################################################################
 # Build Rust dependencies, caching stage
 ################################################################################
-FROM base AS dependencies
+# This stage exists so that dependencies of Nosey Parker can be preserved in
+# the Docker cache.
+#
+# Building dependencies only is not naturally supported out-of-the box with
+# Cargo, and so requires some machinations.
+
+FROM base_builder AS dependencies_builder
 
 WORKDIR "/noseyparker"
 
@@ -57,12 +63,12 @@ RUN mkdir -p  ./src/bin/noseyparker &&\
     # Stub main required for compile
     echo "fn main() {}" > ./src/bin/noseyparker/main.rs &&\
     # Run the build
-    cargo build --release
+    cargo build --release --profile release --locked
 
 ################################################################################
 # Build application
 ################################################################################
-FROM dependencies AS build
+FROM dependencies_builder AS app_builder
 
 WORKDIR "/noseyparker"
 
@@ -76,14 +82,14 @@ RUN touch \
     ./src/lib.rs \
     ./src/bin/noseyparker/main.rs
 
-RUN cargo build --release
+RUN cargo install --root /usr/local --profile release --locked --path .
 
 ################################################################################
 # Build a smaller image just for running the `noseyparker` binary
 ################################################################################
 FROM debian:11-slim
 
-COPY --from=build /noseyparker/target/release/noseyparker /usr/bin/noseyparker
+COPY --from=app_builder /usr/local/bin/noseyparker /usr/local/bin/noseyparker
 
 # Tip when running: use a volume mount: `-v "$PWD:/scan"` to make for handling of paths on the command line
 WORKDIR "/scan"

README.md

@@ -211,7 +211,7 @@ https://github.com/octocat/octocat.github.io.git
 https://github.com/octocat/test-repo1.git
 ```
 
-An optional GitHub Personal Access Token can be provided via the `GITHUB_TOKEN` environment variable.
+An optional GitHub Personal Access Token can be provided via the `NP_GITHUB_TOKEN` environment variable.
 Providing an access token gives a higher API rate limit and may make additional repositories accessible to you.
 
 Additional output formats are supported, including JSON and JSON lines, via the `--format=FORMAT` option.

data/default/rules/github.yml

@@ -30,6 +30,7 @@ rules:
 
 
 - name: GitHub App Token
+  # Note: `ghu_` prefix is for user-to-server tokens; `ghs_` is for server-to-server tokens
   pattern: '\b((?:ghu|ghs)_[a-zA-Z0-9]{36})\b'
 
   references:
@@ -39,6 +40,9 @@ rules:
 
   examples:
   - '            "token": "ghu_16C7e42F292c69C2E7C10c838347Ae178B4a",'
+  - |
+      Example usage:
+      git clone http://ghs_RguXIkihJjwHAP6eXEYxaPNvywurTr5IOAbg@github.com/username/repo.git
 
 
 - name: GitHub Refresh Token