2.2.0 (#106)

* Refactor constants

* Add apikey-maybe detection

* BSD 3-Clause license

* Add XML case

* Update severity levels

* Update release notes

* Fix python 3.6 dependencies

* Bump dependencies

CONTRIBUTING.md

@@ -8,6 +8,6 @@ Take note of the build status of your pull request, only builds that pass will b
 
 ## License
 
-By contributing your code, you agree to license your contribution under the terms of the [GNU GPLv3.0](https://github.com/adeptex/whispers/blob/master/LICENSE)
+By contributing your code, you agree to license your contribution under the terms of the [BSD 3-Clause "New" or "Revised" License](https://github.com/adeptex/whispers/blob/master/LICENSE)
 
-All files are released with the **GNU General Public License v3.0**.
+All files are released with the **BSD 3-Clause "New" or "Revised" License**.

Makefile

@@ -1,8 +1,8 @@
 install:
-	pip3 install -e .
+	pip3 install -r requirements.txt -e .
 
 install-dev:
-	pip3 install -e ".[dev]"
+	pip3 install -r requirements.txt -e ".[dev]"
 
 flake8-lint:
 	flake8 whispers/ tests/

README.md

@@ -78,9 +78,9 @@ whispers dir/or/file | jq '.[].value'
 
 # Custom usage:
 #   - only check 'keys' rule group
-#   - with BLOCKER or CRITICAL severity
+#   - with Critical or High severity
 #   - everywhere in target/dir except for .log & .raw files (regex)
-whispers -g keys -s BLOCKER,CRITICAL -F '.*\.(log|raw)' target/dir
+whispers -g keys -s Critical,High -F '.*\.(log|raw)' target/dir
 ```
 
 ```bash
@@ -111,11 +111,11 @@ whispers --xgroups files dir/or/file
 ```
 
 ```bash
-# Include only BLOCKER & CRITICAL severity
-whispers --severity BLOCKER,CRITICAL dir/or/file
+# Include only Critical & High severity
+whispers --severity Critical,High dir/or/file
 
-# Exclude all MINOR severity
-whispers --xseverity MINOR dir/or/file
+# Exclude all Low severity
+whispers --xseverity Low dir/or/file
 ```
 
 ```bash
@@ -132,7 +132,7 @@ whispers --xfiles '.*\.(log|cfg)' dir/or/file
 ```py
 import whispers
 
-args = "-c whispers/config.yml -R file-known -S INFO tests/fixtures"
+args = "-c whispers/config.yml -R file-known -S Info tests/fixtures"
 
 for secret in whispers.secrets(args):
   print(f"[{secret.file}:{secret.line}] {secret.key} = {secret.value}")
@@ -173,14 +173,14 @@ exclude:
     - .*\.log
 ```
 
-Only scan for CRITICAL level findings in .npmrc files, excluding a known testing value:
+Only scan for  level findings in .npmrc files, excluding a known testing value:
 
 ```yaml
 include:
   files:
     - "**/*.npmrc"
   severity:
-    - CRITICAL
+    - High
 
 exclude:
   values: 
@@ -199,16 +199,16 @@ include:
     - uri
     - id: starks  # inline rule
       message: Whispers from the North
-      severity: CRITICAL
+      severity: Critical
       value:
         regex: (Aria|Ned) Stark
         ignorecase: True
   groups:
     - keys
   severity:
-    - CRITICAL
-    - BLOCKER
-    - MAJOR
+    - Critical
+    - High
+    - Medium
 
 exclude:
   files:
@@ -236,28 +236,29 @@ Simple filtering based on rules and severity can also be done with CLI arguments
 
 ## Rules
 
-| Group                | Rule ID              | Severity            |
-|----------------------|----------------------|---------------------|
-| files                | file-known           | MINOR               |
-| infra                | dockercfg            | CRITICAL            |
-| infra                | htpasswd             | MAJOR               |
-| infra                | npmrc                | CRITICAL            |
-| infra                | pip                  | CRITICAL            |
-| infra                | pypirc               | CRITICAL            |
-| keys                 | apikey               | MAJOR               |
-| keys                 | apikey-known         | CRITICAL            |
-| keys                 | aws-id               | BLOCKER             |
-| keys                 | aws-secret           | BLOCKER             |
-| keys                 | aws-token            | BLOCKER             |
-| keys                 | privatekey           | CRITICAL            |
-| misc                 | comment              | INFO                |
-| misc                 | creditcard           | MINOR               |
-| misc                 | secret               | MINOR               |
-| misc                 | webhook              | MINOR               |
-| passwords            | password             | CRITICAL            |
-| passwords            | uri                  | CRITICAL            |
-| python               | cors                 | MINOR               |
-| python               | system               | MINOR               |
+| Group                | Rule ID              | Severity        |
+|----------------------|----------------------|-----------------|
+| files                | file-known           | Low             |
+| infra                | dockercfg            | High            |
+| infra                | htpasswd             | Medium          |
+| infra                | npmrc                | High            |
+| infra                | pip                  | High            |
+| infra                | pypirc               | High            |
+| keys                 | apikey               | Medium          |
+| keys                 | apikey-known         | High            |
+| keys                 | apikey-maybe         | Low             |
+| keys                 | aws-id               | Critical        |
+| keys                 | aws-secret           | Critical        |
+| keys                 | aws-token            | Critical        |
+| keys                 | privatekey           | High            |
+| misc                 | comment              | Info            |
+| misc                 | creditcard           | Low             |
+| misc                 | secret               | Low             |
+| misc                 | webhook              | Low             |
+| passwords            | password             | High            |
+| passwords            | uri                  | High            |
+| python               | cors                 | Low             |
+| python               | system               | Low             |
 
 
 ### Custom rules
@@ -275,7 +276,7 @@ Rules specify the actual things that should be pulled out from key-value pairs.
   group: rule-group           # rule group name
   description: Values formatted like AWS Session Token
   message: AWS Session Token  # report will show this message
-  severity: BLOCKER           # one of BLOCKER, CRITICAL, MAJOR, MINOR, INFO
+  severity: Critical           # one of Critical, High, Medium, Low, Info
 
   key:                        # specify key format
     regex: (aws.?session.?token)?
@@ -325,4 +326,4 @@ make test
 
 ## License
 
-[GNU General Public License v3.0](https://github.com/adeptex/whispers/blob/master/LICENSE)
+[BSD 3-Clause License](https://github.com/adeptex/whispers/blob/master/LICENSE)

RELEASE_NOTES.md

@@ -1,36 +1,49 @@
-# Whispers 2.1.0 release notes
+# Whispers 2.2.0 release notes
 
-## :x: Breaking changes :x:
+* License change
+* Compatibility improvements
+    * Standardize severity levels
+    * Minor code refactor
+* Detection improvements
+    * Add XML cases
+    * Add `apikey-maybe` rule
 
-### :x: Arguments :x:
 
-Several arguments have been modified and/or adapted to improve usability.
 
-- Human readable output is shown in logs (2.1), `-H` and `--human` (2.0) are removed.
+## 💫 Licensing changes (again) 💫
 
-- Version can be shown with `--version` (2.1), `-v` (2.0) is removed.
+Version 2.1 was released under [GNU General Public License v3.0](https://github.com/adeptex/whispers/blob/3f5282ea3855d658ea37ec96dfc693598c16d7a7/LICENSE), which is `intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users.` 
 
-- Extended help can be shown with `--info` (2.1), `-i` (2.0) is removed.
+Version 2.2 is released under [BSD 3-Clause License](https://github.com/adeptex/whispers/blob/master/LICENSE), which is a permissive license that `prohibits others from using the name of the copyright holder or its contributors to promote derived products without written consent.` 
 
-- Debug mode can be enabled with `--debug` (2.1), `-d` (2.0) is removed.
+This change removes source code disclosure requirement 🕵️
 
-- Logs can be redirected to a file with `--log log.txt` (2.1), constant `/tmp/whispers.log` (2.0) is removed.
 
-- Configuration template can be created with `--init` (2.1), `--print_config` (2.0) is removed.
+## ❌ Breaking changes ❌
 
+### ❌ Severity levels ❌
 
-### :x: Logging :x:
+Severity level names have been adapted to a more common format. For example, `BLOCKER` is replaced by `Critical` and so on. The full list is as follows:
 
-**Version 2.0:** Opt-in logging for tracing execution flow, useful only for debugging. Results printed to `stdout` using `print()` as a JSON dict, one result per line. Enabling logging required adding the `--log` argument.
+| Version 2.1 (before) | Version 2.2 (now) |
+|---|---|
+| `BLOCKER` | `Critical` |
+| `CRITICAL` | `High` |
+| `MAJOR` | `Medium` |
+| `MINOR` | `Low` |
+| `INFO` | `Info` |
+
+⚠️ **Please update your custom rules and CLI args to reflect these changes** ⚠️
 
-**Version 2.1:** Logging is used to alert identified secrets during execution with `WARNING` level. Results are written to `stdout` as a JSON list at the end. This improves results parseability as a JSON list, while maintaining live results display that was previously achieved by printing secrets as JSON one per line.
+Instead of 
 
+> `whispers -s BLOCKER dir/or/file`
 
-## :white_check_mark: New features :white_check_mark:
+use
 
-### :white_check_mark: Results as JSON list :white_check_mark:
+> `whispers -s Critical dir/or/file`
 
-To improve integration and downstream processing, Whispers now outputs results as a JSON list of dictionaries with all detected secrets together (2.1), instead of one JSON dictionary per line (2.0). This list is directly loadable and parsable as JSON.
+See [README](https://github.com/adeptex/whispers#readme) for details and examples.
 
 
 # Changelog
@@ -39,3 +52,4 @@ To improve integration and downstream processing, Whispers now outputs results a
 |---|---|
 |2.0.0|https://github.com/adeptex/whispers/releases/tag/2.0.0|
 |2.1.0|https://github.com/adeptex/whispers/releases/tag/2.1.0|
+|2.2.0|https://github.com/adeptex/whispers/releases/tag/2.2.0|

requirements.txt

@@ -4,17 +4,23 @@
 #
 #    make freeze-upgrade
 #
-astroid==2.15.5
+astroid==2.15.6; python_version > '3.6'
+astroid==2.11.7; python_version <= '3.6'
 beautifulsoup4==4.12.2
-jellyfish==1.0.0
+dataclasses==0.8; python_version <= '3.6'
+jellyfish==1.0.1; python_version > '3.6'
+jellyfish==0.10.0; python_version <= '3.6'
 jproperties==2.1.1
-lazy-object-proxy==1.9.0
+lazy-object-proxy==1.9.0; python_version > '3.6'
+lazy-object-proxy==1.7.1; python_version <= '3.6'
 luhn==0.2.0
 lxml==4.9.3
-pyyaml==6.0
+pyyaml==6.0.1
 six==1.16.0
-soupsieve==2.4.1
-typing-extensions==4.7.1
+soupsieve==2.5; python_version > '3.6'
+soupsieve==2.3.2.post1; python_version <= '3.6'
+typing-extensions==4.8.0; python_version > '3.6'
+typing-extensions==4.1.1; python_version <= '3.6'
 wrapt==1.15.0
 
 # The following packages are considered to be unsafe in a requirements file:

setup.py

@@ -15,10 +15,6 @@
     "pyyaml",
 ]
 
-# Python 3.6 requirements
-if version_info < (3, 7):
-    install_requires += ["dataclasses"]
-
 dev_requires = [
     "autoflake~=1.4",
     "autopep8~=1.7",
@@ -61,4 +57,7 @@ def get_readme():
     tests_require=dev_requires,
     extras_require={"dev": dev_requires},
     entry_points={"console_scripts": ["whispers=whispers.main:main"]},
+    classifiers=[
+        "License :: BSD-3-Clause",
+    ],
 )

tests/configs/inline_rule.yml

@@ -16,7 +16,7 @@ include:
       group: test
       description: This rule is defined directly in the config file
       message: Inline Rule
-      severity: INFO
+      severity: Info
       key:
         regex: inline
         ignorecase: False

tests/fixtures/apikeys.json

@@ -15,6 +15,7 @@
       "GITHUBKEY=YXNkZmZmZmZm_HARDcoded",
       {"license_key": "YXNkZmZmZmZm_HARDcoded"}
     ],
-    "NUMERIC_APIKEY": 1925483168813050783076
+    "NUMERIC_APIKEY": 1925483168813050783076,
+    "bearer": [{"key": "token", "value": "hardcoded-bearer-v41u3"}]
   }
 }

tests/fixtures/apikeys.xml

@@ -15,5 +15,10 @@
     <environment>GITHUBKEY=YXNkZmZmZmZm_HARDcoded</environment>
     <license_key>YXNkZmZmZmZm_HARDcoded</license_key>
     <NUMERIC_APIKEY>1925483168813050783076</NUMERIC_APIKEY>
+    <bearer>
+      <key>token</key>
+      <value>hardcoded-bearer-v41u3-01</value>
+    </bearer>
+    <bearer key="token" value="hardcoded-bearer-v41u3-02" />
   </noncompliant>
 </tests>

tests/fixtures/apikeys.yml

@@ -18,4 +18,7 @@ noncompliant:
   environment:
     - GITHUBKEY=YXNkZmZmZmZm_HARDcoded
     - license_key: YXNkZmZmZmZm_HARDcoded
-  NUMERIC_APIKEY: 1925483168813050783076
+  NUMERIC_APIKEY: 1925483168813050783076
+  bearer:
+    - key: token
+      value: hardcoded-bearer-v41u3

tests/integration/test_whispers.py

@@ -10,7 +10,7 @@
         (f"-c {config_path('integration.yml')} {fixture_path()}", 5),
         (f"-r apikey-known {fixture_path('apikeys-known.yml')}", 56),
         (f"--rules file-known {fixture_path('files')}", 3),
-        (f"-s BLOCKER {fixture_path('aws.yml')}", 3),
+        (f"-s Critical {fixture_path('aws.yml')}", 3),
     ],
 )
 def test_whispers(args, expected):

tests/rules/multiple.yml

@@ -1,19 +1,19 @@
 - id: dup1
   description: This rule is ok
   message: Valid
-  severity: INFO
+  severity: Info
 
 - id: dup2
   description: This rule is duplicated
   message: Rule ID already exists
-  severity: CRITICAL
+  severity: High
 
 - id: dup3
   description: This rule is ok
   message: Valid
-  severity: INFO
+  severity: Info
 
 - id: dup4
   description: This rule is duplicated
   message: Rule ID already exists
-  severity: CRITICAL
+  severity: High

tests/rules/valid.yml

@@ -2,7 +2,7 @@
   group: test
   description: Valid rule config
   message: Valid
-  severity: INFO
+  severity: Info
   value:
     regex: ^test$
     ignorecase: False

tests/unit/conftest.py

@@ -19,7 +19,7 @@ def does_not_raise():
 
 @pytest.fixture
 def rule_fixture():
-    return Rule({"id": "fixture", "group": "tests", "message": "test", "severity": "INFO", "key": {}, "value": {}})
+    return Rule({"id": "fixture", "group": "tests", "message": "test", "severity": "Info", "key": {}, "value": {}})
 
 
 def fixture_path(filename: str = "") -> str:

tests/unit/core/test_args.py

@@ -75,7 +75,7 @@ def test_show_info():
     with patch("sys.stdout", mock_print):
         show_info()
         result = mock_print.getvalue()
-        expected = ["keys", "apikey-known", "CRITICAL"]
+        expected = ["keys", "apikey-known", "High"]
         for item in expected:
             assert item in result
 

tests/unit/core/test_rules.py

@@ -33,7 +33,7 @@ def test_load_rules_groups():
 
 
 def test_load_rules_severity():
-    args = parse_args(["-s", "BLOCKER,CRITICAL,MINOR", "-S", "MINOR", "tests/fixtures"])
+    args = parse_args(["-s", "Critical,High,Low", "-S", "Low", "tests/fixtures"])
     config = load_config(args)
     rules = load_rules(args, config)
     assert len(rules) == 11