Rework regex list

.gitignore

@@ -51,3 +51,4 @@ src/build/
 .vscode
 todo
 src/main/java/com/cys4/sensitivediscoverer/tests
+/tests

src/main/resources/regexes/regex_general.jsonc

@@ -28,18 +28,6 @@
       "{'my_super_secret' : 'ABCDEFGH-12'}"
     ]
   },
-//  {
-//    // Try to match only IPs and not section numbers of js libraries
-//    "active": true,
-//    "description": "IP Address",
-//    "regex": "(?<![\\.\\da-zA-Z%\\-])(?<!\\/\\s{0,8})((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?![\\.\\da-zA-Z\\-])",
-//    "tests":[
-//      "0.0.0.0",
-//      "10.255.255.255",
-//      "127.0.0.1",
-//      "192.168.0.254/24"
-//    ]
-//  },
   {
     "active": true,
     "description": "Environment configuration file",
@@ -48,5 +36,49 @@
       "{\"env_file\":\"../.env\"}",
       "production.env"
     ]
+  },
+  {
+      "active": true,
+      "description": "Private IPv4 address",
+      "regex": "1(0(\\.[0-2]\\d{0,2}){3}|27(\\.[0-2]\\d{0,2}){3}|92\\.168(\\.[0-2]\\d{0,2}){2}|72\\.(1[6-9]|2\\d|3[0-2])(\\.[0-2]\\d{0,2}){2})(?![\\d.])",
+      "urls": [
+        "https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml"
+      ],
+      "tests": [
+        "127.0.0.1",
+        "127.255.255.254",
+        "10.0.0.1",
+        "10.255.255.254",
+        "172.16.0.1",
+        "172.32.255.254",
+        "192.168.0.1",
+        "192.168.255.254",
+        "10.1.10.100",
+        "10.2.20.200",
+        "!172.33.0.1",
+        "!172.15.0.1",
+        "!10.0.0.1.1",
+        "!10.255.255.2550",
+        "!10.999.999.999"
+      ]
+  },
+  {
+    "active": true,
+    "description": "Email",
+    "regex": "[a-zA-Z0-9]@[a-zA-Z0-9\\-\\.]{3,128}\\.[a-zA-Z0-9]{2,32}(?![\\w\\\\])",
+    "refinerRegex": "[\\w\\-\\.+]{1,128}$",
+    "urls": [
+      "https://learn.microsoft.com/en-us/exchange/recipients-in-exchange-online/plus-addressing-in-exchange-online"
+    ],
+    "tests": [
+      "username@domain.com",
+      "user-nam_e@domain.com",
+      "user-nam_e+user-nam_e@domain.com",
+      "sean+newsletter@contoso.com",
+      "test2+x@a.b.s23",
+      "1@domain.mail",
+      "!username@domain.com\\",
+      "!@domain.com"
+    ]
   }
 ]

src/main/resources/regexes/regex_token.jsonc

@@ -15,25 +15,42 @@
     AROA    | Role
     ASCA    | Certificate
     ASIA    | Temporary (AWS STS) access key IDs use this prefix, but are unique only in combination with the secret access key and the session token.
+    //TODO find sources on keys starting with "A3T[A-Z0-9]"
     */
     "active": true,
     "description": "AWS Access Key ID",
-    //TODO length should be {16}, but some examples have {17}
-    //TODO what is A3T. ?
-    "regex": "(?i)(A3T[A-Z0-9]|ABIA|ACCA|AGPA|AIDA|AIPA|AKIA|ANPA|ANVA|APKA|AROA|ASCA|ASIA)[a-zA-Z0-9]{14,20}(?![a-zA-Z0-9+/=])",
+    "regex": "A(BIA|CCA|GPA|I(DA|PA)|KIA|N(PA|VA)|PKA|ROA|S(CA|IA))[a-zA-Z0-9]{16,17}(?![a-zA-Z0-9+/=])",
     "tests": [
+      // check all prefixes
+      "ABIA000000000EXAMPLE",
+      "ACCA000000000EXAMPLE",
+      "AGPA000000000EXAMPLE",
+      "AIDA000000000EXAMPLE",
+      "AIPA000000000EXAMPLE",
+      "AKIA000000000EXAMPLE",
+      "ANPA000000000EXAMPLE",
+      "ANVA000000000EXAMPLE",
+      "APKA000000000EXAMPLE",
+      "AROA000000000EXAMPLE",
+      "ASCA000000000EXAMPLE",
+      "ASIA000000000EXAMPLE",
+      // key samples with different lengths
       "AKIAIOSFODNN7EXAMPLE",
       "ASIAIOSFODNN7EXAMPLE",
       "AIDACKCEVSQ6C2EXAMPLE",
       "AROADBQP57FF2AEXAMPLE",
       "AIDAJQABLZS4A3QDU576Q",
       "AKIAIU5C4FEZYEXAMPLE",
-      "AKIA2EKLCMMJCE6KTO42"
+      "AKIA2EKLCMMJCE6KTO42",
+      // check false positives
+      "\"AKIAIOSFODNN7EXAMPLE\"",
+      "!AKIAIOSFODNN7EXAMPLE="
     ],
     "urls": [
-      "https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html",
+      "https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html", // token length 16-17
       "https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html",
-      "https://docs.gitguardian.com/secrets-detection/secrets-detection-engine/detectors/specifics/aws_ses_keys"
+      "https://docs.gitguardian.com/secrets-detection/secrets-detection-engine/detectors/specifics/aws_ses_keys",
+      "https://docs.aws.amazon.com/STS/latest/APIReference/API_GetAccessKeyInfo.html" // AKIA tokens
     ]
   },
   {
@@ -63,24 +80,6 @@
       "https://amazonpaylegacyintegrationguide.s3.amazonaws.com/docs/eu/amazon-pay-automatic/add-address-and-wallet-widgets.html"
     ]
   },
-  //todo
-  //  {
-  //    // Do not match "https://facebook.com" or fa-facebook
-  //    "active": true,
-  //    "description": "Facebook Secret",
-  //    "regex": "(?i)(?<!https://)(?<!fa-)(facebook)([^;,]{0,32}?)['\\\"][\\w\\d]{10,}?['\\\"]",
-  //    "tests": [
-  //      "!fa-facebook",
-  //      "!https://facebook.com"
-  //    ]
-  //  },
-  //todo
-  //  {
-  //    "active": true,
-  //    "description": "Facebook Page Access Token",
-  //    "urls": ["https://developers.facebook.com/docs/facebook-login/guides/access-tokens"],
-  //    "regex": "(EAACEdE\\w+|EAA\\w+)"
-  //  },
   {
     "active": true,
     "description": "Google API Key",
@@ -112,7 +111,8 @@
   {
     "active": true,
     "description": "Google OAuth Client ID",
-    "regex": "\\d{1,20}-\\w{32}\\.apps\\.googleusercontent\\.com",
+    "regex": "\\.apps\\.googleusercontent\\.com",
+    "refinerRegex": "\\d{1,20}-\\w{32}$",
     "tests": [
       "462858740426-tkpv8n03opijg7erd3s9ccued2pfllsd.apps.googleusercontent.com",
       "332292536747-zbepioq1127w4lv5r6dbbddd0jdbdchl.apps.googleusercontent.com",
@@ -136,29 +136,18 @@
       "https://docs.gitguardian.com/secrets-detection/secrets-detection-engine/detectors/specifics/google_oauth2_keys"
     ]
   },
-  //todo
-  //  {
-  //    "active": true,
-  //    "description": "LinkedIn Secret Key",
-  //    "regex": "(?i)linkedin([^;]{0,32}?)['\"][0-9a-zA-Z]{16}['\"]"
-  //  },
-  //todo
-  //  {
-  //    "active": true,
-  //    "description": "LinkedIn Client ID (API Key)",
-  //    "regex": "(?i)linkedin([^;]{0,32}?)(?-i)['\"][0-9a-z]{14}['\"]"
-  //  },
-  {
-    "active": true,
-    "description": "MailChimp API Key",
-    "regex": "[0-9a-f]{32}-us[0-9]{1,2}(?!\\d)",
-    "tests": [
-      "ae54fcc23ade65fa404a65e78c56f898-us1"
-    ],
-    "urls": [
-      "https://docs.gitguardian.com/secrets-detection/secrets-detection-engine/detectors/specifics/mailchimp_api_key"
-    ]
-  },
+//  {
+//    //todo: improve regex, now it's too slow with the initial "[0-9a-f]{32}",
+//    "active": false,
+//    "description": "MailChimp API Key",
+//    "regex": "[0-9a-f]{32}-us[0-9]{1,2}(?!\\d)",
+//    "tests": [
+//      "ae54fcc23ade65fa404a65e78c56f898-us1"
+//    ],
+//    "urls": [
+//      "https://docs.gitguardian.com/secrets-detection/secrets-detection-engine/detectors/specifics/mailchimp_api_key"
+//    ]
+//  },
   {
     "active": true,
     "description": "MailGun API Key",
@@ -183,18 +172,6 @@
       "https://docs.gitguardian.com/secrets-detection/secrets-detection-engine/detectors/specifics/nuget_api_key"
     ]
   },
-  //todo
-//  {
-//    "active": true,
-//    "description": "Braintree SDK Token",
-//    "regex": "(production|sandbox)_[a-z0-9]{8}_[a-z0-9]{16}",
-//    "tests": [
-//      "production_696htvk3y5xcfnsn_42cde001d7a240d98ded578b4c1321c9"
-//    ],
-//    "urls": [
-//      ""
-//    ]
-//  },
   {
     /*
     SG.ID.VALUE
@@ -225,7 +202,8 @@
     */
     "active": true,
     "description": "Slack Token",
-    "regex": "(xoxe\\.)?(xox[psboare]|xapp)(-[a-zA-Z0-9]{1,64}){1,5}",
+    "regex": "x(ox[psboare]|app)(-[a-zA-Z0-9]{1,64}){1,5}",
+    "refinerRegex": "xoxe\\.$",
     "urls": [
       "https://api.slack.com/authentication/token-types",
       "https://learn.microsoft.com/en-us/purview/sit-defn-slack-access-token",
@@ -274,25 +252,6 @@
       "sq0idp-Y7WFLOjXXp00XXXp00_0_A"
     ]
   },
-//  {
-//    "active": true,
-//    "description": "Square Auth Token",
-//    "regex": "(?<![a-zA-Z0-9\/+])EAAA[a-zA-Z0-9\\-_]{60}",
-//    "urls": [
-//      "https://developer.squareup.com/docs/build-basics/access-tokens"
-//    ],
-//    "tests": [
-//      "EAAAEB0XZqkuXAPRKhusNNaHhrZUsZNKw_ay92e-kmhYSDrKo73FgPX9RsrFr3Fg",
-//      "!RandomEAAAEB0XZqkuXAPRKhusNNaHhrZUsZNKw_ay92e-kmhYSDrKo73FgPX9RsrFr3Fg",
-//      "!/EAAAEB0XZqkuXAPRKhusNNaHhrZUsZNKw_ay92e-kmhYSDrKo73FgPX9RsrFr3Fg",
-//      "!+EAAAEB0XZqkuXAPRKhusNNaHhrZUsZNKw_ay92e-kmhYSDrKo73FgPX9RsrFr3Fg"
-//    ]
-//  },
-  //    {
-  //      "active": true,
-  //      "description": "StackHawk API Key",
-  //      "regex": "hawk(\\.[\\w\\-]{20})+"
-  //    },
   {
     "active": true,
     "description": "Stripe API Secret Key",
@@ -325,7 +284,7 @@
   {
     "active": true,
     "description": "Twilio Account SID",
-    "regex": "SK[0-9a-zA-Z]{32}(?![a-zA-Z0-9+/=])",
+    "regex": "SK[0-9a-zA-Z]{32}(?![a-zA-Z0-9+/=?_%)])",
     "urls": [
       "https://www.twilio.com/docs/iam/api-keys/api-key",
       "https://www.twilio.com/docs/glossary/what-is-a-sid"
@@ -334,26 +293,6 @@
       "SKa3aiydsctvbbgam9tpp75jsduidvwi9t"
     ]
   },
-  //todo
-  //  {
-  //    "active": true,
-  //    "description": "Twitter Access Token",
-  //    "regex": "(?i)twitter([^;,]{0,32}?)['\"]\\d+\\-[0-9a-zA-Z]{40}['\"]",
-  //    // oauth_token=
-  //    "urls": [
-  //      "https://developer.twitter.com/en/docs/authentication/oauth-1-0a/obtaining-user-access-tokens"
-  //    ]
-  //  },
-  //todo
-  //  {
-  //    "active": true,
-  //    "description": "Twitter Secret Key",
-  //    "regex": "(?i)twitter([^;,]{0,32}?)['\"][0-9a-zA-Z]{35,44}['\"]",
-  //    // oauth_token_secret=
-  //    "urls": [
-  //      "https://developer.twitter.com/en/docs/authentication/oauth-1-0a/obtaining-user-access-tokens"
-  //    ]
-  //  },
   {
     /*
     Token type                                                      | Prefix
@@ -392,5 +331,17 @@
     "tests": [
       "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"
     ]
+  },
+  {
+    "active": true,
+    "description": "OpenAI API Key",
+    "regex": "sk-[a-zA-Z0-9]{40,128}(?![\\w\\-])",
+    "urls": [
+      "https://docs.gitguardian.com/secrets-detection/secrets-detection-engine/detectors/specifics/openai_apikey"
+    ],
+    "tests": [
+      "sk-4dF6gH8jK9lM0nO1pQ2rS4tU6vW8xY0zA2C3E4G5I6J7K8L9M",
+      "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"
+    ]
   }
 ]