feat(proxy): add --proxy-domains flag for domain-specific proxying wi…

…th wildcard support (#582)

- Rename “proxy-internal” to “proxy-target”.
- Rename “proxy-external” to “proxy-bright”. 
- Add a new “proxy-domains” flag as an optional flag. It can be accepted
when the user is setting either “proxy-target” or “proxy”. It can
receive a comma-separated list of domains which will be proxied. Any
domain that is not part of the list won’t be proxied. Notes: Support for
wildcard is required. If I specify “*.foo.bar” as one of the domains,
“a.foo.bar”, “b.foo.bar”, etc. will all be proxied.
- Bright domains will always be proxied when using with the “proxy”
flag, without a need to have them explicitly listed as part of the
“proxy-domains” list. if the user doesn’t want to proxy bright domains,
he should use only "proxy-target"

src/Commands/GetEntryPoints.ts

@@ -32,7 +32,7 @@ export class GetEntryPoints implements CommandModule {
             insecure: args.insecure as boolean,
             baseURL: args.api as string,
             apiKey: args.token as string,
-            proxyURL: (args.proxyExternal ?? args.proxy) as string
+            proxyURL: (args.proxyBright ?? args.proxy) as string
           }
         })
       );

src/Commands/PollingScanStatus.ts

@@ -53,7 +53,7 @@ export class PollingScanStatus implements CommandModule {
             insecure: args.insecure as boolean,
             baseURL: args.api as string,
             apiKey: args.token as string,
-            proxyURL: (args.proxyExternal ?? args.proxy) as string
+            proxyURL: (args.proxyBright ?? args.proxy) as string
           }
         })
       );

src/Commands/RetestScan.ts

@@ -27,7 +27,7 @@ export class RetestScan implements CommandModule {
             insecure: args.insecure as boolean,
             baseURL: args.api as string,
             apiKey: args.token as string,
-            proxyURL: (args.proxyExternal ?? args.proxy) as string
+            proxyURL: (args.proxyBright ?? args.proxy) as string
           }
         })
       );

src/Commands/RunRepeater.ts

@@ -131,6 +131,12 @@ export class RunRepeater implements CommandModule {
         alias: ['rm', 'remove'],
         describe: 'Stop and remove repeater daemon'
       })
+      .option('proxy-domains', {
+        requiresArg: true,
+        array: true,
+        describe:
+          'Comma-separated list of domains that should be routed through the proxy. This option is only applicable when using the --proxy option'
+      })
       .conflicts({
         daemon: 'remove-daemon',
         ntlm: [
@@ -165,7 +171,7 @@ export class RunRepeater implements CommandModule {
             useValue: {
               headers: (args.header ?? args.headers) as Record<string, string>,
               timeout: args.timeout as number,
-              proxyUrl: (args.proxyInternal ?? args.proxy) as string,
+              proxyUrl: (args.proxyTarget ?? args.proxy) as string,
               certs: args.cert as Cert[],
               maxContentLength: 100,
               reuseConnection:
@@ -185,7 +191,8 @@ export class RunRepeater implements CommandModule {
                 'application/msgpack',
                 'application/ld+json',
                 'application/graphql'
-              ]
+              ],
+              proxyDomains: args.proxyDomains as string[]
             }
           })
           .register<DefaultRepeaterServerOptions>(
@@ -195,7 +202,7 @@ export class RunRepeater implements CommandModule {
                 uri: args.repeaterServer as string,
                 token: args.token as string,
                 connectTimeout: 10000,
-                proxyUrl: (args.proxyExternal ?? args.proxy) as string,
+                proxyUrl: (args.proxyBright ?? args.proxy) as string,
                 insecure: args.insecure as boolean
               }
             }

src/Commands/RunScan.ts

@@ -172,7 +172,7 @@ export class RunScan implements CommandModule {
             insecure: args.insecure as boolean,
             baseURL: args.api as string,
             apiKey: args.token as string,
-            proxyURL: (args.proxyExternal ?? args.proxy) as string
+            proxyURL: (args.proxyBright ?? args.proxy) as string
           }
         })
       );

src/Commands/StopScan.ts

@@ -27,7 +27,7 @@ export class StopScan implements CommandModule {
             insecure: args.insecure as boolean,
             baseURL: args.api as string,
             apiKey: args.token as string,
-            proxyURL: (args.proxyExternal ?? args.proxy) as string
+            proxyURL: (args.proxyBright ?? args.proxy) as string
           }
         })
       );

src/Commands/UploadArchive.ts

@@ -86,7 +86,7 @@ export class UploadArchive implements CommandModule {
             insecure: args.insecure as boolean,
             baseURL: args.api as string,
             apiKey: args.token as string,
-            proxyURL: (args.proxyExternal ?? args.proxy) as string
+            proxyURL: (args.proxyBright ?? args.proxy) as string
           }
         });
       });

src/Config/CliBuilder.ts

@@ -64,12 +64,14 @@ export class CliBuilder {
         describe:
           'Specify a proxy URL to route all traffic through. This should be an HTTP(S), SOCKS4, or SOCKS5 URL. By default, if you specify SOCKS://<URL>, then SOCKS5h is applied.'
       })
-      .option('proxy-external', {
+      .option('proxy-bright', {
+        alias: 'proxy-external',
         requiresArg: true,
         describe:
           'Specify a proxy URL to route all outbound traffic through. For more information, see the --proxy option.'
       })
-      .option('proxy-internal', {
+      .option('proxy-target', {
+        alias: 'proxy-internal',
         requiresArg: true,
         describe:
           'Specify a proxy URL to route all inbound traffic through. For more information, see the --proxy option.'

src/EntryPoint/RestEntryPoints.ts

@@ -11,6 +11,7 @@ export interface RestProjectsOptions {
   timeout?: number;
   insecure?: boolean;
   proxyURL?: string;
+  proxyDomains?: string[];
 }
 
 export const RestProjectsOptions: unique symbol = Symbol('RestProjectsOptions');

src/Repeater/DefaultRepeaterServer.ts

@@ -32,6 +32,7 @@ export interface DefaultRepeaterServerOptions {
   readonly connectTimeout?: number;
   readonly proxyUrl?: string;
   readonly insecure?: boolean;
+  readonly proxyDomains?: string[];
 }
 
 export const DefaultRepeaterServerOptions: unique symbol = Symbol(

src/RequestExecutor/HttpRequestExecutor.ts

@@ -1,7 +1,7 @@
 import { RequestExecutor } from './RequestExecutor';
 import { Response } from './Response';
 import { Request, RequestOptions } from './Request';
-import { logger, ProxyFactory } from '../Utils';
+import { Helpers, logger, ProxyFactory } from '../Utils';
 import { VirtualScripts } from '../Scripts';
 import { Protocol } from './Protocol';
 import { RequestExecutorOptions } from './RequestExecutorOptions';
@@ -39,6 +39,7 @@ export class HttpRequestExecutor implements RequestExecutor {
   private readonly httpsProxyAgent?: https.Agent;
   private readonly httpAgent?: http.Agent;
   private readonly httpsAgent?: https.Agent;
+  private readonly proxyDomains?: RegExp[];
 
   get protocol(): Protocol {
     return Protocol.HTTP;
@@ -65,6 +66,12 @@ export class HttpRequestExecutor implements RequestExecutor {
       this.httpsAgent = new https.Agent(agentOptions);
       this.httpAgent = new http.Agent(agentOptions);
     }
+
+    if (this.options.proxyDomains) {
+      this.proxyDomains = this.options.proxyDomains.map((domain) =>
+        Helpers.wildcardToRegExp(domain)
+      );
+    }
   }
 
   public async execute(options: Request): Promise<Response> {
@@ -190,6 +197,15 @@ export class HttpRequestExecutor implements RequestExecutor {
   }
 
   private getRequestAgent(options: Request) {
+    if (
+      this.proxyDomains &&
+      !this.proxyDomains.some((domain) =>
+        domain.test(parseUrl(options.url).hostname)
+      )
+    ) {
+      return options.secureEndpoint ? this.httpsAgent : this.httpAgent;
+    }
+
     return options.secureEndpoint
       ? this.httpsProxyAgent ?? this.httpsAgent
       : this.httpProxyAgent ?? this.httpAgent;

src/RequestExecutor/RequestExecutorOptions.ts

@@ -8,6 +8,7 @@ export interface RequestExecutorOptions {
   whitelistMimes?: string[];
   maxContentLength?: number;
   reuseConnection?: boolean;
+  proxyDomains?: string[];
 }
 
 export const RequestExecutorOptions: unique symbol = Symbol(

src/Scan/RestScans.ts

@@ -22,6 +22,7 @@ export interface RestScansOptions {
   timeout?: number;
   insecure?: boolean;
   proxyURL?: string;
+  proxyDomains?: string[];
 }
 
 export const RestScansOptions: unique symbol = Symbol('RestScansOptions');