From b3eb1344aeabb874ac47e8a40681d149c4c3884b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 27 May 2024 12:52:54 +0300 Subject: [PATCH 1/7] introduce timeout variants --- cmd/integration-test/library.go | 1 + cmd/nuclei/main.go | 2 -- internal/runner/runner.go | 1 + lib/multi.go | 1 + lib/sdk_private.go | 1 + pkg/protocols/common/protocolstate/state.go | 5 +-- .../http/httpclientpool/clientpool.go | 14 ++------ pkg/protocols/javascript/js_test.go | 19 +++++----- pkg/protocols/network/request.go | 4 +-- pkg/protocols/protocols.go | 1 + pkg/templates/compile_test.go | 19 +++++----- pkg/testutils/testutils.go | 24 ++++++------- pkg/tmplexec/flow/flow_executor_test.go | 19 +++++----- pkg/tmplexec/multiproto/multi_test.go | 19 +++++----- pkg/types/types.go | 35 ++++++++++++++++--- 15 files changed, 95 insertions(+), 70 deletions(-) diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go index 1324688e05..d341f576c6 100644 --- a/cmd/integration-test/library.go +++ b/cmd/integration-test/library.go @@ -112,6 +112,7 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) Colorizer: aurora.NewAurora(true), ResumeCfg: types.NewResumeCfg(), Parser: templates.NewParser(), + TimeoutVariants: defaultOpts.BuildTimeoutVariants(), } engine := core.New(defaultOpts) engine.SetExecuterOptions(executerOpts) diff --git a/cmd/nuclei/main.go b/cmd/nuclei/main.go index e93a4c4441..8878c93cba 100644 --- a/cmd/nuclei/main.go +++ b/cmd/nuclei/main.go @@ -296,7 +296,6 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.BoolVarP(&options.ShowMatchLine, "show-match-line", "sml", false, "show match lines for file templates, works with extractors only"), flagSet.BoolVar(&options.ZTLS, "ztls", false, "use ztls library with autofallback to standard one for tls13 [Deprecated] autofallback to ztls is enabled by default"), //nolint:all flagSet.StringVar(&options.SNI, "sni", "", "tls sni hostname to use (default: input domain name)"), - flagSet.DurationVarP(&options.DialerTimeout, "dialer-timeout", "dt", 0, "timeout for network requests."), flagSet.DurationVarP(&options.DialerKeepAlive, "dialer-keep-alive", "dka", 0, "keep-alive duration for network requests."), flagSet.BoolVarP(&options.AllowLocalFileAccess, "allow-local-file-access", "lfa", false, "allows file (payload) access anywhere on the system"), flagSet.BoolVarP(&options.RestrictLocalNetworkAccess, "restrict-local-network-access", "lna", false, "blocks connections to the local / private network"), @@ -305,7 +304,6 @@ on extensive configurability, massive extensibility and ease of use.`) flagSet.StringVarP(&options.SourceIP, "source-ip", "sip", "", "source ip address to use for network scan"), flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"), flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", 1*1024*1024, "max response size to read in bytes"), - flagSet.DurationVarP(&options.ResponseReadTimeout, "response-read-timeout", "rrt", time.Duration(5*time.Second), "response read timeout in seconds"), flagSet.CallbackVar(resetCallback, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"), flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"), flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"), diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 24010d7537..865acf0666 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -466,6 +466,7 @@ func (r *Runner) RunEnumeration() error { InputHelper: input.NewHelper(), TemporaryDirectory: r.tmpDir, Parser: r.parser, + TimeoutVariants: r.options.BuildTimeoutVariants(), } if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) { diff --git a/lib/multi.go b/lib/multi.go index 7c713e8aa0..764c300d11 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -40,6 +40,7 @@ func createEphemeralObjects(base *NucleiEngine, opts *types.Options) (*unsafeOpt Colorizer: aurora.NewAurora(true), ResumeCfg: types.NewResumeCfg(), Parser: base.parser, + TimeoutVariants: opts.BuildTimeoutVariants(), } if opts.RateLimitMinute > 0 { opts.RateLimit = opts.RateLimitMinute diff --git a/lib/sdk_private.go b/lib/sdk_private.go index cfe9c88a25..aef658ad72 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -166,6 +166,7 @@ func (e *NucleiEngine) init() error { ResumeCfg: types.NewResumeCfg(), Browser: e.browserInstance, Parser: e.parser, + TimeoutVariants: e.opts.BuildTimeoutVariants(), } if len(e.opts.SecretsFile) > 0 { authTmplStore, err := runner.GetAuthTmplStore(*e.opts, e.catalog, e.executerOpts) diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index 89675e7696..9b8e0b57f8 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -30,8 +30,9 @@ func Init(options *types.Options) error { lfaAllowed = options.AllowLocalFileAccess opts := fastdialer.DefaultOptions - if options.DialerTimeout > 0 { - opts.DialerTimeout = options.DialerTimeout + timeoutVariants := options.BuildTimeoutVariants() + if timeoutVariants.DialTimeout > 0 { + opts.DialerTimeout = timeoutVariants.DialTimeout } if options.DialerKeepAlive > 0 { opts.DialerKeepAlive = options.DialerKeepAlive diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 4200e25b78..8670c41d60 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -35,17 +35,11 @@ var ( forceMaxRedirects int normalClient *retryablehttp.Client clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] - // MaxResponseHeaderTimeout is the timeout for response headers - // to be read from the server (this prevents infinite hang started by server if any) - // Note: this will be overridden temporarily when using @timeout request annotation - MaxResponseHeaderTimeout = time.Duration(10) * time.Second - // HttpTimeoutMultiplier is the multiplier for the http timeout - HttpTimeoutMultiplier = 3 ) // GetHttpTimeout returns the http timeout for the client func GetHttpTimeout(opts *types.Options) time.Duration { - return time.Duration(opts.Timeout*HttpTimeoutMultiplier) * time.Second + return opts.BuildTimeoutVariants().HttpTimeout } // Init initializes the clientpool implementation @@ -54,9 +48,6 @@ func Init(options *types.Options) error { if normalClient != nil { return nil } - if options.Timeout > 10 { - MaxResponseHeaderTimeout = time.Duration(options.Timeout) * time.Second - } if options.ShouldFollowHTTPRedirects() { forceMaxRedirects = options.MaxRedirects } @@ -247,7 +238,8 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl } // responseHeaderTimeout is max timeout for response headers to be read - responseHeaderTimeout := MaxResponseHeaderTimeout + timeoutVariants := options.BuildTimeoutVariants() + responseHeaderTimeout := timeoutVariants.MaxResponseHeaderTimeout if configuration.ResponseHeaderTimeout != 0 { responseHeaderTimeout = configuration.ResponseHeaderTimeout } diff --git a/pkg/protocols/javascript/js_test.go b/pkg/protocols/javascript/js_test.go index efb78ef6e2..4074eff55f 100644 --- a/pkg/protocols/javascript/js_test.go +++ b/pkg/protocols/javascript/js_test.go @@ -32,15 +32,16 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(options.OmitTemplate), - Options: options, - Progress: progressImpl, - ProjectFile: nil, - IssuesClient: nil, - Browser: nil, - Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), - RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - Parser: templates.NewParser(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + Parser: templates.NewParser(), + TimeoutVariants: options.BuildTimeoutVariants(), } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 62807abf93..802d73539f 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -327,7 +327,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac } if input.Read > 0 { - buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.ResponseReadTimeout) + buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.TimeoutVariants.ResponseReadTimeout) if err != nil { return errorutil.NewWithErr(err).Msgf("could not read response from connection") } @@ -377,7 +377,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac bufferSize = -1 } - final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.Options.ResponseReadTimeout) + final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.TimeoutVariants.ResponseReadTimeout) if err != nil { request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) gologger.Verbose().Msgf("could not read more data from %s: %s", actualAddress, err) diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index 6b4904c8d8..d90ccf4738 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -120,6 +120,7 @@ type ExecutorOptions struct { // ExportReqURLPattern exports the request URL pattern // in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request ExportReqURLPattern bool + TimeoutVariants types.TimeoutVariants } // todo: centralizing components is not feasible with current clogged architecture diff --git a/pkg/templates/compile_test.go b/pkg/templates/compile_test.go index 3a230cc527..82536fb79a 100644 --- a/pkg/templates/compile_test.go +++ b/pkg/templates/compile_test.go @@ -39,15 +39,16 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(options.OmitTemplate), - Options: options, - Progress: progressImpl, - ProjectFile: nil, - IssuesClient: nil, - Browser: nil, - Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), - RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - Parser: templates.NewParser(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + Parser: templates.NewParser(), + TimeoutVariants: options.BuildTimeoutVariants(), } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index 4ce40ace96..fffe08a3cf 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -75,7 +75,6 @@ var DefaultOptions = &types.Options{ InteractionsPollDuration: 5, GitHubTemplateRepo: []string{}, GitHubToken: "", - ResponseReadTimeout: time.Second * 5, } // TemplateInfo contains info for a mock executed template. @@ -89,17 +88,18 @@ type TemplateInfo struct { func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protocols.ExecutorOptions { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts := &protocols.ExecutorOptions{ - TemplateID: info.ID, - TemplateInfo: info.Info, - TemplatePath: info.Path, - Output: NewMockOutputWriter(options.OmitTemplate), - Options: options, - Progress: progressImpl, - ProjectFile: nil, - IssuesClient: nil, - Browser: nil, - Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), - RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + TemplateID: info.ID, + TemplateInfo: info.Info, + TemplatePath: info.Path, + Output: NewMockOutputWriter(options.OmitTemplate), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + TimeoutVariants: options.BuildTimeoutVariants(), } executerOpts.CreateTemplateCtxStore() return executerOpts diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go index b47b38a2a0..4f907e4437 100644 --- a/pkg/tmplexec/flow/flow_executor_test.go +++ b/pkg/tmplexec/flow/flow_executor_test.go @@ -27,15 +27,16 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(options.OmitTemplate), - Options: options, - Progress: progressImpl, - ProjectFile: nil, - IssuesClient: nil, - Browser: nil, - Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), - RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - Parser: templates.NewParser(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + Parser: templates.NewParser(), + TimeoutVariants: options.BuildTimeoutVariants(), } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/tmplexec/multiproto/multi_test.go b/pkg/tmplexec/multiproto/multi_test.go index 4f2aa25e4e..34df7ff2d3 100644 --- a/pkg/tmplexec/multiproto/multi_test.go +++ b/pkg/tmplexec/multiproto/multi_test.go @@ -27,15 +27,16 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(options.OmitTemplate), - Options: options, - Progress: progressImpl, - ProjectFile: nil, - IssuesClient: nil, - Browser: nil, - Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), - RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - Parser: templates.NewParser(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + Parser: templates.NewParser(), + TimeoutVariants: options.BuildTimeoutVariants(), } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/types/types.go b/pkg/types/types.go index 0c31769ee1..c7c8a44c9f 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -277,8 +277,6 @@ type Options struct { SNI string // InputFileMode specifies the mode of input file (jsonl, burp, openapi, swagger, etc) InputFileMode string - // DialerTimeout sets the timeout for network requests. - DialerTimeout time.Duration // DialerKeepAlive sets the keep alive duration for network requests. DialerKeepAlive time.Duration // Interface to use for network scan @@ -291,8 +289,6 @@ type Options struct { ResponseReadSize int // ResponseSaveSize is the maximum size of response to save ResponseSaveSize int - // ResponseReadTimeout is response read timeout in seconds - ResponseReadTimeout time.Duration // Health Check HealthCheck bool // Time to wait between each input read operation before closing the stream @@ -401,6 +397,36 @@ type Options struct { ListTemplateProfiles bool } +type TimeoutVariants struct { + MaxResponseHeaderTimeout time.Duration + ResponseReadTimeout time.Duration + JsCompilerExecutionTimeout time.Duration + HttpTimeout time.Duration + DialTimeout time.Duration +} + +func (options *Options) BuildTimeoutVariants() TimeoutVariants { + timeoutVariants := TimeoutVariants{ + // MaxResponseHeaderTimeout is the timeout for response headers + // to be read from the server (this prevents infinite hang started by server if any) + // Note: this will be overridden temporarily when using @timeout request annotation + MaxResponseHeaderTimeout: time.Second * 10, + //response read timeout in seconds + ResponseReadTimeout: time.Second * 5, + JsCompilerExecutionTimeout: time.Second * time.Duration(int(float64(options.Timeout)*1.5)), + //http timeout for the client + HttpTimeout: time.Second * time.Duration(options.Timeout*3), + //timeout for network requests + DialTimeout: time.Second * time.Duration(options.Timeout), + } + + if options.Timeout > 10 { + timeoutVariants.MaxResponseHeaderTimeout = time.Second * time.Duration(options.Timeout) + } + + return timeoutVariants +} + // ShouldLoadResume resume file func (options *Options) ShouldLoadResume() bool { return options.Resume != "" && fileutil.FileExists(options.Resume) @@ -437,7 +463,6 @@ func DefaultOptions() *Options { MaxHostError: 30, ResponseReadSize: 10 * 1024 * 1024, ResponseSaveSize: 1024 * 1024, - ResponseReadTimeout: 5 * time.Second, } } From 32a73bf006f999a9e9b36eab29ff8dd9cc904512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Fri, 7 Jun 2024 13:26:31 +0300 Subject: [PATCH 2/7] update instances and add codeexectimeout --- pkg/js/compiler/compiler.go | 7 +++++-- pkg/protocols/code/code.go | 13 +++++++------ pkg/protocols/javascript/js.go | 20 +++++++++++++++----- pkg/types/types.go | 2 ++ 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index 367bd013b2..c552ccf362 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -4,11 +4,11 @@ package compiler import ( "context" "fmt" - "time" "github.com/dop251/goja" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators" + "github.com/projectdiscovery/nuclei/v3/pkg/types" contextutil "github.com/projectdiscovery/utils/context" "github.com/projectdiscovery/utils/errkit" stringsutil "github.com/projectdiscovery/utils/strings" @@ -45,6 +45,8 @@ type ExecuteOptions struct { Context context.Context + TimeoutVariants types.TimeoutVariants + // Manually exported objects exports map[string]interface{} } @@ -113,7 +115,8 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, } // execute with context and timeout - ctx, cancel := context.WithTimeoutCause(opts.Context, time.Duration(opts.Timeout)*time.Second, ErrJSExecDeadline) + + ctx, cancel := context.WithTimeoutCause(opts.Context, opts.TimeoutVariants.JsCompilerExecutionTimeout, ErrJSExecDeadline) defer cancel() // execute the script results, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (val goja.Value, err error) { diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index f103f50d0c..30c00e3efe 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -199,11 +199,12 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, args, &compiler.ExecuteOptions{ - Timeout: timeout, - Source: &request.PreCondition, - Callback: registerPreConditionFunctions, - Cleanup: cleanUpPreConditionFunctions, - Context: input.Context(), + Timeout: timeout, + TimeoutVariants: request.options.TimeoutVariants, + Source: &request.PreCondition, + Callback: registerPreConditionFunctions, + Cleanup: cleanUpPreConditionFunctions, + Context: input.Context(), }) if err != nil { return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err) @@ -218,7 +219,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } } - ctx, cancel := context.WithTimeoutCause(input.Context(), time.Duration(timeout)*time.Second, ErrCodeExecutionDeadline) + ctx, cancel := context.WithTimeoutCause(input.Context(), request.options.TimeoutVariants.CodeExecutionTimeout, ErrCodeExecutionDeadline) defer cancel() // Note: we use contextutil despite the fact that gozero accepts context as argument gOutput, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (*gozerotypes.Result, error) { diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 71f9f53413..1a41feea29 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -153,9 +153,10 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } opts := &compiler.ExecuteOptions{ - Timeout: request.Timeout, - Source: &request.Init, - Context: context.Background(), + Timeout: request.Timeout, + TimeoutVariants: request.options.TimeoutVariants, + Source: &request.Init, + Context: context.Background(), } // register 'export' function to export variables from init code // these are saved in args and are available in pre-condition and request code @@ -345,7 +346,11 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV argsCopy.TemplateCtx = templateCtx.GetAll() result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, argsCopy, - &compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.PreCondition, Context: target.Context()}) + &compiler.ExecuteOptions{ + Timeout: request.Timeout, + TimeoutVariants: requestOptions.TimeoutVariants, + Source: &request.PreCondition, Context: target.Context(), + }) if err != nil { return errorutil.NewWithTag(request.TemplateID, "could not execute pre-condition: %s", err) } @@ -500,7 +505,12 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte } results, err := request.options.JsCompiler.ExecuteWithOptions(request.scriptCompiled, argsCopy, - &compiler.ExecuteOptions{Timeout: request.Timeout, Source: &request.Code, Context: input.Context()}) + &compiler.ExecuteOptions{ + Timeout: request.Timeout, + TimeoutVariants: requestOptions.TimeoutVariants, + Source: &request.Code, + Context: input.Context(), + }) if err != nil { // shouldn't fail even if it returned error instead create a failure event results = compiler.ExecuteResult{"success": false, "error": err.Error()} diff --git a/pkg/types/types.go b/pkg/types/types.go index c7c8a44c9f..308cce0ae0 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -401,6 +401,7 @@ type TimeoutVariants struct { MaxResponseHeaderTimeout time.Duration ResponseReadTimeout time.Duration JsCompilerExecutionTimeout time.Duration + CodeExecutionTimeout time.Duration HttpTimeout time.Duration DialTimeout time.Duration } @@ -414,6 +415,7 @@ func (options *Options) BuildTimeoutVariants() TimeoutVariants { //response read timeout in seconds ResponseReadTimeout: time.Second * 5, JsCompilerExecutionTimeout: time.Second * time.Duration(int(float64(options.Timeout)*1.5)), + CodeExecutionTimeout: time.Second * 3, //http timeout for the client HttpTimeout: time.Second * time.Duration(options.Timeout*3), //timeout for network requests From 2fbd88e33cd899cd2649ee456fc224f485c7805c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 27 Jun 2024 14:25:12 +0300 Subject: [PATCH 3/7] fix test --- pkg/js/compiler/compiler_test.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/js/compiler/compiler_test.go b/pkg/js/compiler/compiler_test.go index ca09e7782e..7858b0f07f 100644 --- a/pkg/js/compiler/compiler_test.go +++ b/pkg/js/compiler/compiler_test.go @@ -1,11 +1,14 @@ package compiler import ( + "context" "strings" "testing" + "time" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/nuclei/v3/pkg/types" ) func TestNewCompilerConsoleDebug(t *testing.T) { @@ -18,7 +21,15 @@ func TestNewCompilerConsoleDebug(t *testing.T) { }) compiler := New() - _, err := compiler.Execute("console.log('hello world');", NewExecuteArgs()) + p, err := WrapScriptNCompile("console.log('hello world');", false) + if err != nil { + t.Fatal(err) + } + + _, err = compiler.ExecuteWithOptions(p, NewExecuteArgs(), &ExecuteOptions{Context: context.Background(), + Timeout: 10, + TimeoutVariants: types.TimeoutVariants{JsCompilerExecutionTimeout: time.Duration(10) * time.Second}}, + ) if err != nil { t.Fatal(err) } From 40822a9f87cb1dfb09922d2a913c85c732ca4aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 27 Jun 2024 14:29:31 +0300 Subject: [PATCH 4/7] default to 10s --- pkg/types/types.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/types/types.go b/pkg/types/types.go index 3ef03cf1b9..8ce9706198 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -414,6 +414,10 @@ type TimeoutVariants struct { } func (options *Options) BuildTimeoutVariants() TimeoutVariants { + if options.Timeout == 0 { + options.Timeout = 10 + } + timeoutVariants := TimeoutVariants{ // MaxResponseHeaderTimeout is the timeout for response headers // to be read from the server (this prevents infinite hang started by server if any) From b67eee88cdd11f3b37fc8e63e65b8e1d66ee4c98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 27 Jun 2024 14:42:56 +0300 Subject: [PATCH 5/7] minor --- pkg/js/compiler/compiler_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/js/compiler/compiler_test.go b/pkg/js/compiler/compiler_test.go index 7858b0f07f..7dfcd7f46e 100644 --- a/pkg/js/compiler/compiler_test.go +++ b/pkg/js/compiler/compiler_test.go @@ -27,8 +27,8 @@ func TestNewCompilerConsoleDebug(t *testing.T) { } _, err = compiler.ExecuteWithOptions(p, NewExecuteArgs(), &ExecuteOptions{Context: context.Background(), - Timeout: 10, - TimeoutVariants: types.TimeoutVariants{JsCompilerExecutionTimeout: time.Duration(10) * time.Second}}, + Timeout: 20, + TimeoutVariants: types.TimeoutVariants{JsCompilerExecutionTimeout: time.Duration(20) * time.Second}}, ) if err != nil { t.Fatal(err) From 2e9ace20c365a23aee8a31227215e8325bf62631 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 8 Jul 2024 17:27:01 +0530 Subject: [PATCH 6/7] make timeouts pluggable and rename --- cmd/integration-test/library.go | 1 - internal/runner/runner.go | 1 - lib/multi.go | 1 - lib/sdk_private.go | 1 - pkg/js/compiler/compiler.go | 10 +- pkg/js/compiler/compiler_test.go | 3 +- pkg/js/compiler/init.go | 11 +-- pkg/protocols/code/code.go | 11 +-- pkg/protocols/common/protocolstate/state.go | 5 +- pkg/protocols/http/http.go | 2 +- .../http/httpclientpool/clientpool.go | 15 +-- pkg/protocols/http/request.go | 2 +- pkg/protocols/http/request_annotations.go | 6 +- pkg/protocols/javascript/js.go | 12 +-- pkg/protocols/javascript/js_test.go | 19 ++-- pkg/protocols/network/request.go | 4 +- pkg/protocols/protocols.go | 1 - pkg/templates/compile_test.go | 1 - pkg/testutils/testutils.go | 1 - pkg/tmplexec/flow/flow_executor_test.go | 1 - pkg/tmplexec/multiproto/multi_test.go | 19 ++-- pkg/types/types.go | 91 +++++++++++++------ 22 files changed, 106 insertions(+), 112 deletions(-) diff --git a/cmd/integration-test/library.go b/cmd/integration-test/library.go index d341f576c6..1324688e05 100644 --- a/cmd/integration-test/library.go +++ b/cmd/integration-test/library.go @@ -112,7 +112,6 @@ func executeNucleiAsLibrary(templatePath, templateURL string) ([]string, error) Colorizer: aurora.NewAurora(true), ResumeCfg: types.NewResumeCfg(), Parser: templates.NewParser(), - TimeoutVariants: defaultOpts.BuildTimeoutVariants(), } engine := core.New(defaultOpts) engine.SetExecuterOptions(executerOpts) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 7699f0f3be..bc436500af 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -472,7 +472,6 @@ func (r *Runner) RunEnumeration() error { TemporaryDirectory: r.tmpDir, Parser: r.parser, FuzzParamsFrequency: fuzzFreqCache, - TimeoutVariants: r.options.BuildTimeoutVariants(), } if config.DefaultConfig.IsDebugArgEnabled(config.DebugExportURLPattern) { diff --git a/lib/multi.go b/lib/multi.go index 643d76fe75..44c13ddfe6 100644 --- a/lib/multi.go +++ b/lib/multi.go @@ -40,7 +40,6 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types Colorizer: aurora.NewAurora(true), ResumeCfg: types.NewResumeCfg(), Parser: base.parser, - TimeoutVariants: opts.BuildTimeoutVariants(), } if opts.RateLimitMinute > 0 { opts.RateLimit = opts.RateLimitMinute diff --git a/lib/sdk_private.go b/lib/sdk_private.go index fa96313b4c..b65ecf2a2f 100644 --- a/lib/sdk_private.go +++ b/lib/sdk_private.go @@ -171,7 +171,6 @@ func (e *NucleiEngine) init(ctx context.Context) error { ResumeCfg: types.NewResumeCfg(), Browser: e.browserInstance, Parser: e.parser, - TimeoutVariants: e.opts.BuildTimeoutVariants(), } if len(e.opts.SecretsFile) > 0 { authTmplStore, err := runner.GetAuthTmplStore(*e.opts, e.catalog, e.executerOpts) diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index c552ccf362..e47633ac6d 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -38,14 +38,12 @@ type ExecuteOptions struct { // Cleanup is extra cleanup function to be called after execution Cleanup func(runtime *goja.Runtime) - /// Timeout for this script execution - Timeout int // Source is original source of the script Source *string Context context.Context - TimeoutVariants types.TimeoutVariants + TimeoutVariants *types.Timeouts // Manually exported objects exports map[string]interface{} @@ -108,12 +106,6 @@ func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, // merge all args into templatectx args.TemplateCtx = generators.MergeMaps(args.TemplateCtx, args.Args) - if opts.Timeout <= 0 || opts.Timeout > 180 { - // some js scripts can take longer time so allow configuring timeout - // from template but keep it within sane limits (180s) - opts.Timeout = JsProtocolTimeout - } - // execute with context and timeout ctx, cancel := context.WithTimeoutCause(opts.Context, opts.TimeoutVariants.JsCompilerExecutionTimeout, ErrJSExecDeadline) diff --git a/pkg/js/compiler/compiler_test.go b/pkg/js/compiler/compiler_test.go index 7dfcd7f46e..23ffd47557 100644 --- a/pkg/js/compiler/compiler_test.go +++ b/pkg/js/compiler/compiler_test.go @@ -27,8 +27,7 @@ func TestNewCompilerConsoleDebug(t *testing.T) { } _, err = compiler.ExecuteWithOptions(p, NewExecuteArgs(), &ExecuteOptions{Context: context.Background(), - Timeout: 20, - TimeoutVariants: types.TimeoutVariants{JsCompilerExecutionTimeout: time.Duration(20) * time.Second}}, + TimeoutVariants: &types.Timeouts{JsCompilerExecutionTimeout: time.Duration(20) * time.Second}}, ) if err != nil { t.Fatal(err) diff --git a/pkg/js/compiler/init.go b/pkg/js/compiler/init.go index 8a171dc3da..92301df5e4 100644 --- a/pkg/js/compiler/init.go +++ b/pkg/js/compiler/init.go @@ -7,26 +7,17 @@ import ( // jsprotocolInit var ( - // Per Execution Javascript timeout in seconds - JsProtocolTimeout = 10 PoolingJsVmConcurrency = 100 NonPoolingVMConcurrency = 20 - JsTimeoutMultiplier = 1.5 ) // Init initializes the javascript protocol func Init(opts *types.Options) error { - if opts.Timeout < 10 { - // keep existing 10s timeout - return nil - } + if opts.JsConcurrency < 100 { // 100 is reasonable default opts.JsConcurrency = 100 } - // we have dialer timeout set to 10s so js needs to be at least - // 15s to return the actual error if not it will be a dialer timeout - JsProtocolTimeout = int(float64(opts.Timeout) * JsTimeoutMultiplier) PoolingJsVmConcurrency = opts.JsConcurrency PoolingJsVmConcurrency -= NonPoolingVMConcurrency return nil diff --git a/pkg/protocols/code/code.go b/pkg/protocols/code/code.go index 30c00e3efe..7736113dca 100644 --- a/pkg/protocols/code/code.go +++ b/pkg/protocols/code/code.go @@ -37,8 +37,7 @@ import ( ) const ( - pythonEnvRegex = `os\.getenv\(['"]([^'"]+)['"]\)` - TimeoutMultiplier = 6 // timeout multiplier for code protocol + pythonEnvRegex = `os\.getenv\(['"]([^'"]+)['"]\)` ) var ( @@ -179,9 +178,6 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa metaSrc.AddVariable(gozerotypes.Variable{Name: name, Value: v}) } - // set timeout using multiplier - timeout := TimeoutMultiplier * request.options.Options.Timeout - if request.PreCondition != "" { if request.options.Options.Debug || request.options.Options.DebugRequests { gologger.Debug().Msgf("[%s] Executing Precondition for Code request\n", request.TemplateID) @@ -199,8 +195,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, args, &compiler.ExecuteOptions{ - Timeout: timeout, - TimeoutVariants: request.options.TimeoutVariants, + TimeoutVariants: request.options.Options.GetTimeouts(), Source: &request.PreCondition, Callback: registerPreConditionFunctions, Cleanup: cleanUpPreConditionFunctions, @@ -219,7 +214,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa } } - ctx, cancel := context.WithTimeoutCause(input.Context(), request.options.TimeoutVariants.CodeExecutionTimeout, ErrCodeExecutionDeadline) + ctx, cancel := context.WithTimeoutCause(input.Context(), request.options.Options.GetTimeouts().CodeExecutionTimeout, ErrCodeExecutionDeadline) defer cancel() // Note: we use contextutil despite the fact that gozero accepts context as argument gOutput, err := contextutil.ExecFuncWithTwoReturns(ctx, func() (*gozerotypes.Result, error) { diff --git a/pkg/protocols/common/protocolstate/state.go b/pkg/protocols/common/protocolstate/state.go index d39f0fcd3b..b7c7796fa8 100644 --- a/pkg/protocols/common/protocolstate/state.go +++ b/pkg/protocols/common/protocolstate/state.go @@ -34,10 +34,7 @@ func Init(options *types.Options) error { lfaAllowed = options.AllowLocalFileAccess opts := fastdialer.DefaultOptions - timeoutVariants := options.BuildTimeoutVariants() - if timeoutVariants.DialTimeout > 0 { - opts.DialerTimeout = timeoutVariants.DialTimeout - } + opts.DialerTimeout = options.GetTimeouts().DialTimeout if options.DialerKeepAlive > 0 { opts.DialerKeepAlive = options.DialerKeepAlive } diff --git a/pkg/protocols/http/http.go b/pkg/protocols/http/http.go index c20cbbfa66..fc347ce88d 100644 --- a/pkg/protocols/http/http.go +++ b/pkg/protocols/http/http.go @@ -336,7 +336,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { request.Raw[i] = strings.ReplaceAll(raw, "\n", "\r\n") } } - request.rawhttpClient = httpclientpool.GetRawHTTP(options.Options) + request.rawhttpClient = httpclientpool.GetRawHTTP(options) } if len(request.Matchers) > 0 || len(request.Extractors) > 0 { compiled := &request.Operators diff --git a/pkg/protocols/http/httpclientpool/clientpool.go b/pkg/protocols/http/httpclientpool/clientpool.go index 0f9887c17b..2c244556ae 100644 --- a/pkg/protocols/http/httpclientpool/clientpool.go +++ b/pkg/protocols/http/httpclientpool/clientpool.go @@ -17,6 +17,7 @@ import ( "golang.org/x/net/publicsuffix" "github.com/projectdiscovery/fastdialer/fastdialer/ja3/impersonate" + "github.com/projectdiscovery/nuclei/v3/pkg/protocols" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" "github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils" "github.com/projectdiscovery/nuclei/v3/pkg/types" @@ -33,11 +34,6 @@ var ( clientPool *mapsutil.SyncLockMap[string, *retryablehttp.Client] ) -// GetHttpTimeout returns the http timeout for the client -func GetHttpTimeout(opts *types.Options) time.Duration { - return opts.BuildTimeoutVariants().HttpTimeout -} - // Init initializes the clientpool implementation func Init(options *types.Options) error { // Don't create clients if already created in the past. @@ -134,7 +130,7 @@ func (c *Configuration) HasStandardOptions() bool { } // GetRawHTTP returns the rawhttp request client -func GetRawHTTP(options *types.Options) *rawhttp.Client { +func GetRawHTTP(options *protocols.ExecutorOptions) *rawhttp.Client { if rawHttpClient == nil { rawHttpOptions := rawhttp.DefaultOptions if types.ProxyURL != "" { @@ -144,7 +140,7 @@ func GetRawHTTP(options *types.Options) *rawhttp.Client { } else if protocolstate.Dialer != nil { rawHttpOptions.FastDialer = protocolstate.Dialer } - rawHttpOptions.Timeout = GetHttpTimeout(options) + rawHttpOptions.Timeout = options.Options.GetTimeouts().HttpTimeout rawHttpClient = rawhttp.NewClient(rawHttpOptions) } return rawHttpClient @@ -230,8 +226,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl } // responseHeaderTimeout is max timeout for response headers to be read - timeoutVariants := options.BuildTimeoutVariants() - responseHeaderTimeout := timeoutVariants.MaxResponseHeaderTimeout + responseHeaderTimeout := options.GetTimeouts().HttpResponseHeaderTimeout if configuration.ResponseHeaderTimeout != 0 { responseHeaderTimeout = configuration.ResponseHeaderTimeout } @@ -300,7 +295,7 @@ func wrappedGet(options *types.Options, configuration *Configuration) (*retryabl CheckRedirect: makeCheckRedirectFunc(redirectFlow, maxRedirects), } if !configuration.NoTimeout { - httpclient.Timeout = GetHttpTimeout(options) + httpclient.Timeout = options.GetTimeouts().HttpTimeout } client := retryablehttp.NewWithHTTPClient(httpclient, retryableHttpOptions) if jar != nil { diff --git a/pkg/protocols/http/request.go b/pkg/protocols/http/request.go index 7e5e069348..c17e746e44 100644 --- a/pkg/protocols/http/request.go +++ b/pkg/protocols/http/request.go @@ -483,7 +483,7 @@ func (request *Request) ExecuteWithResults(input *contextargs.Context, dynamicVa request.options.RateLimitTake() ctx := request.newContext(input) - ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, httpclientpool.GetHttpTimeout(request.options.Options), ErrHttpEngineRequestDeadline) + ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline) defer cancel() generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue) diff --git a/pkg/protocols/http/request_annotations.go b/pkg/protocols/http/request_annotations.go index c0f4b69774..fa2a5eaedd 100644 --- a/pkg/protocols/http/request_annotations.go +++ b/pkg/protocols/http/request_annotations.go @@ -130,6 +130,10 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req if duration := reTimeoutAnnotation.FindStringSubmatch(rawRequest); len(duration) > 0 { value := strings.TrimSpace(duration[1]) if parsed, err := time.ParseDuration(value); err == nil { + // to avoid dos via timeout request annotation in http template we set it to maximum of 2 minutes + if parsed > 2*time.Minute { + parsed = 2 * time.Minute + } //nolint:govet // cancelled automatically by withTimeout // global timeout is overridden by annotation by replacing context ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), parsed, ErrTimeoutAnnotationDeadline) @@ -140,7 +144,7 @@ func (r *Request) parseAnnotations(rawRequest string, request *retryablehttp.Req } else { //nolint:govet // cancelled automatically by withTimeout // global timeout is overridden by annotation by replacing context - ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), httpclientpool.GetHttpTimeout(r.options.Options), ErrRequestTimeoutDeadline) + ctx, overrides.cancelFunc = context.WithTimeoutCause(context.TODO(), r.options.Options.GetTimeouts().HttpTimeout, ErrRequestTimeoutDeadline) request = request.Clone(ctx) } } diff --git a/pkg/protocols/javascript/js.go b/pkg/protocols/javascript/js.go index 1a41feea29..2e43c61cd9 100644 --- a/pkg/protocols/javascript/js.go +++ b/pkg/protocols/javascript/js.go @@ -65,9 +65,6 @@ type Request struct { // Code contains code to execute for the javascript request. Code string `yaml:"code,omitempty" json:"code,omitempty" jsonschema:"title=code to execute in javascript,description=Executes inline javascript code for the request"` // description: | - // Timeout in seconds is optional timeout for each javascript script execution (i.e init, pre-condition, code) - Timeout int `yaml:"timeout,omitempty" json:"timeout,omitempty" jsonschema:"title=timeout for javascript execution,description=Timeout in seconds is optional timeout for entire javascript script execution"` - // description: | // StopAtFirstMatch stops processing the request at first match. StopAtFirstMatch bool `yaml:"stop-at-first-match,omitempty" json:"stop-at-first-match,omitempty" jsonschema:"title=stop at first match,description=Stop the execution after a match is found"` // description: | @@ -153,8 +150,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error { } opts := &compiler.ExecuteOptions{ - Timeout: request.Timeout, - TimeoutVariants: request.options.TimeoutVariants, + TimeoutVariants: request.options.Options.GetTimeouts(), Source: &request.Init, Context: context.Background(), } @@ -347,8 +343,7 @@ func (request *Request) ExecuteWithResults(target *contextargs.Context, dynamicV result, err := request.options.JsCompiler.ExecuteWithOptions(request.preConditionCompiled, argsCopy, &compiler.ExecuteOptions{ - Timeout: request.Timeout, - TimeoutVariants: requestOptions.TimeoutVariants, + TimeoutVariants: requestOptions.Options.GetTimeouts(), Source: &request.PreCondition, Context: target.Context(), }) if err != nil { @@ -506,8 +501,7 @@ func (request *Request) executeRequestWithPayloads(hostPort string, input *conte results, err := request.options.JsCompiler.ExecuteWithOptions(request.scriptCompiled, argsCopy, &compiler.ExecuteOptions{ - Timeout: request.Timeout, - TimeoutVariants: requestOptions.TimeoutVariants, + TimeoutVariants: requestOptions.Options.GetTimeouts(), Source: &request.Code, Context: input.Context(), }) diff --git a/pkg/protocols/javascript/js_test.go b/pkg/protocols/javascript/js_test.go index 4074eff55f..efb78ef6e2 100644 --- a/pkg/protocols/javascript/js_test.go +++ b/pkg/protocols/javascript/js_test.go @@ -32,16 +32,15 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(options.OmitTemplate), - Options: options, - Progress: progressImpl, - ProjectFile: nil, - IssuesClient: nil, - Browser: nil, - Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), - RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - Parser: templates.NewParser(), - TimeoutVariants: options.BuildTimeoutVariants(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + Parser: templates.NewParser(), } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/protocols/network/request.go b/pkg/protocols/network/request.go index 802d73539f..90390e53c3 100644 --- a/pkg/protocols/network/request.go +++ b/pkg/protocols/network/request.go @@ -327,7 +327,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac } if input.Read > 0 { - buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.TimeoutVariants.ResponseReadTimeout) + buffer, err := ConnReadNWithTimeout(conn, int64(input.Read), request.options.Options.GetTimeouts().TcpReadTimeout) if err != nil { return errorutil.NewWithErr(err).Msgf("could not read response from connection") } @@ -377,7 +377,7 @@ func (request *Request) executeRequestWithPayloads(variables map[string]interfac bufferSize = -1 } - final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.TimeoutVariants.ResponseReadTimeout) + final, err := ConnReadNWithTimeout(conn, int64(bufferSize), request.options.Options.GetTimeouts().TcpReadTimeout) if err != nil { request.options.Output.Request(request.options.TemplatePath, address, request.Type().String(), err) gologger.Verbose().Msgf("could not read more data from %s: %s", actualAddress, err) diff --git a/pkg/protocols/protocols.go b/pkg/protocols/protocols.go index f3f73214bb..af7d2b4766 100644 --- a/pkg/protocols/protocols.go +++ b/pkg/protocols/protocols.go @@ -124,7 +124,6 @@ type ExecutorOptions struct { // ExportReqURLPattern exports the request URL pattern // in ResultEvent it contains the exact url pattern (ex: {{BaseURL}}/{{randstr}}/xyz) used in the request ExportReqURLPattern bool - TimeoutVariants types.TimeoutVariants } // todo: centralizing components is not feasible with current clogged architecture diff --git a/pkg/templates/compile_test.go b/pkg/templates/compile_test.go index 9bae1fe3d6..91a858bd7e 100644 --- a/pkg/templates/compile_test.go +++ b/pkg/templates/compile_test.go @@ -48,7 +48,6 @@ func setup() { Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), Parser: templates.NewParser(), - TimeoutVariants: options.BuildTimeoutVariants(), } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/testutils/testutils.go b/pkg/testutils/testutils.go index ea2b1d0315..930787aab9 100644 --- a/pkg/testutils/testutils.go +++ b/pkg/testutils/testutils.go @@ -100,7 +100,6 @@ func NewMockExecuterOptions(options *types.Options, info *TemplateInfo) *protoco Browser: nil, Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - TimeoutVariants: options.BuildTimeoutVariants(), } executerOpts.CreateTemplateCtxStore() return executerOpts diff --git a/pkg/tmplexec/flow/flow_executor_test.go b/pkg/tmplexec/flow/flow_executor_test.go index 4f907e4437..cf7b1790a6 100644 --- a/pkg/tmplexec/flow/flow_executor_test.go +++ b/pkg/tmplexec/flow/flow_executor_test.go @@ -36,7 +36,6 @@ func setup() { Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), Parser: templates.NewParser(), - TimeoutVariants: options.BuildTimeoutVariants(), } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/tmplexec/multiproto/multi_test.go b/pkg/tmplexec/multiproto/multi_test.go index 34df7ff2d3..4f2aa25e4e 100644 --- a/pkg/tmplexec/multiproto/multi_test.go +++ b/pkg/tmplexec/multiproto/multi_test.go @@ -27,16 +27,15 @@ func setup() { progressImpl, _ := progress.NewStatsTicker(0, false, false, false, 0) executerOpts = protocols.ExecutorOptions{ - Output: testutils.NewMockOutputWriter(options.OmitTemplate), - Options: options, - Progress: progressImpl, - ProjectFile: nil, - IssuesClient: nil, - Browser: nil, - Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), - RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), - Parser: templates.NewParser(), - TimeoutVariants: options.BuildTimeoutVariants(), + Output: testutils.NewMockOutputWriter(options.OmitTemplate), + Options: options, + Progress: progressImpl, + ProjectFile: nil, + IssuesClient: nil, + Browser: nil, + Catalog: disk.NewCatalog(config.DefaultConfig.TemplatesDirectory), + RateLimiter: ratelimit.New(context.Background(), uint(options.RateLimit), time.Second), + Parser: templates.NewParser(), } workflowLoader, err := workflow.NewLoader(&executerOpts) if err != nil { diff --git a/pkg/types/types.go b/pkg/types/types.go index 8ce9706198..48e9039d37 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -402,42 +402,79 @@ type Options struct { HttpApiEndpoint string // ListTemplateProfiles lists all available template profiles ListTemplateProfiles bool + // timeouts contains various types of timeouts used in nuclei + // these timeouts are derived from dial-timeout (-timeout) with known multipliers + // This is internally managed and does not need to be set by user by explicitly setting + // this overrides the default/derived one + timeouts *Timeouts } -type TimeoutVariants struct { - MaxResponseHeaderTimeout time.Duration - ResponseReadTimeout time.Duration - JsCompilerExecutionTimeout time.Duration - CodeExecutionTimeout time.Duration - HttpTimeout time.Duration - DialTimeout time.Duration +// SetTimeouts sets the timeout variants to use for the executor +func (opts *Options) SetTimeouts(t *Timeouts) { + opts.timeouts = t } -func (options *Options) BuildTimeoutVariants() TimeoutVariants { - if options.Timeout == 0 { - options.Timeout = 10 +// GetTimeouts returns the timeout variants to use for the executor +func (eo *Options) GetTimeouts() *Timeouts { + if eo.timeouts != nil { + // redundant but apply to avoid any potential issues + eo.timeouts.ApplyDefaults() + return eo.timeouts } + // set timeout variant value + eo.timeouts = NewTimeoutVariant(eo.Timeout) + eo.timeouts.ApplyDefaults() + return eo.timeouts +} - timeoutVariants := TimeoutVariants{ - // MaxResponseHeaderTimeout is the timeout for response headers - // to be read from the server (this prevents infinite hang started by server if any) - // Note: this will be overridden temporarily when using @timeout request annotation - MaxResponseHeaderTimeout: time.Second * 10, - //response read timeout in seconds - ResponseReadTimeout: time.Second * 5, - JsCompilerExecutionTimeout: time.Second * time.Duration(int(float64(options.Timeout)*1.5)), - CodeExecutionTimeout: time.Second * 3, - //http timeout for the client - HttpTimeout: time.Second * time.Duration(options.Timeout*3), - //timeout for network requests - DialTimeout: time.Second * time.Duration(options.Timeout), - } +// Timeouts is a struct that contains all the timeout variants for nuclei +// dialer timeout is used to derive other timeouts +type Timeouts struct { + // DialTimeout for fastdialer (default 10s) + DialTimeout time.Duration + // Tcp(Network Protocol) Read From Connection Timeout (default 5s) + TcpReadTimeout time.Duration + // Http Response Header Timeout (default 10s) + // this timeout prevents infinite hangs started by server if any + // this is temporarily overridden when using @timeout request annotation + HttpResponseHeaderTimeout time.Duration + // HttpTimeout for http client (default -> 3 x dial-timeout = 30s) + HttpTimeout time.Duration + // JsCompilerExec timeout/deadline (default -> 2 x dial-timeout = 20s) + JsCompilerExecutionTimeout time.Duration + // CodeExecutionTimeout for code execution (default -> 3 x dial-timeout = 30s) + CodeExecutionTimeout time.Duration +} - if options.Timeout > 10 { - timeoutVariants.MaxResponseHeaderTimeout = time.Second * time.Duration(options.Timeout) +// NewTimeoutVariant creates a new timeout variant with the given dial timeout in seconds +func NewTimeoutVariant(dialTimeoutSec int) *Timeouts { + tv := &Timeouts{ + DialTimeout: time.Duration(dialTimeoutSec) * time.Second, } + tv.ApplyDefaults() + return tv +} - return timeoutVariants +// ApplyDefaults applies default values to timeout variants when missing +func (tv *Timeouts) ApplyDefaults() { + if tv.DialTimeout == 0 { + tv.DialTimeout = 10 * time.Second + } + if tv.TcpReadTimeout == 0 { + tv.TcpReadTimeout = 5 * time.Second + } + if tv.HttpResponseHeaderTimeout == 0 { + tv.HttpResponseHeaderTimeout = 10 * time.Second + } + if tv.HttpTimeout == 0 { + tv.HttpTimeout = 3 * tv.DialTimeout + } + if tv.JsCompilerExecutionTimeout == 0 { + tv.JsCompilerExecutionTimeout = 2 * tv.DialTimeout + } + if tv.CodeExecutionTimeout == 0 { + tv.CodeExecutionTimeout = 3 * tv.DialTimeout + } } // ShouldLoadResume resume file From 4e5961ff5014029b8923b1666caa5e0ab6b3db34 Mon Sep 17 00:00:00 2001 From: Tarun Koyalwar Date: Mon, 8 Jul 2024 17:35:01 +0530 Subject: [PATCH 7/7] remove residual code --- pkg/js/compiler/compiler.go | 9 --------- pkg/js/compiler/compiler_test.go | 11 ----------- 2 files changed, 20 deletions(-) diff --git a/pkg/js/compiler/compiler.go b/pkg/js/compiler/compiler.go index e47633ac6d..f50e44ff2a 100644 --- a/pkg/js/compiler/compiler.go +++ b/pkg/js/compiler/compiler.go @@ -79,15 +79,6 @@ func (e ExecuteResult) GetSuccess() bool { return val } -// Execute executes a script with the default options. -func (c *Compiler) Execute(code string, args *ExecuteArgs) (ExecuteResult, error) { - p, err := WrapScriptNCompile(code, false) - if err != nil { - return nil, err - } - return c.ExecuteWithOptions(p, args, &ExecuteOptions{Context: context.Background()}) -} - // ExecuteWithOptions executes a script with the provided options. func (c *Compiler) ExecuteWithOptions(program *goja.Program, args *ExecuteArgs, opts *ExecuteOptions) (ExecuteResult, error) { if opts == nil { diff --git a/pkg/js/compiler/compiler_test.go b/pkg/js/compiler/compiler_test.go index 23ffd47557..3a43a4016e 100644 --- a/pkg/js/compiler/compiler_test.go +++ b/pkg/js/compiler/compiler_test.go @@ -37,17 +37,6 @@ func TestNewCompilerConsoleDebug(t *testing.T) { } } -func TestExecuteResultGetSuccess(t *testing.T) { - compiler := New() - result, err := compiler.Execute("1+1 == 2", NewExecuteArgs()) - if err != nil { - t.Fatal(err) - } - if result.GetSuccess() != true { - t.Fatalf("expected true, got=%v", result.GetSuccess()) - } -} - type noopWriter struct { Callback func(data []byte, level levels.Level) }