-
Notifications
You must be signed in to change notification settings - Fork 2.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Transparent Memoization via func Annotation (#4742)
* initial implementation with manual code generation * testing generation * refactor to package methods + auto memoize * more memos * fixing signatures * refactor * adding gen util * adding util * regenerate memoized files --------- Co-authored-by: Tarun Koyalwar <tarun@projectdiscovery.io>
- Loading branch information
1 parent
e7252a4
commit 4c7a0f4
Showing
37 changed files
with
787 additions
and
18 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| // Warning - This is generated code | ||
| package {{.SourcePackage}} | ||
|
|
||
| import ( | ||
| "github.com/projectdiscovery/utils/memoize" | ||
|
|
||
| {{range .Imports}} | ||
| {{.Name}} {{.Path}} | ||
| {{end}} | ||
| ) | ||
|
|
||
| {{range .Functions}} | ||
| {{ .SignatureWithPrefix "memoized" }} { | ||
| hash := "{{ .Name }}" {{range .Params}} + ":" + fmt.Sprint({{.Name}}) {{end}} | ||
|
|
||
| v, err, _ := protocolstate.Memoizer.Do(hash, func() (interface{}, error) { | ||
| return {{.Name}}({{.ParamsNames}}) | ||
| }) | ||
| if err != nil { | ||
| return {{.ResultFirstFieldDefaultValue}}, err | ||
| } | ||
| if value, ok := v.({{.ResultFirstFieldType}}); ok { | ||
| return value, nil | ||
| } | ||
|
|
||
| return {{.ResultFirstFieldDefaultValue}}, errors.New("could not convert cached result") | ||
| } | ||
| {{end}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| // this small cli tool is specific for those functions with arbitrary parameters and with result-error tuple as return values | ||
| // func(x,y) => result, error | ||
| // it works by creating a new memoized version of the functions in the same path as memo.original.file.go | ||
| // some parts are specific for nuclei and hardcoded within the template | ||
| package main | ||
|
|
||
| import ( | ||
| "flag" | ||
| "io/fs" | ||
| "log" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| "github.com/projectdiscovery/utils/memoize" | ||
| stringsutil "github.com/projectdiscovery/utils/strings" | ||
| ) | ||
|
|
||
| var ( | ||
| srcPath = flag.String("src", "", "nuclei source path") | ||
| tplPath = flag.String("tpl", "function.tpl", "template path") | ||
| tplSrc []byte | ||
| ) | ||
|
|
||
| func main() { | ||
| flag.Parse() | ||
|
|
||
| var err error | ||
| tplSrc, err = os.ReadFile(*tplPath) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
|
|
||
| err = filepath.Walk(*srcPath, walk) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| } | ||
|
|
||
| func walk(path string, info fs.FileInfo, err error) error { | ||
| if info.IsDir() { | ||
| return nil | ||
| } | ||
|
|
||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| ext := filepath.Ext(path) | ||
| base := filepath.Base(path) | ||
|
|
||
| if !stringsutil.EqualFoldAny(ext, ".go") { | ||
| return nil | ||
| } | ||
|
|
||
| basePath := filepath.Dir(path) | ||
| outPath := filepath.Join(basePath, "memo."+base) | ||
|
|
||
| // filename := filepath.Base(path) | ||
| data, err := os.ReadFile(path) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if !stringsutil.ContainsAnyI(string(data), "@memo") { | ||
| return nil | ||
| } | ||
| log.Println("processing:", path) | ||
| out, err := memoize.Src(string(tplSrc), path, data, "") | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if err := os.WriteFile(outPath, out, os.ModePerm); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| // Warning - This is generated code | ||
| package mssql | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| _ "github.com/denisenkom/go-mssqldb" | ||
|
|
||
| "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" | ||
| ) | ||
|
|
||
| func memoizedconnect(host string, port int, username string, password string, dbName string) (bool, error) { | ||
| hash := "connect" + ":" + fmt.Sprint(host) + ":" + fmt.Sprint(port) + ":" + fmt.Sprint(username) + ":" + fmt.Sprint(password) + ":" + fmt.Sprint(dbName) | ||
|
|
||
| v, err, _ := protocolstate.Memoizer.Do(hash, func() (interface{}, error) { | ||
| return connect(host, port, username, password, dbName) | ||
| }) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| if value, ok := v.(bool); ok { | ||
| return value, nil | ||
| } | ||
|
|
||
| return false, errors.New("could not convert cached result") | ||
| } | ||
|
|
||
| func memoizedisMssql(host string, port int) (bool, error) { | ||
| hash := "isMssql" + ":" + fmt.Sprint(host) + ":" + fmt.Sprint(port) | ||
|
|
||
| v, err, _ := protocolstate.Memoizer.Do(hash, func() (interface{}, error) { | ||
| return isMssql(host, port) | ||
| }) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| if value, ok := v.(bool); ok { | ||
| return value, nil | ||
| } | ||
|
|
||
| return false, errors.New("could not convert cached result") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| // Warning - This is generated code | ||
| package mysql | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" | ||
| ) | ||
|
|
||
| func memoizedisMySQL(host string, port int) (bool, error) { | ||
| hash := "isMySQL" + ":" + fmt.Sprint(host) + ":" + fmt.Sprint(port) | ||
|
|
||
| v, err, _ := protocolstate.Memoizer.Do(hash, func() (interface{}, error) { | ||
| return isMySQL(host, port) | ||
| }) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| if value, ok := v.(bool); ok { | ||
| return value, nil | ||
| } | ||
|
|
||
| return false, errors.New("could not convert cached result") | ||
| } | ||
|
|
||
| func memoizedfingerprintMySQL(host string, port int) (MySQLInfo, error) { | ||
| hash := "fingerprintMySQL" + ":" + fmt.Sprint(host) + ":" + fmt.Sprint(port) | ||
|
|
||
| v, err, _ := protocolstate.Memoizer.Do(hash, func() (interface{}, error) { | ||
| return fingerprintMySQL(host, port) | ||
| }) | ||
| if err != nil { | ||
| return MySQLInfo{}, err | ||
| } | ||
| if value, ok := v.(MySQLInfo); ok { | ||
| return value, nil | ||
| } | ||
|
|
||
| return MySQLInfo{}, errors.New("could not convert cached result") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| // Warning - This is generated code | ||
| package mysql | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
|
|
||
| "github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/protocolstate" | ||
| ) | ||
|
|
||
| func memoizedconnectWithDSN(dsn string) (bool, error) { | ||
| hash := "connectWithDSN" + ":" + fmt.Sprint(dsn) | ||
|
|
||
| v, err, _ := protocolstate.Memoizer.Do(hash, func() (interface{}, error) { | ||
| return connectWithDSN(dsn) | ||
| }) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
| if value, ok := v.(bool); ok { | ||
| return value, nil | ||
| } | ||
|
|
||
| return false, errors.New("could not convert cached result") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.