Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix missing results in flow template + feature: internal matchers using internal: true #4582

Merged
merged 7 commits into from
Jan 7, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat: internal matcher
  • Loading branch information
tarunKoyalwar committed Jan 5, 2024

Unverified

This commit is not signed, but one or more authors requires that any commit attributed to them is signed.
commit 71d5962605aa69d047820d93426c13bb2dd3b768
4 changes: 2 additions & 2 deletions cmd/integration-test/flow.go
Original file line number Diff line number Diff line change
@@ -67,7 +67,7 @@ func (t *iterateValuesFlow) Execute(filePath string) error {
if err != nil {
return err
}
return expectResultsCount(results, 1)
return expectResultsCount(results, 2)
}

type dnsNsProbe struct{}
@@ -77,7 +77,7 @@ func (t *dnsNsProbe) Execute(filePath string) error {
if err != nil {
return err
}
return expectResultsCount(results, 1)
return expectResultsCount(results, 2)
}

func getBase64(input string) string {
1 change: 1 addition & 0 deletions integration_tests/flow/conditional-flow-negative.yaml
Original file line number Diff line number Diff line change
@@ -15,6 +15,7 @@ dns:
- type: word
words:
- "ghost.io"
internal: true

http:
- method: GET
1 change: 1 addition & 0 deletions integration_tests/flow/conditional-flow.yaml
Original file line number Diff line number Diff line change
@@ -15,6 +15,7 @@ dns:
- type: word
words:
- "ghost.io"
internal: true

http:
- method: GET
1 change: 1 addition & 0 deletions integration_tests/flow/dns-ns-probe.yaml
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@ dns:
- type: word
words:
- "IN\tNS"
# internal: true
extractors:
- type: regex
internal: true
5 changes: 3 additions & 2 deletions integration_tests/flow/flow-hide-matcher.yaml
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
id: flow-hide-matcher

info:
name: Test HTTP Template
name: Test Flow Hide Matcher
author: pdteam
severity: info
description: In flow matcher output of previous step is hidden and only last event matcher output is shown
description: In Template any matcher can be marked as internal which hides it from the output.

flow: http(1) && http(2)

@@ -17,6 +17,7 @@ http:
- type: word
words:
- ok
internal: true

- method: GET
path:
10 changes: 8 additions & 2 deletions integration_tests/flow/iterate-values-flow.yaml
Original file line number Diff line number Diff line change
@@ -21,9 +21,9 @@ http:
extractors:
- type: regex
name: emails
internal: true
regex:
- '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}'
internal: true

- method: GET
path:
@@ -32,4 +32,10 @@ http:
matchers:
- type: word
words:
- "Welcome"
- "Welcome"

extractors:
- type: dsl
name: email
dsl:
- email
8 changes: 8 additions & 0 deletions pkg/operators/matchers/matchers.go
Original file line number Diff line number Diff line change
@@ -120,6 +120,14 @@ type Matcher struct {
// - false
// - true
MatchAll bool `yaml:"match-all,omitempty" json:"match-all,omitempty" jsonschema:"title=match all values,description=match all matcher values ignoring condition"`
// description: |
// Internal when true hides the matcher from output. Default is false.
// It is meant to be used in multiprotocol / flow templates to create internal matcher condition without printing it in output.
// or other similar use cases.
// values:
// - false
// - true
Internal bool `yaml:"internal,omitempty" json:"internal,omitempty" jsonschema:"title=hide matcher from output,description=hide matcher from output"`

// cached data for the compiled matcher
condition ConditionType // todo: this field should be the one used for overridden marshal ops
2 changes: 1 addition & 1 deletion pkg/operators/matchers/validate.go
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@ import (
"gopkg.in/yaml.v3"
)

var commonExpectedFields = []string{"Type", "Condition", "Name", "MatchAll", "Negative"}
var commonExpectedFields = []string{"Type", "Condition", "Name", "MatchAll", "Negative", "Internal"}

// Validate perform initial validation on the matcher structure
func (matcher *Matcher) Validate() error {
3 changes: 3 additions & 0 deletions pkg/operators/operators.go
Original file line number Diff line number Diff line change
@@ -90,6 +90,8 @@ type Result struct {

// Optional lineCounts for file protocol
LineCount string
// Operators is reference to operators that generated this result (Read-Only)
Operators *Operators
}

func (result *Result) HasMatch(name string) bool {
@@ -217,6 +219,7 @@ func (operators *Operators) Execute(data map[string]interface{}, match MatchFunc
Extracts: make(map[string][]string),
DynamicValues: make(map[string][]string),
outputUnique: make(map[string]struct{}),
Operators: operators,
}

// state variable to check if all extractors are internal
16 changes: 16 additions & 0 deletions pkg/tmplexec/exec.go
Original file line number Diff line number Diff line change
@@ -117,6 +117,22 @@ func (e *TemplateExecuter) Execute(ctx *scan.ScanContext) (bool, error) {
// something went wrong
return
}
// check for internal true matcher event
if event.HasOperatorResult() && event.OperatorsResult.Matched && event.OperatorsResult.Operators != nil {
// note all matchers should have internal:true if it is a combination then print it
allInternalMatchers := true
for _, matcher := range event.OperatorsResult.Operators.Matchers {
if allInternalMatchers && !matcher.Internal {
allInternalMatchers = false
break
}
}
if allInternalMatchers {
// this is a internal event and no meant to be printed
return
}
}

// If no results were found, and also interactsh is not being used
// in that case we can skip it, otherwise we've to show failure in
// case of matcher-status flag.
Loading