Skip to content

Commit

Permalink
Merge pull request #1396 from sundowndev/feat/scanner-options
Browse files Browse the repository at this point in the history
Scanner options
  • Loading branch information
sundowndev authored Feb 20, 2024
2 parents b38394a + 8b9598a commit 72e43a7
Show file tree
Hide file tree
Showing 30 changed files with 532 additions and 215 deletions.
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@ lint:
.PHONY: install-tools
install-tools:
$(GOINSTALL) gotest.tools/gotestsum@v1.6.3
$(GOINSTALL) github.com/vektra/mockery/v2@v2.8.0
$(GOINSTALL) github.com/swaggo/swag/cmd/swag@v1.16.1
$(GOINSTALL) github.com/vektra/mockery/v2@v2.38.0
$(GOINSTALL) github.com/swaggo/swag/cmd/swag@v1.16.3
@which golangci-lint > /dev/null 2>&1 || (curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash -s -- -b $(GOBINPATH) v1.46.2)

go.mod: FORCE
Expand Down
3 changes: 2 additions & 1 deletion cmd/scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ func runScan(opts *ScanCmdOptions) {
remoteLibrary := remote.NewLibrary(f)
remote.InitScanners(remoteLibrary)

result, errs := remoteLibrary.Scan(num)
// Scanner options are currently not used in CLI
result, errs := remoteLibrary.Scan(num, remote.ScannerOptions{})

err = output.GetOutput(output.Console, color.Output).Write(result, errs)
if err != nil {
Expand Down
23 changes: 15 additions & 8 deletions docs/getting-started/scanners.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ GOOGLE_API_KEY="value"
phoneinfoga scan -n +4176418xxxx --env-file=.env.local
```

### Scanner options

When using the **REST API**, you can also specify those values on a per-request basis. Each scanner supports its own options, see below. For details on how to specify those options, see [API docs](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/sundowndev/phoneinfoga/master/web/docs/swagger.yaml#/Numbers/RunScanner). For readability and simplicity, options are named exactly like their environment variable equivalent.

!!! warning
Scanner options will override environment variables for the current request.

## Building your own scanner

PhoneInfoga can now be extended with plugins! You can build your own scanner and PhoneInfoga will use it to scan the given phone number.
Expand Down Expand Up @@ -63,9 +70,9 @@ Numverify provide standard but useful information such as country code, location
2. Go to "Number Verification API" in the marketplace, click on "Subscribe for free", then choose whatever plan you want
3. Copy the new API token and use it as an environment variable

| Environment variable | Default | Description |
|----------------------|---------|------------------------------------------------------------------------|
| NUMVERIFY_API_KEY | | API key to authenticate to the Numverify API. |
| Environment variable | Option | Default | Description |
|----------------------|------------|---------|-------------------------------------------------------|
| NUMVERIFY_API_KEY | NUMVERIFY_API_KEY | | API key to authenticate to the Numverify API. |

??? example "Output example"

Expand Down Expand Up @@ -209,11 +216,11 @@ Follow the steps below to create a new search engine :

??? info "Configuration"

| Environment variable | Default | Description |
|-----------------------|---------|------------------------------------------------------------------------|
| GOOGLECSE_CX | | Search engine ID. |
| GOOGLE_API_KEY | | API key to authenticate to the Google API. |
| GOOGLECSE_MAX_RESULTS | 10 | Maximum results for each request. Each 10 results requires an additional request. This value cannot go above 100. |
| Environment variable | Option | Default | Description |
|-----------------------|----------|----------|-------------------------------------------------------------|
| GOOGLECSE_CX | GOOGLECSE_CX | | Search engine ID. |
| GOOGLE_API_KEY | GOOGLE_API_KEY | | API key to authenticate to the Google API. |
| GOOGLECSE_MAX_RESULTS | | 10 | Maximum results for each request. Each 10 results requires an additional request. This value cannot go above 100. |

??? example "Output example"

Expand Down
4 changes: 2 additions & 2 deletions examples/plugin/customscanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,13 @@ func (s *customScanner) Description() string {
// This can be useful to check for authentication or
// country code support for example, and avoid running
// the scanner when it just can't work.
func (s *customScanner) DryRun(n number.Number) error {
func (s *customScanner) DryRun(n number.Number, opts remote.ScannerOptions) error {
return nil
}

// Run does the actual scan of the phone number.
// Note this function will be executed in a goroutine.
func (s *customScanner) Run(n number.Number) (interface{}, error) {
func (s *customScanner) Run(n number.Number, opts remote.ScannerOptions) (interface{}, error) {
data := customScannerResponse{
Valid: true,
Info: "This number is known for scams!",
Expand Down
5 changes: 3 additions & 2 deletions examples/plugin/customscanner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"github.com/stretchr/testify/assert"
"github.com/sundowndev/phoneinfoga/v2/lib/number"
"github.com/sundowndev/phoneinfoga/v2/lib/remote"
"testing"
)

Expand Down Expand Up @@ -37,11 +38,11 @@ func TestCustomScanner(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
scanner := &customScanner{}

if scanner.DryRun(*tt.number) != nil {
if scanner.DryRun(*tt.number, remote.ScannerOptions{}) != nil {
t.Fatal("DryRun() should return nil")
}

got, err := scanner.Run(*tt.number)
got, err := scanner.Run(*tt.number, remote.ScannerOptions{})
if tt.wantError != "" {
assert.EqualError(t, err, tt.wantError)
} else {
Expand Down
25 changes: 12 additions & 13 deletions lib/remote/googlecse_scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@ import (
const GoogleCSE = "googlecse"

type googleCSEScanner struct {
Cx string
ApiKey string
MaxResults int64
httpClient *http.Client
}
Expand Down Expand Up @@ -52,8 +50,6 @@ func NewGoogleCSEScanner(HTTPclient *http.Client) Scanner {
}

return &googleCSEScanner{
Cx: os.Getenv("GOOGLECSE_CX"),
ApiKey: os.Getenv("GOOGLE_API_KEY"),
MaxResults: int64(maxResults),
httpClient: HTTPclient,
}
Expand All @@ -67,32 +63,34 @@ func (s *googleCSEScanner) Description() string {
return "Googlecse searches for footprints of a given phone number on the web using Google Custom Search Engine."
}

func (s *googleCSEScanner) DryRun(_ number.Number) error {
if s.Cx == "" || s.ApiKey == "" {
func (s *googleCSEScanner) DryRun(_ number.Number, opts ScannerOptions) error {
if opts.GetStringEnv("GOOGLECSE_CX") == "" || opts.GetStringEnv("GOOGLE_API_KEY") == "" {
return errors.New("search engine ID and/or API key is not defined")
}
return nil
}

func (s *googleCSEScanner) Run(n number.Number) (interface{}, error) {
func (s *googleCSEScanner) Run(n number.Number, opts ScannerOptions) (interface{}, error) {
var allItems []*customsearch.Result
var dorks []*GoogleSearchDork
var totalResultCount int
var totalRequestCount int
var cx = opts.GetStringEnv("GOOGLECSE_CX")
var apikey = opts.GetStringEnv("GOOGLE_API_KEY")

dorks = append(dorks, s.generateDorkQueries(n)...)

customsearchService, err := customsearch.NewService(
context.Background(),
option.WithAPIKey(s.ApiKey),
option.WithAPIKey(apikey),
option.WithHTTPClient(s.httpClient),
)
if err != nil {
return nil, err
}

for _, req := range dorks {
n, items, err := s.search(customsearchService, req.Dork)
n, items, err := s.search(customsearchService, req.Dork, cx)
if err != nil {
if s.isRateLimit(err) {
return nil, errors.New("rate limit exceeded, see https://developers.google.com/custom-search/v1/overview#pricing")
Expand All @@ -111,22 +109,22 @@ func (s *googleCSEScanner) Run(n number.Number) (interface{}, error) {
URL: item.Link,
})
}
data.Homepage = fmt.Sprintf("https://cse.google.com/cse?cx=%s", s.Cx)
data.Homepage = fmt.Sprintf("https://cse.google.com/cse?cx=%s", cx)
data.ResultCount = len(allItems)
data.TotalResultCount = totalResultCount
data.TotalRequestCount = totalRequestCount

return data, nil
}

func (s *googleCSEScanner) search(service *customsearch.Service, q string) (int, []*customsearch.Result, error) {
func (s *googleCSEScanner) search(service *customsearch.Service, q string, cx string) (int, []*customsearch.Result, error) {
var results []*customsearch.Result
var totalResultCount int

offset := int64(0)
for offset < s.MaxResults {
search := service.Cse.List()
search.Cx(s.Cx)
search.Cx(cx)
search.Q(q)
search.Start(offset)
searchQuery, err := search.Do()
Expand All @@ -151,7 +149,8 @@ func (s *googleCSEScanner) isRateLimit(theError error) bool {
if theError == nil {
return false
}
if _, ok := theError.(*googleapi.Error); !ok {
var err *googleapi.Error
if !errors.As(theError, &err) {
return false
}
if theError.(*googleapi.Error).Code != 429 {
Expand Down
91 changes: 86 additions & 5 deletions lib/remote/googlecse_scanner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) {
testcases := []struct {
name string
number *number.Number
opts ScannerOptions
expected map[string]interface{}
wantErrors map[string]error
mocks func()
Expand Down Expand Up @@ -89,6 +90,75 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) {
})
},
},
{
name: "test with options and no results",
number: test.NewFakeUSNumber(),
opts: ScannerOptions{
"GOOGLECSE_CX": "custom_cx",
"GOOGLE_API_KEY": "secret",
},
expected: map[string]interface{}{
"googlecse": GoogleCSEScannerResponse{
Homepage: "https://cse.google.com/cse?cx=custom_cx",
ResultCount: 0,
TotalResultCount: 0,
TotalRequestCount: 2,
Items: nil,
},
},
wantErrors: map[string]error{},
mocks: func() {
gock.New("https://customsearch.googleapis.com").
Get("/customsearch/v1").
MatchParam("cx", "custom_cx").
// TODO: ensure that custom api key is used
// MatchHeader("Authorization", "secret").
// TODO: the matcher below doesn't work for some reason
//MatchParam("q", "intext:\"14152229670\" OR intext:\"+14152229670\" OR intext:\"4152229670\" OR intext:\"(415) 222-9670\"").
MatchParam("start", "0").
Reply(200).
JSON(&customsearch.Search{
ServerResponse: googleapi.ServerResponse{
Header: http.Header{},
HTTPStatusCode: 200,
},
SearchInformation: &customsearch.SearchSearchInformation{
FormattedSearchTime: "0",
FormattedTotalResults: "0",
SearchTime: 0,
TotalResults: "0",
ForceSendFields: nil,
NullFields: nil,
},
Items: []*customsearch.Result{},
})

gock.New("https://customsearch.googleapis.com").
Get("/customsearch/v1").
MatchParam("cx", "custom_cx").
// TODO: ensure that custom api key is used
// MatchHeader("Authorization", "secret").
// TODO: the matcher below doesn't work for some reason
//MatchParam("q", "(ext:doc OR ext:docx OR ext:odt OR ext:pdf OR ext:rtf OR ext:sxw OR ext:psw OR ext:ppt OR ext:pptx OR ext:pps OR ext:csv OR ext:txt OR ext:xls) intext:\"14152229670\" OR intext:\"+14152229670\" OR intext:\"4152229670\" OR intext:\"(415)+222-9670\"").
MatchParam("start", "0").
Reply(200).
JSON(&customsearch.Search{
ServerResponse: googleapi.ServerResponse{
Header: http.Header{},
HTTPStatusCode: 200,
},
SearchInformation: &customsearch.SearchSearchInformation{
FormattedSearchTime: "0",
FormattedTotalResults: "0",
SearchTime: 0,
TotalResults: "0",
ForceSendFields: nil,
NullFields: nil,
},
Items: []*customsearch.Result{},
})
},
},
{
name: "test with results",
number: test.NewFakeUSNumber(),
Expand Down Expand Up @@ -229,7 +299,8 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
_ = os.Setenv("GOOGLECSE_CX", "fake_search_engine_id")
_ = os.Setenv("GOOGLE_API_KEY", "fake_api_key")
defer os.Clearenv()
defer os.Unsetenv("GOOGLECSE_CX")
defer os.Unsetenv("GOOGLE_API_KEY")

tt.mocks()
defer gock.Off() // Flush pending mocks after test execution
Expand All @@ -238,11 +309,11 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) {
remote := NewLibrary(filter.NewEngine())
remote.AddScanner(scanner)

if scanner.DryRun(*tt.number) != nil {
if scanner.DryRun(*tt.number, tt.opts) != nil {
t.Fatal("DryRun() should return nil")
}

got, errs := remote.Scan(tt.number)
got, errs := remote.Scan(tt.number, tt.opts)
if len(tt.wantErrors) > 0 {
assert.Equal(t, tt.wantErrors, errs)
} else {
Expand All @@ -259,12 +330,22 @@ func TestGoogleCSEScanner_DryRun(t *testing.T) {
defer os.Unsetenv("GOOGLECSE_CX")
defer os.Unsetenv("GOOGLE_API_KEY")
scanner := NewGoogleCSEScanner(&http.Client{})
assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber()))
assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{}))
}

func TestGoogleCSEScanner_DryRunWithOptions(t *testing.T) {
errStr := "search engine ID and/or API key is not defined"

scanner := NewGoogleCSEScanner(&http.Client{})
assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLECSE_CX": "test", "GOOGLE_API_KEY": "secret"}))
assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLECSE_CX": "", "GOOGLE_API_KEY": ""}), errStr)
assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLECSE_CX": "test"}), errStr)
assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLE_API_KEY": "test"}), errStr)
}

func TestGoogleCSEScanner_DryRun_Error(t *testing.T) {
scanner := NewGoogleCSEScanner(&http.Client{})
assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber()), "search engine ID and/or API key is not defined")
assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{}), "search engine ID and/or API key is not defined")
}

func TestGoogleCSEScanner_MaxResults(t *testing.T) {
Expand Down
4 changes: 2 additions & 2 deletions lib/remote/googlesearch_scanner.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,11 +39,11 @@ func (s *googlesearchScanner) Description() string {
return "Generate several Google dork requests for a given phone number."
}

func (s *googlesearchScanner) DryRun(_ number.Number) error {
func (s *googlesearchScanner) DryRun(_ number.Number, _ ScannerOptions) error {
return nil
}

func (s *googlesearchScanner) Run(n number.Number) (interface{}, error) {
func (s *googlesearchScanner) Run(n number.Number, _ ScannerOptions) (interface{}, error) {
res := GoogleSearchResponse{
SocialMedia: getSocialMediaDorks(n),
DisposableProviders: getDisposableProvidersDorks(n),
Expand Down
Loading

0 comments on commit 72e43a7

Please sign in to comment.