Merge pull request #1396 from sundowndev/feat/scanner-options

Scanner options

Makefile

@@ -53,8 +53,8 @@ lint:
 .PHONY: install-tools
 install-tools:
 	$(GOINSTALL) gotest.tools/[email protected]
-	$(GOINSTALL) github.com/vektra/mockery/v2@v2.8.0
-	$(GOINSTALL) github.com/swaggo/swag/cmd/[email protected].1
+	$(GOINSTALL) github.com/vektra/mockery/v2@v2.38.0
+	$(GOINSTALL) github.com/swaggo/swag/cmd/[email protected].3
 	@which golangci-lint > /dev/null 2>&1 || (curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash -s -- -b $(GOBINPATH) v1.46.2)
 
 go.mod: FORCE

cmd/scan.go

@@ -80,7 +80,8 @@ func runScan(opts *ScanCmdOptions) {
 	remoteLibrary := remote.NewLibrary(f)
 	remote.InitScanners(remoteLibrary)
 
-	result, errs := remoteLibrary.Scan(num)
+	// Scanner options are currently not used in CLI
+	result, errs := remoteLibrary.Scan(num, remote.ScannerOptions{})
 
 	err = output.GetOutput(output.Console, color.Output).Write(result, errs)
 	if err != nil {

docs/getting-started/scanners.md

@@ -19,6 +19,13 @@ GOOGLE_API_KEY="value"
 phoneinfoga scan -n +4176418xxxx --env-file=.env.local
 ```
 
+### Scanner options
+
+When using the **REST API**, you can also specify those values on a per-request basis. Each scanner supports its own options, see below. For details on how to specify those options, see [API docs](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/sundowndev/phoneinfoga/master/web/docs/swagger.yaml#/Numbers/RunScanner). For readability and simplicity, options are named exactly like their environment variable equivalent.
+
+!!! warning
+    Scanner options will override environment variables for the current request.
+
 ## Building your own scanner
 
 PhoneInfoga can now be extended with plugins! You can build your own scanner and PhoneInfoga will use it to scan the given phone number.
@@ -63,9 +70,9 @@ Numverify provide standard but useful information such as country code, location
     2. Go to "Number Verification API" in the marketplace, click on "Subscribe for free", then choose whatever plan you want
     3. Copy the new API token and use it as an environment variable
 
-    | Environment variable | Default | Description                                                            |
-    |----------------------|---------|------------------------------------------------------------------------|
-    | NUMVERIFY_API_KEY    |         | API key to authenticate to the Numverify API.                          |
+    | Environment variable |   Option   | Default | Description                                          |
+    |----------------------|------------|---------|-------------------------------------------------------|
+    | NUMVERIFY_API_KEY    |   NUMVERIFY_API_KEY  |         | API key to authenticate to the Numverify API.        |
 
 ??? example "Output example"
 
@@ -209,11 +216,11 @@ Follow the steps below to create a new search engine :
 
 ??? info "Configuration"
 
-    | Environment variable | Default | Description                                                            |
-    |-----------------------|---------|------------------------------------------------------------------------|
-    | GOOGLECSE_CX          |         | Search engine ID.            |
-    | GOOGLE_API_KEY     |         | API key to authenticate to the Google API.  |
-    | GOOGLECSE_MAX_RESULTS |   10    | Maximum results for each request. Each 10 results requires an additional request. This value cannot go above 100.  |
+    |  Environment variable |  Option  | Default  | Description                                                 |
+    |-----------------------|----------|----------|-------------------------------------------------------------|
+    | GOOGLECSE_CX          |    GOOGLECSE_CX    |          | Search engine ID.            |
+    | GOOGLE_API_KEY        |  GOOGLE_API_KEY |          | API key to authenticate to the Google API.  |
+    | GOOGLECSE_MAX_RESULTS |          |   10     | Maximum results for each request. Each 10 results requires an additional request. This value cannot go above 100.  |
 
 ??? example "Output example"
 

examples/plugin/customscanner.go

@@ -28,13 +28,13 @@ func (s *customScanner) Description() string {
 // This can be useful to check for authentication or
 // country code support for example, and avoid running
 // the scanner when it just can't work.
-func (s *customScanner) DryRun(n number.Number) error {
+func (s *customScanner) DryRun(n number.Number, opts remote.ScannerOptions) error {
 	return nil
 }
 
 // Run does the actual scan of the phone number.
 // Note this function will be executed in a goroutine.
-func (s *customScanner) Run(n number.Number) (interface{}, error) {
+func (s *customScanner) Run(n number.Number, opts remote.ScannerOptions) (interface{}, error) {
 	data := customScannerResponse{
 		Valid:  true,
 		Info:   "This number is known for scams!",

examples/plugin/customscanner_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/stretchr/testify/assert"
 	"github.com/sundowndev/phoneinfoga/v2/lib/number"
+	"github.com/sundowndev/phoneinfoga/v2/lib/remote"
 	"testing"
 )
 
@@ -37,11 +38,11 @@ func TestCustomScanner(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			scanner := &customScanner{}
 
-			if scanner.DryRun(*tt.number) != nil {
+			if scanner.DryRun(*tt.number, remote.ScannerOptions{}) != nil {
 				t.Fatal("DryRun() should return nil")
 			}
 
-			got, err := scanner.Run(*tt.number)
+			got, err := scanner.Run(*tt.number, remote.ScannerOptions{})
 			if tt.wantError != "" {
 				assert.EqualError(t, err, tt.wantError)
 			} else {

lib/remote/googlecse_scanner.go

@@ -18,8 +18,6 @@ import (
 const GoogleCSE = "googlecse"
 
 type googleCSEScanner struct {
-	Cx         string
-	ApiKey     string
 	MaxResults int64
 	httpClient *http.Client
 }
@@ -52,8 +50,6 @@ func NewGoogleCSEScanner(HTTPclient *http.Client) Scanner {
 	}
 
 	return &googleCSEScanner{
-		Cx:         os.Getenv("GOOGLECSE_CX"),
-		ApiKey:     os.Getenv("GOOGLE_API_KEY"),
 		MaxResults: int64(maxResults),
 		httpClient: HTTPclient,
 	}
@@ -67,32 +63,34 @@ func (s *googleCSEScanner) Description() string {
 	return "Googlecse searches for footprints of a given phone number on the web using Google Custom Search Engine."
 }
 
-func (s *googleCSEScanner) DryRun(_ number.Number) error {
-	if s.Cx == "" || s.ApiKey == "" {
+func (s *googleCSEScanner) DryRun(_ number.Number, opts ScannerOptions) error {
+	if opts.GetStringEnv("GOOGLECSE_CX") == "" || opts.GetStringEnv("GOOGLE_API_KEY") == "" {
 		return errors.New("search engine ID and/or API key is not defined")
 	}
 	return nil
 }
 
-func (s *googleCSEScanner) Run(n number.Number) (interface{}, error) {
+func (s *googleCSEScanner) Run(n number.Number, opts ScannerOptions) (interface{}, error) {
 	var allItems []*customsearch.Result
 	var dorks []*GoogleSearchDork
 	var totalResultCount int
 	var totalRequestCount int
+	var cx = opts.GetStringEnv("GOOGLECSE_CX")
+	var apikey = opts.GetStringEnv("GOOGLE_API_KEY")
 
 	dorks = append(dorks, s.generateDorkQueries(n)...)
 
 	customsearchService, err := customsearch.NewService(
 		context.Background(),
-		option.WithAPIKey(s.ApiKey),
+		option.WithAPIKey(apikey),
 		option.WithHTTPClient(s.httpClient),
 	)
 	if err != nil {
 		return nil, err
 	}
 
 	for _, req := range dorks {
-		n, items, err := s.search(customsearchService, req.Dork)
+		n, items, err := s.search(customsearchService, req.Dork, cx)
 		if err != nil {
 			if s.isRateLimit(err) {
 				return nil, errors.New("rate limit exceeded, see https://developers.google.com/custom-search/v1/overview#pricing")
@@ -111,22 +109,22 @@ func (s *googleCSEScanner) Run(n number.Number) (interface{}, error) {
 			URL:   item.Link,
 		})
 	}
-	data.Homepage = fmt.Sprintf("https://cse.google.com/cse?cx=%s", s.Cx)
+	data.Homepage = fmt.Sprintf("https://cse.google.com/cse?cx=%s", cx)
 	data.ResultCount = len(allItems)
 	data.TotalResultCount = totalResultCount
 	data.TotalRequestCount = totalRequestCount
 
 	return data, nil
 }
 
-func (s *googleCSEScanner) search(service *customsearch.Service, q string) (int, []*customsearch.Result, error) {
+func (s *googleCSEScanner) search(service *customsearch.Service, q string, cx string) (int, []*customsearch.Result, error) {
 	var results []*customsearch.Result
 	var totalResultCount int
 
 	offset := int64(0)
 	for offset < s.MaxResults {
 		search := service.Cse.List()
-		search.Cx(s.Cx)
+		search.Cx(cx)
 		search.Q(q)
 		search.Start(offset)
 		searchQuery, err := search.Do()
@@ -151,7 +149,8 @@ func (s *googleCSEScanner) isRateLimit(theError error) bool {
 	if theError == nil {
 		return false
 	}
-	if _, ok := theError.(*googleapi.Error); !ok {
+	var err *googleapi.Error
+	if !errors.As(theError, &err) {
 		return false
 	}
 	if theError.(*googleapi.Error).Code != 429 {

lib/remote/googlecse_scanner_test.go

@@ -24,6 +24,7 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) {
 	testcases := []struct {
 		name       string
 		number     *number.Number
+		opts       ScannerOptions
 		expected   map[string]interface{}
 		wantErrors map[string]error
 		mocks      func()
@@ -89,6 +90,75 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) {
 					})
 			},
 		},
+		{
+			name:   "test with options and no results",
+			number: test.NewFakeUSNumber(),
+			opts: ScannerOptions{
+				"GOOGLECSE_CX":   "custom_cx",
+				"GOOGLE_API_KEY": "secret",
+			},
+			expected: map[string]interface{}{
+				"googlecse": GoogleCSEScannerResponse{
+					Homepage:          "https://cse.google.com/cse?cx=custom_cx",
+					ResultCount:       0,
+					TotalResultCount:  0,
+					TotalRequestCount: 2,
+					Items:             nil,
+				},
+			},
+			wantErrors: map[string]error{},
+			mocks: func() {
+				gock.New("https://customsearch.googleapis.com").
+					Get("/customsearch/v1").
+					MatchParam("cx", "custom_cx").
+					// TODO: ensure that custom api key is used
+					// MatchHeader("Authorization", "secret").
+					// TODO: the matcher below doesn't work for some reason
+					//MatchParam("q", "intext:\"14152229670\" OR intext:\"+14152229670\" OR intext:\"4152229670\" OR intext:\"(415) 222-9670\"").
+					MatchParam("start", "0").
+					Reply(200).
+					JSON(&customsearch.Search{
+						ServerResponse: googleapi.ServerResponse{
+							Header:         http.Header{},
+							HTTPStatusCode: 200,
+						},
+						SearchInformation: &customsearch.SearchSearchInformation{
+							FormattedSearchTime:   "0",
+							FormattedTotalResults: "0",
+							SearchTime:            0,
+							TotalResults:          "0",
+							ForceSendFields:       nil,
+							NullFields:            nil,
+						},
+						Items: []*customsearch.Result{},
+					})
+
+				gock.New("https://customsearch.googleapis.com").
+					Get("/customsearch/v1").
+					MatchParam("cx", "custom_cx").
+					// TODO: ensure that custom api key is used
+					// MatchHeader("Authorization", "secret").
+					// TODO: the matcher below doesn't work for some reason
+					//MatchParam("q", "(ext:doc OR ext:docx OR ext:odt OR ext:pdf OR ext:rtf OR ext:sxw OR ext:psw OR ext:ppt OR ext:pptx OR ext:pps OR ext:csv OR ext:txt OR ext:xls) intext:\"14152229670\" OR intext:\"+14152229670\" OR intext:\"4152229670\" OR intext:\"(415)+222-9670\"").
+					MatchParam("start", "0").
+					Reply(200).
+					JSON(&customsearch.Search{
+						ServerResponse: googleapi.ServerResponse{
+							Header:         http.Header{},
+							HTTPStatusCode: 200,
+						},
+						SearchInformation: &customsearch.SearchSearchInformation{
+							FormattedSearchTime:   "0",
+							FormattedTotalResults: "0",
+							SearchTime:            0,
+							TotalResults:          "0",
+							ForceSendFields:       nil,
+							NullFields:            nil,
+						},
+						Items: []*customsearch.Result{},
+					})
+			},
+		},
 		{
 			name:   "test with results",
 			number: test.NewFakeUSNumber(),
@@ -229,7 +299,8 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			_ = os.Setenv("GOOGLECSE_CX", "fake_search_engine_id")
 			_ = os.Setenv("GOOGLE_API_KEY", "fake_api_key")
-			defer os.Clearenv()
+			defer os.Unsetenv("GOOGLECSE_CX")
+			defer os.Unsetenv("GOOGLE_API_KEY")
 
 			tt.mocks()
 			defer gock.Off() // Flush pending mocks after test execution
@@ -238,11 +309,11 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) {
 			remote := NewLibrary(filter.NewEngine())
 			remote.AddScanner(scanner)
 
-			if scanner.DryRun(*tt.number) != nil {
+			if scanner.DryRun(*tt.number, tt.opts) != nil {
 				t.Fatal("DryRun() should return nil")
 			}
 
-			got, errs := remote.Scan(tt.number)
+			got, errs := remote.Scan(tt.number, tt.opts)
 			if len(tt.wantErrors) > 0 {
 				assert.Equal(t, tt.wantErrors, errs)
 			} else {
@@ -259,12 +330,22 @@ func TestGoogleCSEScanner_DryRun(t *testing.T) {
 	defer os.Unsetenv("GOOGLECSE_CX")
 	defer os.Unsetenv("GOOGLE_API_KEY")
 	scanner := NewGoogleCSEScanner(&http.Client{})
-	assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber()))
+	assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{}))
+}
+
+func TestGoogleCSEScanner_DryRunWithOptions(t *testing.T) {
+	errStr := "search engine ID and/or API key is not defined"
+
+	scanner := NewGoogleCSEScanner(&http.Client{})
+	assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLECSE_CX": "test", "GOOGLE_API_KEY": "secret"}))
+	assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLECSE_CX": "", "GOOGLE_API_KEY": ""}), errStr)
+	assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLECSE_CX": "test"}), errStr)
+	assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLE_API_KEY": "test"}), errStr)
 }
 
 func TestGoogleCSEScanner_DryRun_Error(t *testing.T) {
 	scanner := NewGoogleCSEScanner(&http.Client{})
-	assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber()), "search engine ID and/or API key is not defined")
+	assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{}), "search engine ID and/or API key is not defined")
 }
 
 func TestGoogleCSEScanner_MaxResults(t *testing.T) {

lib/remote/googlesearch_scanner.go

@@ -39,11 +39,11 @@ func (s *googlesearchScanner) Description() string {
 	return "Generate several Google dork requests for a given phone number."
 }
 
-func (s *googlesearchScanner) DryRun(_ number.Number) error {
+func (s *googlesearchScanner) DryRun(_ number.Number, _ ScannerOptions) error {
 	return nil
 }
 
-func (s *googlesearchScanner) Run(n number.Number) (interface{}, error) {
+func (s *googlesearchScanner) Run(n number.Number, _ ScannerOptions) (interface{}, error) {
 	res := GoogleSearchResponse{
 		SocialMedia:         getSocialMediaDorks(n),
 		DisposableProviders: getDisposableProvidersDorks(n),