From 644b520f77fbf0ccd44f9a1caf2279c6e410d15b Mon Sep 17 00:00:00 2001 From: sundowndev <16480203+sundowndev@users.noreply.github.com> Date: Thu, 15 Feb 2024 20:53:54 +0400 Subject: [PATCH 01/10] chore: update mockery tool --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9ff1d0475..27ab7265e 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ lint: .PHONY: install-tools install-tools: $(GOINSTALL) gotest.tools/gotestsum@v1.6.3 - $(GOINSTALL) github.com/vektra/mockery/v2@v2.8.0 + $(GOINSTALL) github.com/vektra/mockery/v2@v2.38.0 $(GOINSTALL) github.com/swaggo/swag/cmd/swag@v1.16.1 @which golangci-lint > /dev/null 2>&1 || (curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash -s -- -b $(GOBINPATH) v1.46.2) From 56fa6ea63c0077116788b8e3b037d848342a5d7b Mon Sep 17 00:00:00 2001 From: sundowndev <16480203+sundowndev@users.noreply.github.com> Date: Thu, 15 Feb 2024 20:54:44 +0400 Subject: [PATCH 02/10] docs: scanner options --- docs/getting-started/scanners.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/getting-started/scanners.md b/docs/getting-started/scanners.md index e1b384fa0..67602f267 100644 --- a/docs/getting-started/scanners.md +++ b/docs/getting-started/scanners.md @@ -19,6 +19,8 @@ GOOGLE_API_KEY="value" phoneinfoga scan -n +4176418xxxx --env-file=.env.local ``` +**HTTP API consumers**: You can also specify scanner options such as api keys on a per-request basis. Each scanner supports its own options, see below. For details on how to specify those options, see [API docs](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/sundowndev/phoneinfoga/master/web/docs/swagger.yaml). Scanner options will override environment variables for the current request. + ## Building your own scanner PhoneInfoga can now be extended with plugins! You can build your own scanner and PhoneInfoga will use it to scan the given phone number. @@ -63,9 +65,9 @@ Numverify provide standard but useful information such as country code, location 2. Go to "Number Verification API" in the marketplace, click on "Subscribe for free", then choose whatever plan you want 3. Copy the new API token and use it as an environment variable - | Environment variable | Default | Description | - |----------------------|---------|------------------------------------------------------------------------| - | NUMVERIFY_API_KEY | | API key to authenticate to the Numverify API. | + | Environment variable | Option | Default | Description | + |----------------------|------------|---------|-------------------------------------------------------| + | NUMVERIFY_API_KEY | api_key | | API key to authenticate to the Numverify API. | ??? example "Output example" @@ -209,11 +211,11 @@ Follow the steps below to create a new search engine : ??? info "Configuration" - | Environment variable | Default | Description | - |-----------------------|---------|------------------------------------------------------------------------| - | GOOGLECSE_CX | | Search engine ID. | - | GOOGLE_API_KEY | | API key to authenticate to the Google API. | - | GOOGLECSE_MAX_RESULTS | 10 | Maximum results for each request. Each 10 results requires an additional request. This value cannot go above 100. | + | Environment variable | Option | Default | Description | + |-----------------------|----------|----------|-------------------------------------------------------------| + | GOOGLECSE_CX | cx | | Search engine ID. | + | GOOGLE_API_KEY | api_key | | API key to authenticate to the Google API. | + | GOOGLECSE_MAX_RESULTS | | 10 | Maximum results for each request. Each 10 results requires an additional request. This value cannot go above 100. | ??? example "Output example" From 4726fffb561ce060a0adecee18a79ec1d0467898 Mon Sep 17 00:00:00 2001 From: sundowndev <16480203+sundowndev@users.noreply.github.com> Date: Thu, 15 Feb 2024 20:55:15 +0400 Subject: [PATCH 03/10] feat: implement scanner options --- examples/plugin/customscanner.go | 4 +-- examples/plugin/customscanner_test.go | 5 +-- lib/remote/googlecse_scanner.go | 34 +++++++++++++++----- lib/remote/googlecse_scanner_test.go | 9 +++--- lib/remote/googlesearch_scanner.go | 4 +-- lib/remote/googlesearch_scanner_test.go | 29 ++++++++--------- lib/remote/local_scanner.go | 4 +-- lib/remote/local_scanner_test.go | 2 +- lib/remote/numverify_scanner.go | 15 +++++++-- lib/remote/numverify_scanner_test.go | 21 +++++++------ lib/remote/ovh_scanner.go | 4 +-- lib/remote/ovh_scanner_test.go | 17 +++++----- lib/remote/remote.go | 4 +-- lib/remote/remote_test.go | 41 +++++++++++++------------ lib/remote/scanner.go | 6 ++-- lib/remote/suppliers/numverify.go | 13 ++++++-- lib/remote/suppliers/numverify_test.go | 41 +++++++++++++++++++++++-- web/controllers.go | 8 ++--- web/v2/api/handlers/scanners.go | 19 +++++++++--- web/v2/api/handlers/scanners_test.go | 40 +++++++++++++++++++++--- 20 files changed, 220 insertions(+), 100 deletions(-) diff --git a/examples/plugin/customscanner.go b/examples/plugin/customscanner.go index 0eaf6b7ff..499fc3af2 100644 --- a/examples/plugin/customscanner.go +++ b/examples/plugin/customscanner.go @@ -28,13 +28,13 @@ func (s *customScanner) Description() string { // This can be useful to check for authentication or // country code support for example, and avoid running // the scanner when it just can't work. -func (s *customScanner) DryRun(n number.Number) error { +func (s *customScanner) DryRun(n number.Number, opts remote.ScannerOptions) error { return nil } // Run does the actual scan of the phone number. // Note this function will be executed in a goroutine. -func (s *customScanner) Run(n number.Number) (interface{}, error) { +func (s *customScanner) Run(n number.Number, opts remote.ScannerOptions) (interface{}, error) { data := customScannerResponse{ Valid: true, Info: "This number is known for scams!", diff --git a/examples/plugin/customscanner_test.go b/examples/plugin/customscanner_test.go index 4c5691a97..dd475eadb 100644 --- a/examples/plugin/customscanner_test.go +++ b/examples/plugin/customscanner_test.go @@ -3,6 +3,7 @@ package main import ( "github.com/stretchr/testify/assert" "github.com/sundowndev/phoneinfoga/v2/lib/number" + "github.com/sundowndev/phoneinfoga/v2/lib/remote" "testing" ) @@ -37,11 +38,11 @@ func TestCustomScanner(t *testing.T) { t.Run(tt.name, func(t *testing.T) { scanner := &customScanner{} - if scanner.DryRun(*tt.number) != nil { + if scanner.DryRun(*tt.number, remote.ScannerOptions{}) != nil { t.Fatal("DryRun() should return nil") } - got, err := scanner.Run(*tt.number) + got, err := scanner.Run(*tt.number, remote.ScannerOptions{}) if tt.wantError != "" { assert.EqualError(t, err, tt.wantError) } else { diff --git a/lib/remote/googlecse_scanner.go b/lib/remote/googlecse_scanner.go index 58eecc29a..9b41dd143 100644 --- a/lib/remote/googlecse_scanner.go +++ b/lib/remote/googlecse_scanner.go @@ -67,24 +67,42 @@ func (s *googleCSEScanner) Description() string { return "Googlecse searches for footprints of a given phone number on the web using Google Custom Search Engine." } -func (s *googleCSEScanner) DryRun(_ number.Number) error { - if s.Cx == "" || s.ApiKey == "" { +func (s *googleCSEScanner) DryRun(_ number.Number, opts ScannerOptions) error { + var cx = s.Cx + var apikey = s.ApiKey + + if v, ok := opts["cx"].(string); ok { + cx = v + } + if v, ok := opts["api_key"].(string); ok { + apikey = v + } + if cx == "" || apikey == "" { return errors.New("search engine ID and/or API key is not defined") } return nil } -func (s *googleCSEScanner) Run(n number.Number) (interface{}, error) { +func (s *googleCSEScanner) Run(n number.Number, opts ScannerOptions) (interface{}, error) { var allItems []*customsearch.Result var dorks []*GoogleSearchDork var totalResultCount int var totalRequestCount int + var cx = s.Cx + var apikey = s.ApiKey + + if v, ok := opts["cx"].(string); ok { + cx = v + } + if v, ok := opts["api_key"].(string); ok { + apikey = v + } dorks = append(dorks, s.generateDorkQueries(n)...) customsearchService, err := customsearch.NewService( context.Background(), - option.WithAPIKey(s.ApiKey), + option.WithAPIKey(apikey), option.WithHTTPClient(s.httpClient), ) if err != nil { @@ -92,7 +110,7 @@ func (s *googleCSEScanner) Run(n number.Number) (interface{}, error) { } for _, req := range dorks { - n, items, err := s.search(customsearchService, req.Dork) + n, items, err := s.search(customsearchService, req.Dork, cx) if err != nil { if s.isRateLimit(err) { return nil, errors.New("rate limit exceeded, see https://developers.google.com/custom-search/v1/overview#pricing") @@ -111,7 +129,7 @@ func (s *googleCSEScanner) Run(n number.Number) (interface{}, error) { URL: item.Link, }) } - data.Homepage = fmt.Sprintf("https://cse.google.com/cse?cx=%s", s.Cx) + data.Homepage = fmt.Sprintf("https://cse.google.com/cse?cx=%s", cx) data.ResultCount = len(allItems) data.TotalResultCount = totalResultCount data.TotalRequestCount = totalRequestCount @@ -119,14 +137,14 @@ func (s *googleCSEScanner) Run(n number.Number) (interface{}, error) { return data, nil } -func (s *googleCSEScanner) search(service *customsearch.Service, q string) (int, []*customsearch.Result, error) { +func (s *googleCSEScanner) search(service *customsearch.Service, q string, cx string) (int, []*customsearch.Result, error) { var results []*customsearch.Result var totalResultCount int offset := int64(0) for offset < s.MaxResults { search := service.Cse.List() - search.Cx(s.Cx) + search.Cx(cx) search.Q(q) search.Start(offset) searchQuery, err := search.Do() diff --git a/lib/remote/googlecse_scanner_test.go b/lib/remote/googlecse_scanner_test.go index 0d3f7d97e..e21effa2a 100644 --- a/lib/remote/googlecse_scanner_test.go +++ b/lib/remote/googlecse_scanner_test.go @@ -229,7 +229,8 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) { t.Run(tt.name, func(t *testing.T) { _ = os.Setenv("GOOGLECSE_CX", "fake_search_engine_id") _ = os.Setenv("GOOGLE_API_KEY", "fake_api_key") - defer os.Clearenv() + defer os.Unsetenv("GOOGLECSE_CX") + defer os.Unsetenv("GOOGLE_API_KEY") tt.mocks() defer gock.Off() // Flush pending mocks after test execution @@ -238,7 +239,7 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) { remote := NewLibrary(filter.NewEngine()) remote.AddScanner(scanner) - if scanner.DryRun(*tt.number) != nil { + if scanner.DryRun(*tt.number, ScannerOptions{}) != nil { t.Fatal("DryRun() should return nil") } @@ -259,12 +260,12 @@ func TestGoogleCSEScanner_DryRun(t *testing.T) { defer os.Unsetenv("GOOGLECSE_CX") defer os.Unsetenv("GOOGLE_API_KEY") scanner := NewGoogleCSEScanner(&http.Client{}) - assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber())) + assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{})) } func TestGoogleCSEScanner_DryRun_Error(t *testing.T) { scanner := NewGoogleCSEScanner(&http.Client{}) - assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber()), "search engine ID and/or API key is not defined") + assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{}), "search engine ID and/or API key is not defined") } func TestGoogleCSEScanner_MaxResults(t *testing.T) { diff --git a/lib/remote/googlesearch_scanner.go b/lib/remote/googlesearch_scanner.go index 42380b186..885ea621b 100644 --- a/lib/remote/googlesearch_scanner.go +++ b/lib/remote/googlesearch_scanner.go @@ -39,11 +39,11 @@ func (s *googlesearchScanner) Description() string { return "Generate several Google dork requests for a given phone number." } -func (s *googlesearchScanner) DryRun(_ number.Number) error { +func (s *googlesearchScanner) DryRun(_ number.Number, _ ScannerOptions) error { return nil } -func (s *googlesearchScanner) Run(n number.Number) (interface{}, error) { +func (s *googlesearchScanner) Run(n number.Number, _ ScannerOptions) (interface{}, error) { res := GoogleSearchResponse{ SocialMedia: getSocialMediaDorks(n), DisposableProviders: getDisposableProvidersDorks(n), diff --git a/lib/remote/googlesearch_scanner_test.go b/lib/remote/googlesearch_scanner_test.go index 5854ae691..40b2919ba 100644 --- a/lib/remote/googlesearch_scanner_test.go +++ b/lib/remote/googlesearch_scanner_test.go @@ -1,15 +1,16 @@ -package remote +package remote_test import ( "github.com/stretchr/testify/assert" "github.com/sundowndev/phoneinfoga/v2/lib/filter" "github.com/sundowndev/phoneinfoga/v2/lib/number" + "github.com/sundowndev/phoneinfoga/v2/lib/remote" "testing" ) func TestGoogleSearchScanner_Metadata(t *testing.T) { - scanner := NewGoogleSearchScanner() - assert.Equal(t, Googlesearch, scanner.Name()) + scanner := remote.NewGoogleSearchScanner() + assert.Equal(t, remote.Googlesearch, scanner.Name()) assert.NotEmpty(t, scanner.Description()) } @@ -27,8 +28,8 @@ func TestGoogleSearchScanner(t *testing.T) { return n }(), expected: map[string]interface{}{ - "googlesearch": GoogleSearchResponse{ - SocialMedia: []*GoogleSearchDork{ + "googlesearch": remote.GoogleSearchResponse{ + SocialMedia: []*remote.GoogleSearchDork{ { Number: "+15556661212", Dork: "site:facebook.com intext:\"15556661212\" | intext:\"+15556661212\" | intext:\"5556661212\"", @@ -55,7 +56,7 @@ func TestGoogleSearchScanner(t *testing.T) { URL: "https://www.google.com/search?q=site%3Avk.com+intext%3A%2215556661212%22+%7C+intext%3A%22%2B15556661212%22+%7C+intext%3A%225556661212%22", }, }, - DisposableProviders: []*GoogleSearchDork{ + DisposableProviders: []*remote.GoogleSearchDork{ { Number: "+15556661212", Dork: "site:hs3x.com intext:\"15556661212\"", @@ -161,7 +162,7 @@ func TestGoogleSearchScanner(t *testing.T) { URL: "https://www.google.com/search?q=site%3Asmslive.co+intext%3A%2215556661212%22+%7C+intext%3A%225556661212%22", }, }, - Reputation: []*GoogleSearchDork{ + Reputation: []*remote.GoogleSearchDork{ { Number: "+15556661212", Dork: "site:whosenumber.info intext:\"+15556661212\" intitle:\"who called\"", @@ -213,7 +214,7 @@ func TestGoogleSearchScanner(t *testing.T) { URL: "https://www.google.com/search?q=site%3Auk.popularphotolook.com+inurl%3A%225556661212%22", }, }, - Individuals: []*GoogleSearchDork{ + Individuals: []*remote.GoogleSearchDork{ { Number: "+15556661212", Dork: "site:numinfo.net intext:\"15556661212\" | intext:\"+15556661212\" | intext:\"5556661212\"", @@ -250,7 +251,7 @@ func TestGoogleSearchScanner(t *testing.T) { URL: "https://www.google.com/search?q=site%3Aspytox.com+intext%3A%225556661212%22", }, }, - General: []*GoogleSearchDork{ + General: []*remote.GoogleSearchDork{ { Number: "+15556661212", Dork: "intext:\"15556661212\" | intext:\"+15556661212\" | intext:\"5556661212\" | intext:\"(555) 666-1212\"", @@ -270,15 +271,15 @@ func TestGoogleSearchScanner(t *testing.T) { for _, tt := range testcases { t.Run(tt.name, func(t *testing.T) { - scanner := NewGoogleSearchScanner() - remote := NewLibrary(filter.NewEngine()) - remote.AddScanner(scanner) + scanner := remote.NewGoogleSearchScanner() + lib := remote.NewLibrary(filter.NewEngine()) + lib.AddScanner(scanner) - if scanner.DryRun(*tt.number) != nil { + if scanner.DryRun(*tt.number, remote.ScannerOptions{}) != nil { t.Fatal("DryRun() should return nil") } - got, errs := remote.Scan(tt.number) + got, errs := lib.Scan(tt.number) if len(tt.wantErrors) > 0 { assert.Equal(t, tt.wantErrors, errs) } else { diff --git a/lib/remote/local_scanner.go b/lib/remote/local_scanner.go index 5eb68f20e..1b9baaa54 100644 --- a/lib/remote/local_scanner.go +++ b/lib/remote/local_scanner.go @@ -30,11 +30,11 @@ func (s *localScanner) Description() string { return "Gather offline info about a given phone number." } -func (s *localScanner) DryRun(_ number.Number) error { +func (s *localScanner) DryRun(_ number.Number, _ ScannerOptions) error { return nil } -func (s *localScanner) Run(n number.Number) (interface{}, error) { +func (s *localScanner) Run(n number.Number, _ ScannerOptions) (interface{}, error) { data := LocalScannerResponse{ RawLocal: n.RawLocal, Local: n.Local, diff --git a/lib/remote/local_scanner_test.go b/lib/remote/local_scanner_test.go index 05b7ebeeb..a50f559cd 100644 --- a/lib/remote/local_scanner_test.go +++ b/lib/remote/local_scanner_test.go @@ -45,7 +45,7 @@ func TestLocalScanner(t *testing.T) { remote := NewLibrary(filter.NewEngine()) remote.AddScanner(scanner) - if scanner.DryRun(*tt.number) != nil { + if scanner.DryRun(*tt.number, map[string]interface{}{}) != nil { t.Fatal("DryRun() should return nil") } diff --git a/lib/remote/numverify_scanner.go b/lib/remote/numverify_scanner.go index a48b7ad2c..2ef516b58 100644 --- a/lib/remote/numverify_scanner.go +++ b/lib/remote/numverify_scanner.go @@ -37,15 +37,24 @@ func (s *numverifyScanner) Description() string { return "Request info about a given phone number through the Numverify API." } -func (s *numverifyScanner) DryRun(_ number.Number) error { +func (s *numverifyScanner) DryRun(_ number.Number, opts ScannerOptions) error { + if _, ok := opts["api_key"]; ok { + return nil + } if !s.client.IsAvailable() { return errors.New("API key is not defined") } return nil } -func (s *numverifyScanner) Run(n number.Number) (interface{}, error) { - res, err := s.client.Validate(n.International) +func (s *numverifyScanner) Run(n number.Number, opts ScannerOptions) (interface{}, error) { + var apiKey string + + if v, ok := opts["api_key"].(string); ok { + apiKey = v + } + + res, err := s.client.Validate(n.International, apiKey) if err != nil { return nil, err } diff --git a/lib/remote/numverify_scanner_test.go b/lib/remote/numverify_scanner_test.go index 307f1c7ff..ea73bf447 100644 --- a/lib/remote/numverify_scanner_test.go +++ b/lib/remote/numverify_scanner_test.go @@ -1,18 +1,19 @@ -package remote +package remote_test import ( "errors" "github.com/stretchr/testify/assert" "github.com/sundowndev/phoneinfoga/v2/lib/filter" "github.com/sundowndev/phoneinfoga/v2/lib/number" + "github.com/sundowndev/phoneinfoga/v2/lib/remote" "github.com/sundowndev/phoneinfoga/v2/lib/remote/suppliers" "github.com/sundowndev/phoneinfoga/v2/mocks" "testing" ) func TestNumverifyScanner_Metadata(t *testing.T) { - scanner := NewNumverifyScanner(&mocks.NumverifySupplier{}) - assert.Equal(t, Numverify, scanner.Name()) + scanner := remote.NewNumverifyScanner(&mocks.NumverifySupplier{}) + assert.Equal(t, remote.Numverify, scanner.Name()) assert.NotEmpty(t, scanner.Description()) } @@ -34,7 +35,7 @@ func TestNumverifyScanner(t *testing.T) { }(), mocks: func(s *mocks.NumverifySupplier) { s.On("IsAvailable").Return(true) - s.On("Validate", "15556661212").Return(&suppliers.NumverifyValidateResponse{ + s.On("Validate", "15556661212", "").Return(&suppliers.NumverifyValidateResponse{ Valid: true, Number: "test", LocalFormat: "test", @@ -48,7 +49,7 @@ func TestNumverifyScanner(t *testing.T) { }, nil).Once() }, expected: map[string]interface{}{ - "numverify": NumverifyScannerResponse{ + "numverify": remote.NumverifyScannerResponse{ Valid: true, Number: "test", LocalFormat: "test", @@ -71,7 +72,7 @@ func TestNumverifyScanner(t *testing.T) { }(), mocks: func(s *mocks.NumverifySupplier) { s.On("IsAvailable").Return(true) - s.On("Validate", "15556661212").Return(nil, dummyError).Once() + s.On("Validate", "15556661212", "").Return(nil, dummyError).Once() }, expected: map[string]interface{}{}, wantErrors: map[string]error{ @@ -97,11 +98,11 @@ func TestNumverifyScanner(t *testing.T) { numverifySupplierMock := &mocks.NumverifySupplier{} tt.mocks(numverifySupplierMock) - scanner := NewNumverifyScanner(numverifySupplierMock) - remote := NewLibrary(filter.NewEngine()) - remote.AddScanner(scanner) + scanner := remote.NewNumverifyScanner(numverifySupplierMock) + lib := remote.NewLibrary(filter.NewEngine()) + lib.AddScanner(scanner) - got, errs := remote.Scan(tt.number) + got, errs := lib.Scan(tt.number) if len(tt.wantErrors) > 0 { assert.Equal(t, tt.wantErrors, errs) } else { diff --git a/lib/remote/ovh_scanner.go b/lib/remote/ovh_scanner.go index f435ea877..eea2be386 100644 --- a/lib/remote/ovh_scanner.go +++ b/lib/remote/ovh_scanner.go @@ -32,14 +32,14 @@ func (s *ovhScanner) Description() string { return "Search a phone number through the OVH Telecom REST API." } -func (s *ovhScanner) DryRun(n number.Number) error { +func (s *ovhScanner) DryRun(n number.Number, _ ScannerOptions) error { if !s.isSupported(n.CountryCode) { return fmt.Errorf("country code %d is not supported", n.CountryCode) } return nil } -func (s *ovhScanner) Run(n number.Number) (interface{}, error) { +func (s *ovhScanner) Run(n number.Number, _ ScannerOptions) (interface{}, error) { res, err := s.client.Search(n) if err != nil { return nil, err diff --git a/lib/remote/ovh_scanner_test.go b/lib/remote/ovh_scanner_test.go index dfc084039..79b3d5cf2 100644 --- a/lib/remote/ovh_scanner_test.go +++ b/lib/remote/ovh_scanner_test.go @@ -1,18 +1,19 @@ -package remote +package remote_test import ( "errors" "github.com/stretchr/testify/assert" "github.com/sundowndev/phoneinfoga/v2/lib/filter" "github.com/sundowndev/phoneinfoga/v2/lib/number" + "github.com/sundowndev/phoneinfoga/v2/lib/remote" "github.com/sundowndev/phoneinfoga/v2/lib/remote/suppliers" "github.com/sundowndev/phoneinfoga/v2/mocks" "testing" ) func TestOVHScanner_Metadata(t *testing.T) { - scanner := NewOVHScanner(&mocks.OVHSupplier{}) - assert.Equal(t, OVH, scanner.Name()) + scanner := remote.NewOVHScanner(&mocks.OVHSupplier{}) + assert.Equal(t, remote.OVH, scanner.Name()) assert.NotEmpty(t, scanner.Description()) } @@ -39,7 +40,7 @@ func TestOVHScanner(t *testing.T) { }, nil).Once() }, expected: map[string]interface{}{ - "ovh": OVHScannerResponse{ + "ovh": remote.OVHScannerResponse{ Found: false, }, }, @@ -75,11 +76,11 @@ func TestOVHScanner(t *testing.T) { OVHSupplierMock := &mocks.OVHSupplier{} tt.mocks(OVHSupplierMock) - scanner := NewOVHScanner(OVHSupplierMock) - remote := NewLibrary(filter.NewEngine()) - remote.AddScanner(scanner) + scanner := remote.NewOVHScanner(OVHSupplierMock) + lib := remote.NewLibrary(filter.NewEngine()) + lib.AddScanner(scanner) - got, errs := remote.Scan(tt.number) + got, errs := lib.Scan(tt.number) if len(tt.wantErrors) > 0 { assert.Equal(t, tt.wantErrors, errs) } else { diff --git a/lib/remote/remote.go b/lib/remote/remote.go index 40ee543c3..f12a0578e 100644 --- a/lib/remote/remote.go +++ b/lib/remote/remote.go @@ -69,7 +69,7 @@ func (r *Library) Scan(n *number.Number) (map[string]interface{}, map[string]err } }() - if err := s.DryRun(*n); err != nil { + if err := s.DryRun(*n, make(ScannerOptions)); err != nil { logrus. WithField("scanner", s.Name()). WithField("reason", err.Error()). @@ -77,7 +77,7 @@ func (r *Library) Scan(n *number.Number) (map[string]interface{}, map[string]err return } - data, err := s.Run(*n) + data, err := s.Run(*n, make(ScannerOptions)) if err != nil { r.addError(s.Name(), err) return diff --git a/lib/remote/remote_test.go b/lib/remote/remote_test.go index 3d9dc9cd1..12e5f7141 100644 --- a/lib/remote/remote_test.go +++ b/lib/remote/remote_test.go @@ -1,10 +1,11 @@ -package remote +package remote_test import ( "errors" "github.com/stretchr/testify/assert" "github.com/sundowndev/phoneinfoga/v2/lib/filter" "github.com/sundowndev/phoneinfoga/v2/lib/number" + "github.com/sundowndev/phoneinfoga/v2/lib/remote" "github.com/sundowndev/phoneinfoga/v2/mocks" "testing" ) @@ -25,16 +26,16 @@ func TestRemoteLibrary_SuccessScan(t *testing.T) { } fakeScanner := &mocks.Scanner{} - fakeScanner.On("DryRun", *num).Return(nil).Once() fakeScanner.On("Name").Return("fake").Times(2) - fakeScanner.On("Run", *num).Return(fakeScannerResponse{Valid: true}, nil).Once() + fakeScanner.On("DryRun", *num, remote.ScannerOptions{}).Return(nil).Once() + fakeScanner.On("Run", *num, remote.ScannerOptions{}).Return(fakeScannerResponse{Valid: true}, nil).Once() fakeScanner2 := &mocks.Scanner{} - fakeScanner2.On("DryRun", *num).Return(nil).Once() fakeScanner2.On("Name").Return("fake2").Times(2) - fakeScanner2.On("Run", *num).Return(fakeScannerResponse{Valid: false}, nil).Once() + fakeScanner2.On("DryRun", *num, remote.ScannerOptions{}).Return(nil).Once() + fakeScanner2.On("Run", *num, remote.ScannerOptions{}).Return(fakeScannerResponse{Valid: false}, nil).Once() - lib := NewLibrary(filter.NewEngine()) + lib := remote.NewLibrary(filter.NewEngine()) lib.AddScanner(fakeScanner) lib.AddScanner(fakeScanner2) @@ -56,11 +57,11 @@ func TestRemoteLibrary_FailedScan(t *testing.T) { dummyError := errors.New("test") fakeScanner := &mocks.Scanner{} - fakeScanner.On("DryRun", *num).Return(nil).Once() fakeScanner.On("Name").Return("fake").Times(2) - fakeScanner.On("Run", *num).Return(nil, dummyError).Once() + fakeScanner.On("DryRun", *num, remote.ScannerOptions{}).Return(nil).Once() + fakeScanner.On("Run", *num, remote.ScannerOptions{}).Return(nil, dummyError).Once() - lib := NewLibrary(filter.NewEngine()) + lib := remote.NewLibrary(filter.NewEngine()) lib.AddScanner(fakeScanner) @@ -79,9 +80,9 @@ func TestRemoteLibrary_EmptyScan(t *testing.T) { fakeScanner := &mocks.Scanner{} fakeScanner.On("Name").Return("mockscanner").Times(2) - fakeScanner.On("DryRun", *num).Return(errors.New("dummy error")).Once() + fakeScanner.On("DryRun", *num, remote.ScannerOptions{}).Return(errors.New("dummy error")).Once() - lib := NewLibrary(filter.NewEngine()) + lib := remote.NewLibrary(filter.NewEngine()) lib.AddScanner(fakeScanner) @@ -100,10 +101,10 @@ func TestRemoteLibrary_PanicRun(t *testing.T) { fakeScanner := &mocks.Scanner{} fakeScanner.On("Name").Return("fake") - fakeScanner.On("DryRun", *num).Return(nil).Once() - fakeScanner.On("Run", *num).Panic("dummy panic").Once() + fakeScanner.On("DryRun", *num, remote.ScannerOptions{}).Return(nil).Once() + fakeScanner.On("Run", *num, remote.ScannerOptions{}).Panic("dummy panic").Once() - lib := NewLibrary(filter.NewEngine()) + lib := remote.NewLibrary(filter.NewEngine()) lib.AddScanner(fakeScanner) @@ -122,9 +123,9 @@ func TestRemoteLibrary_PanicDryRun(t *testing.T) { fakeScanner := &mocks.Scanner{} fakeScanner.On("Name").Return("fake") - fakeScanner.On("DryRun", *num).Panic("dummy panic").Once() + fakeScanner.On("DryRun", *num, remote.ScannerOptions{}).Panic("dummy panic").Once() - lib := NewLibrary(filter.NewEngine()) + lib := remote.NewLibrary(filter.NewEngine()) lib.AddScanner(fakeScanner) @@ -142,12 +143,12 @@ func TestRemoteLibrary_GetAllScanners(t *testing.T) { fakeScanner2 := &mocks.Scanner{} fakeScanner2.On("Name").Return("fake2") - lib := NewLibrary(filter.NewEngine()) + lib := remote.NewLibrary(filter.NewEngine()) lib.AddScanner(fakeScanner) lib.AddScanner(fakeScanner2) - assert.Equal(t, []Scanner{fakeScanner, fakeScanner2}, lib.GetAllScanners()) + assert.Equal(t, []remote.Scanner{fakeScanner, fakeScanner2}, lib.GetAllScanners()) } func TestRemoteLibrary_AddIgnoredScanner(t *testing.T) { @@ -159,10 +160,10 @@ func TestRemoteLibrary_AddIgnoredScanner(t *testing.T) { f := filter.NewEngine() f.AddRule("fake2") - lib := NewLibrary(f) + lib := remote.NewLibrary(f) lib.AddScanner(fakeScanner) lib.AddScanner(fakeScanner2) - assert.Equal(t, []Scanner{fakeScanner}, lib.GetAllScanners()) + assert.Equal(t, []remote.Scanner{fakeScanner}, lib.GetAllScanners()) } diff --git a/lib/remote/scanner.go b/lib/remote/scanner.go index 5532eef0f..a7911204b 100644 --- a/lib/remote/scanner.go +++ b/lib/remote/scanner.go @@ -8,6 +8,8 @@ import ( "github.com/sundowndev/phoneinfoga/v2/lib/number" ) +type ScannerOptions map[string]interface{} + type Plugin interface { Lookup(string) (plugin.Symbol, error) } @@ -15,8 +17,8 @@ type Plugin interface { type Scanner interface { Name() string Description() string - DryRun(number.Number) error - Run(number.Number) (interface{}, error) + DryRun(number.Number, ScannerOptions) error + Run(number.Number, ScannerOptions) (interface{}, error) } func OpenPlugin(path string) error { diff --git a/lib/remote/suppliers/numverify.go b/lib/remote/suppliers/numverify.go index 23fa11b9c..df2a8b06e 100644 --- a/lib/remote/suppliers/numverify.go +++ b/lib/remote/suppliers/numverify.go @@ -12,7 +12,7 @@ import ( type NumverifySupplierInterface interface { IsAvailable() bool - Validate(string) (*NumverifyValidateResponse, error) + Validate(string, string) (*NumverifyValidateResponse, error) } type NumverifyErrorResponse struct { @@ -47,7 +47,14 @@ func (s *NumverifySupplier) IsAvailable() bool { return s.ApiKey != "" } -func (s *NumverifySupplier) Validate(internationalNumber string) (res *NumverifyValidateResponse, err error) { +func (s *NumverifySupplier) Validate(internationalNumber string, customApiKey string) (res *NumverifyValidateResponse, err error) { + apiKey := s.ApiKey + + // User-provided credentials + if customApiKey != "" { + apiKey = customApiKey + } + logrus. WithField("number", internationalNumber). Debug("Running validate operation through Numverify API") @@ -57,7 +64,7 @@ func (s *NumverifySupplier) Validate(internationalNumber string) (res *Numverify // Build the request client := &http.Client{} req, _ := http.NewRequest("GET", url, nil) - req.Header.Set("Apikey", s.ApiKey) + req.Header.Set("Apikey", apiKey) response, err := client.Do(req) diff --git a/lib/remote/suppliers/numverify_test.go b/lib/remote/suppliers/numverify_test.go index 367d5cb07..2f1b9bebd 100644 --- a/lib/remote/suppliers/numverify_test.go +++ b/lib/remote/suppliers/numverify_test.go @@ -41,7 +41,42 @@ func TestNumverifySupplierSuccess(t *testing.T) { assert.True(t, s.IsAvailable()) - got, err := s.Validate(number) + got, err := s.Validate(number, "") + assert.Nil(t, err) + + assert.Equal(t, expectedResult, got) +} + +func TestNumverifySupplierSuccessCustomApiKey(t *testing.T) { + defer gock.Off() // Flush pending mocks after test execution + + number := "11115551212" + + expectedResult := &NumverifyValidateResponse{ + Valid: true, + Number: "79516566591", + LocalFormat: "9516566591", + InternationalFormat: "+79516566591", + CountryPrefix: "+7", + CountryCode: "RU", + CountryName: "Russian Federation", + Location: "Saint Petersburg and Leningrad Oblast", + Carrier: "OJSC St. Petersburg Telecom (OJSC Tele2-Saint-Petersburg)", + LineType: "mobile", + } + + gock.New("https://api.apilayer.com"). + Get("/number_verification/validate"). + MatchHeader("Apikey", "5ad5554ac240e4d3d31107941b35a5eb"). + MatchParam("number", number). + Reply(200). + JSON(expectedResult) + + s := NewNumverifySupplier() + + assert.False(t, s.IsAvailable()) + + got, err := s.Validate(number, "5ad5554ac240e4d3d31107941b35a5eb") assert.Nil(t, err) assert.Equal(t, expectedResult, got) @@ -70,7 +105,7 @@ func TestNumverifySupplierError(t *testing.T) { assert.True(t, s.IsAvailable()) - got, err := s.Validate(number) + got, err := s.Validate(number, "") assert.Nil(t, got) assert.Equal(t, errors.New("You have exceeded your daily\\/monthly API rate limit. Please review and upgrade your subscription plan at https:\\/\\/apilayer.com\\/subscriptions to continue."), err) } @@ -93,7 +128,7 @@ func TestNumverifySupplierHTTPError(t *testing.T) { assert.True(t, s.IsAvailable()) - got, err := s.Validate(number) + got, err := s.Validate(number, "") assert.Nil(t, got) assert.Equal(t, &url.Error{ Op: "Get", diff --git a/web/controllers.go b/web/controllers.go index b2b61f642..1cd774ec5 100644 --- a/web/controllers.go +++ b/web/controllers.go @@ -76,7 +76,7 @@ func localScan(c *gin.Context) { return } - result, err := remote.NewLocalScanner().Run(*num) + result, err := remote.NewLocalScanner().Run(*num, make(remote.ScannerOptions)) if err != nil { handleError(c, errors.NewInternalError(err)) return @@ -104,7 +104,7 @@ func numverifyScan(c *gin.Context) { return } - result, err := remote.NewNumverifyScanner(suppliers.NewNumverifySupplier()).Run(*num) + result, err := remote.NewNumverifyScanner(suppliers.NewNumverifySupplier()).Run(*num, make(remote.ScannerOptions)) if err != nil { handleError(c, errors.NewInternalError(err)) return @@ -132,7 +132,7 @@ func googleSearchScan(c *gin.Context) { return } - result, err := remote.NewGoogleSearchScanner().Run(*num) + result, err := remote.NewGoogleSearchScanner().Run(*num, make(remote.ScannerOptions)) if err != nil { handleError(c, errors.NewInternalError(err)) return @@ -160,7 +160,7 @@ func ovhScan(c *gin.Context) { return } - result, err := remote.NewOVHScanner(suppliers.NewOVHSupplier()).Run(*num) + result, err := remote.NewOVHScanner(suppliers.NewOVHSupplier()).Run(*num, make(remote.ScannerOptions)) if err != nil { handleError(c, errors.NewInternalError(err)) return diff --git a/web/v2/api/handlers/scanners.go b/web/v2/api/handlers/scanners.go index 5294a2462..4d54a1b80 100644 --- a/web/v2/api/handlers/scanners.go +++ b/web/v2/api/handlers/scanners.go @@ -3,6 +3,7 @@ package handlers import ( "github.com/gin-gonic/gin" "github.com/sundowndev/phoneinfoga/v2/lib/number" + "github.com/sundowndev/phoneinfoga/v2/lib/remote" "github.com/sundowndev/phoneinfoga/v2/web/v2/api" "net/http" ) @@ -43,7 +44,8 @@ func GetAllScanners(*gin.Context) *api.Response { } type DryRunScannerInput struct { - Number string `json:"number" binding:"number,required"` + Number string `json:"number" binding:"number,required"` + Options remote.ScannerOptions `json:"options" validate:"dive,required"` } type DryRunScannerResponse struct { @@ -74,6 +76,10 @@ func DryRunScanner(ctx *gin.Context) *api.Response { } } + if input.Options == nil { + input.Options = make(remote.ScannerOptions) + } + scanner := RemoteLibrary.GetScanner(ctx.Param("scanner")) if scanner == nil { return &api.Response{ @@ -92,7 +98,7 @@ func DryRunScanner(ctx *gin.Context) *api.Response { } } - err = scanner.DryRun(*num) + err = scanner.DryRun(*num, input.Options) if err != nil { return &api.Response{ Code: http.StatusBadRequest, @@ -114,7 +120,8 @@ func DryRunScanner(ctx *gin.Context) *api.Response { } type RunScannerInput struct { - Number string `json:"number" binding:"number,required"` + Number string `json:"number" binding:"number,required"` + Options remote.ScannerOptions `json:"options" validate:"dive,required"` } type RunScannerResponse struct { @@ -144,6 +151,10 @@ func RunScanner(ctx *gin.Context) *api.Response { } } + if input.Options == nil { + input.Options = make(remote.ScannerOptions) + } + scanner := RemoteLibrary.GetScanner(ctx.Param("scanner")) if scanner == nil { return &api.Response{ @@ -162,7 +173,7 @@ func RunScanner(ctx *gin.Context) *api.Response { } } - result, err := scanner.Run(*num) + result, err := scanner.Run(*num, input.Options) if err != nil { return &api.Response{ Code: http.StatusInternalServerError, diff --git a/web/v2/api/handlers/scanners_test.go b/web/v2/api/handlers/scanners_test.go index 9cc67c564..39ba5931b 100644 --- a/web/v2/api/handlers/scanners_test.go +++ b/web/v2/api/handlers/scanners_test.go @@ -100,7 +100,39 @@ func TestDryRunScanner(t *testing.T) { }, Mocks: func(s *mocks.Scanner) { s.On("Name").Return("fakeScanner") - s.On("DryRun", *test.NewFakeUSNumber()).Return(nil) + s.On("DryRun", *test.NewFakeUSNumber(), remote.ScannerOptions{}).Return(nil) + }, + }, + { + Name: "test dry running scanner with options", + Params: params{Supplier: "fakeScanner"}, + Body: handlers.DryRunScannerInput{ + Number: "14152229670", + Options: remote.ScannerOptions{"api_key": "secret"}, + }, + Expected: expectedResponse{ + Code: 200, + Body: handlers.DryRunScannerResponse{Success: true}, + }, + Mocks: func(s *mocks.Scanner) { + s.On("Name").Return("fakeScanner") + s.On("DryRun", *test.NewFakeUSNumber(), remote.ScannerOptions{"api_key": "secret"}).Return(nil) + }, + }, + { + Name: "test dry running scanner with empty options", + Params: params{Supplier: "fakeScanner"}, + Body: handlers.DryRunScannerInput{ + Number: "14152229670", + Options: remote.ScannerOptions{}, + }, + Expected: expectedResponse{ + Code: 200, + Body: handlers.DryRunScannerResponse{Success: true}, + }, + Mocks: func(s *mocks.Scanner) { + s.On("Name").Return("fakeScanner") + s.On("DryRun", *test.NewFakeUSNumber(), remote.ScannerOptions{}).Return(nil) }, }, { @@ -113,7 +145,7 @@ func TestDryRunScanner(t *testing.T) { }, Mocks: func(s *mocks.Scanner) { s.On("Name").Return("fakeScanner") - s.On("DryRun", *test.NewFakeUSNumber()).Return(errors.New("dummy error")) + s.On("DryRun", *test.NewFakeUSNumber(), make(remote.ScannerOptions)).Return(errors.New("dummy error")) }, }, { @@ -218,7 +250,7 @@ func TestRunScanner(t *testing.T) { }, Mocks: func(s *mocks.Scanner) { s.On("Name").Return("fakeScanner") - s.On("Run", *test.NewFakeUSNumber()).Return(FakeScannerResponse{Info: "test"}, nil) + s.On("Run", *test.NewFakeUSNumber(), remote.ScannerOptions{}).Return(FakeScannerResponse{Info: "test"}, nil) }, }, { @@ -231,7 +263,7 @@ func TestRunScanner(t *testing.T) { }, Mocks: func(s *mocks.Scanner) { s.On("Name").Return("fakeScanner") - s.On("Run", *test.NewFakeUSNumber()).Return(nil, errors.New("dummy error")) + s.On("Run", *test.NewFakeUSNumber(), remote.ScannerOptions{}).Return(nil, errors.New("dummy error")) }, }, { From 514e889cf598b85d45783e70103544ee74bb38e0 Mon Sep 17 00:00:00 2001 From: sundowndev <16480203+sundowndev@users.noreply.github.com> Date: Thu, 15 Feb 2024 20:55:23 +0400 Subject: [PATCH 04/10] chore: update mocks --- mocks/NumverifySupplier.go | 43 ++++++++++++++++++++++------ mocks/Scanner.go | 57 +++++++++++++++++++++++++------------- 2 files changed, 72 insertions(+), 28 deletions(-) diff --git a/mocks/NumverifySupplier.go b/mocks/NumverifySupplier.go index 9bbb04691..77606f53b 100644 --- a/mocks/NumverifySupplier.go +++ b/mocks/NumverifySupplier.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.8.0. DO NOT EDIT. +// Code generated by mockery v2.38.0. DO NOT EDIT. package mocks @@ -16,6 +16,10 @@ type NumverifySupplier struct { func (_m *NumverifySupplier) IsAvailable() bool { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for IsAvailable") + } + var r0 bool if rf, ok := ret.Get(0).(func() bool); ok { r0 = rf() @@ -26,25 +30,46 @@ func (_m *NumverifySupplier) IsAvailable() bool { return r0 } -// Validate provides a mock function with given fields: _a0 -func (_m *NumverifySupplier) Validate(_a0 string) (*suppliers.NumverifyValidateResponse, error) { - ret := _m.Called(_a0) +// Validate provides a mock function with given fields: _a0, _a1 +func (_m *NumverifySupplier) Validate(_a0 string, _a1 string) (*suppliers.NumverifyValidateResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for Validate") + } var r0 *suppliers.NumverifyValidateResponse - if rf, ok := ret.Get(0).(func(string) *suppliers.NumverifyValidateResponse); ok { - r0 = rf(_a0) + var r1 error + if rf, ok := ret.Get(0).(func(string, string) (*suppliers.NumverifyValidateResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(string, string) *suppliers.NumverifyValidateResponse); ok { + r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(*suppliers.NumverifyValidateResponse) } } - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(_a0) + if rf, ok := ret.Get(1).(func(string, string) error); ok { + r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) } return r0, r1 } + +// NewNumverifySupplier creates a new instance of NumverifySupplier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewNumverifySupplier(t interface { + mock.TestingT + Cleanup(func()) +}) *NumverifySupplier { + mock := &NumverifySupplier{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/mocks/Scanner.go b/mocks/Scanner.go index 72a5196a9..f6b6ea616 100644 --- a/mocks/Scanner.go +++ b/mocks/Scanner.go @@ -1,10 +1,11 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. +// Code generated by mockery v2.38.0. DO NOT EDIT. package mocks import ( mock "github.com/stretchr/testify/mock" number "github.com/sundowndev/phoneinfoga/v2/lib/number" + remote "github.com/sundowndev/phoneinfoga/v2/lib/remote" ) // Scanner is an autogenerated mock type for the Scanner type @@ -16,6 +17,10 @@ type Scanner struct { func (_m *Scanner) Description() string { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Description") + } + var r0 string if rf, ok := ret.Get(0).(func() string); ok { r0 = rf() @@ -26,13 +31,17 @@ func (_m *Scanner) Description() string { return r0 } -// DryRun provides a mock function with given fields: _a0 -func (_m *Scanner) DryRun(_a0 number.Number) error { - ret := _m.Called(_a0) +// DryRun provides a mock function with given fields: _a0, _a1 +func (_m *Scanner) DryRun(_a0 number.Number, _a1 remote.ScannerOptions) error { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for DryRun") + } var r0 error - if rf, ok := ret.Get(0).(func(number.Number) error); ok { - r0 = rf(_a0) + if rf, ok := ret.Get(0).(func(number.Number, remote.ScannerOptions) error); ok { + r0 = rf(_a0, _a1) } else { r0 = ret.Error(0) } @@ -44,6 +53,10 @@ func (_m *Scanner) DryRun(_a0 number.Number) error { func (_m *Scanner) Name() string { ret := _m.Called() + if len(ret) == 0 { + panic("no return value specified for Name") + } + var r0 string if rf, ok := ret.Get(0).(func() string); ok { r0 = rf() @@ -54,22 +67,29 @@ func (_m *Scanner) Name() string { return r0 } -// Run provides a mock function with given fields: _a0 -func (_m *Scanner) Run(_a0 number.Number) (interface{}, error) { - ret := _m.Called(_a0) +// Run provides a mock function with given fields: _a0, _a1 +func (_m *Scanner) Run(_a0 number.Number, _a1 remote.ScannerOptions) (interface{}, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for Run") + } var r0 interface{} - if rf, ok := ret.Get(0).(func(number.Number) interface{}); ok { - r0 = rf(_a0) + var r1 error + if rf, ok := ret.Get(0).(func(number.Number, remote.ScannerOptions) (interface{}, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(number.Number, remote.ScannerOptions) interface{}); ok { + r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { r0 = ret.Get(0).(interface{}) } } - var r1 error - if rf, ok := ret.Get(1).(func(number.Number) error); ok { - r1 = rf(_a0) + if rf, ok := ret.Get(1).(func(number.Number, remote.ScannerOptions) error); ok { + r1 = rf(_a0, _a1) } else { r1 = ret.Error(1) } @@ -77,13 +97,12 @@ func (_m *Scanner) Run(_a0 number.Number) (interface{}, error) { return r0, r1 } -type mockConstructorTestingTNewScanner interface { +// NewScanner creates a new instance of Scanner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewScanner(t interface { mock.TestingT Cleanup(func()) -} - -// NewScanner creates a new instance of Scanner. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewScanner(t mockConstructorTestingTNewScanner) *Scanner { +}) *Scanner { mock := &Scanner{} mock.Mock.Test(t) From 880a5a77c7a781371b8f7262f3558ab5046f16d8 Mon Sep 17 00:00:00 2001 From: sundowndev <16480203+sundowndev@users.noreply.github.com> Date: Thu, 15 Feb 2024 20:55:34 +0400 Subject: [PATCH 05/10] chore: update openapi specs --- web/docs/docs.go | 16 ++++++++++++++-- web/docs/swagger.json | 16 ++++++++++++++-- web/docs/swagger.yaml | 9 +++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/web/docs/docs.go b/web/docs/docs.go index 00dba7312..96cb0a541 100644 --- a/web/docs/docs.go +++ b/web/docs/docs.go @@ -519,11 +519,15 @@ const docTemplate = `{ "handlers.DryRunScannerInput": { "type": "object", "required": [ - "number" + "number", + "options" ], "properties": { "number": { "type": "string" + }, + "options": { + "$ref": "#/definitions/remote.ScannerOptions" } } }, @@ -552,11 +556,15 @@ const docTemplate = `{ "handlers.RunScannerInput": { "type": "object", "required": [ - "number" + "number", + "options" ], "properties": { "number": { "type": "string" + }, + "options": { + "$ref": "#/definitions/remote.ScannerOptions" } } }, @@ -707,6 +715,10 @@ const docTemplate = `{ } } }, + "remote.ScannerOptions": { + "type": "object", + "additionalProperties": true + }, "web.JSONResponse": { "type": "object", "properties": { diff --git a/web/docs/swagger.json b/web/docs/swagger.json index 92866cde5..5b1715dad 100644 --- a/web/docs/swagger.json +++ b/web/docs/swagger.json @@ -516,11 +516,15 @@ "handlers.DryRunScannerInput": { "type": "object", "required": [ - "number" + "number", + "options" ], "properties": { "number": { "type": "string" + }, + "options": { + "$ref": "#/definitions/remote.ScannerOptions" } } }, @@ -549,11 +553,15 @@ "handlers.RunScannerInput": { "type": "object", "required": [ - "number" + "number", + "options" ], "properties": { "number": { "type": "string" + }, + "options": { + "$ref": "#/definitions/remote.ScannerOptions" } } }, @@ -704,6 +712,10 @@ } } }, + "remote.ScannerOptions": { + "type": "object", + "additionalProperties": true + }, "web.JSONResponse": { "type": "object", "properties": { diff --git a/web/docs/swagger.yaml b/web/docs/swagger.yaml index af932b99f..6a99a4db9 100644 --- a/web/docs/swagger.yaml +++ b/web/docs/swagger.yaml @@ -35,8 +35,11 @@ definitions: properties: number: type: string + options: + $ref: '#/definitions/remote.ScannerOptions' required: - number + - options type: object handlers.DryRunScannerResponse: properties: @@ -56,8 +59,11 @@ definitions: properties: number: type: string + options: + $ref: '#/definitions/remote.ScannerOptions' required: - number + - options type: object handlers.RunScannerResponse: properties: @@ -155,6 +161,9 @@ definitions: zip_code: type: string type: object + remote.ScannerOptions: + additionalProperties: true + type: object web.JSONResponse: properties: error: From 7bec7f883214c6973c1ac78354ef9c3bebb67d84 Mon Sep 17 00:00:00 2001 From: sundowndev <16480203+sundowndev@users.noreply.github.com> Date: Thu, 15 Feb 2024 23:13:18 +0400 Subject: [PATCH 06/10] test(remote): scanner options improve coverage for scanner options --- cmd/scan.go | 3 +- lib/remote/googlecse_scanner_test.go | 84 ++++++++++++++++++++++++- lib/remote/googlesearch_scanner_test.go | 2 +- lib/remote/local_scanner_test.go | 4 +- lib/remote/numverify_scanner_test.go | 42 ++++++++++++- lib/remote/ovh_scanner_test.go | 2 +- lib/remote/remote.go | 6 +- lib/remote/remote_test.go | 10 +-- 8 files changed, 137 insertions(+), 16 deletions(-) diff --git a/cmd/scan.go b/cmd/scan.go index b2bc32474..b7f77eb33 100644 --- a/cmd/scan.go +++ b/cmd/scan.go @@ -80,7 +80,8 @@ func runScan(opts *ScanCmdOptions) { remoteLibrary := remote.NewLibrary(f) remote.InitScanners(remoteLibrary) - result, errs := remoteLibrary.Scan(num) + // Scanner options are currently not used in CLI + result, errs := remoteLibrary.Scan(num, remote.ScannerOptions{}) err = output.GetOutput(output.Console, color.Output).Write(result, errs) if err != nil { diff --git a/lib/remote/googlecse_scanner_test.go b/lib/remote/googlecse_scanner_test.go index e21effa2a..8507f803f 100644 --- a/lib/remote/googlecse_scanner_test.go +++ b/lib/remote/googlecse_scanner_test.go @@ -24,6 +24,7 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) { testcases := []struct { name string number *number.Number + opts ScannerOptions expected map[string]interface{} wantErrors map[string]error mocks func() @@ -89,6 +90,75 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) { }) }, }, + { + name: "test with options and no results", + number: test.NewFakeUSNumber(), + opts: ScannerOptions{ + "cx": "custom_cx", + "api_key": "secret", + }, + expected: map[string]interface{}{ + "googlecse": GoogleCSEScannerResponse{ + Homepage: "https://cse.google.com/cse?cx=custom_cx", + ResultCount: 0, + TotalResultCount: 0, + TotalRequestCount: 2, + Items: nil, + }, + }, + wantErrors: map[string]error{}, + mocks: func() { + gock.New("https://customsearch.googleapis.com"). + Get("/customsearch/v1"). + MatchParam("cx", "custom_cx"). + // TODO: ensure that custom api key is used + // MatchHeader("Authorization", "secret"). + // TODO: the matcher below doesn't work for some reason + //MatchParam("q", "intext:\"14152229670\" OR intext:\"+14152229670\" OR intext:\"4152229670\" OR intext:\"(415) 222-9670\""). + MatchParam("start", "0"). + Reply(200). + JSON(&customsearch.Search{ + ServerResponse: googleapi.ServerResponse{ + Header: http.Header{}, + HTTPStatusCode: 200, + }, + SearchInformation: &customsearch.SearchSearchInformation{ + FormattedSearchTime: "0", + FormattedTotalResults: "0", + SearchTime: 0, + TotalResults: "0", + ForceSendFields: nil, + NullFields: nil, + }, + Items: []*customsearch.Result{}, + }) + + gock.New("https://customsearch.googleapis.com"). + Get("/customsearch/v1"). + MatchParam("cx", "custom_cx"). + // TODO: ensure that custom api key is used + // MatchHeader("Authorization", "secret"). + // TODO: the matcher below doesn't work for some reason + //MatchParam("q", "(ext:doc OR ext:docx OR ext:odt OR ext:pdf OR ext:rtf OR ext:sxw OR ext:psw OR ext:ppt OR ext:pptx OR ext:pps OR ext:csv OR ext:txt OR ext:xls) intext:\"14152229670\" OR intext:\"+14152229670\" OR intext:\"4152229670\" OR intext:\"(415)+222-9670\""). + MatchParam("start", "0"). + Reply(200). + JSON(&customsearch.Search{ + ServerResponse: googleapi.ServerResponse{ + Header: http.Header{}, + HTTPStatusCode: 200, + }, + SearchInformation: &customsearch.SearchSearchInformation{ + FormattedSearchTime: "0", + FormattedTotalResults: "0", + SearchTime: 0, + TotalResults: "0", + ForceSendFields: nil, + NullFields: nil, + }, + Items: []*customsearch.Result{}, + }) + }, + }, { name: "test with results", number: test.NewFakeUSNumber(), @@ -239,11 +309,11 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) { remote := NewLibrary(filter.NewEngine()) remote.AddScanner(scanner) - if scanner.DryRun(*tt.number, ScannerOptions{}) != nil { + if scanner.DryRun(*tt.number, tt.opts) != nil { t.Fatal("DryRun() should return nil") } - got, errs := remote.Scan(tt.number) + got, errs := remote.Scan(tt.number, tt.opts) if len(tt.wantErrors) > 0 { assert.Equal(t, tt.wantErrors, errs) } else { @@ -263,6 +333,16 @@ func TestGoogleCSEScanner_DryRun(t *testing.T) { assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{})) } +func TestGoogleCSEScanner_DryRunWithOptions(t *testing.T) { + errStr := "search engine ID and/or API key is not defined" + + scanner := NewGoogleCSEScanner(&http.Client{}) + assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"cx": "test", "api_key": "secret"})) + assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"cx": "", "api_key": ""}), errStr) + assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"cx": "test"}), errStr) + assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"api_key": "test"}), errStr) +} + func TestGoogleCSEScanner_DryRun_Error(t *testing.T) { scanner := NewGoogleCSEScanner(&http.Client{}) assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{}), "search engine ID and/or API key is not defined") diff --git a/lib/remote/googlesearch_scanner_test.go b/lib/remote/googlesearch_scanner_test.go index 40b2919ba..4eac094a1 100644 --- a/lib/remote/googlesearch_scanner_test.go +++ b/lib/remote/googlesearch_scanner_test.go @@ -279,7 +279,7 @@ func TestGoogleSearchScanner(t *testing.T) { t.Fatal("DryRun() should return nil") } - got, errs := lib.Scan(tt.number) + got, errs := lib.Scan(tt.number, remote.ScannerOptions{}) if len(tt.wantErrors) > 0 { assert.Equal(t, tt.wantErrors, errs) } else { diff --git a/lib/remote/local_scanner_test.go b/lib/remote/local_scanner_test.go index a50f559cd..c13ca86dc 100644 --- a/lib/remote/local_scanner_test.go +++ b/lib/remote/local_scanner_test.go @@ -45,11 +45,11 @@ func TestLocalScanner(t *testing.T) { remote := NewLibrary(filter.NewEngine()) remote.AddScanner(scanner) - if scanner.DryRun(*tt.number, map[string]interface{}{}) != nil { + if scanner.DryRun(*tt.number, ScannerOptions{}) != nil { t.Fatal("DryRun() should return nil") } - got, errs := remote.Scan(tt.number) + got, errs := remote.Scan(tt.number, ScannerOptions{}) if len(tt.wantErrors) > 0 { assert.Equal(t, tt.wantErrors, errs) } else { diff --git a/lib/remote/numverify_scanner_test.go b/lib/remote/numverify_scanner_test.go index ea73bf447..faa51c5d9 100644 --- a/lib/remote/numverify_scanner_test.go +++ b/lib/remote/numverify_scanner_test.go @@ -23,6 +23,7 @@ func TestNumverifyScanner(t *testing.T) { testcases := []struct { name string number *number.Number + opts remote.ScannerOptions mocks func(s *mocks.NumverifySupplier) expected map[string]interface{} wantErrors map[string]error @@ -91,6 +92,45 @@ func TestNumverifyScanner(t *testing.T) { expected: map[string]interface{}{}, wantErrors: map[string]error{}, }, + { + name: "should run with options defined", + opts: remote.ScannerOptions{ + "api_key": "secret", + }, + number: func() *number.Number { + n, _ := number.NewNumber("15556661212") + return n + }(), + mocks: func(s *mocks.NumverifySupplier) { + s.On("Validate", "15556661212", "secret").Return(&suppliers.NumverifyValidateResponse{ + Valid: true, + Number: "test", + LocalFormat: "test", + InternationalFormat: "test", + CountryPrefix: "test", + CountryCode: "test", + CountryName: "test", + Location: "test", + Carrier: "test", + LineType: "test", + }, nil).Once() + }, + expected: map[string]interface{}{ + "numverify": remote.NumverifyScannerResponse{ + Valid: true, + Number: "test", + LocalFormat: "test", + InternationalFormat: "test", + CountryPrefix: "test", + CountryCode: "test", + CountryName: "test", + Location: "test", + Carrier: "test", + LineType: "test", + }, + }, + wantErrors: map[string]error{}, + }, } for _, tt := range testcases { @@ -102,7 +142,7 @@ func TestNumverifyScanner(t *testing.T) { lib := remote.NewLibrary(filter.NewEngine()) lib.AddScanner(scanner) - got, errs := lib.Scan(tt.number) + got, errs := lib.Scan(tt.number, tt.opts) if len(tt.wantErrors) > 0 { assert.Equal(t, tt.wantErrors, errs) } else { diff --git a/lib/remote/ovh_scanner_test.go b/lib/remote/ovh_scanner_test.go index 79b3d5cf2..4b1186c5d 100644 --- a/lib/remote/ovh_scanner_test.go +++ b/lib/remote/ovh_scanner_test.go @@ -80,7 +80,7 @@ func TestOVHScanner(t *testing.T) { lib := remote.NewLibrary(filter.NewEngine()) lib.AddScanner(scanner) - got, errs := lib.Scan(tt.number) + got, errs := lib.Scan(tt.number, remote.ScannerOptions{}) if len(tt.wantErrors) > 0 { assert.Equal(t, tt.wantErrors, errs) } else { diff --git a/lib/remote/remote.go b/lib/remote/remote.go index f12a0578e..64349586a 100644 --- a/lib/remote/remote.go +++ b/lib/remote/remote.go @@ -55,7 +55,7 @@ func (r *Library) addError(k string, err error) { r.errors[k] = err } -func (r *Library) Scan(n *number.Number) (map[string]interface{}, map[string]error) { +func (r *Library) Scan(n *number.Number, opts ScannerOptions) (map[string]interface{}, map[string]error) { var wg sync.WaitGroup for _, s := range r.scanners { @@ -69,7 +69,7 @@ func (r *Library) Scan(n *number.Number) (map[string]interface{}, map[string]err } }() - if err := s.DryRun(*n, make(ScannerOptions)); err != nil { + if err := s.DryRun(*n, opts); err != nil { logrus. WithField("scanner", s.Name()). WithField("reason", err.Error()). @@ -77,7 +77,7 @@ func (r *Library) Scan(n *number.Number) (map[string]interface{}, map[string]err return } - data, err := s.Run(*n, make(ScannerOptions)) + data, err := s.Run(*n, opts) if err != nil { r.addError(s.Name(), err) return diff --git a/lib/remote/remote_test.go b/lib/remote/remote_test.go index 12e5f7141..6c0de510b 100644 --- a/lib/remote/remote_test.go +++ b/lib/remote/remote_test.go @@ -40,7 +40,7 @@ func TestRemoteLibrary_SuccessScan(t *testing.T) { lib.AddScanner(fakeScanner) lib.AddScanner(fakeScanner2) - result, errs := lib.Scan(num) + result, errs := lib.Scan(num, remote.ScannerOptions{}) assert.Equal(t, expected, result) assert.Equal(t, map[string]error{}, errs) @@ -65,7 +65,7 @@ func TestRemoteLibrary_FailedScan(t *testing.T) { lib.AddScanner(fakeScanner) - result, errs := lib.Scan(num) + result, errs := lib.Scan(num, remote.ScannerOptions{}) assert.Equal(t, map[string]interface{}{}, result) assert.Equal(t, map[string]error{"fake": dummyError}, errs) @@ -86,7 +86,7 @@ func TestRemoteLibrary_EmptyScan(t *testing.T) { lib.AddScanner(fakeScanner) - result, errs := lib.Scan(num) + result, errs := lib.Scan(num, remote.ScannerOptions{}) assert.Equal(t, map[string]interface{}{}, result) assert.Equal(t, map[string]error{}, errs) @@ -108,7 +108,7 @@ func TestRemoteLibrary_PanicRun(t *testing.T) { lib.AddScanner(fakeScanner) - result, errs := lib.Scan(num) + result, errs := lib.Scan(num, remote.ScannerOptions{}) assert.Equal(t, map[string]interface{}{}, result) assert.Equal(t, map[string]error{"fake": errors.New("panic occurred while running scan, see debug logs")}, errs) @@ -129,7 +129,7 @@ func TestRemoteLibrary_PanicDryRun(t *testing.T) { lib.AddScanner(fakeScanner) - result, errs := lib.Scan(num) + result, errs := lib.Scan(num, remote.ScannerOptions{}) assert.Equal(t, map[string]interface{}{}, result) assert.Equal(t, map[string]error{"fake": errors.New("panic occurred while running scan, see debug logs")}, errs) From bd2a4cd027801f5643599876d97934e53be2f7d6 Mon Sep 17 00:00:00 2001 From: sundowndev <16480203+sundowndev@users.noreply.github.com> Date: Tue, 20 Feb 2024 17:55:45 +0400 Subject: [PATCH 07/10] refactor(remote): numverify supplier Implement new interface for numverify supplier so it doesn't use a global api key anymore --- lib/remote/numverify_scanner.go | 15 ++--- lib/remote/numverify_scanner_test.go | 68 ++++++----------------- lib/remote/scanner.go | 7 +++ lib/remote/scanner_test.go | 40 +++++++++++++ lib/remote/suppliers/numverify.go | 39 +++++++------ lib/remote/suppliers/numverify_test.go | 64 +++------------------ mocks/NumverifySupplier.go | 42 +++----------- mocks/NumverifySupplierRequest.go | 77 ++++++++++++++++++++++++++ 8 files changed, 182 insertions(+), 170 deletions(-) create mode 100644 mocks/NumverifySupplierRequest.go diff --git a/lib/remote/numverify_scanner.go b/lib/remote/numverify_scanner.go index 2ef516b58..e34e3c6b6 100644 --- a/lib/remote/numverify_scanner.go +++ b/lib/remote/numverify_scanner.go @@ -38,23 +38,16 @@ func (s *numverifyScanner) Description() string { } func (s *numverifyScanner) DryRun(_ number.Number, opts ScannerOptions) error { - if _, ok := opts["api_key"]; ok { + if opts.GetStringEnv("NUMVERIFY_API_KEY") != "" { return nil } - if !s.client.IsAvailable() { - return errors.New("API key is not defined") - } - return nil + return errors.New("API key is not defined") } func (s *numverifyScanner) Run(n number.Number, opts ScannerOptions) (interface{}, error) { - var apiKey string - - if v, ok := opts["api_key"].(string); ok { - apiKey = v - } + apiKey := opts.GetStringEnv("NUMVERIFY_API_KEY") - res, err := s.client.Validate(n.International, apiKey) + res, err := s.client.Request().SetApiKey(apiKey).ValidateNumber(n.International) if err != nil { return nil, err } diff --git a/lib/remote/numverify_scanner_test.go b/lib/remote/numverify_scanner_test.go index faa51c5d9..f70e80fad 100644 --- a/lib/remote/numverify_scanner_test.go +++ b/lib/remote/numverify_scanner_test.go @@ -24,7 +24,7 @@ func TestNumverifyScanner(t *testing.T) { name string number *number.Number opts remote.ScannerOptions - mocks func(s *mocks.NumverifySupplier) + mocks func(*mocks.NumverifySupplier, *mocks.NumverifySupplierReq) expected map[string]interface{} wantErrors map[string]error }{ @@ -34,9 +34,13 @@ func TestNumverifyScanner(t *testing.T) { n, _ := number.NewNumber("15556661212") return n }(), - mocks: func(s *mocks.NumverifySupplier) { - s.On("IsAvailable").Return(true) - s.On("Validate", "15556661212", "").Return(&suppliers.NumverifyValidateResponse{ + opts: map[string]interface{}{ + "NUMVERIFY_API_KEY": "secret", + }, + mocks: func(s *mocks.NumverifySupplier, r *mocks.NumverifySupplierReq) { + s.On("Request").Return(r) + r.On("SetApiKey", "secret").Return(r) + r.On("ValidateNumber", "15556661212").Return(&suppliers.NumverifyValidateResponse{ Valid: true, Number: "test", LocalFormat: "test", @@ -71,9 +75,13 @@ func TestNumverifyScanner(t *testing.T) { n, _ := number.NewNumber("15556661212") return n }(), - mocks: func(s *mocks.NumverifySupplier) { - s.On("IsAvailable").Return(true) - s.On("Validate", "15556661212", "").Return(nil, dummyError).Once() + opts: map[string]interface{}{ + "NUMVERIFY_API_KEY": "secret", + }, + mocks: func(s *mocks.NumverifySupplier, r *mocks.NumverifySupplierReq) { + s.On("Request").Return(r) + r.On("SetApiKey", "secret").Return(r) + r.On("ValidateNumber", "15556661212").Return(nil, dummyError).Once() }, expected: map[string]interface{}{}, wantErrors: map[string]error{ @@ -86,57 +94,17 @@ func TestNumverifyScanner(t *testing.T) { n, _ := number.NewNumber("15556661212") return n }(), - mocks: func(s *mocks.NumverifySupplier) { - s.On("IsAvailable").Return(false) - }, + mocks: func(s *mocks.NumverifySupplier, r *mocks.NumverifySupplierReq) {}, expected: map[string]interface{}{}, wantErrors: map[string]error{}, }, - { - name: "should run with options defined", - opts: remote.ScannerOptions{ - "api_key": "secret", - }, - number: func() *number.Number { - n, _ := number.NewNumber("15556661212") - return n - }(), - mocks: func(s *mocks.NumverifySupplier) { - s.On("Validate", "15556661212", "secret").Return(&suppliers.NumverifyValidateResponse{ - Valid: true, - Number: "test", - LocalFormat: "test", - InternationalFormat: "test", - CountryPrefix: "test", - CountryCode: "test", - CountryName: "test", - Location: "test", - Carrier: "test", - LineType: "test", - }, nil).Once() - }, - expected: map[string]interface{}{ - "numverify": remote.NumverifyScannerResponse{ - Valid: true, - Number: "test", - LocalFormat: "test", - InternationalFormat: "test", - CountryPrefix: "test", - CountryCode: "test", - CountryName: "test", - Location: "test", - Carrier: "test", - LineType: "test", - }, - }, - wantErrors: map[string]error{}, - }, } for _, tt := range testcases { t.Run(tt.name, func(t *testing.T) { numverifySupplierMock := &mocks.NumverifySupplier{} - tt.mocks(numverifySupplierMock) + numverifySupplierReqMock := &mocks.NumverifySupplierReq{} + tt.mocks(numverifySupplierMock, numverifySupplierReqMock) scanner := remote.NewNumverifyScanner(numverifySupplierMock) lib := remote.NewLibrary(filter.NewEngine()) diff --git a/lib/remote/scanner.go b/lib/remote/scanner.go index a7911204b..ec0a87981 100644 --- a/lib/remote/scanner.go +++ b/lib/remote/scanner.go @@ -10,6 +10,13 @@ import ( type ScannerOptions map[string]interface{} +func (o ScannerOptions) GetStringEnv(k string) string { + if v, ok := o[k].(string); ok { + return v + } + return os.Getenv(k) +} + type Plugin interface { Lookup(string) (plugin.Symbol, error) } diff --git a/lib/remote/scanner_test.go b/lib/remote/scanner_test.go index 9de53afa3..c277351e6 100644 --- a/lib/remote/scanner_test.go +++ b/lib/remote/scanner_test.go @@ -2,6 +2,7 @@ package remote import ( "fmt" + "os" "path/filepath" "testing" @@ -38,3 +39,42 @@ func Test_ValidatePlugin_Errors(t *testing.T) { }) } } + +func TestScannerOptions(t *testing.T) { + testcases := []struct { + name string + opts ScannerOptions + check func(*testing.T, ScannerOptions) + }{ + { + name: "test GetStringEnv with simple options", + opts: map[string]interface{}{ + "foo": "bar", + }, + check: func(t *testing.T, opts ScannerOptions) { + assert.Equal(t, opts.GetStringEnv("foo"), "bar") + assert.Equal(t, opts.GetStringEnv("bar"), "") + }, + }, + { + name: "test GetStringEnv with env vars", + opts: map[string]interface{}{ + "foo_bar": "bar", + }, + check: func(t *testing.T, opts ScannerOptions) { + _ = os.Setenv("FOO_BAR", "secret") + defer os.Unsetenv("FOO_BAR") + + assert.Equal(t, opts.GetStringEnv("FOO_BAR"), "secret") + assert.Equal(t, opts.GetStringEnv("foo_bar"), "bar") + assert.Equal(t, opts.GetStringEnv("foo"), "") + }, + }, + } + + for _, tt := range testcases { + t.Run(tt.name, func(t *testing.T) { + tt.check(t, tt.opts) + }) + } +} diff --git a/lib/remote/suppliers/numverify.go b/lib/remote/suppliers/numverify.go index df2a8b06e..96d0bbfbf 100644 --- a/lib/remote/suppliers/numverify.go +++ b/lib/remote/suppliers/numverify.go @@ -4,15 +4,17 @@ import ( "encoding/json" "errors" "fmt" - "net/http" - "os" - "github.com/sirupsen/logrus" + "net/http" ) type NumverifySupplierInterface interface { - IsAvailable() bool - Validate(string, string) (*NumverifyValidateResponse, error) + Request() NumverifySupplierRequestInterface +} + +type NumverifySupplierRequestInterface interface { + SetApiKey(string) NumverifySupplierRequestInterface + ValidateNumber(string) (*NumverifyValidateResponse, error) } type NumverifyErrorResponse struct { @@ -34,37 +36,40 @@ type NumverifyValidateResponse struct { } type NumverifySupplier struct { - ApiKey string + Uri string } func NewNumverifySupplier() *NumverifySupplier { return &NumverifySupplier{ - ApiKey: os.Getenv("NUMVERIFY_API_KEY"), + Uri: "https://api.apilayer.com", } } -func (s *NumverifySupplier) IsAvailable() bool { - return s.ApiKey != "" +type NumverifyRequest struct { + apiKey string + uri string } -func (s *NumverifySupplier) Validate(internationalNumber string, customApiKey string) (res *NumverifyValidateResponse, err error) { - apiKey := s.ApiKey +func (s *NumverifySupplier) Request() NumverifySupplierRequestInterface { + return &NumverifyRequest{uri: s.Uri} +} - // User-provided credentials - if customApiKey != "" { - apiKey = customApiKey - } +func (r *NumverifyRequest) SetApiKey(k string) NumverifySupplierRequestInterface { + r.apiKey = k + return r +} +func (r *NumverifyRequest) ValidateNumber(internationalNumber string) (res *NumverifyValidateResponse, err error) { logrus. WithField("number", internationalNumber). Debug("Running validate operation through Numverify API") - url := fmt.Sprintf("https://api.apilayer.com/number_verification/validate?number=%s", internationalNumber) + url := fmt.Sprintf("%s/number_verification/validate?number=%s", r.uri, internationalNumber) // Build the request client := &http.Client{} req, _ := http.NewRequest("GET", url, nil) - req.Header.Set("Apikey", apiKey) + req.Header.Set("Apikey", r.apiKey) response, err := client.Do(req) diff --git a/lib/remote/suppliers/numverify_test.go b/lib/remote/suppliers/numverify_test.go index 2f1b9bebd..5f752a15b 100644 --- a/lib/remote/suppliers/numverify_test.go +++ b/lib/remote/suppliers/numverify_test.go @@ -9,48 +9,11 @@ import ( "testing" ) -func TestNumverifySupplierSuccess(t *testing.T) { - defer gock.Off() // Flush pending mocks after test execution - - number := "11115551212" - - _ = os.Setenv("NUMVERIFY_API_KEY", "5ad5554ac240e4d3d31107941b35a5eb") - defer os.Clearenv() - - expectedResult := &NumverifyValidateResponse{ - Valid: true, - Number: "79516566591", - LocalFormat: "9516566591", - InternationalFormat: "+79516566591", - CountryPrefix: "+7", - CountryCode: "RU", - CountryName: "Russian Federation", - Location: "Saint Petersburg and Leningrad Oblast", - Carrier: "OJSC St. Petersburg Telecom (OJSC Tele2-Saint-Petersburg)", - LineType: "mobile", - } - - gock.New("https://api.apilayer.com"). - Get("/number_verification/validate"). - MatchHeader("Apikey", "5ad5554ac240e4d3d31107941b35a5eb"). - MatchParam("number", number). - Reply(200). - JSON(expectedResult) - - s := NewNumverifySupplier() - - assert.True(t, s.IsAvailable()) - - got, err := s.Validate(number, "") - assert.Nil(t, err) - - assert.Equal(t, expectedResult, got) -} - func TestNumverifySupplierSuccessCustomApiKey(t *testing.T) { defer gock.Off() // Flush pending mocks after test execution number := "11115551212" + apikey := "5ad5554ac240e4d3d31107941b35a5eb" expectedResult := &NumverifyValidateResponse{ Valid: true, @@ -67,16 +30,14 @@ func TestNumverifySupplierSuccessCustomApiKey(t *testing.T) { gock.New("https://api.apilayer.com"). Get("/number_verification/validate"). - MatchHeader("Apikey", "5ad5554ac240e4d3d31107941b35a5eb"). + MatchHeader("Apikey", apikey). MatchParam("number", number). Reply(200). JSON(expectedResult) s := NewNumverifySupplier() - assert.False(t, s.IsAvailable()) - - got, err := s.Validate(number, "5ad5554ac240e4d3d31107941b35a5eb") + got, err := s.Request().SetApiKey(apikey).ValidateNumber(number) assert.Nil(t, err) assert.Equal(t, expectedResult, got) @@ -86,9 +47,7 @@ func TestNumverifySupplierError(t *testing.T) { defer gock.Off() // Flush pending mocks after test execution number := "11115551212" - - _ = os.Setenv("NUMVERIFY_API_KEY", "5ad5554ac240e4d3d31107941b35a5eb") - defer os.Clearenv() + apikey := "5ad5554ac240e4d3d31107941b35a5eb" expectedResult := &NumverifyErrorResponse{ Message: "You have exceeded your daily\\/monthly API rate limit. Please review and upgrade your subscription plan at https:\\/\\/apilayer.com\\/subscriptions to continue.", @@ -96,16 +55,14 @@ func TestNumverifySupplierError(t *testing.T) { gock.New("https://api.apilayer.com"). Get("/number_verification/validate"). - MatchHeader("Apikey", "5ad5554ac240e4d3d31107941b35a5eb"). + MatchHeader("Apikey", apikey). MatchParam("number", number). Reply(429). JSON(expectedResult) s := NewNumverifySupplier() - assert.True(t, s.IsAvailable()) - - got, err := s.Validate(number, "") + got, err := s.Request().SetApiKey(apikey).ValidateNumber(number) assert.Nil(t, got) assert.Equal(t, errors.New("You have exceeded your daily\\/monthly API rate limit. Please review and upgrade your subscription plan at https:\\/\\/apilayer.com\\/subscriptions to continue."), err) } @@ -126,9 +83,7 @@ func TestNumverifySupplierHTTPError(t *testing.T) { s := NewNumverifySupplier() - assert.True(t, s.IsAvailable()) - - got, err := s.Validate(number, "") + got, err := s.Request().ValidateNumber(number) assert.Nil(t, got) assert.Equal(t, &url.Error{ Op: "Get", @@ -136,8 +91,3 @@ func TestNumverifySupplierHTTPError(t *testing.T) { Err: dummyError, }, err) } - -func TestNumverifySupplierWithoutAPIKey(t *testing.T) { - s := NewNumverifySupplier() - assert.False(t, s.IsAvailable()) -} diff --git a/mocks/NumverifySupplier.go b/mocks/NumverifySupplier.go index 77606f53b..5dcefd33b 100644 --- a/mocks/NumverifySupplier.go +++ b/mocks/NumverifySupplier.go @@ -12,52 +12,24 @@ type NumverifySupplier struct { mock.Mock } -// IsAvailable provides a mock function with given fields: -func (_m *NumverifySupplier) IsAvailable() bool { +// Request provides a mock function with given fields: +func (_m *NumverifySupplier) Request() suppliers.NumverifySupplierRequestInterface { ret := _m.Called() if len(ret) == 0 { - panic("no return value specified for IsAvailable") + panic("no return value specified for Request") } - var r0 bool - if rf, ok := ret.Get(0).(func() bool); ok { + var r0 suppliers.NumverifySupplierRequestInterface + if rf, ok := ret.Get(0).(func() suppliers.NumverifySupplierRequestInterface); ok { r0 = rf() - } else { - r0 = ret.Get(0).(bool) - } - - return r0 -} - -// Validate provides a mock function with given fields: _a0, _a1 -func (_m *NumverifySupplier) Validate(_a0 string, _a1 string) (*suppliers.NumverifyValidateResponse, error) { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for Validate") - } - - var r0 *suppliers.NumverifyValidateResponse - var r1 error - if rf, ok := ret.Get(0).(func(string, string) (*suppliers.NumverifyValidateResponse, error)); ok { - return rf(_a0, _a1) - } - if rf, ok := ret.Get(0).(func(string, string) *suppliers.NumverifyValidateResponse); ok { - r0 = rf(_a0, _a1) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*suppliers.NumverifyValidateResponse) + r0 = ret.Get(0).(suppliers.NumverifySupplierRequestInterface) } } - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(_a0, _a1) - } else { - r1 = ret.Error(1) - } - - return r0, r1 + return r0 } // NewNumverifySupplier creates a new instance of NumverifySupplier. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. diff --git a/mocks/NumverifySupplierRequest.go b/mocks/NumverifySupplierRequest.go new file mode 100644 index 000000000..879dbf7fb --- /dev/null +++ b/mocks/NumverifySupplierRequest.go @@ -0,0 +1,77 @@ +// Code generated by mockery v2.38.0. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + suppliers "github.com/sundowndev/phoneinfoga/v2/lib/remote/suppliers" +) + +// NumverifySupplierReq is an autogenerated mock type for the NumverifySupplierRequestInterface type +type NumverifySupplierReq struct { + mock.Mock +} + +// SetApiKey provides a mock function with given fields: _a0 +func (_m *NumverifySupplierReq) SetApiKey(_a0 string) suppliers.NumverifySupplierRequestInterface { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for SetApiKey") + } + + var r0 suppliers.NumverifySupplierRequestInterface + if rf, ok := ret.Get(0).(func(string) suppliers.NumverifySupplierRequestInterface); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(suppliers.NumverifySupplierRequestInterface) + } + } + + return r0 +} + +// ValidateNumber provides a mock function with given fields: _a0 +func (_m *NumverifySupplierReq) ValidateNumber(_a0 string) (*suppliers.NumverifyValidateResponse, error) { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for ValidateNumber") + } + + var r0 *suppliers.NumverifyValidateResponse + var r1 error + if rf, ok := ret.Get(0).(func(string) (*suppliers.NumverifyValidateResponse, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(string) *suppliers.NumverifyValidateResponse); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*suppliers.NumverifyValidateResponse) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewNumverifySupplierReq creates a new instance of NumverifySupplierReq. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewNumverifySupplierReq(t interface { + mock.TestingT + Cleanup(func()) +}) *NumverifySupplierReq { + mock := &NumverifySupplierReq{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} From ef94bbe26d47c4520215bdf636b660d41ebadc9e Mon Sep 17 00:00:00 2001 From: sundowndev <16480203+sundowndev@users.noreply.github.com> Date: Tue, 20 Feb 2024 17:56:35 +0400 Subject: [PATCH 08/10] refactor(remote): googlecse scanner Improve handling of options. Use the same key for both options and env vars. --- lib/remote/googlecse_scanner.go | 29 +++++----------------------- lib/remote/googlecse_scanner_test.go | 12 ++++++------ 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/lib/remote/googlecse_scanner.go b/lib/remote/googlecse_scanner.go index 9b41dd143..f56617e8b 100644 --- a/lib/remote/googlecse_scanner.go +++ b/lib/remote/googlecse_scanner.go @@ -18,8 +18,6 @@ import ( const GoogleCSE = "googlecse" type googleCSEScanner struct { - Cx string - ApiKey string MaxResults int64 httpClient *http.Client } @@ -52,8 +50,6 @@ func NewGoogleCSEScanner(HTTPclient *http.Client) Scanner { } return &googleCSEScanner{ - Cx: os.Getenv("GOOGLECSE_CX"), - ApiKey: os.Getenv("GOOGLE_API_KEY"), MaxResults: int64(maxResults), httpClient: HTTPclient, } @@ -68,16 +64,7 @@ func (s *googleCSEScanner) Description() string { } func (s *googleCSEScanner) DryRun(_ number.Number, opts ScannerOptions) error { - var cx = s.Cx - var apikey = s.ApiKey - - if v, ok := opts["cx"].(string); ok { - cx = v - } - if v, ok := opts["api_key"].(string); ok { - apikey = v - } - if cx == "" || apikey == "" { + if opts.GetStringEnv("GOOGLECSE_CX") == "" || opts.GetStringEnv("GOOGLE_API_KEY") == "" { return errors.New("search engine ID and/or API key is not defined") } return nil @@ -88,15 +75,8 @@ func (s *googleCSEScanner) Run(n number.Number, opts ScannerOptions) (interface{ var dorks []*GoogleSearchDork var totalResultCount int var totalRequestCount int - var cx = s.Cx - var apikey = s.ApiKey - - if v, ok := opts["cx"].(string); ok { - cx = v - } - if v, ok := opts["api_key"].(string); ok { - apikey = v - } + var cx = opts.GetStringEnv("GOOGLECSE_CX") + var apikey = opts.GetStringEnv("GOOGLE_API_KEY") dorks = append(dorks, s.generateDorkQueries(n)...) @@ -169,7 +149,8 @@ func (s *googleCSEScanner) isRateLimit(theError error) bool { if theError == nil { return false } - if _, ok := theError.(*googleapi.Error); !ok { + var err *googleapi.Error + if !errors.As(theError, &err) { return false } if theError.(*googleapi.Error).Code != 429 { diff --git a/lib/remote/googlecse_scanner_test.go b/lib/remote/googlecse_scanner_test.go index 8507f803f..e06a62ed4 100644 --- a/lib/remote/googlecse_scanner_test.go +++ b/lib/remote/googlecse_scanner_test.go @@ -94,8 +94,8 @@ func TestGoogleCSEScanner_Scan_Success(t *testing.T) { name: "test with options and no results", number: test.NewFakeUSNumber(), opts: ScannerOptions{ - "cx": "custom_cx", - "api_key": "secret", + "GOOGLECSE_CX": "custom_cx", + "GOOGLE_API_KEY": "secret", }, expected: map[string]interface{}{ "googlecse": GoogleCSEScannerResponse{ @@ -337,10 +337,10 @@ func TestGoogleCSEScanner_DryRunWithOptions(t *testing.T) { errStr := "search engine ID and/or API key is not defined" scanner := NewGoogleCSEScanner(&http.Client{}) - assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"cx": "test", "api_key": "secret"})) - assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"cx": "", "api_key": ""}), errStr) - assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"cx": "test"}), errStr) - assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"api_key": "test"}), errStr) + assert.Nil(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLECSE_CX": "test", "GOOGLE_API_KEY": "secret"})) + assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLECSE_CX": "", "GOOGLE_API_KEY": ""}), errStr) + assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLECSE_CX": "test"}), errStr) + assert.EqualError(t, scanner.DryRun(*test.NewFakeUSNumber(), ScannerOptions{"GOOGLE_API_KEY": "test"}), errStr) } func TestGoogleCSEScanner_DryRun_Error(t *testing.T) { From 792965a13797e17a96fac9524edc426586581d41 Mon Sep 17 00:00:00 2001 From: sundowndev <16480203+sundowndev@users.noreply.github.com> Date: Tue, 20 Feb 2024 18:03:20 +0400 Subject: [PATCH 09/10] docs: update scanner options --- docs/getting-started/scanners.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/getting-started/scanners.md b/docs/getting-started/scanners.md index 67602f267..e5d3bbd2a 100644 --- a/docs/getting-started/scanners.md +++ b/docs/getting-started/scanners.md @@ -19,7 +19,12 @@ GOOGLE_API_KEY="value" phoneinfoga scan -n +4176418xxxx --env-file=.env.local ``` -**HTTP API consumers**: You can also specify scanner options such as api keys on a per-request basis. Each scanner supports its own options, see below. For details on how to specify those options, see [API docs](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/sundowndev/phoneinfoga/master/web/docs/swagger.yaml). Scanner options will override environment variables for the current request. +### Scanner options + +When using the **REST API**, you can also specify those values on a per-request basis. Each scanner supports its own options, see below. For details on how to specify those options, see [API docs](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/sundowndev/phoneinfoga/master/web/docs/swagger.yaml#/Numbers/RunScanner). For readability and simplicity, options are named exactly like their environment variable equivalent. + +!!! warning + Scanner options will override environment variables for the current request. ## Building your own scanner @@ -67,7 +72,7 @@ Numverify provide standard but useful information such as country code, location | Environment variable | Option | Default | Description | |----------------------|------------|---------|-------------------------------------------------------| - | NUMVERIFY_API_KEY | api_key | | API key to authenticate to the Numverify API. | + | NUMVERIFY_API_KEY | NUMVERIFY_API_KEY | | API key to authenticate to the Numverify API. | ??? example "Output example" @@ -213,8 +218,8 @@ Follow the steps below to create a new search engine : | Environment variable | Option | Default | Description | |-----------------------|----------|----------|-------------------------------------------------------------| - | GOOGLECSE_CX | cx | | Search engine ID. | - | GOOGLE_API_KEY | api_key | | API key to authenticate to the Google API. | + | GOOGLECSE_CX | GOOGLECSE_CX | | Search engine ID. | + | GOOGLE_API_KEY | GOOGLE_API_KEY | | API key to authenticate to the Google API. | | GOOGLECSE_MAX_RESULTS | | 10 | Maximum results for each request. Each 10 results requires an additional request. This value cannot go above 100. | ??? example "Output example" From 8b9598aa5914405b5547ba04af89d5b2fa2baf77 Mon Sep 17 00:00:00 2001 From: sundowndev <16480203+sundowndev@users.noreply.github.com> Date: Tue, 20 Feb 2024 18:53:04 +0400 Subject: [PATCH 10/10] chore: update swag cli --- Makefile | 2 +- web/docs/docs.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 27ab7265e..9b1feb347 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ lint: install-tools: $(GOINSTALL) gotest.tools/gotestsum@v1.6.3 $(GOINSTALL) github.com/vektra/mockery/v2@v2.38.0 - $(GOINSTALL) github.com/swaggo/swag/cmd/swag@v1.16.1 + $(GOINSTALL) github.com/swaggo/swag/cmd/swag@v1.16.3 @which golangci-lint > /dev/null 2>&1 || (curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | bash -s -- -b $(GOBINPATH) v1.46.2) go.mod: FORCE diff --git a/web/docs/docs.go b/web/docs/docs.go index 96cb0a541..4316dd458 100644 --- a/web/docs/docs.go +++ b/web/docs/docs.go @@ -1,5 +1,4 @@ -// Code generated by swaggo/swag. DO NOT EDIT. - +// Package docs Code generated by swaggo/swag. DO NOT EDIT package docs import "github.com/swaggo/swag"