From 3f86c01f1adc40413e9625dc4925bd3395b2d59b Mon Sep 17 00:00:00 2001 From: macdoos Date: Fri, 14 Jun 2024 16:13:45 +0200 Subject: [PATCH 1/4] Fetch robots.txt on 301 --- sif.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) mode change 100644 => 100755 sif.go diff --git a/sif.go b/sif.go old mode 100644 new mode 100755 index f235450..b50bf9c --- a/sif.go +++ b/sif.go @@ -4,6 +4,8 @@ import ( "bufio" "errors" "fmt" + "io" + "net/http" "os" "strings" @@ -60,6 +62,33 @@ func New(settings *config.Settings) (*App, error) { return app, nil } +func fetchRobotsTXT(url string) (string, error) { + resp, err := http.Get(url) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusMovedPermanently { + redirectURL := resp.Header.Get("Location") + if redirectURL == "" { + return "", errors.New("redirect location is empty") + } + return fetchRobotsTXT(redirectURL) + } + + if resp.StatusCode != http.StatusOK { + return "", errors.New(fmt.Sprintf("failed to fetch robots.txt: %s", resp.Status)) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return string(body), nil +} + // Run runs the pentesting suite, with the targets specified, according to the // settings specified. func (app *App) Run() error { @@ -87,6 +116,12 @@ func (app *App) Run() error { } if !app.settings.NoScan { + robotsTxt, err := fetchRobotsTXT(fmt.Sprintf("%s/robots.txt", url)) + if err != nil { + log.Errorf("Failed to fetch robots.txt for %s: %v", url, err) + } else { + log.Infof("robots.txt content for %s:\n%s", url, robotsTxt) + } scan.Scan(url, app.settings.Timeout, app.settings.Threads, app.settings.LogDir) } From dd1af9744f2b763078f768dc3dfde508e8d448e4 Mon Sep 17 00:00:00 2001 From: macdoos Date: Sat, 15 Jun 2024 22:22:03 +0200 Subject: [PATCH 2/4] Add function to scan.go --- pkg/scan/scan.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/pkg/scan/scan.go b/pkg/scan/scan.go index 2452b33..6835b8e 100644 --- a/pkg/scan/scan.go +++ b/pkg/scan/scan.go @@ -15,6 +15,24 @@ import ( "github.com/dropalldatabases/sif/pkg/logger" ) +func fetchRobotsTXT(url string, client *http.Client) (*http.Response, error) { + resp, err := client.Get(url) + if err != nil { + return nil, err + } + + if resp.StatusCode == http.StatusMovedPermanently { + redirectURL := resp.Header.Get("Location") + if redirectURL == "" { + return nil, fmt.Errorf("redirect location is empty") + } + resp.Body.Close() + return fetchRobotsTXT(redirectURL, client) + } + + return resp, nil +} + func Scan(url string, timeout time.Duration, threads int, logdir string) { fmt.Println(styles.Separator.Render("🐾 Starting " + styles.Status.Render("base url scanning") + "...")) @@ -39,11 +57,13 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) { }, } - resp, err := client.Get(url + "/robots.txt") + resp, err := fetchRobotsTXT(url+"/robots.txt", client) if err != nil { - log.Debugf("Error: %s", err) + log.Debugf("Error fetching robots.txt: %s", err) + return } defer resp.Body.Close() + if resp.StatusCode != 404 && resp.StatusCode != 301 && resp.StatusCode != 302 && resp.StatusCode != 307 { scanlog.Infof("file [%s] found", styles.Status.Render("robots.txt")) From 4f58f22b69df9ebd03b0cfe91657ce122d7082da Mon Sep 17 00:00:00 2001 From: macdoos Date: Sat, 15 Jun 2024 22:22:42 +0200 Subject: [PATCH 3/4] Revert "Fetch robots.txt on 301" This reverts commit 3f86c01f1adc40413e9625dc4925bd3395b2d59b. --- sif.go | 35 ----------------------------------- 1 file changed, 35 deletions(-) mode change 100755 => 100644 sif.go diff --git a/sif.go b/sif.go old mode 100755 new mode 100644 index b50bf9c..f235450 --- a/sif.go +++ b/sif.go @@ -4,8 +4,6 @@ import ( "bufio" "errors" "fmt" - "io" - "net/http" "os" "strings" @@ -62,33 +60,6 @@ func New(settings *config.Settings) (*App, error) { return app, nil } -func fetchRobotsTXT(url string) (string, error) { - resp, err := http.Get(url) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusMovedPermanently { - redirectURL := resp.Header.Get("Location") - if redirectURL == "" { - return "", errors.New("redirect location is empty") - } - return fetchRobotsTXT(redirectURL) - } - - if resp.StatusCode != http.StatusOK { - return "", errors.New(fmt.Sprintf("failed to fetch robots.txt: %s", resp.Status)) - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - - return string(body), nil -} - // Run runs the pentesting suite, with the targets specified, according to the // settings specified. func (app *App) Run() error { @@ -116,12 +87,6 @@ func (app *App) Run() error { } if !app.settings.NoScan { - robotsTxt, err := fetchRobotsTXT(fmt.Sprintf("%s/robots.txt", url)) - if err != nil { - log.Errorf("Failed to fetch robots.txt for %s: %v", url, err) - } else { - log.Infof("robots.txt content for %s:\n%s", url, robotsTxt) - } scan.Scan(url, app.settings.Timeout, app.settings.Threads, app.settings.LogDir) } From 80062533c934b0abc036724c9212aa6d8760cdd7 Mon Sep 17 00:00:00 2001 From: macdoos Date: Sat, 15 Jun 2024 23:31:34 +0200 Subject: [PATCH 4/4] Proper logging --- pkg/scan/scan.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/pkg/scan/scan.go b/pkg/scan/scan.go index 6835b8e..4d748db 100644 --- a/pkg/scan/scan.go +++ b/pkg/scan/scan.go @@ -15,26 +15,27 @@ import ( "github.com/dropalldatabases/sif/pkg/logger" ) -func fetchRobotsTXT(url string, client *http.Client) (*http.Response, error) { +func fetchRobotsTXT(url string, client *http.Client) *http.Response { resp, err := client.Get(url) if err != nil { - return nil, err + log.Debugf("Error fetching robots.txt: %s", err) + return nil } if resp.StatusCode == http.StatusMovedPermanently { redirectURL := resp.Header.Get("Location") if redirectURL == "" { - return nil, fmt.Errorf("redirect location is empty") + log.Debugf("Redirect location is empty for %s", url) + return nil } resp.Body.Close() return fetchRobotsTXT(redirectURL, client) } - return resp, nil + return resp } func Scan(url string, timeout time.Duration, threads int, logdir string) { - fmt.Println(styles.Separator.Render("🐾 Starting " + styles.Status.Render("base url scanning") + "...")) sanitizedURL := strings.Split(url, "://")[1] @@ -57,9 +58,8 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) { }, } - resp, err := fetchRobotsTXT(url+"/robots.txt", client) - if err != nil { - log.Debugf("Error fetching robots.txt: %s", err) + resp := fetchRobotsTXT(url+"/robots.txt", client) + if resp == nil { return } defer resp.Body.Close() @@ -90,12 +90,13 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) { } _, sanitizedRobot, _ := strings.Cut(robot, ": ") - log.Debugf("%s", robot) + scanlog.Debugf("%s", robot) resp, err := client.Get(url + "/" + sanitizedRobot) if err != nil { - log.Debugf("Error %s: %s", sanitizedRobot, err) - return + scanlog.Debugf("Error %s: %s", sanitizedRobot, err) + continue } + defer resp.Body.Close() if resp.StatusCode != 404 { scanlog.Infof("%s from robots: [%s]", styles.Status.Render(strconv.Itoa(resp.StatusCode)), styles.Highlight.Render(sanitizedRobot))