Merge pull request #25 from macdoos/main

working thank you <3

pkg/scan/scan.go

@@ -15,8 +15,27 @@ import (
 	"github.com/dropalldatabases/sif/pkg/logger"
 )
 
-func Scan(url string, timeout time.Duration, threads int, logdir string) {
+func fetchRobotsTXT(url string, client *http.Client) *http.Response {
+	resp, err := client.Get(url)
+	if err != nil {
+		log.Debugf("Error fetching robots.txt: %s", err)
+		return nil
+	}
+
+	if resp.StatusCode == http.StatusMovedPermanently {
+		redirectURL := resp.Header.Get("Location")
+		if redirectURL == "" {
+			log.Debugf("Redirect location is empty for %s", url)
+			return nil
+		}
+		resp.Body.Close()
+		return fetchRobotsTXT(redirectURL, client)
+	}
 
+	return resp
+}
+
+func Scan(url string, timeout time.Duration, threads int, logdir string) {
 	fmt.Println(styles.Separator.Render("🐾 Starting " + styles.Status.Render("base url scanning") + "..."))
 
 	sanitizedURL := strings.Split(url, "://")[1]
@@ -39,11 +58,12 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) {
 		},
 	}
 
-	resp, err := client.Get(url + "/robots.txt")
-	if err != nil {
-		log.Debugf("Error: %s", err)
+	resp := fetchRobotsTXT(url+"/robots.txt", client)
+	if resp == nil {
+		return
 	}
 	defer resp.Body.Close()
+
 	if resp.StatusCode != 404 && resp.StatusCode != 301 && resp.StatusCode != 302 && resp.StatusCode != 307 {
 		scanlog.Infof("file [%s] found", styles.Status.Render("robots.txt"))
 
@@ -70,12 +90,13 @@ func Scan(url string, timeout time.Duration, threads int, logdir string) {
 					}
 
 					_, sanitizedRobot, _ := strings.Cut(robot, ": ")
-					log.Debugf("%s", robot)
+					scanlog.Debugf("%s", robot)
 					resp, err := client.Get(url + "/" + sanitizedRobot)
 					if err != nil {
-						log.Debugf("Error %s: %s", sanitizedRobot, err)
-						return
+						scanlog.Debugf("Error %s: %s", sanitizedRobot, err)
+						continue
 					}
+					defer resp.Body.Close()
 
 					if resp.StatusCode != 404 {
 						scanlog.Infof("%s from robots: [%s]", styles.Status.Render(strconv.Itoa(resp.StatusCode)), styles.Highlight.Render(sanitizedRobot))