diff --git a/main.go b/main.go index 3d0fa2e..97a9262 100644 --- a/main.go +++ b/main.go @@ -17,72 +17,72 @@ type finishState struct { err error } -var ( // opts - help = flag.Bool("h", false, "Show help menu") - useOrigFilename = flag.Bool("o", false, "Download with the original filename") - customDownloadDir = flag.String("c", "", "Set a custom directory for the images to download to") -) +// opts +var useOrigFilename = flag.Bool("o", false, "Download with the original filename") +var customDownloadDir = flag.String("c", "", "Set a custom directory for the images to download to") func main() { flag.Parse() - if *help { - flag.Usage() - return - } - - args := flag.Args() - if len(args) < 1 { + urls := flag.Args() + if len(urls) < 1 { flag.Usage() - return + os.Exit(0) } - urls := strings.Split(args[0], " ") origDir, err := os.Getwd() if err != nil { - fmt.Printf("Could not grab working directory! | %v\n", err) - return + fmt.Printf("Could not grab working directory! %v\n", err) + os.Exit(1) } + Gochan := godesu.New() // intialize godesu for urlNum, url := range urls { // loop through all urls purl := strings.Split(url, "/") + if len(purl) < 6 { // check length to avoid runtime error + fmt.Println("Invalid URL!") + os.Exit(1) + } + ThreadNum, err := strconv.Atoi(purl[5]) if err != nil { - fmt.Printf("Make sure the URL is correct! | %v\n", err) - return + fmt.Printf("Inavlid URL! %v\n", err) + os.Exit(1) } err, Thread := Gochan.Board(purl[3]).GetThread(ThreadNum) if err != nil { - fmt.Printf("Could not fetch thread! | %v\n", err) - return + fmt.Printf("Could not fetch thread! %v\n", err) + os.Exit(1) } images := Thread.Images() finishStateChan := make(chan finishState, len(images)) // make the download channel with proper buffer size if *customDownloadDir != "" { - if err := os.Chdir(*customDownloadDir + "/"); err != nil { - if err := os.MkdirAll(*customDownloadDir+"/", os.ModePerm); err != nil { - fmt.Printf("Cannot create directory! %v\n", err) - return - } + if err := os.MkdirAll(*customDownloadDir+"/", os.ModePerm); err != nil { + fmt.Printf("Cannot create '%s'! %v\n", *customDownloadDir+"/", err) + os.Exit(1) + } - os.Chdir(*customDownloadDir + "/") + if err := os.Chdir(*customDownloadDir + "/"); err != nil { // this should return an error + fmt.Printf("Cannot change into '%s'! %v\n", *customDownloadDir+"/", err) + os.Exit(2) } } else { - if err := os.Chdir(purl[3] + "/" + purl[5]); err != nil { - if err := os.MkdirAll(purl[3]+"/"+purl[5], os.ModePerm); err != nil { - fmt.Printf("Cannot create directory! %v\n", err) - return - } + if err := os.MkdirAll(purl[3]+"/"+purl[5], os.ModePerm); err != nil { + fmt.Printf("Cannot create '%s'! %v\n", purl[3]+"/"+purl[5], err) + os.Exit(1) + } - os.Chdir(purl[3] + "/" + purl[5]) + if err := os.Chdir(purl[3] + "/" + purl[5]); err != nil { // this should never return an error + fmt.Printf("Cannot change into '%s'! %v\n", purl[3]+"/"+purl[5], err) + os.Exit(2) } } - fmt.Printf("Downloading '%v' %v of %v\n", url, urlNum+1, len(urls)) - for _, image := range images { // get the images downloading + fmt.Printf("Downloading '%s' %d of %d\n", url, urlNum+1, len(urls)) + for _, image := range images { // start downloading images go func(image godesu.Image) { var fs finishState if *useOrigFilename { @@ -92,18 +92,18 @@ func main() { } if _, err := os.Stat(fs.filename); err == nil { - fs.err = fmt.Errorf("'%v' exists! Skipping...", fs.filename) + fs.err = fmt.Errorf("'%s' exists! Skipping...", fs.filename) finishStateChan <- fs return } resp, err := http.Get(image.URL) if err != nil { - fs.err = fmt.Errorf("Error downloading '%v'! %v", image.URL, err) + fs.err = fmt.Errorf("Error downloading '%s'! %v", image.URL, err) finishStateChan <- fs return } else if resp.StatusCode != http.StatusOK { - fs.err = fmt.Errorf("Error downloading '%v'! Http status not ok: %d", image.URL, resp.StatusCode) + fs.err = fmt.Errorf("Error downloading '%s'! Http status not ok: %d", image.URL, resp.StatusCode) finishStateChan <- fs return } @@ -113,7 +113,7 @@ func main() { file, err := os.Create(tmpFilename) if err != nil { - fs.err = fmt.Errorf("Cannot create '%v'! %v", tmpFilename, err) + fs.err = fmt.Errorf("Cannot create '%s'! %v", tmpFilename, err) finishStateChan <- fs return } @@ -122,7 +122,7 @@ func main() { io.Copy(file, resp.Body) if err := os.Rename(tmpFilename, fs.filename); err != nil { - fs.err = fmt.Errorf("Unable to rename '%v' to '%v'! %v", tmpFilename, fs.filename, err) + fs.err = fmt.Errorf("Unable to rename '%s' to '%s'! %v", tmpFilename, fs.filename, err) } finishStateChan <- fs @@ -133,9 +133,9 @@ func main() { for i := 0; i < len(images); i++ { // watch for images to finish fs := <-finishStateChan if fs.err != nil { - fmt.Printf("%v %v of %v\n", fs.err, i+1, len(images)) + fmt.Printf("%v %d of %d\n", fs.err, i+1, len(images)) } else { - fmt.Printf("Finished downloading '%v' %v of %v\n", fs.filename, i+1, len(images)) + fmt.Printf("Finished downloading '%s' %d of %d\n", fs.filename, i+1, len(images)) } } diff --git a/readme.md b/readme.md index 95d117b..aa5a467 100644 --- a/readme.md +++ b/readme.md @@ -6,6 +6,8 @@ Goscrape is a very simple and efficient 4chan media scraper written in go that s ## How to install `go get github.com/lordrusk/goscrape` +Binaries included in each release. Latest release found [here.](https://github.com/LordRusk/goscrape/releases/tag/v1.3.2) + ## How to use `-h` for help menu. @@ -13,11 +15,10 @@ Goscrape is a very simple and efficient 4chan media scraper written in go that s `-c` to set a custom directory. -Put in quotes for multiple threads. +Each arguement after opts treated as a link to thread. ## Features * Goscrape is upwards of 4x faster then other scrapers, goscrape does this by using go's concurrency to download multiple images at the same time, taking advantage of more bandwidth. You won't find download speeds like this anywhere else. -* Goscrape uses the [godesu](https://github.com/mtarnawa/godesu) 4chan read-only api to interact with 4chan. The old system used a request library, regex, maps, etc, but this seems to fit more with the go style. * Goscrape is cross-platform, works everywhere that go does out of the box! (Plan9, etc, etc) ## Why?