Skip to content

Commit

Permalink
use os.Exit(), lots of small changes here and there. Push to v1.3.2
Browse files Browse the repository at this point in the history
  • Loading branch information
@LordRusk
LordRusk committed Jan 16, 2021
1 parent 43f23ff commit 03aa8bc
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 43 deletions.
82 changes: 41 additions & 41 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,72 +17,72 @@ type finishState struct {
err error
}

var ( // opts
help = flag.Bool("h", false, "Show help menu")
useOrigFilename = flag.Bool("o", false, "Download with the original filename")
customDownloadDir = flag.String("c", "", "Set a custom directory for the images to download to")
)
// opts
var useOrigFilename = flag.Bool("o", false, "Download with the original filename")
var customDownloadDir = flag.String("c", "", "Set a custom directory for the images to download to")

func main() {
flag.Parse()
if *help {
flag.Usage()
return
}

args := flag.Args()
if len(args) < 1 {
urls := flag.Args()
if len(urls) < 1 {
flag.Usage()
return
os.Exit(0)
}

urls := strings.Split(args[0], " ")
origDir, err := os.Getwd()
if err != nil {
fmt.Printf("Could not grab working directory! | %v\n", err)
return
fmt.Printf("Could not grab working directory! %v\n", err)
os.Exit(1)
}

Gochan := godesu.New() // intialize godesu

for urlNum, url := range urls { // loop through all urls
purl := strings.Split(url, "/")
if len(purl) < 6 { // check length to avoid runtime error
fmt.Println("Invalid URL!")
os.Exit(1)
}

ThreadNum, err := strconv.Atoi(purl[5])
if err != nil {
fmt.Printf("Make sure the URL is correct! | %v\n", err)
return
fmt.Printf("Inavlid URL! %v\n", err)
os.Exit(1)
}

err, Thread := Gochan.Board(purl[3]).GetThread(ThreadNum)
if err != nil {
fmt.Printf("Could not fetch thread! | %v\n", err)
return
fmt.Printf("Could not fetch thread! %v\n", err)
os.Exit(1)
}

images := Thread.Images()
finishStateChan := make(chan finishState, len(images)) // make the download channel with proper buffer size

if *customDownloadDir != "" {
if err := os.Chdir(*customDownloadDir + "/"); err != nil {
if err := os.MkdirAll(*customDownloadDir+"/", os.ModePerm); err != nil {
fmt.Printf("Cannot create directory! %v\n", err)
return
}
if err := os.MkdirAll(*customDownloadDir+"/", os.ModePerm); err != nil {
fmt.Printf("Cannot create '%s'! %v\n", *customDownloadDir+"/", err)
os.Exit(1)
}

os.Chdir(*customDownloadDir + "/")
if err := os.Chdir(*customDownloadDir + "/"); err != nil { // this should return an error
fmt.Printf("Cannot change into '%s'! %v\n", *customDownloadDir+"/", err)
os.Exit(2)
}
} else {
if err := os.Chdir(purl[3] + "/" + purl[5]); err != nil {
if err := os.MkdirAll(purl[3]+"/"+purl[5], os.ModePerm); err != nil {
fmt.Printf("Cannot create directory! %v\n", err)
return
}
if err := os.MkdirAll(purl[3]+"/"+purl[5], os.ModePerm); err != nil {
fmt.Printf("Cannot create '%s'! %v\n", purl[3]+"/"+purl[5], err)
os.Exit(1)
}

os.Chdir(purl[3] + "/" + purl[5])
if err := os.Chdir(purl[3] + "/" + purl[5]); err != nil { // this should never return an error
fmt.Printf("Cannot change into '%s'! %v\n", purl[3]+"/"+purl[5], err)
os.Exit(2)
}
}

fmt.Printf("Downloading '%v' %v of %v\n", url, urlNum+1, len(urls))
for _, image := range images { // get the images downloading
fmt.Printf("Downloading '%s' %d of %d\n", url, urlNum+1, len(urls))
for _, image := range images { // start downloading images
go func(image godesu.Image) {
var fs finishState
if *useOrigFilename {
Expand All @@ -92,18 +92,18 @@ func main() {
}

if _, err := os.Stat(fs.filename); err == nil {
fs.err = fmt.Errorf("'%v' exists! Skipping...", fs.filename)
fs.err = fmt.Errorf("'%s' exists! Skipping...", fs.filename)
finishStateChan <- fs
return
}

resp, err := http.Get(image.URL)
if err != nil {
fs.err = fmt.Errorf("Error downloading '%v'! %v", image.URL, err)
fs.err = fmt.Errorf("Error downloading '%s'! %v", image.URL, err)
finishStateChan <- fs
return
} else if resp.StatusCode != http.StatusOK {
fs.err = fmt.Errorf("Error downloading '%v'! Http status not ok: %d", image.URL, resp.StatusCode)
fs.err = fmt.Errorf("Error downloading '%s'! Http status not ok: %d", image.URL, resp.StatusCode)
finishStateChan <- fs
return
}
Expand All @@ -113,7 +113,7 @@ func main() {

file, err := os.Create(tmpFilename)
if err != nil {
fs.err = fmt.Errorf("Cannot create '%v'! %v", tmpFilename, err)
fs.err = fmt.Errorf("Cannot create '%s'! %v", tmpFilename, err)
finishStateChan <- fs
return
}
Expand All @@ -122,7 +122,7 @@ func main() {
io.Copy(file, resp.Body)

if err := os.Rename(tmpFilename, fs.filename); err != nil {
fs.err = fmt.Errorf("Unable to rename '%v' to '%v'! %v", tmpFilename, fs.filename, err)
fs.err = fmt.Errorf("Unable to rename '%s' to '%s'! %v", tmpFilename, fs.filename, err)
}

finishStateChan <- fs
Expand All @@ -133,9 +133,9 @@ func main() {
for i := 0; i < len(images); i++ { // watch for images to finish
fs := <-finishStateChan
if fs.err != nil {
fmt.Printf("%v %v of %v\n", fs.err, i+1, len(images))
fmt.Printf("%v %d of %d\n", fs.err, i+1, len(images))
} else {
fmt.Printf("Finished downloading '%v' %v of %v\n", fs.filename, i+1, len(images))
fmt.Printf("Finished downloading '%s' %d of %d\n", fs.filename, i+1, len(images))
}
}

Expand Down
5 changes: 3 additions & 2 deletions readme.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,19 @@ Goscrape is a very simple and efficient 4chan media scraper written in go that s
## How to install
`go get github.com/lordrusk/goscrape`

Binaries included in each release. Latest release found [here.](https://github.com/LordRusk/goscrape/releases/tag/v1.3.2)

## How to use
`-h` for help menu.

`-o` for original filenames.

`-c` to set a custom directory.

Put in quotes for multiple threads.
Each arguement after opts treated as a link to thread.

## Features
* Goscrape is upwards of 4x faster then other scrapers, goscrape does this by using go's concurrency to download multiple images at the same time, taking advantage of more bandwidth. You won't find download speeds like this anywhere else.
* Goscrape uses the [godesu](https://github.com/mtarnawa/godesu) 4chan read-only api to interact with 4chan. The old system used a request library, regex, maps, etc, but this seems to fit more with the go style.
* Goscrape is cross-platform, works everywhere that go does out of the box! (Plan9, etc, etc)

## Why?
Expand Down

0 comments on commit 03aa8bc

Please sign in to comment.