Merge pull request #2 from gsxhnd/fix/crawl-create-dir

fix: dir not create

.vscode/launch.json

@@ -49,7 +49,7 @@
       "showLog": true,
       "debugAdapter": "dlv-dap",
       "program": "${workspaceRoot}/",
-      "args": ["code", "--proxy=http://127.0.0.1:7890", "TEST-001"],
+      "args": ["jav_code", "--proxy=http://127.0.0.1:7890", "PCDE-016"],
       "output": "${workspaceRoot}/build/garage",
       "trace": "verbose",
       "cwd": "${workspaceRoot}"

CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## Version 0.0.7
+
+- FIX: 爬虫部分没有提前创建下载数据
+
 ## Version 0.0.6
 
 - CHORE: 编译去除路径

cmd/flag.go

@@ -5,8 +5,9 @@ import (
 )
 
 var (
-	proxyFlag = &cli.StringFlag{Name: "proxy", Usage: "代理配置,如: http://127.0.0.1:1080"}
-	siteFlag  = &cli.StringFlag{
+	proxyFlag   = &cli.StringFlag{Name: "proxy", Usage: "代理配置,如: http://127.0.0.1:1080"}
+	destDirFlag = &cli.StringFlag{Name: "dest_dir", Usage: "设置下载目录", Value: "./javbus"}
+	siteFlag    = &cli.StringFlag{
 		Name:        "site",
 		Usage:       "选择爬取数据的网站,支持网站(javbus)",
 		Destination: nil,

cmd/jav_code.go

@@ -1,6 +1,8 @@
 package cmd
 
 import (
+	"path"
+
 	"github.com/gsxhnd/garage/crawl"
 	"github.com/gsxhnd/garage/utils"
 	"github.com/urfave/cli/v2"
@@ -14,14 +16,22 @@ var javCodeCmd = &cli.Command{
 	Flags: []cli.Flag{
 		proxyFlag,
 		siteFlag,
+		destDirFlag,
 	},
 	Action: func(ctx *cli.Context) error {
 		var (
-			newLogger = utils.GetLogger()
-			code      = ctx.Args().Get(0)
-			proxy     = ctx.String("proxy")
+			logger  = utils.GetLogger()
+			code    = ctx.Args().Get(0)
+			proxy   = ctx.String("proxy")
+			destDir = "./javs"
 		)
-		c := crawl.NewCrawlClient(newLogger)
+
+		if err := utils.MkdirDestDir(path.Join(destDir, code)); err != nil {
+			logger.Panic("目录创建失败， Error: " + err.Error())
+			return err
+		}
+
+		c := crawl.NewCrawlClient(logger)
 		_ = c.SetProxy(proxy)
 		c.StarCrawlJavbusMovie(code)
 		return nil

crawl/javbus.go

@@ -3,13 +3,13 @@ package crawl
 import (
 	"bytes"
 	"encoding/json"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"strings"
 
 	"github.com/gocolly/colly/v2"
+	"go.uber.org/zap"
 )
 
 func (c2 *Client) StarCrawlJavbusMovie(code string) {
@@ -58,15 +58,20 @@ func (c2 *Client) DownloadInfo(code string) (*JavMovie, error) {
 			star := element.Attr("title")
 			data.Stars = append(data.Stars, star)
 		})
+
+		// e.ForEach("#magnet-table , tr", func(i int, h *colly.HTMLElement) {
+		// 	fmt.Println(i)
+		// 	fmt.Println(h)
+		// })
 	})
 	err := c2.collector.Visit(c2.javbusUrl + code)
 	if err != nil {
-		fmt.Println("err:", err)
 		return nil, err
 	}
 	saveData, _ := json.Marshal(&data)
 	err = ioutil.WriteFile("./javs/"+code+"/info.json", saveData, os.ModeAppend)
 	if err != nil {
+		c2.logger.Error("", zap.Error(err))
 		return nil, err
 	} else {
 		return &data, nil

utils/dest.go

@@ -6,7 +6,7 @@ func MkdirDestDir(dir_name string) error {
 	_, err := os.Stat(dir_name)
 	if err != nil {
 		if os.IsNotExist(err) {
-			err = os.Mkdir(dir_name, os.ModePerm)
+			err = os.MkdirAll(dir_name, os.ModePerm)
 			if err != nil {
 				return err
 			}