From d54777fe28300443ebe6c0558afedb66c99c4f3f Mon Sep 17 00:00:00 2001 From: Axel Berardino Date: Mon, 2 Sep 2019 11:57:49 +0200 Subject: [PATCH] Update documentation and fix quality issues Update readme to handle badges. Make travis works. Fix all go report issues. Fix bad import path. --- .gitignore | 1 + .travis.yml | 11 +++- README.md | 19 ++++++- cmd/cli/main.go | 87 +++++++++++++++++++----------- cmd/server/main.go | 8 +-- cmd/server/page/download_file.go | 2 +- cmd/server/page/gen_account.go | 6 +-- cmd/server/page/main_page.go | 4 +- cmd/server/page/view_account.go | 2 +- docs/index.md | 7 +-- gen_demo.sh | 2 +- generate/generator.go | 27 +++++++--- misc/version.go | 2 +- models/character.go | 2 +- models/item.go | 2 + scraper/characters.go | 2 +- scraper/rate_limit_manager_test.go | 8 +-- scraper/scraper.go | 76 +++++++++++++++----------- scraper/stash.go | 2 +- 19 files changed, 174 insertions(+), 96 deletions(-) diff --git a/.gitignore b/.gitignore index fb2d1cc..0367428 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ pass.txt .vscode *.tar.gz *.zip +coverage.txt diff --git a/.travis.yml b/.travis.yml index 82d70ad..673f7f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ language: go -go: master -script: go build ./... && go test ./... + +go: + - 1.12.x git: # for cloning @@ -8,3 +9,9 @@ git: before_install: - go get -u github.com/gin-gonic/gin + +script: + - go test -v ./... -race -coverprofile=coverage.txt -covermode=atomic + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/README.md b/README.md index 5f22023..9bcc570 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ -## POE STASH +# POE STASH + +[![GoReportCard Badge]][GoReportCard] +[![Travis Badge]][Travis] +[![License Badge]][License] +[![Issue Badge]][Issue] +[![Pull Request Badge]][Pull Request] Share your stuff with others! @@ -112,3 +118,14 @@ Check the FAQ: [Here](/docs/faq.md) To contact me, either send a mail at cptpingu@gmail.com, or open an issue on this github. + +[GoReportCard]: https://goreportcard.com/report/github.com/cptpingu/poe-stash +[GoReportCard Badge]: https://goreportcard.com/badge/github.com/cptpingu/poe-stash +[License]: https://opensource.org/licenses/MIT +[License Badge]: https://img.shields.io/github/license/cptpingu/poe-stash +[Travis]: https://travis-ci.com/cptpingu/poe-stash +[Travis Badge]: https://api.travis-ci.org/cptpingu/poe-stash.svg?branch=master +[Issue]: https://github.com/cptpingu/poe-stash/issues +[Issue Badge]: https://img.shields.io/github/issues-raw/cptpingu/poe-stash +[Pull Request]: https://github.com/cptpingu/poe-stash/pulls +[Pull Request Badge]: https://img.shields.io/github/issues-pr-raw/cptpingu/poe-stash diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 700949f..71f1228 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -6,9 +6,9 @@ import ( "fmt" "os" - "github.com/poe-stash/generate" - "github.com/poe-stash/misc" - "github.com/poe-stash/scraper" + "github.com/cptpingu/poe-stash/generate" + "github.com/cptpingu/poe-stash/misc" + "github.com/cptpingu/poe-stash/scraper" ) // mandatoryOption ensure an option is not empty. @@ -20,6 +20,55 @@ func mandatoryOption(opt string, name string) bool { return true } +// scrapData scraps all data. +func scrapData(account, poeSessID, realm, league string, demo, cache bool, verbosity int) (*scraper.ScrapedData, error) { + scraper := scraper.NewScraper(account, poeSessID, realm, league) + scraper.SetDemo(demo) + if cache { + scraper.EnableCache() + } + scraper.SetVerbosity(verbosity) + data, err := scraper.ScrapEverything() + if err != nil { + return nil, err + } + + return data, nil +} + +// generateData generates html file from the given data. +func generateData(data *scraper.ScrapedData, output string) (resErr error) { + var file *os.File + var err error + + if output == "-" { + file = os.Stdout + } else { + file, err = os.Create(output) + if err != nil { + return err + } + defer func() { + if err := file.Close(); err != nil { + if resErr != nil { + resErr = err + } + } + }() + } + + w := bufio.NewWriter(file) + gen := generate.NewGenerator(w) + if errGen := gen.GenerateHTML(data); errGen != nil { + return err + } + if errFlush := w.Flush(); err != nil { + return errFlush + } + + return nil +} + // main is the main routine for this CLI. // This CLI allows to generate an html file which contains all // inventories, characters and items for a given account. @@ -76,44 +125,20 @@ func main() { } } - scraper := scraper.NewScraper(*account, *poeSessID, *realm, *league) - scraper.SetDemo(*demo) - if *cache { - scraper.EnableCache() - } - scraper.SetVerbosity(*verbosity) - data, errScrap := scraper.ScrapEverything() + data, errScrap := scrapData(*account, *poeSessID, *realm, *league, *demo, *cache, *verbosity) if errScrap != nil { fmt.Println("can't scrap data", errScrap) os.Exit(2) } - var file *os.File - var err error if *output == "" { *output = *account + "-" + *league + ".html" } - if *output == "-" { - file = os.Stdout - } else { - file, err = os.Create(*output) - if err != nil { - panic(err) - } - defer func() { - if err := file.Close(); err != nil { - panic(err) - } - }() - } - - w := bufio.NewWriter(file) - gen := generate.NewGenerator(w) - if errGen := gen.GenerateHTML(data); errGen != nil { + if errGen := generateData(data, *output); errGen != nil { fmt.Println("can't generate data", errGen) os.Exit(3) } - w.Flush() - fmt.Println("File sucessfully generated:", *output) + + fmt.Println("File successfully generated:", *output) } diff --git a/cmd/server/main.go b/cmd/server/main.go index 3ad8069..62cd05a 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -12,10 +12,10 @@ import ( "github.com/gin-gonic/gin" - "github.com/poe-stash/cmd/server/page" - "github.com/poe-stash/generate" - "github.com/poe-stash/misc" - "github.com/poe-stash/scraper" + "github.com/cptpingu/poe-stash/cmd/server/page" + "github.com/cptpingu/poe-stash/generate" + "github.com/cptpingu/poe-stash/misc" + "github.com/cptpingu/poe-stash/scraper" ) // EnvMiddleware will add env to query. diff --git a/cmd/server/page/download_file.go b/cmd/server/page/download_file.go index 09a1575..d1a8c05 100644 --- a/cmd/server/page/download_file.go +++ b/cmd/server/page/download_file.go @@ -5,7 +5,7 @@ import ( "github.com/gin-gonic/gin" - "github.com/poe-stash/scraper" + "github.com/cptpingu/poe-stash/scraper" ) // DownloadFileHandler handles force download of a file. diff --git a/cmd/server/page/gen_account.go b/cmd/server/page/gen_account.go index a06094e..27c81c3 100644 --- a/cmd/server/page/gen_account.go +++ b/cmd/server/page/gen_account.go @@ -9,8 +9,8 @@ import ( "github.com/gin-gonic/gin" - "github.com/poe-stash/generate" - "github.com/poe-stash/scraper" + "github.com/cptpingu/poe-stash/generate" + "github.com/cptpingu/poe-stash/scraper" ) // GenAccountHandler handles refresh of an account. @@ -75,7 +75,7 @@ func GenAccountHandler(c *gin.Context) { // Everything is fine, let's store the poeSessID. if errFile := ioutil.WriteFile(scraper.DataCacheDir+account+".poesessid", []byte(poeSessID), 0644); errFile != nil { // Non fatal error, storing the session is not mandatory. - fmt.Println("error occured:", errFile) + fmt.Println("error occurred:", errFile) } c.HTML(http.StatusOK, "redirect", "/view/"+name) } diff --git a/cmd/server/page/main_page.go b/cmd/server/page/main_page.go index 9378062..c311ac5 100644 --- a/cmd/server/page/main_page.go +++ b/cmd/server/page/main_page.go @@ -10,8 +10,8 @@ import ( "github.com/gin-gonic/gin" - "github.com/poe-stash/models" - "github.com/poe-stash/scraper" + "github.com/cptpingu/poe-stash/models" + "github.com/cptpingu/poe-stash/scraper" ) // listAllAccounts list all fetch accounts. diff --git a/cmd/server/page/view_account.go b/cmd/server/page/view_account.go index 87d6335..c33d8da 100644 --- a/cmd/server/page/view_account.go +++ b/cmd/server/page/view_account.go @@ -5,7 +5,7 @@ import ( "net/http" "github.com/gin-gonic/gin" - "github.com/poe-stash/scraper" + "github.com/cptpingu/poe-stash/scraper" ) // ViewAccountHandler handles viewing an account diff --git a/docs/index.md b/docs/index.md index b3ad0d0..b2c39c3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,9 +19,10 @@ Share a stash with a friend: ## Download Released versions are available here: - * [Windows](https://github.com/cptpingu/poe-stash/releases/download/v0.5/poe-stash-windows-amd64.zip) - * [Linux](https://github.com/cptpingu/poe-stash/releases/download/v0.5/poe-stash-linux-x86_64.tar.gz) - * [MacOS](https://github.com/cptpingu/poe-stash/releases/download/v0.5/poe-stash-darwin-x86_64.tar.gz) + + * [Windows](https://gitreleases.dev/gh/cptpingu/poe-stash/latest/poe-stash-windows-amd64.zip) + * [Linux](https://gitreleases.dev/gh/cptpingu/poe-stash/latest/poe-stash-linux-x86_64.tar.gz) + * [MacOS](https://gitreleases.dev/gh/cptpingu/poe-stash/latest/poe-stash-darwin-x86_64.tar.gz) ## Getting started diff --git a/gen_demo.sh b/gen_demo.sh index 948ae99..d8b4e84 100755 --- a/gen_demo.sh +++ b/gen_demo.sh @@ -4,5 +4,5 @@ for file in $(\ls demo/*.json); do name=$(basename $file) account=${name%%.json} echo "Generating $account..." - go run cmd/cli/main.go --account all_stash_types --demo + go run cmd/cli/main.go --account $account --demo done diff --git a/generate/generator.go b/generate/generator.go index bad6bb1..31a7c7b 100644 --- a/generate/generator.go +++ b/generate/generator.go @@ -16,9 +16,9 @@ import ( "time" "unicode" - "github.com/poe-stash/misc" - "github.com/poe-stash/models" - "github.com/poe-stash/scraper" + "github.com/cptpingu/poe-stash/misc" + "github.com/cptpingu/poe-stash/models" + "github.com/cptpingu/poe-stash/scraper" ) const ( @@ -672,9 +672,9 @@ func GenNaiveSearchIndex(item models.Item) string { return strings.Join(keys, " ") } -// ItemCategory returns a text item category from categories. -func ItemCategory(item models.Item) string { - res := make([]string, 0, 10) +// itemCategoryAttribute returns attributes of an item category. +func itemCategoryAttribute(item models.Item) []string { + res := make([]string, 0, 5) if item.IsShaper { res = append(res, "shaper") @@ -712,7 +712,12 @@ func ItemCategory(item models.Item) string { res = append(res, "divination", "divine", "divcard", "divinationcard") } - categories := item.Category + return res +} + +// itemCategoryType returns types of an item category. +func itemCategoryType(categories models.Category) []string { + res := make([]string, 0, 5) if categories.Armor != nil { res = append(res, "armor", "armour", "armors", "armours") @@ -757,6 +762,14 @@ func ItemCategory(item models.Item) string { } } + return res +} + +// ItemCategory returns a text item category from categories. +func ItemCategory(item models.Item) string { + attributes := itemCategoryAttribute(item) + types := itemCategoryType(item.Category) + res := append(attributes, types...) sort.Strings(res) return strings.Join(res, " ") } diff --git a/misc/version.go b/misc/version.go index 22bd293..44aa5e0 100644 --- a/misc/version.go +++ b/misc/version.go @@ -1,4 +1,4 @@ package misc // Version of the application. -const Version = "v0.5" +const Version = "v0.6" diff --git a/models/character.go b/models/character.go index 1548fc4..83d0b2d 100644 --- a/models/character.go +++ b/models/character.go @@ -97,7 +97,7 @@ func ParseInventory(data []byte) (*CharacterInventory, error) { return &inventory, nil } -// CharacterSkills holds all skills choosen by the character +// CharacterSkills holds all skills chosen by the character // and also all items (jewels or abyss) put in the slots. type CharacterSkills struct { Hashes []int `json:"hashes"` diff --git a/models/item.go b/models/item.go index 9f26311..d2f8cbb 100644 --- a/models/item.go +++ b/models/item.go @@ -79,6 +79,7 @@ type ItemProperty struct { // FrameType is a type of rarity of an item. type FrameType int +// Frame type represents the type of frame to draw for an item. const ( NormalItemFrameType FrameType = iota MagicItemFrameType @@ -123,6 +124,7 @@ type IncubatedItemType struct { // LayoutType is the type of layout. type LayoutType string +// Layout is the type of layout to use (type of grid to place items). const ( DefaultLayout LayoutType = "" CurrencyLayout = "currency" diff --git a/scraper/characters.go b/scraper/characters.go index 66e7f1d..f250923 100644 --- a/scraper/characters.go +++ b/scraper/characters.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "net/url" - "github.com/poe-stash/models" + "github.com/cptpingu/poe-stash/models" ) // ScrapCharacters scraps all characters owned by a user. diff --git a/scraper/rate_limit_manager_test.go b/scraper/rate_limit_manager_test.go index 7f629b5..5ae7ab1 100644 --- a/scraper/rate_limit_manager_test.go +++ b/scraper/rate_limit_manager_test.go @@ -38,13 +38,13 @@ func TestExtractFirstRuleFromString(t *testing.T) { var r RateRules var err error - r, err = ExtractFirstRuleFromString("") + _, err = ExtractFirstRuleFromString("") errorEqual(t, errRuleParseFirst, err) - r, err = ExtractFirstRuleFromString("errRuleParseFirst") + _, err = ExtractFirstRuleFromString("errRuleParseFirst") errorEqual(t, errRuleParseFirst, err) - r, err = ExtractFirstRuleFromString("34:34:34:34") + _, err = ExtractFirstRuleFromString("34:34:34:34") errorEqual(t, errRuleParseFirst, err) - r, err = ExtractFirstRuleFromString("a:b:c") + _, err = ExtractFirstRuleFromString("a:b:c") errorEqual(t, errInvalidRule, err) r, err = ExtractFirstRuleFromString("1:2:3") diff --git a/scraper/scraper.go b/scraper/scraper.go index 2279a8d..761590d 100644 --- a/scraper/scraper.go +++ b/scraper/scraper.go @@ -8,7 +8,7 @@ import ( "strconv" "time" - "github.com/poe-stash/models" + "github.com/cptpingu/poe-stash/models" ) const ( @@ -99,30 +99,8 @@ func hash(s string) string { return strconv.Itoa(int(h.Sum32())) } -// CallAPI calls a distant API and returns the content. -func (s *Scraper) CallAPI(apiURL string) ([]byte, error) { - var fileCache string - if s.cache { - fileCache = s.cacheDir + hash(apiURL) - if b, err := ioutil.ReadFile(fileCache); err != nil { - fmt.Println("can't read cache", err) - } else { - return b, nil - } - } - - req, err := http.NewRequest("GET", apiURL, nil) - if err != nil { - return nil, err - } - cookie := http.Cookie{ - Name: "POESESSID", - Value: s.poeSessionID, - } - req.AddCookie(&cookie) - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - - // Handle rate limiting. +// rateLimitWait waits for an amount of time depending of the rate limit. +func (s *Scraper) rateLimitWait(req *http.Request, apiURL string) (string, func()) { baseURL := req.URL.Hostname() + req.URL.EscapedPath() rateLimiter := s.rateLimitManager.GetRateLimiter(s.poeSessionID, baseURL) @@ -134,14 +112,11 @@ func (s *Scraper) CallAPI(apiURL string) ([]byte, error) { } } time.Sleep(waitTime) + return baseURL, queryDone +} - // Query the server. - resp, errResponse := s.client.Do(req) - queryDone() - if errResponse != nil { - return nil, errResponse - } - +// updateRateLimit updates the dynamic rate limiters. +func (s *Scraper) updateRateLimit(resp *http.Response, baseURL string) { // Let check if there are some rate limiting rules rateLimitRules := resp.Header.Get("X-Rate-Limit-Account") if rateLimitRules == "" { @@ -166,6 +141,43 @@ func (s *Scraper) CallAPI(apiURL string) ([]byte, error) { } } +} + +// CallAPI calls a distant API and returns the content. +func (s *Scraper) CallAPI(apiURL string) ([]byte, error) { + var fileCache string + if s.cache { + fileCache = s.cacheDir + hash(apiURL) + if b, err := ioutil.ReadFile(fileCache); err != nil { + fmt.Println("can't read cache", err) + } else { + return b, nil + } + } + + req, err := http.NewRequest("GET", apiURL, nil) + if err != nil { + return nil, err + } + cookie := http.Cookie{ + Name: "POESESSID", + Value: s.poeSessionID, + } + req.AddCookie(&cookie) + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + + // Handle rate limiting. + baseURL, queryDone := s.rateLimitWait(req, apiURL) + + // Query the server. + resp, errResponse := s.client.Do(req) + queryDone() + if errResponse != nil { + return nil, errResponse + } + + s.updateRateLimit(resp, baseURL) + defer func() { localErr := resp.Body.Close() if err == nil { diff --git a/scraper/stash.go b/scraper/stash.go index c4b8f87..c4e1618 100644 --- a/scraper/stash.go +++ b/scraper/stash.go @@ -6,7 +6,7 @@ import ( "net/url" "strconv" - "github.com/poe-stash/models" + "github.com/cptpingu/poe-stash/models" ) // ScrapStash scraps a stash from the official website.