From 1b982cec6b887a55d4331f480140339cecafae74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paulo=20Roberto=20Brand=C3=A3o?= Date: Thu, 9 Aug 2018 11:14:12 -0300 Subject: [PATCH 1/8] Fetching Organization Private Repos Added function GetRepositoriesFromOrganization for downloading of private repo when target is an Organization using SSH URL --- core/github.go | 220 +++++++++++++++++++++++++++++-------------------- main.go | 8 +- 2 files changed, 136 insertions(+), 92 deletions(-) diff --git a/core/github.go b/core/github.go index a1941701..a32552be 100644 --- a/core/github.go +++ b/core/github.go @@ -1,113 +1,151 @@ package core import ( - "context" + "context" - "github.com/google/go-github/github" + "github.com/google/go-github/github" ) type GithubOwner struct { - Login *string - ID *int64 - Type *string - Name *string - AvatarURL *string - URL *string - Company *string - Blog *string - Location *string - Email *string - Bio *string + Login *string + ID *int64 + Type *string + Name *string + AvatarURL *string + URL *string + Company *string + Blog *string + Location *string + Email *string + Bio *string } type GithubRepository struct { - Owner *string - ID *int64 - Name *string - FullName *string - CloneURL *string - URL *string - DefaultBranch *string - Description *string - Homepage *string + Owner *string + ID *int64 + Name *string + FullName *string + CloneURL *string + URL *string + DefaultBranch *string + Description *string + Homepage *string } func GetUserOrOrganization(login string, client *github.Client) (*GithubOwner, error) { - ctx := context.Background() - user, _, err := client.Users.Get(ctx, login) - if err != nil { - return nil, err - } - return &GithubOwner{ - Login: user.Login, - ID: user.ID, - Type: user.Type, - Name: user.Name, - AvatarURL: user.AvatarURL, - URL: user.HTMLURL, - Company: user.Company, - Blog: user.Blog, - Location: user.Location, - Email: user.Email, - Bio: user.Bio, - }, nil + ctx := context.Background() + user, _, err := client.Users.Get(ctx, login) + if err != nil { + return nil, err + } + return &GithubOwner{ + Login: user.Login, + ID: user.ID, + Type: user.Type, + Name: user.Name, + AvatarURL: user.AvatarURL, + URL: user.HTMLURL, + Company: user.Company, + Blog: user.Blog, + Location: user.Location, + Email: user.Email, + Bio: user.Bio, + }, nil } func GetRepositoriesFromOwner(login *string, client *github.Client) ([]*GithubRepository, error) { - var allRepos []*GithubRepository - loginVal := *login - ctx := context.Background() - opt := &github.RepositoryListOptions{ - Type: "sources", - } + var allRepos []*GithubRepository + loginVal := *login + ctx := context.Background() + opt := &github.RepositoryListOptions{ + Type: "sources", + } - for { - repos, resp, err := client.Repositories.List(ctx, loginVal, opt) - if err != nil { - return allRepos, err - } - for _, repo := range repos { - if !*repo.Fork { - r := GithubRepository{ - Owner: repo.Owner.Login, - ID: repo.ID, - Name: repo.Name, - FullName: repo.FullName, - CloneURL: repo.CloneURL, - URL: repo.HTMLURL, - DefaultBranch: repo.DefaultBranch, - Description: repo.Description, - Homepage: repo.Homepage, - } - allRepos = append(allRepos, &r) - } - } - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } + for { + repos, resp, err := client.Repositories.List(ctx, loginVal, opt) + if err != nil { + return allRepos, err + } + for _, repo := range repos { + if !*repo.Fork { + r := GithubRepository{ + Owner: repo.Owner.Login, + ID: repo.ID, + Name: repo.Name, + FullName: repo.FullName, + CloneURL: repo.CloneURL, + URL: repo.HTMLURL, + DefaultBranch: repo.DefaultBranch, + Description: repo.Description, + Homepage: repo.Homepage, + } + allRepos = append(allRepos, &r) + } + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } - return allRepos, nil + return allRepos, nil +} + +func GetRepositoriesFromOrganization(login *string, client *github.Client) ([]*GithubRepository, error) { + var allRepos []*GithubRepository + loginVal := *login + ctx := context.Background() + opt := &github.RepositoryListByOrgOptions{ + Type: "sources", + } + + for { + repos, resp, err := client.Repositories.ListByOrg(ctx, loginVal, opt) + if err != nil { + return allRepos, err + } + for _, repo := range repos { + if !*repo.Fork { + r := GithubRepository{ + Owner: repo.Owner.Login, + ID: repo.ID, + Name: repo.Name, + FullName: repo.FullName, + CloneURL: repo.SSHURL, + URL: repo.HTMLURL, + DefaultBranch: repo.DefaultBranch, + Description: repo.Description, + Homepage: repo.Homepage, + } + allRepos = append(allRepos, &r) + } + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + + return allRepos, nil } func GetOrganizationMembers(login *string, client *github.Client) ([]*GithubOwner, error) { - var allMembers []*GithubOwner - loginVal := *login - ctx := context.Background() - opt := &github.ListMembersOptions{} - for { - members, resp, err := client.Organizations.ListMembers(ctx, loginVal, opt) - if err != nil { - return allMembers, err - } - for _, member := range members { - allMembers = append(allMembers, &GithubOwner{Login: member.Login, ID: member.ID, Type: member.Type}) - } - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } - return allMembers, nil + var allMembers []*GithubOwner + loginVal := *login + ctx := context.Background() + opt := &github.ListMembersOptions{} + for { + members, resp, err := client.Organizations.ListMembers(ctx, loginVal, opt) + if err != nil { + return allMembers, err + } + for _, member := range members { + allMembers = append(allMembers, &GithubOwner{Login: member.Login, ID: member.ID, Type: member.Type}) + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return allMembers, nil } diff --git a/main.go b/main.go index a693ad89..26020505 100644 --- a/main.go +++ b/main.go @@ -57,12 +57,18 @@ func GatherRepositories(sess *core.Session) { for i := 0; i < threadNum; i++ { go func() { for { + var repos []*core.GithubRepository + var err error target, ok := <-ch if !ok { wg.Done() return } - repos, err := core.GetRepositoriesFromOwner(target.Login, sess.GithubClient) + if *target.Type == "Organization" { + repos, err = core.GetRepositoriesFromOrganization(target.Login, sess.GithubClient) + } else { + repos, err = core.GetRepositoriesFromOwner(target.Login, sess.GithubClient) + } if err != nil { sess.Out.Error(" Failed to retrieve repositories from %s: %s\n", *target.Login, err) } From 0f67d48942ddd6c4f9d2c34e01c34d00d18f012c Mon Sep 17 00:00:00 2001 From: Mick Grove Date: Tue, 14 Aug 2018 15:11:38 -0700 Subject: [PATCH 2/8] Fixed bug where gitrob does not look at files in the very initial commit. --- core/git.go | 189 +++++++++++++++++++++++++++------------------------- 1 file changed, 99 insertions(+), 90 deletions(-) diff --git a/core/git.go b/core/git.go index f5abbc5c..e3179117 100644 --- a/core/git.go +++ b/core/git.go @@ -1,120 +1,129 @@ package core import ( - "fmt" - "io/ioutil" + "fmt" + "io/ioutil" - "gopkg.in/src-d/go-git.v4" - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/object" - "gopkg.in/src-d/go-git.v4/utils/merkletrie" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/utils/merkletrie" ) const ( - EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" ) func CloneRepository(url *string, branch *string, depth int) (*git.Repository, string, error) { - urlVal := *url - branchVal := *branch - dir, err := ioutil.TempDir("", "gitrob") - if err != nil { - return nil, "", err - } - repository, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: urlVal, - Depth: depth, - ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), - SingleBranch: true, - Tags: git.NoTags, - }) - if err != nil { - return nil, dir, err - } - return repository, dir, nil + urlVal := *url + branchVal := *branch + dir, err := ioutil.TempDir("", "gitrob") + if err != nil { + return nil, "", err + } + repository, err := git.PlainClone(dir, false, &git.CloneOptions{ + URL: urlVal, + Depth: depth, + ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), + SingleBranch: true, + Tags: git.NoTags, + }) + if err != nil { + return nil, dir, err + } + return repository, dir, nil } func GetRepositoryHistory(repository *git.Repository) ([]*object.Commit, error) { - var commits []*object.Commit - ref, err := repository.Head() - if err != nil { - return nil, err - } - cIter, err := repository.Log(&git.LogOptions{From: ref.Hash()}) - if err != nil { - return nil, err - } - cIter.ForEach(func(c *object.Commit) error { - commits = append(commits, c) - return nil - }) - return commits, nil + var commits []*object.Commit + ref, err := repository.Head() + if err != nil { + return nil, err + } + cIter, err := repository.Log(&git.LogOptions{From: ref.Hash()}) + if err != nil { + return nil, err + } + cIter.ForEach(func(c *object.Commit) error { + commits = append(commits, c) + return nil + }) + return commits, nil } func GetChanges(commit *object.Commit, repo *git.Repository) (object.Changes, error) { - parentCommit, err := GetParentCommit(commit, repo) - if err != nil { - return nil, err - } + parentCommit, err := GetParentCommit(commit, repo) + if err != nil { + //this may be the parent commit + parentCommit = commit + //return nil, err + } - commitTree, err := commit.Tree() - if err != nil { - return nil, err - } + commitTree, err := commit.Tree() + if err != nil { + return nil, err + } - parentCommitTree, err := parentCommit.Tree() - if err != nil { - return nil, err - } + parentCommitTree, err := parentCommit.Tree() + if err != nil { + return nil, err + } - changes, err := object.DiffTree(parentCommitTree, commitTree) - if err != nil { - return nil, err - } - return changes, nil + //changes, err := object.DiffTree(parentCommitTree, commitTree) + var changes object.Changes + if parentCommit == commit { + changes, err = object.DiffTree(nil, parentCommitTree) + } else { + changes, err = object.DiffTree(parentCommitTree, commitTree) + } + + if err != nil { + return nil, err + } + return changes, nil } func GetParentCommit(commit *object.Commit, repo *git.Repository) (*object.Commit, error) { - if commit.NumParents() == 0 { - parentCommit, err := repo.CommitObject(plumbing.NewHash(EmptyTreeCommitId)) - if err != nil { - return nil, err - } - return parentCommit, nil - } - parentCommit, err := commit.Parents().Next() - if err != nil { - return nil, err - } - return parentCommit, nil + if commit.NumParents() == 0 { + parentCommit, err := repo.CommitObject(plumbing.NewHash(EmptyTreeCommitId)) + if err != nil { + return nil, err + } + return parentCommit, nil + } + parentCommit, err := commit.Parents().Next() + if err != nil { + return nil, err + } + return parentCommit, nil } func GetChangeAction(change *object.Change) string { - action, err := change.Action() - if err != nil { - return "Unknown" - } - switch action { - case merkletrie.Insert: - return "Insert" - case merkletrie.Modify: - return "Modify" - case merkletrie.Delete: - return "Delete" - default: - return "Unknown" - } + action, err := change.Action() + if err != nil { + return "Unknown" + } + switch action { + case merkletrie.Insert: + return "Insert" + case merkletrie.Modify: + return "Modify" + case merkletrie.Delete: + return "Delete" + default: + return "Unknown" + } } func GetChangePath(change *object.Change) string { - action, err := change.Action() - if err != nil { - return change.To.Name - } + action, err := change.Action() + if err != nil { + return change.To.Name + } - if action == merkletrie.Delete { - return change.From.Name - } else { - return change.To.Name - } + if action == merkletrie.Delete { + return change.From.Name + } else { + return change.To.Name + } } From 885ed725b12099d9ba81d7328be0120b061717ba Mon Sep 17 00:00:00 2001 From: David Epler Date: Mon, 27 Aug 2018 13:34:38 -0400 Subject: [PATCH 3/8] Add `-no-server` option Added option to stop *gitrob* from starting its own web server so it is possible to just get findings. Closes #142 --- core/options.go | 2 ++ core/session.go | 4 +++- main.go | 10 +++++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/core/options.go b/core/options.go index cf610816..a9e90227 100644 --- a/core/options.go +++ b/core/options.go @@ -15,6 +15,7 @@ type Options struct { Port *int Silent *bool Debug *bool + NoServer *bool Logins []string } @@ -30,6 +31,7 @@ func ParseOptions() (Options, error) { Port: flag.Int("port", 9393, "Port to run web server on"), Silent: flag.Bool("silent", false, "Suppress all output except for errors"), Debug: flag.Bool("debug", false, "Print debugging information"), + NoServer: flag.Bool("no-server", false, "Disables web server"), } flag.Parse() diff --git a/core/session.go b/core/session.go index bf58134e..dc80a4f0 100644 --- a/core/session.go +++ b/core/session.go @@ -60,7 +60,9 @@ func (s *Session) Start() { s.InitThreads() s.InitGithubAccessToken() s.InitGithubClient() - s.InitRouter() + if !*s.Options.NoServer { + s.InitRouter() + } } func (s *Session) Finish() { diff --git a/main.go b/main.go index a693ad89..a8a7394e 100644 --- a/main.go +++ b/main.go @@ -218,7 +218,9 @@ func main() { sess.Out.Info("%s\n\n", core.ASCIIBanner) sess.Out.Important("%s v%s started at %s\n", core.Name, core.Version, sess.Stats.StartedAt.Format(time.RFC3339)) sess.Out.Important("Loaded %d signatures\n", len(core.Signatures)) - sess.Out.Important("Web interface available at http://%s:%d\n", *sess.Options.BindAddress, *sess.Options.Port) + if !*sess.Options.NoServer { + sess.Out.Important("Web interface available at http://%s:%d\n", *sess.Options.BindAddress, *sess.Options.Port) + } if sess.Stats.Status == "finished" { sess.Out.Important("Loaded session file: %s\n", *sess.Options.Load) @@ -242,6 +244,8 @@ func main() { } PrintSessionStats(sess) - sess.Out.Important("Press Ctrl+C to stop web server and exit.\n\n") - select {} + if !*sess.Options.NoServer { + sess.Out.Important("Press Ctrl+C to stop web server and exit.\n\n") + select {} + } } From 0f6710ae7be3260cf927adcde0682d15c70a5edd Mon Sep 17 00:00:00 2001 From: David Epler Date: Mon, 27 Aug 2018 14:07:48 -0400 Subject: [PATCH 4/8] update README.md Add option to README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b093ae0d..893f1484 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,8 @@ Gitrob is a tool to help find potentially sensitive files pushed to public repos Load session file -no-expand-orgs Don't add members to targets when processing organizations +-no-server + Disables web server -port int Port to run web server on (default 9393) -save string From 23ce0e9920d915934ea5bb60e61de707434e3b17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Marschollek?= Date: Thu, 6 Sep 2018 17:02:52 +0100 Subject: [PATCH 5/8] =?UTF-8?q?=E2=9C=A8=20Add=20support=20for=20Github=20?= =?UTF-8?q?Enterprise?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds support for GitHub Enterprise which can be configured using the three new command line arguments with the enterprise prefix (see documentation). Closes #145 Closes #148 --- README.md | 14 ++++++++++++++ core/git.go | 18 +++++++++++++----- core/options.go | 8 +++++++- core/session.go | 48 +++++++++++++++++++++++++++++++++++++++++++++++- main.go | 2 +- 5 files changed, 82 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b093ae0d..6ef1d8c7 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,12 @@ Gitrob is a tool to help find potentially sensitive files pushed to public repos Number of repository commits to process (default 500) -debug Print debugging information +-enterprise-upload-url string + Upload URL for Github Enterprise (defaults to the URL set in -enterprise-url if any) +-enterprise-url string + URL for Github Enterprise (/api/v3/ will be appended if not included) +-enterprise-user string + Username for Github Enterprise (defaults to first target) -github-access-token string GitHub access token to use for API requests -load string @@ -54,6 +60,14 @@ A session stored in a file can be loaded with the `-load` option: Gitrob will start its web interface and serve the results for analysis. +### Use with Github Enterprise + +To configure Gitrob for Github Enterprise, the following switches can be used: + +- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github Webinterface can be found. +- `enterprise-upload-url:` Optional, defaults to `enterprise-url`; full path to the upload URL if different from the main Github Enterprise URL. +- `enterprise-user`: Optional, defaults to the first target. + ## Installation A [precompiled version is available](https://github.com/michenriksen/gitrob/releases) for each release, alternatively you can use the latest version of the source code from this repository in order to build your own binary. diff --git a/core/git.go b/core/git.go index f5abbc5c..96c5cb9a 100644 --- a/core/git.go +++ b/core/git.go @@ -7,6 +7,7 @@ import ( "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/plumbing/transport/http" "gopkg.in/src-d/go-git.v4/utils/merkletrie" ) @@ -14,20 +15,27 @@ const ( EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" ) -func CloneRepository(url *string, branch *string, depth int) (*git.Repository, string, error) { +func CloneRepository(url *string, branch *string, sess *Session) (*git.Repository, string, error) { urlVal := *url branchVal := *branch dir, err := ioutil.TempDir("", "gitrob") if err != nil { return nil, "", err } - repository, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: urlVal, - Depth: depth, + + options := &git.CloneOptions{ + URL: urlVal, + Depth: *sess.Options.CommitDepth, ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), SingleBranch: true, Tags: git.NoTags, - }) + } + + if sess.GithubAccessToken != "" && *sess.Options.EnterpriseUser != "" { + options.Auth = &http.BasicAuth{Username: *sess.Options.EnterpriseUser, Password: sess.GithubAccessToken} + } + + repository, err := git.PlainClone(dir, false, options) if err != nil { return nil, dir, err } diff --git a/core/options.go b/core/options.go index cf610816..f48c60ee 100644 --- a/core/options.go +++ b/core/options.go @@ -7,6 +7,9 @@ import ( type Options struct { CommitDepth *int GithubAccessToken *string `json:"-"` + EnterpriseAPI *string + EnterpriseUpload *string + EnterpriseUser *string NoExpandOrgs *bool Threads *int Save *string `json:"-"` @@ -21,7 +24,10 @@ type Options struct { func ParseOptions() (Options, error) { options := Options{ CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), - GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + EnterpriseAPI: flag.String("enterprise-url", "", "Base URL of the GitHub Enterprise API"), + EnterpriseUpload: flag.String("enterprise-upload-url", "", "Upload URL for GitHub Enterprise"), + EnterpriseUser: flag.String("enterprise-user", "", "Username for your GitHub Enterprise account"), NoExpandOrgs: flag.Bool("no-expand-orgs", false, "Don't add members to targets when processing organizations"), Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), Save: flag.String("save", "", "Save session to file"), diff --git a/core/session.go b/core/session.go index bf58134e..d327c764 100644 --- a/core/session.go +++ b/core/session.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "os" "runtime" + "strings" "sync" "time" @@ -59,6 +60,7 @@ func (s *Session) Start() { s.InitLogger() s.InitThreads() s.InitGithubAccessToken() + s.initEnterpriseConfig() s.InitGithubClient() s.InitRouter() } @@ -130,13 +132,57 @@ func (s *Session) InitGithubAccessToken() { } } +func (s *Session) initEnterpriseConfig() { + apiURL := *s.Options.EnterpriseAPI + + if apiURL == "" { + return + } + + if !strings.HasSuffix(apiURL, "/") { + apiURL += "/" + } + + if !strings.HasSuffix(apiURL, "/api/v3/") { + apiURL += "/api/v3/" + } + + *s.Options.EnterpriseAPI = apiURL + + uploadURL := *s.Options.EnterpriseUpload + + if uploadURL == "" { + uploadURL = *s.Options.EnterpriseAPI + } else { + if !strings.HasSuffix(uploadURL, "/") { + uploadURL += "/" + *s.Options.EnterpriseUpload = uploadURL + } + } + + if *s.Options.EnterpriseUser == "" && len(s.Options.Logins) > 0 { + *s.Options.EnterpriseUser = s.Options.Logins[0] + } + } + func (s *Session) InitGithubClient() { ctx := context.Background() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: s.GithubAccessToken}, ) tc := oauth2.NewClient(ctx, ts) - s.GithubClient = github.NewClient(tc) + + if *s.Options.EnterpriseAPI != "" { + enterpriseClient, err := github.NewEnterpriseClient(*s.Options.EnterpriseAPI, *s.Options.EnterpriseUpload, tc) + if err != nil { + s.Out.Fatal("Error creating GitHub Enterprise client: %s\n", err) + } + + s.GithubClient = enterpriseClient + } else { + s.GithubClient = github.NewClient(tc) + } + s.GithubClient.UserAgent = fmt.Sprintf("%s v%s", Name, Version) } diff --git a/main.go b/main.go index a693ad89..06f682cf 100644 --- a/main.go +++ b/main.go @@ -115,7 +115,7 @@ func AnalyzeRepositories(sess *core.Session) { } sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", tid, *repo.FullName) - clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, *sess.Options.CommitDepth) + clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, sess) if err != nil { if err.Error() != "remote repository is empty" { sess.Out.Error("Error cloning repository %s: %s\n", *repo.FullName, err) From 85ad655c8662bce74a01d9991b4b83b3f685fb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Marschollek?= Date: Sat, 20 Oct 2018 12:13:56 +0100 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=90=9B=20Fix=20preview=20of=20files?= =?UTF-8?q?=20on=20Github=20Enterprise?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes download of files when using Github Enterprise. It now uses the existing Github client to download the files instead of making a manual HTTP request. --- README.md | 8 +++---- core/options.go | 7 ++++--- core/router.go | 52 ++++++++++++++++++++-------------------------- core/session.go | 29 ++++++++++++++++---------- core/signatures.go | 8 +++---- main.go | 4 +++- 6 files changed, 55 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 6ef1d8c7..3438314b 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Gitrob is a tool to help find potentially sensitive files pushed to public repos -enterprise-upload-url string Upload URL for Github Enterprise (defaults to the URL set in -enterprise-url if any) -enterprise-url string - URL for Github Enterprise (/api/v3/ will be appended if not included) + URL for Github Enterprise -enterprise-user string Username for Github Enterprise (defaults to first target) -github-access-token string @@ -64,9 +64,9 @@ Gitrob will start its web interface and serve the results for analysis. To configure Gitrob for Github Enterprise, the following switches can be used: -- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github Webinterface can be found. -- `enterprise-upload-url:` Optional, defaults to `enterprise-url`; full path to the upload URL if different from the main Github Enterprise URL. -- `enterprise-user`: Optional, defaults to the first target. +- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github Webinterface can be found. Example: `-enterprise-url=https://github.yourcompany.com` +- `enterprise-upload-url:` Optional, defaults to `enterprise-url`; full path to the upload URL if different from the main Github Enterprise URL. Example: `-enterprise-upload-url=https://github.yourcompany.com/api/v3/upload` +- `enterprise-user`: Optional, defaults to the first target. Example: `-enterprise-user=your.username` ## Installation diff --git a/core/options.go b/core/options.go index f48c60ee..ff820eae 100644 --- a/core/options.go +++ b/core/options.go @@ -7,6 +7,7 @@ import ( type Options struct { CommitDepth *int GithubAccessToken *string `json:"-"` + EnterpriseURL *string EnterpriseAPI *string EnterpriseUpload *string EnterpriseUser *string @@ -24,9 +25,9 @@ type Options struct { func ParseOptions() (Options, error) { options := Options{ CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), - GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), - EnterpriseAPI: flag.String("enterprise-url", "", "Base URL of the GitHub Enterprise API"), - EnterpriseUpload: flag.String("enterprise-upload-url", "", "Upload URL for GitHub Enterprise"), + GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + EnterpriseURL: flag.String("enterprise-url", "", "URL of the GitHub Enterprise instance, e.g. https://github.yourcompany.com"), + EnterpriseUpload: flag.String("enterprise-upload-url", "", "Upload URL for GitHub Enterprise, e.g. https://github.yourcompany.com/api/v3/upload"), EnterpriseUser: flag.String("enterprise-user", "", "Username for your GitHub Enterprise account"), NoExpandOrgs: flag.Bool("no-expand-orgs", false, "Don't add members to targets when processing organizations"), Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), diff --git a/core/router.go b/core/router.go index 227e29bb..56d6fcde 100644 --- a/core/router.go +++ b/core/router.go @@ -1,8 +1,8 @@ package core import ( + "context" "fmt" - "io/ioutil" "net/http" "strings" @@ -10,10 +10,12 @@ import ( "github.com/gin-contrib/secure" "github.com/gin-contrib/static" "github.com/gin-gonic/gin" + "github.com/google/go-github/github" ) const ( - GithubBaseUri = "https://raw.githubusercontent.com" + contextKeyGithubClient = "kGithubClient" + MaximumFileSize = 102400 CspPolicy = "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; font-src 'self'" ReferrerPolicy = "no-referrer" @@ -74,36 +76,26 @@ func NewRouter(s *Session) *gin.Engine { router.GET("/repositories", func(c *gin.Context) { c.JSON(200, s.Repositories) }) - router.GET("/files/:owner/:repo/:commit/*path", fetchFile) + + router.GET("/files/:owner/:repo/:commit/*path", func (c *gin.Context) { + c.Set(contextKeyGithubClient, s.GithubClient) + fetchFile(c) + }) return router } func fetchFile(c *gin.Context) { - fileUrl := fmt.Sprintf("%s/%s/%s/%s%s", GithubBaseUri, c.Param("owner"), c.Param("repo"), c.Param("commit"), c.Param("path")) - resp, err := http.Head(fileUrl) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "message": err, - }) - return + client, _ := c.Get(contextKeyGithubClient) + githubClient := client.(*github.Client) + + ctx := context.Background() + options := &github.RepositoryContentGetOptions{ + Ref: c.Param("commit"), } - if resp.StatusCode == http.StatusNotFound { - c.JSON(http.StatusNotFound, gin.H{ - "message": "No content", - }) - return - } + fileResponse, _, _, err := githubClient.Repositories.GetContents(ctx, c.Param("owner"), c.Param("repo"), c.Param("path"), options) - if resp.ContentLength > MaximumFileSize { - c.JSON(http.StatusUnprocessableEntity, gin.H{ - "message": fmt.Sprintf("File size exceeds maximum of %d bytes", MaximumFileSize), - }) - return - } - - resp, err = http.Get(fileUrl) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{ "message": err, @@ -111,14 +103,14 @@ func fetchFile(c *gin.Context) { return } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - c.JSON(http.StatusInternalServerError, gin.H{ - "message": err, + if fileResponse.GetSize() > MaximumFileSize { + c.JSON(http.StatusUnprocessableEntity, gin.H{ + "message": fmt.Sprintf("File size exceeds maximum of %d bytes", MaximumFileSize), }) return } + + content, _ := fileResponse.GetContent() - c.String(http.StatusOK, string(body[:])) + c.String(http.StatusOK, content) } diff --git a/core/session.go b/core/session.go index d327c764..2076bf01 100644 --- a/core/session.go +++ b/core/session.go @@ -24,6 +24,9 @@ const ( StatusGathering = "gathering" StatusAnalyzing = "analyzing" StatusFinished = "finished" + + githubDotComURL = "https://github.com" + githubAPIPath = "/api/v3/" ) type Stats struct { @@ -133,21 +136,17 @@ func (s *Session) InitGithubAccessToken() { } func (s *Session) initEnterpriseConfig() { - apiURL := *s.Options.EnterpriseAPI + apiURL := *s.Options.EnterpriseURL if apiURL == "" { return } - if !strings.HasSuffix(apiURL, "/") { - apiURL += "/" - } - - if !strings.HasSuffix(apiURL, "/api/v3/") { - apiURL += "/api/v3/" - } + apiURL = strings.TrimSuffix(apiURL, "/") - *s.Options.EnterpriseAPI = apiURL + *s.Options.EnterpriseURL = apiURL + apiPath := apiURL + githubAPIPath + s.Options.EnterpriseAPI = &apiPath uploadURL := *s.Options.EnterpriseUpload @@ -163,8 +162,16 @@ func (s *Session) initEnterpriseConfig() { if *s.Options.EnterpriseUser == "" && len(s.Options.Logins) > 0 { *s.Options.EnterpriseUser = s.Options.Logins[0] } +} + +func (s *Session) GithubURL() string { + if s.Options.EnterpriseURL != nil && *s.Options.EnterpriseURL != "" { + return *s.Options.EnterpriseURL } + return githubDotComURL +} + func (s *Session) InitGithubClient() { ctx := context.Background() ts := oauth2.StaticTokenSource( @@ -172,7 +179,7 @@ func (s *Session) InitGithubClient() { ) tc := oauth2.NewClient(ctx, ts) - if *s.Options.EnterpriseAPI != "" { + if s.Options.EnterpriseAPI != nil && *s.Options.EnterpriseAPI != "" { enterpriseClient, err := github.NewEnterpriseClient(*s.Options.EnterpriseAPI, *s.Options.EnterpriseUpload, tc) if err != nil { s.Out.Fatal("Error creating GitHub Enterprise client: %s\n", err) @@ -180,7 +187,7 @@ func (s *Session) InitGithubClient() { s.GithubClient = enterpriseClient } else { - s.GithubClient = github.NewClient(tc) + s.GithubClient = github.NewClient(tc) } s.GithubClient.UserAgent = fmt.Sprintf("%s v%s", Name, Version) diff --git a/core/signatures.go b/core/signatures.go index fe36cc03..65b91e7d 100644 --- a/core/signatures.go +++ b/core/signatures.go @@ -59,8 +59,8 @@ type Finding struct { RepositoryUrl string } -func (f *Finding) setupUrls() { - f.RepositoryUrl = fmt.Sprintf("https://github.com/%s/%s", f.RepositoryOwner, f.RepositoryName) +func (f *Finding) setupUrls(githubURL string) { + f.RepositoryUrl = strings.Join([]string {githubURL, f.RepositoryOwner, f.RepositoryName}, "/") f.FileUrl = fmt.Sprintf("%s/blob/%s/%s", f.RepositoryUrl, f.CommitHash, f.FilePath) f.CommitUrl = fmt.Sprintf("%s/commit/%s", f.RepositoryUrl, f.CommitHash) } @@ -77,8 +77,8 @@ func (f *Finding) generateID() { f.Id = fmt.Sprintf("%x", h.Sum(nil)) } -func (f *Finding) Initialize() { - f.setupUrls() +func (f *Finding) Initialize(githubURL string) { + f.setupUrls(githubURL) f.generateID() } diff --git a/main.go b/main.go index 06f682cf..b7ea8915 100644 --- a/main.go +++ b/main.go @@ -103,6 +103,8 @@ func AnalyzeRepositories(sess *core.Session) { sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories")) + githubURL := sess.GithubURL() + for i := 0; i < threadNum; i++ { go func(tid int) { for { @@ -163,7 +165,7 @@ func AnalyzeRepositories(sess *core.Session) { CommitMessage: strings.TrimSpace(commit.Message), CommitAuthor: commit.Author.String(), } - finding.Initialize() + finding.Initialize(githubURL) sess.AddFinding(finding) sess.Out.Warn(" %s: %s\n", strings.ToUpper(changeAction), finding.Description) From 24ad583da61708b578b6e4b857d501bf16c91adb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Marschollek?= Date: Sat, 20 Oct 2018 14:49:35 +0100 Subject: [PATCH 7/8] =?UTF-8?q?=F0=9F=93=9D=20Fix=20a=20typo=20in=20the=20?= =?UTF-8?q?README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3438314b..8cae8839 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Gitrob will start its web interface and serve the results for analysis. To configure Gitrob for Github Enterprise, the following switches can be used: -- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github Webinterface can be found. Example: `-enterprise-url=https://github.yourcompany.com` +- `enterprise-url`: Must be specified; this is the URL where the path `/api/v3/` exists. This is usually the URL where the Github web interface can be found. Example: `-enterprise-url=https://github.yourcompany.com` - `enterprise-upload-url:` Optional, defaults to `enterprise-url`; full path to the upload URL if different from the main Github Enterprise URL. Example: `-enterprise-upload-url=https://github.yourcompany.com/api/v3/upload` - `enterprise-user`: Optional, defaults to the first target. Example: `-enterprise-user=your.username` From 7385f86a8212b07c11ffa4d91f5c07bbaa3e670a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Marschollek?= Date: Sat, 20 Oct 2018 14:57:39 +0100 Subject: [PATCH 8/8] =?UTF-8?q?=F0=9F=8E=A8=20Fix=20(some=20of)=20the=20in?= =?UTF-8?q?dentation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/git.go | 2 +- core/options.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/git.go b/core/git.go index 96c5cb9a..635de023 100644 --- a/core/git.go +++ b/core/git.go @@ -24,7 +24,7 @@ func CloneRepository(url *string, branch *string, sess *Session) (*git.Repositor } options := &git.CloneOptions{ - URL: urlVal, + URL: urlVal, Depth: *sess.Options.CommitDepth, ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), SingleBranch: true, diff --git a/core/options.go b/core/options.go index ff820eae..d4ef4cb7 100644 --- a/core/options.go +++ b/core/options.go @@ -25,7 +25,7 @@ type Options struct { func ParseOptions() (Options, error) { options := Options{ CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), - GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), EnterpriseURL: flag.String("enterprise-url", "", "URL of the GitHub Enterprise instance, e.g. https://github.yourcompany.com"), EnterpriseUpload: flag.String("enterprise-upload-url", "", "Upload URL for GitHub Enterprise, e.g. https://github.yourcompany.com/api/v3/upload"), EnterpriseUser: flag.String("enterprise-user", "", "Username for your GitHub Enterprise account"),