From bfad3228ffafbe8550081dc1bdece091884624f8 Mon Sep 17 00:00:00 2001 From: plasticuproject Date: Thu, 28 Feb 2019 13:02:18 -0500 Subject: [PATCH 1/6] Added include-branches feature Added a feature where you can add the option to scan and search all branches of the repositories. Also added micksmix's bug fix where it wasn't searching or logging found signatures in initial commits. --- README.md | 2 + core/git.go | 189 ++++--- core/github.go | 211 ++++---- core/options.go | 56 +- core/session.go | 349 ++++++------ core/signatures.go | 1294 ++++++++++++++++++++++---------------------- main.go | 451 +++++++-------- 7 files changed, 1303 insertions(+), 1249 deletions(-) diff --git a/README.md b/README.md index b093ae0d..e8c70624 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ Gitrob is a tool to help find potentially sensitive files pushed to public repos Print debugging information -github-access-token string GitHub access token to use for API requests +-include-branches + Include repository branches in scan -load string Load session file -no-expand-orgs diff --git a/core/git.go b/core/git.go index f5abbc5c..e3179117 100644 --- a/core/git.go +++ b/core/git.go @@ -1,120 +1,129 @@ package core import ( - "fmt" - "io/ioutil" + "fmt" + "io/ioutil" - "gopkg.in/src-d/go-git.v4" - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/object" - "gopkg.in/src-d/go-git.v4/utils/merkletrie" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/utils/merkletrie" ) const ( - EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" ) func CloneRepository(url *string, branch *string, depth int) (*git.Repository, string, error) { - urlVal := *url - branchVal := *branch - dir, err := ioutil.TempDir("", "gitrob") - if err != nil { - return nil, "", err - } - repository, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: urlVal, - Depth: depth, - ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), - SingleBranch: true, - Tags: git.NoTags, - }) - if err != nil { - return nil, dir, err - } - return repository, dir, nil + urlVal := *url + branchVal := *branch + dir, err := ioutil.TempDir("", "gitrob") + if err != nil { + return nil, "", err + } + repository, err := git.PlainClone(dir, false, &git.CloneOptions{ + URL: urlVal, + Depth: depth, + ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), + SingleBranch: true, + Tags: git.NoTags, + }) + if err != nil { + return nil, dir, err + } + return repository, dir, nil } func GetRepositoryHistory(repository *git.Repository) ([]*object.Commit, error) { - var commits []*object.Commit - ref, err := repository.Head() - if err != nil { - return nil, err - } - cIter, err := repository.Log(&git.LogOptions{From: ref.Hash()}) - if err != nil { - return nil, err - } - cIter.ForEach(func(c *object.Commit) error { - commits = append(commits, c) - return nil - }) - return commits, nil + var commits []*object.Commit + ref, err := repository.Head() + if err != nil { + return nil, err + } + cIter, err := repository.Log(&git.LogOptions{From: ref.Hash()}) + if err != nil { + return nil, err + } + cIter.ForEach(func(c *object.Commit) error { + commits = append(commits, c) + return nil + }) + return commits, nil } func GetChanges(commit *object.Commit, repo *git.Repository) (object.Changes, error) { - parentCommit, err := GetParentCommit(commit, repo) - if err != nil { - return nil, err - } + parentCommit, err := GetParentCommit(commit, repo) + if err != nil { + //this may be the parent commit + parentCommit = commit + //return nil, err + } - commitTree, err := commit.Tree() - if err != nil { - return nil, err - } + commitTree, err := commit.Tree() + if err != nil { + return nil, err + } - parentCommitTree, err := parentCommit.Tree() - if err != nil { - return nil, err - } + parentCommitTree, err := parentCommit.Tree() + if err != nil { + return nil, err + } - changes, err := object.DiffTree(parentCommitTree, commitTree) - if err != nil { - return nil, err - } - return changes, nil + //changes, err := object.DiffTree(parentCommitTree, commitTree) + var changes object.Changes + if parentCommit == commit { + changes, err = object.DiffTree(nil, parentCommitTree) + } else { + changes, err = object.DiffTree(parentCommitTree, commitTree) + } + + if err != nil { + return nil, err + } + return changes, nil } func GetParentCommit(commit *object.Commit, repo *git.Repository) (*object.Commit, error) { - if commit.NumParents() == 0 { - parentCommit, err := repo.CommitObject(plumbing.NewHash(EmptyTreeCommitId)) - if err != nil { - return nil, err - } - return parentCommit, nil - } - parentCommit, err := commit.Parents().Next() - if err != nil { - return nil, err - } - return parentCommit, nil + if commit.NumParents() == 0 { + parentCommit, err := repo.CommitObject(plumbing.NewHash(EmptyTreeCommitId)) + if err != nil { + return nil, err + } + return parentCommit, nil + } + parentCommit, err := commit.Parents().Next() + if err != nil { + return nil, err + } + return parentCommit, nil } func GetChangeAction(change *object.Change) string { - action, err := change.Action() - if err != nil { - return "Unknown" - } - switch action { - case merkletrie.Insert: - return "Insert" - case merkletrie.Modify: - return "Modify" - case merkletrie.Delete: - return "Delete" - default: - return "Unknown" - } + action, err := change.Action() + if err != nil { + return "Unknown" + } + switch action { + case merkletrie.Insert: + return "Insert" + case merkletrie.Modify: + return "Modify" + case merkletrie.Delete: + return "Delete" + default: + return "Unknown" + } } func GetChangePath(change *object.Change) string { - action, err := change.Action() - if err != nil { - return change.To.Name - } + action, err := change.Action() + if err != nil { + return change.To.Name + } - if action == merkletrie.Delete { - return change.From.Name - } else { - return change.To.Name - } + if action == merkletrie.Delete { + return change.From.Name + } else { + return change.To.Name + } } diff --git a/core/github.go b/core/github.go index a1941701..ca5d319b 100644 --- a/core/github.go +++ b/core/github.go @@ -1,113 +1,138 @@ package core import ( - "context" + "context" - "github.com/google/go-github/github" + "github.com/google/go-github/github" ) type GithubOwner struct { - Login *string - ID *int64 - Type *string - Name *string - AvatarURL *string - URL *string - Company *string - Blog *string - Location *string - Email *string - Bio *string + Login *string + ID *int64 + Type *string + Name *string + AvatarURL *string + URL *string + Company *string + Blog *string + Location *string + Email *string + Bio *string } type GithubRepository struct { - Owner *string - ID *int64 - Name *string - FullName *string - CloneURL *string - URL *string - DefaultBranch *string - Description *string - Homepage *string + Owner *string + ID *int64 + Name *string + FullName *string + CloneURL *string + URL *string + DefaultBranch *string + BranchName *string + Description *string + Homepage *string } func GetUserOrOrganization(login string, client *github.Client) (*GithubOwner, error) { - ctx := context.Background() - user, _, err := client.Users.Get(ctx, login) - if err != nil { - return nil, err - } - return &GithubOwner{ - Login: user.Login, - ID: user.ID, - Type: user.Type, - Name: user.Name, - AvatarURL: user.AvatarURL, - URL: user.HTMLURL, - Company: user.Company, - Blog: user.Blog, - Location: user.Location, - Email: user.Email, - Bio: user.Bio, - }, nil + ctx := context.Background() + user, _, err := client.Users.Get(ctx, login) + if err != nil { + return nil, err + } + return &GithubOwner{ + Login: user.Login, + ID: user.ID, + Type: user.Type, + Name: user.Name, + AvatarURL: user.AvatarURL, + URL: user.HTMLURL, + Company: user.Company, + Blog: user.Blog, + Location: user.Location, + Email: user.Email, + Bio: user.Bio, + }, nil } -func GetRepositoriesFromOwner(login *string, client *github.Client) ([]*GithubRepository, error) { - var allRepos []*GithubRepository - loginVal := *login - ctx := context.Background() - opt := &github.RepositoryListOptions{ - Type: "sources", - } +func GetRepositoriesFromOwner(login *string, client *github.Client, includeBranches *bool) ([]*GithubRepository, error) { + var allRepos []*GithubRepository + loginVal := *login + ctx := context.Background() + opt := &github.RepositoryListOptions{ + Type: "sources", + } + opts := &github.ListOptions{} + for { + repos, resp, err := client.Repositories.List(ctx, loginVal, opt) + if err != nil { + return allRepos, err + } + for _, repo := range repos { + if !*repo.Fork { + t := GithubRepository{ + Owner: repo.Owner.Login, + ID: repo.ID, + Name: repo.Name, + FullName: repo.FullName, + CloneURL: repo.CloneURL, + URL: repo.HTMLURL, + DefaultBranch: repo.DefaultBranch, + BranchName: repo.DefaultBranch, + Description: repo.Description, + Homepage: repo.Homepage, + } + branches, resp, err := client.Repositories.ListBranches(ctx, *t.Owner, *t.Name, opts) + if err != nil { + return allRepos, err + } + for _, branch := range branches { + r := GithubRepository{ + Owner: t.Owner, + ID: t.ID, + Name: t.Name, + FullName: t.FullName, + CloneURL: t.CloneURL, + URL: t.URL, + DefaultBranch: t.DefaultBranch, + BranchName: branch.Name, + Description: t.Description, + Homepage: t.Homepage, + } + if *includeBranches { + allRepos = append(allRepos, &r) + } else { + allRepos = append(allRepos, &t) + } + } + opts.Page = resp.NextPage + } + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } - for { - repos, resp, err := client.Repositories.List(ctx, loginVal, opt) - if err != nil { - return allRepos, err - } - for _, repo := range repos { - if !*repo.Fork { - r := GithubRepository{ - Owner: repo.Owner.Login, - ID: repo.ID, - Name: repo.Name, - FullName: repo.FullName, - CloneURL: repo.CloneURL, - URL: repo.HTMLURL, - DefaultBranch: repo.DefaultBranch, - Description: repo.Description, - Homepage: repo.Homepage, - } - allRepos = append(allRepos, &r) - } - } - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } - - return allRepos, nil + return allRepos, nil } func GetOrganizationMembers(login *string, client *github.Client) ([]*GithubOwner, error) { - var allMembers []*GithubOwner - loginVal := *login - ctx := context.Background() - opt := &github.ListMembersOptions{} - for { - members, resp, err := client.Organizations.ListMembers(ctx, loginVal, opt) - if err != nil { - return allMembers, err - } - for _, member := range members { - allMembers = append(allMembers, &GithubOwner{Login: member.Login, ID: member.ID, Type: member.Type}) - } - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } - return allMembers, nil + var allMembers []*GithubOwner + loginVal := *login + ctx := context.Background() + opt := &github.ListMembersOptions{} + for { + members, resp, err := client.Organizations.ListMembers(ctx, loginVal, opt) + if err != nil { + return allMembers, err + } + for _, member := range members { + allMembers = append(allMembers, &GithubOwner{Login: member.Login, ID: member.ID, Type: member.Type}) + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return allMembers, nil } diff --git a/core/options.go b/core/options.go index cf610816..a65fab51 100644 --- a/core/options.go +++ b/core/options.go @@ -1,39 +1,41 @@ package core import ( - "flag" + "flag" ) type Options struct { - CommitDepth *int - GithubAccessToken *string `json:"-"` - NoExpandOrgs *bool - Threads *int - Save *string `json:"-"` - Load *string `json:"-"` - BindAddress *string - Port *int - Silent *bool - Debug *bool - Logins []string + CommitDepth *int + GithubAccessToken *string `json:"-"` + NoExpandOrgs *bool + Threads *int + Save *string `json:"-"` + Load *string `json:"-"` + BindAddress *string + Port *int + Silent *bool + Debug *bool + IncludeBranches *bool + Logins []string } func ParseOptions() (Options, error) { - options := Options{ - CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), - GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), - NoExpandOrgs: flag.Bool("no-expand-orgs", false, "Don't add members to targets when processing organizations"), - Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), - Save: flag.String("save", "", "Save session to file"), - Load: flag.String("load", "", "Load session file"), - BindAddress: flag.String("bind-address", "127.0.0.1", "Address to bind web server to"), - Port: flag.Int("port", 9393, "Port to run web server on"), - Silent: flag.Bool("silent", false, "Suppress all output except for errors"), - Debug: flag.Bool("debug", false, "Print debugging information"), - } + options := Options{ + CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), + GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + NoExpandOrgs: flag.Bool("no-expand-orgs", false, "Don't add members to targets when processing organizations"), + Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), + Save: flag.String("save", "", "Save session to file"), + Load: flag.String("load", "", "Load session file"), + BindAddress: flag.String("bind-address", "127.0.0.1", "Address to bind web server to"), + Port: flag.Int("port", 9393, "Port to run web server on"), + Silent: flag.Bool("silent", false, "Suppress all output except for errors"), + Debug: flag.Bool("debug", false, "Print debugging information"), + IncludeBranches: flag.Bool("include-branches", false, "Include repository branches in scan"), + } - flag.Parse() - options.Logins = flag.Args() + flag.Parse() + options.Logins = flag.Args() - return options, nil + return options, nil } diff --git a/core/session.go b/core/session.go index bf58134e..70dcf659 100644 --- a/core/session.go +++ b/core/session.go @@ -1,242 +1,253 @@ package core import ( - "context" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "os" - "runtime" - "sync" - "time" - - "github.com/gin-gonic/gin" - "github.com/google/go-github/github" - "golang.org/x/oauth2" + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "runtime" + "sync" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/go-github/github" + "golang.org/x/oauth2" ) const ( - AccessTokenEnvVariable = "GITROB_ACCESS_TOKEN" + AccessTokenEnvVariable = "GITROB_ACCESS_TOKEN" - StatusInitializing = "initializing" - StatusGathering = "gathering" - StatusAnalyzing = "analyzing" - StatusFinished = "finished" + StatusInitializing = "initializing" + StatusGathering = "gathering" + StatusAnalyzing = "analyzing" + StatusFinished = "finished" ) type Stats struct { - sync.Mutex + sync.Mutex - StartedAt time.Time - FinishedAt time.Time - Status string - Progress float64 - Targets int - Repositories int - Commits int - Files int - Findings int + StartedAt time.Time + FinishedAt time.Time + Status string + Progress float64 + Targets int + Repositories int + Commits int + Files int + Findings int } type Session struct { - sync.Mutex + sync.Mutex - Version string - Options Options `json:"-"` - Out *Logger `json:"-"` - Stats *Stats - GithubAccessToken string `json:"-"` - GithubClient *github.Client `json:"-"` - Router *gin.Engine `json:"-"` - Targets []*GithubOwner - Repositories []*GithubRepository - Findings []*Finding + Version string + Options Options `json:"-"` + Out *Logger `json:"-"` + Stats *Stats + GithubAccessToken string `json:"-"` + GithubClient *github.Client `json:"-"` + Router *gin.Engine `json:"-"` + Targets []*GithubOwner + Repositories []*GithubRepository + Findings []*Finding } func (s *Session) Start() { - s.InitStats() - s.InitLogger() - s.InitThreads() - s.InitGithubAccessToken() - s.InitGithubClient() - s.InitRouter() + s.InitStats() + s.InitLogger() + s.InitThreads() + s.InitGithubAccessToken() + s.InitGithubClient() + s.InitRouter() } func (s *Session) Finish() { - s.Stats.FinishedAt = time.Now() - s.Stats.Status = StatusFinished + s.Stats.FinishedAt = time.Now() + s.Stats.Status = StatusFinished } func (s *Session) AddTarget(target *GithubOwner) { - s.Lock() - defer s.Unlock() - for _, t := range s.Targets { - if *target.ID == *t.ID { - return - } - } - s.Targets = append(s.Targets, target) + s.Lock() + defer s.Unlock() + for _, t := range s.Targets { + if *target.ID == *t.ID { + return + } + } + s.Targets = append(s.Targets, target) } func (s *Session) AddRepository(repository *GithubRepository) { - s.Lock() - defer s.Unlock() - for _, r := range s.Repositories { - if *repository.ID == *r.ID { - return - } - } - s.Repositories = append(s.Repositories, repository) + s.Lock() + defer s.Unlock() + for _, r := range s.Repositories { + if *repository.ID == *r.ID { + return + } + } + s.Repositories = append(s.Repositories, repository) +} + +func (s *Session) AddBranches(repository *GithubRepository) { + s.Lock() + defer s.Unlock() + for _, r := range s.Repositories { + if *repository.BranchName == *r.BranchName { + return + } + } + s.Repositories = append(s.Repositories, repository) } func (s *Session) AddFinding(finding *Finding) { - s.Lock() - defer s.Unlock() - s.Findings = append(s.Findings, finding) + s.Lock() + defer s.Unlock() + s.Findings = append(s.Findings, finding) } func (s *Session) InitStats() { - if s.Stats != nil { - return - } - s.Stats = &Stats{ - StartedAt: time.Now(), - Status: StatusInitializing, - Progress: 0.0, - Targets: 0, - Repositories: 0, - Commits: 0, - Files: 0, - Findings: 0, - } + if s.Stats != nil { + return + } + s.Stats = &Stats{ + StartedAt: time.Now(), + Status: StatusInitializing, + Progress: 0.0, + Targets: 0, + Repositories: 0, + Commits: 0, + Files: 0, + Findings: 0, + } } func (s *Session) InitLogger() { - s.Out = &Logger{} - s.Out.SetDebug(*s.Options.Debug) - s.Out.SetSilent(*s.Options.Silent) + s.Out = &Logger{} + s.Out.SetDebug(*s.Options.Debug) + s.Out.SetSilent(*s.Options.Silent) } func (s *Session) InitGithubAccessToken() { - if *s.Options.GithubAccessToken == "" { - accessToken := os.Getenv(AccessTokenEnvVariable) - if accessToken == "" { - s.Out.Fatal("No GitHub access token given. Please provide via command line option or in the %s environment variable.\n", AccessTokenEnvVariable) - } - s.GithubAccessToken = accessToken - } else { - s.GithubAccessToken = *s.Options.GithubAccessToken - } + if *s.Options.GithubAccessToken == "" { + accessToken := os.Getenv(AccessTokenEnvVariable) + if accessToken == "" { + s.Out.Fatal("No GitHub access token given. Please provide via command line option or in the %s environment variable.\n", AccessTokenEnvVariable) + } + s.GithubAccessToken = accessToken + } else { + s.GithubAccessToken = *s.Options.GithubAccessToken + } } func (s *Session) InitGithubClient() { - ctx := context.Background() - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: s.GithubAccessToken}, - ) - tc := oauth2.NewClient(ctx, ts) - s.GithubClient = github.NewClient(tc) - s.GithubClient.UserAgent = fmt.Sprintf("%s v%s", Name, Version) + ctx := context.Background() + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: s.GithubAccessToken}, + ) + tc := oauth2.NewClient(ctx, ts) + s.GithubClient = github.NewClient(tc) + s.GithubClient.UserAgent = fmt.Sprintf("%s v%s", Name, Version) } func (s *Session) InitThreads() { - if *s.Options.Threads == 0 { - numCPUs := runtime.NumCPU() - s.Options.Threads = &numCPUs - } - runtime.GOMAXPROCS(*s.Options.Threads + 2) // thread count + main + web server + if *s.Options.Threads == 0 { + numCPUs := runtime.NumCPU() + s.Options.Threads = &numCPUs + } + runtime.GOMAXPROCS(*s.Options.Threads + 2) // thread count + main + web server } func (s *Session) InitRouter() { - bind := fmt.Sprintf("%s:%d", *s.Options.BindAddress, *s.Options.Port) - s.Router = NewRouter(s) - go func(sess *Session) { - if err := sess.Router.Run(bind); err != nil { - sess.Out.Fatal("Error when starting web server: %s\n", err) - } - }(s) + bind := fmt.Sprintf("%s:%d", *s.Options.BindAddress, *s.Options.Port) + s.Router = NewRouter(s) + go func(sess *Session) { + if err := sess.Router.Run(bind); err != nil { + sess.Out.Fatal("Error when starting web server: %s\n", err) + } + }(s) } func (s *Session) SaveToFile(location string) error { - sessionJson, err := json.Marshal(s) - if err != nil { - return err - } - err = ioutil.WriteFile(location, sessionJson, 0644) - if err != nil { - return err - } - return nil + sessionJson, err := json.Marshal(s) + if err != nil { + return err + } + err = ioutil.WriteFile(location, sessionJson, 0644) + if err != nil { + return err + } + return nil } func (s *Stats) IncrementTargets() { - s.Lock() - defer s.Unlock() - s.Targets++ + s.Lock() + defer s.Unlock() + s.Targets++ } func (s *Stats) IncrementRepositories() { - s.Lock() - defer s.Unlock() - s.Repositories++ + s.Lock() + defer s.Unlock() + s.Repositories++ } func (s *Stats) IncrementCommits() { - s.Lock() - defer s.Unlock() - s.Commits++ + s.Lock() + defer s.Unlock() + s.Commits++ } func (s *Stats) IncrementFiles() { - s.Lock() - defer s.Unlock() - s.Files++ + s.Lock() + defer s.Unlock() + s.Files++ } func (s *Stats) IncrementFindings() { - s.Lock() - defer s.Unlock() - s.Findings++ + s.Lock() + defer s.Unlock() + s.Findings++ } func (s *Stats) UpdateProgress(current int, total int) { - s.Lock() - defer s.Unlock() - if current >= total { - s.Progress = 100.0 - } else { - s.Progress = (float64(current) * float64(100)) / float64(total) - } + s.Lock() + defer s.Unlock() + if current >= total { + s.Progress = 100.0 + } else { + s.Progress = (float64(current) * float64(100)) / float64(total) + } } func NewSession() (*Session, error) { - var err error - var session Session - - if session.Options, err = ParseOptions(); err != nil { - return nil, err - } - - if *session.Options.Save != "" && FileExists(*session.Options.Save) { - return nil, errors.New(fmt.Sprintf("File: %s already exists.", *session.Options.Save)) - } - - if *session.Options.Load != "" { - if !FileExists(*session.Options.Load) { - return nil, errors.New(fmt.Sprintf("Session file %s does not exist or is not readable.", *session.Options.Load)) - } - data, err := ioutil.ReadFile(*session.Options.Load) - if err != nil { - return nil, err - } - if err := json.Unmarshal(data, &session); err != nil { - return nil, errors.New(fmt.Sprintf("Session file %s is corrupt or generated by an old version of Gitrob.", *session.Options.Load)) - } - } - - session.Version = Version - session.Start() - - return &session, nil + var err error + var session Session + + if session.Options, err = ParseOptions(); err != nil { + return nil, err + } + + if *session.Options.Save != "" && FileExists(*session.Options.Save) { + return nil, errors.New(fmt.Sprintf("File: %s already exists.", *session.Options.Save)) + } + + if *session.Options.Load != "" { + if !FileExists(*session.Options.Load) { + return nil, errors.New(fmt.Sprintf("Session file %s does not exist or is not readable.", *session.Options.Load)) + } + data, err := ioutil.ReadFile(*session.Options.Load) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &session); err != nil { + return nil, errors.New(fmt.Sprintf("Session file %s is corrupt or generated by an old version of Gitrob.", *session.Options.Load)) + } + } + + session.Version = Version + session.Start() + + return &session, nil } diff --git a/core/signatures.go b/core/signatures.go index fe36cc03..508f4932 100644 --- a/core/signatures.go +++ b/core/signatures.go @@ -1,710 +1,712 @@ package core import ( - "crypto/sha1" - "fmt" - "io" - "path/filepath" - "regexp" - "strings" + "crypto/sha1" + "fmt" + "io" + "path/filepath" + "regexp" + "strings" ) const ( - TypeSimple = "simple" - TypePattern = "pattern" + TypeSimple = "simple" + TypePattern = "pattern" - PartExtension = "extension" - PartFilename = "filename" - PartPath = "path" + PartExtension = "extension" + PartFilename = "filename" + PartPath = "path" ) var skippableExtensions = []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf"} var skippablePathIndicators = []string{"node_modules/", "vendor/bundle", "vendor/cache"} type MatchFile struct { - Path string - Filename string - Extension string + Path string + Filename string + Extension string } func (f *MatchFile) IsSkippable() bool { - ext := strings.ToLower(f.Extension) - path := strings.ToLower(f.Path) - for _, skippableExt := range skippableExtensions { - if ext == skippableExt { - return true - } - } - for _, skippablePathIndicator := range skippablePathIndicators { - if strings.Contains(path, skippablePathIndicator) { - return true - } - } - return false + ext := strings.ToLower(f.Extension) + path := strings.ToLower(f.Path) + for _, skippableExt := range skippableExtensions { + if ext == skippableExt { + return true + } + } + for _, skippablePathIndicator := range skippablePathIndicators { + if strings.Contains(path, skippablePathIndicator) { + return true + } + } + return false } type Finding struct { - Id string - FilePath string - Action string - Description string - Comment string - RepositoryOwner string - RepositoryName string - CommitHash string - CommitMessage string - CommitAuthor string - FileUrl string - CommitUrl string - RepositoryUrl string + Id string + FilePath string + Action string + Description string + Comment string + RepositoryOwner string + RepositoryName string + BranchName string + CommitHash string + CommitMessage string + CommitAuthor string + FileUrl string + CommitUrl string + RepositoryUrl string } func (f *Finding) setupUrls() { - f.RepositoryUrl = fmt.Sprintf("https://github.com/%s/%s", f.RepositoryOwner, f.RepositoryName) - f.FileUrl = fmt.Sprintf("%s/blob/%s/%s", f.RepositoryUrl, f.CommitHash, f.FilePath) - f.CommitUrl = fmt.Sprintf("%s/commit/%s", f.RepositoryUrl, f.CommitHash) + f.RepositoryUrl = fmt.Sprintf("https://github.com/%s/%s", f.RepositoryOwner, f.RepositoryName) + f.FileUrl = fmt.Sprintf("%s/blob/%s/%s", f.RepositoryUrl, f.CommitHash, f.FilePath) + f.CommitUrl = fmt.Sprintf("%s/commit/%s", f.RepositoryUrl, f.CommitHash) } func (f *Finding) generateID() { - h := sha1.New() - io.WriteString(h, f.FilePath) - io.WriteString(h, f.Action) - io.WriteString(h, f.RepositoryOwner) - io.WriteString(h, f.RepositoryName) - io.WriteString(h, f.CommitHash) - io.WriteString(h, f.CommitMessage) - io.WriteString(h, f.CommitAuthor) - f.Id = fmt.Sprintf("%x", h.Sum(nil)) + h := sha1.New() + io.WriteString(h, f.FilePath) + io.WriteString(h, f.Action) + io.WriteString(h, f.RepositoryOwner) + io.WriteString(h, f.RepositoryName) + io.WriteString(h, f.BranchName) + io.WriteString(h, f.CommitHash) + io.WriteString(h, f.CommitMessage) + io.WriteString(h, f.CommitAuthor) + f.Id = fmt.Sprintf("%x", h.Sum(nil)) } func (f *Finding) Initialize() { - f.setupUrls() - f.generateID() + f.setupUrls() + f.generateID() } type Signature interface { - Match(file MatchFile) bool - Description() string - Comment() string + Match(file MatchFile) bool + Description() string + Comment() string } type SimpleSignature struct { - part string - match string - description string - comment string + part string + match string + description string + comment string } type PatternSignature struct { - part string - match *regexp.Regexp - description string - comment string + part string + match *regexp.Regexp + description string + comment string } func (s SimpleSignature) Match(file MatchFile) bool { - var haystack *string - switch s.part { - case PartPath: - haystack = &file.Path - case PartFilename: - haystack = &file.Filename - case PartExtension: - haystack = &file.Extension - default: - return false - } + var haystack *string + switch s.part { + case PartPath: + haystack = &file.Path + case PartFilename: + haystack = &file.Filename + case PartExtension: + haystack = &file.Extension + default: + return false + } - return (s.match == *haystack) + return (s.match == *haystack) } func (s SimpleSignature) Description() string { - return s.description + return s.description } func (s SimpleSignature) Comment() string { - return s.comment + return s.comment } func (s PatternSignature) Match(file MatchFile) bool { - var haystack *string - switch s.part { - case PartPath: - haystack = &file.Path - case PartFilename: - haystack = &file.Filename - case PartExtension: - haystack = &file.Extension - default: - return false - } + var haystack *string + switch s.part { + case PartPath: + haystack = &file.Path + case PartFilename: + haystack = &file.Filename + case PartExtension: + haystack = &file.Extension + default: + return false + } - return s.match.MatchString(*haystack) + return s.match.MatchString(*haystack) } func (s PatternSignature) Description() string { - return s.description + return s.description } func (s PatternSignature) Comment() string { - return s.comment + return s.comment } func NewMatchFile(path string) MatchFile { - _, filename := filepath.Split(path) - extension := filepath.Ext(path) - return MatchFile{ - Path: path, - Filename: filename, - Extension: extension, - } + _, filename := filepath.Split(path) + extension := filepath.Ext(path) + return MatchFile{ + Path: path, + Filename: filename, + Extension: extension, + } } var Signatures = []Signature{ - SimpleSignature{ - part: PartExtension, - match: ".pem", - description: "Potential cryptographic private key", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".log", - description: "Log file", - comment: "Log files can contain secret HTTP endpoints, session IDs, API keys and other goodies", - }, - SimpleSignature{ - part: PartExtension, - match: ".pkcs12", - description: "Potential cryptographic key bundle", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".p12", - description: "Potential cryptographic key bundle", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".pfx", - description: "Potential cryptographic key bundle", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".asc", - description: "Potential cryptographic key bundle", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "otr.private_key", - description: "Pidgin OTR private key", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".ovpn", - description: "OpenVPN client configuration file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".cscfg", - description: "Azure service configuration schema file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".rdp", - description: "Remote Desktop connection file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".mdf", - description: "Microsoft SQL database file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".sdf", - description: "Microsoft SQL server compact database file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".sqlite", - description: "SQLite database file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".bek", - description: "Microsoft BitLocker recovery key file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".tpm", - description: "Microsoft BitLocker Trusted Platform Module password file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".fve", - description: "Windows BitLocker full volume encrypted data file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".jks", - description: "Java keystore file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".psafe3", - description: "Password Safe database file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "secret_token.rb", - description: "Ruby On Rails secret token configuration file", - comment: "If the Rails secret token is known, it can allow for remote code execution (http://www.exploit-db.com/exploits/27527/)", - }, - SimpleSignature{ - part: PartFilename, - match: "carrierwave.rb", - description: "Carrierwave configuration file", - comment: "Can contain credentials for cloud storage systems such as Amazon S3 and Google Storage", - }, - SimpleSignature{ - part: PartFilename, - match: "database.yml", - description: "Potential Ruby On Rails database configuration file", - comment: "Can contain database credentials", - }, - SimpleSignature{ - part: PartFilename, - match: "omniauth.rb", - description: "OmniAuth configuration file", - comment: "The OmniAuth configuration file can contain client application secrets", - }, - SimpleSignature{ - part: PartFilename, - match: "settings.py", - description: "Django configuration file", - comment: "Can contain database credentials, cloud storage system credentials, and other secrets", - }, - SimpleSignature{ - part: PartExtension, - match: ".agilekeychain", - description: "1Password password manager database file", - comment: "Feed it to Hashcat and see if you're lucky", - }, - SimpleSignature{ - part: PartExtension, - match: ".keychain", - description: "Apple Keychain database file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".pcap", - description: "Network traffic capture file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".gnucash", - description: "GnuCash database file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml", - description: "Jenkins publish over SSH plugin file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "credentials.xml", - description: "Potential Jenkins credentials file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".kwallet", - description: "KDE Wallet Manager database file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "LocalSettings.php", - description: "Potential MediaWiki configuration file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".tblk", - description: "Tunnelblick VPN configuration file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "Favorites.plist", - description: "Sequel Pro MySQL database manager bookmark file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "configuration.user.xpl", - description: "Little Snitch firewall configuration file", - comment: "Contains traffic rules for applications", - }, - SimpleSignature{ - part: PartExtension, - match: ".dayone", - description: "Day One journal file", - comment: "Now it's getting creepy...", - }, - SimpleSignature{ - part: PartFilename, - match: "journal.txt", - description: "Potential jrnl journal file", - comment: "Now it's getting creepy...", - }, - SimpleSignature{ - part: PartFilename, - match: "knife.rb", - description: "Chef Knife configuration file", - comment: "Can contain references to Chef servers", - }, - SimpleSignature{ - part: PartFilename, - match: "proftpdpasswd", - description: "cPanel backup ProFTPd credentials file", - comment: "Contains usernames and password hashes for FTP accounts", - }, - SimpleSignature{ - part: PartFilename, - match: "robomongo.json", - description: "Robomongo MongoDB manager configuration file", - comment: "Can contain credentials for MongoDB databases", - }, - SimpleSignature{ - part: PartFilename, - match: "filezilla.xml", - description: "FileZilla FTP configuration file", - comment: "Can contain credentials for FTP servers", - }, - SimpleSignature{ - part: PartFilename, - match: "recentservers.xml", - description: "FileZilla FTP recent servers file", - comment: "Can contain credentials for FTP servers", - }, - SimpleSignature{ - part: PartFilename, - match: "ventrilo_srv.ini", - description: "Ventrilo server configuration file", - comment: "Can contain passwords", - }, - SimpleSignature{ - part: PartFilename, - match: "terraform.tfvars", - description: "Terraform variable config file", - comment: "Can contain credentials for terraform providers", - }, - SimpleSignature{ - part: PartFilename, - match: ".exports", - description: "Shell configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - SimpleSignature{ - part: PartFilename, - match: ".functions", - description: "Shell configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - SimpleSignature{ - part: PartFilename, - match: ".extra", - description: "Shell configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^.*_rsa$`), - description: "Private SSH key", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^.*_dsa$`), - description: "Private SSH key", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^.*_ed25519$`), - description: "Private SSH key", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^.*_ecdsa$`), - description: "Private SSH key", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?ssh/config$`), - description: "SSH configuration file", - comment: "", - }, - PatternSignature{ - part: PartExtension, - match: regexp.MustCompile(`^key(pair)?$`), - description: "Potential cryptographic private key", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?(bash_|zsh_|sh_|z)?history$`), - description: "Shell command history file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?mysql_history$`), - description: "MySQL client command history file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?psql_history$`), - description: "PostgreSQL client command history file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?pgpass$`), - description: "PostgreSQL password file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?irb_history$`), - description: "Ruby IRB console history file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?purple/accounts\.xml$`), - description: "Pidgin chat client account configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?xchat2?/servlist_?\.conf$`), - description: "Hexchat/XChat IRC client server list configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?irssi/config$`), - description: "Irssi IRC client configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?recon-ng/keys\.db$`), - description: "Recon-ng web reconnaissance framework API key database", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?dbeaver-data-sources.xml$`), - description: "DBeaver SQL database manager configuration file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?muttrc$`), - description: "Mutt e-mail client configuration file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?s3cfg$`), - description: "S3cmd configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?aws/credentials$`), - description: "AWS CLI credentials file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^sftp-config(\.json)?$`), - description: "SFTP connection configuration file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?trc$`), - description: "T command-line Twitter client configuration file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?gitrobrc$`), - description: "Well, this is awkward... Gitrob configuration file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?(bash|zsh|csh)rc$`), - description: "Shell configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?(bash_|zsh_)?profile$`), - description: "Shell profile configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?(bash_|zsh_)?aliases$`), - description: "Shell command alias configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`config(\.inc)?\.php$`), - description: "PHP configuration file", - comment: "", - }, - PatternSignature{ - part: PartExtension, - match: regexp.MustCompile(`^key(store|ring)$`), - description: "GNOME Keyring database file", - comment: "", - }, - PatternSignature{ - part: PartExtension, - match: regexp.MustCompile(`^kdbx?$`), - description: "KeePass password manager database file", - comment: "Feed it to Hashcat and see if you're lucky", - }, - PatternSignature{ - part: PartExtension, - match: regexp.MustCompile(`^sql(dump)?$`), - description: "SQL dump file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?htpasswd$`), - description: "Apache htpasswd file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^(\.|_)?netrc$`), - description: "Configuration file for auto-login process", - comment: "Can contain username and password", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?gem/credentials$`), - description: "Rubygems credentials file", - comment: "Can contain API key for a rubygems.org account", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?tugboat$`), - description: "Tugboat DigitalOcean management tool configuration", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`doctl/config.yaml$`), - description: "DigitalOcean doctl command-line client configuration file", - comment: "Contains DigitalOcean API key and other information", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?git-credentials$`), - description: "git-credential-store helper credentials file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`config/hub$`), - description: "GitHub Hub command-line client configuration file", - comment: "Can contain GitHub API access token", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?gitconfig$`), - description: "Git configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?chef/(.*)\.pem$`), - description: "Chef private key", - comment: "Can be used to authenticate against Chef servers", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`etc/shadow$`), - description: "Potential Linux shadow file", - comment: "Contains hashed passwords for system users", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`etc/passwd$`), - description: "Potential Linux passwd file", - comment: "Contains system user information", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?dockercfg$`), - description: "Docker configuration file", - comment: "Can contain credentials for public or private Docker registries", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?npmrc$`), - description: "NPM configuration file", - comment: "Can contain credentials for NPM registries", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?env$`), - description: "Environment configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`credential`), - description: "Contains word: credential", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`password`), - description: "Contains word: password", - comment: "", - }, + SimpleSignature{ + part: PartExtension, + match: ".pem", + description: "Potential cryptographic private key", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".log", + description: "Log file", + comment: "Log files can contain secret HTTP endpoints, session IDs, API keys and other goodies", + }, + SimpleSignature{ + part: PartExtension, + match: ".pkcs12", + description: "Potential cryptographic key bundle", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".p12", + description: "Potential cryptographic key bundle", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".pfx", + description: "Potential cryptographic key bundle", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".asc", + description: "Potential cryptographic key bundle", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "otr.private_key", + description: "Pidgin OTR private key", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".ovpn", + description: "OpenVPN client configuration file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".cscfg", + description: "Azure service configuration schema file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".rdp", + description: "Remote Desktop connection file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".mdf", + description: "Microsoft SQL database file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".sdf", + description: "Microsoft SQL server compact database file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".sqlite", + description: "SQLite database file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".bek", + description: "Microsoft BitLocker recovery key file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".tpm", + description: "Microsoft BitLocker Trusted Platform Module password file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".fve", + description: "Windows BitLocker full volume encrypted data file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".jks", + description: "Java keystore file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".psafe3", + description: "Password Safe database file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "secret_token.rb", + description: "Ruby On Rails secret token configuration file", + comment: "If the Rails secret token is known, it can allow for remote code execution (http://www.exploit-db.com/exploits/27527/)", + }, + SimpleSignature{ + part: PartFilename, + match: "carrierwave.rb", + description: "Carrierwave configuration file", + comment: "Can contain credentials for cloud storage systems such as Amazon S3 and Google Storage", + }, + SimpleSignature{ + part: PartFilename, + match: "database.yml", + description: "Potential Ruby On Rails database configuration file", + comment: "Can contain database credentials", + }, + SimpleSignature{ + part: PartFilename, + match: "omniauth.rb", + description: "OmniAuth configuration file", + comment: "The OmniAuth configuration file can contain client application secrets", + }, + SimpleSignature{ + part: PartFilename, + match: "settings.py", + description: "Django configuration file", + comment: "Can contain database credentials, cloud storage system credentials, and other secrets", + }, + SimpleSignature{ + part: PartExtension, + match: ".agilekeychain", + description: "1Password password manager database file", + comment: "Feed it to Hashcat and see if you're lucky", + }, + SimpleSignature{ + part: PartExtension, + match: ".keychain", + description: "Apple Keychain database file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".pcap", + description: "Network traffic capture file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".gnucash", + description: "GnuCash database file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml", + description: "Jenkins publish over SSH plugin file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "credentials.xml", + description: "Potential Jenkins credentials file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".kwallet", + description: "KDE Wallet Manager database file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "LocalSettings.php", + description: "Potential MediaWiki configuration file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".tblk", + description: "Tunnelblick VPN configuration file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "Favorites.plist", + description: "Sequel Pro MySQL database manager bookmark file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "configuration.user.xpl", + description: "Little Snitch firewall configuration file", + comment: "Contains traffic rules for applications", + }, + SimpleSignature{ + part: PartExtension, + match: ".dayone", + description: "Day One journal file", + comment: "Now it's getting creepy...", + }, + SimpleSignature{ + part: PartFilename, + match: "journal.txt", + description: "Potential jrnl journal file", + comment: "Now it's getting creepy...", + }, + SimpleSignature{ + part: PartFilename, + match: "knife.rb", + description: "Chef Knife configuration file", + comment: "Can contain references to Chef servers", + }, + SimpleSignature{ + part: PartFilename, + match: "proftpdpasswd", + description: "cPanel backup ProFTPd credentials file", + comment: "Contains usernames and password hashes for FTP accounts", + }, + SimpleSignature{ + part: PartFilename, + match: "robomongo.json", + description: "Robomongo MongoDB manager configuration file", + comment: "Can contain credentials for MongoDB databases", + }, + SimpleSignature{ + part: PartFilename, + match: "filezilla.xml", + description: "FileZilla FTP configuration file", + comment: "Can contain credentials for FTP servers", + }, + SimpleSignature{ + part: PartFilename, + match: "recentservers.xml", + description: "FileZilla FTP recent servers file", + comment: "Can contain credentials for FTP servers", + }, + SimpleSignature{ + part: PartFilename, + match: "ventrilo_srv.ini", + description: "Ventrilo server configuration file", + comment: "Can contain passwords", + }, + SimpleSignature{ + part: PartFilename, + match: "terraform.tfvars", + description: "Terraform variable config file", + comment: "Can contain credentials for terraform providers", + }, + SimpleSignature{ + part: PartFilename, + match: ".exports", + description: "Shell configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + SimpleSignature{ + part: PartFilename, + match: ".functions", + description: "Shell configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + SimpleSignature{ + part: PartFilename, + match: ".extra", + description: "Shell configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^.*_rsa$`), + description: "Private SSH key", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^.*_dsa$`), + description: "Private SSH key", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^.*_ed25519$`), + description: "Private SSH key", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^.*_ecdsa$`), + description: "Private SSH key", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?ssh/config$`), + description: "SSH configuration file", + comment: "", + }, + PatternSignature{ + part: PartExtension, + match: regexp.MustCompile(`^key(pair)?$`), + description: "Potential cryptographic private key", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?(bash_|zsh_|sh_|z)?history$`), + description: "Shell command history file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?mysql_history$`), + description: "MySQL client command history file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?psql_history$`), + description: "PostgreSQL client command history file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?pgpass$`), + description: "PostgreSQL password file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?irb_history$`), + description: "Ruby IRB console history file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?purple/accounts\.xml$`), + description: "Pidgin chat client account configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?xchat2?/servlist_?\.conf$`), + description: "Hexchat/XChat IRC client server list configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?irssi/config$`), + description: "Irssi IRC client configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?recon-ng/keys\.db$`), + description: "Recon-ng web reconnaissance framework API key database", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?dbeaver-data-sources.xml$`), + description: "DBeaver SQL database manager configuration file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?muttrc$`), + description: "Mutt e-mail client configuration file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?s3cfg$`), + description: "S3cmd configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?aws/credentials$`), + description: "AWS CLI credentials file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^sftp-config(\.json)?$`), + description: "SFTP connection configuration file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?trc$`), + description: "T command-line Twitter client configuration file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?gitrobrc$`), + description: "Well, this is awkward... Gitrob configuration file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?(bash|zsh|csh)rc$`), + description: "Shell configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?(bash_|zsh_)?profile$`), + description: "Shell profile configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?(bash_|zsh_)?aliases$`), + description: "Shell command alias configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`config(\.inc)?\.php$`), + description: "PHP configuration file", + comment: "", + }, + PatternSignature{ + part: PartExtension, + match: regexp.MustCompile(`^key(store|ring)$`), + description: "GNOME Keyring database file", + comment: "", + }, + PatternSignature{ + part: PartExtension, + match: regexp.MustCompile(`^kdbx?$`), + description: "KeePass password manager database file", + comment: "Feed it to Hashcat and see if you're lucky", + }, + PatternSignature{ + part: PartExtension, + match: regexp.MustCompile(`^sql(dump)?$`), + description: "SQL dump file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?htpasswd$`), + description: "Apache htpasswd file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^(\.|_)?netrc$`), + description: "Configuration file for auto-login process", + comment: "Can contain username and password", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?gem/credentials$`), + description: "Rubygems credentials file", + comment: "Can contain API key for a rubygems.org account", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?tugboat$`), + description: "Tugboat DigitalOcean management tool configuration", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`doctl/config.yaml$`), + description: "DigitalOcean doctl command-line client configuration file", + comment: "Contains DigitalOcean API key and other information", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?git-credentials$`), + description: "git-credential-store helper credentials file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`config/hub$`), + description: "GitHub Hub command-line client configuration file", + comment: "Can contain GitHub API access token", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?gitconfig$`), + description: "Git configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?chef/(.*)\.pem$`), + description: "Chef private key", + comment: "Can be used to authenticate against Chef servers", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`etc/shadow$`), + description: "Potential Linux shadow file", + comment: "Contains hashed passwords for system users", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`etc/passwd$`), + description: "Potential Linux passwd file", + comment: "Contains system user information", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?dockercfg$`), + description: "Docker configuration file", + comment: "Can contain credentials for public or private Docker registries", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?npmrc$`), + description: "NPM configuration file", + comment: "Can contain credentials for NPM registries", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?env$`), + description: "Environment configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`credential`), + description: "Contains word: credential", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`password`), + description: "Contains word: password", + comment: "", + }, } diff --git a/main.go b/main.go index a693ad89..d76bd5e6 100644 --- a/main.go +++ b/main.go @@ -1,247 +1,250 @@ package main import ( - "fmt" - "os" - "strings" - "sync" - "time" + "fmt" + "os" + "strings" + "sync" + "time" - "github.com/michenriksen/gitrob/core" + "github.com/michenriksen/gitrob/core" ) var ( - sess *core.Session - err error + sess *core.Session + err error ) func GatherTargets(sess *core.Session) { - sess.Stats.Status = core.StatusGathering - sess.Out.Important("Gathering targets...\n") - for _, login := range sess.Options.Logins { - target, err := core.GetUserOrOrganization(login, sess.GithubClient) - if err != nil { - sess.Out.Error(" Error retrieving information on %s: %s\n", login, err) - continue - } - sess.Out.Debug("%s (ID: %d) type: %s\n", *target.Login, *target.ID, *target.Type) - sess.AddTarget(target) - if *sess.Options.NoExpandOrgs == false && *target.Type == "Organization" { - sess.Out.Debug("Gathering members of %s (ID: %d)...\n", *target.Login, *target.ID) - members, err := core.GetOrganizationMembers(target.Login, sess.GithubClient) - if err != nil { - sess.Out.Error(" Error retrieving members of %s: %s\n", *target.Login, err) - continue - } - for _, member := range members { - sess.Out.Debug("Adding organization member %s (ID: %d) to targets\n", *member.Login, *member.ID) - sess.AddTarget(member) - } - } - } + sess.Stats.Status = core.StatusGathering + sess.Out.Important("Gathering targets...\n") + for _, login := range sess.Options.Logins { + target, err := core.GetUserOrOrganization(login, sess.GithubClient) + if err != nil { + sess.Out.Error(" Error retrieving information on %s: %s\n", login, err) + continue + } + sess.Out.Debug("%s (ID: %d) type: %s\n", *target.Login, *target.ID, *target.Type) + sess.AddTarget(target) + if *sess.Options.NoExpandOrgs == false && *target.Type == "Organization" { + sess.Out.Debug("Gathering members of %s (ID: %d)...\n", *target.Login, *target.ID) + members, err := core.GetOrganizationMembers(target.Login, sess.GithubClient) + if err != nil { + sess.Out.Error(" Error retrieving members of %s: %s\n", *target.Login, err) + continue + } + for _, member := range members { + sess.Out.Debug("Adding organization member %s (ID: %d) to targets\n", *member.Login, *member.ID) + sess.AddTarget(member) + } + } + } } func GatherRepositories(sess *core.Session) { - var ch = make(chan *core.GithubOwner, len(sess.Targets)) - var wg sync.WaitGroup - var threadNum int - if len(sess.Targets) == 1 { - threadNum = 1 - } else if len(sess.Targets) <= *sess.Options.Threads { - threadNum = len(sess.Targets) - 1 - } else { - threadNum = *sess.Options.Threads - } - wg.Add(threadNum) - sess.Out.Debug("Threads for repository gathering: %d\n", threadNum) - for i := 0; i < threadNum; i++ { - go func() { - for { - target, ok := <-ch - if !ok { - wg.Done() - return - } - repos, err := core.GetRepositoriesFromOwner(target.Login, sess.GithubClient) - if err != nil { - sess.Out.Error(" Failed to retrieve repositories from %s: %s\n", *target.Login, err) - } - if len(repos) == 0 { - continue - } - for _, repo := range repos { - sess.Out.Debug(" Retrieved repository: %s\n", *repo.FullName) - sess.AddRepository(repo) - } - sess.Stats.IncrementTargets() - sess.Out.Info(" Retrieved %d %s from %s\n", len(repos), core.Pluralize(len(repos), "repository", "repositories"), *target.Login) - } - }() - } - - for _, target := range sess.Targets { - ch <- target - } - close(ch) - wg.Wait() + var ch = make(chan *core.GithubOwner, len(sess.Targets)) + var wg sync.WaitGroup + var threadNum int + if len(sess.Targets) == 1 { + threadNum = 1 + } else if len(sess.Targets) <= *sess.Options.Threads { + threadNum = len(sess.Targets) - 1 + } else { + threadNum = *sess.Options.Threads + } + wg.Add(threadNum) + sess.Out.Debug("Threads for repository gathering: %d\n", threadNum) + for i := 0; i < threadNum; i++ { + go func() { + for { + target, ok := <-ch + if !ok { + wg.Done() + return + } + repos, err := core.GetRepositoriesFromOwner(target.Login, sess.GithubClient, sess.Options.IncludeBranches) + if err != nil { + sess.Out.Error(" Failed to retrieve repositories from %s: %s\n", *target.Login, err) + } + if len(repos) == 0 { + continue + } + for _, repo := range repos { + sess.Out.Debug(" Retrieved repository: %s\n", *repo.FullName) + sess.AddRepository(repo) + sess.AddBranches(repo) + } + sess.Stats.IncrementTargets() + sess.Out.Info(" Retrieved %d %s from %s\n", len(repos), core.Pluralize(len(repos), "repository", "repositories"), *target.Login) + } + }() + } + + for _, target := range sess.Targets { + ch <- target + } + close(ch) + wg.Wait() } func AnalyzeRepositories(sess *core.Session) { - sess.Stats.Status = core.StatusAnalyzing - var ch = make(chan *core.GithubRepository, len(sess.Repositories)) - var wg sync.WaitGroup - var threadNum int - if len(sess.Repositories) <= 1 { - threadNum = 1 - } else if len(sess.Repositories) <= *sess.Options.Threads { - threadNum = len(sess.Repositories) - 1 - } else { - threadNum = *sess.Options.Threads - } - wg.Add(threadNum) - sess.Out.Debug("Threads for repository analysis: %d\n", threadNum) - - sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories")) - - for i := 0; i < threadNum; i++ { - go func(tid int) { - for { - sess.Out.Debug("[THREAD #%d] Requesting new repository to analyze...\n", tid) - repo, ok := <-ch - if !ok { - sess.Out.Debug("[THREAD #%d] No more tasks, marking WaitGroup as done\n", tid) - wg.Done() - return - } - - sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", tid, *repo.FullName) - clone, path, err := core.CloneRepository(repo.CloneURL, repo.DefaultBranch, *sess.Options.CommitDepth) - if err != nil { - if err.Error() != "remote repository is empty" { - sess.Out.Error("Error cloning repository %s: %s\n", *repo.FullName, err) - } - sess.Stats.IncrementRepositories() - sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) - continue - } - sess.Out.Debug("[THREAD #%d][%s] Cloned repository to: %s\n", tid, *repo.FullName, path) - - history, err := core.GetRepositoryHistory(clone) - if err != nil { - sess.Out.Error("[THREAD #%d][%s] Error getting commit history: %s\n", tid, *repo.FullName, err) - os.RemoveAll(path) - sess.Stats.IncrementRepositories() - sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) - continue - } - sess.Out.Debug("[THREAD #%d][%s] Number of commits: %d\n", tid, *repo.FullName, len(history)) - - for _, commit := range history { - sess.Out.Debug("[THREAD #%d][%s] Analyzing commit: %s\n", tid, *repo.FullName, commit.Hash) - changes, _ := core.GetChanges(commit, clone) - sess.Out.Debug("[THREAD #%d][%s] Changes in %s: %d\n", tid, *repo.FullName, commit.Hash, len(changes)) - for _, change := range changes { - changeAction := core.GetChangeAction(change) - path := core.GetChangePath(change) - matchFile := core.NewMatchFile(path) - if matchFile.IsSkippable() { - sess.Out.Debug("[THREAD #%d][%s] Skipping %s\n", tid, *repo.FullName, matchFile.Path) - continue - } - sess.Out.Debug("[THREAD #%d][%s] Matching: %s...\n", tid, *repo.FullName, matchFile.Path) - for _, signature := range core.Signatures { - if signature.Match(matchFile) { - - finding := &core.Finding{ - FilePath: path, - Action: changeAction, - Description: signature.Description(), - Comment: signature.Comment(), - RepositoryOwner: *repo.Owner, - RepositoryName: *repo.Name, - CommitHash: commit.Hash.String(), - CommitMessage: strings.TrimSpace(commit.Message), - CommitAuthor: commit.Author.String(), - } - finding.Initialize() - sess.AddFinding(finding) - - sess.Out.Warn(" %s: %s\n", strings.ToUpper(changeAction), finding.Description) - sess.Out.Info(" Path.......: %s\n", finding.FilePath) - sess.Out.Info(" Repo.......: %s\n", *repo.FullName) - sess.Out.Info(" Message....: %s\n", core.TruncateString(finding.CommitMessage, 100)) - sess.Out.Info(" Author.....: %s\n", finding.CommitAuthor) - if finding.Comment != "" { - sess.Out.Info(" Comment....: %s\n", finding.Comment) - } - sess.Out.Info(" File URL...: %s\n", finding.FileUrl) - sess.Out.Info(" Commit URL.: %s\n", finding.CommitUrl) - sess.Out.Info(" ------------------------------------------------\n\n") - sess.Stats.IncrementFindings() - break - } - } - sess.Stats.IncrementFiles() - } - sess.Stats.IncrementCommits() - sess.Out.Debug("[THREAD #%d][%s] Done analyzing changes in %s\n", tid, *repo.FullName, commit.Hash) - } - sess.Out.Debug("[THREAD #%d][%s] Done analyzing commits\n", tid, *repo.FullName) - os.RemoveAll(path) - sess.Out.Debug("[THREAD #%d][%s] Deleted %s\n", tid, *repo.FullName, path) - sess.Stats.IncrementRepositories() - sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) - } - }(i) - } - for _, repo := range sess.Repositories { - ch <- repo - } - close(ch) - wg.Wait() + sess.Stats.Status = core.StatusAnalyzing + var ch = make(chan *core.GithubRepository, len(sess.Repositories)) + var wg sync.WaitGroup + var threadNum int + if len(sess.Repositories) <= 1 { + threadNum = 1 + } else if len(sess.Repositories) <= *sess.Options.Threads { + threadNum = len(sess.Repositories) - 1 + } else { + threadNum = *sess.Options.Threads + } + wg.Add(threadNum) + sess.Out.Debug("Threads for repository analysis: %d\n", threadNum) + + sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories")) + + for i := 0; i < threadNum; i++ { + go func(tid int) { + for { + sess.Out.Debug("[THREAD #%d] Requesting new repository to analyze...\n", tid) + repo, ok := <-ch + if !ok { + sess.Out.Debug("[THREAD #%d] No more tasks, marking WaitGroup as done\n", tid) + wg.Done() + return + } + + sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", tid, *repo.FullName) + clone, path, err := core.CloneRepository(repo.CloneURL, repo.BranchName, *sess.Options.CommitDepth) + if err != nil { + if err.Error() != "remote repository is empty" { + sess.Out.Error("Error cloning repository %s: %s\n", *repo.FullName, err) + } + sess.Stats.IncrementRepositories() + sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) + continue + } + sess.Out.Debug("[THREAD #%d][%s] Cloned repository to: %s\n", tid, *repo.FullName, path) + + history, err := core.GetRepositoryHistory(clone) + if err != nil { + sess.Out.Error("[THREAD #%d][%s] Error getting commit history: %s\n", tid, *repo.FullName, err) + os.RemoveAll(path) + sess.Stats.IncrementRepositories() + sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) + continue + } + sess.Out.Debug("[THREAD #%d][%s] Number of commits: %d\n", tid, *repo.FullName, len(history)) + + for _, commit := range history { + sess.Out.Debug("[THREAD #%d][%s] Analyzing commit: %s\n", tid, *repo.FullName, commit.Hash) + changes, _ := core.GetChanges(commit, clone) + sess.Out.Debug("[THREAD #%d][%s] Changes in %s: %d\n", tid, *repo.FullName, commit.Hash, len(changes)) + for _, change := range changes { + changeAction := core.GetChangeAction(change) + path := core.GetChangePath(change) + matchFile := core.NewMatchFile(path) + if matchFile.IsSkippable() { + sess.Out.Debug("[THREAD #%d][%s] Skipping %s\n", tid, *repo.FullName, matchFile.Path) + continue + } + sess.Out.Debug("[THREAD #%d][%s] Matching: %s...\n", tid, *repo.FullName, matchFile.Path) + for _, signature := range core.Signatures { + if signature.Match(matchFile) { + + finding := &core.Finding{ + FilePath: path, + Action: changeAction, + Description: signature.Description(), + Comment: signature.Comment(), + RepositoryOwner: *repo.Owner, + RepositoryName: *repo.Name, + BranchName: *repo.BranchName, + CommitHash: commit.Hash.String(), + CommitMessage: strings.TrimSpace(commit.Message), + CommitAuthor: commit.Author.String(), + } + finding.Initialize() + sess.AddFinding(finding) + + sess.Out.Warn(" %s: %s\n", strings.ToUpper(changeAction), finding.Description) + sess.Out.Info(" Path.......: %s\n", finding.FilePath) + sess.Out.Info(" Repo.......: %s\n", *repo.FullName) + sess.Out.Info(" Branch.....: %s\n", *repo.BranchName) + sess.Out.Info(" Message....: %s\n", core.TruncateString(finding.CommitMessage, 100)) + sess.Out.Info(" Author.....: %s\n", finding.CommitAuthor) + if finding.Comment != "" { + sess.Out.Info(" Comment....: %s\n", finding.Comment) + } + sess.Out.Info(" File URL...: %s\n", finding.FileUrl) + sess.Out.Info(" Commit URL.: %s\n", finding.CommitUrl) + sess.Out.Info(" ------------------------------------------------\n\n") + sess.Stats.IncrementFindings() + break + } + } + sess.Stats.IncrementFiles() + } + sess.Stats.IncrementCommits() + sess.Out.Debug("[THREAD #%d][%s] Done analyzing changes in %s\n", tid, *repo.FullName, commit.Hash) + } + sess.Out.Debug("[THREAD #%d][%s] Done analyzing commits\n", tid, *repo.FullName) + os.RemoveAll(path) + sess.Out.Debug("[THREAD #%d][%s] Deleted %s\n", tid, *repo.FullName, path) + sess.Stats.IncrementRepositories() + sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) + } + }(i) + } + for _, repo := range sess.Repositories { + ch <- repo + } + close(ch) + wg.Wait() } func PrintSessionStats(sess *core.Session) { - sess.Out.Info("\nFindings....: %d\n", sess.Stats.Findings) - sess.Out.Info("Files.......: %d\n", sess.Stats.Files) - sess.Out.Info("Commits.....: %d\n", sess.Stats.Commits) - sess.Out.Info("Repositories: %d\n", sess.Stats.Repositories) - sess.Out.Info("Targets.....: %d\n\n", sess.Stats.Targets) + sess.Out.Info("\nFindings....: %d\n", sess.Stats.Findings) + sess.Out.Info("Files.......: %d\n", sess.Stats.Files) + sess.Out.Info("Commits.....: %d\n", sess.Stats.Commits) + sess.Out.Info("Repositories: %d\n", sess.Stats.Repositories) + sess.Out.Info("Targets.....: %d\n\n", sess.Stats.Targets) } func main() { - if sess, err = core.NewSession(); err != nil { - fmt.Println(err) - os.Exit(1) - } - - sess.Out.Info("%s\n\n", core.ASCIIBanner) - sess.Out.Important("%s v%s started at %s\n", core.Name, core.Version, sess.Stats.StartedAt.Format(time.RFC3339)) - sess.Out.Important("Loaded %d signatures\n", len(core.Signatures)) - sess.Out.Important("Web interface available at http://%s:%d\n", *sess.Options.BindAddress, *sess.Options.Port) - - if sess.Stats.Status == "finished" { - sess.Out.Important("Loaded session file: %s\n", *sess.Options.Load) - } else { - if len(sess.Options.Logins) == 0 { - sess.Out.Fatal("Please provide at least one GitHub organization or user\n") - } - - GatherTargets(sess) - GatherRepositories(sess) - AnalyzeRepositories(sess) - sess.Finish() - - if *sess.Options.Save != "" { - err := sess.SaveToFile(*sess.Options.Save) - if err != nil { - sess.Out.Error("Error saving session to %s: %s\n", *sess.Options.Save, err) - } - sess.Out.Important("Saved session to: %s\n\n", *sess.Options.Save) - } - } - - PrintSessionStats(sess) - sess.Out.Important("Press Ctrl+C to stop web server and exit.\n\n") - select {} + if sess, err = core.NewSession(); err != nil { + fmt.Println(err) + os.Exit(1) + } + + sess.Out.Info("%s\n\n", core.ASCIIBanner) + sess.Out.Important("%s v%s started at %s\n", core.Name, core.Version, sess.Stats.StartedAt.Format(time.RFC3339)) + sess.Out.Important("Loaded %d signatures\n", len(core.Signatures)) + sess.Out.Important("Web interface available at http://%s:%d\n", *sess.Options.BindAddress, *sess.Options.Port) + + if sess.Stats.Status == "finished" { + sess.Out.Important("Loaded session file: %s\n", *sess.Options.Load) + } else { + if len(sess.Options.Logins) == 0 { + sess.Out.Fatal("Please provide at least one GitHub organization or user\n") + } + + GatherTargets(sess) + GatherRepositories(sess) + AnalyzeRepositories(sess) + sess.Finish() + + if *sess.Options.Save != "" { + err := sess.SaveToFile(*sess.Options.Save) + if err != nil { + sess.Out.Error("Error saving session to %s: %s\n", *sess.Options.Save, err) + } + sess.Out.Important("Saved session to: %s\n\n", *sess.Options.Save) + } + } + + PrintSessionStats(sess) + sess.Out.Important("Press Ctrl+C to stop web server and exit.\n\n") + select {} } From d9b3e6861cf0f5fd62b5b2857a2d826601ddc17b Mon Sep 17 00:00:00 2001 From: plasticuproject Date: Thu, 28 Feb 2019 14:32:26 -0500 Subject: [PATCH 2/6] Add codeship.aes signature Added signature for codeship.aes Credit to @dcepler --- core/signatures.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/signatures.go b/core/signatures.go index 508f4932..42625c70 100644 --- a/core/signatures.go +++ b/core/signatures.go @@ -439,6 +439,12 @@ var Signatures = []Signature{ description: "Shell configuration file", comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", }, + SimpleSignature{ + part: PartFilename, + match: "codeship.aes", + description: "Codeship Pro project-specific AES key", + comment: "AES key to encrypt/decrypt environment variables for Codeship Pro", + }, PatternSignature{ part: PartFilename, match: regexp.MustCompile(`^.*_rsa$`), From 65c1598c9b80704ed689b42959630e91ae69badf Mon Sep 17 00:00:00 2001 From: plasticuproject Date: Thu, 28 Feb 2019 14:46:23 -0500 Subject: [PATCH 3/6] Removed codeship.aes signature Removed codeship.aes signature. This was commited to the wrong branch. --- core/signatures.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/core/signatures.go b/core/signatures.go index 42625c70..508f4932 100644 --- a/core/signatures.go +++ b/core/signatures.go @@ -439,12 +439,6 @@ var Signatures = []Signature{ description: "Shell configuration file", comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", }, - SimpleSignature{ - part: PartFilename, - match: "codeship.aes", - description: "Codeship Pro project-specific AES key", - comment: "AES key to encrypt/decrypt environment variables for Codeship Pro", - }, PatternSignature{ part: PartFilename, match: regexp.MustCompile(`^.*_rsa$`), From 42ac22c75871e590070c6d4219780abc75815d83 Mon Sep 17 00:00:00 2001 From: plasticuproject Date: Fri, 1 Mar 2019 15:31:03 -0500 Subject: [PATCH 4/6] Formated some output text. --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index d76bd5e6..0918bf6e 100644 --- a/main.go +++ b/main.go @@ -102,7 +102,7 @@ func AnalyzeRepositories(sess *core.Session) { wg.Add(threadNum) sess.Out.Debug("Threads for repository analysis: %d\n", threadNum) - sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories")) + sess.Out.Important("Analyzing %d %s branches...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories")) for i := 0; i < threadNum; i++ { go func(tid int) { From 200a31dba4ed57ef0b3fdef809c75161304c0ba7 Mon Sep 17 00:00:00 2001 From: plasticuproject Date: Fri, 1 Mar 2019 16:08:19 -0500 Subject: [PATCH 5/6] Fixed text output --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 0918bf6e..103fd35f 100644 --- a/main.go +++ b/main.go @@ -102,7 +102,7 @@ func AnalyzeRepositories(sess *core.Session) { wg.Add(threadNum) sess.Out.Debug("Threads for repository analysis: %d\n", threadNum) - sess.Out.Important("Analyzing %d %s branches...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "repository", "repositories")) + sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "branch", "branches")) for i := 0; i < threadNum; i++ { go func(tid int) { From 2b5b277195f6ff541227a539fb839ac14a88d1d1 Mon Sep 17 00:00:00 2001 From: plasticuproject Date: Tue, 17 Dec 2019 22:12:16 -0500 Subject: [PATCH 6/6] Corrected tabbed indentations to 2 space indendtation --- core/git.go | 196 +++---- core/github.go | 234 ++++---- core/options.go | 58 +- core/session.go | 354 ++++++------ core/signatures.go | 1296 ++++++++++++++++++++++---------------------- main.go | 454 ++++++++-------- 6 files changed, 1296 insertions(+), 1296 deletions(-) diff --git a/core/git.go b/core/git.go index e3179117..c19aa951 100644 --- a/core/git.go +++ b/core/git.go @@ -1,129 +1,129 @@ package core import ( - "fmt" - "io/ioutil" + "fmt" + "io/ioutil" - "gopkg.in/src-d/go-git.v4" - "gopkg.in/src-d/go-git.v4/plumbing" - "gopkg.in/src-d/go-git.v4/plumbing/object" - "gopkg.in/src-d/go-git.v4/utils/merkletrie" + "gopkg.in/src-d/go-git.v4" + "gopkg.in/src-d/go-git.v4/plumbing" + "gopkg.in/src-d/go-git.v4/plumbing/object" + "gopkg.in/src-d/go-git.v4/utils/merkletrie" ) const ( - EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + EmptyTreeCommitId = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" ) func CloneRepository(url *string, branch *string, depth int) (*git.Repository, string, error) { - urlVal := *url - branchVal := *branch - dir, err := ioutil.TempDir("", "gitrob") - if err != nil { - return nil, "", err - } - repository, err := git.PlainClone(dir, false, &git.CloneOptions{ - URL: urlVal, - Depth: depth, - ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), - SingleBranch: true, - Tags: git.NoTags, - }) - if err != nil { - return nil, dir, err - } - return repository, dir, nil + urlVal := *url + branchVal := *branch + dir, err := ioutil.TempDir("", "gitrob") + if err != nil { + return nil, "", err + } + repository, err := git.PlainClone(dir, false, &git.CloneOptions{ + URL: urlVal, + Depth: depth, + ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branchVal)), + SingleBranch: true, + Tags: git.NoTags, + }) + if err != nil { + return nil, dir, err + } + return repository, dir, nil } func GetRepositoryHistory(repository *git.Repository) ([]*object.Commit, error) { - var commits []*object.Commit - ref, err := repository.Head() - if err != nil { - return nil, err - } - cIter, err := repository.Log(&git.LogOptions{From: ref.Hash()}) - if err != nil { - return nil, err - } - cIter.ForEach(func(c *object.Commit) error { - commits = append(commits, c) - return nil - }) - return commits, nil + var commits []*object.Commit + ref, err := repository.Head() + if err != nil { + return nil, err + } + cIter, err := repository.Log(&git.LogOptions{From: ref.Hash()}) + if err != nil { + return nil, err + } + cIter.ForEach(func(c *object.Commit) error { + commits = append(commits, c) + return nil + }) + return commits, nil } func GetChanges(commit *object.Commit, repo *git.Repository) (object.Changes, error) { - parentCommit, err := GetParentCommit(commit, repo) - if err != nil { - //this may be the parent commit - parentCommit = commit - //return nil, err - } + parentCommit, err := GetParentCommit(commit, repo) + if err != nil { + //this may be the parent commit + parentCommit = commit + //return nil, err + } - commitTree, err := commit.Tree() - if err != nil { - return nil, err - } + commitTree, err := commit.Tree() + if err != nil { + return nil, err + } - parentCommitTree, err := parentCommit.Tree() - if err != nil { - return nil, err - } + parentCommitTree, err := parentCommit.Tree() + if err != nil { + return nil, err + } - //changes, err := object.DiffTree(parentCommitTree, commitTree) - var changes object.Changes - if parentCommit == commit { - changes, err = object.DiffTree(nil, parentCommitTree) - } else { - changes, err = object.DiffTree(parentCommitTree, commitTree) - } + //changes, err := object.DiffTree(parentCommitTree, commitTree) + var changes object.Changes + if parentCommit == commit { + changes, err = object.DiffTree(nil, parentCommitTree) + } else { + changes, err = object.DiffTree(parentCommitTree, commitTree) + } - if err != nil { - return nil, err - } - return changes, nil + if err != nil { + return nil, err + } + return changes, nil } func GetParentCommit(commit *object.Commit, repo *git.Repository) (*object.Commit, error) { - if commit.NumParents() == 0 { - parentCommit, err := repo.CommitObject(plumbing.NewHash(EmptyTreeCommitId)) - if err != nil { - return nil, err - } - return parentCommit, nil - } - parentCommit, err := commit.Parents().Next() - if err != nil { - return nil, err - } - return parentCommit, nil + if commit.NumParents() == 0 { + parentCommit, err := repo.CommitObject(plumbing.NewHash(EmptyTreeCommitId)) + if err != nil { + return nil, err + } + return parentCommit, nil + } + parentCommit, err := commit.Parents().Next() + if err != nil { + return nil, err + } + return parentCommit, nil } func GetChangeAction(change *object.Change) string { - action, err := change.Action() - if err != nil { - return "Unknown" - } - switch action { - case merkletrie.Insert: - return "Insert" - case merkletrie.Modify: - return "Modify" - case merkletrie.Delete: - return "Delete" - default: - return "Unknown" - } + action, err := change.Action() + if err != nil { + return "Unknown" + } + switch action { + case merkletrie.Insert: + return "Insert" + case merkletrie.Modify: + return "Modify" + case merkletrie.Delete: + return "Delete" + default: + return "Unknown" + } } func GetChangePath(change *object.Change) string { - action, err := change.Action() - if err != nil { - return change.To.Name - } + action, err := change.Action() + if err != nil { + return change.To.Name + } - if action == merkletrie.Delete { - return change.From.Name - } else { - return change.To.Name - } + if action == merkletrie.Delete { + return change.From.Name + } else { + return change.To.Name + } } diff --git a/core/github.go b/core/github.go index ca5d319b..e10a5988 100644 --- a/core/github.go +++ b/core/github.go @@ -1,138 +1,138 @@ package core import ( - "context" + "context" - "github.com/google/go-github/github" + "github.com/google/go-github/github" ) type GithubOwner struct { - Login *string - ID *int64 - Type *string - Name *string - AvatarURL *string - URL *string - Company *string - Blog *string - Location *string - Email *string - Bio *string + Login *string + ID *int64 + Type *string + Name *string + AvatarURL *string + URL *string + Company *string + Blog *string + Location *string + Email *string + Bio *string } type GithubRepository struct { - Owner *string - ID *int64 - Name *string - FullName *string - CloneURL *string - URL *string - DefaultBranch *string - BranchName *string - Description *string - Homepage *string + Owner *string + ID *int64 + Name *string + FullName *string + CloneURL *string + URL *string + DefaultBranch *string + BranchName *string + Description *string + Homepage *string } func GetUserOrOrganization(login string, client *github.Client) (*GithubOwner, error) { - ctx := context.Background() - user, _, err := client.Users.Get(ctx, login) - if err != nil { - return nil, err - } - return &GithubOwner{ - Login: user.Login, - ID: user.ID, - Type: user.Type, - Name: user.Name, - AvatarURL: user.AvatarURL, - URL: user.HTMLURL, - Company: user.Company, - Blog: user.Blog, - Location: user.Location, - Email: user.Email, - Bio: user.Bio, - }, nil + ctx := context.Background() + user, _, err := client.Users.Get(ctx, login) + if err != nil { + return nil, err + } + return &GithubOwner{ + Login: user.Login, + ID: user.ID, + Type: user.Type, + Name: user.Name, + AvatarURL: user.AvatarURL, + URL: user.HTMLURL, + Company: user.Company, + Blog: user.Blog, + Location: user.Location, + Email: user.Email, + Bio: user.Bio, + }, nil } func GetRepositoriesFromOwner(login *string, client *github.Client, includeBranches *bool) ([]*GithubRepository, error) { - var allRepos []*GithubRepository - loginVal := *login - ctx := context.Background() - opt := &github.RepositoryListOptions{ - Type: "sources", - } - opts := &github.ListOptions{} - for { - repos, resp, err := client.Repositories.List(ctx, loginVal, opt) - if err != nil { - return allRepos, err - } - for _, repo := range repos { - if !*repo.Fork { - t := GithubRepository{ - Owner: repo.Owner.Login, - ID: repo.ID, - Name: repo.Name, - FullName: repo.FullName, - CloneURL: repo.CloneURL, - URL: repo.HTMLURL, - DefaultBranch: repo.DefaultBranch, - BranchName: repo.DefaultBranch, - Description: repo.Description, - Homepage: repo.Homepage, - } - branches, resp, err := client.Repositories.ListBranches(ctx, *t.Owner, *t.Name, opts) - if err != nil { - return allRepos, err - } - for _, branch := range branches { - r := GithubRepository{ - Owner: t.Owner, - ID: t.ID, - Name: t.Name, - FullName: t.FullName, - CloneURL: t.CloneURL, - URL: t.URL, - DefaultBranch: t.DefaultBranch, - BranchName: branch.Name, - Description: t.Description, - Homepage: t.Homepage, - } - if *includeBranches { - allRepos = append(allRepos, &r) - } else { - allRepos = append(allRepos, &t) - } - } - opts.Page = resp.NextPage - } - } - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } + var allRepos []*GithubRepository + loginVal := *login + ctx := context.Background() + opt := &github.RepositoryListOptions{ + Type: "sources", + } + opts := &github.ListOptions{} + for { + repos, resp, err := client.Repositories.List(ctx, loginVal, opt) + if err != nil { + return allRepos, err + } + for _, repo := range repos { + if !*repo.Fork { + t := GithubRepository{ + Owner: repo.Owner.Login, + ID: repo.ID, + Name: repo.Name, + FullName: repo.FullName, + CloneURL: repo.CloneURL, + URL: repo.HTMLURL, + DefaultBranch: repo.DefaultBranch, + BranchName: repo.DefaultBranch, + Description: repo.Description, + Homepage: repo.Homepage, + } + branches, resp, err := client.Repositories.ListBranches(ctx, *t.Owner, *t.Name, opts) + if err != nil { + return allRepos, err + } + for _, branch := range branches { + r := GithubRepository{ + Owner: t.Owner, + ID: t.ID, + Name: t.Name, + FullName: t.FullName, + CloneURL: t.CloneURL, + URL: t.URL, + DefaultBranch: t.DefaultBranch, + BranchName: branch.Name, + Description: t.Description, + Homepage: t.Homepage, + } + if *includeBranches { + allRepos = append(allRepos, &r) + } else { + allRepos = append(allRepos, &t) + } + } + opts.Page = resp.NextPage + } + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } - return allRepos, nil + return allRepos, nil } func GetOrganizationMembers(login *string, client *github.Client) ([]*GithubOwner, error) { - var allMembers []*GithubOwner - loginVal := *login - ctx := context.Background() - opt := &github.ListMembersOptions{} - for { - members, resp, err := client.Organizations.ListMembers(ctx, loginVal, opt) - if err != nil { - return allMembers, err - } - for _, member := range members { - allMembers = append(allMembers, &GithubOwner{Login: member.Login, ID: member.ID, Type: member.Type}) - } - if resp.NextPage == 0 { - break - } - opt.Page = resp.NextPage - } - return allMembers, nil + var allMembers []*GithubOwner + loginVal := *login + ctx := context.Background() + opt := &github.ListMembersOptions{} + for { + members, resp, err := client.Organizations.ListMembers(ctx, loginVal, opt) + if err != nil { + return allMembers, err + } + for _, member := range members { + allMembers = append(allMembers, &GithubOwner{Login: member.Login, ID: member.ID, Type: member.Type}) + } + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return allMembers, nil } diff --git a/core/options.go b/core/options.go index a65fab51..06f7eeaf 100644 --- a/core/options.go +++ b/core/options.go @@ -1,41 +1,41 @@ package core import ( - "flag" + "flag" ) type Options struct { - CommitDepth *int - GithubAccessToken *string `json:"-"` - NoExpandOrgs *bool - Threads *int - Save *string `json:"-"` - Load *string `json:"-"` - BindAddress *string - Port *int - Silent *bool - Debug *bool - IncludeBranches *bool - Logins []string + CommitDepth *int + GithubAccessToken *string `json:"-"` + NoExpandOrgs *bool + Threads *int + Save *string `json:"-"` + Load *string `json:"-"` + BindAddress *string + Port *int + Silent *bool + Debug *bool + IncludeBranches *bool + Logins []string } func ParseOptions() (Options, error) { - options := Options{ - CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), - GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), - NoExpandOrgs: flag.Bool("no-expand-orgs", false, "Don't add members to targets when processing organizations"), - Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), - Save: flag.String("save", "", "Save session to file"), - Load: flag.String("load", "", "Load session file"), - BindAddress: flag.String("bind-address", "127.0.0.1", "Address to bind web server to"), - Port: flag.Int("port", 9393, "Port to run web server on"), - Silent: flag.Bool("silent", false, "Suppress all output except for errors"), - Debug: flag.Bool("debug", false, "Print debugging information"), - IncludeBranches: flag.Bool("include-branches", false, "Include repository branches in scan"), - } + options := Options{ + CommitDepth: flag.Int("commit-depth", 500, "Number of repository commits to process"), + GithubAccessToken: flag.String("github-access-token", "", "GitHub access token to use for API requests"), + NoExpandOrgs: flag.Bool("no-expand-orgs", false, "Don't add members to targets when processing organizations"), + Threads: flag.Int("threads", 0, "Number of concurrent threads (default number of logical CPUs)"), + Save: flag.String("save", "", "Save session to file"), + Load: flag.String("load", "", "Load session file"), + BindAddress: flag.String("bind-address", "127.0.0.1", "Address to bind web server to"), + Port: flag.Int("port", 9393, "Port to run web server on"), + Silent: flag.Bool("silent", false, "Suppress all output except for errors"), + Debug: flag.Bool("debug", false, "Print debugging information"), + IncludeBranches: flag.Bool("include-branches", false, "Include repository branches in scan"), + } - flag.Parse() - options.Logins = flag.Args() + flag.Parse() + options.Logins = flag.Args() - return options, nil + return options, nil } diff --git a/core/session.go b/core/session.go index 70dcf659..2c003948 100644 --- a/core/session.go +++ b/core/session.go @@ -1,253 +1,253 @@ package core import ( - "context" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "os" - "runtime" - "sync" - "time" - - "github.com/gin-gonic/gin" - "github.com/google/go-github/github" - "golang.org/x/oauth2" + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "runtime" + "sync" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/go-github/github" + "golang.org/x/oauth2" ) const ( - AccessTokenEnvVariable = "GITROB_ACCESS_TOKEN" + AccessTokenEnvVariable = "GITROB_ACCESS_TOKEN" - StatusInitializing = "initializing" - StatusGathering = "gathering" - StatusAnalyzing = "analyzing" - StatusFinished = "finished" + StatusInitializing = "initializing" + StatusGathering = "gathering" + StatusAnalyzing = "analyzing" + StatusFinished = "finished" ) type Stats struct { - sync.Mutex + sync.Mutex - StartedAt time.Time - FinishedAt time.Time - Status string - Progress float64 - Targets int - Repositories int - Commits int - Files int - Findings int + StartedAt time.Time + FinishedAt time.Time + Status string + Progress float64 + Targets int + Repositories int + Commits int + Files int + Findings int } type Session struct { - sync.Mutex + sync.Mutex - Version string - Options Options `json:"-"` - Out *Logger `json:"-"` - Stats *Stats - GithubAccessToken string `json:"-"` - GithubClient *github.Client `json:"-"` - Router *gin.Engine `json:"-"` - Targets []*GithubOwner - Repositories []*GithubRepository - Findings []*Finding + Version string + Options Options `json:"-"` + Out *Logger `json:"-"` + Stats *Stats + GithubAccessToken string `json:"-"` + GithubClient *github.Client `json:"-"` + Router *gin.Engine `json:"-"` + Targets []*GithubOwner + Repositories []*GithubRepository + Findings []*Finding } func (s *Session) Start() { - s.InitStats() - s.InitLogger() - s.InitThreads() - s.InitGithubAccessToken() - s.InitGithubClient() - s.InitRouter() + s.InitStats() + s.InitLogger() + s.InitThreads() + s.InitGithubAccessToken() + s.InitGithubClient() + s.InitRouter() } func (s *Session) Finish() { - s.Stats.FinishedAt = time.Now() - s.Stats.Status = StatusFinished + s.Stats.FinishedAt = time.Now() + s.Stats.Status = StatusFinished } func (s *Session) AddTarget(target *GithubOwner) { - s.Lock() - defer s.Unlock() - for _, t := range s.Targets { - if *target.ID == *t.ID { - return - } - } - s.Targets = append(s.Targets, target) + s.Lock() + defer s.Unlock() + for _, t := range s.Targets { + if *target.ID == *t.ID { + return + } + } + s.Targets = append(s.Targets, target) } func (s *Session) AddRepository(repository *GithubRepository) { - s.Lock() - defer s.Unlock() - for _, r := range s.Repositories { - if *repository.ID == *r.ID { - return - } - } - s.Repositories = append(s.Repositories, repository) + s.Lock() + defer s.Unlock() + for _, r := range s.Repositories { + if *repository.ID == *r.ID { + return + } + } + s.Repositories = append(s.Repositories, repository) } func (s *Session) AddBranches(repository *GithubRepository) { - s.Lock() - defer s.Unlock() - for _, r := range s.Repositories { - if *repository.BranchName == *r.BranchName { - return - } - } - s.Repositories = append(s.Repositories, repository) + s.Lock() + defer s.Unlock() + for _, r := range s.Repositories { + if *repository.BranchName == *r.BranchName { + return + } + } + s.Repositories = append(s.Repositories, repository) } func (s *Session) AddFinding(finding *Finding) { - s.Lock() - defer s.Unlock() - s.Findings = append(s.Findings, finding) + s.Lock() + defer s.Unlock() + s.Findings = append(s.Findings, finding) } func (s *Session) InitStats() { - if s.Stats != nil { - return - } - s.Stats = &Stats{ - StartedAt: time.Now(), - Status: StatusInitializing, - Progress: 0.0, - Targets: 0, - Repositories: 0, - Commits: 0, - Files: 0, - Findings: 0, - } + if s.Stats != nil { + return + } + s.Stats = &Stats{ + StartedAt: time.Now(), + Status: StatusInitializing, + Progress: 0.0, + Targets: 0, + Repositories: 0, + Commits: 0, + Files: 0, + Findings: 0, + } } func (s *Session) InitLogger() { - s.Out = &Logger{} - s.Out.SetDebug(*s.Options.Debug) - s.Out.SetSilent(*s.Options.Silent) + s.Out = &Logger{} + s.Out.SetDebug(*s.Options.Debug) + s.Out.SetSilent(*s.Options.Silent) } func (s *Session) InitGithubAccessToken() { - if *s.Options.GithubAccessToken == "" { - accessToken := os.Getenv(AccessTokenEnvVariable) - if accessToken == "" { - s.Out.Fatal("No GitHub access token given. Please provide via command line option or in the %s environment variable.\n", AccessTokenEnvVariable) - } - s.GithubAccessToken = accessToken - } else { - s.GithubAccessToken = *s.Options.GithubAccessToken - } + if *s.Options.GithubAccessToken == "" { + accessToken := os.Getenv(AccessTokenEnvVariable) + if accessToken == "" { + s.Out.Fatal("No GitHub access token given. Please provide via command line option or in the %s environment variable.\n", AccessTokenEnvVariable) + } + s.GithubAccessToken = accessToken + } else { + s.GithubAccessToken = *s.Options.GithubAccessToken + } } func (s *Session) InitGithubClient() { - ctx := context.Background() - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: s.GithubAccessToken}, - ) - tc := oauth2.NewClient(ctx, ts) - s.GithubClient = github.NewClient(tc) - s.GithubClient.UserAgent = fmt.Sprintf("%s v%s", Name, Version) + ctx := context.Background() + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: s.GithubAccessToken}, + ) + tc := oauth2.NewClient(ctx, ts) + s.GithubClient = github.NewClient(tc) + s.GithubClient.UserAgent = fmt.Sprintf("%s v%s", Name, Version) } func (s *Session) InitThreads() { - if *s.Options.Threads == 0 { - numCPUs := runtime.NumCPU() - s.Options.Threads = &numCPUs - } - runtime.GOMAXPROCS(*s.Options.Threads + 2) // thread count + main + web server + if *s.Options.Threads == 0 { + numCPUs := runtime.NumCPU() + s.Options.Threads = &numCPUs + } + runtime.GOMAXPROCS(*s.Options.Threads + 2) // thread count + main + web server } func (s *Session) InitRouter() { - bind := fmt.Sprintf("%s:%d", *s.Options.BindAddress, *s.Options.Port) - s.Router = NewRouter(s) - go func(sess *Session) { - if err := sess.Router.Run(bind); err != nil { - sess.Out.Fatal("Error when starting web server: %s\n", err) - } - }(s) + bind := fmt.Sprintf("%s:%d", *s.Options.BindAddress, *s.Options.Port) + s.Router = NewRouter(s) + go func(sess *Session) { + if err := sess.Router.Run(bind); err != nil { + sess.Out.Fatal("Error when starting web server: %s\n", err) + } + }(s) } func (s *Session) SaveToFile(location string) error { - sessionJson, err := json.Marshal(s) - if err != nil { - return err - } - err = ioutil.WriteFile(location, sessionJson, 0644) - if err != nil { - return err - } - return nil + sessionJson, err := json.Marshal(s) + if err != nil { + return err + } + err = ioutil.WriteFile(location, sessionJson, 0644) + if err != nil { + return err + } + return nil } func (s *Stats) IncrementTargets() { - s.Lock() - defer s.Unlock() - s.Targets++ + s.Lock() + defer s.Unlock() + s.Targets++ } func (s *Stats) IncrementRepositories() { - s.Lock() - defer s.Unlock() - s.Repositories++ + s.Lock() + defer s.Unlock() + s.Repositories++ } func (s *Stats) IncrementCommits() { - s.Lock() - defer s.Unlock() - s.Commits++ + s.Lock() + defer s.Unlock() + s.Commits++ } func (s *Stats) IncrementFiles() { - s.Lock() - defer s.Unlock() - s.Files++ + s.Lock() + defer s.Unlock() + s.Files++ } func (s *Stats) IncrementFindings() { - s.Lock() - defer s.Unlock() - s.Findings++ + s.Lock() + defer s.Unlock() + s.Findings++ } func (s *Stats) UpdateProgress(current int, total int) { - s.Lock() - defer s.Unlock() - if current >= total { - s.Progress = 100.0 - } else { - s.Progress = (float64(current) * float64(100)) / float64(total) - } + s.Lock() + defer s.Unlock() + if current >= total { + s.Progress = 100.0 + } else { + s.Progress = (float64(current) * float64(100)) / float64(total) + } } func NewSession() (*Session, error) { - var err error - var session Session - - if session.Options, err = ParseOptions(); err != nil { - return nil, err - } - - if *session.Options.Save != "" && FileExists(*session.Options.Save) { - return nil, errors.New(fmt.Sprintf("File: %s already exists.", *session.Options.Save)) - } - - if *session.Options.Load != "" { - if !FileExists(*session.Options.Load) { - return nil, errors.New(fmt.Sprintf("Session file %s does not exist or is not readable.", *session.Options.Load)) - } - data, err := ioutil.ReadFile(*session.Options.Load) - if err != nil { - return nil, err - } - if err := json.Unmarshal(data, &session); err != nil { - return nil, errors.New(fmt.Sprintf("Session file %s is corrupt or generated by an old version of Gitrob.", *session.Options.Load)) - } - } - - session.Version = Version - session.Start() - - return &session, nil + var err error + var session Session + + if session.Options, err = ParseOptions(); err != nil { + return nil, err + } + + if *session.Options.Save != "" && FileExists(*session.Options.Save) { + return nil, errors.New(fmt.Sprintf("File: %s already exists.", *session.Options.Save)) + } + + if *session.Options.Load != "" { + if !FileExists(*session.Options.Load) { + return nil, errors.New(fmt.Sprintf("Session file %s does not exist or is not readable.", *session.Options.Load)) + } + data, err := ioutil.ReadFile(*session.Options.Load) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &session); err != nil { + return nil, errors.New(fmt.Sprintf("Session file %s is corrupt or generated by an old version of Gitrob.", *session.Options.Load)) + } + } + + session.Version = Version + session.Start() + + return &session, nil } diff --git a/core/signatures.go b/core/signatures.go index 508f4932..245a5124 100644 --- a/core/signatures.go +++ b/core/signatures.go @@ -1,712 +1,712 @@ package core import ( - "crypto/sha1" - "fmt" - "io" - "path/filepath" - "regexp" - "strings" + "crypto/sha1" + "fmt" + "io" + "path/filepath" + "regexp" + "strings" ) const ( - TypeSimple = "simple" - TypePattern = "pattern" + TypeSimple = "simple" + TypePattern = "pattern" - PartExtension = "extension" - PartFilename = "filename" - PartPath = "path" + PartExtension = "extension" + PartFilename = "filename" + PartPath = "path" ) var skippableExtensions = []string{".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".psd", ".xcf"} var skippablePathIndicators = []string{"node_modules/", "vendor/bundle", "vendor/cache"} type MatchFile struct { - Path string - Filename string - Extension string + Path string + Filename string + Extension string } func (f *MatchFile) IsSkippable() bool { - ext := strings.ToLower(f.Extension) - path := strings.ToLower(f.Path) - for _, skippableExt := range skippableExtensions { - if ext == skippableExt { - return true - } - } - for _, skippablePathIndicator := range skippablePathIndicators { - if strings.Contains(path, skippablePathIndicator) { - return true - } - } - return false + ext := strings.ToLower(f.Extension) + path := strings.ToLower(f.Path) + for _, skippableExt := range skippableExtensions { + if ext == skippableExt { + return true + } + } + for _, skippablePathIndicator := range skippablePathIndicators { + if strings.Contains(path, skippablePathIndicator) { + return true + } + } + return false } type Finding struct { - Id string - FilePath string - Action string - Description string - Comment string - RepositoryOwner string - RepositoryName string - BranchName string - CommitHash string - CommitMessage string - CommitAuthor string - FileUrl string - CommitUrl string - RepositoryUrl string + Id string + FilePath string + Action string + Description string + Comment string + RepositoryOwner string + RepositoryName string + BranchName string + CommitHash string + CommitMessage string + CommitAuthor string + FileUrl string + CommitUrl string + RepositoryUrl string } func (f *Finding) setupUrls() { - f.RepositoryUrl = fmt.Sprintf("https://github.com/%s/%s", f.RepositoryOwner, f.RepositoryName) - f.FileUrl = fmt.Sprintf("%s/blob/%s/%s", f.RepositoryUrl, f.CommitHash, f.FilePath) - f.CommitUrl = fmt.Sprintf("%s/commit/%s", f.RepositoryUrl, f.CommitHash) + f.RepositoryUrl = fmt.Sprintf("https://github.com/%s/%s", f.RepositoryOwner, f.RepositoryName) + f.FileUrl = fmt.Sprintf("%s/blob/%s/%s", f.RepositoryUrl, f.CommitHash, f.FilePath) + f.CommitUrl = fmt.Sprintf("%s/commit/%s", f.RepositoryUrl, f.CommitHash) } func (f *Finding) generateID() { - h := sha1.New() - io.WriteString(h, f.FilePath) - io.WriteString(h, f.Action) - io.WriteString(h, f.RepositoryOwner) - io.WriteString(h, f.RepositoryName) - io.WriteString(h, f.BranchName) - io.WriteString(h, f.CommitHash) - io.WriteString(h, f.CommitMessage) - io.WriteString(h, f.CommitAuthor) - f.Id = fmt.Sprintf("%x", h.Sum(nil)) + h := sha1.New() + io.WriteString(h, f.FilePath) + io.WriteString(h, f.Action) + io.WriteString(h, f.RepositoryOwner) + io.WriteString(h, f.RepositoryName) + io.WriteString(h, f.BranchName) + io.WriteString(h, f.CommitHash) + io.WriteString(h, f.CommitMessage) + io.WriteString(h, f.CommitAuthor) + f.Id = fmt.Sprintf("%x", h.Sum(nil)) } func (f *Finding) Initialize() { - f.setupUrls() - f.generateID() + f.setupUrls() + f.generateID() } type Signature interface { - Match(file MatchFile) bool - Description() string - Comment() string + Match(file MatchFile) bool + Description() string + Comment() string } type SimpleSignature struct { - part string - match string - description string - comment string + part string + match string + description string + comment string } type PatternSignature struct { - part string - match *regexp.Regexp - description string - comment string + part string + match *regexp.Regexp + description string + comment string } func (s SimpleSignature) Match(file MatchFile) bool { - var haystack *string - switch s.part { - case PartPath: - haystack = &file.Path - case PartFilename: - haystack = &file.Filename - case PartExtension: - haystack = &file.Extension - default: - return false - } + var haystack *string + switch s.part { + case PartPath: + haystack = &file.Path + case PartFilename: + haystack = &file.Filename + case PartExtension: + haystack = &file.Extension + default: + return false + } - return (s.match == *haystack) + return (s.match == *haystack) } func (s SimpleSignature) Description() string { - return s.description + return s.description } func (s SimpleSignature) Comment() string { - return s.comment + return s.comment } func (s PatternSignature) Match(file MatchFile) bool { - var haystack *string - switch s.part { - case PartPath: - haystack = &file.Path - case PartFilename: - haystack = &file.Filename - case PartExtension: - haystack = &file.Extension - default: - return false - } + var haystack *string + switch s.part { + case PartPath: + haystack = &file.Path + case PartFilename: + haystack = &file.Filename + case PartExtension: + haystack = &file.Extension + default: + return false + } - return s.match.MatchString(*haystack) + return s.match.MatchString(*haystack) } func (s PatternSignature) Description() string { - return s.description + return s.description } func (s PatternSignature) Comment() string { - return s.comment + return s.comment } func NewMatchFile(path string) MatchFile { - _, filename := filepath.Split(path) - extension := filepath.Ext(path) - return MatchFile{ - Path: path, - Filename: filename, - Extension: extension, - } + _, filename := filepath.Split(path) + extension := filepath.Ext(path) + return MatchFile{ + Path: path, + Filename: filename, + Extension: extension, + } } var Signatures = []Signature{ - SimpleSignature{ - part: PartExtension, - match: ".pem", - description: "Potential cryptographic private key", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".log", - description: "Log file", - comment: "Log files can contain secret HTTP endpoints, session IDs, API keys and other goodies", - }, - SimpleSignature{ - part: PartExtension, - match: ".pkcs12", - description: "Potential cryptographic key bundle", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".p12", - description: "Potential cryptographic key bundle", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".pfx", - description: "Potential cryptographic key bundle", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".asc", - description: "Potential cryptographic key bundle", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "otr.private_key", - description: "Pidgin OTR private key", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".ovpn", - description: "OpenVPN client configuration file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".cscfg", - description: "Azure service configuration schema file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".rdp", - description: "Remote Desktop connection file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".mdf", - description: "Microsoft SQL database file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".sdf", - description: "Microsoft SQL server compact database file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".sqlite", - description: "SQLite database file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".bek", - description: "Microsoft BitLocker recovery key file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".tpm", - description: "Microsoft BitLocker Trusted Platform Module password file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".fve", - description: "Windows BitLocker full volume encrypted data file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".jks", - description: "Java keystore file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".psafe3", - description: "Password Safe database file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "secret_token.rb", - description: "Ruby On Rails secret token configuration file", - comment: "If the Rails secret token is known, it can allow for remote code execution (http://www.exploit-db.com/exploits/27527/)", - }, - SimpleSignature{ - part: PartFilename, - match: "carrierwave.rb", - description: "Carrierwave configuration file", - comment: "Can contain credentials for cloud storage systems such as Amazon S3 and Google Storage", - }, - SimpleSignature{ - part: PartFilename, - match: "database.yml", - description: "Potential Ruby On Rails database configuration file", - comment: "Can contain database credentials", - }, - SimpleSignature{ - part: PartFilename, - match: "omniauth.rb", - description: "OmniAuth configuration file", - comment: "The OmniAuth configuration file can contain client application secrets", - }, - SimpleSignature{ - part: PartFilename, - match: "settings.py", - description: "Django configuration file", - comment: "Can contain database credentials, cloud storage system credentials, and other secrets", - }, - SimpleSignature{ - part: PartExtension, - match: ".agilekeychain", - description: "1Password password manager database file", - comment: "Feed it to Hashcat and see if you're lucky", - }, - SimpleSignature{ - part: PartExtension, - match: ".keychain", - description: "Apple Keychain database file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".pcap", - description: "Network traffic capture file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".gnucash", - description: "GnuCash database file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml", - description: "Jenkins publish over SSH plugin file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "credentials.xml", - description: "Potential Jenkins credentials file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".kwallet", - description: "KDE Wallet Manager database file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "LocalSettings.php", - description: "Potential MediaWiki configuration file", - comment: "", - }, - SimpleSignature{ - part: PartExtension, - match: ".tblk", - description: "Tunnelblick VPN configuration file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "Favorites.plist", - description: "Sequel Pro MySQL database manager bookmark file", - comment: "", - }, - SimpleSignature{ - part: PartFilename, - match: "configuration.user.xpl", - description: "Little Snitch firewall configuration file", - comment: "Contains traffic rules for applications", - }, - SimpleSignature{ - part: PartExtension, - match: ".dayone", - description: "Day One journal file", - comment: "Now it's getting creepy...", - }, - SimpleSignature{ - part: PartFilename, - match: "journal.txt", - description: "Potential jrnl journal file", - comment: "Now it's getting creepy...", - }, - SimpleSignature{ - part: PartFilename, - match: "knife.rb", - description: "Chef Knife configuration file", - comment: "Can contain references to Chef servers", - }, - SimpleSignature{ - part: PartFilename, - match: "proftpdpasswd", - description: "cPanel backup ProFTPd credentials file", - comment: "Contains usernames and password hashes for FTP accounts", - }, - SimpleSignature{ - part: PartFilename, - match: "robomongo.json", - description: "Robomongo MongoDB manager configuration file", - comment: "Can contain credentials for MongoDB databases", - }, - SimpleSignature{ - part: PartFilename, - match: "filezilla.xml", - description: "FileZilla FTP configuration file", - comment: "Can contain credentials for FTP servers", - }, - SimpleSignature{ - part: PartFilename, - match: "recentservers.xml", - description: "FileZilla FTP recent servers file", - comment: "Can contain credentials for FTP servers", - }, - SimpleSignature{ - part: PartFilename, - match: "ventrilo_srv.ini", - description: "Ventrilo server configuration file", - comment: "Can contain passwords", - }, - SimpleSignature{ - part: PartFilename, - match: "terraform.tfvars", - description: "Terraform variable config file", - comment: "Can contain credentials for terraform providers", - }, - SimpleSignature{ - part: PartFilename, - match: ".exports", - description: "Shell configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - SimpleSignature{ - part: PartFilename, - match: ".functions", - description: "Shell configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - SimpleSignature{ - part: PartFilename, - match: ".extra", - description: "Shell configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^.*_rsa$`), - description: "Private SSH key", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^.*_dsa$`), - description: "Private SSH key", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^.*_ed25519$`), - description: "Private SSH key", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^.*_ecdsa$`), - description: "Private SSH key", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?ssh/config$`), - description: "SSH configuration file", - comment: "", - }, - PatternSignature{ - part: PartExtension, - match: regexp.MustCompile(`^key(pair)?$`), - description: "Potential cryptographic private key", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?(bash_|zsh_|sh_|z)?history$`), - description: "Shell command history file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?mysql_history$`), - description: "MySQL client command history file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?psql_history$`), - description: "PostgreSQL client command history file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?pgpass$`), - description: "PostgreSQL password file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?irb_history$`), - description: "Ruby IRB console history file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?purple/accounts\.xml$`), - description: "Pidgin chat client account configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?xchat2?/servlist_?\.conf$`), - description: "Hexchat/XChat IRC client server list configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?irssi/config$`), - description: "Irssi IRC client configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?recon-ng/keys\.db$`), - description: "Recon-ng web reconnaissance framework API key database", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?dbeaver-data-sources.xml$`), - description: "DBeaver SQL database manager configuration file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?muttrc$`), - description: "Mutt e-mail client configuration file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?s3cfg$`), - description: "S3cmd configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?aws/credentials$`), - description: "AWS CLI credentials file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^sftp-config(\.json)?$`), - description: "SFTP connection configuration file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?trc$`), - description: "T command-line Twitter client configuration file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?gitrobrc$`), - description: "Well, this is awkward... Gitrob configuration file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?(bash|zsh|csh)rc$`), - description: "Shell configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?(bash_|zsh_)?profile$`), - description: "Shell profile configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?(bash_|zsh_)?aliases$`), - description: "Shell command alias configuration file", - comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`config(\.inc)?\.php$`), - description: "PHP configuration file", - comment: "", - }, - PatternSignature{ - part: PartExtension, - match: regexp.MustCompile(`^key(store|ring)$`), - description: "GNOME Keyring database file", - comment: "", - }, - PatternSignature{ - part: PartExtension, - match: regexp.MustCompile(`^kdbx?$`), - description: "KeePass password manager database file", - comment: "Feed it to Hashcat and see if you're lucky", - }, - PatternSignature{ - part: PartExtension, - match: regexp.MustCompile(`^sql(dump)?$`), - description: "SQL dump file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?htpasswd$`), - description: "Apache htpasswd file", - comment: "", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^(\.|_)?netrc$`), - description: "Configuration file for auto-login process", - comment: "Can contain username and password", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?gem/credentials$`), - description: "Rubygems credentials file", - comment: "Can contain API key for a rubygems.org account", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?tugboat$`), - description: "Tugboat DigitalOcean management tool configuration", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`doctl/config.yaml$`), - description: "DigitalOcean doctl command-line client configuration file", - comment: "Contains DigitalOcean API key and other information", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?git-credentials$`), - description: "git-credential-store helper credentials file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`config/hub$`), - description: "GitHub Hub command-line client configuration file", - comment: "Can contain GitHub API access token", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?gitconfig$`), - description: "Git configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`\.?chef/(.*)\.pem$`), - description: "Chef private key", - comment: "Can be used to authenticate against Chef servers", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`etc/shadow$`), - description: "Potential Linux shadow file", - comment: "Contains hashed passwords for system users", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`etc/passwd$`), - description: "Potential Linux passwd file", - comment: "Contains system user information", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?dockercfg$`), - description: "Docker configuration file", - comment: "Can contain credentials for public or private Docker registries", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?npmrc$`), - description: "NPM configuration file", - comment: "Can contain credentials for NPM registries", - }, - PatternSignature{ - part: PartFilename, - match: regexp.MustCompile(`^\.?env$`), - description: "Environment configuration file", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`credential`), - description: "Contains word: credential", - comment: "", - }, - PatternSignature{ - part: PartPath, - match: regexp.MustCompile(`password`), - description: "Contains word: password", - comment: "", - }, + SimpleSignature{ + part: PartExtension, + match: ".pem", + description: "Potential cryptographic private key", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".log", + description: "Log file", + comment: "Log files can contain secret HTTP endpoints, session IDs, API keys and other goodies", + }, + SimpleSignature{ + part: PartExtension, + match: ".pkcs12", + description: "Potential cryptographic key bundle", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".p12", + description: "Potential cryptographic key bundle", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".pfx", + description: "Potential cryptographic key bundle", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".asc", + description: "Potential cryptographic key bundle", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "otr.private_key", + description: "Pidgin OTR private key", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".ovpn", + description: "OpenVPN client configuration file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".cscfg", + description: "Azure service configuration schema file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".rdp", + description: "Remote Desktop connection file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".mdf", + description: "Microsoft SQL database file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".sdf", + description: "Microsoft SQL server compact database file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".sqlite", + description: "SQLite database file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".bek", + description: "Microsoft BitLocker recovery key file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".tpm", + description: "Microsoft BitLocker Trusted Platform Module password file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".fve", + description: "Windows BitLocker full volume encrypted data file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".jks", + description: "Java keystore file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".psafe3", + description: "Password Safe database file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "secret_token.rb", + description: "Ruby On Rails secret token configuration file", + comment: "If the Rails secret token is known, it can allow for remote code execution (http://www.exploit-db.com/exploits/27527/)", + }, + SimpleSignature{ + part: PartFilename, + match: "carrierwave.rb", + description: "Carrierwave configuration file", + comment: "Can contain credentials for cloud storage systems such as Amazon S3 and Google Storage", + }, + SimpleSignature{ + part: PartFilename, + match: "database.yml", + description: "Potential Ruby On Rails database configuration file", + comment: "Can contain database credentials", + }, + SimpleSignature{ + part: PartFilename, + match: "omniauth.rb", + description: "OmniAuth configuration file", + comment: "The OmniAuth configuration file can contain client application secrets", + }, + SimpleSignature{ + part: PartFilename, + match: "settings.py", + description: "Django configuration file", + comment: "Can contain database credentials, cloud storage system credentials, and other secrets", + }, + SimpleSignature{ + part: PartExtension, + match: ".agilekeychain", + description: "1Password password manager database file", + comment: "Feed it to Hashcat and see if you're lucky", + }, + SimpleSignature{ + part: PartExtension, + match: ".keychain", + description: "Apple Keychain database file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".pcap", + description: "Network traffic capture file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".gnucash", + description: "GnuCash database file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "jenkins.plugins.publish_over_ssh.BapSshPublisherPlugin.xml", + description: "Jenkins publish over SSH plugin file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "credentials.xml", + description: "Potential Jenkins credentials file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".kwallet", + description: "KDE Wallet Manager database file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "LocalSettings.php", + description: "Potential MediaWiki configuration file", + comment: "", + }, + SimpleSignature{ + part: PartExtension, + match: ".tblk", + description: "Tunnelblick VPN configuration file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "Favorites.plist", + description: "Sequel Pro MySQL database manager bookmark file", + comment: "", + }, + SimpleSignature{ + part: PartFilename, + match: "configuration.user.xpl", + description: "Little Snitch firewall configuration file", + comment: "Contains traffic rules for applications", + }, + SimpleSignature{ + part: PartExtension, + match: ".dayone", + description: "Day One journal file", + comment: "Now it's getting creepy...", + }, + SimpleSignature{ + part: PartFilename, + match: "journal.txt", + description: "Potential jrnl journal file", + comment: "Now it's getting creepy...", + }, + SimpleSignature{ + part: PartFilename, + match: "knife.rb", + description: "Chef Knife configuration file", + comment: "Can contain references to Chef servers", + }, + SimpleSignature{ + part: PartFilename, + match: "proftpdpasswd", + description: "cPanel backup ProFTPd credentials file", + comment: "Contains usernames and password hashes for FTP accounts", + }, + SimpleSignature{ + part: PartFilename, + match: "robomongo.json", + description: "Robomongo MongoDB manager configuration file", + comment: "Can contain credentials for MongoDB databases", + }, + SimpleSignature{ + part: PartFilename, + match: "filezilla.xml", + description: "FileZilla FTP configuration file", + comment: "Can contain credentials for FTP servers", + }, + SimpleSignature{ + part: PartFilename, + match: "recentservers.xml", + description: "FileZilla FTP recent servers file", + comment: "Can contain credentials for FTP servers", + }, + SimpleSignature{ + part: PartFilename, + match: "ventrilo_srv.ini", + description: "Ventrilo server configuration file", + comment: "Can contain passwords", + }, + SimpleSignature{ + part: PartFilename, + match: "terraform.tfvars", + description: "Terraform variable config file", + comment: "Can contain credentials for terraform providers", + }, + SimpleSignature{ + part: PartFilename, + match: ".exports", + description: "Shell configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + SimpleSignature{ + part: PartFilename, + match: ".functions", + description: "Shell configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + SimpleSignature{ + part: PartFilename, + match: ".extra", + description: "Shell configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^.*_rsa$`), + description: "Private SSH key", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^.*_dsa$`), + description: "Private SSH key", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^.*_ed25519$`), + description: "Private SSH key", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^.*_ecdsa$`), + description: "Private SSH key", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?ssh/config$`), + description: "SSH configuration file", + comment: "", + }, + PatternSignature{ + part: PartExtension, + match: regexp.MustCompile(`^key(pair)?$`), + description: "Potential cryptographic private key", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?(bash_|zsh_|sh_|z)?history$`), + description: "Shell command history file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?mysql_history$`), + description: "MySQL client command history file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?psql_history$`), + description: "PostgreSQL client command history file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?pgpass$`), + description: "PostgreSQL password file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?irb_history$`), + description: "Ruby IRB console history file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?purple/accounts\.xml$`), + description: "Pidgin chat client account configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?xchat2?/servlist_?\.conf$`), + description: "Hexchat/XChat IRC client server list configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?irssi/config$`), + description: "Irssi IRC client configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?recon-ng/keys\.db$`), + description: "Recon-ng web reconnaissance framework API key database", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?dbeaver-data-sources.xml$`), + description: "DBeaver SQL database manager configuration file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?muttrc$`), + description: "Mutt e-mail client configuration file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?s3cfg$`), + description: "S3cmd configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?aws/credentials$`), + description: "AWS CLI credentials file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^sftp-config(\.json)?$`), + description: "SFTP connection configuration file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?trc$`), + description: "T command-line Twitter client configuration file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?gitrobrc$`), + description: "Well, this is awkward... Gitrob configuration file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?(bash|zsh|csh)rc$`), + description: "Shell configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?(bash_|zsh_)?profile$`), + description: "Shell profile configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?(bash_|zsh_)?aliases$`), + description: "Shell command alias configuration file", + comment: "Shell configuration files can contain passwords, API keys, hostnames and other goodies", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`config(\.inc)?\.php$`), + description: "PHP configuration file", + comment: "", + }, + PatternSignature{ + part: PartExtension, + match: regexp.MustCompile(`^key(store|ring)$`), + description: "GNOME Keyring database file", + comment: "", + }, + PatternSignature{ + part: PartExtension, + match: regexp.MustCompile(`^kdbx?$`), + description: "KeePass password manager database file", + comment: "Feed it to Hashcat and see if you're lucky", + }, + PatternSignature{ + part: PartExtension, + match: regexp.MustCompile(`^sql(dump)?$`), + description: "SQL dump file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?htpasswd$`), + description: "Apache htpasswd file", + comment: "", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^(\.|_)?netrc$`), + description: "Configuration file for auto-login process", + comment: "Can contain username and password", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?gem/credentials$`), + description: "Rubygems credentials file", + comment: "Can contain API key for a rubygems.org account", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?tugboat$`), + description: "Tugboat DigitalOcean management tool configuration", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`doctl/config.yaml$`), + description: "DigitalOcean doctl command-line client configuration file", + comment: "Contains DigitalOcean API key and other information", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?git-credentials$`), + description: "git-credential-store helper credentials file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`config/hub$`), + description: "GitHub Hub command-line client configuration file", + comment: "Can contain GitHub API access token", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?gitconfig$`), + description: "Git configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`\.?chef/(.*)\.pem$`), + description: "Chef private key", + comment: "Can be used to authenticate against Chef servers", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`etc/shadow$`), + description: "Potential Linux shadow file", + comment: "Contains hashed passwords for system users", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`etc/passwd$`), + description: "Potential Linux passwd file", + comment: "Contains system user information", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?dockercfg$`), + description: "Docker configuration file", + comment: "Can contain credentials for public or private Docker registries", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?npmrc$`), + description: "NPM configuration file", + comment: "Can contain credentials for NPM registries", + }, + PatternSignature{ + part: PartFilename, + match: regexp.MustCompile(`^\.?env$`), + description: "Environment configuration file", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`credential`), + description: "Contains word: credential", + comment: "", + }, + PatternSignature{ + part: PartPath, + match: regexp.MustCompile(`password`), + description: "Contains word: password", + comment: "", + }, } diff --git a/main.go b/main.go index 103fd35f..f1e5489c 100644 --- a/main.go +++ b/main.go @@ -1,250 +1,250 @@ package main import ( - "fmt" - "os" - "strings" - "sync" - "time" + "fmt" + "os" + "strings" + "sync" + "time" - "github.com/michenriksen/gitrob/core" + "github.com/michenriksen/gitrob/core" ) var ( - sess *core.Session - err error + sess *core.Session + err error ) func GatherTargets(sess *core.Session) { - sess.Stats.Status = core.StatusGathering - sess.Out.Important("Gathering targets...\n") - for _, login := range sess.Options.Logins { - target, err := core.GetUserOrOrganization(login, sess.GithubClient) - if err != nil { - sess.Out.Error(" Error retrieving information on %s: %s\n", login, err) - continue - } - sess.Out.Debug("%s (ID: %d) type: %s\n", *target.Login, *target.ID, *target.Type) - sess.AddTarget(target) - if *sess.Options.NoExpandOrgs == false && *target.Type == "Organization" { - sess.Out.Debug("Gathering members of %s (ID: %d)...\n", *target.Login, *target.ID) - members, err := core.GetOrganizationMembers(target.Login, sess.GithubClient) - if err != nil { - sess.Out.Error(" Error retrieving members of %s: %s\n", *target.Login, err) - continue - } - for _, member := range members { - sess.Out.Debug("Adding organization member %s (ID: %d) to targets\n", *member.Login, *member.ID) - sess.AddTarget(member) - } - } - } + sess.Stats.Status = core.StatusGathering + sess.Out.Important("Gathering targets...\n") + for _, login := range sess.Options.Logins { + target, err := core.GetUserOrOrganization(login, sess.GithubClient) + if err != nil { + sess.Out.Error(" Error retrieving information on %s: %s\n", login, err) + continue + } + sess.Out.Debug("%s (ID: %d) type: %s\n", *target.Login, *target.ID, *target.Type) + sess.AddTarget(target) + if *sess.Options.NoExpandOrgs == false && *target.Type == "Organization" { + sess.Out.Debug("Gathering members of %s (ID: %d)...\n", *target.Login, *target.ID) + members, err := core.GetOrganizationMembers(target.Login, sess.GithubClient) + if err != nil { + sess.Out.Error(" Error retrieving members of %s: %s\n", *target.Login, err) + continue + } + for _, member := range members { + sess.Out.Debug("Adding organization member %s (ID: %d) to targets\n", *member.Login, *member.ID) + sess.AddTarget(member) + } + } + } } func GatherRepositories(sess *core.Session) { - var ch = make(chan *core.GithubOwner, len(sess.Targets)) - var wg sync.WaitGroup - var threadNum int - if len(sess.Targets) == 1 { - threadNum = 1 - } else if len(sess.Targets) <= *sess.Options.Threads { - threadNum = len(sess.Targets) - 1 - } else { - threadNum = *sess.Options.Threads - } - wg.Add(threadNum) - sess.Out.Debug("Threads for repository gathering: %d\n", threadNum) - for i := 0; i < threadNum; i++ { - go func() { - for { - target, ok := <-ch - if !ok { - wg.Done() - return - } - repos, err := core.GetRepositoriesFromOwner(target.Login, sess.GithubClient, sess.Options.IncludeBranches) - if err != nil { - sess.Out.Error(" Failed to retrieve repositories from %s: %s\n", *target.Login, err) - } - if len(repos) == 0 { - continue - } - for _, repo := range repos { - sess.Out.Debug(" Retrieved repository: %s\n", *repo.FullName) - sess.AddRepository(repo) - sess.AddBranches(repo) - } - sess.Stats.IncrementTargets() - sess.Out.Info(" Retrieved %d %s from %s\n", len(repos), core.Pluralize(len(repos), "repository", "repositories"), *target.Login) - } - }() - } - - for _, target := range sess.Targets { - ch <- target - } - close(ch) - wg.Wait() + var ch = make(chan *core.GithubOwner, len(sess.Targets)) + var wg sync.WaitGroup + var threadNum int + if len(sess.Targets) == 1 { + threadNum = 1 + } else if len(sess.Targets) <= *sess.Options.Threads { + threadNum = len(sess.Targets) - 1 + } else { + threadNum = *sess.Options.Threads + } + wg.Add(threadNum) + sess.Out.Debug("Threads for repository gathering: %d\n", threadNum) + for i := 0; i < threadNum; i++ { + go func() { + for { + target, ok := <-ch + if !ok { + wg.Done() + return + } + repos, err := core.GetRepositoriesFromOwner(target.Login, sess.GithubClient, sess.Options.IncludeBranches) + if err != nil { + sess.Out.Error(" Failed to retrieve repositories from %s: %s\n", *target.Login, err) + } + if len(repos) == 0 { + continue + } + for _, repo := range repos { + sess.Out.Debug(" Retrieved repository: %s\n", *repo.FullName) + sess.AddRepository(repo) + sess.AddBranches(repo) + } + sess.Stats.IncrementTargets() + sess.Out.Info(" Retrieved %d %s from %s\n", len(repos), core.Pluralize(len(repos), "repository", "repositories"), *target.Login) + } + }() + } + + for _, target := range sess.Targets { + ch <- target + } + close(ch) + wg.Wait() } func AnalyzeRepositories(sess *core.Session) { - sess.Stats.Status = core.StatusAnalyzing - var ch = make(chan *core.GithubRepository, len(sess.Repositories)) - var wg sync.WaitGroup - var threadNum int - if len(sess.Repositories) <= 1 { - threadNum = 1 - } else if len(sess.Repositories) <= *sess.Options.Threads { - threadNum = len(sess.Repositories) - 1 - } else { - threadNum = *sess.Options.Threads - } - wg.Add(threadNum) - sess.Out.Debug("Threads for repository analysis: %d\n", threadNum) - - sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "branch", "branches")) - - for i := 0; i < threadNum; i++ { - go func(tid int) { - for { - sess.Out.Debug("[THREAD #%d] Requesting new repository to analyze...\n", tid) - repo, ok := <-ch - if !ok { - sess.Out.Debug("[THREAD #%d] No more tasks, marking WaitGroup as done\n", tid) - wg.Done() - return - } - - sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", tid, *repo.FullName) - clone, path, err := core.CloneRepository(repo.CloneURL, repo.BranchName, *sess.Options.CommitDepth) - if err != nil { - if err.Error() != "remote repository is empty" { - sess.Out.Error("Error cloning repository %s: %s\n", *repo.FullName, err) - } - sess.Stats.IncrementRepositories() - sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) - continue - } - sess.Out.Debug("[THREAD #%d][%s] Cloned repository to: %s\n", tid, *repo.FullName, path) - - history, err := core.GetRepositoryHistory(clone) - if err != nil { - sess.Out.Error("[THREAD #%d][%s] Error getting commit history: %s\n", tid, *repo.FullName, err) - os.RemoveAll(path) - sess.Stats.IncrementRepositories() - sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) - continue - } - sess.Out.Debug("[THREAD #%d][%s] Number of commits: %d\n", tid, *repo.FullName, len(history)) - - for _, commit := range history { - sess.Out.Debug("[THREAD #%d][%s] Analyzing commit: %s\n", tid, *repo.FullName, commit.Hash) - changes, _ := core.GetChanges(commit, clone) - sess.Out.Debug("[THREAD #%d][%s] Changes in %s: %d\n", tid, *repo.FullName, commit.Hash, len(changes)) - for _, change := range changes { - changeAction := core.GetChangeAction(change) - path := core.GetChangePath(change) - matchFile := core.NewMatchFile(path) - if matchFile.IsSkippable() { - sess.Out.Debug("[THREAD #%d][%s] Skipping %s\n", tid, *repo.FullName, matchFile.Path) - continue - } - sess.Out.Debug("[THREAD #%d][%s] Matching: %s...\n", tid, *repo.FullName, matchFile.Path) - for _, signature := range core.Signatures { - if signature.Match(matchFile) { - - finding := &core.Finding{ - FilePath: path, - Action: changeAction, - Description: signature.Description(), - Comment: signature.Comment(), - RepositoryOwner: *repo.Owner, - RepositoryName: *repo.Name, - BranchName: *repo.BranchName, - CommitHash: commit.Hash.String(), - CommitMessage: strings.TrimSpace(commit.Message), - CommitAuthor: commit.Author.String(), - } - finding.Initialize() - sess.AddFinding(finding) - - sess.Out.Warn(" %s: %s\n", strings.ToUpper(changeAction), finding.Description) - sess.Out.Info(" Path.......: %s\n", finding.FilePath) - sess.Out.Info(" Repo.......: %s\n", *repo.FullName) - sess.Out.Info(" Branch.....: %s\n", *repo.BranchName) - sess.Out.Info(" Message....: %s\n", core.TruncateString(finding.CommitMessage, 100)) - sess.Out.Info(" Author.....: %s\n", finding.CommitAuthor) - if finding.Comment != "" { - sess.Out.Info(" Comment....: %s\n", finding.Comment) - } - sess.Out.Info(" File URL...: %s\n", finding.FileUrl) - sess.Out.Info(" Commit URL.: %s\n", finding.CommitUrl) - sess.Out.Info(" ------------------------------------------------\n\n") - sess.Stats.IncrementFindings() - break - } - } - sess.Stats.IncrementFiles() - } - sess.Stats.IncrementCommits() - sess.Out.Debug("[THREAD #%d][%s] Done analyzing changes in %s\n", tid, *repo.FullName, commit.Hash) - } - sess.Out.Debug("[THREAD #%d][%s] Done analyzing commits\n", tid, *repo.FullName) - os.RemoveAll(path) - sess.Out.Debug("[THREAD #%d][%s] Deleted %s\n", tid, *repo.FullName, path) - sess.Stats.IncrementRepositories() - sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) - } - }(i) - } - for _, repo := range sess.Repositories { - ch <- repo - } - close(ch) - wg.Wait() + sess.Stats.Status = core.StatusAnalyzing + var ch = make(chan *core.GithubRepository, len(sess.Repositories)) + var wg sync.WaitGroup + var threadNum int + if len(sess.Repositories) <= 1 { + threadNum = 1 + } else if len(sess.Repositories) <= *sess.Options.Threads { + threadNum = len(sess.Repositories) - 1 + } else { + threadNum = *sess.Options.Threads + } + wg.Add(threadNum) + sess.Out.Debug("Threads for repository analysis: %d\n", threadNum) + + sess.Out.Important("Analyzing %d %s...\n", len(sess.Repositories), core.Pluralize(len(sess.Repositories), "branch", "branches")) + + for i := 0; i < threadNum; i++ { + go func(tid int) { + for { + sess.Out.Debug("[THREAD #%d] Requesting new repository to analyze...\n", tid) + repo, ok := <-ch + if !ok { + sess.Out.Debug("[THREAD #%d] No more tasks, marking WaitGroup as done\n", tid) + wg.Done() + return + } + + sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", tid, *repo.FullName) + clone, path, err := core.CloneRepository(repo.CloneURL, repo.BranchName, *sess.Options.CommitDepth) + if err != nil { + if err.Error() != "remote repository is empty" { + sess.Out.Error("Error cloning repository %s: %s\n", *repo.FullName, err) + } + sess.Stats.IncrementRepositories() + sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) + continue + } + sess.Out.Debug("[THREAD #%d][%s] Cloned repository to: %s\n", tid, *repo.FullName, path) + + history, err := core.GetRepositoryHistory(clone) + if err != nil { + sess.Out.Error("[THREAD #%d][%s] Error getting commit history: %s\n", tid, *repo.FullName, err) + os.RemoveAll(path) + sess.Stats.IncrementRepositories() + sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) + continue + } + sess.Out.Debug("[THREAD #%d][%s] Number of commits: %d\n", tid, *repo.FullName, len(history)) + + for _, commit := range history { + sess.Out.Debug("[THREAD #%d][%s] Analyzing commit: %s\n", tid, *repo.FullName, commit.Hash) + changes, _ := core.GetChanges(commit, clone) + sess.Out.Debug("[THREAD #%d][%s] Changes in %s: %d\n", tid, *repo.FullName, commit.Hash, len(changes)) + for _, change := range changes { + changeAction := core.GetChangeAction(change) + path := core.GetChangePath(change) + matchFile := core.NewMatchFile(path) + if matchFile.IsSkippable() { + sess.Out.Debug("[THREAD #%d][%s] Skipping %s\n", tid, *repo.FullName, matchFile.Path) + continue + } + sess.Out.Debug("[THREAD #%d][%s] Matching: %s...\n", tid, *repo.FullName, matchFile.Path) + for _, signature := range core.Signatures { + if signature.Match(matchFile) { + + finding := &core.Finding{ + FilePath: path, + Action: changeAction, + Description: signature.Description(), + Comment: signature.Comment(), + RepositoryOwner: *repo.Owner, + RepositoryName: *repo.Name, + BranchName: *repo.BranchName, + CommitHash: commit.Hash.String(), + CommitMessage: strings.TrimSpace(commit.Message), + CommitAuthor: commit.Author.String(), + } + finding.Initialize() + sess.AddFinding(finding) + + sess.Out.Warn(" %s: %s\n", strings.ToUpper(changeAction), finding.Description) + sess.Out.Info(" Path.......: %s\n", finding.FilePath) + sess.Out.Info(" Repo.......: %s\n", *repo.FullName) + sess.Out.Info(" Branch.....: %s\n", *repo.BranchName) + sess.Out.Info(" Message....: %s\n", core.TruncateString(finding.CommitMessage, 100)) + sess.Out.Info(" Author.....: %s\n", finding.CommitAuthor) + if finding.Comment != "" { + sess.Out.Info(" Comment....: %s\n", finding.Comment) + } + sess.Out.Info(" File URL...: %s\n", finding.FileUrl) + sess.Out.Info(" Commit URL.: %s\n", finding.CommitUrl) + sess.Out.Info(" ------------------------------------------------\n\n") + sess.Stats.IncrementFindings() + break + } + } + sess.Stats.IncrementFiles() + } + sess.Stats.IncrementCommits() + sess.Out.Debug("[THREAD #%d][%s] Done analyzing changes in %s\n", tid, *repo.FullName, commit.Hash) + } + sess.Out.Debug("[THREAD #%d][%s] Done analyzing commits\n", tid, *repo.FullName) + os.RemoveAll(path) + sess.Out.Debug("[THREAD #%d][%s] Deleted %s\n", tid, *repo.FullName, path) + sess.Stats.IncrementRepositories() + sess.Stats.UpdateProgress(sess.Stats.Repositories, len(sess.Repositories)) + } + }(i) + } + for _, repo := range sess.Repositories { + ch <- repo + } + close(ch) + wg.Wait() } func PrintSessionStats(sess *core.Session) { - sess.Out.Info("\nFindings....: %d\n", sess.Stats.Findings) - sess.Out.Info("Files.......: %d\n", sess.Stats.Files) - sess.Out.Info("Commits.....: %d\n", sess.Stats.Commits) - sess.Out.Info("Repositories: %d\n", sess.Stats.Repositories) - sess.Out.Info("Targets.....: %d\n\n", sess.Stats.Targets) + sess.Out.Info("\nFindings....: %d\n", sess.Stats.Findings) + sess.Out.Info("Files.......: %d\n", sess.Stats.Files) + sess.Out.Info("Commits.....: %d\n", sess.Stats.Commits) + sess.Out.Info("Repositories: %d\n", sess.Stats.Repositories) + sess.Out.Info("Targets.....: %d\n\n", sess.Stats.Targets) } func main() { - if sess, err = core.NewSession(); err != nil { - fmt.Println(err) - os.Exit(1) - } - - sess.Out.Info("%s\n\n", core.ASCIIBanner) - sess.Out.Important("%s v%s started at %s\n", core.Name, core.Version, sess.Stats.StartedAt.Format(time.RFC3339)) - sess.Out.Important("Loaded %d signatures\n", len(core.Signatures)) - sess.Out.Important("Web interface available at http://%s:%d\n", *sess.Options.BindAddress, *sess.Options.Port) - - if sess.Stats.Status == "finished" { - sess.Out.Important("Loaded session file: %s\n", *sess.Options.Load) - } else { - if len(sess.Options.Logins) == 0 { - sess.Out.Fatal("Please provide at least one GitHub organization or user\n") - } - - GatherTargets(sess) - GatherRepositories(sess) - AnalyzeRepositories(sess) - sess.Finish() - - if *sess.Options.Save != "" { - err := sess.SaveToFile(*sess.Options.Save) - if err != nil { - sess.Out.Error("Error saving session to %s: %s\n", *sess.Options.Save, err) - } - sess.Out.Important("Saved session to: %s\n\n", *sess.Options.Save) - } - } - - PrintSessionStats(sess) - sess.Out.Important("Press Ctrl+C to stop web server and exit.\n\n") - select {} + if sess, err = core.NewSession(); err != nil { + fmt.Println(err) + os.Exit(1) + } + + sess.Out.Info("%s\n\n", core.ASCIIBanner) + sess.Out.Important("%s v%s started at %s\n", core.Name, core.Version, sess.Stats.StartedAt.Format(time.RFC3339)) + sess.Out.Important("Loaded %d signatures\n", len(core.Signatures)) + sess.Out.Important("Web interface available at http://%s:%d\n", *sess.Options.BindAddress, *sess.Options.Port) + + if sess.Stats.Status == "finished" { + sess.Out.Important("Loaded session file: %s\n", *sess.Options.Load) + } else { + if len(sess.Options.Logins) == 0 { + sess.Out.Fatal("Please provide at least one GitHub organization or user\n") + } + + GatherTargets(sess) + GatherRepositories(sess) + AnalyzeRepositories(sess) + sess.Finish() + + if *sess.Options.Save != "" { + err := sess.SaveToFile(*sess.Options.Save) + if err != nil { + sess.Out.Error("Error saving session to %s: %s\n", *sess.Options.Save, err) + } + sess.Out.Important("Saved session to: %s\n\n", *sess.Options.Save) + } + } + + PrintSessionStats(sess) + sess.Out.Important("Press Ctrl+C to stop web server and exit.\n\n") + select {} }