Modified Gitrob scan to include all GitHub organization private repositories.

Author: PaulTurner-awin

common/interfaces.go

@@ -3,5 +3,6 @@ package common
 type IClient interface {
 	GetUserOrOrganization(login string) (*Owner, error)
 	GetRepositoriesFromOwner(target Owner) ([]*Repository, error)
+	GetRepositoriesFromOrganization(target Owner) ([]*Repository, error)
 	GetOrganizationMembers(target Owner) ([]*Owner, error)
 }

core/analysis.go

@@ -64,12 +64,19 @@ func GatherRepositories(sess *Session) {
 	for i := 0; i < threadNum; i++ {
 		go func() {
 			for {
+				var repos []*common.Repository
+				var err error
 				target, ok := <-ch
 				if !ok {
 					wg.Done()
 					return
 				}
-				repos, err := sess.Client.GetRepositoriesFromOwner(*target)
+				//repos, err := sess.Client.GetRepositoriesFromOwner(*target)
+				if *target.Type == "Organization" {
+					repos, err = sess.Client.GetRepositoriesFromOrganization(*target)
+				} else {
+					repos, err = sess.Client.GetRepositoriesFromOwner(*target)
+				}
 				if err != nil {
 					sess.Out.Error(" Failed to retrieve repositories from %s: %s\n", *target.Login, err)
 				}
@@ -216,23 +223,25 @@ func findSecrets(sess *Session, repo *common.Repository, commit *object.Commit,
 func cloneRepository(sess *Session, repo *common.Repository, threadId int) (*git.Repository, string, error) {
 	sess.Out.Debug("[THREAD #%d][%s] Cloning repository...\n", threadId, *repo.CloneURL)
 
+	userName := "oauth2"
+
 	cloneConfig := common.CloneConfiguration{
 		Url:        repo.CloneURL,
 		Branch:     repo.DefaultBranch,
 		Depth:      sess.Options.CommitDepth,
-		Token:      &sess.GitLab.AccessToken,
 		InMemClone: sess.Options.InMemClone,
+		Username: &userName,
 	}
 
 	var clone *git.Repository
 	var path string
 	var err error
 
 	if sess.IsGithubSession {
+		cloneConfig.Token = &sess.Github.AccessToken
 		clone, path, err = github.CloneRepository(&cloneConfig)
 	} else {
-		userName := "oauth2"
-		cloneConfig.Username = &userName
+		cloneConfig.Token = &sess.GitLab.AccessToken
 		clone, path, err = gitlab.CloneRepository(&cloneConfig)
 	}
 	if err != nil {

github/apiClient.go

@@ -81,6 +81,43 @@ func (c Client) GetRepositoriesFromOwner(target common.Owner) ([]*common.Reposit
 	return allRepos, nil
 }
 
+func (c Client) GetRepositoriesFromOrganization(target common.Owner) ([]*common.Repository, error) {
+	var allRepos []*common.Repository
+	ctx := context.Background()
+	opt := &github.RepositoryListByOrgOptions{
+		Type: "sources",
+	}
+
+	for {
+		repos, resp, err := c.apiClient.Repositories.ListByOrg(ctx, *target.Login, opt)
+		if err != nil {
+			return allRepos, err
+		}
+		for _, repo := range repos {
+			if !*repo.Fork {
+				r := common.Repository{
+					Owner:         repo.Owner.Login,
+					ID:            repo.ID,
+					Name:          repo.Name,
+					FullName:      repo.FullName,
+					CloneURL:      repo.SSHURL,
+					URL:           repo.HTMLURL,
+					DefaultBranch: repo.DefaultBranch,
+					Description:   repo.Description,
+					Homepage:      repo.Homepage,
+				}
+				allRepos = append(allRepos, &r)
+			}
+		}
+		if resp.NextPage == 0 {
+			break
+		}
+		opt.Page = resp.NextPage
+	}
+
+	return allRepos, nil
+}
+
 func (c Client) GetOrganizationMembers(target common.Owner) ([]*common.Owner, error) {
 	var allMembers []*common.Owner
 	ctx := context.Background()

github/git.go

@@ -2,13 +2,15 @@ package github
 
 import (
 	"fmt"
+	"strings"
 	"gopkg.in/src-d/go-git.v4/storage/memory"
 	"io/ioutil"
 
 	"gitrob/common"
 
 	"gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/plumbing"
+	"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
 )
 
 func CloneRepository(cloneConfig *common.CloneConfiguration) (*git.Repository, string, error) {
@@ -19,8 +21,15 @@ func CloneRepository(cloneConfig *common.CloneConfiguration) (*git.Repository, s
 		ReferenceName: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", *cloneConfig.Branch)),
 		SingleBranch:  true,
 		Tags:          git.NoTags,
+		Auth: &http.BasicAuth{
+			Username: *cloneConfig.Username,
+			Password: *cloneConfig.Token,
+		},
 	}
 
+	cloneOptions.URL = strings.Replace(cloneOptions.URL, "[email protected]:", "https://github.com/", -1)
+
+
 	var repository *git.Repository
 	var err error
 	var dir string

gitlab/apiClient.go

@@ -112,6 +112,29 @@ func (c Client) GetRepositoriesFromOwner(target common.Owner) ([]*common.Reposit
 	return allProjects, nil
 }
 
+func (c Client) GetRepositoriesFromOrganization(target common.Owner) ([]*common.Repository, error) {
+        var allProjects []*common.Repository
+        id := int(*target.ID)
+        if *target.Type == common.TargetTypeUser {
+                userProjects, err := c.getUserProjects(id)
+                if err != nil {
+                        return nil, err
+                }
+                for _, project := range userProjects {
+                        allProjects = append(allProjects, project)
+                }
+        } else {
+                groupProjects, err := c.getGroupProjects(target)
+                if err != nil {
+                        return nil, err
+                }
+                for _, project := range groupProjects {
+                        allProjects = append(allProjects, project)
+                }
+        }
+        return allProjects, nil
+}
+
 func (c Client) getUser(login string) (*gitlab.User, error) {
 	users, _, err := c.apiClient.Users.ListUsers(&gitlab.ListUsersOptions{Username: gitlab.String(login)})
 	if err != nil {