Enable cloning repository to a specified location with retention option (trufflesecurity#4408)

* Enabled cloning repositories to a specified path with retention option

* Fixes after testing

* resolved lint issue

* resolved comments

* enabled clone path for github basic auth

Author: kashifkhan0771

hack/snifftest/main.go

@@ -181,7 +181,7 @@ func main() {
 				defer sem.Release(1)
 				defer wgChunkers.Done()
 				logger.Info("cloning repo", "repo", r)
-				path, repo, err := git.CloneRepoUsingUnauthenticated(ctx, r)
+				path, repo, err := git.CloneRepoUsingUnauthenticated(ctx, r, "")
 				if err != nil {
 					logFatal(err, "error cloning repo", "repo", r)
 				}

main.go

@@ -99,6 +99,8 @@ var (
 	gitScanBranch       = gitScan.Flag("branch", "Branch to scan.").String()
 	gitScanMaxDepth     = gitScan.Flag("max-depth", "Maximum depth of commits to scan.").Int()
 	gitScanBare         = gitScan.Flag("bare", "Scan bare repository (e.g. useful while using in pre-receive hooks)").Bool()
+	gitClonePath        = gitScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String()
+	gitNoCleanup        = gitScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool()
 	_                   = gitScan.Flag("allow", "No-op flag for backwards compat.").Bool()
 	_                   = gitScan.Flag("entropy", "No-op flag for backwards compat.").Bool()
 	_                   = gitScan.Flag("regex", "No-op flag for backwards compat.").Bool()
@@ -120,6 +122,8 @@ var (
 	githubScanGistComments      = githubScan.Flag("gist-comments", "Include gist comments in scan.").Bool()
 	githubCommentsTimeframeDays = githubScan.Flag("comments-timeframe", "Number of days in the past to review when scanning issue, PR, and gist comments.").Uint32()
 	githubAuthInUrl             = githubScan.Flag("auth-in-url", "Embed authentication credentials in repository URLs instead of using secure HTTP headers").Bool()
+	githubClonePath             = githubScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir).").String()
+	githubNoCleanup             = githubScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool()
 
 	// GitHub Cross Fork Object Reference Experimental Feature
 	githubExperimentalScan = cli.Command("github-experimental", "Run an experimental GitHub scan. Must specify at least one experimental sub-module to run: object-discovery.")
@@ -142,6 +146,8 @@ var (
 	gitlabScanIncludeRepos = gitlabScan.Flag("include-repos", `Repositories to include in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/trufflehog", "trufflesecurity/t*"`).Strings()
 	gitlabScanExcludeRepos = gitlabScan.Flag("exclude-repos", `Repositories to exclude in an org scan. This can also be a glob pattern. You can repeat this flag. Must use Gitlab repo full name. Example: "trufflesecurity/driftwood", "trufflesecurity/d*"`).Strings()
 	gitlabAuthInUrl        = gitlabScan.Flag("auth-in-url", "Embed authentication credentials in repository URLs instead of using secure HTTP headers").Bool()
+	gitlabClonePath        = gitlabScan.Flag("clone-path", "Custom path where the repository should be cloned (default: temp dir)").String()
+	gitlabNoCleanup        = gitlabScan.Flag("no-cleanup", "Do not delete cloned repositories after scanning (can only be used with --clone-path).").Bool()
 
 	filesystemScan  = cli.Command("filesystem", "Find credentials in a filesystem.")
 	filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings()
@@ -719,6 +725,15 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			}
 		}
 
+		// clone path is only supported for HTTPS repository URLs, not for local repositories.
+		if *gitClonePath != "" && strings.HasPrefix(*gitScanURI, "file://") {
+			return scanMetrics, fmt.Errorf("invalid configuration: --clone-path cannot be used with a local repository URL")
+		}
+
+		if err := validateClonePath(*gitClonePath, *gitNoCleanup); err != nil {
+			return scanMetrics, err
+		}
+
 		gitCfg := sources.GitConfig{
 			URI:              *gitScanURI,
 			IncludePathsFile: *gitScanIncludePaths,
@@ -728,6 +743,8 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			MaxDepth:         *gitScanMaxDepth,
 			Bare:             *gitScanBare,
 			ExcludeGlobs:     *gitScanExcludeGlobs,
+			ClonePath:        *gitClonePath,
+			NoCleanup:        *gitNoCleanup,
 		}
 		if ref, err := eng.ScanGit(ctx, gitCfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Git: %v", err)
@@ -746,6 +763,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			return scanMetrics, fmt.Errorf("invalid config: you cannot specify both organizations and repositories at the same time")
 		}
 
+		if err := validateClonePath(*githubClonePath, *githubNoCleanup); err != nil {
+			return scanMetrics, err
+		}
+
 		cfg := sources.GithubConfig{
 			Endpoint:                   *githubScanEndpoint,
 			Token:                      *githubScanToken,
@@ -763,7 +784,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			CommentsTimeframeDays:      *githubCommentsTimeframeDays,
 			Filter:                     filter,
 			AuthInUrl:                  *githubAuthInUrl,
+			ClonePath:                  *githubClonePath,
+			NoCleanup:                  *githubNoCleanup,
 		}
+
 		if ref, err := eng.ScanGitHub(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan Github: %v", err)
 		} else {
@@ -792,6 +816,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			return scanMetrics, fmt.Errorf("invalid config: you cannot specify both repositories and groups at the same time")
 		}
 
+		if err := validateClonePath(*gitlabClonePath, *gitlabNoCleanup); err != nil {
+			return scanMetrics, err
+		}
+
 		cfg := sources.GitlabConfig{
 			Endpoint:     *gitlabScanEndpoint,
 			Token:        *gitlabScanToken,
@@ -801,7 +829,10 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
 			ExcludeRepos: *gitlabScanExcludeRepos,
 			Filter:       filter,
 			AuthInUrl:    *gitlabAuthInUrl,
+			ClonePath:    *gitlabClonePath,
+			NoCleanup:    *gitlabNoCleanup,
 		}
+
 		if ref, err := eng.ScanGitLab(ctx, cfg); err != nil {
 			return scanMetrics, fmt.Errorf("failed to scan GitLab: %v", err)
 		} else {
@@ -1120,3 +1151,31 @@ func isValidCommit(uri, commit string) bool {
 
 	return strings.TrimSpace(string(output)) == "commit"
 }
+
+// validateClonePath ensures that --clone-path, if provided, exists and is a directory.
+// It also verifies that --no-cleanup is only allowed when --clone-path is set.
+// Note: without a custom clone path, repositories are cloned into temporary directories, which should never be retained.
+func validateClonePath(clonePath string, noCleanup bool) error {
+	if noCleanup && clonePath == "" {
+		return fmt.Errorf("invalid configuration: --no-cleanup can only be used together with --clone-path")
+	}
+
+	if clonePath == "" {
+		return nil
+	}
+
+	info, err := os.Stat(clonePath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("path provided to --clone-path: %q does not exist", clonePath)
+		}
+
+		return fmt.Errorf("failed to access --clone-path %q: %w", clonePath, err)
+	}
+
+	if !info.IsDir() {
+		return fmt.Errorf("path provided to --clone-path: %q is not a directory", clonePath)
+	}
+
+	return nil
+}

pkg/engine/git.go

@@ -24,7 +24,10 @@ func (e *Engine) ScanGit(ctx context.Context, c sources.GitConfig) (sources.JobP
 		ExcludePathsFile: c.ExcludePathsFile,
 		MaxDepth:         int64(c.MaxDepth),
 		SkipBinaries:     c.SkipBinaries,
+		ClonePath:        c.ClonePath,
+		NoCleanup:        c.NoCleanup,
 	}
+
 	var conn anypb.Any
 	if err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{}); err != nil {
 		ctx.Logger().Error(err, "failed to marshal git connection")

pkg/engine/git_test.go

@@ -32,7 +32,7 @@ func (p *discardPrinter) Print(context.Context, *detectors.ResultWithMetadata) e
 func TestGitEngine(t *testing.T) {
 	ctx := context.Background()
 	repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
-	path, _, err := git.PrepareRepo(ctx, repoUrl)
+	path, _, err := git.PrepareRepo(ctx, repoUrl, "")
 	if err != nil {
 		t.Error(err)
 	}

pkg/engine/github.go

@@ -29,7 +29,10 @@ func (e *Engine) ScanGitHub(ctx context.Context, c sources.GithubConfig) (source
 		SkipBinaries:               c.SkipBinaries,
 		CommentsTimeframeDays:      c.CommentsTimeframeDays,
 		RemoveAuthInUrl:            !c.AuthInUrl, // configuration uses the opposite field in proto to keep credentials in the URL by default.
+		ClonePath:                  c.ClonePath,
+		NoCleanup:                  c.NoCleanup,
 	}
+
 	if len(c.Token) > 0 {
 		connection.Credential = &sourcespb.GitHub_Token{
 			Token: c.Token,

pkg/engine/gitlab.go

@@ -58,6 +58,12 @@ func (e *Engine) ScanGitLab(ctx context.Context, c sources.GitlabConfig) (source
 		connection.IgnoreRepos = c.ExcludeRepos
 	}
 
+	if c.ClonePath != "" {
+		connection.ClonePath = c.ClonePath
+	}
+
+	connection.NoCleanup = c.NoCleanup
+
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
 	if err != nil {

pkg/output/legacy_json.go

@@ -38,7 +38,7 @@ func (p *LegacyJSONPrinter) Print(ctx context.Context, r *detectors.ResultWithMe
 	}
 
 	// cloning the repo again here is not great and only works with unauthed repos
-	repoPath, remote, err := git.PrepareRepo(ctx, repo)
+	repoPath, remote, err := git.PrepareRepo(ctx, repo, "")
 	if err != nil || repoPath == "" {
 		return fmt.Errorf("error preparing git repo for scanning: %w", err)
 	}