From a927028b9bc2661540a7c17797f54ce4a8d86ba3 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 31 Jan 2024 11:21:39 +0530 Subject: [PATCH 01/24] code refactoring --- api/GrpcHandler.go | 1 + internal/Configuration.go | 5 +- pkg/RepoManages.go | 20 ++++++-- pkg/git/GitBaseManager.go | 4 +- pkg/git/GitContextUtils.go | 5 ++ pkg/git/RepositoryManager.go | 73 ++++++++++++++++++++++----- pkg/git/RepositoryManagerAnalytics.go | 7 +-- pkg/git/Util.go | 35 ++++++------- pkg/git/Watcher.go | 24 ++++++--- 9 files changed, 119 insertions(+), 55 deletions(-) diff --git a/api/GrpcHandler.go b/api/GrpcHandler.go index 43b7bbe7..8fac0b63 100644 --- a/api/GrpcHandler.go +++ b/api/GrpcHandler.go @@ -355,6 +355,7 @@ func (impl *GrpcHandlerImpl) GetCommitMetadataForPipelineMaterial(ctx context.Co return nil, err } + if res == nil { res1 := &pb.GitCommit{} return res1, nil diff --git a/internal/Configuration.go b/internal/Configuration.go index 997cb557..e72b830d 100644 --- a/internal/Configuration.go +++ b/internal/Configuration.go @@ -1,13 +1,12 @@ package internal -import ( - "github.com/caarlos0/env" -) +import "github.com/caarlos0/env" type Configuration struct { CommitStatsTimeoutInSec int `env:"COMMIT_STATS_TIMEOUT_IN_SEC" envDefault:"2"` EnableFileStats bool `env:"ENABLE_FILE_STATS" envDefault:"false"` GitHistoryCount int `env:"GIT_HISTORY_COUNT" envDefault:"15"` + CloningMode string `env:"CLONING_MODE" envDefault:"FULL"` MinLimit int `env:"MIN_LIMIT_FOR_PVC" envDefault:"1"` // in MB UseGitCli bool `env:"USE_GIT_CLI" envDefault:"false"` AnalyticsDebug bool `env:"ANALYTICS_DEBUG" envDefault:"false"` diff --git a/pkg/RepoManages.go b/pkg/RepoManages.go index 7a70a702..b48a3f3a 100644 --- a/pkg/RepoManages.go +++ b/pkg/RepoManages.go @@ -195,6 +195,9 @@ func (impl RepoManagerImpl) updatePipelineMaterialCommit(gitCtx git.GitContext, continue } + gitCtx = gitCtx.WithCredentials(material.GitProvider.UserName, material.GitProvider.Password). + WithCloningMode(impl.configuration.CloningMode) + commits, err := impl.repositoryManager.ChangesSince(gitCtx, material.CheckoutLocation, pipelineMaterial.Value, "", "", impl.configuration.GitHistoryCount) //commits, err := impl.FetchChanges(pipelineMaterial.Id, "", "", 0) if err == nil { @@ -345,11 +348,14 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq if err != nil { return material, nil } - checkoutPath, err := git.GetLocationForMaterial(material) + + gitCtx = gitCtx.WithCredentials(userName, password). + WithCloningMode(impl.configuration.CloningMode) + + checkoutPath, err := impl.repositoryManager.GetLocationForMaterial(material) if err != nil { return material, err } - gitCtx = gitCtx.WithCredentials(userName, password) err = impl.repositoryManager.Add(gitCtx, material.GitProviderId, checkoutPath, material.Url, gitProvider.AuthMode, gitProvider.SshPrivateKey) if err == nil { @@ -627,7 +633,10 @@ func (impl RepoManagerImpl) GetLatestCommitForBranch(gitCtx git.GitContext, pipe }() userName, password, err := git.GetUserNamePassword(gitMaterial.GitProvider) - gitCtx = gitCtx.WithCredentials(userName, password) + + gitCtx = gitCtx.WithCredentials(userName, password). + WithCloningMode(impl.configuration.CloningMode) + updated, repo, err := impl.repositoryManager.Fetch(gitCtx, gitMaterial.Url, gitMaterial.CheckoutLocation) if !updated { impl.logger.Warn("repository is up to date") @@ -659,7 +668,7 @@ func (impl RepoManagerImpl) GetLatestCommitForBranch(gitCtx git.GitContext, pipe return nil, err } - commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, branchName, "", "", 1) + commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, branchName, "", "", 1, gitMaterial.CheckoutLocation) if commits == nil { return nil, err @@ -694,6 +703,8 @@ func (impl RepoManagerImpl) GetCommitMetadataForPipelineMaterial(gitCtx git.GitC return nil, err } + gitCtx = gitCtx.WithCredentials(gitMaterial.GitProvider.UserName, gitMaterial.GitProvider.Password). + WithCloningMode(impl.configuration.CloningMode) // validate checkout status of gitMaterial if !gitMaterial.CheckoutStatus { impl.logger.Errorw("checkout not success", "gitMaterialId", gitMaterialId) @@ -707,7 +718,6 @@ func (impl RepoManagerImpl) GetCommitMetadataForPipelineMaterial(gitCtx git.GitC repoLock.Mutex.Unlock() impl.locker.ReturnLocker(gitMaterial.Id) }() - commits, err := impl.repositoryManager.ChangesSince(gitCtx, gitMaterial.CheckoutLocation, branchName, "", gitHash, 1) if err != nil { impl.logger.Errorw("error while fetching commit info", "pipelineMaterialId", pipelineMaterialId, "gitHash", gitHash, "err", err) diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index 59fa1af9..409c560e 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -29,8 +29,6 @@ type GitManager interface { OpenRepoPlain(checkoutPath string) (*GitRepository, error) // Init initializes a git repo Init(gitCtx GitContext, rootDir string, remoteUrl string, isBare bool) error - // FetchDiffStatBetweenCommits returns the file stats reponse on executing git action - FetchDiffStatBetweenCommits(gitCtx GitContext, oldHash string, newHash string, rootDir string) (response, errMsg string, err error) } // GitManagerBase Base methods which will be available to all implementation of the parent interface @@ -43,6 +41,8 @@ type GitManagerBase interface { Checkout(gitCtx GitContext, rootDir, branch string) (response, errMsg string, err error) // ConfigureSshCommand configures ssh in git repo ConfigureSshCommand(gitCtx GitContext, rootDir string, sshPrivateKeyPath string) (response, errMsg string, err error) + // FetchDiffStatBetweenCommits returns the file stats reponse on executing git action + FetchDiffStatBetweenCommits(gitCtx GitContext, oldHash string, newHash string, rootDir string) (response, errMsg string, err error) // LogMergeBase get the commit diff between using a merge base strategy LogMergeBase(gitCtx GitContext, rootDir, from string, to string) ([]*Commit, error) } diff --git a/pkg/git/GitContextUtils.go b/pkg/git/GitContextUtils.go index 13b179aa..b1476a6d 100644 --- a/pkg/git/GitContextUtils.go +++ b/pkg/git/GitContextUtils.go @@ -30,6 +30,11 @@ func (gitCtx GitContext) WithTimeout(timeoutSeconds int) (GitContext, context.Ca return gitCtx, cancel } +func (gitCtx GitContext) WithCloningMode(CloningMode string) GitContext { + gitCtx.CloningMode = CloningMode + return gitCtx +} + func RunWithTimeout[T any](ctx context.Context, f func() ([]*T, error)) ([]*T, error) { resultCh := make(chan []*T) errCh := make(chan error) diff --git a/pkg/git/RepositoryManager.go b/pkg/git/RepositoryManager.go index 266be686..553ae7f3 100644 --- a/pkg/git/RepositoryManager.go +++ b/pkg/git/RepositoryManager.go @@ -18,11 +18,15 @@ package git import ( "errors" + "fmt" "github.com/devtron-labs/git-sensor/internal" "github.com/devtron-labs/git-sensor/util" "golang.org/x/sys/unix" "io" "os" + "path" + "regexp" + "strconv" "strings" "time" @@ -37,18 +41,19 @@ type RepositoryManager interface { Fetch(gitCtx GitContext, url string, location string) (updated bool, repo *GitRepository, err error) // Add adds and initializes a new git repo , cleans the directory if not empty and fetches latest commits Add(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error + GetLocationForMaterial(material *sql.GitMaterial) (location string, err error) // Clean cleans a directory Clean(cloneDir string) error // ChangesSince given the checkput path, retrieves the latest commits for the gt repo existing on the path ChangesSince(gitCtx GitContext, checkoutPath string, branch string, from string, to string, count int) ([]*GitCommitBase, error) // ChangesSinceByRepository returns the latest commits list for the given range and count for an existing repo - ChangesSinceByRepository(gitCtx GitContext, repository *GitRepository, branch string, from string, to string, count int) ([]*GitCommitBase, error) + ChangesSinceByRepository(gitCtx GitContext, repository *GitRepository, branch string, from string, to string, count int, checkoutPath string) ([]*GitCommitBase, error) // GetCommitMetadata retrieves the commit metadata for given hash GetCommitMetadata(gitCtx GitContext, checkoutPath, commitHash string) (*GitCommitBase, error) // GetCommitForTag retrieves the commit metadata for given tag GetCommitForTag(gitCtx GitContext, checkoutPath, tag string) (*GitCommitBase, error) // CreateSshFileIfNotExistsAndConfigureSshCommand creates ssh file with creds and configures it at the location - CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx GitContext, location string, gitProviderId int, sshPrivateKeyContent string) error + CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx GitContext, location string, gitProviderId int, sshPrivateKeyContent string) (string, error) } type RepositoryManagerImpl struct { @@ -75,7 +80,35 @@ func (impl RepositoryManagerImpl) IsSpaceAvailableOnDisk() bool { return availableSpace > int64(impl.configuration.MinLimit)*1024*1024 } +func (impl RepositoryManagerImpl) GetLocationForMaterial(material *sql.GitMaterial) (location string, err error) { + //gitRegex := `/(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)(\/?|\#[-\d\w._]+?)$/` + httpsRegex := `^https.*` + httpsMatched, err := regexp.MatchString(httpsRegex, material.Url) + if httpsMatched { + locationWithoutProtocol := strings.ReplaceAll(material.Url, "https://", "") + checkoutPath := path.Join(GIT_BASE_DIR, strconv.Itoa(material.Id), locationWithoutProtocol) + return checkoutPath, nil + } + + sshRegex := `^git@.*` + sshMatched, err := regexp.MatchString(sshRegex, material.Url) + if sshMatched { + checkoutPath := path.Join(GIT_BASE_DIR, strconv.Itoa(material.Id), material.Url) + return checkoutPath, nil + } + + return "", fmt.Errorf("unsupported format url %s", material.Url) +} + func (impl RepositoryManagerImpl) Add(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error { + err := impl.Add1(gitCtx, gitProviderId, location, url, authMode, sshPrivateKeyContent) + if err != nil { + return err + } + return impl.Add2(gitCtx, location) +} + +func (impl RepositoryManagerImpl) Add1(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error { var err error start := time.Now() defer func() { @@ -95,18 +128,23 @@ func (impl RepositoryManagerImpl) Add(gitCtx GitContext, gitProviderId int, loca impl.logger.Errorw("err in git init", "err", err) return err } - + var sshPrivateKeyPath string // check ssh if authMode == sql.AUTH_MODE_SSH { - err = impl.CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx, location, gitProviderId, sshPrivateKeyContent) + sshPrivateKeyPath, err = impl.CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx, location, gitProviderId, sshPrivateKeyContent) if err != nil { + impl.logger.Errorw("error while creating ssh file for shallow clone", "checkoutPath", location, "sshPrivateKeyPath", sshPrivateKeyPath, "err", err) return err } } + return nil +} + +func (impl RepositoryManagerImpl) Add2(gitCtx GitContext, location string) error { opt, errorMsg, err := impl.gitManager.Fetch(gitCtx, location) if err != nil { - impl.logger.Errorw("error in cloning repo", "errorMsg", errorMsg, "err", err) + impl.logger.Errorw("error in fetching repo", "errorMsg", errorMsg, "err", err) return err } impl.logger.Debugw("opt msg", "opt", opt) @@ -186,7 +224,7 @@ func (impl RepositoryManagerImpl) GetCommitMetadata(gitCtx GitContext, checkoutP // from -> old commit // to -> new commit -func (impl RepositoryManagerImpl) ChangesSinceByRepository(gitCtx GitContext, repository *GitRepository, branch string, from string, to string, count int) ([]*GitCommitBase, error) { +func (impl RepositoryManagerImpl) ChangesSinceByRepository(gitCtx GitContext, repository *GitRepository, branch string, from string, to string, count int, checkoutPath string) ([]*GitCommitBase, error) { // fix for azure devops (manual trigger webhook bases pipeline) : // branch name comes as 'refs/heads/master', we need to extract actual branch name out of it. // https://stackoverflow.com/questions/59956206/how-to-get-a-branch-name-with-a-slash-in-azure-devops @@ -205,6 +243,7 @@ func (impl RepositoryManagerImpl) ChangesSinceByRepository(gitCtx GitContext, re ToCommitHash: to, }) if err != nil { + impl.logger.Errorw("error in getting iterator", "branch", branch, "err", err) return nil, err } var gitCommits []*GitCommitBase @@ -255,7 +294,7 @@ func (impl RepositoryManagerImpl) ChangesSinceByRepository(gitCtx GitContext, re if impl.configuration.EnableFileStats { defer func() { if err := recover(); err != nil { - impl.logger.Error("file stats function panicked for commit", "err", err, "commit", commit) + impl.logger.Error("file stats function panicked for commit", "err", err, "commit", commit, "count", count) } }() //TODO: implement below Stats() function using git CLI as it panics in some cases, remove defer function after using git CLI @@ -271,6 +310,13 @@ func (impl RepositoryManagerImpl) ChangesSinceByRepository(gitCtx GitContext, re return gitCommits, err } +func trimLastGitCommit(gitCommits []*GitCommitBase, count int) []*GitCommitBase { + if len(gitCommits) > count { + gitCommits = gitCommits[:len(gitCommits)-1] + } + return gitCommits +} + func (impl RepositoryManagerImpl) ChangesSince(gitCtx GitContext, checkoutPath string, branch string, from string, to string, count int) ([]*GitCommitBase, error) { var err error start := time.Now() @@ -285,30 +331,31 @@ func (impl RepositoryManagerImpl) ChangesSince(gitCtx GitContext, checkoutPath s return nil, err } ///--------------------- - return impl.ChangesSinceByRepository(gitCtx, r, branch, from, to, count) + return impl.ChangesSinceByRepository(gitCtx, r, branch, from, to, count, checkoutPath) ///---------------------- } -func (impl RepositoryManagerImpl) CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx GitContext, location string, gitProviderId int, sshPrivateKeyContent string) error { +func (impl RepositoryManagerImpl) CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx GitContext, location string, gitProviderId int, sshPrivateKeyContent string) (string, error) { // add private key var err error + var sshPrivateKeyPath string start := time.Now() defer func() { util.TriggerGitOperationMetrics("createSshFileIfNotExistsAndConfigureSshCommand", start, err) }() - sshPrivateKeyPath, err := GetOrCreateSshPrivateKeyOnDisk(gitProviderId, sshPrivateKeyContent) + sshPrivateKeyPath, err = GetOrCreateSshPrivateKeyOnDisk(gitProviderId, sshPrivateKeyContent) if err != nil { impl.logger.Errorw("error in creating ssh private key", "err", err) - return err + return sshPrivateKeyPath, err } //git config core.sshCommand _, errorMsg, err := impl.gitManager.ConfigureSshCommand(gitCtx, location, sshPrivateKeyPath) if err != nil { impl.logger.Errorw("error in configuring ssh command while adding repo", "errorMsg", errorMsg, "err", err) - return err + return sshPrivateKeyPath, err } - return nil + return sshPrivateKeyPath, nil } diff --git a/pkg/git/RepositoryManagerAnalytics.go b/pkg/git/RepositoryManagerAnalytics.go index 70873ce9..064d043f 100644 --- a/pkg/git/RepositoryManagerAnalytics.go +++ b/pkg/git/RepositoryManagerAnalytics.go @@ -20,11 +20,8 @@ type RepositoryManagerAnalyticsImpl struct { *RepositoryManagerImpl } -func NewRepositoryManagerAnalyticsImpl( - repositoryManagerImpl *RepositoryManagerImpl, -) *RepositoryManagerAnalyticsImpl { - return &RepositoryManagerAnalyticsImpl{ - RepositoryManagerImpl: repositoryManagerImpl} +func NewRepositoryManagerAnalyticsImpl(repositoryManagerImpl *RepositoryManagerImpl) *RepositoryManagerAnalyticsImpl { + return &RepositoryManagerAnalyticsImpl{RepositoryManagerImpl: repositoryManagerImpl} } func computeDiff(r *git.Repository, newHash *plumbing.Hash, oldHash *plumbing.Hash) ([]*object.Commit, error) { diff --git a/pkg/git/Util.go b/pkg/git/Util.go index 6f956fa1..ee4da8cc 100644 --- a/pkg/git/Util.go +++ b/pkg/git/Util.go @@ -22,7 +22,6 @@ import ( "io/ioutil" "os" "path" - "regexp" "strconv" "strings" ) @@ -33,6 +32,10 @@ const ( SSH_PRIVATE_KEY_FILE_NAME = "ssh_pvt_key" CLONE_TIMEOUT_SEC = 600 FETCH_TIMEOUT_SEC = 30 + GITHUB_PROVIDER = "github.com" + GITLAB_PROVIDER = "gitlab.com" + CloningModeShallow = "SHALLOW" + CloningModeFull = "FULL" ) //git@gitlab.com:devtron-client-gitops/wms-user-management.git @@ -41,24 +44,18 @@ const ( //git@bitbucket.org:DelhiveryTech/kafka-consumer-config.git //https://prashant-delhivery@bitbucket.org/DelhiveryTech/kafka-consumer-config.git -func GetLocationForMaterial(material *sql.GitMaterial) (location string, err error) { - //gitRegex := `/(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)(\/?|\#[-\d\w._]+?)$/` - httpsRegex := `^https.*` - httpsMatched, err := regexp.MatchString(httpsRegex, material.Url) - if httpsMatched { - locationWithoutProtocol := strings.ReplaceAll(material.Url, "https://", "") - checkoutPath := path.Join(GIT_BASE_DIR, strconv.Itoa(material.Id), locationWithoutProtocol) - return checkoutPath, nil - } - - sshRegex := `^git@.*` - sshMatched, err := regexp.MatchString(sshRegex, material.Url) - if sshMatched { - checkoutPath := path.Join(GIT_BASE_DIR, strconv.Itoa(material.Id), material.Url) - return checkoutPath, nil - } - - return "", fmt.Errorf("unsupported format url %s", material.Url) +func GetProjectName(url string) string { + //if url = https://github.com/devtron-labs/git-sensor.git then it will return git-sensor + projName := strings.Split(url, ".")[1] + projectName := projName[strings.LastIndex(projName, "/")+1:] + return projectName +} +func GetCheckoutPath(url string, cloneLocation string) string { + //url= https://github.com/devtron-labs/git-sensor.git cloneLocation= git-base/1/github.com/prakash100198 + //then this function returns git-base/1/github.com/prakash100198/SampleGoLangProject/.git + projectName := GetProjectName(url) + projRootDir := cloneLocation + "/" + projectName + "/.git" + return projRootDir } func GetUserNamePassword(gitProvider *sql.GitProvider) (userName, password string, err error) { diff --git a/pkg/git/Watcher.go b/pkg/git/Watcher.go index 86bcb1db..104eff7e 100644 --- a/pkg/git/Watcher.go +++ b/pkg/git/Watcher.go @@ -31,6 +31,7 @@ import ( "github.com/robfig/cron/v3" "go.uber.org/zap" "runtime/debug" + "strings" "time" ) @@ -183,22 +184,28 @@ func (impl GitWatcherImpl) pollAndUpdateGitMaterial(materialReq *sql.GitMaterial func (impl GitWatcherImpl) pollGitMaterialAndNotify(material *sql.GitMaterial) error { gitProvider := material.GitProvider userName, password, err := GetUserNamePassword(gitProvider) - - gitCtx := BuildGitContext(context.Background()). - WithCredentials(userName, password) - - location, err := GetLocationForMaterial(material) + location := material.CheckoutLocation if err != nil { impl.logger.Errorw("error in determining location", "url", material.Url, "err", err) return err } + gitCtx := BuildGitContext(context.Background()). + WithCredentials(userName, password). + WithCloningMode(impl.configuration.CloningMode) updated, repo, err := impl.FetchAndUpdateMaterial(gitCtx, material, location) if err != nil { impl.logger.Errorw("error in fetching material details ", "repo", material.Url, "err", err) - // there might be the case if ssh private key gets flush from disk, so creating and single retrying in this case + // there might be the case if ssh priclvate key gets flush from disk, so creating and single retrying in this case if gitProvider.AuthMode == sql.AUTH_MODE_SSH { - err = impl.repositoryManager.CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx, location, gitProvider.Id, gitProvider.SshPrivateKey) + if strings.Contains(material.CheckoutLocation, "/.git") { + location, err = impl.repositoryManager.GetLocationForMaterial(material) + if err != nil { + impl.logger.Errorw("error in getting clone location ", "material", material, "err", err) + return err + } + } + _, err = impl.repositoryManager.CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx, location, gitProvider.Id, gitProvider.SshPrivateKey) if err != nil { impl.logger.Errorw("error in creating/configuring ssh private key on disk ", "repo", material.Url, "gitProviderId", gitProvider.Id, "err", err) return err @@ -225,6 +232,7 @@ func (impl GitWatcherImpl) pollGitMaterialAndNotify(material *sql.GitMaterial) e var updatedMaterials []*CiPipelineMaterialBean var updatedMaterialsModel []*sql.CiPipelineMaterial var erroredMaterialsModels []*sql.CiPipelineMaterial + checkoutLocation := material.CheckoutLocation for _, material := range materials { if material.Type != sql.SOURCE_TYPE_BRANCH_FIXED { continue @@ -232,7 +240,7 @@ func (impl GitWatcherImpl) pollGitMaterialAndNotify(material *sql.GitMaterial) e impl.logger.Debugw("Running changesBySinceRepository for material - ", material) impl.logger.Debugw("---------------------------------------------------------- ") // parse env variables here, then search for the count field and pass here. - commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, material.Value, "", "", impl.configuration.GitHistoryCount) + commits, err := impl.repositoryManager.ChangesSinceByRepository(gitCtx, repo, material.Value, "", "", impl.configuration.GitHistoryCount, checkoutLocation) if err != nil { material.Errored = true material.ErrorMsg = err.Error() From 870512af8833fe2525b2b0e8a99900ed2721a879 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 31 Jan 2024 11:29:08 +0530 Subject: [PATCH 02/24] internal name changed to internals --- App.go | 2 +- api/GrpcHandler.go | 2 +- api/GrpcHandler_test.go | 2 +- api/RestHandler.go | 2 +- {internal => internals}/Configuration.go | 2 +- {internal => internals}/Lockutil.go | 2 +- {internal => internals}/logger/logger.go | 0 .../middleware/delegator.go | 0 .../middleware/instrument.go | 0 .../queueManager/LocalWorker.go | 0 .../sql/CiPipelineMaterial.go | 0 {internal => internals}/sql/GitMaterial.go | 0 .../sql/GitProviderRepository.go | 0 ...kEventDataMappingFilterResultRepository.go | 0 .../sql/WebhookEventDataMappingRepository.go | 0 .../sql/WebhookEventParsedDataRepository.go | 0 .../sql/WebhookEventRepository.go | 0 {internal => internals}/sql/connection.go | 0 {internal => internals}/util/HelperUtil.go | 2 +- pkg/RepoManages.go | 18 ++++--- pkg/git/Bean.go | 2 +- pkg/git/GitBaseManager.go | 12 ++--- pkg/git/GitCliManager_test.go | 6 +-- pkg/git/RepositoryManager.go | 10 ++-- pkg/git/RepositoryManagerIT_test.go | 6 +-- pkg/git/Util.go | 2 +- pkg/git/Watcher.go | 14 +++--- pkg/git/WebhookEventBeanConverter.go | 47 +++++++++---------- pkg/git/WebhookEventParser.go | 2 +- pkg/git/WebhookEventService.go | 4 +- pkg/mocks/RepoManager.go | 2 +- util/BasicUtil.go | 2 +- wire.go | 10 ++-- wire_gen.go | 10 ++-- 34 files changed, 81 insertions(+), 80 deletions(-) rename {internal => internals}/Configuration.go (98%) rename {internal => internals}/Lockutil.go (99%) rename {internal => internals}/logger/logger.go (100%) rename {internal => internals}/middleware/delegator.go (100%) rename {internal => internals}/middleware/instrument.go (100%) rename {internal => internals}/queueManager/LocalWorker.go (100%) rename {internal => internals}/sql/CiPipelineMaterial.go (100%) rename {internal => internals}/sql/GitMaterial.go (100%) rename {internal => internals}/sql/GitProviderRepository.go (100%) rename {internal => internals}/sql/WebhookEventDataMappingFilterResultRepository.go (100%) rename {internal => internals}/sql/WebhookEventDataMappingRepository.go (100%) rename {internal => internals}/sql/WebhookEventParsedDataRepository.go (100%) rename {internal => internals}/sql/WebhookEventRepository.go (100%) rename {internal => internals}/sql/connection.go (100%) rename {internal => internals}/util/HelperUtil.go (93%) diff --git a/App.go b/App.go index 2d0cd012..b73e6929 100644 --- a/App.go +++ b/App.go @@ -24,7 +24,7 @@ import ( pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/git-sensor/api" "github.com/devtron-labs/git-sensor/bean" - "github.com/devtron-labs/git-sensor/internal/middleware" + "github.com/devtron-labs/git-sensor/internals/middleware" "github.com/devtron-labs/git-sensor/pkg/git" pb "github.com/devtron-labs/protos/gitSensor" "github.com/go-pg/pg" diff --git a/api/GrpcHandler.go b/api/GrpcHandler.go index 8fac0b63..6e6350c5 100644 --- a/api/GrpcHandler.go +++ b/api/GrpcHandler.go @@ -2,7 +2,7 @@ package api import ( "context" - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals/sql" "github.com/devtron-labs/git-sensor/pkg" "github.com/devtron-labs/git-sensor/pkg/git" pb "github.com/devtron-labs/protos/gitSensor" diff --git a/api/GrpcHandler_test.go b/api/GrpcHandler_test.go index ccd33709..b3f80ff0 100644 --- a/api/GrpcHandler_test.go +++ b/api/GrpcHandler_test.go @@ -3,7 +3,7 @@ package api import ( "context" "encoding/json" - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals/sql" "github.com/devtron-labs/git-sensor/pkg" "github.com/devtron-labs/git-sensor/pkg/git" "github.com/devtron-labs/git-sensor/pkg/mocks" diff --git a/api/RestHandler.go b/api/RestHandler.go index 2cfccd4f..2c83edf5 100644 --- a/api/RestHandler.go +++ b/api/RestHandler.go @@ -18,7 +18,7 @@ package api import ( "encoding/json" - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals/sql" "github.com/devtron-labs/git-sensor/pkg" "github.com/devtron-labs/git-sensor/pkg/git" "github.com/gorilla/mux" diff --git a/internal/Configuration.go b/internals/Configuration.go similarity index 98% rename from internal/Configuration.go rename to internals/Configuration.go index e72b830d..ae559c2d 100644 --- a/internal/Configuration.go +++ b/internals/Configuration.go @@ -1,4 +1,4 @@ -package internal +package internals import "github.com/caarlos0/env" diff --git a/internal/Lockutil.go b/internals/Lockutil.go similarity index 99% rename from internal/Lockutil.go rename to internals/Lockutil.go index d5b56b66..e850fd8a 100644 --- a/internal/Lockutil.go +++ b/internals/Lockutil.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package internal +package internals import ( "go.uber.org/zap" diff --git a/internal/logger/logger.go b/internals/logger/logger.go similarity index 100% rename from internal/logger/logger.go rename to internals/logger/logger.go diff --git a/internal/middleware/delegator.go b/internals/middleware/delegator.go similarity index 100% rename from internal/middleware/delegator.go rename to internals/middleware/delegator.go diff --git a/internal/middleware/instrument.go b/internals/middleware/instrument.go similarity index 100% rename from internal/middleware/instrument.go rename to internals/middleware/instrument.go diff --git a/internal/queueManager/LocalWorker.go b/internals/queueManager/LocalWorker.go similarity index 100% rename from internal/queueManager/LocalWorker.go rename to internals/queueManager/LocalWorker.go diff --git a/internal/sql/CiPipelineMaterial.go b/internals/sql/CiPipelineMaterial.go similarity index 100% rename from internal/sql/CiPipelineMaterial.go rename to internals/sql/CiPipelineMaterial.go diff --git a/internal/sql/GitMaterial.go b/internals/sql/GitMaterial.go similarity index 100% rename from internal/sql/GitMaterial.go rename to internals/sql/GitMaterial.go diff --git a/internal/sql/GitProviderRepository.go b/internals/sql/GitProviderRepository.go similarity index 100% rename from internal/sql/GitProviderRepository.go rename to internals/sql/GitProviderRepository.go diff --git a/internal/sql/WebhookEventDataMappingFilterResultRepository.go b/internals/sql/WebhookEventDataMappingFilterResultRepository.go similarity index 100% rename from internal/sql/WebhookEventDataMappingFilterResultRepository.go rename to internals/sql/WebhookEventDataMappingFilterResultRepository.go diff --git a/internal/sql/WebhookEventDataMappingRepository.go b/internals/sql/WebhookEventDataMappingRepository.go similarity index 100% rename from internal/sql/WebhookEventDataMappingRepository.go rename to internals/sql/WebhookEventDataMappingRepository.go diff --git a/internal/sql/WebhookEventParsedDataRepository.go b/internals/sql/WebhookEventParsedDataRepository.go similarity index 100% rename from internal/sql/WebhookEventParsedDataRepository.go rename to internals/sql/WebhookEventParsedDataRepository.go diff --git a/internal/sql/WebhookEventRepository.go b/internals/sql/WebhookEventRepository.go similarity index 100% rename from internal/sql/WebhookEventRepository.go rename to internals/sql/WebhookEventRepository.go diff --git a/internal/sql/connection.go b/internals/sql/connection.go similarity index 100% rename from internal/sql/connection.go rename to internals/sql/connection.go diff --git a/internal/util/HelperUtil.go b/internals/util/HelperUtil.go similarity index 93% rename from internal/util/HelperUtil.go rename to internals/util/HelperUtil.go index 051391d5..278ae8dc 100644 --- a/internal/util/HelperUtil.go +++ b/internals/util/HelperUtil.go @@ -1,7 +1,7 @@ package util import ( - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals/sql" "strings" ) diff --git a/pkg/RepoManages.go b/pkg/RepoManages.go index b48a3f3a..703d2fba 100644 --- a/pkg/RepoManages.go +++ b/pkg/RepoManages.go @@ -20,9 +20,9 @@ import ( "encoding/json" "errors" "fmt" - "github.com/devtron-labs/git-sensor/internal" - "github.com/devtron-labs/git-sensor/internal/sql" - "github.com/devtron-labs/git-sensor/internal/util" + "github.com/devtron-labs/git-sensor/internals" + "github.com/devtron-labs/git-sensor/internals/sql" + "github.com/devtron-labs/git-sensor/internals/util" "github.com/devtron-labs/git-sensor/pkg/git" _ "github.com/robfig/cron/v3" "go.uber.org/zap" @@ -58,14 +58,14 @@ type RepoManagerImpl struct { repositoryManagerAnalytics git.RepositoryManagerAnalytics gitProviderRepository sql.GitProviderRepository ciPipelineMaterialRepository sql.CiPipelineMaterialRepository - locker *internal.RepositoryLocker + locker *internals.RepositoryLocker gitWatcher git.GitWatcher webhookEventRepository sql.WebhookEventRepository webhookEventParsedDataRepository sql.WebhookEventParsedDataRepository webhookEventDataMappingRepository sql.WebhookEventDataMappingRepository webhookEventDataMappingFilterResultRepository sql.WebhookEventDataMappingFilterResultRepository webhookEventBeanConverter git.WebhookEventBeanConverter - configuration *internal.Configuration + configuration *internals.Configuration gitManager git.GitManagerImpl } @@ -76,13 +76,13 @@ func NewRepoManagerImpl( repositoryManagerAnalytics git.RepositoryManagerAnalytics, gitProviderRepository sql.GitProviderRepository, ciPipelineMaterialRepository sql.CiPipelineMaterialRepository, - locker *internal.RepositoryLocker, + locker *internals.RepositoryLocker, gitWatcher git.GitWatcher, webhookEventRepository sql.WebhookEventRepository, webhookEventParsedDataRepository sql.WebhookEventParsedDataRepository, webhookEventDataMappingRepository sql.WebhookEventDataMappingRepository, webhookEventDataMappingFilterResultRepository sql.WebhookEventDataMappingFilterResultRepository, webhookEventBeanConverter git.WebhookEventBeanConverter, - configuration *internal.Configuration, + configuration *internals.Configuration, gitManager git.GitManagerImpl, ) *RepoManagerImpl { return &RepoManagerImpl{ @@ -757,6 +757,10 @@ func (impl RepoManagerImpl) GetReleaseChanges(gitCtx git.GitContext, request *Re repoLock.Mutex.Unlock() impl.locker.ReturnLocker(gitMaterial.Id) }() + + gitCtx = gitCtx.WithCredentials(gitMaterial.GitProvider.UserName, gitMaterial.GitProvider.Password). + WithCloningMode(impl.configuration.CloningMode) + gitChanges, err := impl.repositoryManagerAnalytics.ChangesSinceByRepositoryForAnalytics(gitCtx, gitMaterial.CheckoutLocation, request.OldCommit, request.NewCommit) if err != nil { impl.logger.Errorw("error in computing changes", "req", request, "err", err) diff --git a/pkg/git/Bean.go b/pkg/git/Bean.go index 9aaea416..be3f19cb 100644 --- a/pkg/git/Bean.go +++ b/pkg/git/Bean.go @@ -17,7 +17,7 @@ package git import ( - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals/sql" "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing/object" "io" diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index 409c560e..85e504a4 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -5,8 +5,8 @@ import ( "encoding/json" "errors" "fmt" - "github.com/devtron-labs/git-sensor/internal" - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals" + "github.com/devtron-labs/git-sensor/internals/sql" "github.com/devtron-labs/git-sensor/util" "go.uber.org/zap" "os" @@ -48,11 +48,11 @@ type GitManagerBase interface { } type GitManagerBaseImpl struct { logger *zap.SugaredLogger - conf *internal.Configuration + conf *internals.Configuration commandTimeoutMap map[string]int } -func NewGitManagerBaseImpl(logger *zap.SugaredLogger, config *internal.Configuration) *GitManagerBaseImpl { +func NewGitManagerBaseImpl(logger *zap.SugaredLogger, config *internals.Configuration) *GitManagerBaseImpl { commandTimeoutMap, err := parseCmdTimeoutJson(config) if err != nil { @@ -62,7 +62,7 @@ func NewGitManagerBaseImpl(logger *zap.SugaredLogger, config *internal.Configura return &GitManagerBaseImpl{logger: logger, conf: config, commandTimeoutMap: commandTimeoutMap} } -func parseCmdTimeoutJson(config *internal.Configuration) (map[string]int, error) { +func parseCmdTimeoutJson(config *internals.Configuration) (map[string]int, error) { commandTimeoutMap := make(map[string]int) var err error if config.CliCmdTimeoutJson != "" { @@ -75,7 +75,7 @@ type GitManagerImpl struct { GitManager } -func NewGitManagerImpl(configuration *internal.Configuration, +func NewGitManagerImpl(configuration *internals.Configuration, cliGitManager GitCliManager, goGitManager GoGitSDKManager) GitManagerImpl { diff --git a/pkg/git/GitCliManager_test.go b/pkg/git/GitCliManager_test.go index 325d3be1..d2b60ed1 100644 --- a/pkg/git/GitCliManager_test.go +++ b/pkg/git/GitCliManager_test.go @@ -3,7 +3,7 @@ package git import ( "context" "github.com/devtron-labs/common-lib/utils" - "github.com/devtron-labs/git-sensor/internal" + "github.com/devtron-labs/git-sensor/internals" "testing" ) @@ -12,7 +12,7 @@ func TestGitCliManagerImpl_processGitLogOutput(t *testing.T) { t.Run("tt.name", func(t *testing.T) { logger, _ := utils.NewSugardLogger() impl := &GitCliManagerImpl{ - GitManagerBaseImpl: NewGitManagerBaseImpl(logger, &internal.Configuration{}), + GitManagerBaseImpl: NewGitManagerBaseImpl(logger, &internals.Configuration{}), } got, err := impl.GetCommits(BuildGitContext(context.Background()), "main", "", "/Users/subhashish/workspace/lens", 15, "", "") print("err", got, err) @@ -20,7 +20,7 @@ func TestGitCliManagerImpl_processGitLogOutput(t *testing.T) { t.Run("analytics", func(t *testing.T) { logger, _ := utils.NewSugardLogger() - conf := &internal.Configuration{UseGitCli: false, AnalyticsDebug: true, GoGitTimeout: 10, CliCmdTimeoutGlobal: 8, CliCmdTimeoutJson: `{"log":4}`} + conf := &internals.Configuration{UseGitCli: false, AnalyticsDebug: true, GoGitTimeout: 10, CliCmdTimeoutGlobal: 8, CliCmdTimeoutJson: `{"log":4}`} impl := &GoGitSDKManagerImpl{ GitManagerBaseImpl: NewGitManagerBaseImpl(logger, conf), } diff --git a/pkg/git/RepositoryManager.go b/pkg/git/RepositoryManager.go index 553ae7f3..98ca8df2 100644 --- a/pkg/git/RepositoryManager.go +++ b/pkg/git/RepositoryManager.go @@ -19,7 +19,7 @@ package git import ( "errors" "fmt" - "github.com/devtron-labs/git-sensor/internal" + "github.com/devtron-labs/git-sensor/internals" "github.com/devtron-labs/git-sensor/util" "golang.org/x/sys/unix" "io" @@ -30,8 +30,8 @@ import ( "strings" "time" - "github.com/devtron-labs/git-sensor/internal/middleware" - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals/middleware" + "github.com/devtron-labs/git-sensor/internals/sql" "go.uber.org/zap" ) @@ -59,12 +59,12 @@ type RepositoryManager interface { type RepositoryManagerImpl struct { logger *zap.SugaredLogger gitManager GitManagerImpl - configuration *internal.Configuration + configuration *internals.Configuration } func NewRepositoryManagerImpl( logger *zap.SugaredLogger, - configuration *internal.Configuration, + configuration *internals.Configuration, gitManager GitManagerImpl, ) *RepositoryManagerImpl { return &RepositoryManagerImpl{logger: logger, configuration: configuration, gitManager: gitManager} diff --git a/pkg/git/RepositoryManagerIT_test.go b/pkg/git/RepositoryManagerIT_test.go index 4312c269..c62e6721 100644 --- a/pkg/git/RepositoryManagerIT_test.go +++ b/pkg/git/RepositoryManagerIT_test.go @@ -3,8 +3,8 @@ package git import ( "context" "github.com/devtron-labs/common-lib/utils" - "github.com/devtron-labs/git-sensor/internal" - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals" + "github.com/devtron-labs/git-sensor/internals/sql" "github.com/stretchr/testify/assert" "os" "reflect" @@ -32,7 +32,7 @@ func getRepoManagerAnalyticsImpl(t *testing.T) *RepositoryManagerAnalyticsImpl { func getRepoManagerImpl(t *testing.T) *RepositoryManagerImpl { logger, err := utils.NewSugardLogger() assert.Nil(t, err) - conf := &internal.Configuration{ + conf := &internals.Configuration{ CommitStatsTimeoutInSec: 0, EnableFileStats: true, GitHistoryCount: 2, diff --git a/pkg/git/Util.go b/pkg/git/Util.go index ee4da8cc..334bc7e8 100644 --- a/pkg/git/Util.go +++ b/pkg/git/Util.go @@ -18,7 +18,7 @@ package git import ( "fmt" - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals/sql" "io/ioutil" "os" "path" diff --git a/pkg/git/Watcher.go b/pkg/git/Watcher.go index 104eff7e..cd793064 100644 --- a/pkg/git/Watcher.go +++ b/pkg/git/Watcher.go @@ -24,9 +24,9 @@ import ( "github.com/devtron-labs/common-lib/constants" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/common-lib/pubsub-lib/model" - "github.com/devtron-labs/git-sensor/internal" - "github.com/devtron-labs/git-sensor/internal/middleware" - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals" + "github.com/devtron-labs/git-sensor/internals/middleware" + "github.com/devtron-labs/git-sensor/internals/sql" "github.com/gammazero/workerpool" "github.com/robfig/cron/v3" "go.uber.org/zap" @@ -42,10 +42,10 @@ type GitWatcherImpl struct { logger *zap.SugaredLogger ciPipelineMaterialRepository sql.CiPipelineMaterialRepository pubSubClient *pubsub.PubSubClientServiceImpl - locker *internal.RepositoryLocker + locker *internals.RepositoryLocker pollConfig *PollConfig webhookHandler WebhookHandler - configuration *internal.Configuration + configuration *internals.Configuration gitManager GitManagerImpl } @@ -62,8 +62,8 @@ func NewGitWatcherImpl(repositoryManager RepositoryManager, materialRepo sql.MaterialRepository, logger *zap.SugaredLogger, ciPipelineMaterialRepository sql.CiPipelineMaterialRepository, - locker *internal.RepositoryLocker, - pubSubClient *pubsub.PubSubClientServiceImpl, webhookHandler WebhookHandler, configuration *internal.Configuration, + locker *internals.RepositoryLocker, + pubSubClient *pubsub.PubSubClientServiceImpl, webhookHandler WebhookHandler, configuration *internals.Configuration, gitmanager GitManagerImpl, ) (*GitWatcherImpl, error) { diff --git a/pkg/git/WebhookEventBeanConverter.go b/pkg/git/WebhookEventBeanConverter.go index 283d0a05..e43ef9fa 100644 --- a/pkg/git/WebhookEventBeanConverter.go +++ b/pkg/git/WebhookEventBeanConverter.go @@ -17,7 +17,7 @@ package git -import "github.com/devtron-labs/git-sensor/internal/sql" +import "github.com/devtron-labs/git-sensor/internals/sql" type WebhookEventBeanConverter interface { ConvertFromWebhookParsedDataSqlBean(sqlBean *sql.WebhookEventParsedData) *WebhookData @@ -25,51 +25,48 @@ type WebhookEventBeanConverter interface { } type WebhookEventBeanConverterImpl struct { - } func NewWebhookEventBeanConverterImpl() *WebhookEventBeanConverterImpl { - return &WebhookEventBeanConverterImpl{ - - } + return &WebhookEventBeanConverterImpl{} } func (impl WebhookEventBeanConverterImpl) ConvertFromWebhookParsedDataSqlBean(sqlBean *sql.WebhookEventParsedData) *WebhookData { return &WebhookData{ - Id : sqlBean.Id, + Id: sqlBean.Id, EventActionType: sqlBean.EventActionType, - Data : sqlBean.Data, + Data: sqlBean.Data, } } func (impl WebhookEventBeanConverterImpl) ConvertFromWebhookEventSqlBean(webhookEventFromDb *sql.GitHostWebhookEvent) *WebhookEventConfig { webhookEvent := &WebhookEventConfig{ - Id : webhookEventFromDb.Id, - GitHostId: webhookEventFromDb.GitHostId, - Name: webhookEventFromDb.Name, + Id: webhookEventFromDb.Id, + GitHostId: webhookEventFromDb.GitHostId, + Name: webhookEventFromDb.Name, EventTypesCsv: webhookEventFromDb.EventTypesCsv, - ActionType: webhookEventFromDb.ActionType, - IsActive: webhookEventFromDb.IsActive, - CreatedOn: webhookEventFromDb.CreatedOn, - UpdatedOn: webhookEventFromDb.UpdatedOn, + ActionType: webhookEventFromDb.ActionType, + IsActive: webhookEventFromDb.IsActive, + CreatedOn: webhookEventFromDb.CreatedOn, + UpdatedOn: webhookEventFromDb.UpdatedOn, } // build selectors var webhookEventSelectors []*WebhookEventSelectors for _, selectorFromDb := range webhookEventFromDb.Selectors { selector := &WebhookEventSelectors{ - Id : selectorFromDb.Id, - EventId: selectorFromDb.EventId, - Name: selectorFromDb.Name, - Selector: selectorFromDb.Selector, - ToShow: selectorFromDb.ToShow, + Id: selectorFromDb.Id, + EventId: selectorFromDb.EventId, + Name: selectorFromDb.Name, + Selector: selectorFromDb.Selector, + ToShow: selectorFromDb.ToShow, ToShowInCiFilter: selectorFromDb.ToShowInCiFilter, - FixValue: selectorFromDb.FixValue, - PossibleValues: selectorFromDb.PossibleValues, - IsActive: selectorFromDb.IsActive, - CreatedOn: selectorFromDb.CreatedOn, - UpdatedOn: selectorFromDb.UpdatedOn, + FixValue: selectorFromDb.FixValue, + PossibleValues: selectorFromDb.PossibleValues, + IsActive: selectorFromDb.IsActive, + CreatedOn: selectorFromDb.CreatedOn, + UpdatedOn: selectorFromDb.UpdatedOn, } webhookEventSelectors = append(webhookEventSelectors, selector) } @@ -78,4 +75,4 @@ func (impl WebhookEventBeanConverterImpl) ConvertFromWebhookEventSqlBean(webhook return webhookEvent -} \ No newline at end of file +} diff --git a/pkg/git/WebhookEventParser.go b/pkg/git/WebhookEventParser.go index 596e7d55..0b9c3c08 100644 --- a/pkg/git/WebhookEventParser.go +++ b/pkg/git/WebhookEventParser.go @@ -18,7 +18,7 @@ package git import ( - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals/sql" "github.com/tidwall/gjson" "go.uber.org/zap" "time" diff --git a/pkg/git/WebhookEventService.go b/pkg/git/WebhookEventService.go index e1452cbb..bad9b8f2 100644 --- a/pkg/git/WebhookEventService.go +++ b/pkg/git/WebhookEventService.go @@ -20,8 +20,8 @@ import ( "encoding/json" "fmt" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - "github.com/devtron-labs/git-sensor/internal/sql" - "github.com/devtron-labs/git-sensor/internal/util" + "github.com/devtron-labs/git-sensor/internals/sql" + "github.com/devtron-labs/git-sensor/internals/util" _ "github.com/robfig/cron/v3" "go.uber.org/zap" "regexp" diff --git a/pkg/mocks/RepoManager.go b/pkg/mocks/RepoManager.go index 453d9451..3b73c688 100644 --- a/pkg/mocks/RepoManager.go +++ b/pkg/mocks/RepoManager.go @@ -8,7 +8,7 @@ import ( pkg "github.com/devtron-labs/git-sensor/pkg" - sql "github.com/devtron-labs/git-sensor/internal/sql" + sql "github.com/devtron-labs/git-sensor/internals/sql" ) // RepoManager is an autogenerated mock type for the RepoManager type diff --git a/util/BasicUtil.go b/util/BasicUtil.go index 4c6109ab..e3761ab3 100644 --- a/util/BasicUtil.go +++ b/util/BasicUtil.go @@ -17,7 +17,7 @@ package util import ( - "github.com/devtron-labs/git-sensor/internal/middleware" + "github.com/devtron-labs/git-sensor/internals/middleware" "math/rand" "strings" "time" diff --git a/wire.go b/wire.go index afb34140..cee631d6 100644 --- a/wire.go +++ b/wire.go @@ -23,9 +23,9 @@ import ( "github.com/devtron-labs/common-lib/monitoring" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/git-sensor/api" - "github.com/devtron-labs/git-sensor/internal" - "github.com/devtron-labs/git-sensor/internal/logger" - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals" + "github.com/devtron-labs/git-sensor/internals/logger" + "github.com/devtron-labs/git-sensor/internals/sql" "github.com/devtron-labs/git-sensor/pkg" "github.com/devtron-labs/git-sensor/pkg/git" "github.com/google/wire" @@ -36,7 +36,7 @@ func InitializeApp() (*App, error) { wire.Build( NewApp, api.NewMuxRouter, - internal.ParseConfiguration, + internals.ParseConfiguration, logger.NewSugaredLogger, api.NewRestHandlerImpl, wire.Bind(new(api.RestHandler), new(*api.RestHandlerImpl)), @@ -63,7 +63,7 @@ func InitializeApp() (*App, error) { git.NewGitManagerBaseImpl, git.NewGitWatcherImpl, wire.Bind(new(git.GitWatcher), new(*git.GitWatcherImpl)), - internal.NewRepositoryLocker, + internals.NewRepositoryLocker, //internal.NewNatsConnection, pubsub.NewPubSubClientServiceImpl, sql.NewWebhookEventRepositoryImpl, diff --git a/wire_gen.go b/wire_gen.go index 3886ce60..d189fa75 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -10,9 +10,9 @@ import ( "github.com/devtron-labs/common-lib/monitoring" "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/git-sensor/api" - "github.com/devtron-labs/git-sensor/internal" - "github.com/devtron-labs/git-sensor/internal/logger" - "github.com/devtron-labs/git-sensor/internal/sql" + "github.com/devtron-labs/git-sensor/internals" + "github.com/devtron-labs/git-sensor/internals/logger" + "github.com/devtron-labs/git-sensor/internals/sql" "github.com/devtron-labs/git-sensor/pkg" "github.com/devtron-labs/git-sensor/pkg/git" ) @@ -30,7 +30,7 @@ func InitializeApp() (*App, error) { return nil, err } materialRepositoryImpl := sql.NewMaterialRepositoryImpl(db) - configuration, err := internal.ParseConfiguration() + configuration, err := internals.ParseConfiguration() if err != nil { return nil, err } @@ -42,7 +42,7 @@ func InitializeApp() (*App, error) { repositoryManagerAnalyticsImpl := git.NewRepositoryManagerAnalyticsImpl(repositoryManagerImpl) gitProviderRepositoryImpl := sql.NewGitProviderRepositoryImpl(db) ciPipelineMaterialRepositoryImpl := sql.NewCiPipelineMaterialRepositoryImpl(db, sugaredLogger) - repositoryLocker := internal.NewRepositoryLocker(sugaredLogger) + repositoryLocker := internals.NewRepositoryLocker(sugaredLogger) pubSubClientServiceImpl := pubsub_lib.NewPubSubClientServiceImpl(sugaredLogger) webhookEventRepositoryImpl := sql.NewWebhookEventRepositoryImpl(db) webhookEventParsedDataRepositoryImpl := sql.NewWebhookEventParsedDataRepositoryImpl(db) From afea9c774162329dc7c5303a2a20aa92cba287e8 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 31 Jan 2024 12:15:33 +0530 Subject: [PATCH 03/24] code made common --- pkg/RepoManages.go | 4 ++-- pkg/git/RepositoryManager.go | 22 +++++++++++++++++----- pkg/git/Watcher.go | 2 +- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/pkg/RepoManages.go b/pkg/RepoManages.go index 703d2fba..434ee99d 100644 --- a/pkg/RepoManages.go +++ b/pkg/RepoManages.go @@ -352,14 +352,14 @@ func (impl RepoManagerImpl) checkoutMaterial(gitCtx git.GitContext, material *sq gitCtx = gitCtx.WithCredentials(userName, password). WithCloningMode(impl.configuration.CloningMode) - checkoutPath, err := impl.repositoryManager.GetLocationForMaterial(material) + checkoutPath, checkoutLocationForFetching, err := impl.repositoryManager.GetCheckoutPathAndLocation(gitCtx, material, gitProvider.Url) if err != nil { return material, err } err = impl.repositoryManager.Add(gitCtx, material.GitProviderId, checkoutPath, material.Url, gitProvider.AuthMode, gitProvider.SshPrivateKey) if err == nil { - material.CheckoutLocation = checkoutPath + material.CheckoutLocation = checkoutLocationForFetching material.CheckoutStatus = true } else { material.CheckoutStatus = false diff --git a/pkg/git/RepositoryManager.go b/pkg/git/RepositoryManager.go index 98ca8df2..9f76c264 100644 --- a/pkg/git/RepositoryManager.go +++ b/pkg/git/RepositoryManager.go @@ -41,7 +41,8 @@ type RepositoryManager interface { Fetch(gitCtx GitContext, url string, location string) (updated bool, repo *GitRepository, err error) // Add adds and initializes a new git repo , cleans the directory if not empty and fetches latest commits Add(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error - GetLocationForMaterial(material *sql.GitMaterial) (location string, err error) + GetLocationForMaterial(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) + GetCheckoutPathAndLocation(gitCtx GitContext, material *sql.GitMaterial, url string) (string, string, error) // Clean cleans a directory Clean(cloneDir string) error // ChangesSince given the checkput path, retrieves the latest commits for the gt repo existing on the path @@ -80,24 +81,35 @@ func (impl RepositoryManagerImpl) IsSpaceAvailableOnDisk() bool { return availableSpace > int64(impl.configuration.MinLimit)*1024*1024 } -func (impl RepositoryManagerImpl) GetLocationForMaterial(material *sql.GitMaterial) (location string, err error) { +func (impl RepositoryManagerImpl) GetLocationForMaterial(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) { //gitRegex := `/(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)(\/?|\#[-\d\w._]+?)$/` httpsRegex := `^https.*` httpsMatched, err := regexp.MatchString(httpsRegex, material.Url) if httpsMatched { locationWithoutProtocol := strings.ReplaceAll(material.Url, "https://", "") checkoutPath := path.Join(GIT_BASE_DIR, strconv.Itoa(material.Id), locationWithoutProtocol) - return checkoutPath, nil + return checkoutPath, httpsMatched, false, nil } sshRegex := `^git@.*` sshMatched, err := regexp.MatchString(sshRegex, material.Url) if sshMatched { checkoutPath := path.Join(GIT_BASE_DIR, strconv.Itoa(material.Id), material.Url) - return checkoutPath, nil + return checkoutPath, httpsMatched, sshMatched, nil } - return "", fmt.Errorf("unsupported format url %s", material.Url) + return "", httpsMatched, sshMatched, fmt.Errorf("unsupported format url %s", material.Url) +} + +func (impl RepositoryManagerImpl) GetCheckoutPathAndLocation(gitCtx GitContext, material *sql.GitMaterial, url string) (string, string, error) { + var checkoutPath string + var checkoutLocationForFetching string + checkoutPath, _, _, err := impl.GetLocationForMaterial(material, gitCtx.CloningMode) + if err != nil { + return checkoutPath, checkoutLocationForFetching, err + } + checkoutLocationForFetching = checkoutPath + return checkoutPath, checkoutLocationForFetching, nil } func (impl RepositoryManagerImpl) Add(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error { diff --git a/pkg/git/Watcher.go b/pkg/git/Watcher.go index cd793064..aa991ed7 100644 --- a/pkg/git/Watcher.go +++ b/pkg/git/Watcher.go @@ -199,7 +199,7 @@ func (impl GitWatcherImpl) pollGitMaterialAndNotify(material *sql.GitMaterial) e // there might be the case if ssh priclvate key gets flush from disk, so creating and single retrying in this case if gitProvider.AuthMode == sql.AUTH_MODE_SSH { if strings.Contains(material.CheckoutLocation, "/.git") { - location, err = impl.repositoryManager.GetLocationForMaterial(material) + location, _, _, err = impl.repositoryManager.GetLocationForMaterial(material, gitCtx.CloningMode) if err != nil { impl.logger.Errorw("error in getting clone location ", "material", material, "err", err) return err From 12688e0ccf4573f8fcf904b9933dddd208b1911f Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 31 Jan 2024 12:40:21 +0530 Subject: [PATCH 04/24] code refactoring in GitBaseManager.go --- pkg/git/GitBaseManager.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index 85e504a4..dbeb64a6 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -45,6 +45,8 @@ type GitManagerBase interface { FetchDiffStatBetweenCommits(gitCtx GitContext, oldHash string, newHash string, rootDir string) (response, errMsg string, err error) // LogMergeBase get the commit diff between using a merge base strategy LogMergeBase(gitCtx GitContext, rootDir, from string, to string) ([]*Commit, error) + CreateCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc) + RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) } type GitManagerBaseImpl struct { logger *zap.SugaredLogger @@ -109,7 +111,7 @@ func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (respon impl.logger.Debugw("git fetch ", "location", rootDir) cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force") defer cancel() - output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password) + output, errMsg, err := impl.RunCommandWithCred(cmd, gitCtx.Username, gitCtx.Password) impl.logger.Debugw("fetch output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } @@ -148,7 +150,7 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st return commits, nil } -func (impl *GitManagerBaseImpl) runCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) { +func (impl *GitManagerBaseImpl) RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) { cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_ASKPASS=%s", GIT_ASK_PASS), fmt.Sprintf("GIT_USERNAME=%s", userName), @@ -280,7 +282,7 @@ func (impl *GitManagerBaseImpl) FetchDiffStatBetweenCommits(gitCtx GitContext, o cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "diff", "--numstat", oldHash, newHash) defer cancel() - output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password) + output, errMsg, err := impl.RunCommandWithCred(cmd, gitCtx.Username, gitCtx.Password) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } From ab32ad70e02c6183731c037c8cb7021ad47e577e Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 31 Jan 2024 13:03:28 +0530 Subject: [PATCH 05/24] code refactoring in GitBaseManager.go --- pkg/git/RepositoryManager.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/git/RepositoryManager.go b/pkg/git/RepositoryManager.go index 9f76c264..66ff6c14 100644 --- a/pkg/git/RepositoryManager.go +++ b/pkg/git/RepositoryManager.go @@ -113,14 +113,14 @@ func (impl RepositoryManagerImpl) GetCheckoutPathAndLocation(gitCtx GitContext, } func (impl RepositoryManagerImpl) Add(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error { - err := impl.Add1(gitCtx, gitProviderId, location, url, authMode, sshPrivateKeyContent) + _, err := impl.Add1(gitCtx, gitProviderId, location, url, authMode, sshPrivateKeyContent) if err != nil { return err } return impl.Add2(gitCtx, location) } -func (impl RepositoryManagerImpl) Add1(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error { +func (impl RepositoryManagerImpl) Add1(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) (string, error) { var err error start := time.Now() defer func() { @@ -129,16 +129,16 @@ func (impl RepositoryManagerImpl) Add1(gitCtx GitContext, gitProviderId int, loc err = os.RemoveAll(location) if err != nil { impl.logger.Errorw("error in cleaning checkout path", "err", err) - return err + return "", err } if !impl.IsSpaceAvailableOnDisk() { err = errors.New("git-sensor PVC - disk full, please increase space") - return err + return "", err } err = impl.gitManager.Init(gitCtx, location, url, true) if err != nil { impl.logger.Errorw("err in git init", "err", err) - return err + return "", err } var sshPrivateKeyPath string // check ssh @@ -146,11 +146,11 @@ func (impl RepositoryManagerImpl) Add1(gitCtx GitContext, gitProviderId int, loc sshPrivateKeyPath, err = impl.CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx, location, gitProviderId, sshPrivateKeyContent) if err != nil { impl.logger.Errorw("error while creating ssh file for shallow clone", "checkoutPath", location, "sshPrivateKeyPath", sshPrivateKeyPath, "err", err) - return err + return "", err } } - return nil + return sshPrivateKeyPath, nil } func (impl RepositoryManagerImpl) Add2(gitCtx GitContext, location string) error { From 36e964e7fc7ad606ca8d0af1af9da5c859f981c2 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 31 Jan 2024 13:32:06 +0530 Subject: [PATCH 06/24] code refactoring in GitBaseManager.go --- pkg/git/RepositoryManager.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/git/RepositoryManager.go b/pkg/git/RepositoryManager.go index 66ff6c14..b624841f 100644 --- a/pkg/git/RepositoryManager.go +++ b/pkg/git/RepositoryManager.go @@ -43,6 +43,7 @@ type RepositoryManager interface { Add(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error GetLocationForMaterial(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) GetCheckoutPathAndLocation(gitCtx GitContext, material *sql.GitMaterial, url string) (string, string, error) + TrimLastGitCommit(gitCommits []*GitCommitBase, count int) []*GitCommitBase // Clean cleans a directory Clean(cloneDir string) error // ChangesSince given the checkput path, retrieves the latest commits for the gt repo existing on the path @@ -322,7 +323,7 @@ func (impl RepositoryManagerImpl) ChangesSinceByRepository(gitCtx GitContext, re return gitCommits, err } -func trimLastGitCommit(gitCommits []*GitCommitBase, count int) []*GitCommitBase { +func (impl RepositoryManagerImpl) TrimLastGitCommit(gitCommits []*GitCommitBase, count int) []*GitCommitBase { if len(gitCommits) > count { gitCommits = gitCommits[:len(gitCommits)-1] } From 135bca7168f0edec2deae2f57878bc779ff6ab0c Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Wed, 31 Jan 2024 14:55:40 +0530 Subject: [PATCH 07/24] code refactoring --- pkg/git/RepositoryManager.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/git/RepositoryManager.go b/pkg/git/RepositoryManager.go index b624841f..5f0c2332 100644 --- a/pkg/git/RepositoryManager.go +++ b/pkg/git/RepositoryManager.go @@ -41,6 +41,8 @@ type RepositoryManager interface { Fetch(gitCtx GitContext, url string, location string) (updated bool, repo *GitRepository, err error) // Add adds and initializes a new git repo , cleans the directory if not empty and fetches latest commits Add(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error + Add1(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) (string, error) + Add2(gitCtx GitContext, location string) error GetLocationForMaterial(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) GetCheckoutPathAndLocation(gitCtx GitContext, material *sql.GitMaterial, url string) (string, string, error) TrimLastGitCommit(gitCommits []*GitCommitBase, count int) []*GitCommitBase From 22a0c6aa0d73a02fe4d814969c07f8e19c07f105 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 1 Feb 2024 12:18:24 +0530 Subject: [PATCH 08/24] code refactoring --- pkg/RepoManages.go | 4 +-- pkg/git/GitBaseManager.go | 39 +++++++++------------------ pkg/git/GitCliManager.go | 38 +++++++++++++++++++++----- pkg/git/GoGitSDKManager.go | 29 +++++++++++++++++--- pkg/git/RepositoryManager.go | 30 ++++++++++++++++++--- pkg/git/RepositoryManagerAnalytics.go | 17 +++++++++--- pkg/git/Watcher.go | 4 +-- wire.go | 16 ++++++----- wire_gen.go | 6 ++--- 9 files changed, 126 insertions(+), 57 deletions(-) diff --git a/pkg/RepoManages.go b/pkg/RepoManages.go index 434ee99d..706403a2 100644 --- a/pkg/RepoManages.go +++ b/pkg/RepoManages.go @@ -66,7 +66,7 @@ type RepoManagerImpl struct { webhookEventDataMappingFilterResultRepository sql.WebhookEventDataMappingFilterResultRepository webhookEventBeanConverter git.WebhookEventBeanConverter configuration *internals.Configuration - gitManager git.GitManagerImpl + gitManager git.GitManager } func NewRepoManagerImpl( @@ -83,7 +83,7 @@ func NewRepoManagerImpl( webhookEventDataMappingFilterResultRepository sql.WebhookEventDataMappingFilterResultRepository, webhookEventBeanConverter git.WebhookEventBeanConverter, configuration *internals.Configuration, - gitManager git.GitManagerImpl, + gitManager git.GitManager, ) *RepoManagerImpl { return &RepoManagerImpl{ logger: logger, diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index dbeb64a6..6444c94e 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -47,6 +47,7 @@ type GitManagerBase interface { LogMergeBase(gitCtx GitContext, rootDir, from string, to string) ([]*Commit, error) CreateCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc) RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) + RunCommand(cmd *exec.Cmd) (response, errMsg string, err error) } type GitManagerBaseImpl struct { logger *zap.SugaredLogger @@ -79,32 +80,16 @@ type GitManagerImpl struct { func NewGitManagerImpl(configuration *internals.Configuration, cliGitManager GitCliManager, - goGitManager GoGitSDKManager) GitManagerImpl { + goGitManager GoGitSDKManager) *GitManagerImpl { if configuration.UseGitCli { - return GitManagerImpl{cliGitManager} - } - return GitManagerImpl{goGitManager} -} - -func (impl *GitManagerImpl) OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) { - - r, err := impl.OpenRepoPlain(location) - if err != nil { - err = os.RemoveAll(location) - if err != nil { - return r, fmt.Errorf("error in cleaning checkout path: %s", err) - } - err = impl.Init(gitCtx, location, url, true) - if err != nil { - return r, fmt.Errorf("err in git init: %s", err) - } - r, err = impl.OpenRepoPlain(location) - if err != nil { - return r, fmt.Errorf("err in git init: %s", err) + return &GitManagerImpl{ + cliGitManager, } } - return r, nil + return &GitManagerImpl{ + goGitManager, + } } func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (response, errMsg string, err error) { @@ -120,7 +105,7 @@ func (impl *GitManagerBaseImpl) Checkout(gitCtx GitContext, rootDir, branch stri impl.logger.Debugw("git checkout ", "location", rootDir) cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "checkout", branch, "--force") defer cancel() - output, errMsg, err := impl.runCommand(cmd) + output, errMsg, err := impl.RunCommand(cmd) impl.logger.Debugw("checkout output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } @@ -137,7 +122,7 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st impl.logger.Debugw("git", cmdArgs) cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", cmdArgs...) defer cancel() - output, errMsg, err := impl.runCommand(cmd) + output, errMsg, err := impl.RunCommand(cmd) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil { return nil, err @@ -156,10 +141,10 @@ func (impl *GitManagerBaseImpl) RunCommandWithCred(cmd *exec.Cmd, userName, pass fmt.Sprintf("GIT_USERNAME=%s", userName), fmt.Sprintf("GIT_PASSWORD=%s", password), ) - return impl.runCommand(cmd) + return impl.RunCommand(cmd) } -func (impl *GitManagerBaseImpl) runCommand(cmd *exec.Cmd) (response, errMsg string, err error) { +func (impl *GitManagerBaseImpl) RunCommand(cmd *exec.Cmd) (response, errMsg string, err error) { cmd.Env = append(cmd.Env, "HOME=/dev/null") outBytes, err := cmd.CombinedOutput() if err != nil { @@ -185,7 +170,7 @@ func (impl *GitManagerBaseImpl) ConfigureSshCommand(gitCtx GitContext, rootDir s coreSshCommand := fmt.Sprintf("ssh -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", sshPrivateKeyPath) cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "config", "core.sshCommand", coreSshCommand) defer cancel() - output, errMsg, err := impl.runCommand(cmd) + output, errMsg, err := impl.RunCommand(cmd) impl.logger.Debugw("configure ssh command output ", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } diff --git a/pkg/git/GitCliManager.go b/pkg/git/GitCliManager.go index dd5bb922..c998cf5b 100644 --- a/pkg/git/GitCliManager.go +++ b/pkg/git/GitCliManager.go @@ -1,6 +1,8 @@ package git import ( + "fmt" + "go.uber.org/zap" "gopkg.in/src-d/go-billy.v4/osfs" "os" "path/filepath" @@ -12,13 +14,15 @@ type GitCliManager interface { } type GitCliManagerImpl struct { - *GitManagerBaseImpl + GitManagerBase + logger *zap.SugaredLogger } -func NewGitCliManagerImpl(baseManagerImpl *GitManagerBaseImpl) *GitCliManagerImpl { +func NewGitCliManagerImpl(baseManager GitManagerBase, logger *zap.SugaredLogger) *GitCliManagerImpl { return &GitCliManagerImpl{ - GitManagerBaseImpl: baseManagerImpl, + GitManagerBase: baseManager, + logger: logger, } } @@ -89,7 +93,7 @@ func (impl *GitCliManagerImpl) GitInit(gitCtx GitContext, rootDir string) error impl.logger.Debugw("git", "-C", rootDir, "init") cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "init") defer cancel() - output, errMsg, err := impl.runCommand(cmd) + output, errMsg, err := impl.RunCommand(cmd) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return err } @@ -98,7 +102,7 @@ func (impl *GitCliManagerImpl) GitCreateRemote(gitCtx GitContext, rootDir string impl.logger.Debugw("git", "-C", rootDir, "remote", "add", "origin", url) cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "remote", "add", "origin", url) defer cancel() - output, errMsg, err := impl.runCommand(cmd) + output, errMsg, err := impl.GitManagerBase.RunCommand(cmd) impl.logger.Debugw("url", url, "opt", output, "errMsg", errMsg, "error", err) return err } @@ -112,7 +116,7 @@ func (impl *GitCliManagerImpl) GetCommits(gitCtx GitContext, branchRef string, b impl.logger.Debugw("git", cmdArgs) cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", cmdArgs...) defer cancel() - output, errMsg, err := impl.runCommand(cmd) + output, errMsg, err := impl.GitManagerBase.RunCommand(cmd) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil { return nil, err @@ -139,7 +143,7 @@ func (impl *GitCliManagerImpl) GitShow(gitCtx GitContext, rootDir string, hash s impl.logger.Debugw("git", "-C", rootDir, "show", hash, "--date=iso-strict", GITFORMAT, "-s") cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "show", hash, "--date=iso-strict", GITFORMAT, "-s") defer cancel() - output, errMsg, err := impl.runCommand(cmd) + output, errMsg, err := impl.GitManagerBase.RunCommand(cmd) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil { return nil, err @@ -187,3 +191,23 @@ func (impl *GitCliManagerImpl) processGitLogOutput(out string) ([]GitCommit, err } return gitCommits, nil } + +func (impl *GitCliManagerImpl) OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) { + + r, err := impl.OpenRepoPlain(location) + if err != nil { + err = os.RemoveAll(location) + if err != nil { + return r, fmt.Errorf("error in cleaning checkout path: %s", err) + } + err = impl.Init(gitCtx, location, url, true) + if err != nil { + return r, fmt.Errorf("err in git init: %s", err) + } + r, err = impl.OpenRepoPlain(location) + if err != nil { + return r, fmt.Errorf("err in git init: %s", err) + } + } + return r, nil +} diff --git a/pkg/git/GoGitSDKManager.go b/pkg/git/GoGitSDKManager.go index e1ee433b..0f863410 100644 --- a/pkg/git/GoGitSDKManager.go +++ b/pkg/git/GoGitSDKManager.go @@ -2,6 +2,7 @@ package git import ( "fmt" + "go.uber.org/zap" "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/config" "gopkg.in/src-d/go-git.v4/plumbing" @@ -12,12 +13,14 @@ type GoGitSDKManager interface { GitManager } type GoGitSDKManagerImpl struct { - *GitManagerBaseImpl + GitManagerBase + logger *zap.SugaredLogger } -func NewGoGitSDKManagerImpl(baseManagerImpl *GitManagerBaseImpl) *GoGitSDKManagerImpl { +func NewGoGitSDKManagerImpl(baseManager GitManagerBase, logger *zap.SugaredLogger) *GoGitSDKManagerImpl { return &GoGitSDKManagerImpl{ - GitManagerBaseImpl: baseManagerImpl, + GitManagerBase: baseManager, + logger: logger, } } @@ -131,3 +134,23 @@ func (impl *GoGitSDKManagerImpl) GetCommitStats(gitCtx GitContext, commit GitCom } return transformFileStats(stats), err } + +func (impl *GoGitSDKManagerImpl) OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) { + + r, err := impl.OpenRepoPlain(location) + if err != nil { + err = os.RemoveAll(location) + if err != nil { + return r, fmt.Errorf("error in cleaning checkout path: %s", err) + } + err = impl.Init(gitCtx, location, url, true) + if err != nil { + return r, fmt.Errorf("err in git init: %s", err) + } + r, err = impl.OpenRepoPlain(location) + if err != nil { + return r, fmt.Errorf("err in git init: %s", err) + } + } + return r, nil +} diff --git a/pkg/git/RepositoryManager.go b/pkg/git/RepositoryManager.go index 5f0c2332..9b663278 100644 --- a/pkg/git/RepositoryManager.go +++ b/pkg/git/RepositoryManager.go @@ -58,18 +58,19 @@ type RepositoryManager interface { GetCommitForTag(gitCtx GitContext, checkoutPath, tag string) (*GitCommitBase, error) // CreateSshFileIfNotExistsAndConfigureSshCommand creates ssh file with creds and configures it at the location CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx GitContext, location string, gitProviderId int, sshPrivateKeyContent string) (string, error) + OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) } type RepositoryManagerImpl struct { logger *zap.SugaredLogger - gitManager GitManagerImpl + gitManager GitManager configuration *internals.Configuration } func NewRepositoryManagerImpl( logger *zap.SugaredLogger, configuration *internals.Configuration, - gitManager GitManagerImpl, + gitManager GitManager, ) *RepositoryManagerImpl { return &RepositoryManagerImpl{logger: logger, configuration: configuration, gitManager: gitManager} } @@ -186,7 +187,7 @@ func (impl RepositoryManagerImpl) Fetch(gitCtx GitContext, url string, location err = errors.New("git-sensor PVC - disk full, please increase space") return false, nil, err } - r, err := impl.gitManager.OpenNewRepo(gitCtx, location, url) + r, err := impl.OpenNewRepo(gitCtx, location, url) if err != nil { return false, r, err } @@ -374,3 +375,26 @@ func (impl RepositoryManagerImpl) CreateSshFileIfNotExistsAndConfigureSshCommand return sshPrivateKeyPath, nil } + +func (impl RepositoryManagerImpl) OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) { + + r, err := impl.gitManager.OpenRepoPlain(location) + if err != nil { + err = os.RemoveAll(location) + if err != nil { + impl.logger.Errorw("error in cleaning checkout path: %s", err) + return r, err + } + err = impl.gitManager.Init(gitCtx, location, url, true) + if err != nil { + impl.logger.Errorw("err in git init: %s", err) + return r, err + } + r, err = impl.gitManager.OpenRepoPlain(location) + if err != nil { + impl.logger.Errorw("err in git init: %s", err) + return r, err + } + } + return r, nil +} diff --git a/pkg/git/RepositoryManagerAnalytics.go b/pkg/git/RepositoryManagerAnalytics.go index 064d043f..6bb08641 100644 --- a/pkg/git/RepositoryManagerAnalytics.go +++ b/pkg/git/RepositoryManagerAnalytics.go @@ -2,7 +2,9 @@ package git import ( "fmt" + "github.com/devtron-labs/git-sensor/internals" "github.com/devtron-labs/git-sensor/util" + "go.uber.org/zap" "gopkg.in/src-d/go-git.v4" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/object" @@ -17,11 +19,20 @@ type RepositoryManagerAnalytics interface { } type RepositoryManagerAnalyticsImpl struct { - *RepositoryManagerImpl + repoManager RepositoryManager + gitManager GitManager + configuration *internals.Configuration + logger *zap.SugaredLogger } -func NewRepositoryManagerAnalyticsImpl(repositoryManagerImpl *RepositoryManagerImpl) *RepositoryManagerAnalyticsImpl { - return &RepositoryManagerAnalyticsImpl{RepositoryManagerImpl: repositoryManagerImpl} +func NewRepositoryManagerAnalyticsImpl(repoManager RepositoryManager, gitManager GitManager, + configuration *internals.Configuration, logger *zap.SugaredLogger) *RepositoryManagerAnalyticsImpl { + return &RepositoryManagerAnalyticsImpl{ + repoManager: repoManager, + gitManager: gitManager, + configuration: configuration, + logger: logger, + } } func computeDiff(r *git.Repository, newHash *plumbing.Hash, oldHash *plumbing.Hash) ([]*object.Commit, error) { diff --git a/pkg/git/Watcher.go b/pkg/git/Watcher.go index aa991ed7..c5eeb87a 100644 --- a/pkg/git/Watcher.go +++ b/pkg/git/Watcher.go @@ -46,7 +46,7 @@ type GitWatcherImpl struct { pollConfig *PollConfig webhookHandler WebhookHandler configuration *internals.Configuration - gitManager GitManagerImpl + gitManager GitManager } type GitWatcher interface { @@ -64,7 +64,7 @@ func NewGitWatcherImpl(repositoryManager RepositoryManager, ciPipelineMaterialRepository sql.CiPipelineMaterialRepository, locker *internals.RepositoryLocker, pubSubClient *pubsub.PubSubClientServiceImpl, webhookHandler WebhookHandler, configuration *internals.Configuration, - gitmanager GitManagerImpl, + gitmanager GitManager, ) (*GitWatcherImpl, error) { cfg := &PollConfig{} diff --git a/wire.go b/wire.go index cee631d6..e839964b 100644 --- a/wire.go +++ b/wire.go @@ -41,8 +41,6 @@ func InitializeApp() (*App, error) { api.NewRestHandlerImpl, wire.Bind(new(api.RestHandler), new(*api.RestHandlerImpl)), api.NewGrpcHandlerImpl, - pkg.NewRepoManagerImpl, - wire.Bind(new(pkg.RepoManager), new(*pkg.RepoManagerImpl)), sql.NewMaterialRepositoryImpl, wire.Bind(new(sql.MaterialRepository), new(*sql.MaterialRepositoryImpl)), sql.NewDbConnection, @@ -51,16 +49,20 @@ func InitializeApp() (*App, error) { wire.Bind(new(sql.CiPipelineMaterialRepository), new(*sql.CiPipelineMaterialRepositoryImpl)), sql.NewGitProviderRepositoryImpl, wire.Bind(new(sql.GitProviderRepository), new(*sql.GitProviderRepositoryImpl)), - git.NewRepositoryManagerImpl, - wire.Bind(new(git.RepositoryManager), new(*git.RepositoryManagerImpl)), - git.NewRepositoryManagerAnalyticsImpl, - wire.Bind(new(git.RepositoryManagerAnalytics), new(*git.RepositoryManagerAnalyticsImpl)), + git.NewGitManagerBaseImpl, + wire.Bind(new(git.GitManagerBase), new(*git.GitManagerBaseImpl)), git.NewGoGitSDKManagerImpl, wire.Bind(new(git.GoGitSDKManager), new(*git.GoGitSDKManagerImpl)), git.NewGitCliManagerImpl, wire.Bind(new(git.GitCliManager), new(*git.GitCliManagerImpl)), git.NewGitManagerImpl, - git.NewGitManagerBaseImpl, + wire.Bind(new(git.GitManager), new(*git.GitManagerImpl)), + git.NewRepositoryManagerImpl, + wire.Bind(new(git.RepositoryManager), new(*git.RepositoryManagerImpl)), + git.NewRepositoryManagerAnalyticsImpl, + wire.Bind(new(git.RepositoryManagerAnalytics), new(*git.RepositoryManagerAnalyticsImpl)), + pkg.NewRepoManagerImpl, + wire.Bind(new(pkg.RepoManager), new(*pkg.RepoManagerImpl)), git.NewGitWatcherImpl, wire.Bind(new(git.GitWatcher), new(*git.GitWatcherImpl)), internals.NewRepositoryLocker, diff --git a/wire_gen.go b/wire_gen.go index d189fa75..3157d659 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -35,11 +35,11 @@ func InitializeApp() (*App, error) { return nil, err } gitManagerBaseImpl := git.NewGitManagerBaseImpl(sugaredLogger, configuration) - gitCliManagerImpl := git.NewGitCliManagerImpl(gitManagerBaseImpl) - goGitSDKManagerImpl := git.NewGoGitSDKManagerImpl(gitManagerBaseImpl) + gitCliManagerImpl := git.NewGitCliManagerImpl(gitManagerBaseImpl, sugaredLogger) + goGitSDKManagerImpl := git.NewGoGitSDKManagerImpl(gitManagerBaseImpl, sugaredLogger) gitManagerImpl := git.NewGitManagerImpl(configuration, gitCliManagerImpl, goGitSDKManagerImpl) repositoryManagerImpl := git.NewRepositoryManagerImpl(sugaredLogger, configuration, gitManagerImpl) - repositoryManagerAnalyticsImpl := git.NewRepositoryManagerAnalyticsImpl(repositoryManagerImpl) + repositoryManagerAnalyticsImpl := git.NewRepositoryManagerAnalyticsImpl(repositoryManagerImpl, gitManagerImpl, configuration, sugaredLogger) gitProviderRepositoryImpl := sql.NewGitProviderRepositoryImpl(db) ciPipelineMaterialRepositoryImpl := sql.NewCiPipelineMaterialRepositoryImpl(db, sugaredLogger) repositoryLocker := internals.NewRepositoryLocker(sugaredLogger) From 69598026d2ad7e051c571dd1f7dc529bdc0bb891 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 1 Feb 2024 12:35:41 +0530 Subject: [PATCH 09/24] code refactoring --- pkg/git/RepositoryManager.go | 38 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/pkg/git/RepositoryManager.go b/pkg/git/RepositoryManager.go index 9b663278..5d6f595d 100644 --- a/pkg/git/RepositoryManager.go +++ b/pkg/git/RepositoryManager.go @@ -41,8 +41,8 @@ type RepositoryManager interface { Fetch(gitCtx GitContext, url string, location string) (updated bool, repo *GitRepository, err error) // Add adds and initializes a new git repo , cleans the directory if not empty and fetches latest commits Add(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error - Add1(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) (string, error) - Add2(gitCtx GitContext, location string) error + GetSshPrivateKeyPath(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) (string, error) + FetchRepo(gitCtx GitContext, location string) error GetLocationForMaterial(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) GetCheckoutPathAndLocation(gitCtx GitContext, material *sql.GitMaterial, url string) (string, string, error) TrimLastGitCommit(gitCommits []*GitCommitBase, count int) []*GitCommitBase @@ -75,7 +75,7 @@ func NewRepositoryManagerImpl( return &RepositoryManagerImpl{logger: logger, configuration: configuration, gitManager: gitManager} } -func (impl RepositoryManagerImpl) IsSpaceAvailableOnDisk() bool { +func (impl *RepositoryManagerImpl) IsSpaceAvailableOnDisk() bool { var statFs unix.Statfs_t err := unix.Statfs(GIT_BASE_DIR, &statFs) if err != nil { @@ -85,7 +85,7 @@ func (impl RepositoryManagerImpl) IsSpaceAvailableOnDisk() bool { return availableSpace > int64(impl.configuration.MinLimit)*1024*1024 } -func (impl RepositoryManagerImpl) GetLocationForMaterial(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) { +func (impl *RepositoryManagerImpl) GetLocationForMaterial(material *sql.GitMaterial, cloningMode string) (location string, httpMatched bool, shMatched bool, err error) { //gitRegex := `/(?:git|ssh|https?|git@[-\w.]+):(\/\/)?(.*?)(\.git)(\/?|\#[-\d\w._]+?)$/` httpsRegex := `^https.*` httpsMatched, err := regexp.MatchString(httpsRegex, material.Url) @@ -105,7 +105,7 @@ func (impl RepositoryManagerImpl) GetLocationForMaterial(material *sql.GitMateri return "", httpsMatched, sshMatched, fmt.Errorf("unsupported format url %s", material.Url) } -func (impl RepositoryManagerImpl) GetCheckoutPathAndLocation(gitCtx GitContext, material *sql.GitMaterial, url string) (string, string, error) { +func (impl *RepositoryManagerImpl) GetCheckoutPathAndLocation(gitCtx GitContext, material *sql.GitMaterial, url string) (string, string, error) { var checkoutPath string var checkoutLocationForFetching string checkoutPath, _, _, err := impl.GetLocationForMaterial(material, gitCtx.CloningMode) @@ -116,15 +116,15 @@ func (impl RepositoryManagerImpl) GetCheckoutPathAndLocation(gitCtx GitContext, return checkoutPath, checkoutLocationForFetching, nil } -func (impl RepositoryManagerImpl) Add(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error { - _, err := impl.Add1(gitCtx, gitProviderId, location, url, authMode, sshPrivateKeyContent) +func (impl *RepositoryManagerImpl) Add(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) error { + _, err := impl.GetSshPrivateKeyPath(gitCtx, gitProviderId, location, url, authMode, sshPrivateKeyContent) if err != nil { return err } - return impl.Add2(gitCtx, location) + return impl.FetchRepo(gitCtx, location) } -func (impl RepositoryManagerImpl) Add1(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) (string, error) { +func (impl *RepositoryManagerImpl) GetSshPrivateKeyPath(gitCtx GitContext, gitProviderId int, location, url string, authMode sql.AuthMode, sshPrivateKeyContent string) (string, error) { var err error start := time.Now() defer func() { @@ -157,7 +157,7 @@ func (impl RepositoryManagerImpl) Add1(gitCtx GitContext, gitProviderId int, loc return sshPrivateKeyPath, nil } -func (impl RepositoryManagerImpl) Add2(gitCtx GitContext, location string) error { +func (impl *RepositoryManagerImpl) FetchRepo(gitCtx GitContext, location string) error { opt, errorMsg, err := impl.gitManager.Fetch(gitCtx, location) if err != nil { impl.logger.Errorw("error in fetching repo", "errorMsg", errorMsg, "err", err) @@ -167,7 +167,7 @@ func (impl RepositoryManagerImpl) Add2(gitCtx GitContext, location string) error return nil } -func (impl RepositoryManagerImpl) Clean(dir string) error { +func (impl *RepositoryManagerImpl) Clean(dir string) error { var err error start := time.Now() defer func() { @@ -177,7 +177,7 @@ func (impl RepositoryManagerImpl) Clean(dir string) error { return err } -func (impl RepositoryManagerImpl) Fetch(gitCtx GitContext, url string, location string) (updated bool, repo *GitRepository, err error) { +func (impl *RepositoryManagerImpl) Fetch(gitCtx GitContext, url string, location string) (updated bool, repo *GitRepository, err error) { start := time.Now() defer func() { util.TriggerGitOperationMetrics("fetch", start, err) @@ -210,7 +210,7 @@ func (impl RepositoryManagerImpl) Fetch(gitCtx GitContext, url string, location } -func (impl RepositoryManagerImpl) GetCommitForTag(gitCtx GitContext, checkoutPath, tag string) (*GitCommitBase, error) { +func (impl *RepositoryManagerImpl) GetCommitForTag(gitCtx GitContext, checkoutPath, tag string) (*GitCommitBase, error) { var err error start := time.Now() defer func() { @@ -224,7 +224,7 @@ func (impl RepositoryManagerImpl) GetCommitForTag(gitCtx GitContext, checkoutPat return commit.GetCommit(), nil } -func (impl RepositoryManagerImpl) GetCommitMetadata(gitCtx GitContext, checkoutPath, commitHash string) (*GitCommitBase, error) { +func (impl *RepositoryManagerImpl) GetCommitMetadata(gitCtx GitContext, checkoutPath, commitHash string) (*GitCommitBase, error) { var err error start := time.Now() defer func() { @@ -240,7 +240,7 @@ func (impl RepositoryManagerImpl) GetCommitMetadata(gitCtx GitContext, checkoutP // from -> old commit // to -> new commit -func (impl RepositoryManagerImpl) ChangesSinceByRepository(gitCtx GitContext, repository *GitRepository, branch string, from string, to string, count int, checkoutPath string) ([]*GitCommitBase, error) { +func (impl *RepositoryManagerImpl) ChangesSinceByRepository(gitCtx GitContext, repository *GitRepository, branch string, from string, to string, count int, checkoutPath string) ([]*GitCommitBase, error) { // fix for azure devops (manual trigger webhook bases pipeline) : // branch name comes as 'refs/heads/master', we need to extract actual branch name out of it. // https://stackoverflow.com/questions/59956206/how-to-get-a-branch-name-with-a-slash-in-azure-devops @@ -326,14 +326,14 @@ func (impl RepositoryManagerImpl) ChangesSinceByRepository(gitCtx GitContext, re return gitCommits, err } -func (impl RepositoryManagerImpl) TrimLastGitCommit(gitCommits []*GitCommitBase, count int) []*GitCommitBase { +func (impl *RepositoryManagerImpl) TrimLastGitCommit(gitCommits []*GitCommitBase, count int) []*GitCommitBase { if len(gitCommits) > count { gitCommits = gitCommits[:len(gitCommits)-1] } return gitCommits } -func (impl RepositoryManagerImpl) ChangesSince(gitCtx GitContext, checkoutPath string, branch string, from string, to string, count int) ([]*GitCommitBase, error) { +func (impl *RepositoryManagerImpl) ChangesSince(gitCtx GitContext, checkoutPath string, branch string, from string, to string, count int) ([]*GitCommitBase, error) { var err error start := time.Now() defer func() { @@ -352,7 +352,7 @@ func (impl RepositoryManagerImpl) ChangesSince(gitCtx GitContext, checkoutPath s } -func (impl RepositoryManagerImpl) CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx GitContext, location string, gitProviderId int, sshPrivateKeyContent string) (string, error) { +func (impl *RepositoryManagerImpl) CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx GitContext, location string, gitProviderId int, sshPrivateKeyContent string) (string, error) { // add private key var err error var sshPrivateKeyPath string @@ -376,7 +376,7 @@ func (impl RepositoryManagerImpl) CreateSshFileIfNotExistsAndConfigureSshCommand return sshPrivateKeyPath, nil } -func (impl RepositoryManagerImpl) OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) { +func (impl *RepositoryManagerImpl) OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) { r, err := impl.gitManager.OpenRepoPlain(location) if err != nil { From 4590a56c87a557c6890b38518cd19cff397fa75b Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 1 Feb 2024 13:50:34 +0530 Subject: [PATCH 10/24] code refactoring --- pkg/git/GitCliManager.go | 21 --------------------- pkg/git/GoGitSDKManager.go | 20 -------------------- 2 files changed, 41 deletions(-) diff --git a/pkg/git/GitCliManager.go b/pkg/git/GitCliManager.go index c998cf5b..37ea06c5 100644 --- a/pkg/git/GitCliManager.go +++ b/pkg/git/GitCliManager.go @@ -1,7 +1,6 @@ package git import ( - "fmt" "go.uber.org/zap" "gopkg.in/src-d/go-billy.v4/osfs" "os" @@ -191,23 +190,3 @@ func (impl *GitCliManagerImpl) processGitLogOutput(out string) ([]GitCommit, err } return gitCommits, nil } - -func (impl *GitCliManagerImpl) OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) { - - r, err := impl.OpenRepoPlain(location) - if err != nil { - err = os.RemoveAll(location) - if err != nil { - return r, fmt.Errorf("error in cleaning checkout path: %s", err) - } - err = impl.Init(gitCtx, location, url, true) - if err != nil { - return r, fmt.Errorf("err in git init: %s", err) - } - r, err = impl.OpenRepoPlain(location) - if err != nil { - return r, fmt.Errorf("err in git init: %s", err) - } - } - return r, nil -} diff --git a/pkg/git/GoGitSDKManager.go b/pkg/git/GoGitSDKManager.go index 0f863410..66e43170 100644 --- a/pkg/git/GoGitSDKManager.go +++ b/pkg/git/GoGitSDKManager.go @@ -134,23 +134,3 @@ func (impl *GoGitSDKManagerImpl) GetCommitStats(gitCtx GitContext, commit GitCom } return transformFileStats(stats), err } - -func (impl *GoGitSDKManagerImpl) OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) { - - r, err := impl.OpenRepoPlain(location) - if err != nil { - err = os.RemoveAll(location) - if err != nil { - return r, fmt.Errorf("error in cleaning checkout path: %s", err) - } - err = impl.Init(gitCtx, location, url, true) - if err != nil { - return r, fmt.Errorf("err in git init: %s", err) - } - r, err = impl.OpenRepoPlain(location) - if err != nil { - return r, fmt.Errorf("err in git init: %s", err) - } - } - return r, nil -} From 87c7fd6101802a47ea93369e8a6de6ebfb81c2c2 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 1 Feb 2024 13:52:06 +0530 Subject: [PATCH 11/24] code refactoring --- pkg/git/GitCliManager.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/git/GitCliManager.go b/pkg/git/GitCliManager.go index 37ea06c5..a0278439 100644 --- a/pkg/git/GitCliManager.go +++ b/pkg/git/GitCliManager.go @@ -101,7 +101,7 @@ func (impl *GitCliManagerImpl) GitCreateRemote(gitCtx GitContext, rootDir string impl.logger.Debugw("git", "-C", rootDir, "remote", "add", "origin", url) cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "remote", "add", "origin", url) defer cancel() - output, errMsg, err := impl.GitManagerBase.RunCommand(cmd) + output, errMsg, err := impl.RunCommand(cmd) impl.logger.Debugw("url", url, "opt", output, "errMsg", errMsg, "error", err) return err } @@ -115,7 +115,7 @@ func (impl *GitCliManagerImpl) GetCommits(gitCtx GitContext, branchRef string, b impl.logger.Debugw("git", cmdArgs) cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", cmdArgs...) defer cancel() - output, errMsg, err := impl.GitManagerBase.RunCommand(cmd) + output, errMsg, err := impl.RunCommand(cmd) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil { return nil, err @@ -142,7 +142,7 @@ func (impl *GitCliManagerImpl) GitShow(gitCtx GitContext, rootDir string, hash s impl.logger.Debugw("git", "-C", rootDir, "show", hash, "--date=iso-strict", GITFORMAT, "-s") cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "show", hash, "--date=iso-strict", GITFORMAT, "-s") defer cancel() - output, errMsg, err := impl.GitManagerBase.RunCommand(cmd) + output, errMsg, err := impl.RunCommand(cmd) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil { return nil, err From 1e4829a00c4caabf7076551969cb11ac5b1afbbe Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 1 Feb 2024 16:01:17 +0530 Subject: [PATCH 12/24] code refactoring as suggested in code review --- pkg/git/GitBaseManager.go | 23 +++++++---------------- pkg/git/GitCliManager.go | 14 +++++++------- pkg/git/GoGitSDKManager.go | 6 +++--- pkg/git/RepositoryManagerIT_test.go | 2 +- wire.go | 16 ++++++++-------- wire_gen.go | 13 +++++-------- 6 files changed, 31 insertions(+), 43 deletions(-) diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index 6444c94e..f473a072 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -45,9 +45,9 @@ type GitManagerBase interface { FetchDiffStatBetweenCommits(gitCtx GitContext, oldHash string, newHash string, rootDir string) (response, errMsg string, err error) // LogMergeBase get the commit diff between using a merge base strategy LogMergeBase(gitCtx GitContext, rootDir, from string, to string) ([]*Commit, error) - CreateCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc) - RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) - RunCommand(cmd *exec.Cmd) (response, errMsg string, err error) + //CreateCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc) + //RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) + //RunCommand(cmd *exec.Cmd) (response, errMsg string, err error) } type GitManagerBaseImpl struct { logger *zap.SugaredLogger @@ -74,22 +74,13 @@ func parseCmdTimeoutJson(config *internals.Configuration) (map[string]int, error return commandTimeoutMap, err } -type GitManagerImpl struct { - GitManager -} - -func NewGitManagerImpl(configuration *internals.Configuration, - cliGitManager GitCliManager, - goGitManager GoGitSDKManager) *GitManagerImpl { +func NewGitManager(logger *zap.SugaredLogger, configuration *internals.Configuration) GitManager { + baseImpl := NewGitManagerBaseImpl(logger, configuration) if configuration.UseGitCli { - return &GitManagerImpl{ - cliGitManager, - } - } - return &GitManagerImpl{ - goGitManager, + return NewGitCliManagerImpl(baseImpl, logger) } + return NewGoGitSDKManagerImpl(baseImpl, logger) } func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (response, errMsg string, err error) { diff --git a/pkg/git/GitCliManager.go b/pkg/git/GitCliManager.go index a0278439..9819e631 100644 --- a/pkg/git/GitCliManager.go +++ b/pkg/git/GitCliManager.go @@ -8,20 +8,20 @@ import ( "strconv" ) -type GitCliManager interface { - GitManager -} +//type GitCliManager interface { +// //GitManager +//} type GitCliManagerImpl struct { - GitManagerBase + *GitManagerBaseImpl logger *zap.SugaredLogger } -func NewGitCliManagerImpl(baseManager GitManagerBase, logger *zap.SugaredLogger) *GitCliManagerImpl { +func NewGitCliManagerImpl(baseManager *GitManagerBaseImpl, logger *zap.SugaredLogger) *GitCliManagerImpl { return &GitCliManagerImpl{ - GitManagerBase: baseManager, - logger: logger, + GitManagerBaseImpl: baseManager, + logger: logger, } } diff --git a/pkg/git/GoGitSDKManager.go b/pkg/git/GoGitSDKManager.go index 66e43170..49935a2f 100644 --- a/pkg/git/GoGitSDKManager.go +++ b/pkg/git/GoGitSDKManager.go @@ -9,9 +9,9 @@ import ( "os" ) -type GoGitSDKManager interface { - GitManager -} +// type GoGitSDKManager interface { +// GitManager +// } type GoGitSDKManagerImpl struct { GitManagerBase logger *zap.SugaredLogger diff --git a/pkg/git/RepositoryManagerIT_test.go b/pkg/git/RepositoryManagerIT_test.go index c62e6721..428e2b40 100644 --- a/pkg/git/RepositoryManagerIT_test.go +++ b/pkg/git/RepositoryManagerIT_test.go @@ -43,7 +43,7 @@ func getRepoManagerImpl(t *testing.T) *RepositoryManagerImpl { gitCliImpl := NewGitCliManagerImpl(base) gogitImpl := NewGoGitSDKManagerImpl(base) - gitUtil := NewGitManagerImpl(conf, gitCliImpl, gogitImpl) + gitUtil := NewGitManager(conf, gitCliImpl, gogitImpl) repositoryManagerImpl := NewRepositoryManagerImpl(logger, conf, gitUtil) return repositoryManagerImpl } diff --git a/wire.go b/wire.go index e839964b..8b2ace7a 100644 --- a/wire.go +++ b/wire.go @@ -49,14 +49,14 @@ func InitializeApp() (*App, error) { wire.Bind(new(sql.CiPipelineMaterialRepository), new(*sql.CiPipelineMaterialRepositoryImpl)), sql.NewGitProviderRepositoryImpl, wire.Bind(new(sql.GitProviderRepository), new(*sql.GitProviderRepositoryImpl)), - git.NewGitManagerBaseImpl, - wire.Bind(new(git.GitManagerBase), new(*git.GitManagerBaseImpl)), - git.NewGoGitSDKManagerImpl, - wire.Bind(new(git.GoGitSDKManager), new(*git.GoGitSDKManagerImpl)), - git.NewGitCliManagerImpl, - wire.Bind(new(git.GitCliManager), new(*git.GitCliManagerImpl)), - git.NewGitManagerImpl, - wire.Bind(new(git.GitManager), new(*git.GitManagerImpl)), + //git.NewGitManagerBaseImpl, + //wire.Bind(new(git.GitManagerBase), new(*git.GitManagerBaseImpl)), + //git.NewGoGitSDKManagerImpl, + //wire.Bind(new(git.GoGitSDKManager), new(*git.GoGitSDKManagerImpl)), + //git.NewGitCliManagerImpl, + //wire.Bind(new(git.GitCliManager), new(*git.GitCliManagerImpl)), + git.NewGitManager, + //wire.Bind(new(git.GitManager), new(*git.GitManagerImpl)), git.NewRepositoryManagerImpl, wire.Bind(new(git.RepositoryManager), new(*git.RepositoryManagerImpl)), git.NewRepositoryManagerAnalyticsImpl, diff --git a/wire_gen.go b/wire_gen.go index 3157d659..8973a288 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -34,12 +34,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - gitManagerBaseImpl := git.NewGitManagerBaseImpl(sugaredLogger, configuration) - gitCliManagerImpl := git.NewGitCliManagerImpl(gitManagerBaseImpl, sugaredLogger) - goGitSDKManagerImpl := git.NewGoGitSDKManagerImpl(gitManagerBaseImpl, sugaredLogger) - gitManagerImpl := git.NewGitManagerImpl(configuration, gitCliManagerImpl, goGitSDKManagerImpl) - repositoryManagerImpl := git.NewRepositoryManagerImpl(sugaredLogger, configuration, gitManagerImpl) - repositoryManagerAnalyticsImpl := git.NewRepositoryManagerAnalyticsImpl(repositoryManagerImpl, gitManagerImpl, configuration, sugaredLogger) + gitManager := git.NewGitManager(sugaredLogger, configuration) + repositoryManagerImpl := git.NewRepositoryManagerImpl(sugaredLogger, configuration, gitManager) + repositoryManagerAnalyticsImpl := git.NewRepositoryManagerAnalyticsImpl(repositoryManagerImpl, gitManager, configuration, sugaredLogger) gitProviderRepositoryImpl := sql.NewGitProviderRepositoryImpl(db) ciPipelineMaterialRepositoryImpl := sql.NewCiPipelineMaterialRepositoryImpl(db, sugaredLogger) repositoryLocker := internals.NewRepositoryLocker(sugaredLogger) @@ -52,11 +49,11 @@ func InitializeApp() (*App, error) { webhookEventServiceImpl := git.NewWebhookEventServiceImpl(sugaredLogger, webhookEventRepositoryImpl, webhookEventParsedDataRepositoryImpl, webhookEventDataMappingRepositoryImpl, webhookEventDataMappingFilterResultRepositoryImpl, materialRepositoryImpl, pubSubClientServiceImpl, webhookEventBeanConverterImpl) webhookEventParserImpl := git.NewWebhookEventParserImpl(sugaredLogger) webhookHandlerImpl := git.NewWebhookHandlerImpl(sugaredLogger, webhookEventServiceImpl, webhookEventParserImpl) - gitWatcherImpl, err := git.NewGitWatcherImpl(repositoryManagerImpl, materialRepositoryImpl, sugaredLogger, ciPipelineMaterialRepositoryImpl, repositoryLocker, pubSubClientServiceImpl, webhookHandlerImpl, configuration, gitManagerImpl) + gitWatcherImpl, err := git.NewGitWatcherImpl(repositoryManagerImpl, materialRepositoryImpl, sugaredLogger, ciPipelineMaterialRepositoryImpl, repositoryLocker, pubSubClientServiceImpl, webhookHandlerImpl, configuration, gitManager) if err != nil { return nil, err } - repoManagerImpl := pkg.NewRepoManagerImpl(sugaredLogger, materialRepositoryImpl, repositoryManagerImpl, repositoryManagerAnalyticsImpl, gitProviderRepositoryImpl, ciPipelineMaterialRepositoryImpl, repositoryLocker, gitWatcherImpl, webhookEventRepositoryImpl, webhookEventParsedDataRepositoryImpl, webhookEventDataMappingRepositoryImpl, webhookEventDataMappingFilterResultRepositoryImpl, webhookEventBeanConverterImpl, configuration, gitManagerImpl) + repoManagerImpl := pkg.NewRepoManagerImpl(sugaredLogger, materialRepositoryImpl, repositoryManagerImpl, repositoryManagerAnalyticsImpl, gitProviderRepositoryImpl, ciPipelineMaterialRepositoryImpl, repositoryLocker, gitWatcherImpl, webhookEventRepositoryImpl, webhookEventParsedDataRepositoryImpl, webhookEventDataMappingRepositoryImpl, webhookEventDataMappingFilterResultRepositoryImpl, webhookEventBeanConverterImpl, configuration, gitManager) restHandlerImpl := api.NewRestHandlerImpl(repoManagerImpl, sugaredLogger) monitoringRouter := monitoring.NewMonitoringRouter(sugaredLogger) muxRouter := api.NewMuxRouter(sugaredLogger, restHandlerImpl, monitoringRouter) From 2bbe2964423734e780f894c29fd42169b93c294c Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 1 Feb 2024 16:01:46 +0530 Subject: [PATCH 13/24] commented code removed --- wire.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/wire.go b/wire.go index 8b2ace7a..a1e1484b 100644 --- a/wire.go +++ b/wire.go @@ -49,14 +49,7 @@ func InitializeApp() (*App, error) { wire.Bind(new(sql.CiPipelineMaterialRepository), new(*sql.CiPipelineMaterialRepositoryImpl)), sql.NewGitProviderRepositoryImpl, wire.Bind(new(sql.GitProviderRepository), new(*sql.GitProviderRepositoryImpl)), - //git.NewGitManagerBaseImpl, - //wire.Bind(new(git.GitManagerBase), new(*git.GitManagerBaseImpl)), - //git.NewGoGitSDKManagerImpl, - //wire.Bind(new(git.GoGitSDKManager), new(*git.GoGitSDKManagerImpl)), - //git.NewGitCliManagerImpl, - //wire.Bind(new(git.GitCliManager), new(*git.GitCliManagerImpl)), git.NewGitManager, - //wire.Bind(new(git.GitManager), new(*git.GitManagerImpl)), git.NewRepositoryManagerImpl, wire.Bind(new(git.RepositoryManager), new(*git.RepositoryManagerImpl)), git.NewRepositoryManagerAnalyticsImpl, From a02805a64786e8a2dc37d073bc69a60cff74d890 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 1 Feb 2024 17:02:34 +0530 Subject: [PATCH 14/24] commented code retrieved --- pkg/git/GitBaseManager.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index f473a072..0f5094b9 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -45,9 +45,9 @@ type GitManagerBase interface { FetchDiffStatBetweenCommits(gitCtx GitContext, oldHash string, newHash string, rootDir string) (response, errMsg string, err error) // LogMergeBase get the commit diff between using a merge base strategy LogMergeBase(gitCtx GitContext, rootDir, from string, to string) ([]*Commit, error) - //CreateCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc) - //RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) - //RunCommand(cmd *exec.Cmd) (response, errMsg string, err error) + CreateCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc) + RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) + RunCommand(cmd *exec.Cmd) (response, errMsg string, err error) } type GitManagerBaseImpl struct { logger *zap.SugaredLogger From e9b3856446d3cf0ea46ca9a5d8ae0b18de78f2a8 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 1 Feb 2024 17:09:28 +0530 Subject: [PATCH 15/24] GitManagerImpl retrieved --- pkg/git/GitBaseManager.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index 0f5094b9..3ead1c12 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -65,13 +65,8 @@ func NewGitManagerBaseImpl(logger *zap.SugaredLogger, config *internals.Configur return &GitManagerBaseImpl{logger: logger, conf: config, commandTimeoutMap: commandTimeoutMap} } -func parseCmdTimeoutJson(config *internals.Configuration) (map[string]int, error) { - commandTimeoutMap := make(map[string]int) - var err error - if config.CliCmdTimeoutJson != "" { - err = json.Unmarshal([]byte(config.CliCmdTimeoutJson), &commandTimeoutMap) - } - return commandTimeoutMap, err +type GitManagerImpl struct { + GitManager } func NewGitManager(logger *zap.SugaredLogger, configuration *internals.Configuration) GitManager { @@ -83,6 +78,15 @@ func NewGitManager(logger *zap.SugaredLogger, configuration *internals.Configura return NewGoGitSDKManagerImpl(baseImpl, logger) } +func parseCmdTimeoutJson(config *internals.Configuration) (map[string]int, error) { + commandTimeoutMap := make(map[string]int) + var err error + if config.CliCmdTimeoutJson != "" { + err = json.Unmarshal([]byte(config.CliCmdTimeoutJson), &commandTimeoutMap) + } + return commandTimeoutMap, err +} + func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (response, errMsg string, err error) { impl.logger.Debugw("git fetch ", "location", rootDir) cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force") From 2a6a05ae0160fab461d5020fa0d15d6070a89f7e Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Thu, 1 Feb 2024 17:30:12 +0530 Subject: [PATCH 16/24] GitManagerImpl - wire file corrected --- pkg/git/GitBaseManager.go | 11 ++++++++--- pkg/git/RepositoryManagerIT_test.go | 2 +- wire.go | 3 ++- wire_gen.go | 10 +++++----- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index 3ead1c12..52dcd5c8 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -69,13 +69,18 @@ type GitManagerImpl struct { GitManager } -func NewGitManager(logger *zap.SugaredLogger, configuration *internals.Configuration) GitManager { +func NewGitManagerImpl(logger *zap.SugaredLogger, configuration *internals.Configuration) *GitManagerImpl { baseImpl := NewGitManagerBaseImpl(logger, configuration) if configuration.UseGitCli { - return NewGitCliManagerImpl(baseImpl, logger) + return &GitManagerImpl{ + GitManager: NewGitCliManagerImpl(baseImpl, logger), + } + + } + return &GitManagerImpl{ + GitManager: NewGoGitSDKManagerImpl(baseImpl, logger), } - return NewGoGitSDKManagerImpl(baseImpl, logger) } func parseCmdTimeoutJson(config *internals.Configuration) (map[string]int, error) { diff --git a/pkg/git/RepositoryManagerIT_test.go b/pkg/git/RepositoryManagerIT_test.go index 428e2b40..c62e6721 100644 --- a/pkg/git/RepositoryManagerIT_test.go +++ b/pkg/git/RepositoryManagerIT_test.go @@ -43,7 +43,7 @@ func getRepoManagerImpl(t *testing.T) *RepositoryManagerImpl { gitCliImpl := NewGitCliManagerImpl(base) gogitImpl := NewGoGitSDKManagerImpl(base) - gitUtil := NewGitManager(conf, gitCliImpl, gogitImpl) + gitUtil := NewGitManagerImpl(conf, gitCliImpl, gogitImpl) repositoryManagerImpl := NewRepositoryManagerImpl(logger, conf, gitUtil) return repositoryManagerImpl } diff --git a/wire.go b/wire.go index a1e1484b..0cb358ee 100644 --- a/wire.go +++ b/wire.go @@ -49,7 +49,8 @@ func InitializeApp() (*App, error) { wire.Bind(new(sql.CiPipelineMaterialRepository), new(*sql.CiPipelineMaterialRepositoryImpl)), sql.NewGitProviderRepositoryImpl, wire.Bind(new(sql.GitProviderRepository), new(*sql.GitProviderRepositoryImpl)), - git.NewGitManager, + git.NewGitManagerImpl, + wire.Bind(new(git.GitManager), new(*git.GitManagerImpl)), git.NewRepositoryManagerImpl, wire.Bind(new(git.RepositoryManager), new(*git.RepositoryManagerImpl)), git.NewRepositoryManagerAnalyticsImpl, diff --git a/wire_gen.go b/wire_gen.go index 8973a288..32d80fa4 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -34,9 +34,9 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - gitManager := git.NewGitManager(sugaredLogger, configuration) - repositoryManagerImpl := git.NewRepositoryManagerImpl(sugaredLogger, configuration, gitManager) - repositoryManagerAnalyticsImpl := git.NewRepositoryManagerAnalyticsImpl(repositoryManagerImpl, gitManager, configuration, sugaredLogger) + gitManagerImpl := git.NewGitManagerImpl(sugaredLogger, configuration) + repositoryManagerImpl := git.NewRepositoryManagerImpl(sugaredLogger, configuration, gitManagerImpl) + repositoryManagerAnalyticsImpl := git.NewRepositoryManagerAnalyticsImpl(repositoryManagerImpl, gitManagerImpl, configuration, sugaredLogger) gitProviderRepositoryImpl := sql.NewGitProviderRepositoryImpl(db) ciPipelineMaterialRepositoryImpl := sql.NewCiPipelineMaterialRepositoryImpl(db, sugaredLogger) repositoryLocker := internals.NewRepositoryLocker(sugaredLogger) @@ -49,11 +49,11 @@ func InitializeApp() (*App, error) { webhookEventServiceImpl := git.NewWebhookEventServiceImpl(sugaredLogger, webhookEventRepositoryImpl, webhookEventParsedDataRepositoryImpl, webhookEventDataMappingRepositoryImpl, webhookEventDataMappingFilterResultRepositoryImpl, materialRepositoryImpl, pubSubClientServiceImpl, webhookEventBeanConverterImpl) webhookEventParserImpl := git.NewWebhookEventParserImpl(sugaredLogger) webhookHandlerImpl := git.NewWebhookHandlerImpl(sugaredLogger, webhookEventServiceImpl, webhookEventParserImpl) - gitWatcherImpl, err := git.NewGitWatcherImpl(repositoryManagerImpl, materialRepositoryImpl, sugaredLogger, ciPipelineMaterialRepositoryImpl, repositoryLocker, pubSubClientServiceImpl, webhookHandlerImpl, configuration, gitManager) + gitWatcherImpl, err := git.NewGitWatcherImpl(repositoryManagerImpl, materialRepositoryImpl, sugaredLogger, ciPipelineMaterialRepositoryImpl, repositoryLocker, pubSubClientServiceImpl, webhookHandlerImpl, configuration, gitManagerImpl) if err != nil { return nil, err } - repoManagerImpl := pkg.NewRepoManagerImpl(sugaredLogger, materialRepositoryImpl, repositoryManagerImpl, repositoryManagerAnalyticsImpl, gitProviderRepositoryImpl, ciPipelineMaterialRepositoryImpl, repositoryLocker, gitWatcherImpl, webhookEventRepositoryImpl, webhookEventParsedDataRepositoryImpl, webhookEventDataMappingRepositoryImpl, webhookEventDataMappingFilterResultRepositoryImpl, webhookEventBeanConverterImpl, configuration, gitManager) + repoManagerImpl := pkg.NewRepoManagerImpl(sugaredLogger, materialRepositoryImpl, repositoryManagerImpl, repositoryManagerAnalyticsImpl, gitProviderRepositoryImpl, ciPipelineMaterialRepositoryImpl, repositoryLocker, gitWatcherImpl, webhookEventRepositoryImpl, webhookEventParsedDataRepositoryImpl, webhookEventDataMappingRepositoryImpl, webhookEventDataMappingFilterResultRepositoryImpl, webhookEventBeanConverterImpl, configuration, gitManagerImpl) restHandlerImpl := api.NewRestHandlerImpl(repoManagerImpl, sugaredLogger) monitoringRouter := monitoring.NewMonitoringRouter(sugaredLogger) muxRouter := api.NewMuxRouter(sugaredLogger, restHandlerImpl, monitoringRouter) From b23e4f322da8503da286943a5f5e11fdf10651bc Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 2 Feb 2024 13:11:10 +0530 Subject: [PATCH 17/24] code review comments incorporated --- pkg/git/GitBaseManager.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index 52dcd5c8..4da38cde 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -45,9 +45,7 @@ type GitManagerBase interface { FetchDiffStatBetweenCommits(gitCtx GitContext, oldHash string, newHash string, rootDir string) (response, errMsg string, err error) // LogMergeBase get the commit diff between using a merge base strategy LogMergeBase(gitCtx GitContext, rootDir, from string, to string) ([]*Commit, error) - CreateCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc) - RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) - RunCommand(cmd *exec.Cmd) (response, errMsg string, err error) + ExecuteCustomCommand(gitContext GitContext, name string, arg ...string) (response, errMsg string, err error) } type GitManagerBaseImpl struct { logger *zap.SugaredLogger @@ -295,3 +293,10 @@ func (impl *GitManagerBaseImpl) getCommandTimeout(command string) int { } return timeout } + +func (impl *GitManagerBaseImpl) ExecuteCustomCommand(gitContext GitContext, name string, arg ...string) (response, errMsg string, err error) { + cmd, cancel := impl.CreateCmdWithContext(gitContext, name, arg...) + defer cancel() + output, errMsg, err := impl.RunCommandWithCred(cmd, gitContext.Username, gitContext.Password) + return output, errMsg, err +} From 32b9bd3cb756ebd4ec6649aa514749624a3667d7 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 2 Feb 2024 13:14:55 +0530 Subject: [PATCH 18/24] code review comments incorporated --- pkg/git/RepositoryManager.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/git/RepositoryManager.go b/pkg/git/RepositoryManager.go index 5d6f595d..58c3ffa0 100644 --- a/pkg/git/RepositoryManager.go +++ b/pkg/git/RepositoryManager.go @@ -58,7 +58,6 @@ type RepositoryManager interface { GetCommitForTag(gitCtx GitContext, checkoutPath, tag string) (*GitCommitBase, error) // CreateSshFileIfNotExistsAndConfigureSshCommand creates ssh file with creds and configures it at the location CreateSshFileIfNotExistsAndConfigureSshCommand(gitCtx GitContext, location string, gitProviderId int, sshPrivateKeyContent string) (string, error) - OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) } type RepositoryManagerImpl struct { @@ -187,7 +186,7 @@ func (impl *RepositoryManagerImpl) Fetch(gitCtx GitContext, url string, location err = errors.New("git-sensor PVC - disk full, please increase space") return false, nil, err } - r, err := impl.OpenNewRepo(gitCtx, location, url) + r, err := impl.openNewRepo(gitCtx, location, url) if err != nil { return false, r, err } @@ -376,7 +375,7 @@ func (impl *RepositoryManagerImpl) CreateSshFileIfNotExistsAndConfigureSshComman return sshPrivateKeyPath, nil } -func (impl *RepositoryManagerImpl) OpenNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) { +func (impl *RepositoryManagerImpl) openNewRepo(gitCtx GitContext, location string, url string) (*GitRepository, error) { r, err := impl.gitManager.OpenRepoPlain(location) if err != nil { From 95435836e461756f2946187bfccef55bae59333f Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 2 Feb 2024 13:20:36 +0530 Subject: [PATCH 19/24] app directory created --- App.go => app/App.go | 2 +- wire.go | 7 ++++--- wire_gen.go | 7 ++++--- 3 files changed, 9 insertions(+), 7 deletions(-) rename App.go => app/App.go (99%) diff --git a/App.go b/app/App.go similarity index 99% rename from App.go rename to app/App.go index b73e6929..95a0d104 100644 --- a/App.go +++ b/app/App.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package main +package app import ( "context" diff --git a/wire.go b/wire.go index 0cb358ee..4aefd62d 100644 --- a/wire.go +++ b/wire.go @@ -23,6 +23,7 @@ import ( "github.com/devtron-labs/common-lib/monitoring" pubsub "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/git-sensor/api" + "github.com/devtron-labs/git-sensor/app" "github.com/devtron-labs/git-sensor/internals" "github.com/devtron-labs/git-sensor/internals/logger" "github.com/devtron-labs/git-sensor/internals/sql" @@ -31,10 +32,10 @@ import ( "github.com/google/wire" ) -func InitializeApp() (*App, error) { +func InitializeApp() (*app.App, error) { wire.Build( - NewApp, + app.NewApp, api.NewMuxRouter, internals.ParseConfiguration, logger.NewSugaredLogger, @@ -80,5 +81,5 @@ func InitializeApp() (*App, error) { wire.Bind(new(git.WebhookHandler), new(*git.WebhookHandlerImpl)), monitoring.NewMonitoringRouter, ) - return &App{}, nil + return &app.App{}, nil } diff --git a/wire_gen.go b/wire_gen.go index 32d80fa4..c7b7edf5 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -10,6 +10,7 @@ import ( "github.com/devtron-labs/common-lib/monitoring" "github.com/devtron-labs/common-lib/pubsub-lib" "github.com/devtron-labs/git-sensor/api" + "github.com/devtron-labs/git-sensor/app" "github.com/devtron-labs/git-sensor/internals" "github.com/devtron-labs/git-sensor/internals/logger" "github.com/devtron-labs/git-sensor/internals/sql" @@ -19,7 +20,7 @@ import ( // Injectors from wire.go: -func InitializeApp() (*App, error) { +func InitializeApp() (*app.App, error) { sugaredLogger := logger.NewSugaredLogger() config, err := sql.GetConfig() if err != nil { @@ -58,6 +59,6 @@ func InitializeApp() (*App, error) { monitoringRouter := monitoring.NewMonitoringRouter(sugaredLogger) muxRouter := api.NewMuxRouter(sugaredLogger, restHandlerImpl, monitoringRouter) grpcHandlerImpl := api.NewGrpcHandlerImpl(repoManagerImpl, sugaredLogger) - app := NewApp(muxRouter, sugaredLogger, gitWatcherImpl, db, pubSubClientServiceImpl, grpcHandlerImpl) - return app, nil + appApp := app.NewApp(muxRouter, sugaredLogger, gitWatcherImpl, db, pubSubClientServiceImpl, grpcHandlerImpl) + return appApp, nil } From 49f149882af67f35bf205bc614b84f1342086dd6 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 2 Feb 2024 16:08:40 +0530 Subject: [PATCH 20/24] code review comments incorporated --- pkg/git/GitBaseManager.go | 22 +++++++++++----------- pkg/git/GitCliManager.go | 25 ++++++++----------------- pkg/git/GoGitSDKManager.go | 3 --- pkg/git/Watcher.go | 2 +- 4 files changed, 20 insertions(+), 32 deletions(-) diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index 4da38cde..caa7dd27 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -92,16 +92,16 @@ func parseCmdTimeoutJson(config *internals.Configuration) (map[string]int, error func (impl *GitManagerBaseImpl) Fetch(gitCtx GitContext, rootDir string) (response, errMsg string, err error) { impl.logger.Debugw("git fetch ", "location", rootDir) - cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force") + cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "fetch", "origin", "--tags", "--force") defer cancel() - output, errMsg, err := impl.RunCommandWithCred(cmd, gitCtx.Username, gitCtx.Password) + output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password) impl.logger.Debugw("fetch output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } func (impl *GitManagerBaseImpl) Checkout(gitCtx GitContext, rootDir, branch string) (response, errMsg string, err error) { impl.logger.Debugw("git checkout ", "location", rootDir) - cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "checkout", branch, "--force") + cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "checkout", branch, "--force") defer cancel() output, errMsg, err := impl.RunCommand(cmd) impl.logger.Debugw("checkout output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) @@ -118,7 +118,7 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st } cmdArgs := []string{"-C", rootDir, "log", from + "..." + toCommitHash, "--date=iso-strict", GITFORMAT} impl.logger.Debugw("git", cmdArgs) - cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", cmdArgs...) + cmd, cancel := impl.createCmdWithContext(gitCtx, "git", cmdArgs...) defer cancel() output, errMsg, err := impl.RunCommand(cmd) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) @@ -133,7 +133,7 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st return commits, nil } -func (impl *GitManagerBaseImpl) RunCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) { +func (impl *GitManagerBaseImpl) runCommandWithCred(cmd *exec.Cmd, userName, password string) (response, errMsg string, err error) { cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_ASKPASS=%s", GIT_ASK_PASS), fmt.Sprintf("GIT_USERNAME=%s", userName), @@ -166,7 +166,7 @@ func (impl *GitManagerBaseImpl) RunCommand(cmd *exec.Cmd) (response, errMsg stri func (impl *GitManagerBaseImpl) ConfigureSshCommand(gitCtx GitContext, rootDir string, sshPrivateKeyPath string) (response, errMsg string, err error) { impl.logger.Debugw("configuring ssh command on ", "location", rootDir) coreSshCommand := fmt.Sprintf("ssh -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", sshPrivateKeyPath) - cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "config", "core.sshCommand", coreSshCommand) + cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "config", "core.sshCommand", coreSshCommand) defer cancel() output, errMsg, err := impl.RunCommand(cmd) impl.logger.Debugw("configure ssh command output ", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) @@ -262,15 +262,15 @@ func (impl *GitManagerBaseImpl) FetchDiffStatBetweenCommits(gitCtx GitContext, o newHash = oldHash oldHash = oldHash + "^" } - cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "diff", "--numstat", oldHash, newHash) + cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "diff", "--numstat", oldHash, newHash) defer cancel() - output, errMsg, err := impl.RunCommandWithCred(cmd, gitCtx.Username, gitCtx.Password) + output, errMsg, err := impl.runCommandWithCred(cmd, gitCtx.Username, gitCtx.Password) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } -func (impl *GitManagerBaseImpl) CreateCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc) { +func (impl *GitManagerBaseImpl) createCmdWithContext(ctx GitContext, name string, arg ...string) (*exec.Cmd, context.CancelFunc) { newCtx := ctx cancel := func() {} @@ -295,8 +295,8 @@ func (impl *GitManagerBaseImpl) getCommandTimeout(command string) int { } func (impl *GitManagerBaseImpl) ExecuteCustomCommand(gitContext GitContext, name string, arg ...string) (response, errMsg string, err error) { - cmd, cancel := impl.CreateCmdWithContext(gitContext, name, arg...) + cmd, cancel := impl.createCmdWithContext(gitContext, name, arg...) defer cancel() - output, errMsg, err := impl.RunCommandWithCred(cmd, gitContext.Username, gitContext.Password) + output, errMsg, err := impl.runCommandWithCred(cmd, gitContext.Username, gitContext.Password) return output, errMsg, err } diff --git a/pkg/git/GitCliManager.go b/pkg/git/GitCliManager.go index 9819e631..61346385 100644 --- a/pkg/git/GitCliManager.go +++ b/pkg/git/GitCliManager.go @@ -13,15 +13,15 @@ import ( //} type GitCliManagerImpl struct { - *GitManagerBaseImpl + GitManagerBase logger *zap.SugaredLogger } -func NewGitCliManagerImpl(baseManager *GitManagerBaseImpl, logger *zap.SugaredLogger) *GitCliManagerImpl { +func NewGitCliManagerImpl(baseManager GitManagerBase, logger *zap.SugaredLogger) *GitCliManagerImpl { return &GitCliManagerImpl{ - GitManagerBaseImpl: baseManager, - logger: logger, + GitManagerBase: baseManager, + logger: logger, } } @@ -90,18 +90,14 @@ func openGitRepo(path string) error { } func (impl *GitCliManagerImpl) GitInit(gitCtx GitContext, rootDir string) error { impl.logger.Debugw("git", "-C", rootDir, "init") - cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "init") - defer cancel() - output, errMsg, err := impl.RunCommand(cmd) + output, errMsg, err := impl.GitManagerBase.ExecuteCustomCommand(gitCtx, "git", "-C", rootDir, "init") impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return err } func (impl *GitCliManagerImpl) GitCreateRemote(gitCtx GitContext, rootDir string, url string) error { impl.logger.Debugw("git", "-C", rootDir, "remote", "add", "origin", url) - cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "remote", "add", "origin", url) - defer cancel() - output, errMsg, err := impl.RunCommand(cmd) + output, errMsg, err := impl.GitManagerBase.ExecuteCustomCommand(gitCtx, "git", "-C", rootDir, "remote", "add", "origin", url) impl.logger.Debugw("url", url, "opt", output, "errMsg", errMsg, "error", err) return err } @@ -111,11 +107,8 @@ func (impl *GitCliManagerImpl) GetCommits(gitCtx GitContext, branchRef string, b rangeCmdArgs := []string{branchRef} extraCmdArgs := []string{"-n", strconv.Itoa(numCommits), "--date=iso-strict", GITFORMAT} cmdArgs := impl.getCommandForLogRange(branchRef, from, to, rangeCmdArgs, baseCmdArgs, extraCmdArgs) - impl.logger.Debugw("git", cmdArgs) - cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", cmdArgs...) - defer cancel() - output, errMsg, err := impl.RunCommand(cmd) + output, errMsg, err := impl.GitManagerBase.ExecuteCustomCommand(gitCtx, "git", cmdArgs...) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil { return nil, err @@ -140,9 +133,7 @@ func (impl *GitCliManagerImpl) getCommandForLogRange(branchRef string, from stri func (impl *GitCliManagerImpl) GitShow(gitCtx GitContext, rootDir string, hash string) (GitCommit, error) { impl.logger.Debugw("git", "-C", rootDir, "show", hash, "--date=iso-strict", GITFORMAT, "-s") - cmd, cancel := impl.CreateCmdWithContext(gitCtx, "git", "-C", rootDir, "show", hash, "--date=iso-strict", GITFORMAT, "-s") - defer cancel() - output, errMsg, err := impl.RunCommand(cmd) + output, errMsg, err := impl.GitManagerBase.ExecuteCustomCommand(gitCtx, "git", "-C", rootDir, "show", hash, "--date=iso-strict", GITFORMAT, "-s") impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil { return nil, err diff --git a/pkg/git/GoGitSDKManager.go b/pkg/git/GoGitSDKManager.go index 49935a2f..48ea05f8 100644 --- a/pkg/git/GoGitSDKManager.go +++ b/pkg/git/GoGitSDKManager.go @@ -9,9 +9,6 @@ import ( "os" ) -// type GoGitSDKManager interface { -// GitManager -// } type GoGitSDKManagerImpl struct { GitManagerBase logger *zap.SugaredLogger diff --git a/pkg/git/Watcher.go b/pkg/git/Watcher.go index c5eeb87a..32511eee 100644 --- a/pkg/git/Watcher.go +++ b/pkg/git/Watcher.go @@ -196,7 +196,7 @@ func (impl GitWatcherImpl) pollGitMaterialAndNotify(material *sql.GitMaterial) e updated, repo, err := impl.FetchAndUpdateMaterial(gitCtx, material, location) if err != nil { impl.logger.Errorw("error in fetching material details ", "repo", material.Url, "err", err) - // there might be the case if ssh priclvate key gets flush from disk, so creating and single retrying in this case + // there might be the case if ssh private key gets flush from disk, so creating and single retrying in this case if gitProvider.AuthMode == sql.AUTH_MODE_SSH { if strings.Contains(material.CheckoutLocation, "/.git") { location, _, _, err = impl.repositoryManager.GetLocationForMaterial(material, gitCtx.CloningMode) From 8d9f6d592f17650c7efdb2878e9820de19331199 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 2 Feb 2024 16:10:03 +0530 Subject: [PATCH 21/24] code review comments incorporated --- pkg/git/GitBaseManager.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/git/GitBaseManager.go b/pkg/git/GitBaseManager.go index caa7dd27..55e49803 100644 --- a/pkg/git/GitBaseManager.go +++ b/pkg/git/GitBaseManager.go @@ -103,7 +103,7 @@ func (impl *GitManagerBaseImpl) Checkout(gitCtx GitContext, rootDir, branch stri impl.logger.Debugw("git checkout ", "location", rootDir) cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "checkout", branch, "--force") defer cancel() - output, errMsg, err := impl.RunCommand(cmd) + output, errMsg, err := impl.runCommand(cmd) impl.logger.Debugw("checkout output", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } @@ -120,7 +120,7 @@ func (impl *GitManagerBaseImpl) LogMergeBase(gitCtx GitContext, rootDir, from st impl.logger.Debugw("git", cmdArgs) cmd, cancel := impl.createCmdWithContext(gitCtx, "git", cmdArgs...) defer cancel() - output, errMsg, err := impl.RunCommand(cmd) + output, errMsg, err := impl.runCommand(cmd) impl.logger.Debugw("root", rootDir, "opt", output, "errMsg", errMsg, "error", err) if err != nil { return nil, err @@ -139,10 +139,10 @@ func (impl *GitManagerBaseImpl) runCommandWithCred(cmd *exec.Cmd, userName, pass fmt.Sprintf("GIT_USERNAME=%s", userName), fmt.Sprintf("GIT_PASSWORD=%s", password), ) - return impl.RunCommand(cmd) + return impl.runCommand(cmd) } -func (impl *GitManagerBaseImpl) RunCommand(cmd *exec.Cmd) (response, errMsg string, err error) { +func (impl *GitManagerBaseImpl) runCommand(cmd *exec.Cmd) (response, errMsg string, err error) { cmd.Env = append(cmd.Env, "HOME=/dev/null") outBytes, err := cmd.CombinedOutput() if err != nil { @@ -168,7 +168,7 @@ func (impl *GitManagerBaseImpl) ConfigureSshCommand(gitCtx GitContext, rootDir s coreSshCommand := fmt.Sprintf("ssh -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no", sshPrivateKeyPath) cmd, cancel := impl.createCmdWithContext(gitCtx, "git", "-C", rootDir, "config", "core.sshCommand", coreSshCommand) defer cancel() - output, errMsg, err := impl.RunCommand(cmd) + output, errMsg, err := impl.runCommand(cmd) impl.logger.Debugw("configure ssh command output ", "root", rootDir, "opt", output, "errMsg", errMsg, "error", err) return output, errMsg, err } From 8028b7e8e796d13d33fffd15ffe8536c42afc364 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 2 Feb 2024 16:10:22 +0530 Subject: [PATCH 22/24] code review comments incorporated --- pkg/git/GitCliManager.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/git/GitCliManager.go b/pkg/git/GitCliManager.go index 61346385..d4cd89bc 100644 --- a/pkg/git/GitCliManager.go +++ b/pkg/git/GitCliManager.go @@ -8,10 +8,6 @@ import ( "strconv" ) -//type GitCliManager interface { -// //GitManager -//} - type GitCliManagerImpl struct { GitManagerBase logger *zap.SugaredLogger From bccadd8eea241ed4628e9aedbba8178cb17c9a98 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Fri, 2 Feb 2024 18:05:51 +0530 Subject: [PATCH 23/24] wireSet generated --- wire.go | 53 ++-------------------------------- wireset/commonWireset.go | 62 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 50 deletions(-) create mode 100644 wireset/commonWireset.go diff --git a/wire.go b/wire.go index 4aefd62d..90bcbee5 100644 --- a/wire.go +++ b/wire.go @@ -20,66 +20,19 @@ package main import ( - "github.com/devtron-labs/common-lib/monitoring" - pubsub "github.com/devtron-labs/common-lib/pubsub-lib" - "github.com/devtron-labs/git-sensor/api" "github.com/devtron-labs/git-sensor/app" - "github.com/devtron-labs/git-sensor/internals" - "github.com/devtron-labs/git-sensor/internals/logger" - "github.com/devtron-labs/git-sensor/internals/sql" - "github.com/devtron-labs/git-sensor/pkg" "github.com/devtron-labs/git-sensor/pkg/git" + "github.com/devtron-labs/git-sensor/wireset" "github.com/google/wire" ) func InitializeApp() (*app.App, error) { wire.Build( - app.NewApp, - api.NewMuxRouter, - internals.ParseConfiguration, - logger.NewSugaredLogger, - api.NewRestHandlerImpl, - wire.Bind(new(api.RestHandler), new(*api.RestHandlerImpl)), - api.NewGrpcHandlerImpl, - sql.NewMaterialRepositoryImpl, - wire.Bind(new(sql.MaterialRepository), new(*sql.MaterialRepositoryImpl)), - sql.NewDbConnection, - sql.GetConfig, - sql.NewCiPipelineMaterialRepositoryImpl, - wire.Bind(new(sql.CiPipelineMaterialRepository), new(*sql.CiPipelineMaterialRepositoryImpl)), - sql.NewGitProviderRepositoryImpl, - wire.Bind(new(sql.GitProviderRepository), new(*sql.GitProviderRepositoryImpl)), - git.NewGitManagerImpl, - wire.Bind(new(git.GitManager), new(*git.GitManagerImpl)), + wireset.CommonWireSet, git.NewRepositoryManagerImpl, wire.Bind(new(git.RepositoryManager), new(*git.RepositoryManagerImpl)), - git.NewRepositoryManagerAnalyticsImpl, - wire.Bind(new(git.RepositoryManagerAnalytics), new(*git.RepositoryManagerAnalyticsImpl)), - pkg.NewRepoManagerImpl, - wire.Bind(new(pkg.RepoManager), new(*pkg.RepoManagerImpl)), - git.NewGitWatcherImpl, - wire.Bind(new(git.GitWatcher), new(*git.GitWatcherImpl)), - internals.NewRepositoryLocker, - //internal.NewNatsConnection, - pubsub.NewPubSubClientServiceImpl, - sql.NewWebhookEventRepositoryImpl, - wire.Bind(new(sql.WebhookEventRepository), new(*sql.WebhookEventRepositoryImpl)), - sql.NewWebhookEventParsedDataRepositoryImpl, - wire.Bind(new(sql.WebhookEventParsedDataRepository), new(*sql.WebhookEventParsedDataRepositoryImpl)), - sql.NewWebhookEventDataMappingRepositoryImpl, - wire.Bind(new(sql.WebhookEventDataMappingRepository), new(*sql.WebhookEventDataMappingRepositoryImpl)), - sql.NewWebhookEventDataMappingFilterResultRepositoryImpl, - wire.Bind(new(sql.WebhookEventDataMappingFilterResultRepository), new(*sql.WebhookEventDataMappingFilterResultRepositoryImpl)), - git.NewWebhookEventBeanConverterImpl, - wire.Bind(new(git.WebhookEventBeanConverter), new(*git.WebhookEventBeanConverterImpl)), - git.NewWebhookEventServiceImpl, - wire.Bind(new(git.WebhookEventService), new(*git.WebhookEventServiceImpl)), - git.NewWebhookEventParserImpl, - wire.Bind(new(git.WebhookEventParser), new(*git.WebhookEventParserImpl)), - git.NewWebhookHandlerImpl, - wire.Bind(new(git.WebhookHandler), new(*git.WebhookHandlerImpl)), - monitoring.NewMonitoringRouter, + wireset.CommonWireSet2, ) return &app.App{}, nil } diff --git a/wireset/commonWireset.go b/wireset/commonWireset.go new file mode 100644 index 00000000..78ad9f78 --- /dev/null +++ b/wireset/commonWireset.go @@ -0,0 +1,62 @@ +package wireset + +import ( + "github.com/devtron-labs/common-lib/monitoring" + pubsub "github.com/devtron-labs/common-lib/pubsub-lib" + "github.com/devtron-labs/git-sensor/api" + "github.com/devtron-labs/git-sensor/app" + "github.com/devtron-labs/git-sensor/internals" + "github.com/devtron-labs/git-sensor/internals/logger" + "github.com/devtron-labs/git-sensor/internals/sql" + "github.com/devtron-labs/git-sensor/pkg" + "github.com/devtron-labs/git-sensor/pkg/git" + "github.com/google/wire" +) + +var CommonWireSet = wire.NewSet(app.NewApp, + api.NewMuxRouter, + internals.ParseConfiguration, + logger.NewSugaredLogger, + api.NewRestHandlerImpl, + wire.Bind(new(api.RestHandler), new(*api.RestHandlerImpl)), + api.NewGrpcHandlerImpl, + sql.NewMaterialRepositoryImpl, + wire.Bind(new(sql.MaterialRepository), new(*sql.MaterialRepositoryImpl)), + sql.NewDbConnection, + sql.GetConfig, + sql.NewCiPipelineMaterialRepositoryImpl, + wire.Bind(new(sql.CiPipelineMaterialRepository), new(*sql.CiPipelineMaterialRepositoryImpl)), + sql.NewGitProviderRepositoryImpl, + wire.Bind(new(sql.GitProviderRepository), new(*sql.GitProviderRepositoryImpl)), + git.NewGitManagerImpl, + wire.Bind(new(git.GitManager), new(*git.GitManagerImpl)), +) + +var CommonWireSet2 = wire.NewSet( + git.NewRepositoryManagerAnalyticsImpl, + wire.Bind(new(git.RepositoryManagerAnalytics), new(*git.RepositoryManagerAnalyticsImpl)), + pkg.NewRepoManagerImpl, + wire.Bind(new(pkg.RepoManager), new(*pkg.RepoManagerImpl)), + git.NewGitWatcherImpl, + wire.Bind(new(git.GitWatcher), new(*git.GitWatcherImpl)), + internals.NewRepositoryLocker, + //internal.NewNatsConnection, + pubsub.NewPubSubClientServiceImpl, + sql.NewWebhookEventRepositoryImpl, + wire.Bind(new(sql.WebhookEventRepository), new(*sql.WebhookEventRepositoryImpl)), + sql.NewWebhookEventParsedDataRepositoryImpl, + wire.Bind(new(sql.WebhookEventParsedDataRepository), new(*sql.WebhookEventParsedDataRepositoryImpl)), + sql.NewWebhookEventDataMappingRepositoryImpl, + wire.Bind(new(sql.WebhookEventDataMappingRepository), new(*sql.WebhookEventDataMappingRepositoryImpl)), + sql.NewWebhookEventDataMappingFilterResultRepositoryImpl, + wire.Bind(new(sql.WebhookEventDataMappingFilterResultRepository), new(*sql.WebhookEventDataMappingFilterResultRepositoryImpl)), + git.NewWebhookEventBeanConverterImpl, + wire.Bind(new(git.WebhookEventBeanConverter), new(*git.WebhookEventBeanConverterImpl)), + git.NewWebhookEventServiceImpl, + wire.Bind(new(git.WebhookEventService), new(*git.WebhookEventServiceImpl)), + git.NewWebhookEventParserImpl, + wire.Bind(new(git.WebhookEventParser), new(*git.WebhookEventParserImpl)), + git.NewWebhookHandlerImpl, + wire.Bind(new(git.WebhookHandler), new(*git.WebhookHandlerImpl)), + monitoring.NewMonitoringRouter, +) From 7e70ef9072c5c4b84616c45a5cff1dfb92929614 Mon Sep 17 00:00:00 2001 From: ShashwatDadhich Date: Mon, 5 Feb 2024 18:12:47 +0530 Subject: [PATCH 24/24] wireSet and test file changed --- pkg/git/GitCliManager_test.go | 15 +++++++-------- pkg/git/RepositoryManagerIT_test.go | 14 +++++++------- wire.go | 1 - wireset/commonWireset.go | 3 --- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/pkg/git/GitCliManager_test.go b/pkg/git/GitCliManager_test.go index d2b60ed1..b0a6c803 100644 --- a/pkg/git/GitCliManager_test.go +++ b/pkg/git/GitCliManager_test.go @@ -12,7 +12,7 @@ func TestGitCliManagerImpl_processGitLogOutput(t *testing.T) { t.Run("tt.name", func(t *testing.T) { logger, _ := utils.NewSugardLogger() impl := &GitCliManagerImpl{ - GitManagerBaseImpl: NewGitManagerBaseImpl(logger, &internals.Configuration{}), + GitManagerBase: NewGitManagerBaseImpl(logger, &internals.Configuration{}), } got, err := impl.GetCommits(BuildGitContext(context.Background()), "main", "", "/Users/subhashish/workspace/lens", 15, "", "") print("err", got, err) @@ -21,14 +21,13 @@ func TestGitCliManagerImpl_processGitLogOutput(t *testing.T) { t.Run("analytics", func(t *testing.T) { logger, _ := utils.NewSugardLogger() conf := &internals.Configuration{UseGitCli: false, AnalyticsDebug: true, GoGitTimeout: 10, CliCmdTimeoutGlobal: 8, CliCmdTimeoutJson: `{"log":4}`} - impl := &GoGitSDKManagerImpl{ - GitManagerBaseImpl: NewGitManagerBaseImpl(logger, conf), + impl := &GitManagerImpl{ + GitManager: NewGitManagerImpl(logger, conf), + } + analyticsImpl := &RepositoryManagerAnalyticsImpl{ + repoManager: NewRepositoryManagerImpl(logger, conf, impl), + gitManager: impl, } - analyticsImpl := &RepositoryManagerAnalyticsImpl{&RepositoryManagerImpl{ - logger: logger, - gitManager: GitManagerImpl{impl}, - configuration: conf, - }} //got, err := impl.GetCommits(GitContext{}, "main", "", "/Users/subhashish/workspace/lens", 15, "", "") got, err := analyticsImpl.ChangesSinceByRepositoryForAnalytics(BuildGitContext(context.Background()), "/Users/subhashish/workspace/lens", "e2d5f17556130e37a9e941a71ffc83a9a2085ec5", "dfb352083ed3131bb2f69cfa2e18c614e5e08700") print("err", got, err) diff --git a/pkg/git/RepositoryManagerIT_test.go b/pkg/git/RepositoryManagerIT_test.go index c62e6721..acce2633 100644 --- a/pkg/git/RepositoryManagerIT_test.go +++ b/pkg/git/RepositoryManagerIT_test.go @@ -26,7 +26,7 @@ var password = "" var sshPrivateKey = `` func getRepoManagerAnalyticsImpl(t *testing.T) *RepositoryManagerAnalyticsImpl { - return &RepositoryManagerAnalyticsImpl{RepositoryManagerImpl: getRepoManagerImpl(t)} + return &RepositoryManagerAnalyticsImpl{repoManager: getRepoManagerImpl(t)} } func getRepoManagerImpl(t *testing.T) *RepositoryManagerImpl { @@ -36,14 +36,14 @@ func getRepoManagerImpl(t *testing.T) *RepositoryManagerImpl { CommitStatsTimeoutInSec: 0, EnableFileStats: true, GitHistoryCount: 2, - UseGitCli: true, + UseGitCli: false, GoGitTimeout: 10, } base := NewGitManagerBaseImpl(logger, conf) - gitCliImpl := NewGitCliManagerImpl(base) - gogitImpl := NewGoGitSDKManagerImpl(base) + _ = NewGitCliManagerImpl(base, logger) + _ = NewGoGitSDKManagerImpl(base, logger) - gitUtil := NewGitManagerImpl(conf, gitCliImpl, gogitImpl) + gitUtil := NewGitManagerImpl(logger, conf) repositoryManagerImpl := NewRepositoryManagerImpl(logger, conf, gitUtil) return repositoryManagerImpl } @@ -132,7 +132,7 @@ func TestRepositoryManager_Add(t *testing.T) { repositoryManagerImpl := getRepoManagerImpl(t) for _, tt := range tests { if tt.payload.authMode == "SSH" { - err := repositoryManagerImpl.CreateSshFileIfNotExistsAndConfigureSshCommand(BuildGitContext(context.Background()), tt.payload.location, tt.payload.gitProviderId, tt.payload.sshPrivateKeyContent) + _, err := repositoryManagerImpl.CreateSshFileIfNotExistsAndConfigureSshCommand(BuildGitContext(context.Background()), tt.payload.location, tt.payload.gitProviderId, tt.payload.sshPrivateKeyContent) assert.Nil(t, err) } t.Run(tt.name, func(t *testing.T) { @@ -583,7 +583,7 @@ func TestRepositoryManager_ChangesSinceByRepository(t *testing.T) { r, err := repositoryManagerImpl.gitManager.OpenRepoPlain(tt.payload.checkoutPath) assert.Nil(t, err) t.Run(tt.name, func(t *testing.T) { - got, err := repositoryManagerImpl.ChangesSinceByRepository(BuildGitContext(context.Background()), r, tt.payload.branch, tt.payload.from, tt.payload.to, tt.payload.count) + got, err := repositoryManagerImpl.ChangesSinceByRepository(BuildGitContext(context.Background()), r, tt.payload.branch, tt.payload.from, tt.payload.to, tt.payload.count, tt.payload.checkoutPath) if (err != nil) != tt.wantErr { t.Errorf("ChangesSinceByRepository() error in %s, error = %v, wantErr %v", tt.name, err, tt.wantErr) return diff --git a/wire.go b/wire.go index 90bcbee5..073d344b 100644 --- a/wire.go +++ b/wire.go @@ -32,7 +32,6 @@ func InitializeApp() (*app.App, error) { wireset.CommonWireSet, git.NewRepositoryManagerImpl, wire.Bind(new(git.RepositoryManager), new(*git.RepositoryManagerImpl)), - wireset.CommonWireSet2, ) return &app.App{}, nil } diff --git a/wireset/commonWireset.go b/wireset/commonWireset.go index 78ad9f78..57e27a80 100644 --- a/wireset/commonWireset.go +++ b/wireset/commonWireset.go @@ -30,9 +30,6 @@ var CommonWireSet = wire.NewSet(app.NewApp, wire.Bind(new(sql.GitProviderRepository), new(*sql.GitProviderRepositoryImpl)), git.NewGitManagerImpl, wire.Bind(new(git.GitManager), new(*git.GitManagerImpl)), -) - -var CommonWireSet2 = wire.NewSet( git.NewRepositoryManagerAnalyticsImpl, wire.Bind(new(git.RepositoryManagerAnalytics), new(*git.RepositoryManagerAnalyticsImpl)), pkg.NewRepoManagerImpl,