Skip to content

Commit 700cce5

Browse files
d4x1d4x1
authored
refactor(gitextractor): move "clone a repo" from "PrePareTaskData" to a new subtask called "CloneGitRepo" (#5992)
1 parent 68baa7d commit 700cce5

4 files changed

Lines changed: 93 additions & 33 deletions

File tree

‎backend/plugins/gitextractor/impl/impl.go‎

Lines changed: 5 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,12 @@ limitations under the License.
1818
package impl
1919

2020
import (
21-
"context"
22-
"fmt"
2321
"github.com/apache/incubator-devlake/core/dal"
2422
"github.com/apache/incubator-devlake/core/errors"
25-
"github.com/apache/incubator-devlake/core/log"
2623
"github.com/apache/incubator-devlake/core/plugin"
2724
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
28-
"github.com/apache/incubator-devlake/plugins/gitextractor/models"
2925
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
30-
"github.com/apache/incubator-devlake/plugins/gitextractor/store"
3126
"github.com/apache/incubator-devlake/plugins/gitextractor/tasks"
32-
"strings"
3327
)
3428

3529
var _ interface {
@@ -55,14 +49,15 @@ func (p GitExtractor) Name() string {
5549
// return all available subtasks, framework will run them for you in order
5650
func (p GitExtractor) SubTaskMetas() []plugin.SubTaskMeta {
5751
return []plugin.SubTaskMeta{
52+
tasks.CloneGitRepoMeta,
5853
tasks.CollectGitCommitMeta,
5954
tasks.CollectGitBranchMeta,
6055
tasks.CollectGitTagMeta,
6156
tasks.CollectGitDiffLineMeta,
6257
}
6358
}
6459

65-
// based on task context and user input options, return data that shared among all subtasks
60+
// PrepareTaskData based on task context and user input options, return data that shared among all subtasks
6661
func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) {
6762
var op tasks.GitExtractorOptions
6863
if err := helper.Decode(options, &op, nil); err != nil {
@@ -71,12 +66,10 @@ func (p GitExtractor) PrepareTaskData(taskCtx plugin.TaskContext, options map[st
7166
if err := op.Valid(); err != nil {
7267
return nil, err
7368
}
74-
storage := store.NewDatabase(taskCtx, op.RepoId)
75-
repo, err := NewGitRepo(taskCtx.GetContext(), taskCtx.GetLogger(), storage, op)
76-
if err != nil {
77-
return nil, err
69+
taskData := &tasks.GitExtractorTaskData{
70+
Options: &op,
7871
}
79-
return repo, nil
72+
return taskData, nil
8073
}
8174

8275
func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error {
@@ -91,20 +84,3 @@ func (p GitExtractor) Close(taskCtx plugin.TaskContext) errors.Error {
9184
func (p GitExtractor) RootPkgPath() string {
9285
return "github.com/apache/incubator-devlake/plugins/gitextractor"
9386
}
94-
95-
// NewGitRepo create and return a new parser git repo
96-
func NewGitRepo(ctx context.Context, logger log.Logger, storage models.Store, op tasks.GitExtractorOptions) (*parser.GitRepo, errors.Error) {
97-
var err errors.Error
98-
var repo *parser.GitRepo
99-
p := parser.NewGitRepoCreator(storage, logger)
100-
if strings.HasPrefix(op.Url, "http") {
101-
repo, err = p.CloneOverHTTP(ctx, op.RepoId, op.Url, op.User, op.Password, op.Proxy)
102-
} else if url := strings.TrimPrefix(op.Url, "ssh://"); strings.HasPrefix(url, "git@") {
103-
repo, err = p.CloneOverSSH(ctx, op.RepoId, url, op.PrivateKey, op.Passphrase)
104-
} else if strings.HasPrefix(op.Url, "/") {
105-
repo, err = p.LocalRepo(op.Url, op.RepoId)
106-
} else {
107-
return nil, errors.BadInput.New(fmt.Sprintf("unsupported url [%s]", op.Url))
108-
}
109-
return repo, err
110-
}

‎backend/plugins/gitextractor/main.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ func main() {
7878
"git extractor",
7979
nil,
8080
)
81-
repo, err := impl.NewGitRepo(subTaskCtx.GetContext(), logger, storage, tasks.GitExtractorOptions{
81+
repo, err := tasks.NewGitRepo(subTaskCtx.GetContext(), logger, storage, &tasks.GitExtractorOptions{
8282
RepoId: *id,
8383
Url: *url,
8484
User: *user,

‎backend/plugins/gitextractor/tasks/clone.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package tasks
19+
20+
import (
21+
"context"
22+
"fmt"
23+
"github.com/apache/incubator-devlake/core/errors"
24+
"github.com/apache/incubator-devlake/core/log"
25+
"github.com/apache/incubator-devlake/core/plugin"
26+
"github.com/apache/incubator-devlake/plugins/gitextractor/models"
27+
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
28+
"github.com/apache/incubator-devlake/plugins/gitextractor/store"
29+
"strings"
30+
)
31+
32+
var CloneGitRepoMeta = plugin.SubTaskMeta{
33+
Name: "cloneGitRepo",
34+
EntryPoint: CloneGitRepo,
35+
EnabledByDefault: true,
36+
Required: true,
37+
Description: "clone a git repo, make it available to later tasks",
38+
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
39+
}
40+
41+
func CloneGitRepo(subTaskCtx plugin.SubTaskContext) errors.Error {
42+
taskData, ok := subTaskCtx.GetData().(*GitExtractorTaskData)
43+
if !ok {
44+
panic("git repo reference not found on context")
45+
}
46+
op := taskData.Options
47+
storage := store.NewDatabase(subTaskCtx, op.RepoId)
48+
repo, err := NewGitRepo(subTaskCtx.GetContext(), subTaskCtx.GetLogger(), storage, op)
49+
if err != nil {
50+
return err
51+
}
52+
taskData.GitRepo = repo
53+
subTaskCtx.TaskContext().SetData(taskData)
54+
return nil
55+
}
56+
57+
// NewGitRepo create and return a new parser git repo
58+
func NewGitRepo(ctx context.Context, logger log.Logger, storage models.Store, op *GitExtractorOptions) (*parser.GitRepo, errors.Error) {
59+
var err errors.Error
60+
var repo *parser.GitRepo
61+
p := parser.NewGitRepoCreator(storage, logger)
62+
if strings.HasPrefix(op.Url, "http") {
63+
repo, err = p.CloneOverHTTP(ctx, op.RepoId, op.Url, op.User, op.Password, op.Proxy)
64+
} else if url := strings.TrimPrefix(op.Url, "ssh://"); strings.HasPrefix(url, "git@") {
65+
repo, err = p.CloneOverSSH(ctx, op.RepoId, url, op.PrivateKey, op.Passphrase)
66+
} else if strings.HasPrefix(op.Url, "/") {
67+
repo, err = p.LocalRepo(op.Url, op.RepoId)
68+
} else {
69+
return nil, errors.BadInput.New(fmt.Sprintf("unsupported url [%s]", op.Url))
70+
}
71+
return repo, err
72+
}

‎backend/plugins/gitextractor/tasks/git_repo_collector.go‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,14 @@ import (
2525
"github.com/apache/incubator-devlake/plugins/gitextractor/parser"
2626
)
2727

28+
type GitExtractorTaskData struct {
29+
Options *GitExtractorOptions
30+
GitRepo *parser.GitRepo
31+
}
32+
2833
type GitExtractorOptions struct {
2934
RepoId string `json:"repoId"`
30-
Name string `jsno:"name"`
35+
Name string `json:"name"`
3136
Url string `json:"url"`
3237
User string `json:"user"`
3338
Password string `json:"password"`
@@ -99,11 +104,14 @@ func CollectGitDiffLines(subTaskCtx plugin.SubTaskContext) errors.Error {
99104
}
100105

101106
func getGitRepo(subTaskCtx plugin.SubTaskContext) *parser.GitRepo {
102-
repo, ok := subTaskCtx.GetData().(*parser.GitRepo)
107+
taskData, ok := subTaskCtx.GetData().(*GitExtractorTaskData)
103108
if !ok {
104109
panic("git repo reference not found on context")
105110
}
106-
return repo
111+
if taskData.GitRepo == nil {
112+
panic("git repo is empty, please check subtask: clone repo")
113+
}
114+
return taskData.GitRepo
107115
}
108116

109117
var CollectGitCommitMeta = plugin.SubTaskMeta{
@@ -112,6 +120,7 @@ var CollectGitCommitMeta = plugin.SubTaskMeta{
112120
EnabledByDefault: true,
113121
Description: "collect git commits into Domain Layer Tables",
114122
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_CROSS},
123+
Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta},
115124
}
116125

117126
var CollectGitBranchMeta = plugin.SubTaskMeta{
@@ -120,6 +129,7 @@ var CollectGitBranchMeta = plugin.SubTaskMeta{
120129
EnabledByDefault: true,
121130
Description: "collect git branch into Domain Layer Tables",
122131
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
132+
Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta},
123133
}
124134

125135
var CollectGitTagMeta = plugin.SubTaskMeta{
@@ -128,6 +138,7 @@ var CollectGitTagMeta = plugin.SubTaskMeta{
128138
EnabledByDefault: true,
129139
Description: "collect git tag into Domain Layer Tables",
130140
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
141+
Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta},
131142
}
132143

133144
var CollectGitDiffLineMeta = plugin.SubTaskMeta{
@@ -136,4 +147,5 @@ var CollectGitDiffLineMeta = plugin.SubTaskMeta{
136147
EnabledByDefault: false,
137148
Description: "collect git commit diff line into Domain Layer Tables",
138149
DomainTypes: []string{plugin.DOMAIN_TYPE_CODE},
150+
Dependencies: []*plugin.SubTaskMeta{&CloneGitRepoMeta},
139151
}

0 commit comments

Comments
 (0)