Skip to content

Commit

Permalink
Perf: Traverse commit history instead of reading the packfile
Browse files Browse the repository at this point in the history 
To avoid show orphan commits and improve performance on commits table, instead of read all the objects from each repository and discard commits, we can execute from go-git a git log --all command and traverse commit history, avoiding read a lot of data from disk.

Signed-off-by: Antonio Jesus Navarro Perez <[email protected]>
  • Loading branch information
@ajnavarro
ajnavarro committed Feb 15, 2019
1 parent 73c1730 commit 9748ee3
Show file tree
Hide file tree
Showing 13 changed files with 117 additions and 75 deletions.
5 changes: 4 additions & 1 deletion commit_blobs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"io"

git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/object"
"gopkg.in/src-d/go-mysql-server.v0/sql"
Expand Down Expand Up @@ -219,7 +220,9 @@ func (i *commitBlobsRowIter) init() error {
if len(i.commits) > 0 {
i.iter, err = NewCommitsByHashIter(i.repo, i.commits)
} else {
i.iter, err = i.repo.CommitObjects()
i.iter, err = i.repo.Log(&git.LogOptions{
All: true,
})
}

return err
Expand Down
11 changes: 9 additions & 2 deletions commit_files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"io"

"github.com/sirupsen/logrus"
git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/filemode"
"gopkg.in/src-d/go-git.v4/plumbing/object"
Expand Down Expand Up @@ -196,7 +197,10 @@ func (i *commitFilesRowIter) init() error {
if len(i.commitHashes) > 0 {
i.commits, err = NewCommitsByHashIter(i.repo, i.commitHashes)
} else {
i.commits, err = i.repo.CommitObjects()
i.commits, err = i.repo.
Log(&git.LogOptions{
All: true,
})
}

return err
Expand Down Expand Up @@ -433,7 +437,10 @@ func newCommitFilesKeyValueIter(
return nil, err
}

commits, err := repo.CommitObjects()
commits, err := repo.
Log(&git.LogOptions{
All: true,
})
if err != nil {
return nil, err
}
Expand Down
5 changes: 4 additions & 1 deletion commit_files_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"testing"

"github.com/stretchr/testify/require"
git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-mysql-server.v0/sql"
"gopkg.in/src-d/go-mysql-server.v0/sql/expression"
Expand Down Expand Up @@ -35,7 +36,9 @@ func TestCommitFilesTableRowIter(t *testing.T) {

require.NoError(err)

commits, err := repo.CommitObjects()
commits, err := repo.Log(&git.LogOptions{
All: true,
})
require.NoError(err)

for {
Expand Down
5 changes: 4 additions & 1 deletion commit_trees.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"io"

git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"

"gopkg.in/src-d/go-git.v4/plumbing/filemode"
Expand Down Expand Up @@ -215,7 +216,9 @@ func (i *commitTreesRowIter) init() error {
if len(i.commitHashes) > 0 {
i.commits, err = NewCommitsByHashIter(i.repo, i.commitHashes)
} else {
i.commits, err = i.repo.CommitObjects()
i.commits, err = i.repo.Log(&git.LogOptions{
All: true,
})
}

return err
Expand Down
20 changes: 10 additions & 10 deletions commit_trees_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,17 @@ func TestCommitTreesRowIter(t *testing.T) {
}

expected := []sql.Row{
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "dbd3641b371024f44d0e469a9c8f5457b0660de1"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "a39771a7651f97faf5c72e08224d857fc35133db"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},

{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a8d315b2b1c615d43042c3a62402b8a54288cf5c"},
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "a39771a7651f97faf5c72e08224d857fc35133db"},
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "cf4aa3b38974fb7d81f367c0830f7d78d65ab86b"},

{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "dbd3641b371024f44d0e469a9c8f5457b0660de1"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "a39771a7651f97faf5c72e08224d857fc35133db"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},

{"918c48b83bd081e863dbe1b80f8998f058cd8294", "fb72698cab7617ac416264415f13224dfd7a165e"},
{"918c48b83bd081e863dbe1b80f8998f058cd8294", "a39771a7651f97faf5c72e08224d857fc35133db"},
{"918c48b83bd081e863dbe1b80f8998f058cd8294", "5a877e6a906a2743ad6e45d99c1793642aaf8eda"},
Expand All @@ -44,13 +44,13 @@ func TestCommitTreesRowIter(t *testing.T) {

{"1669dce138d9b841a518c64b10914d88f5e488ea", "eba74343e2f15d62adedfd8c883ee0262b5c8021"},

{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},

{"b8e471f58bcbca63b07bda20e428190409c2db47", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},

{"35e85108805c84807bc66a02d91535e1e24b38b9", "8dcef98b1d52143e1e2dbc458ffe38f925786bf2"},

{"b029517f6300c2da0f4b651b8642506cd6aaf45d", "aa9b383c260e1d05fbbf6b30a02914555e20c725"},

{"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},

{"b8e471f58bcbca63b07bda20e428190409c2db47", "c2d30fa8ef288618f65f6eed6e168e0d514886f4"},
}

require.Equal(expected, rows)
Expand Down Expand Up @@ -90,8 +90,8 @@ func TestCommitTreesPushdown(t *testing.T) {
),
},
[]sql.Row{
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
{"6ecf0ef2c2dffb796033e5a02219af86ec6584e5", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
{"e8d3ffab552895c19b9fcf7aa264d277cde33881", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
{"918c48b83bd081e863dbe1b80f8998f058cd8294", "586af567d0bb5e771e49bdd9434f5e0fb76d25fa"},
},
},
Expand Down
16 changes: 13 additions & 3 deletions commits.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package gitbase
import (
"io"

git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-mysql-server.v0/sql"

"gopkg.in/src-d/go-git.v4/plumbing"
Expand Down Expand Up @@ -161,7 +162,10 @@ func (i *commitIter) init() error {
if len(i.hashes) > 0 {
i.iter, err = NewCommitsByHashIter(i.repo, i.hashes)
} else {
i.iter, err = i.repo.CommitObjects()
i.iter, err =
i.repo.Log(&git.LogOptions{
All: true,
})
}

return err
Expand Down Expand Up @@ -245,7 +249,10 @@ func NewCommitsByHashIter(
var commitIter object.CommitIter
var err error
if len(hashes) == 0 {
commitIter, err = repo.CommitObjects()
commitIter, err =
repo.Log(&git.LogOptions{
All: true,
})
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -333,7 +340,10 @@ func newCommitsKeyValueIter(
return nil, err
}

commits, err := repo.CommitObjects()
commits, err :=
repo.Log(&git.LogOptions{
All: true,
})
if err != nil {
return nil, err
}
Expand Down
62 changes: 31 additions & 31 deletions commits_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,14 +101,14 @@ func TestCommitsParents(t *testing.T) {
}{
{
name: "test commits parents 1",
hash: "e8d3ffab552895c19b9fcf7aa264d277cde33881",
hash: "6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
parents: []string{
"918c48b83bd081e863dbe1b80f8998f058cd8294",
},
},
{
name: "test commits parents 2",
hash: "6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
hash: "e8d3ffab552895c19b9fcf7aa264d277cde33881",
parents: []string{
"918c48b83bd081e863dbe1b80f8998f058cd8294",
},
Expand Down Expand Up @@ -137,30 +137,30 @@ func TestCommitsParents(t *testing.T) {
},
{
name: "test commits parents 6",
hash: "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
hash: "35e85108805c84807bc66a02d91535e1e24b38b9",
parents: []string{
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
"b8e471f58bcbca63b07bda20e428190409c2db47",
},
},
{
name: "test commits parents 7",
hash: "b8e471f58bcbca63b07bda20e428190409c2db47",
parents: []string{
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
},
name: "test commits parents 7",
hash: "b029517f6300c2da0f4b651b8642506cd6aaf45d",
parents: []string{},
},
{
name: "test commits parents 8",
hash: "35e85108805c84807bc66a02d91535e1e24b38b9",
hash: "a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
parents: []string{
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
"b8e471f58bcbca63b07bda20e428190409c2db47",
},
},
{
name: "test commits parents 9",
hash: "b029517f6300c2da0f4b651b8642506cd6aaf45d",
parents: []string{},
name: "test commits parents 9",
hash: "b8e471f58bcbca63b07bda20e428190409c2db47",
parents: []string{
"b029517f6300c2da0f4b651b8642506cd6aaf45d",
},
},
}

Expand Down Expand Up @@ -188,65 +188,65 @@ func TestCommitsIndexKeyValueIter(t *testing.T) {
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 682,
Offset: 186,
}),
[]interface{}{
"1669dce138d9b841a518c64b10914d88f5e488ea",
"6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 1292,
Offset: 12,
}),
[]interface{}{
"35e85108805c84807bc66a02d91535e1e24b38b9",
"e8d3ffab552895c19b9fcf7aa264d277cde33881",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 186,
Offset: 353,
}),
[]interface{}{
"6ecf0ef2c2dffb796033e5a02219af86ec6584e5",
"918c48b83bd081e863dbe1b80f8998f058cd8294",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 353,
Offset: 516,
}),
[]interface{}{
"918c48b83bd081e863dbe1b80f8998f058cd8294",
"af2d6a6954d532f8ffb47615169c8fdf9d383a1a",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 905,
Offset: 682,
}),
[]interface{}{
"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
"1669dce138d9b841a518c64b10914d88f5e488ea",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 516,
Offset: 1292,
}),
[]interface{}{
"af2d6a6954d532f8ffb47615169c8fdf9d383a1a",
"35e85108805c84807bc66a02d91535e1e24b38b9",
"[email protected]",
},
},
Expand All @@ -265,22 +265,22 @@ func TestCommitsIndexKeyValueIter(t *testing.T) {
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 1130,
Offset: 905,
}),
[]interface{}{
"b8e471f58bcbca63b07bda20e428190409c2db47",
"[email protected]",
"a5b8b09e2f8fcb0bb99d3ccb0958157b40890d69",
"[email protected]",
},
},
{
assertEncodeKey(t, &packOffsetIndexKey{
Repository: path,
Packfile: "323a4b6b5de684f9966953a043bc800154e5dbfa",
Offset: 12,
Offset: 1130,
}),
[]interface{}{
"e8d3ffab552895c19b9fcf7aa264d277cde33881",
"[email protected]",
"b8e471f58bcbca63b07bda20e428190409c2db47",
"[email protected]",
},
},
}
Expand Down
9 changes: 7 additions & 2 deletions files.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bytes"
"io"

git "gopkg.in/src-d/go-git.v4"
"gopkg.in/src-d/go-git.v4/plumbing"
"gopkg.in/src-d/go-git.v4/plumbing/filemode"
"gopkg.in/src-d/go-git.v4/plumbing/object"
Expand Down Expand Up @@ -192,7 +193,9 @@ type filesRowIter struct {
func (i *filesRowIter) init() error {
var err error
i.seen = make(map[plumbing.Hash]struct{})
i.commits, err = i.repo.CommitObjects()
i.commits, err = i.repo.Log(&git.LogOptions{
All: true,
})
return err
}

Expand Down Expand Up @@ -411,7 +414,9 @@ func newFilesKeyValueIter(pool *RepositoryPool, repo *Repository, columns []stri
return nil, err
}

commits, err := repo.CommitObjects()
commits, err := repo.Log(&git.LogOptions{
All: true,
})
if err != nil {
return nil, err
}
Expand Down
Loading

0 comments on commit 9748ee3

Please sign in to comment.