Filter reviews of one pull request in memory instead of database to reduce slow response because of lacking database index (#33106) (#33128)

Backport #33106 by @lunny

This PR fixes a performance problem when reviewing a pull request in a
big instance which have many records in the `review` table.
Traditionally, we should add more indexes in that table. But since
dismissed reviews of 1 pull request will not be too many as expected in
a common repository. Filtering reviews in the memory should be more
quick .

Co-authored-by: Lunny Xiao <[email protected]>
Co-authored-by: wxiaoguang <[email protected]>

Author: 3 people

models/issues/pull.go

@@ -301,7 +301,7 @@ func (pr *PullRequest) LoadRequestedReviewers(ctx context.Context) error {
 		return nil
 	}
 
-	reviews, err := GetReviewsByIssueID(ctx, pr.Issue.ID)
+	reviews, _, err := GetReviewsByIssueID(ctx, pr.Issue.ID)
 	if err != nil {
 		return err
 	}
@@ -320,7 +320,7 @@ func (pr *PullRequest) LoadRequestedReviewers(ctx context.Context) error {
 
 // LoadRequestedReviewersTeams loads the requested reviewers teams.
 func (pr *PullRequest) LoadRequestedReviewersTeams(ctx context.Context) error {
-	reviews, err := GetReviewsByIssueID(ctx, pr.Issue.ID)
+	reviews, _, err := GetReviewsByIssueID(ctx, pr.Issue.ID)
 	if err != nil {
 		return err
 	}

models/issues/review_list.go

@@ -5,6 +5,8 @@ package issues
 
 import (
 	"context"
+	"slices"
+	"sort"
 
 	"code.gitea.io/gitea/models/db"
 	organization_model "code.gitea.io/gitea/models/organization"
@@ -153,43 +155,60 @@ func CountReviews(ctx context.Context, opts FindReviewOptions) (int64, error) {
 	return db.GetEngine(ctx).Where(opts.toCond()).Count(&Review{})
 }
 
-// GetReviewersFromOriginalAuthorsByIssueID gets the latest review of each original authors for a pull request
-func GetReviewersFromOriginalAuthorsByIssueID(ctx context.Context, issueID int64) (ReviewList, error) {
+// GetReviewsByIssueID gets the latest review of each reviewer for a pull request
+// The first returned parameter is the latest review of each individual reviewer or team
+// The second returned parameter is the latest review of each original author which is migrated from other systems
+// The reviews are sorted by updated time
+func GetReviewsByIssueID(ctx context.Context, issueID int64) (latestReviews, migratedOriginalReviews ReviewList, err error) {
 	reviews := make([]*Review, 0, 10)
 
-	// Get latest review of each reviewer, sorted in order they were made
-	if err := db.GetEngine(ctx).SQL("SELECT * FROM review WHERE id IN (SELECT max(id) as id FROM review WHERE issue_id = ? AND reviewer_team_id = 0 AND type in (?, ?, ?) AND original_author_id <> 0 GROUP BY issue_id, original_author_id) ORDER BY review.updated_unix ASC",
-		issueID, ReviewTypeApprove, ReviewTypeReject, ReviewTypeRequest).
-		Find(&reviews); err != nil {
-		return nil, err
+	// Get all reviews for the issue id
+	if err := db.GetEngine(ctx).Where("issue_id=?", issueID).OrderBy("updated_unix ASC").Find(&reviews); err != nil {
+		return nil, nil, err
 	}
 
-	return reviews, nil
-}
-
-// GetReviewsByIssueID gets the latest review of each reviewer for a pull request
-func GetReviewsByIssueID(ctx context.Context, issueID int64) (ReviewList, error) {
-	reviews := make([]*Review, 0, 10)
-
-	sess := db.GetEngine(ctx)
+	// filter them in memory to get the latest review of each reviewer
+	// Since the reviews should not be too many for one issue, less than 100 commonly, it's acceptable to do this in memory
+	// And since there are too less indexes in review table, it will be very slow to filter in the database
+	reviewersMap := make(map[int64][]*Review)         // key is reviewer id
+	originalReviewersMap := make(map[int64][]*Review) // key is original author id
+	reviewTeamsMap := make(map[int64][]*Review)       // key is reviewer team id
+	countedReivewTypes := []ReviewType{ReviewTypeApprove, ReviewTypeReject, ReviewTypeRequest}
+	for _, review := range reviews {
+		if review.ReviewerTeamID == 0 && slices.Contains(countedReivewTypes, review.Type) && !review.Dismissed {
+			if review.OriginalAuthorID != 0 {
+				originalReviewersMap[review.OriginalAuthorID] = append(originalReviewersMap[review.OriginalAuthorID], review)
+			} else {
+				reviewersMap[review.ReviewerID] = append(reviewersMap[review.ReviewerID], review)
+			}
+		} else if review.ReviewerTeamID != 0 && review.OriginalAuthorID == 0 {
+			reviewTeamsMap[review.ReviewerTeamID] = append(reviewTeamsMap[review.ReviewerTeamID], review)
+		}
+	}
 
-	// Get latest review of each reviewer, sorted in order they were made
-	if err := sess.SQL("SELECT * FROM review WHERE id IN (SELECT max(id) as id FROM review WHERE issue_id = ? AND reviewer_team_id = 0 AND type in (?, ?, ?) AND dismissed = ? AND original_author_id = 0 GROUP BY issue_id, reviewer_id) ORDER BY review.updated_unix ASC",
-		issueID, ReviewTypeApprove, ReviewTypeReject, ReviewTypeRequest, false).
-		Find(&reviews); err != nil {
-		return nil, err
+	individualReviews := make([]*Review, 0, 10)
+	for _, reviews := range reviewersMap {
+		individualReviews = append(individualReviews, reviews[len(reviews)-1])
 	}
+	sort.Slice(individualReviews, func(i, j int) bool {
+		return individualReviews[i].UpdatedUnix < individualReviews[j].UpdatedUnix
+	})
 
-	teamReviewRequests := make([]*Review, 0, 5)
-	if err := sess.SQL("SELECT * FROM review WHERE id IN (SELECT max(id) as id FROM review WHERE issue_id = ? AND reviewer_team_id <> 0 AND original_author_id = 0 GROUP BY issue_id, reviewer_team_id) ORDER BY review.updated_unix ASC",
-		issueID).
-		Find(&teamReviewRequests); err != nil {
-		return nil, err
+	originalReviews := make([]*Review, 0, 10)
+	for _, reviews := range originalReviewersMap {
+		originalReviews = append(originalReviews, reviews[len(reviews)-1])
 	}
+	sort.Slice(originalReviews, func(i, j int) bool {
+		return originalReviews[i].UpdatedUnix < originalReviews[j].UpdatedUnix
+	})
 
-	if len(teamReviewRequests) > 0 {
-		reviews = append(reviews, teamReviewRequests...)
+	teamReviewRequests := make([]*Review, 0, 5)
+	for _, reviews := range reviewTeamsMap {
+		teamReviewRequests = append(teamReviewRequests, reviews[len(reviews)-1])
 	}
+	sort.Slice(teamReviewRequests, func(i, j int) bool {
+		return teamReviewRequests[i].UpdatedUnix < teamReviewRequests[j].UpdatedUnix
+	})
 
-	return reviews, nil
+	return append(individualReviews, teamReviewRequests...), originalReviews, nil
 }

models/issues/review_test.go

@@ -162,8 +162,9 @@ func TestGetReviewersByIssueID(t *testing.T) {
 		},
 	)
 
-	allReviews, err := issues_model.GetReviewsByIssueID(db.DefaultContext, issue.ID)
+	allReviews, migratedReviews, err := issues_model.GetReviewsByIssueID(db.DefaultContext, issue.ID)
 	assert.NoError(t, err)
+	assert.Empty(t, migratedReviews)
 	for _, review := range allReviews {
 		assert.NoError(t, review.LoadReviewer(db.DefaultContext))
 	}

routers/web/repo/issue_page_meta.go

@@ -193,6 +193,7 @@ func (d *IssuePageMetaData) retrieveReviewersData(ctx *context.Context) {
 	var posterID int64
 	var isClosed bool
 	var reviews issues_model.ReviewList
+	var err error
 
 	if d.Issue == nil {
 		if ctx.Doer != nil {
@@ -206,14 +207,7 @@ func (d *IssuePageMetaData) retrieveReviewersData(ctx *context.Context) {
 
 		isClosed = d.Issue.IsClosed || d.Issue.PullRequest.HasMerged
 
-		originalAuthorReviews, err := issues_model.GetReviewersFromOriginalAuthorsByIssueID(ctx, d.Issue.ID)
-		if err != nil {
-			ctx.ServerError("GetReviewersFromOriginalAuthorsByIssueID", err)
-			return
-		}
-		data.OriginalReviews = originalAuthorReviews
-
-		reviews, err = issues_model.GetReviewsByIssueID(ctx, d.Issue.ID)
+		reviews, data.OriginalReviews, err = issues_model.GetReviewsByIssueID(ctx, d.Issue.ID)
 		if err != nil {
 			ctx.ServerError("GetReviewersByIssueID", err)
 			return