Add commit status summary table to reduce query from commit status table (#30223)

This PR adds a new table named commit status summary to reduce queries
from the commit status table. After this change, commit status summary
table will be used for the final result, commit status table will be for
details.

---------

Co-authored-by: Jason Song <i@wolfogre.com>
This commit is contained in:
Lunny Xiao 2024-04-12 09:41:50 +08:00 committed by GitHub
parent 26ee66327f
commit fc34481d05
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 170 additions and 31 deletions

View File

@ -292,30 +292,27 @@ func GetLatestCommitStatus(ctx context.Context, repoID int64, sha string, listOp
} }
// GetLatestCommitStatusForPairs returns all statuses with a unique context for a given list of repo-sha pairs // GetLatestCommitStatusForPairs returns all statuses with a unique context for a given list of repo-sha pairs
func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHAs map[int64]string, listOptions db.ListOptions) (map[int64][]*CommitStatus, error) { func GetLatestCommitStatusForPairs(ctx context.Context, repoSHAs []RepoSHA) (map[int64][]*CommitStatus, error) {
type result struct { type result struct {
Index int64 Index int64
RepoID int64 RepoID int64
SHA string
} }
results := make([]result, 0, len(repoIDsToLatestCommitSHAs)) results := make([]result, 0, len(repoSHAs))
getBase := func() *xorm.Session { getBase := func() *xorm.Session {
return db.GetEngine(ctx).Table(&CommitStatus{}) return db.GetEngine(ctx).Table(&CommitStatus{})
} }
// Create a disjunction of conditions for each repoID and SHA pair // Create a disjunction of conditions for each repoID and SHA pair
conds := make([]builder.Cond, 0, len(repoIDsToLatestCommitSHAs)) conds := make([]builder.Cond, 0, len(repoSHAs))
for repoID, sha := range repoIDsToLatestCommitSHAs { for _, repoSHA := range repoSHAs {
conds = append(conds, builder.Eq{"repo_id": repoID, "sha": sha}) conds = append(conds, builder.Eq{"repo_id": repoSHA.RepoID, "sha": repoSHA.SHA})
} }
sess := getBase().Where(builder.Or(conds...)). sess := getBase().Where(builder.Or(conds...)).
Select("max( `index` ) as `index`, repo_id"). Select("max( `index` ) as `index`, repo_id, sha").
GroupBy("context_hash, repo_id").OrderBy("max( `index` ) desc") GroupBy("context_hash, repo_id, sha").OrderBy("max( `index` ) desc")
if !listOptions.IsListAll() {
sess = db.SetSessionPagination(sess, &listOptions)
}
err := sess.Find(&results) err := sess.Find(&results)
if err != nil { if err != nil {
@ -332,7 +329,7 @@ func GetLatestCommitStatusForPairs(ctx context.Context, repoIDsToLatestCommitSHA
cond := builder.Eq{ cond := builder.Eq{
"`index`": result.Index, "`index`": result.Index,
"repo_id": result.RepoID, "repo_id": result.RepoID,
"sha": repoIDsToLatestCommitSHAs[result.RepoID], "sha": result.SHA,
} }
conds = append(conds, cond) conds = append(conds, cond)
} }

View File

@ -0,0 +1,84 @@
// Copyright 2024 Gitea. All rights reserved.
// SPDX-License-Identifier: MIT
package git
import (
"context"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
"xorm.io/builder"
)
// CommitStatusSummary holds the latest commit Status of a single Commit
type CommitStatusSummary struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX UNIQUE(repo_id_sha)"`
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
State api.CommitStatusState `xorm:"VARCHAR(7) NOT NULL"`
}
func init() {
db.RegisterModel(new(CommitStatusSummary))
}
type RepoSHA struct {
RepoID int64
SHA string
}
func GetLatestCommitStatusForRepoAndSHAs(ctx context.Context, repoSHAs []RepoSHA) ([]*CommitStatus, error) {
cond := builder.NewCond()
for _, rs := range repoSHAs {
cond = cond.Or(builder.Eq{"repo_id": rs.RepoID, "sha": rs.SHA})
}
var summaries []CommitStatusSummary
if err := db.GetEngine(ctx).Where(cond).Find(&summaries); err != nil {
return nil, err
}
commitStatuses := make([]*CommitStatus, 0, len(repoSHAs))
for _, summary := range summaries {
commitStatuses = append(commitStatuses, &CommitStatus{
RepoID: summary.RepoID,
SHA: summary.SHA,
State: summary.State,
})
}
return commitStatuses, nil
}
func UpdateCommitStatusSummary(ctx context.Context, repoID int64, sha string) error {
commitStatuses, _, err := GetLatestCommitStatus(ctx, repoID, sha, db.ListOptionsAll)
if err != nil {
return err
}
state := CalcCommitStatus(commitStatuses)
// mysql will return 0 when update a record which state hasn't been changed which behaviour is different from other database,
// so we need to use insert in on duplicate
if setting.Database.Type.IsMySQL() {
_, err := db.GetEngine(ctx).Exec("INSERT INTO commit_status_summary (repo_id,sha,state) VALUES (?,?,?) ON DUPLICATE KEY UPDATE state=?",
repoID, sha, state.State, state.State)
return err
}
if cnt, err := db.GetEngine(ctx).Where("repo_id=? AND sha=?", repoID, sha).
Cols("state").
Update(&CommitStatusSummary{
State: state.State,
}); err != nil {
return err
} else if cnt == 0 {
_, err = db.GetEngine(ctx).Insert(&CommitStatusSummary{
RepoID: repoID,
SHA: sha,
State: state.State,
})
return err
}
return nil
}

View File

@ -576,7 +576,10 @@ var migrations = []Migration{
// Gitea 1.22.0 ends at 294 // Gitea 1.22.0 ends at 294
// v294 -> v295
NewMigration("Add unique index for project issue table", v1_23.AddUniqueIndexForProjectIssue), NewMigration("Add unique index for project issue table", v1_23.AddUniqueIndexForProjectIssue),
// v295 -> v296
NewMigration("Add commit status summary table", v1_23.AddCommitStatusSummary),
} }
// GetCurrentDBVersion returns the current db version // GetCurrentDBVersion returns the current db version

View File

@ -0,0 +1,18 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_23 //nolint
import "xorm.io/xorm"
func AddCommitStatusSummary(x *xorm.Engine) error {
type CommitStatusSummary struct {
ID int64 `xorm:"pk autoincr"`
RepoID int64 `xorm:"INDEX UNIQUE(repo_id_sha)"`
SHA string `xorm:"VARCHAR(64) NOT NULL INDEX UNIQUE(repo_id_sha)"`
State string `xorm:"VARCHAR(7) NOT NULL"`
}
// there is no migrations because if there is no data on this table, it will fall back to get data
// from commit status
return x.Sync2(new(CommitStatusSummary))
}

View File

@ -16,6 +16,7 @@ import (
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
api "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs"
webhook_module "code.gitea.io/gitea/modules/webhook" webhook_module "code.gitea.io/gitea/modules/webhook"
commitstatus_service "code.gitea.io/gitea/services/repository/commitstatus"
"github.com/nektos/act/pkg/jobparser" "github.com/nektos/act/pkg/jobparser"
) )
@ -122,18 +123,13 @@ func createCommitStatus(ctx context.Context, job *actions_model.ActionRunJob) er
if err != nil { if err != nil {
return fmt.Errorf("HashTypeInterfaceFromHashString: %w", err) return fmt.Errorf("HashTypeInterfaceFromHashString: %w", err)
} }
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{ if err := commitstatus_service.CreateCommitStatus(ctx, repo, creator, commitID.String(), &git_model.CommitStatus{
Repo: repo,
SHA: commitID,
Creator: creator,
CommitStatus: &git_model.CommitStatus{
SHA: sha, SHA: sha,
TargetURL: fmt.Sprintf("%s/jobs/%d", run.Link(), index), TargetURL: fmt.Sprintf("%s/jobs/%d", run.Link(), index),
Description: description, Description: description,
Context: ctxname, Context: ctxname,
CreatorID: creator.ID, CreatorID: creator.ID,
State: state, State: state,
},
}); err != nil { }); err != nil {
return fmt.Errorf("NewCommitStatus: %w", err) return fmt.Errorf("NewCommitStatus: %w", err)
} }

View File

@ -7,6 +7,7 @@ import (
"context" "context"
"crypto/sha256" "crypto/sha256"
"fmt" "fmt"
"slices"
"code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git" git_model "code.gitea.io/gitea/models/git"
@ -59,6 +60,7 @@ func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creato
sha = commit.ID.String() sha = commit.ID.String()
} }
if err := db.WithTx(ctx, func(ctx context.Context) error {
if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{ if err := git_model.NewCommitStatus(ctx, git_model.NewCommitStatusOptions{
Repo: repo, Repo: repo,
Creator: creator, Creator: creator,
@ -68,6 +70,11 @@ func CreateCommitStatus(ctx context.Context, repo *repo_model.Repository, creato
return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err) return fmt.Errorf("NewCommitStatus[repo_id: %d, user_id: %d, sha: %s]: %w", repo.ID, creator.ID, sha, err)
} }
return git_model.UpdateCommitStatusSummary(ctx, repo.ID, commit.ID.String())
}); err != nil {
return err
}
defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch) defaultBranchCommit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
if err != nil { if err != nil {
return fmt.Errorf("GetBranchCommit[%s]: %w", repo.DefaultBranch, err) return fmt.Errorf("GetBranchCommit[%s]: %w", repo.DefaultBranch, err)
@ -114,8 +121,35 @@ func FindReposLastestCommitStatuses(ctx context.Context, repos []*repo_model.Rep
return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err) return nil, fmt.Errorf("FindBranchesByRepoAndBranchName: %v", err)
} }
var repoSHAs []git_model.RepoSHA
for id, sha := range repoIDsToLatestCommitSHAs {
repoSHAs = append(repoSHAs, git_model.RepoSHA{RepoID: id, SHA: sha})
}
summaryResults, err := git_model.GetLatestCommitStatusForRepoAndSHAs(ctx, repoSHAs)
if err != nil {
return nil, fmt.Errorf("GetLatestCommitStatusForRepoAndSHAs: %v", err)
}
for _, summary := range summaryResults {
for i, repo := range repos {
if repo.ID == summary.RepoID {
results[i] = summary
_ = slices.DeleteFunc(repoSHAs, func(repoSHA git_model.RepoSHA) bool {
return repoSHA.RepoID == repo.ID
})
if results[i].State != "" {
if err := updateCommitStatusCache(ctx, repo.ID, repo.DefaultBranch, results[i].State); err != nil {
log.Error("updateCommitStatusCache[%d:%s] failed: %v", repo.ID, repo.DefaultBranch, err)
}
}
break
}
}
}
// call the database O(1) times to get the commit statuses for all repos // call the database O(1) times to get the commit statuses for all repos
repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoIDsToLatestCommitSHAs, db.ListOptionsAll) repoToItsLatestCommitStatuses, err := git_model.GetLatestCommitStatusForPairs(ctx, repoSHAs)
if err != nil { if err != nil {
return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err) return nil, fmt.Errorf("GetLatestCommitStatusForPairs: %v", err)
} }

View File

@ -12,6 +12,9 @@ import (
"testing" "testing"
auth_model "code.gitea.io/gitea/models/auth" auth_model "code.gitea.io/gitea/models/auth"
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
api "code.gitea.io/gitea/modules/structs" api "code.gitea.io/gitea/modules/structs"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -90,6 +93,10 @@ func TestPullCreate_CommitStatus(t *testing.T) {
assert.True(t, ok) assert.True(t, ok)
assert.Contains(t, cls, statesIcons[status]) assert.Contains(t, cls, statesIcons[status])
} }
repo1 := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{OwnerName: "user1", Name: "repo1"})
css := unittest.AssertExistsAndLoadBean(t, &git_model.CommitStatusSummary{RepoID: repo1.ID, SHA: commitID})
assert.EqualValues(t, api.CommitStatusWarning, css.State)
}) })
} }