mirror of
https://github.com/go-gitea/gitea.git
synced 2024-09-08 12:30:16 +02:00
# ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png)
353 lines
10 KiB
Go
353 lines
10 KiB
Go
// Copyright 2016 The Gitea Authors. All rights reserved.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
package code
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"runtime/pprof"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"code.gitea.io/gitea/models/db"
|
|
repo_model "code.gitea.io/gitea/models/repo"
|
|
"code.gitea.io/gitea/modules/graceful"
|
|
"code.gitea.io/gitea/modules/log"
|
|
"code.gitea.io/gitea/modules/process"
|
|
"code.gitea.io/gitea/modules/queue"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
"code.gitea.io/gitea/modules/timeutil"
|
|
)
|
|
|
|
// SearchResult result of performing a search in a repo
|
|
type SearchResult struct {
|
|
RepoID int64
|
|
StartIndex int
|
|
EndIndex int
|
|
Filename string
|
|
Content string
|
|
CommitID string
|
|
UpdatedUnix timeutil.TimeStamp
|
|
Language string
|
|
Color string
|
|
}
|
|
|
|
// SearchResultLanguages result of top languages count in search results
|
|
type SearchResultLanguages struct {
|
|
Language string
|
|
Color string
|
|
Count int
|
|
}
|
|
|
|
// Indexer defines an interface to index and search code contents
|
|
type Indexer interface {
|
|
Ping() bool
|
|
Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error
|
|
Delete(repoID int64) error
|
|
Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
|
|
Close()
|
|
}
|
|
|
|
func filenameIndexerID(repoID int64, filename string) string {
|
|
return indexerID(repoID) + "_" + filename
|
|
}
|
|
|
|
func indexerID(id int64) string {
|
|
return strconv.FormatInt(id, 36)
|
|
}
|
|
|
|
func parseIndexerID(indexerID string) (int64, string) {
|
|
index := strings.IndexByte(indexerID, '_')
|
|
if index == -1 {
|
|
log.Error("Unexpected ID in repo indexer: %s", indexerID)
|
|
}
|
|
repoID, _ := strconv.ParseInt(indexerID[:index], 36, 64)
|
|
return repoID, indexerID[index+1:]
|
|
}
|
|
|
|
func filenameOfIndexerID(indexerID string) string {
|
|
index := strings.IndexByte(indexerID, '_')
|
|
if index == -1 {
|
|
log.Error("Unexpected ID in repo indexer: %s", indexerID)
|
|
}
|
|
return indexerID[index+1:]
|
|
}
|
|
|
|
// IndexerData represents data stored in the code indexer
|
|
type IndexerData struct {
|
|
RepoID int64
|
|
}
|
|
|
|
var indexerQueue *queue.WorkerPoolQueue[*IndexerData]
|
|
|
|
func index(ctx context.Context, indexer Indexer, repoID int64) error {
|
|
repo, err := repo_model.GetRepositoryByID(ctx, repoID)
|
|
if repo_model.IsErrRepoNotExist(err) {
|
|
return indexer.Delete(repoID)
|
|
}
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
sha, err := getDefaultBranchSha(ctx, repo)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
changes, err := getRepoChanges(ctx, repo, sha)
|
|
if err != nil {
|
|
return err
|
|
} else if changes == nil {
|
|
return nil
|
|
}
|
|
|
|
if err := indexer.Index(ctx, repo, sha, changes); err != nil {
|
|
return err
|
|
}
|
|
|
|
return repo_model.UpdateIndexerStatus(ctx, repo, repo_model.RepoIndexerTypeCode, sha)
|
|
}
|
|
|
|
// Init initialize the repo indexer
|
|
func Init() {
|
|
if !setting.Indexer.RepoIndexerEnabled {
|
|
indexer.Close()
|
|
return
|
|
}
|
|
|
|
ctx, cancel, finished := process.GetManager().AddTypedContext(context.Background(), "Service: CodeIndexer", process.SystemProcessType, false)
|
|
|
|
graceful.GetManager().RunAtTerminate(func() {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
default:
|
|
}
|
|
cancel()
|
|
log.Debug("Closing repository indexer")
|
|
indexer.Close()
|
|
log.Info("PID: %d Repository Indexer closed", os.Getpid())
|
|
finished()
|
|
})
|
|
|
|
waitChannel := make(chan time.Duration, 1)
|
|
|
|
// Create the Queue
|
|
switch setting.Indexer.RepoType {
|
|
case "bleve", "elasticsearch":
|
|
handler := func(items ...*IndexerData) (unhandled []*IndexerData) {
|
|
idx, err := indexer.get()
|
|
if idx == nil || err != nil {
|
|
log.Error("Codes indexer handler: unable to get indexer!")
|
|
return items
|
|
}
|
|
|
|
for _, indexerData := range items {
|
|
log.Trace("IndexerData Process Repo: %d", indexerData.RepoID)
|
|
|
|
// FIXME: it seems there is a bug in `CatFileBatch` or `nio.Pipe`, which will cause the process to hang forever in rare cases
|
|
/*
|
|
sync.(*Cond).Wait(cond.go:70)
|
|
github.com/djherbis/nio/v3.(*PipeReader).Read(sync.go:106)
|
|
bufio.(*Reader).fill(bufio.go:106)
|
|
bufio.(*Reader).ReadSlice(bufio.go:372)
|
|
bufio.(*Reader).collectFragments(bufio.go:447)
|
|
bufio.(*Reader).ReadString(bufio.go:494)
|
|
code.gitea.io/gitea/modules/git.ReadBatchLine(batch_reader.go:149)
|
|
code.gitea.io/gitea/modules/indexer/code.(*BleveIndexer).addUpdate(bleve.go:214)
|
|
code.gitea.io/gitea/modules/indexer/code.(*BleveIndexer).Index(bleve.go:296)
|
|
code.gitea.io/gitea/modules/indexer/code.(*wrappedIndexer).Index(wrapped.go:74)
|
|
code.gitea.io/gitea/modules/indexer/code.index(indexer.go:105)
|
|
*/
|
|
if err := index(ctx, indexer, indexerData.RepoID); err != nil {
|
|
if !idx.Ping() {
|
|
log.Error("Code indexer handler: indexer is unavailable.")
|
|
unhandled = append(unhandled, indexerData)
|
|
continue
|
|
}
|
|
if !setting.IsInTesting {
|
|
log.Error("Codes indexer handler: index error for repo %v: %v", indexerData.RepoID, err)
|
|
}
|
|
}
|
|
}
|
|
return unhandled
|
|
}
|
|
|
|
indexerQueue = queue.CreateUniqueQueue("code_indexer", handler)
|
|
if indexerQueue == nil {
|
|
log.Fatal("Unable to create codes indexer queue")
|
|
}
|
|
default:
|
|
log.Fatal("Unknown codes indexer type; %s", setting.Indexer.RepoType)
|
|
}
|
|
|
|
go func() {
|
|
pprof.SetGoroutineLabels(ctx)
|
|
start := time.Now()
|
|
var (
|
|
rIndexer Indexer
|
|
populate bool
|
|
err error
|
|
)
|
|
switch setting.Indexer.RepoType {
|
|
case "bleve":
|
|
log.Info("PID: %d Initializing Repository Indexer at: %s", os.Getpid(), setting.Indexer.RepoPath)
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
log.Error("PANIC whilst initializing repository indexer: %v\nStacktrace: %s", err, log.Stack(2))
|
|
log.Error("The indexer files are likely corrupted and may need to be deleted")
|
|
log.Error("You can completely remove the \"%s\" directory to make Gitea recreate the indexes", setting.Indexer.RepoPath)
|
|
}
|
|
}()
|
|
|
|
rIndexer, populate, err = NewBleveIndexer(setting.Indexer.RepoPath)
|
|
if err != nil {
|
|
cancel()
|
|
indexer.Close()
|
|
close(waitChannel)
|
|
log.Fatal("PID: %d Unable to initialize the bleve Repository Indexer at path: %s Error: %v", os.Getpid(), setting.Indexer.RepoPath, err)
|
|
}
|
|
case "elasticsearch":
|
|
log.Info("PID: %d Initializing Repository Indexer at: %s", os.Getpid(), setting.Indexer.RepoConnStr)
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
log.Error("PANIC whilst initializing repository indexer: %v\nStacktrace: %s", err, log.Stack(2))
|
|
log.Error("The indexer files are likely corrupted and may need to be deleted")
|
|
log.Error("You can completely remove the \"%s\" index to make Gitea recreate the indexes", setting.Indexer.RepoConnStr)
|
|
}
|
|
}()
|
|
|
|
rIndexer, populate, err = NewElasticSearchIndexer(setting.Indexer.RepoConnStr, setting.Indexer.RepoIndexerName)
|
|
if err != nil {
|
|
cancel()
|
|
indexer.Close()
|
|
close(waitChannel)
|
|
log.Fatal("PID: %d Unable to initialize the elasticsearch Repository Indexer connstr: %s Error: %v", os.Getpid(), setting.Indexer.RepoConnStr, err)
|
|
}
|
|
default:
|
|
log.Fatal("PID: %d Unknown Indexer type: %s", os.Getpid(), setting.Indexer.RepoType)
|
|
}
|
|
|
|
indexer.set(rIndexer)
|
|
|
|
// Start processing the queue
|
|
go graceful.GetManager().RunWithShutdownFns(indexerQueue.Run)
|
|
|
|
if populate {
|
|
go graceful.GetManager().RunWithShutdownContext(populateRepoIndexer)
|
|
}
|
|
select {
|
|
case waitChannel <- time.Since(start):
|
|
case <-graceful.GetManager().IsShutdown():
|
|
}
|
|
|
|
close(waitChannel)
|
|
}()
|
|
|
|
if setting.Indexer.StartupTimeout > 0 {
|
|
go func() {
|
|
pprof.SetGoroutineLabels(ctx)
|
|
timeout := setting.Indexer.StartupTimeout
|
|
if graceful.GetManager().IsChild() && setting.GracefulHammerTime > 0 {
|
|
timeout += setting.GracefulHammerTime
|
|
}
|
|
select {
|
|
case <-graceful.GetManager().IsShutdown():
|
|
log.Warn("Shutdown before Repository Indexer completed initialization")
|
|
cancel()
|
|
indexer.Close()
|
|
case duration, ok := <-waitChannel:
|
|
if !ok {
|
|
log.Warn("Repository Indexer Initialization failed")
|
|
cancel()
|
|
indexer.Close()
|
|
return
|
|
}
|
|
log.Info("Repository Indexer Initialization took %v", duration)
|
|
case <-time.After(timeout):
|
|
cancel()
|
|
indexer.Close()
|
|
log.Fatal("Repository Indexer Initialization Timed-Out after: %v", timeout)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
// UpdateRepoIndexer update a repository's entries in the indexer
|
|
func UpdateRepoIndexer(repo *repo_model.Repository) {
|
|
indexData := &IndexerData{RepoID: repo.ID}
|
|
if err := indexerQueue.Push(indexData); err != nil {
|
|
log.Error("Update repo index data %v failed: %v", indexData, err)
|
|
}
|
|
}
|
|
|
|
// IsAvailable checks if issue indexer is available
|
|
func IsAvailable() bool {
|
|
idx, err := indexer.get()
|
|
if err != nil {
|
|
log.Error("IsAvailable(): unable to get indexer: %v", err)
|
|
return false
|
|
}
|
|
|
|
return idx.Ping()
|
|
}
|
|
|
|
// populateRepoIndexer populate the repo indexer with pre-existing data. This
|
|
// should only be run when the indexer is created for the first time.
|
|
func populateRepoIndexer(ctx context.Context) {
|
|
log.Info("Populating the repo indexer with existing repositories")
|
|
|
|
exist, err := db.IsTableNotEmpty("repository")
|
|
if err != nil {
|
|
log.Fatal("System error: %v", err)
|
|
} else if !exist {
|
|
return
|
|
}
|
|
|
|
// if there is any existing repo indexer metadata in the DB, delete it
|
|
// since we are starting afresh. Also, xorm requires deletes to have a
|
|
// condition, and we want to delete everything, thus 1=1.
|
|
if err := db.DeleteAllRecords("repo_indexer_status"); err != nil {
|
|
log.Fatal("System error: %v", err)
|
|
}
|
|
|
|
var maxRepoID int64
|
|
if maxRepoID, err = db.GetMaxID("repository"); err != nil {
|
|
log.Fatal("System error: %v", err)
|
|
}
|
|
|
|
// start with the maximum existing repo ID and work backwards, so that we
|
|
// don't include repos that are created after gitea starts; such repos will
|
|
// already be added to the indexer, and we don't need to add them again.
|
|
for maxRepoID > 0 {
|
|
select {
|
|
case <-ctx.Done():
|
|
log.Info("Repository Indexer population shutdown before completion")
|
|
return
|
|
default:
|
|
}
|
|
ids, err := repo_model.GetUnindexedRepos(repo_model.RepoIndexerTypeCode, maxRepoID, 0, 50)
|
|
if err != nil {
|
|
log.Error("populateRepoIndexer: %v", err)
|
|
return
|
|
} else if len(ids) == 0 {
|
|
break
|
|
}
|
|
for _, id := range ids {
|
|
select {
|
|
case <-ctx.Done():
|
|
log.Info("Repository Indexer population shutdown before completion")
|
|
return
|
|
default:
|
|
}
|
|
if err := indexerQueue.Push(&IndexerData{RepoID: id}); err != nil {
|
|
log.Error("indexerQueue.Push: %v", err)
|
|
return
|
|
}
|
|
maxRepoID = id - 1
|
|
}
|
|
}
|
|
log.Info("Done (re)populating the repo indexer with existing repositories")
|
|
}
|