mirror of https://github.com/go-gitea/gitea
Git with a cup of tea, painless self-hosted git service
Mirror for internal git.with.parts use
https://git.with.parts
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
274 lines
6.7 KiB
274 lines
6.7 KiB
// Copyright 2018 The Gitea Authors. All rights reserved.
|
|
// Use of this source code is governed by a MIT-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package issues
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
"code.gitea.io/gitea/models"
|
|
"code.gitea.io/gitea/modules/graceful"
|
|
"code.gitea.io/gitea/modules/log"
|
|
"code.gitea.io/gitea/modules/setting"
|
|
"code.gitea.io/gitea/modules/util"
|
|
)
|
|
|
|
// IndexerData data stored in the issue indexer
|
|
type IndexerData struct {
|
|
ID int64
|
|
RepoID int64
|
|
Title string
|
|
Content string
|
|
Comments []string
|
|
IsDelete bool
|
|
IDs []int64
|
|
}
|
|
|
|
// Match represents on search result
|
|
type Match struct {
|
|
ID int64 `json:"id"`
|
|
Score float64 `json:"score"`
|
|
}
|
|
|
|
// SearchResult represents search results
|
|
type SearchResult struct {
|
|
Total int64
|
|
Hits []Match
|
|
}
|
|
|
|
// Indexer defines an inteface to indexer issues contents
|
|
type Indexer interface {
|
|
Init() (bool, error)
|
|
Index(issue []*IndexerData) error
|
|
Delete(ids ...int64) error
|
|
Search(kw string, repoIDs []int64, limit, start int) (*SearchResult, error)
|
|
}
|
|
|
|
type indexerHolder struct {
|
|
indexer Indexer
|
|
mutex sync.RWMutex
|
|
cond *sync.Cond
|
|
}
|
|
|
|
func newIndexerHolder() *indexerHolder {
|
|
h := &indexerHolder{}
|
|
h.cond = sync.NewCond(h.mutex.RLocker())
|
|
return h
|
|
}
|
|
|
|
func (h *indexerHolder) set(indexer Indexer) {
|
|
h.mutex.Lock()
|
|
defer h.mutex.Unlock()
|
|
h.indexer = indexer
|
|
h.cond.Broadcast()
|
|
}
|
|
|
|
func (h *indexerHolder) get() Indexer {
|
|
h.mutex.RLock()
|
|
defer h.mutex.RUnlock()
|
|
if h.indexer == nil {
|
|
h.cond.Wait()
|
|
}
|
|
return h.indexer
|
|
}
|
|
|
|
var (
|
|
issueIndexerChannel = make(chan *IndexerData, setting.Indexer.UpdateQueueLength)
|
|
// issueIndexerQueue queue of issue ids to be updated
|
|
issueIndexerQueue Queue
|
|
holder = newIndexerHolder()
|
|
)
|
|
|
|
// InitIssueIndexer initialize issue indexer, syncReindex is true then reindex until
|
|
// all issue index done.
|
|
func InitIssueIndexer(syncReindex bool) {
|
|
waitChannel := make(chan time.Duration)
|
|
go func() {
|
|
start := time.Now()
|
|
log.Info("Initializing Issue Indexer")
|
|
var populate bool
|
|
var dummyQueue bool
|
|
switch setting.Indexer.IssueType {
|
|
case "bleve":
|
|
issueIndexer := NewBleveIndexer(setting.Indexer.IssuePath)
|
|
exist, err := issueIndexer.Init()
|
|
if err != nil {
|
|
log.Fatal("Unable to initialize Bleve Issue Indexer: %v", err)
|
|
}
|
|
populate = !exist
|
|
holder.set(issueIndexer)
|
|
case "db":
|
|
issueIndexer := &DBIndexer{}
|
|
holder.set(issueIndexer)
|
|
dummyQueue = true
|
|
default:
|
|
log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType)
|
|
}
|
|
|
|
if dummyQueue {
|
|
issueIndexerQueue = &DummyQueue{}
|
|
} else {
|
|
var err error
|
|
switch setting.Indexer.IssueQueueType {
|
|
case setting.LevelQueueType:
|
|
issueIndexerQueue, err = NewLevelQueue(
|
|
holder.get(),
|
|
setting.Indexer.IssueQueueDir,
|
|
setting.Indexer.IssueQueueBatchNumber)
|
|
if err != nil {
|
|
log.Fatal(
|
|
"Unable create level queue for issue queue dir: %s batch number: %d : %v",
|
|
setting.Indexer.IssueQueueDir,
|
|
setting.Indexer.IssueQueueBatchNumber,
|
|
err)
|
|
}
|
|
case setting.ChannelQueueType:
|
|
issueIndexerQueue = NewChannelQueue(holder.get(), setting.Indexer.IssueQueueBatchNumber)
|
|
case setting.RedisQueueType:
|
|
addrs, pass, idx, err := parseConnStr(setting.Indexer.IssueQueueConnStr)
|
|
if err != nil {
|
|
log.Fatal("Unable to parse connection string for RedisQueueType: %s : %v",
|
|
setting.Indexer.IssueQueueConnStr,
|
|
err)
|
|
}
|
|
issueIndexerQueue, err = NewRedisQueue(addrs, pass, idx, holder.get(), setting.Indexer.IssueQueueBatchNumber)
|
|
if err != nil {
|
|
log.Fatal("Unable to create RedisQueue: %s : %v",
|
|
setting.Indexer.IssueQueueConnStr,
|
|
err)
|
|
}
|
|
default:
|
|
log.Fatal("Unsupported indexer queue type: %v",
|
|
setting.Indexer.IssueQueueType)
|
|
}
|
|
|
|
go func() {
|
|
err = issueIndexerQueue.Run()
|
|
if err != nil {
|
|
log.Error("issueIndexerQueue.Run: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
|
|
go func() {
|
|
for data := range issueIndexerChannel {
|
|
_ = issueIndexerQueue.Push(data)
|
|
}
|
|
}()
|
|
|
|
if populate {
|
|
if syncReindex {
|
|
populateIssueIndexer()
|
|
} else {
|
|
go populateIssueIndexer()
|
|
}
|
|
}
|
|
waitChannel <- time.Since(start)
|
|
}()
|
|
if syncReindex {
|
|
<-waitChannel
|
|
} else if setting.Indexer.StartupTimeout > 0 {
|
|
go func() {
|
|
timeout := setting.Indexer.StartupTimeout
|
|
if graceful.Manager.IsChild() && setting.GracefulHammerTime > 0 {
|
|
timeout += setting.GracefulHammerTime
|
|
}
|
|
select {
|
|
case duration := <-waitChannel:
|
|
log.Info("Issue Indexer Initialization took %v", duration)
|
|
case <-time.After(timeout):
|
|
log.Fatal("Issue Indexer Initialization timed-out after: %v", timeout)
|
|
}
|
|
}()
|
|
}
|
|
}
|
|
|
|
// populateIssueIndexer populate the issue indexer with issue data
|
|
func populateIssueIndexer() {
|
|
for page := 1; ; page++ {
|
|
repos, _, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
|
|
Page: page,
|
|
PageSize: models.RepositoryListDefaultPageSize,
|
|
OrderBy: models.SearchOrderByID,
|
|
Private: true,
|
|
Collaborate: util.OptionalBoolFalse,
|
|
})
|
|
if err != nil {
|
|
log.Error("SearchRepositoryByName: %v", err)
|
|
continue
|
|
}
|
|
if len(repos) == 0 {
|
|
return
|
|
}
|
|
|
|
for _, repo := range repos {
|
|
is, err := models.Issues(&models.IssuesOptions{
|
|
RepoIDs: []int64{repo.ID},
|
|
IsClosed: util.OptionalBoolNone,
|
|
IsPull: util.OptionalBoolNone,
|
|
})
|
|
if err != nil {
|
|
log.Error("Issues: %v", err)
|
|
continue
|
|
}
|
|
if err = models.IssueList(is).LoadDiscussComments(); err != nil {
|
|
log.Error("LoadComments: %v", err)
|
|
continue
|
|
}
|
|
for _, issue := range is {
|
|
UpdateIssueIndexer(issue)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// UpdateIssueIndexer add/update an issue to the issue indexer
|
|
func UpdateIssueIndexer(issue *models.Issue) {
|
|
var comments []string
|
|
for _, comment := range issue.Comments {
|
|
if comment.Type == models.CommentTypeComment {
|
|
comments = append(comments, comment.Content)
|
|
}
|
|
}
|
|
issueIndexerChannel <- &IndexerData{
|
|
ID: issue.ID,
|
|
RepoID: issue.RepoID,
|
|
Title: issue.Title,
|
|
Content: issue.Content,
|
|
Comments: comments,
|
|
}
|
|
}
|
|
|
|
// DeleteRepoIssueIndexer deletes repo's all issues indexes
|
|
func DeleteRepoIssueIndexer(repo *models.Repository) {
|
|
var ids []int64
|
|
ids, err := models.GetIssueIDsByRepoID(repo.ID)
|
|
if err != nil {
|
|
log.Error("getIssueIDsByRepoID failed: %v", err)
|
|
return
|
|
}
|
|
|
|
if len(ids) == 0 {
|
|
return
|
|
}
|
|
|
|
issueIndexerChannel <- &IndexerData{
|
|
IDs: ids,
|
|
IsDelete: true,
|
|
}
|
|
}
|
|
|
|
// SearchIssuesByKeyword search issue ids by keywords and repo id
|
|
func SearchIssuesByKeyword(repoIDs []int64, keyword string) ([]int64, error) {
|
|
var issueIDs []int64
|
|
res, err := holder.get().Search(keyword, repoIDs, 1000, 0)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, r := range res.Hits {
|
|
issueIDs = append(issueIDs, r.ID)
|
|
}
|
|
return issueIDs, nil
|
|
}
|
|
|