From cb01b8691db6ff13db3331dbe90ea291e1d6eadd Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Fri, 7 Jul 2023 13:10:13 -0400 Subject: [PATCH] Add open/closed field support for issue index (#25708) A couple of notes: * Future changes should refactor arguments into a struct * This filtering only is supported by meilisearch right now * Issue index number is bumped which will cause a re-index --- modules/indexer/issues/bleve/bleve.go | 2 +- modules/indexer/issues/bleve/bleve_test.go | 2 +- modules/indexer/issues/db/db.go | 2 +- .../issues/elasticsearch/elasticsearch.go | 2 +- modules/indexer/issues/indexer.go | 20 ++++++++++++------- modules/indexer/issues/indexer_test.go | 16 +++++++-------- modules/indexer/issues/internal/indexer.go | 4 ++-- modules/indexer/issues/internal/model.go | 16 ++++++++------- .../indexer/issues/meilisearch/meilisearch.go | 11 ++++++++-- routers/api/v1/repo/issue.go | 4 ++-- routers/web/repo/issue.go | 6 +++--- routers/web/user/home.go | 2 +- 12 files changed, 51 insertions(+), 36 deletions(-) diff --git a/modules/indexer/issues/bleve/bleve.go b/modules/indexer/issues/bleve/bleve.go index 4cc58acac78..c368a67ab58 100644 --- a/modules/indexer/issues/bleve/bleve.go +++ b/modules/indexer/issues/bleve/bleve.go @@ -138,7 +138,7 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs -func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) { +func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) { var repoQueriesP []*query.NumericRangeQuery for _, repoID := range repoIDs { repoQueriesP = append(repoQueriesP, numericEqualityQuery(repoID, "repo_id")) diff --git a/modules/indexer/issues/bleve/bleve_test.go b/modules/indexer/issues/bleve/bleve_test.go index f890f8eb488..0eb136d22b2 100644 --- a/modules/indexer/issues/bleve/bleve_test.go +++ b/modules/indexer/issues/bleve/bleve_test.go @@ -77,7 +77,7 @@ func TestBleveIndexAndSearch(t *testing.T) { } for _, kw := range keywords { - res, err := indexer.Search(context.TODO(), kw.Keyword, []int64{2}, 10, 0) + res, err := indexer.Search(context.TODO(), kw.Keyword, []int64{2}, 10, 0, "") assert.NoError(t, err) ids := make([]int64, 0, len(res.Hits)) diff --git a/modules/indexer/issues/db/db.go b/modules/indexer/issues/db/db.go index 17ed426b384..b054b9d800e 100644 --- a/modules/indexer/issues/db/db.go +++ b/modules/indexer/issues/db/db.go @@ -36,7 +36,7 @@ func (i *Indexer) Delete(_ context.Context, _ ...int64) error { } // Search searches for issues -func (i *Indexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) { +func (i *Indexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) { total, ids, err := issues_model.SearchIssueIDsByKeyword(ctx, kw, repoIDs, limit, start) if err != nil { return nil, err diff --git a/modules/indexer/issues/elasticsearch/elasticsearch.go b/modules/indexer/issues/elasticsearch/elasticsearch.go index 62889ea578f..cfd3628c185 100644 --- a/modules/indexer/issues/elasticsearch/elasticsearch.go +++ b/modules/indexer/issues/elasticsearch/elasticsearch.go @@ -140,7 +140,7 @@ func (b *Indexer) Delete(ctx context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs -func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) { +func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) { kwQuery := elastic.NewMultiMatchQuery(keyword, "title", "content", "comments") query := elastic.NewBoolQuery() query = query.Must(kwQuery) diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 9e2f13371e4..fe5c5d8f26d 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -242,12 +242,18 @@ func UpdateIssueIndexer(issue *issues_model.Issue) { comments = append(comments, comment.Content) } } + issueType := "issue" + if issue.IsPull { + issueType = "pull" + } indexerData := &internal.IndexerData{ - ID: issue.ID, - RepoID: issue.RepoID, - Title: issue.Title, - Content: issue.Content, - Comments: comments, + ID: issue.ID, + RepoID: issue.RepoID, + State: string(issue.State()), + IssueType: issueType, + Title: issue.Title, + Content: issue.Content, + Comments: comments, } log.Debug("Adding to channel: %v", indexerData) if err := issueIndexerQueue.Push(indexerData); err != nil { @@ -278,10 +284,10 @@ func DeleteRepoIssueIndexer(ctx context.Context, repo *repo_model.Repository) { // SearchIssuesByKeyword search issue ids by keywords and repo id // WARNNING: You have to ensure user have permission to visit repoIDs' issues -func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword string) ([]int64, error) { +func SearchIssuesByKeyword(ctx context.Context, repoIDs []int64, keyword, state string) ([]int64, error) { var issueIDs []int64 indexer := *globalIndexer.Load() - res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0) + res, err := indexer.Search(ctx, keyword, repoIDs, 50, 0, state) if err != nil { return nil, err } diff --git a/modules/indexer/issues/indexer_test.go b/modules/indexer/issues/indexer_test.go index 5962a4ee9cb..757eb2f3d93 100644 --- a/modules/indexer/issues/indexer_test.go +++ b/modules/indexer/issues/indexer_test.go @@ -50,19 +50,19 @@ func TestBleveSearchIssues(t *testing.T) { time.Sleep(5 * time.Second) - ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2") + ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2", "") assert.NoError(t, err) assert.EqualValues(t, []int64{2}, ids) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first") + ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first", "") assert.NoError(t, err) assert.EqualValues(t, []int64{1}, ids) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for") + ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for", "") assert.NoError(t, err) assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good") + ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good", "") assert.NoError(t, err) assert.EqualValues(t, []int64{1}, ids) } @@ -73,19 +73,19 @@ func TestDBSearchIssues(t *testing.T) { setting.Indexer.IssueType = "db" InitIssueIndexer(true) - ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2") + ids, err := SearchIssuesByKeyword(context.TODO(), []int64{1}, "issue2", "") assert.NoError(t, err) assert.EqualValues(t, []int64{2}, ids) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first") + ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "first", "") assert.NoError(t, err) assert.EqualValues(t, []int64{1}, ids) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for") + ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "for", "") assert.NoError(t, err) assert.ElementsMatch(t, []int64{1, 2, 3, 5, 11}, ids) - ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good") + ids, err = SearchIssuesByKeyword(context.TODO(), []int64{1}, "good", "") assert.NoError(t, err) assert.EqualValues(t, []int64{1}, ids) } diff --git a/modules/indexer/issues/internal/indexer.go b/modules/indexer/issues/internal/indexer.go index 553c8a573cd..b96517bb80d 100644 --- a/modules/indexer/issues/internal/indexer.go +++ b/modules/indexer/issues/internal/indexer.go @@ -15,7 +15,7 @@ type Indexer interface { internal.Indexer Index(ctx context.Context, issue []*IndexerData) error Delete(ctx context.Context, ids ...int64) error - Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) + Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*SearchResult, error) } // NewDummyIndexer returns a dummy indexer @@ -37,6 +37,6 @@ func (d *dummyIndexer) Delete(ctx context.Context, ids ...int64) error { return fmt.Errorf("indexer is not ready") } -func (d *dummyIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) { +func (d *dummyIndexer) Search(ctx context.Context, kw string, repoIDs []int64, limit, start int, state string) (*SearchResult, error) { return nil, fmt.Errorf("indexer is not ready") } diff --git a/modules/indexer/issues/internal/model.go b/modules/indexer/issues/internal/model.go index 8c206fc1cfc..2b52d32302a 100644 --- a/modules/indexer/issues/internal/model.go +++ b/modules/indexer/issues/internal/model.go @@ -5,13 +5,15 @@ package internal // IndexerData data stored in the issue indexer type IndexerData struct { - ID int64 `json:"id"` - RepoID int64 `json:"repo_id"` - Title string `json:"title"` - Content string `json:"content"` - Comments []string `json:"comments"` - IsDelete bool `json:"is_delete"` - IDs []int64 `json:"ids"` + ID int64 `json:"id"` + RepoID int64 `json:"repo_id"` + State string `json:"state"` // open, closed, all + IssueType string `json:"type"` // issue or pull + Title string `json:"title"` + Content string `json:"content"` + Comments []string `json:"comments"` + IsDelete bool `json:"is_delete"` + IDs []int64 `json:"ids"` } // Match represents on search result diff --git a/modules/indexer/issues/meilisearch/meilisearch.go b/modules/indexer/issues/meilisearch/meilisearch.go index 2a46b1ab15c..2ea06b576c0 100644 --- a/modules/indexer/issues/meilisearch/meilisearch.go +++ b/modules/indexer/issues/meilisearch/meilisearch.go @@ -16,7 +16,7 @@ import ( ) const ( - issueIndexerLatestVersion = 0 + issueIndexerLatestVersion = 1 ) var _ internal.Indexer = &Indexer{} @@ -70,12 +70,19 @@ func (b *Indexer) Delete(_ context.Context, ids ...int64) error { // Search searches for issues by given conditions. // Returns the matching issue IDs -func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int) (*internal.SearchResult, error) { +func (b *Indexer) Search(ctx context.Context, keyword string, repoIDs []int64, limit, start int, state string) (*internal.SearchResult, error) { repoFilters := make([]string, 0, len(repoIDs)) for _, repoID := range repoIDs { repoFilters = append(repoFilters, "repo_id = "+strconv.FormatInt(repoID, 10)) } filter := strings.Join(repoFilters, " OR ") + if state == "open" || state == "closed" { + if filter != "" { + filter = "(" + filter + ") AND state = " + state + } else { + filter = "state = " + state + } + } searchRes, err := b.inner.Client.Index(b.inner.VersionedIndexName()).Search(keyword, &meilisearch.SearchRequest{ Filter: filter, Limit: int64(limit), diff --git a/routers/api/v1/repo/issue.go b/routers/api/v1/repo/issue.go index 49252f7a4b4..e76775ae822 100644 --- a/routers/api/v1/repo/issue.go +++ b/routers/api/v1/repo/issue.go @@ -195,7 +195,7 @@ func SearchIssues(ctx *context.APIContext) { } var issueIDs []int64 if len(keyword) > 0 && len(repoIDs) > 0 { - if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil { + if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword, ctx.FormString("state")); err != nil { ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err) return } @@ -394,7 +394,7 @@ func ListIssues(ctx *context.APIContext) { var issueIDs []int64 var labelIDs []int64 if len(keyword) > 0 { - issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword) + issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword, ctx.FormString("state")) if err != nil { ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err) return diff --git a/routers/web/repo/issue.go b/routers/web/repo/issue.go index 65a68386ae7..317f762fb1a 100644 --- a/routers/web/repo/issue.go +++ b/routers/web/repo/issue.go @@ -189,7 +189,7 @@ func issues(ctx *context.Context, milestoneID, projectID int64, isPullOption uti var issueIDs []int64 if len(keyword) > 0 { - issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword) + issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{repo.ID}, keyword, ctx.FormString("state")) if err != nil { if issue_indexer.IsAvailable(ctx) { ctx.ServerError("issueIndexer.Search", err) @@ -2466,7 +2466,7 @@ func SearchIssues(ctx *context.Context) { } var issueIDs []int64 if len(keyword) > 0 && len(repoIDs) > 0 { - if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword); err != nil { + if issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, repoIDs, keyword, ctx.FormString("state")); err != nil { ctx.Error(http.StatusInternalServerError, "SearchIssuesByKeyword", err.Error()) return } @@ -2614,7 +2614,7 @@ func ListIssues(ctx *context.Context) { var issueIDs []int64 var labelIDs []int64 if len(keyword) > 0 { - issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword) + issueIDs, err = issue_indexer.SearchIssuesByKeyword(ctx, []int64{ctx.Repo.Repository.ID}, keyword, ctx.FormString("state")) if err != nil { ctx.Error(http.StatusInternalServerError, err.Error()) return diff --git a/routers/web/user/home.go b/routers/web/user/home.go index 6a89c507a9f..5f1e0eb4277 100644 --- a/routers/web/user/home.go +++ b/routers/web/user/home.go @@ -725,7 +725,7 @@ func issueIDsFromSearch(ctx *context.Context, ctxUser *user_model.User, keyword if err != nil { return nil, fmt.Errorf("GetRepoIDsForIssuesOptions: %w", err) } - issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword) + issueIDsFromSearch, err := issue_indexer.SearchIssuesByKeyword(ctx, searchRepoIDs, keyword, ctx.FormString("state")) if err != nil { return nil, fmt.Errorf("SearchIssuesByKeyword: %w", err) }