[Feature] add precise search type for Elastic Search (#12869)

* feat: add type query parameters for specifying precise search

* feat: add select dropdown in search box

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>

modules/context/pagination.go

@@ -53,4 +53,5 @@ func (p *Pagination) SetDefaultParams(ctx *Context) {
 	p.AddParam(ctx, "sort", "SortType")
 	p.AddParam(ctx, "q", "Keyword")
 	p.AddParam(ctx, "tab", "TabName")
+	p.AddParam(ctx, "t", "queryType")
 }

modules/indexer/code/bleve.go

@@ -280,12 +280,23 @@ func (b *BleveIndexer) Delete(repoID int64) error {
 
 // Search searches for files in the specified repo.
 // Returns the matching file-paths
-func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+	var (
+		indexerQuery query.Query
+		keywordQuery query.Query
+	)
+
+	if isMatch {
+		prefixQuery := bleve.NewPrefixQuery(keyword)
+		prefixQuery.FieldVal = "Content"
+		keywordQuery = prefixQuery
+	} else {
 		phraseQuery := bleve.NewMatchPhraseQuery(keyword)
 		phraseQuery.FieldVal = "Content"
 		phraseQuery.Analyzer = repoIndexerAnalyzer
+		keywordQuery = phraseQuery
+	}
 
-	var indexerQuery query.Query
 	if len(repoIDs) > 0 {
 		var repoQueries = make([]query.Query, 0, len(repoIDs))
 		for _, repoID := range repoIDs {
@@ -294,10 +305,10 @@ func (b *BleveIndexer) Search(repoIDs []int64, language, keyword string, page, p
 
 		indexerQuery = bleve.NewConjunctionQuery(
 			bleve.NewDisjunctionQuery(repoQueries...),
-			phraseQuery,
+			keywordQuery,
 		)
 	} else {
-		indexerQuery = phraseQuery
+		indexerQuery = keywordQuery
 	}
 
 	// Save for reuse without language filter

modules/indexer/code/elastic_search.go

@@ -27,6 +27,10 @@ import (
 
 const (
 	esRepoIndexerLatestVersion = 1
+	// multi-match-types, currently only 2 types are used
+	// Reference: https://www.elastic.co/guide/en/elasticsearch/reference/7.0/query-dsl-multi-match-query.html#multi-match-types
+	esMultiMatchTypeBestFields   = "best_fields"
+	esMultiMatchTypePhrasePrefix = "phrase_prefix"
 )
 
 var (
@@ -330,8 +334,13 @@ func extractAggs(searchResult *elastic.SearchResult) []*SearchResultLanguages {
 }
 
 // Search searches for codes and language stats by given conditions.
-func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (b *ElasticSearchIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
-	kwQuery := elastic.NewMultiMatchQuery(keyword, "content")
+	searchType := esMultiMatchTypeBestFields
+	if isMatch {
+		searchType = esMultiMatchTypePhrasePrefix
+	}
+
+	kwQuery := elastic.NewMultiMatchQuery(keyword, "content").Type(searchType)
 	query := elastic.NewBoolQuery()
 	query = query.Must(kwQuery)
 	if len(repoIDs) > 0 {

modules/indexer/code/indexer.go

@@ -43,7 +43,7 @@ type SearchResultLanguages struct {
 type Indexer interface {
 	Index(repo *models.Repository, sha string, changes *repoChanges) error
 	Delete(repoID int64) error
-	Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error)
+	Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error)
 	Close()
 }
 

modules/indexer/code/indexer_test.go

@@ -64,7 +64,7 @@ func testIndexer(name string, t *testing.T, indexer Indexer) {
 
 		for _, kw := range keywords {
 			t.Run(kw.Keyword, func(t *testing.T) {
-				total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10)
+				total, res, langs, err := indexer.Search(kw.RepoIDs, "", kw.Keyword, 1, 10, false)
 				assert.NoError(t, err)
 				assert.EqualValues(t, len(kw.IDs), total)
 				assert.EqualValues(t, kw.Langs, len(langs))

modules/indexer/code/search.go

@@ -106,12 +106,12 @@ func searchResult(result *SearchResult, startIndex, endIndex int) (*Result, erro
 }
 
 // PerformSearch perform a search on a repository
-func PerformSearch(repoIDs []int64, language, keyword string, page, pageSize int) (int, []*Result, []*SearchResultLanguages, error) {
+func PerformSearch(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int, []*Result, []*SearchResultLanguages, error) {
 	if len(keyword) == 0 {
 		return 0, nil, nil, nil
 	}
 
-	total, results, resultLanguages, err := indexer.Search(repoIDs, language, keyword, page, pageSize)
+	total, results, resultLanguages, err := indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch)
 	if err != nil {
 		return 0, nil, nil, err
 	}

modules/indexer/code/wrapped.go

@@ -73,12 +73,12 @@ func (w *wrappedIndexer) Delete(repoID int64) error {
 	return indexer.Delete(repoID)
 }
 
-func (w *wrappedIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int) (int64, []*SearchResult, []*SearchResultLanguages, error) {
+func (w *wrappedIndexer) Search(repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) {
 	indexer, err := w.get()
 	if err != nil {
 		return 0, nil, nil, err
 	}
-	return indexer.Search(repoIDs, language, keyword, page, pageSize)
+	return indexer.Search(repoIDs, language, keyword, page, pageSize, isMatch)
 
 }
 

options/locale/locale_en-US.ini

@@ -237,6 +237,8 @@ users = Users
 organizations = Organizations
 search = Search
 code = Code
+search.fuzzy = Fuzzy
+search.match = Match
 repo_no_results = No matching repositories found.
 user_no_results = No matching users found.
 org_no_results = No matching organizations found.
@@ -1462,6 +1464,8 @@ activity.git_stats_deletion_n = %d deletions
 
 search = Search
 search.search_repo = Search repository
+search.fuzzy = Fuzzy
+search.match = Match
 search.results = Search results for "%s" in <a href="%s">%s</a>
 
 settings = Settings

routers/home.go

@@ -299,6 +299,9 @@ func ExploreCode(ctx *context.Context) {
 		page = 1
 	}
 
+	queryType := strings.TrimSpace(ctx.Query("t"))
+	isMatch := queryType == "match"
+
 	var (
 		repoIDs []int64
 		err     error
@@ -342,14 +345,14 @@ func ExploreCode(ctx *context.Context) {
 
 		ctx.Data["RepoMaps"] = rightRepoMap
 
-		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum)
+		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
 		if err != nil {
 			ctx.ServerError("SearchResults", err)
 			return
 		}
 		// if non-login user or isAdmin, no need to check UnitTypeCode
 	} else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
-		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum)
+		total, searchResults, searchResultLanguages, err = code_indexer.PerformSearch(repoIDs, language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
 		if err != nil {
 			ctx.ServerError("SearchResults", err)
 			return
@@ -380,6 +383,7 @@ func ExploreCode(ctx *context.Context) {
 
 	ctx.Data["Keyword"] = keyword
 	ctx.Data["Language"] = language
+	ctx.Data["queryType"] = queryType
 	ctx.Data["SearchResults"] = searchResults
 	ctx.Data["SearchResultLanguages"] = searchResultLanguages
 	ctx.Data["RequireHighlightJS"] = true

routers/repo/search.go

@@ -28,14 +28,18 @@ func Search(ctx *context.Context) {
 	if page <= 0 {
 		page = 1
 	}
+	queryType := strings.TrimSpace(ctx.Query("t"))
+	isMatch := queryType == "match"
+
 	total, searchResults, searchResultLanguages, err := code_indexer.PerformSearch([]int64{ctx.Repo.Repository.ID},
-		language, keyword, page, setting.UI.RepoSearchPagingNum)
+		language, keyword, page, setting.UI.RepoSearchPagingNum, isMatch)
 	if err != nil {
 		ctx.ServerError("SearchResults", err)
 		return
 	}
 	ctx.Data["Keyword"] = keyword
 	ctx.Data["Language"] = language
+	ctx.Data["queryType"] = queryType
 	ctx.Data["SourcePath"] = setting.AppSubURL + "/" +
 		path.Join(ctx.Repo.Repository.Owner.Name, ctx.Repo.Repository.Name)
 	ctx.Data["SearchResults"] = searchResults

templates/explore/code.tmpl

@@ -5,9 +5,19 @@
 		<form class="ui form ignore-dirty" style="max-width: 100%">
             <input type="hidden" name="tab" value="{{$.TabName}}">
             <div class="ui fluid action input">
+            <div class="twelve wide field">
                 <input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
+            </div>
+            <div class="two wide field">
+                <select name="t">
+                    <option value="">{{.i18n.Tr "explore.search.fuzzy"}}</option>
+                    <option value="match" {{if eq .queryType "match"}}selected{{end}}>{{.i18n.Tr "explore.search.match"}}</option>
+                </select>
+            </div>
+            <div class="three field">
                 <button class="ui blue button">{{.i18n.Tr "explore.search"}}</button>
             </div>
+            </div>
         </form>
         <div class="ui divider"></div>
 
@@ -18,7 +28,7 @@
                 </h3>
 				<div class="df ac fw">
 					{{range $term := .SearchResultLanguages}}
-					<a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{AppSubUrl}}/explore/code?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}">
+					<a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{AppSubUrl}}/explore/code?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if ne $.queryType ""}}&t={{$.queryType}}{{end}}">
 						<i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i>
 						{{$term.Language}}
 						<div class="detail">{{$term.Count}}</div>
@@ -62,4 +72,3 @@
 	</div>
 </div>
 {{template "base/footer" .}}
-

templates/repo/search.tmpl

@@ -5,11 +5,21 @@
 		<div class="ui repo-search">
 			<form class="ui form ignore-dirty" method="get">
 				<div class="ui fluid action input">
+					<div class="twelve wide field">
 						<input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "repo.search.search_repo"}}">
+					</div>
+					<div class="two wide field">
+						<select name="t">
+							<option value="">{{.i18n.Tr "repo.search.fuzzy"}}</option>
+							<option value="match" {{if eq .queryType "match"}}selected{{end}}>{{.i18n.Tr "repo.search.match"}}</option>
+						</select>
+					</div>
+					<div class="three field">
 					  <button class="ui button" type="submit">
 						  <i class="icon df ac jc">{{svg "octicon-search" 16}}</i>
 					  </button>
 					</div>
+				</div>
 			</form>
 		</div>
 		{{if .Keyword}}
@@ -18,7 +28,7 @@
 			</h3>
 			<div class="df ac fw">
 				{{range $term := .SearchResultLanguages}}
-				<a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{EscapePound $.SourcePath}}/search?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}">
+				<a class="ui text-label df ac mr-1 my-1 {{if eq $.Language $term.Language}}primary {{end}}basic label" href="{{EscapePound $.SourcePath}}/search?q={{$.Keyword}}{{if ne $.Language $term.Language}}&l={{$term.Language}}{{end}}{{if ne $.queryType ""}}&t={{$.queryType}}{{end}}">
 					<i class="color-icon mr-3" style="background-color: {{$term.Color}}"></i>
 					{{$term.Language}}
 					<div class="detail">{{$term.Count}}</div>