fix(indexer): index full file paths and real offsets in bleve (#39405)

The bleve path token filter added in
https://github.com/go-gitea/gitea/pull/32210 never generated the full
path of a file, so searching a file by its path (e.g. `potato/ham`)
found nothing. It also gave the path tokens made-up offsets instead of
their position in the path.

The filter is replaced by a tokenizer that emits the raw path suffixes
starting at each segment and word (`potato/ham.md`, `ham.md`). Paths
below the root now match too, as do names containing `-` or spaces and
dotfiles. An exact file name also ranks above files inside a directory
with the same name.

---------

Co-authored-by: silverwind <me@silverwind.io>
Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

modules/indexer/code/bleve/bleve.go

@@ -17,7 +17,6 @@ import (
 	"gitea.dev/modules/git"
 	"gitea.dev/modules/git/gitcmd"
 	"gitea.dev/modules/indexer"
-	path_filter "gitea.dev/modules/indexer/code/bleve/token/path"
 	"gitea.dev/modules/indexer/code/internal"
 	indexer_internal "gitea.dev/modules/indexer/internal"
 	inner_bleve "gitea.dev/modules/indexer/internal/bleve"
@@ -32,8 +31,8 @@ import (
 	analyzer_keyword "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
 	"github.com/blevesearch/bleve/v2/analysis/token/lowercase"
 	"github.com/blevesearch/bleve/v2/analysis/token/unicodenorm"
-	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
 	"github.com/blevesearch/bleve/v2/mapping"
+	"github.com/blevesearch/bleve/v2/registry"
 	"github.com/blevesearch/bleve/v2/search/query"
 	"github.com/go-enry/go-enry/v2"
 )
@@ -69,7 +68,7 @@ const (
 	repoIndexerAnalyzer      = "repoIndexerAnalyzer"
 	filenameIndexerAnalyzer  = "filenameIndexerAnalyzer"
 	repoIndexerDocType       = "repoIndexerDocType"
-	repoIndexerLatestVersion = 10
+	repoIndexerLatestVersion = 11
 )
 
 // generateBleveIndexMapping generates a bleve index mapping for the repo indexer
@@ -114,8 +113,8 @@ func generateBleveIndexMapping() (mapping.IndexMapping, error) {
 	if err := mapping.AddCustomAnalyzer(filenameIndexerAnalyzer, map[string]any{
 		"type":          analyzer_custom.Name,
 		"char_filters":  []string{},
-		"tokenizer":     unicode.Name,
-		"token_filters": []string{unicodeNormalizeName, path_filter.Name, lowercase.Name},
+		"tokenizer":     pathTokenizerName,
+		"token_filters": []string{unicodeNormalizeName, lowercase.Name},
 	}); err != nil {
 		return nil, err
 	}
@@ -139,6 +138,13 @@ func (b *Indexer) SupportedSearchModes() []indexer.SearchMode {
 	return indexer.SearchModesExactWords()
 }
 
+func init() {
+	// due to bleve's design problem, the "Register" must be done in the main goroutine, otherwise data-race
+	util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor))
+	util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor))
+	util.MustNoError(registry.RegisterTokenizer(pathTokenizerName, pathTokenizerConstructor))
+}
+
 // NewIndexer creates a new bleve local indexer
 func NewIndexer(indexDir string) *Indexer {
 	inner := inner_bleve.NewIndexer(indexDir, repoIndexerLatestVersion, generateBleveIndexMapping)

modules/indexer/code/bleve/bleve_test.go

@@ -24,7 +24,7 @@ func TestBleveIndexerTokenFilter(t *testing.T) {
 	require.NoError(t, err)
 
 	batch := inner_bleve.NewFlushingBatch(indexer.inner.Indexer, maxBatchSize)
-	batch.Index("2", &RepoIndexerData{RepoID: 2, Content: "mDNS.port2=12345", UpdatedAt: time.Now()})
+	batch.Index("2", &RepoIndexerData{RepoID: 2, Filename: ".sub-dir/.sub-filename", Content: "mDNS.port2=12345", UpdatedAt: time.Now()})
 	batch.Flush()
 
 	testCases := []struct {
@@ -36,6 +36,10 @@ func TestBleveIndexerTokenFilter(t *testing.T) {
 		{keyword: "mdns", expectedIDs: []int64{2}},
 		{keyword: "port", expectedIDs: []int64{2}},
 		{keyword: "port2", expectedIDs: []int64{2}},
+		{keyword: ".sub-dir", expectedIDs: []int64{2}},
+		{keyword: "sub-file", expectedIDs: []int64{2}}, // prefix matching? not sure whether this behavior is wanted
+		{keyword: "filename", expectedIDs: []int64{2}},
+		{keyword: "name", expectedIDs: []int64{}},
 	}
 
 	for _, testCase := range testCases {

modules/indexer/code/bleve/token/path/path.go

@@ -1,102 +0,0 @@
-// Copyright 2024 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package path
-
-import (
-	"slices"
-	"strings"
-
-	"gitea.dev/modules/util"
-
-	"github.com/blevesearch/bleve/v2/analysis"
-	"github.com/blevesearch/bleve/v2/registry"
-)
-
-const Name = "gitea/path"
-
-type TokenFilter struct{}
-
-func NewTokenFilter() *TokenFilter {
-	return &TokenFilter{}
-}
-
-func TokenFilterConstructor(config map[string]any, cache *registry.Cache) (analysis.TokenFilter, error) {
-	return NewTokenFilter(), nil
-}
-
-func (s *TokenFilter) Filter(input analysis.TokenStream) analysis.TokenStream {
-	if len(input) == 1 {
-		// if there is only one token, we don't need to generate the reversed chain
-		return generatePathTokens(input, false)
-	}
-
-	normal := generatePathTokens(input, false)
-	reversed := generatePathTokens(input, true)
-
-	return append(normal, reversed...)
-}
-
-// Generates path tokens from the input tokens.
-// This mimics the behavior of the path hierarchy tokenizer in ES. It takes the input tokens and combine them, generating a term for each component
-// in tree (e.g., foo/bar/baz.md will generate foo, foo/bar, and foo/bar/baz.md).
-//
-// If the reverse flag is set, the order of the tokens is reversed (the same input will generate baz.md, baz.md/bar, baz.md/bar/foo). This is useful
-// to efficiently search for filenames without supplying the fullpath.
-func generatePathTokens(input analysis.TokenStream, reversed bool) analysis.TokenStream {
-	terms := make([]string, 0, len(input))
-	longestTerm := 0
-
-	if reversed {
-		slices.Reverse(input)
-	}
-
-	for i := range input {
-		var sb strings.Builder
-		sb.Write(input[0].Term)
-
-		for j := 1; j < i; j++ {
-			sb.WriteString("/")
-			sb.Write(input[j].Term)
-		}
-
-		term := sb.String()
-
-		if longestTerm < len(term) {
-			longestTerm = len(term)
-		}
-
-		terms = append(terms, term)
-	}
-
-	output := make(analysis.TokenStream, 0, len(terms))
-
-	for _, term := range terms {
-		var start, end int
-
-		if reversed {
-			start = 0
-			end = len(term)
-		} else {
-			start = longestTerm - len(term)
-			end = longestTerm
-		}
-
-		token := analysis.Token{
-			Position: 1,
-			Start:    start,
-			End:      end,
-			Type:     analysis.AlphaNumeric,
-			Term:     []byte(term),
-		}
-
-		output = append(output, &token)
-	}
-
-	return output
-}
-
-func init() {
-	// FIXME: move it to the bleve's init function, but do not call it in global init
-	util.MustNoError(registry.RegisterTokenFilter(Name, TokenFilterConstructor))
-}

modules/indexer/code/bleve/token/path/path_test.go

@@ -1,76 +0,0 @@
-// Copyright 2024 The Gitea Authors. All rights reserved.
-// SPDX-License-Identifier: MIT
-
-package path
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/blevesearch/bleve/v2/analysis"
-	"github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
-	"github.com/stretchr/testify/assert"
-)
-
-type Scenario struct {
-	Input  string
-	Tokens []string
-}
-
-func TestTokenFilter(t *testing.T) {
-	scenarios := []struct {
-		Input string
-		Terms []string
-	}{
-		{
-			Input: "Dockerfile",
-			Terms: []string{"Dockerfile"},
-		},
-		{
-			Input: "Dockerfile.rootless",
-			Terms: []string{"Dockerfile.rootless"},
-		},
-		{
-			Input: "a/b/c/Dockerfile.rootless",
-			Terms: []string{"a", "a/b", "a/b/c", "a/b/c/Dockerfile.rootless", "Dockerfile.rootless", "Dockerfile.rootless/c", "Dockerfile.rootless/c/b", "Dockerfile.rootless/c/b/a"},
-		},
-		{
-			Input: "",
-			Terms: []string{},
-		},
-	}
-
-	for _, scenario := range scenarios {
-		t.Run(fmt.Sprintf("ensure terms of '%s'", scenario.Input), func(t *testing.T) {
-			terms := extractTerms(scenario.Input)
-
-			assert.Len(t, terms, len(scenario.Terms))
-
-			for _, term := range terms {
-				assert.Contains(t, scenario.Terms, term)
-			}
-		})
-	}
-}
-
-func extractTerms(input string) []string {
-	tokens := tokenize(input)
-	filteredTokens := filter(tokens)
-	terms := make([]string, 0, len(filteredTokens))
-
-	for _, token := range filteredTokens {
-		terms = append(terms, string(token.Term))
-	}
-
-	return terms
-}
-
-func filter(input analysis.TokenStream) analysis.TokenStream {
-	filter := NewTokenFilter()
-	return filter.Filter(input)
-}
-
-func tokenize(input string) analysis.TokenStream {
-	tokenizer := unicode.NewUnicodeTokenizer()
-	return tokenizer.Tokenize([]byte(input))
-}

modules/indexer/code/bleve/token_code.go

@@ -7,8 +7,6 @@ import (
 	"regexp"
 	"unicode"
 
-	"gitea.dev/modules/util"
-
 	"github.com/blevesearch/bleve/v2/analysis"
 	"github.com/blevesearch/bleve/v2/analysis/tokenizer/character"
 	"github.com/blevesearch/bleve/v2/registry"
@@ -58,8 +56,3 @@ func codeTokenFilterConstructor(_ map[string]any, _ *registry.Cache) (analysis.T
 		re: regexp.MustCompile("[a-zA-Z]+|[0-9]+"),
 	}, nil
 }
-
-func init() {
-	util.MustNoError(registry.RegisterTokenizer(codeTokenizerName, codeTokenizerConstructor))
-	util.MustNoError(registry.RegisterTokenFilter(codeTokenFilterName, codeTokenFilterConstructor))
-}

modules/indexer/code/bleve/token_path.go

@@ -0,0 +1,38 @@
+// Copyright 2026 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package bleve
+
+import (
+	"github.com/blevesearch/bleve/v2/analysis"
+	unicode_tokenizer "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode"
+	"github.com/blevesearch/bleve/v2/registry"
+)
+
+const pathTokenizerName = "pathTokenizer"
+
+// pathTokenizer emits the path suffixes starting at each segment and word, for the prefix query in Search
+type pathTokenizer struct{}
+
+func (pathTokenizer) Tokenize(input []byte) (ret analysis.TokenStream) {
+	words := unicode_tokenizer.NewUnicodeTokenizer().Tokenize(input)
+	for start := range input {
+		isWordStart := len(words) > 0 && words[0].Start == start
+		if isWordStart {
+			words = words[1:]
+		}
+		if start == 0 || input[start-1] == '/' || isWordStart {
+			ret = append(ret, &analysis.Token{
+				Start:    start,
+				End:      len(input),
+				Term:     input[start:],
+				Position: len(ret) + 1,
+			})
+		}
+	}
+	return ret
+}
+
+func pathTokenizerConstructor(_ map[string]any, _ *registry.Cache) (analysis.Tokenizer, error) {
+	return pathTokenizer{}, nil
+}

modules/indexer/code/indexer_test.go

@@ -178,6 +178,18 @@ func testIndexer(name string, t *testing.T, indexer internal.Indexer) {
 					},
 				},
 			},
+			// Search for a match on the full path of a file within the repo '62'.
+			{
+				RepoIDs: []int64{62},
+				Keyword: "potato/ham",
+				Langs:   1,
+				Results: []codeSearchResult{
+					{
+						Filename: "potato/ham.md",
+						Content:  "This is not cheese",
+					},
+				},
+			},
 			// Search for matches on the contents of files within the repo '62'.
 			// This scenario yields two results (both are based on contents, the first one is an exact match where as the second is a 'fuzzy' one)
 			{