aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorzeripath2021-04-01 23:50:12 +0100
committerGitHub2021-04-02 00:50:12 +0200
commit1a26f6c7abef670be624c38f626acd1dee5c2bfa (patch)
treee52ae3a7fd6258696aa197b6e4f6dfd97a4d4cdb
parent1062931cf19462cf5ff8d4f14eeead6c6fc58173 (diff)
Speed up `enry.IsVendor` (#15213) (#15246)
Backport #15213 `enry.IsVendor` is kinda slow as it simply iterates across all regexps. This PR ajdusts the regexps to combine them to make this process a little quicker. Related #15143 Signed-off-by: Andrew Thornton <art27@cantab.net>
-rw-r--r--modules/analyze/vendor.go70
-rw-r--r--modules/analyze/vendor_test.go42
-rw-r--r--modules/git/repo_language_stats.go2
-rw-r--r--modules/indexer/code/bleve.go2
-rw-r--r--modules/indexer/code/elastic_search.go2
5 files changed, 115 insertions, 3 deletions
diff --git a/modules/analyze/vendor.go b/modules/analyze/vendor.go
new file mode 100644
index 000000000..12ae8dbd8
--- /dev/null
+++ b/modules/analyze/vendor.go
@@ -0,0 +1,70 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package analyze
+
+import (
+ "regexp"
+ "sort"
+ "strings"
+
+ "github.com/go-enry/go-enry/v2/data"
+)
+
+var isVendorRegExp *regexp.Regexp
+
+func init() {
+ matchers := data.VendorMatchers
+
+ caretStrings := make([]string, 0, 10)
+ caretShareStrings := make([]string, 0, 10)
+
+ matcherStrings := make([]string, 0, len(matchers))
+ for _, matcher := range matchers {
+ str := matcher.String()
+ if str[0] == '^' {
+ caretStrings = append(caretStrings, str[1:])
+ } else if str[0:5] == "(^|/)" {
+ caretShareStrings = append(caretShareStrings, str[5:])
+ } else {
+ matcherStrings = append(matcherStrings, str)
+ }
+ }
+
+ sort.Strings(caretShareStrings)
+ sort.Strings(caretStrings)
+ sort.Strings(matcherStrings)
+
+ sb := &strings.Builder{}
+ sb.WriteString("(?:^(?:")
+ sb.WriteString(caretStrings[0])
+ for _, matcher := range caretStrings[1:] {
+ sb.WriteString(")|(?:")
+ sb.WriteString(matcher)
+ }
+ sb.WriteString("))")
+ sb.WriteString("|")
+ sb.WriteString("(?:(?:^|/)(?:")
+ sb.WriteString(caretShareStrings[0])
+ for _, matcher := range caretShareStrings[1:] {
+ sb.WriteString(")|(?:")
+ sb.WriteString(matcher)
+ }
+ sb.WriteString("))")
+ sb.WriteString("|")
+ sb.WriteString("(?:")
+ sb.WriteString(matcherStrings[0])
+ for _, matcher := range matcherStrings[1:] {
+ sb.WriteString(")|(?:")
+ sb.WriteString(matcher)
+ }
+ sb.WriteString(")")
+ combined := sb.String()
+ isVendorRegExp = regexp.MustCompile(combined)
+}
+
+// IsVendor returns whether or not path is a vendor path.
+func IsVendor(path string) bool {
+ return isVendorRegExp.MatchString(path)
+}
diff --git a/modules/analyze/vendor_test.go b/modules/analyze/vendor_test.go
new file mode 100644
index 000000000..2784e49d3
--- /dev/null
+++ b/modules/analyze/vendor_test.go
@@ -0,0 +1,42 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package analyze
+
+import "testing"
+
+func TestIsVendor(t *testing.T) {
+ tests := []struct {
+ path string
+ want bool
+ }{
+ {"cache/", true},
+ {"random/cache/", true},
+ {"cache", false},
+ {"dependencies/", true},
+ {"Dependencies/", true},
+ {"dependency/", false},
+ {"dist/", true},
+ {"dist", false},
+ {"random/dist/", true},
+ {"random/dist", false},
+ {"deps/", true},
+ {"configure", true},
+ {"a/configure", true},
+ {"config.guess", true},
+ {"config.guess/", false},
+ {".vscode/", true},
+ {"doc/_build/", true},
+ {"a/docs/_build/", true},
+ {"a/dasdocs/_build-vsdoc.js", true},
+ {"a/dasdocs/_build-vsdoc.j", false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.path, func(t *testing.T) {
+ if got := IsVendor(tt.path); got != tt.want {
+ t.Errorf("IsVendor() = %v, want %v", got, tt.want)
+ }
+ })
+ }
+}
diff --git a/modules/git/repo_language_stats.go b/modules/git/repo_language_stats.go
index b721b996e..573cf5df6 100644
--- a/modules/git/repo_language_stats.go
+++ b/modules/git/repo_language_stats.go
@@ -44,7 +44,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
sizes := make(map[string]int64)
err = tree.Files().ForEach(func(f *object.File) error {
- if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
+ if f.Size == 0 || analyze.IsVendor(f.Name) || enry.IsDotFile(f.Name) ||
enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
return nil
}
diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go
index 9caa6528f..7458717cc 100644
--- a/modules/indexer/code/bleve.go
+++ b/modules/indexer/code/bleve.go
@@ -175,7 +175,7 @@ func NewBleveIndexer(indexDir string) (*BleveIndexer, bool, error) {
func (b *BleveIndexer) addUpdate(commitSha string, update fileUpdate, repo *models.Repository, batch rupture.FlushingBatch) error {
// Ignore vendored files in code search
- if setting.Indexer.ExcludeVendored && enry.IsVendor(update.Filename) {
+ if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
return nil
}
diff --git a/modules/indexer/code/elastic_search.go b/modules/indexer/code/elastic_search.go
index 08b20b80a..fd3746205 100644
--- a/modules/indexer/code/elastic_search.go
+++ b/modules/indexer/code/elastic_search.go
@@ -170,7 +170,7 @@ func (b *ElasticSearchIndexer) init() (bool, error) {
func (b *ElasticSearchIndexer) addUpdate(sha string, update fileUpdate, repo *models.Repository) ([]elastic.BulkableRequest, error) {
// Ignore vendored files in code search
- if setting.Indexer.ExcludeVendored && enry.IsVendor(update.Filename) {
+ if setting.Indexer.ExcludeVendored && analyze.IsVendor(update.Filename) {
return nil, nil
}