Custom regexp external issues (#17624) * Implement custom regular expression for external issue tracking. Signed-off-by: Alexander Beyn <malex@fatelectrons.org> * Fix syntax/style * Update repo.go * Set metas['regexp'] * gofmt * fix some tests * fix more tests * refactor frontend * use LRU cache for regexp * Update modules/markup/html_internal_test.go Co-authored-by: Alexander Beyn <malex@fatelectrons.org> Co-authored-by: techknowlogick <techknowlogick@gitea.io> Co-authored-by: Lauris BH <lauris@nix.lv> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com>

Changes

13 changed files (+206/-26)

• models/repo/repo.go
  • @@ -414,6 +414,9 @@ func (repo *Repository) ComposeMetas() map[string]string {
    switch unit.ExternalTrackerConfig().ExternalTrackerStyle { case markup.IssueNameStyleAlphanumeric: metas["style"] = markup.IssueNameStyleAlphanumeric + case markup.IssueNameStyleRegexp: + metas["style"] = markup.IssueNameStyleRegexp + metas["regexp"] = unit.ExternalTrackerConfig().ExternalTrackerRegexpPattern default: metas["style"] = markup.IssueNameStyleNumeric }
• models/repo/repo_unit.go
  • @@ -76,9 +76,10 @@ func (cfg *ExternalWikiConfig) ToDB() ([]byte, error) {
    // ExternalTrackerConfig describes external tracker config type ExternalTrackerConfig struct { - ExternalTrackerURL string - ExternalTrackerFormat string - ExternalTrackerStyle string + ExternalTrackerURL string + ExternalTrackerFormat string + ExternalTrackerStyle string + ExternalTrackerRegexpPattern string } // FromDB fills up a ExternalTrackerConfig from serialized format.
• models/repo_test.go
  • @@ -74,6 +74,9 @@ func TestMetas(t *testing.T) {
    externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markup.IssueNameStyleNumeric testSuccess(markup.IssueNameStyleNumeric) + externalTracker.ExternalTrackerConfig().ExternalTrackerStyle = markup.IssueNameStyleRegexp + testSuccess(markup.IssueNameStyleRegexp) + repo, err := repo_model.GetRepositoryByID(3) assert.NoError(t, err)
• modules/markup/html.go
  • @@ -20,6 +20,7 @@ import (
    "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup/common" "code.gitea.io/gitea/modules/references" + "code.gitea.io/gitea/modules/regexplru" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/templates/vars" "code.gitea.io/gitea/modules/util"
  • @@ -33,6 +34,7 @@ import (
    const ( IssueNameStyleNumeric = "numeric" IssueNameStyleAlphanumeric = "alphanumeric" + IssueNameStyleRegexp = "regexp" ) var (
  • @@ -815,19 +817,35 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
    ) next := node.NextSibling + for node != nil && node != next { - _, exttrack := ctx.Metas["format"] - alphanum := ctx.Metas["style"] == IssueNameStyleAlphanumeric + _, hasExtTrackFormat := ctx.Metas["format"] // Repos with external issue trackers might still need to reference local PRs // We need to concern with the first one that shows up in the text, whichever it is - found, ref = references.FindRenderizableReferenceNumeric(node.Data, exttrack && alphanum) - if exttrack && alphanum { - if found2, ref2 := references.FindRenderizableReferenceAlphanumeric(node.Data); found2 { - if !found || ref2.RefLocation.Start < ref.RefLocation.Start { - found = true - ref = ref2 - } + isNumericStyle := ctx.Metas["style"] == "" || ctx.Metas["style"] == IssueNameStyleNumeric + foundNumeric, refNumeric := references.FindRenderizableReferenceNumeric(node.Data, hasExtTrackFormat && !isNumericStyle) + + switch ctx.Metas["style"] { + case "", IssueNameStyleNumeric: + found, ref = foundNumeric, refNumeric + case IssueNameStyleAlphanumeric: + found, ref = references.FindRenderizableReferenceAlphanumeric(node.Data) + case IssueNameStyleRegexp: + pattern, err := regexplru.GetCompiled(ctx.Metas["regexp"]) + if err != nil { + return + } + found, ref = references.FindRenderizableReferenceRegexp(node.Data, pattern) + } + + // Repos with external issue trackers might still need to reference local PRs + // We need to concern with the first one that shows up in the text, whichever it is + if hasExtTrackFormat && !isNumericStyle { + // If numeric (PR) was found, and it was BEFORE the non-numeric pattern, use that + if foundNumeric && refNumeric.RefLocation.Start < ref.RefLocation.Start { + found = foundNumeric + ref = refNumeric } } if !found {
  • @@ -836,7 +854,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
    var link *html.Node reftext := node.Data[ref.RefLocation.Start:ref.RefLocation.End] - if exttrack && !ref.IsPull { + if hasExtTrackFormat && !ref.IsPull { ctx.Metas["index"] = ref.Issue res, err := vars.Expand(ctx.Metas["format"], ctx.Metas)
  • @@ -869,7 +887,7 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) {
    // Decorate action keywords if actionable var keyword *html.Node - if references.IsXrefActionable(ref, exttrack, alphanum) { + if references.IsXrefActionable(ref, hasExtTrackFormat) { keyword = createKeyword(node.Data[ref.ActionLocation.Start:ref.ActionLocation.End]) } else { keyword = &html.Node{
• modules/markup/html_internal_test.go
  • @@ -21,8 +21,8 @@ const (
    TestRepoURL = TestAppURL + TestOrgRepo + "/" ) - // alphanumLink an HTML link to an alphanumeric-style issue - func alphanumIssueLink(baseURL, class, name string) string { + // externalIssueLink an HTML link to an alphanumeric-style issue + func externalIssueLink(baseURL, class, name string) string { return link(util.URLJoin(baseURL, name), class, name) }
  • @@ -54,6 +54,13 @@ var alphanumericMetas = map[string]string{
    "style": IssueNameStyleAlphanumeric, } + var regexpMetas = map[string]string{ + "format": "https://someurl.com/{user}/{repo}/{index}", + "user": "someUser", + "repo": "someRepo", + "style": IssueNameStyleRegexp, + } + // these values should match the TestOrgRepo const above var localMetas = map[string]string{ "user": "gogits",
  • @@ -184,7 +191,7 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
    test := func(s, expectedFmt string, names ...string) { links := make([]interface{}, len(names)) for i, name := range names { - links[i] = alphanumIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name) + links[i] = externalIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name) } expected := fmt.Sprintf(expectedFmt, links...) testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas})
  • @@ -194,6 +201,43 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
    test("test issue ABCDEFGHIJ-1234567890", "test issue %s", "ABCDEFGHIJ-1234567890") } + func TestRender_IssueIndexPattern5(t *testing.T) { + setting.AppURL = TestAppURL + + // regexp: render inputs without valid mentions + test := func(s, expectedFmt, pattern string, ids, names []string) { + metas := regexpMetas + metas["regexp"] = pattern + links := make([]interface{}, len(ids)) + for i, id := range ids { + links[i] = link(util.URLJoin("https://someurl.com/someUser/someRepo/", id), "ref-issue ref-external-issue", names[i]) + } + + expected := fmt.Sprintf(expectedFmt, links...) + testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: metas}) + } + + test("abc ISSUE-123 def", "abc %s def", + "ISSUE-(\\d+)", + []string{"123"}, + []string{"ISSUE-123"}, + ) + + test("abc (ISSUE 123) def", "abc %s def", + "\\(ISSUE (\\d+)\\)", + []string{"123"}, + []string{"(ISSUE 123)"}, + ) + + test("abc ISSUE-123 def", "abc %s def", + "(ISSUE-(\\d+))", + []string{"ISSUE-123"}, + []string{"ISSUE-123"}, + ) + + testRenderIssueIndexPattern(t, "will not match", "will not match", &RenderContext{Metas: regexpMetas}) + } + func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) { if ctx.URLPrefix == "" { ctx.URLPrefix = TestAppURL
  • @@ -202,7 +246,7 @@ func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *Rend
    var buf strings.Builder err := postProcess(ctx, []processor{issueIndexPatternProcessor}, strings.NewReader(input), &buf) assert.NoError(t, err) - assert.Equal(t, expected, buf.String()) + assert.Equal(t, expected, buf.String(), "input=%q", input) } func TestRender_AutoLink(t *testing.T) {
• modules/references/references.go
  • @@ -351,6 +351,24 @@ func FindRenderizableReferenceNumeric(content string, prOnly bool) (bool, *Rende
    } } + // FindRenderizableReferenceRegexp returns the first regexp unvalidated references found in a string. + func FindRenderizableReferenceRegexp(content string, pattern *regexp.Regexp) (bool, *RenderizableReference) { + match := pattern.FindStringSubmatchIndex(content) + if len(match) < 4 { + return false, nil + } + + action, location := findActionKeywords([]byte(content), match[2]) + + return true, &RenderizableReference{ + Issue: content[match[2]:match[3]], + RefLocation: &RefSpan{Start: match[0], End: match[1]}, + Action: action, + ActionLocation: location, + IsPull: false, + } + } + // FindRenderizableReferenceAlphanumeric returns the first alphanumeric unvalidated references found in a string. func FindRenderizableReferenceAlphanumeric(content string) (bool, *RenderizableReference) { match := issueAlphanumericPattern.FindStringSubmatchIndex(content)
  • @@ -547,7 +565,7 @@ func findActionKeywords(content []byte, start int) (XRefAction, *RefSpan) {
    } // IsXrefActionable returns true if the xref action is actionable (i.e. produces a result when resolved) - func IsXrefActionable(ref *RenderizableReference, extTracker, alphaNum bool) bool { + func IsXrefActionable(ref *RenderizableReference, extTracker bool) bool { if extTracker { // External issues cannot be automatically closed return false
• modules/regexplru/regexplru.go (new)
  • @@ -0,0 +1,45 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package regexplru + + import ( + "regexp" + + "code.gitea.io/gitea/modules/log" + + lru "github.com/hashicorp/golang-lru" + ) + + var lruCache *lru.Cache + + func init() { + var err error + lruCache, err = lru.New(1000) + if err != nil { + log.Fatal("failed to new LRU cache, err: %v", err) + } + } + + // GetCompiled works like regexp.Compile, the compiled expr or error is stored in LRU cache + func GetCompiled(expr string) (r *regexp.Regexp, err error) { + v, ok := lruCache.Get(expr) + if !ok { + r, err = regexp.Compile(expr) + if err != nil { + lruCache.Add(expr, err) + return nil, err + } + lruCache.Add(expr, r) + } else { + r, ok = v.(*regexp.Regexp) + if !ok { + if err, ok = v.(error); ok { + return nil, err + } + panic("impossible") + } + } + return r, nil + }
• modules/regexplru/regexplru_test.go (new)
  • @@ -0,0 +1,27 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package regexplru + + import ( + "testing" + + "github.com/stretchr/testify/assert" + ) + + func TestRegexpLru(t *testing.T) { + r, err := GetCompiled("a") + assert.NoError(t, err) + assert.True(t, r.MatchString("a")) + + r, err = GetCompiled("a") + assert.NoError(t, err) + assert.True(t, r.MatchString("a")) + + assert.EqualValues(t, 1, lruCache.Len()) + + _, err = GetCompiled("(") + assert.Error(t, err) + assert.EqualValues(t, 2, lruCache.Len()) + }
• options/locale/locale_en-US.ini
  • @@ -1811,6 +1811,9 @@ settings.tracker_url_format_error = The external issue tracker URL format is not
    settings.tracker_issue_style = External Issue Tracker Number Format settings.tracker_issue_style.numeric = Numeric settings.tracker_issue_style.alphanumeric = Alphanumeric + settings.tracker_issue_style.regexp = Regular Expression + settings.tracker_issue_style.regexp_pattern = Regular Expression Pattern + settings.tracker_issue_style.regexp_pattern_desc = The first captured group will be used in place of <code>{index}</code>. settings.tracker_url_format_desc = Use the placeholders <code>{user}</code>, <code>{repo}</code> and <code>{index}</code> for the username, repository name and issue index. settings.enable_timetracker = Enable Time Tracking settings.allow_only_contributors_to_track_time = Let Only Contributors Track Time
• routers/web/repo/setting.go
  • @@ -434,9 +434,10 @@ func SettingsPost(ctx *context.Context) {
    RepoID: repo.ID, Type: unit_model.TypeExternalTracker, Config: &repo_model.ExternalTrackerConfig{ - ExternalTrackerURL: form.ExternalTrackerURL, - ExternalTrackerFormat: form.TrackerURLFormat, - ExternalTrackerStyle: form.TrackerIssueStyle, + ExternalTrackerURL: form.ExternalTrackerURL, + ExternalTrackerFormat: form.TrackerURLFormat, + ExternalTrackerStyle: form.TrackerIssueStyle, + ExternalTrackerRegexpPattern: form.ExternalTrackerRegexpPattern, }, }) deleteUnitTypes = append(deleteUnitTypes, unit_model.TypeIssues)
• services/forms/repo_form.go
  • @@ -141,6 +141,7 @@ type RepoSettingForm struct {
    ExternalTrackerURL string TrackerURLFormat string TrackerIssueStyle string + ExternalTrackerRegexpPattern string EnableCloseIssuesViaCommitInAnyBranch bool EnableProjects bool EnablePackages bool
• templates/repo/settings/options.tmpl
  • @@ -361,16 +361,27 @@
    <div class="ui radio checkbox"> {{$externalTracker := (.Repository.MustGetUnit $.UnitTypeExternalTracker)}} {{$externalTrackerStyle := $externalTracker.ExternalTrackerConfig.ExternalTrackerStyle}} - <input class="hidden" tabindex="0" name="tracker_issue_style" type="radio" value="numeric" {{if $externalTrackerStyle}}{{if eq $externalTrackerStyle "numeric"}}checked=""{{end}}{{end}}/> - <label>{{.i18n.Tr "repo.settings.tracker_issue_style.numeric"}} <span class="ui light grey text">(#1234)</span></label> + <input class="js-tracker-issue-style" name="tracker_issue_style" type="radio" value="numeric" {{if eq $externalTrackerStyle "numeric"}}checked{{end}}> + <label>{{.i18n.Tr "repo.settings.tracker_issue_style.numeric"}} <span class="ui light grey text">#1234</span></label> </div> </div> <div class="field"> <div class="ui radio checkbox"> - <input class="hidden" tabindex="0" name="tracker_issue_style" type="radio" value="alphanumeric" {{if $externalTrackerStyle}}{{if eq $externalTracker.ExternalTrackerConfig.ExternalTrackerStyle "alphanumeric"}}checked=""{{end}}{{end}} /> - <label>{{.i18n.Tr "repo.settings.tracker_issue_style.alphanumeric"}} <span class="ui light grey text">(ABC-123, DEFG-234)</span></label> + <input class="js-tracker-issue-style" name="tracker_issue_style" type="radio" value="alphanumeric" {{if eq $externalTrackerStyle "alphanumeric"}}checked{{end}}> + <label>{{.i18n.Tr "repo.settings.tracker_issue_style.alphanumeric"}} <span class="ui light grey text">ABC-123 , DEFG-234</span></label> </div> </div> + <div class="field"> + <div class="ui radio checkbox"> + <input class="js-tracker-issue-style" name="tracker_issue_style" type="radio" value="regexp" {{if eq $externalTrackerStyle "regexp"}}checked{{end}}> + <label>{{.i18n.Tr "repo.settings.tracker_issue_style.regexp"}} <span class="ui light grey text">(ISSUE-\d+) , ISSUE-(\d+)</span></label> + </div> + </div> + </div> + <div class="field {{if ne $externalTrackerStyle "regexp"}}disabled{{end}}" id="tracker-issue-style-regex-box"> + <label for="external_tracker_regexp_pattern">{{.i18n.Tr "repo.settings.tracker_issue_style.regexp_pattern"}}</label> + <input id="external_tracker_regexp_pattern" name="external_tracker_regexp_pattern" value="{{(.Repository.MustGetUnit $.UnitTypeExternalTracker).ExternalTrackerConfig.ExternalTrackerRegexpPattern}}"> + <p class="help">{{.i18n.Tr "repo.settings.tracker_issue_style.regexp_pattern_desc" | Str2html}}</p> </div> </div> </div>
• web_src/js/features/repo-legacy.js
  • @@ -462,6 +462,11 @@ export function initRepository() {
    if (typeof $(this).data('context') !== 'undefined') $($(this).data('context')).addClass('disabled'); } }); + const $trackerIssueStyleRadios = $('.js-tracker-issue-style'); + $trackerIssueStyleRadios.on('change input', () => { + const checkedVal = $trackerIssueStyleRadios.filter(':checked').val(); + $('#tracker-issue-style-regex-box').toggleClass('disabled', checkedVal !== 'regexp'); + }); } // Labels