Improve sync performance for pull-mirrors (#19125) This addresses https://github.com/go-gitea/gitea/issues/18352 It aims to improve performance (and resource use) of the `SyncReleasesWithTags` operation for pull-mirrors. For large repositories with many tags, `SyncReleasesWithTags` can be a costly operation (taking several minutes to complete). The reason is two-fold: 1. on sync, every upstream repo tag is compared (for changes) against existing local entries in the release table to ensure that they are up-to-date. 2. the procedure for getting _each tag_ involves a series of git operations ```bash git show-ref --tags -- v8.2.4477 git cat-file -t 29ab6ce9f36660cffaad3c8789e71162e5db5d2f git cat-file -p 29ab6ce9f36660cffaad3c8789e71162e5db5d2f git rev-list --count 29ab6ce9f36660cffaad3c8789e71162e5db5d2f ``` of which the `git rev-list --count` can be particularly heavy. This PR optimizes performance for pull-mirrors. We utilize the fact that a pull-mirror is always identical to its upstream and rebuild the entire release table on every sync and use a batch `git for-each-ref .. refs/tags` call to retrieve all tags in one go. For large mirror repos, with hundreds of annotated tags, this brings down the duration of the sync operation from several minutes to a few seconds. A few unscientific examples run on my local machine: - https://github.com/spring-projects/spring-boot (223 tags) - before: `0m28,673s` - after: `0m2,244s` - https://github.com/kubernetes/kubernetes (890 tags) - before: `8m00s` - after: `0m8,520s` - https://github.com/vim/vim (13954 tags) - before: `14m20,383s` - after: `0m35,467s` I added a `foreachref` package which contains a flexible way of specifying which reference fields are of interest (`git-for-each-ref(1)`) and to produce a parser for the expected output. These could be reused in other places where `for-each-ref` is used. I'll add unit tests for those if the overall PR looks promising.

Changes

7 changed files (+834/-19)

• modules/git/foreachref/format.go (new)
  • @@ -0,0 +1,84 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package foreachref + + import ( + "encoding/hex" + "fmt" + "io" + "strings" + ) + + var ( + nullChar = []byte("\x00") + dualNullChar = []byte("\x00\x00") + ) + + // Format supports specifying and parsing an output format for 'git + // for-each-ref'. See See git-for-each-ref(1) for available fields. + type Format struct { + // fieldNames hold %(fieldname)s to be passed to the '--format' flag of + // for-each-ref. See git-for-each-ref(1) for available fields. + fieldNames []string + + // fieldDelim is the character sequence that is used to separate fields + // for each reference. fieldDelim and refDelim should be selected to not + // interfere with each other and to not be present in field values. + fieldDelim []byte + // fieldDelimStr is a string representation of fieldDelim. Used to save + // us from repetitive reallocation whenever we need the delimiter as a + // string. + fieldDelimStr string + // refDelim is the character sequence used to separate reference from + // each other in the output. fieldDelim and refDelim should be selected + // to not interfere with each other and to not be present in field + // values. + refDelim []byte + } + + // NewFormat creates a forEachRefFormat using the specified fieldNames. See + // git-for-each-ref(1) for available fields. + func NewFormat(fieldNames ...string) Format { + return Format{ + fieldNames: fieldNames, + fieldDelim: nullChar, + fieldDelimStr: string(nullChar), + refDelim: dualNullChar, + } + } + + // Flag returns a for-each-ref --format flag value that captures the fieldNames. + func (f Format) Flag() string { + var formatFlag strings.Builder + for i, field := range f.fieldNames { + // field key and field value + formatFlag.WriteString(fmt.Sprintf("%s %%(%s)", field, field)) + + if i < len(f.fieldNames)-1 { + // note: escape delimiters to allow control characters as + // delimiters. For example, '%00' for null character or '%0a' + // for newline. + formatFlag.WriteString(f.hexEscaped(f.fieldDelim)) + } + } + formatFlag.WriteString(f.hexEscaped(f.refDelim)) + return formatFlag.String() + } + + // Parser returns a Parser capable of parsing 'git for-each-ref' output produced + // with this Format. + func (f Format) Parser(r io.Reader) *Parser { + return NewParser(r, f) + } + + // hexEscaped produces hex-escpaed characters from a string. For example, "\n\0" + // would turn into "%0a%00". + func (f Format) hexEscaped(delim []byte) string { + escaped := "" + for i := 0; i < len(delim); i++ { + escaped += "%" + hex.EncodeToString([]byte{delim[i]}) + } + return escaped + }
• modules/git/foreachref/format_test.go (new)
  • @@ -0,0 +1,67 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package foreachref_test + + import ( + "testing" + + "code.gitea.io/gitea/modules/git/foreachref" + + "github.com/stretchr/testify/require" + ) + + func TestFormat_Flag(t *testing.T) { + tests := []struct { + name string + + givenFormat foreachref.Format + + wantFlag string + }{ + { + name: "references are delimited by dual null chars", + + // no reference fields requested + givenFormat: foreachref.NewFormat(), + + // only a reference delimiter field in --format + wantFlag: "%00%00", + }, + + { + name: "a field is a space-separated key-value pair", + + givenFormat: foreachref.NewFormat("refname:short"), + + // only a reference delimiter field + wantFlag: "refname:short %(refname:short)%00%00", + }, + + { + name: "fields are separated by a null char field-delimiter", + + givenFormat: foreachref.NewFormat("refname:short", "author"), + + wantFlag: "refname:short %(refname:short)%00author %(author)%00%00", + }, + + { + name: "multiple fields", + + givenFormat: foreachref.NewFormat("refname:short", "objecttype", "objectname"), + + wantFlag: "refname:short %(refname:short)%00objecttype %(objecttype)%00objectname %(objectname)%00%00", + }, + } + + for _, test := range tests { + tc := test // don't close over loop variable + t.Run(tc.name, func(t *testing.T) { + gotFlag := tc.givenFormat.Flag() + + require.Equal(t, tc.wantFlag, gotFlag, "unexpected for-each-ref --format string. wanted: '%s', got: '%s'", tc.wantFlag, gotFlag) + }) + } + }
• modules/git/foreachref/parser.go (new)
  • @@ -0,0 +1,131 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package foreachref + + import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + ) + + // Parser parses 'git for-each-ref' output according to a given output Format. + type Parser struct { + // tokenizes 'git for-each-ref' output into "reference paragraphs". + scanner *bufio.Scanner + + // format represents the '--format' string that describes the expected + // 'git for-each-ref' output structure. + format Format + + // err holds the last encountered error during parsing. + err error + } + + // NewParser creates a 'git for-each-ref' output parser that will parse all + // references in the provided Reader. The references in the output are assumed + // to follow the specified Format. + func NewParser(r io.Reader, format Format) *Parser { + scanner := bufio.NewScanner(r) + + // in addition to the reference delimiter we specified in the --format, + // `git for-each-ref` will always add a newline after every reference. + refDelim := make([]byte, 0, len(format.refDelim)+1) + refDelim = append(refDelim, format.refDelim...) + refDelim = append(refDelim, '\n') + + // Split input into delimiter-separated "reference blocks". + scanner.Split( + func(data []byte, atEOF bool) (advance int, token []byte, err error) { + // Scan until delimiter, marking end of reference. + delimIdx := bytes.Index(data, refDelim) + if delimIdx >= 0 { + token := data[:delimIdx] + advance := delimIdx + len(refDelim) + return advance, token, nil + } + // If we're at EOF, we have a final, non-terminated reference. Return it. + if atEOF { + return len(data), data, nil + } + // Not yet a full field. Request more data. + return 0, nil, nil + }) + + return &Parser{ + scanner: scanner, + format: format, + err: nil, + } + } + + // Next returns the next reference as a collection of key-value pairs. nil + // denotes EOF but is also returned on errors. The Err method should always be + // consulted after Next returning nil. + // + // It could, for example return something like: + // + // { "objecttype": "tag", "refname:short": "v1.16.4", "object": "f460b7543ed500e49c133c2cd85c8c55ee9dbe27" } + // + func (p *Parser) Next() map[string]string { + if !p.scanner.Scan() { + return nil + } + fields, err := p.parseRef(p.scanner.Text()) + if err != nil { + p.err = err + return nil + } + return fields + } + + // Err returns the latest encountered parsing error. + func (p *Parser) Err() error { + return p.err + } + + // parseRef parses out all key-value pairs from a single reference block, such as + // + // "objecttype tag\0refname:short v1.16.4\0object f460b7543ed500e49c133c2cd85c8c55ee9dbe27" + // + func (p *Parser) parseRef(refBlock string) (map[string]string, error) { + if refBlock == "" { + // must be at EOF + return nil, nil + } + + fieldValues := make(map[string]string) + + fields := strings.Split(refBlock, p.format.fieldDelimStr) + if len(fields) != len(p.format.fieldNames) { + return nil, fmt.Errorf("unexpected number of reference fields: wanted %d, was %d", + len(fields), len(p.format.fieldNames)) + } + for i, field := range fields { + field = strings.TrimSpace(field) + + var fieldKey string + var fieldVal string + firstSpace := strings.Index(field, " ") + if firstSpace > 0 { + fieldKey = field[:firstSpace] + fieldVal = field[firstSpace+1:] + } else { + // could be the case if the requested field had no value + fieldKey = field + } + + // enforce the format order of fields + if p.format.fieldNames[i] != fieldKey { + return nil, fmt.Errorf("unexpected field name at position %d: wanted: '%s', was: '%s'", + i, p.format.fieldNames[i], fieldKey) + } + + fieldValues[fieldKey] = fieldVal + } + + return fieldValues, nil + }
• modules/git/foreachref/parser_test.go (new)
  • @@ -0,0 +1,228 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package foreachref_test + + import ( + "errors" + "fmt" + "io" + "strings" + "testing" + + "code.gitea.io/gitea/modules/git/foreachref" + "code.gitea.io/gitea/modules/json" + + "github.com/stretchr/testify/require" + ) + + type refSlice = []map[string]string + + func TestParser(t *testing.T) { + tests := []struct { + name string + + givenFormat foreachref.Format + givenInput io.Reader + + wantRefs refSlice + wantErr bool + expectedErr error + }{ + // this would, for example, be the result when running `git + // for-each-ref refs/tags` on a repo without tags. + { + name: "no references on empty input", + + givenFormat: foreachref.NewFormat("refname:short"), + givenInput: strings.NewReader(``), + + wantRefs: []map[string]string{}, + }, + + // note: `git for-each-ref` will add a newline between every + // reference (in addition to the ref-delimiter we've chosen) + { + name: "single field requested, single reference in output", + + givenFormat: foreachref.NewFormat("refname:short"), + givenInput: strings.NewReader("refname:short v0.0.1\x00\x00" + "\n"), + + wantRefs: []map[string]string{ + {"refname:short": "v0.0.1"}, + }, + }, + { + name: "single field requested, multiple references in output", + + givenFormat: foreachref.NewFormat("refname:short"), + givenInput: strings.NewReader( + "refname:short v0.0.1\x00\x00" + "\n" + + "refname:short v0.0.2\x00\x00" + "\n" + + "refname:short v0.0.3\x00\x00" + "\n"), + + wantRefs: []map[string]string{ + {"refname:short": "v0.0.1"}, + {"refname:short": "v0.0.2"}, + {"refname:short": "v0.0.3"}, + }, + }, + + { + name: "multiple fields requested for each reference", + + givenFormat: foreachref.NewFormat("refname:short", "objecttype", "objectname"), + givenInput: strings.NewReader( + + "refname:short v0.0.1\x00objecttype commit\x00objectname 7b2c5ac9fc04fc5efafb60700713d4fa609b777b\x00\x00" + "\n" + + "refname:short v0.0.2\x00objecttype commit\x00objectname a1f051bc3eba734da4772d60e2d677f47cf93ef4\x00\x00" + "\n" + + "refname:short v0.0.3\x00objecttype commit\x00objectname ef82de70bb3f60c65fb8eebacbb2d122ef517385\x00\x00" + "\n", + ), + + wantRefs: []map[string]string{ + { + "refname:short": "v0.0.1", + "objecttype": "commit", + "objectname": "7b2c5ac9fc04fc5efafb60700713d4fa609b777b", + }, + { + "refname:short": "v0.0.2", + "objecttype": "commit", + "objectname": "a1f051bc3eba734da4772d60e2d677f47cf93ef4", + }, + { + "refname:short": "v0.0.3", + "objecttype": "commit", + "objectname": "ef82de70bb3f60c65fb8eebacbb2d122ef517385", + }, + }, + }, + + { + name: "must handle multi-line fields such as 'content'", + + givenFormat: foreachref.NewFormat("refname:short", "contents", "author"), + givenInput: strings.NewReader( + "refname:short v0.0.1\x00contents Create new buffer if not present yet (#549)\n\nFixes a nil dereference when ProcessFoo is used\nwith multiple commands.\x00author Foo Bar <foo@bar.com> 1507832733 +0200\x00\x00" + "\n" + + "refname:short v0.0.2\x00contents Update CI config (#651)\n\n\x00author John Doe <john.doe@foo.com> 1521643174 +0000\x00\x00" + "\n" + + "refname:short v0.0.3\x00contents Fixed code sample for bash completion (#687)\n\n\x00author Foo Baz <foo@baz.com> 1524836750 +0200\x00\x00" + "\n", + ), + + wantRefs: []map[string]string{ + { + "refname:short": "v0.0.1", + "contents": "Create new buffer if not present yet (#549)\n\nFixes a nil dereference when ProcessFoo is used\nwith multiple commands.", + "author": "Foo Bar <foo@bar.com> 1507832733 +0200", + }, + { + "refname:short": "v0.0.2", + "contents": "Update CI config (#651)", + "author": "John Doe <john.doe@foo.com> 1521643174 +0000", + }, + { + "refname:short": "v0.0.3", + "contents": "Fixed code sample for bash completion (#687)", + "author": "Foo Baz <foo@baz.com> 1524836750 +0200", + }, + }, + }, + + { + name: "must handle fields without values", + + givenFormat: foreachref.NewFormat("refname:short", "object", "objecttype"), + givenInput: strings.NewReader( + "refname:short v0.0.1\x00object \x00objecttype commit\x00\x00" + "\n" + + "refname:short v0.0.2\x00object \x00objecttype commit\x00\x00" + "\n" + + "refname:short v0.0.3\x00object \x00objecttype commit\x00\x00" + "\n", + ), + + wantRefs: []map[string]string{ + { + "refname:short": "v0.0.1", + "object": "", + "objecttype": "commit", + }, + { + "refname:short": "v0.0.2", + "object": "", + "objecttype": "commit", + }, + { + "refname:short": "v0.0.3", + "object": "", + "objecttype": "commit", + }, + }, + }, + + { + name: "must fail when the number of fields in the input doesn't match expected format", + + givenFormat: foreachref.NewFormat("refname:short", "objecttype", "objectname"), + givenInput: strings.NewReader( + "refname:short v0.0.1\x00objecttype commit\x00\x00" + "\n" + + "refname:short v0.0.2\x00objecttype commit\x00\x00" + "\n" + + "refname:short v0.0.3\x00objecttype commit\x00\x00" + "\n", + ), + + wantErr: true, + expectedErr: errors.New("unexpected number of reference fields: wanted 2, was 3"), + }, + + { + name: "must fail input fields don't match expected format", + + givenFormat: foreachref.NewFormat("refname:short", "objectname"), + givenInput: strings.NewReader( + "refname:short v0.0.1\x00objecttype commit\x00\x00" + "\n" + + "refname:short v0.0.2\x00objecttype commit\x00\x00" + "\n" + + "refname:short v0.0.3\x00objecttype commit\x00\x00" + "\n", + ), + + wantErr: true, + expectedErr: errors.New("unexpected field name at position 1: wanted: 'objectname', was: 'objecttype'"), + }, + } + + for _, test := range tests { + tc := test // don't close over loop variable + t.Run(tc.name, func(t *testing.T) { + parser := tc.givenFormat.Parser(tc.givenInput) + + // + // parse references from input + // + gotRefs := make([]map[string]string, 0) + for { + ref := parser.Next() + if ref == nil { + break + } + gotRefs = append(gotRefs, ref) + } + err := parser.Err() + + // + // verify expectations + // + if tc.wantErr { + require.Error(t, err) + require.EqualError(t, err, tc.expectedErr.Error()) + } else { + require.NoError(t, err, "for-each-ref parser unexpectedly failed with: %v", err) + require.Equal(t, tc.wantRefs, gotRefs, "for-each-ref parser produced unexpected reference set. wanted: %v, got: %v", pretty(tc.wantRefs), pretty(gotRefs)) + } + }) + } + } + + func pretty(v interface{}) string { + data, err := json.MarshalIndent(v, "", " ") + if err != nil { + // shouldn't happen + panic(fmt.Sprintf("json-marshalling failed: %v", err)) + } + return string(data) + }
• modules/git/repo_tag.go
  • @@ -8,8 +8,10 @@ package git
    import ( "context" "fmt" + "io" "strings" + "code.gitea.io/gitea/modules/git/foreachref" "code.gitea.io/gitea/modules/util" )
  • @@ -111,37 +113,98 @@ func (repo *Repository) GetTagWithID(idStr, name string) (*Tag, error) {
    // GetTagInfos returns all tag infos of the repository. func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) { - // TODO this a slow implementation, makes one git command per tag - stdout, err := NewCommand(repo.Ctx, "tag").RunInDir(repo.Path) - if err != nil { - return nil, 0, err - } + forEachRefFmt := foreachref.NewFormat("objecttype", "refname:short", "object", "objectname", "creator", "contents", "contents:signature") - tagNames := strings.Split(strings.TrimRight(stdout, "\n"), "\n") - tagsTotal := len(tagNames) - - if page != 0 { - tagNames = util.PaginateSlice(tagNames, page, pageSize).([]string) - } + stdoutReader, stdoutWriter := io.Pipe() + defer stdoutReader.Close() + defer stdoutWriter.Close() + stderr := strings.Builder{} + rc := &RunContext{Dir: repo.Path, Stdout: stdoutWriter, Stderr: &stderr, Timeout: -1} - tags := make([]*Tag, 0, len(tagNames)) - for _, tagName := range tagNames { - tagName = strings.TrimSpace(tagName) - if len(tagName) == 0 { - continue + go func() { + err := NewCommand(repo.Ctx, "for-each-ref", "--format", forEachRefFmt.Flag(), "--sort", "-*creatordate", "refs/tags").RunWithContext(rc) + if err != nil { + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String())) + } else { + _ = stdoutWriter.Close() + } + }() + + var tags []*Tag + parser := forEachRefFmt.Parser(stdoutReader) + for { + ref := parser.Next() + if ref == nil { + break } - tag, err := repo.GetTag(tagName) + tag, err := parseTagRef(ref) if err != nil { - return nil, tagsTotal, err + return nil, 0, fmt.Errorf("GetTagInfos: parse tag: %w", err) } - tag.Name = tagName tags = append(tags, tag) } + if err := parser.Err(); err != nil { + return nil, 0, fmt.Errorf("GetTagInfos: parse output: %w", err) + } + sortTagsByTime(tags) + tagsTotal := len(tags) + if page != 0 { + tags = util.PaginateSlice(tags, page, pageSize).([]*Tag) + } + return tags, tagsTotal, nil } + // parseTagRef parses a tag from a 'git for-each-ref'-produced reference. + func parseTagRef(ref map[string]string) (tag *Tag, err error) { + tag = &Tag{ + Type: ref["objecttype"], + Name: ref["refname:short"], + } + + tag.ID, err = NewIDFromString(ref["objectname"]) + if err != nil { + return nil, fmt.Errorf("parse objectname '%s': %v", ref["objectname"], err) + } + + if tag.Type == "commit" { + // lightweight tag + tag.Object = tag.ID + } else { + // annotated tag + tag.Object, err = NewIDFromString(ref["object"]) + if err != nil { + return nil, fmt.Errorf("parse object '%s': %v", ref["object"], err) + } + } + + tag.Tagger, err = newSignatureFromCommitline([]byte(ref["creator"])) + if err != nil { + return nil, fmt.Errorf("parse tagger: %w", err) + } + + tag.Message = ref["contents"] + // strip PGP signature if present in contents field + pgpStart := strings.Index(tag.Message, beginpgp) + if pgpStart >= 0 { + tag.Message = tag.Message[0:pgpStart] + } + + // annotated tag with GPG signature + if tag.Type == "tag" && ref["contents:signature"] != "" { + payload := fmt.Sprintf("object %s\ntype commit\ntag %s\ntagger %s\n\n%s\n", + tag.Object, tag.Name, ref["creator"], strings.TrimSpace(tag.Message)) + tag.Signature = &CommitGPGSignature{ + Signature: ref["contents:signature"], + Payload: payload, + } + } + + return tag, nil + } + // GetAnnotatedTag returns a Git tag by its SHA, must be an annotated tag func (repo *Repository) GetAnnotatedTag(sha string) (*Tag, error) { id, err := NewIDFromString(sha)
• modules/git/repo_tag_test.go
  • @@ -11,6 +11,7 @@ import (
    "code.gitea.io/gitea/modules/util" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestRepository_GetTags(t *testing.T) {
  • @@ -195,3 +196,184 @@ func TestRepository_GetAnnotatedTag(t *testing.T) {
    assert.True(t, IsErrNotExist(err)) assert.Nil(t, tag4) } + + func TestRepository_parseTagRef(t *testing.T) { + tests := []struct { + name string + + givenRef map[string]string + + want *Tag + wantErr bool + expectedErr error + }{ + { + name: "lightweight tag", + + givenRef: map[string]string{ + "objecttype": "commit", + "refname:short": "v1.9.1", + // object will be empty for lightweight tags + "object": "", + "objectname": "ab23e4b7f4cd0caafe0174c0e7ef6d651ba72889", + "creator": "Foo Bar <foo@bar.com> 1565789218 +0300", + "contents": `Add changelog of v1.9.1 (#7859) + + * add changelog of v1.9.1 + * Update CHANGELOG.md + `, + "contents:signature": "", + }, + + want: &Tag{ + Name: "v1.9.1", + ID: MustIDFromString("ab23e4b7f4cd0caafe0174c0e7ef6d651ba72889"), + Object: MustIDFromString("ab23e4b7f4cd0caafe0174c0e7ef6d651ba72889"), + Type: "commit", + Tagger: parseAuthorLine(t, "Foo Bar <foo@bar.com> 1565789218 +0300"), + Message: "Add changelog of v1.9.1 (#7859)\n\n* add changelog of v1.9.1\n* Update CHANGELOG.md\n", + Signature: nil, + }, + }, + + { + name: "annotated tag", + + givenRef: map[string]string{ + "objecttype": "tag", + "refname:short": "v0.0.1", + // object will refer to commit hash for annotated tag + "object": "3325fd8a973321fd59455492976c042dde3fd1ca", + "objectname": "8c68a1f06fc59c655b7e3905b159d761e91c53c9", + "creator": "Foo Bar <foo@bar.com> 1565789218 +0300", + "contents": `Add changelog of v1.9.1 (#7859) + + * add changelog of v1.9.1 + * Update CHANGELOG.md + `, + "contents:signature": "", + }, + + want: &Tag{ + Name: "v0.0.1", + ID: MustIDFromString("8c68a1f06fc59c655b7e3905b159d761e91c53c9"), + Object: MustIDFromString("3325fd8a973321fd59455492976c042dde3fd1ca"), + Type: "tag", + Tagger: parseAuthorLine(t, "Foo Bar <foo@bar.com> 1565789218 +0300"), + Message: "Add changelog of v1.9.1 (#7859)\n\n* add changelog of v1.9.1\n* Update CHANGELOG.md\n", + Signature: nil, + }, + }, + + { + name: "annotated tag with signature", + + givenRef: map[string]string{ + "objecttype": "tag", + "refname:short": "v0.0.1", + "object": "3325fd8a973321fd59455492976c042dde3fd1ca", + "objectname": "8c68a1f06fc59c655b7e3905b159d761e91c53c9", + "creator": "Foo Bar <foo@bar.com> 1565789218 +0300", + "contents": `Add changelog of v1.9.1 (#7859) + + * add changelog of v1.9.1 + * Update CHANGELOG.md + -----BEGIN PGP SIGNATURE----- + + aBCGzBAABCgAdFiEEyWRwv/q1Q6IjSv+D4IPOwzt33PoFAmI8jbIACgkQ4IPOwzt3 + 3PoRuAv9FVSbPBXvzECubls9KQd7urwEvcfG20Uf79iBwifQJUv+egNQojrs6APT + T4CdIXeGRpwJZaGTUX9RWnoDO1SLXAWnc82CypWraNwrHq8Go2YeoVu0Iy3vb0EU + REdob/tXYZecMuP8AjhUR0XfdYaERYAvJ2dYsH/UkFrqDjM3V4kPXWG+R5DCaZiE + slB5U01i4Dwb/zm/ckzhUGEcOgcnpOKX8SnY5kYRVDY47dl/yJZ1u2XWir3mu60G + 1geIitH7StBddHi/8rz+sJwTfcVaLjn2p59p/Dr9aGbk17GIaKq1j0pZA2lKT0Xt + f9jDqU+9vCxnKgjSDhrwN69LF2jT47ZFjEMGV/wFPOa1EBxVWpgQ/CfEolBlbUqx + yVpbxi/6AOK2lmG130e9jEZJcu+WeZUeq851WgKSEkf2d5f/JpwtSTEOlOedu6V6 + kl845zu5oE2nKM4zMQ7XrYQn538I31ps+VGQ0H8R07WrZP8WKUWugL2cU8KmXFwg + qbHDASXl + =2yGi + -----END PGP SIGNATURE----- + + `, + "contents:signature": `-----BEGIN PGP SIGNATURE----- + + aBCGzBAABCgAdFiEEyWRwv/q1Q6IjSv+D4IPOwzt33PoFAmI8jbIACgkQ4IPOwzt3 + 3PoRuAv9FVSbPBXvzECubls9KQd7urwEvcfG20Uf79iBwifQJUv+egNQojrs6APT + T4CdIXeGRpwJZaGTUX9RWnoDO1SLXAWnc82CypWraNwrHq8Go2YeoVu0Iy3vb0EU + REdob/tXYZecMuP8AjhUR0XfdYaERYAvJ2dYsH/UkFrqDjM3V4kPXWG+R5DCaZiE + slB5U01i4Dwb/zm/ckzhUGEcOgcnpOKX8SnY5kYRVDY47dl/yJZ1u2XWir3mu60G + 1geIitH7StBddHi/8rz+sJwTfcVaLjn2p59p/Dr9aGbk17GIaKq1j0pZA2lKT0Xt + f9jDqU+9vCxnKgjSDhrwN69LF2jT47ZFjEMGV/wFPOa1EBxVWpgQ/CfEolBlbUqx + yVpbxi/6AOK2lmG130e9jEZJcu+WeZUeq851WgKSEkf2d5f/JpwtSTEOlOedu6V6 + kl845zu5oE2nKM4zMQ7XrYQn538I31ps+VGQ0H8R07WrZP8WKUWugL2cU8KmXFwg + qbHDASXl + =2yGi + -----END PGP SIGNATURE----- + + `, + }, + + want: &Tag{ + Name: "v0.0.1", + ID: MustIDFromString("8c68a1f06fc59c655b7e3905b159d761e91c53c9"), + Object: MustIDFromString("3325fd8a973321fd59455492976c042dde3fd1ca"), + Type: "tag", + Tagger: parseAuthorLine(t, "Foo Bar <foo@bar.com> 1565789218 +0300"), + Message: "Add changelog of v1.9.1 (#7859)\n\n* add changelog of v1.9.1\n* Update CHANGELOG.md", + Signature: &CommitGPGSignature{ + Signature: `-----BEGIN PGP SIGNATURE----- + + aBCGzBAABCgAdFiEEyWRwv/q1Q6IjSv+D4IPOwzt33PoFAmI8jbIACgkQ4IPOwzt3 + 3PoRuAv9FVSbPBXvzECubls9KQd7urwEvcfG20Uf79iBwifQJUv+egNQojrs6APT + T4CdIXeGRpwJZaGTUX9RWnoDO1SLXAWnc82CypWraNwrHq8Go2YeoVu0Iy3vb0EU + REdob/tXYZecMuP8AjhUR0XfdYaERYAvJ2dYsH/UkFrqDjM3V4kPXWG+R5DCaZiE + slB5U01i4Dwb/zm/ckzhUGEcOgcnpOKX8SnY5kYRVDY47dl/yJZ1u2XWir3mu60G + 1geIitH7StBddHi/8rz+sJwTfcVaLjn2p59p/Dr9aGbk17GIaKq1j0pZA2lKT0Xt + f9jDqU+9vCxnKgjSDhrwN69LF2jT47ZFjEMGV/wFPOa1EBxVWpgQ/CfEolBlbUqx + yVpbxi/6AOK2lmG130e9jEZJcu+WeZUeq851WgKSEkf2d5f/JpwtSTEOlOedu6V6 + kl845zu5oE2nKM4zMQ7XrYQn538I31ps+VGQ0H8R07WrZP8WKUWugL2cU8KmXFwg + qbHDASXl + =2yGi + -----END PGP SIGNATURE----- + + `, + Payload: `object 3325fd8a973321fd59455492976c042dde3fd1ca + type commit + tag v0.0.1 + tagger Foo Bar <foo@bar.com> 1565789218 +0300 + + Add changelog of v1.9.1 (#7859) + + * add changelog of v1.9.1 + * Update CHANGELOG.md + `, + }, + }, + }, + } + + for _, test := range tests { + tc := test // don't close over loop variable + t.Run(tc.name, func(t *testing.T) { + got, err := parseTagRef(tc.givenRef) + + if tc.wantErr { + require.Error(t, err) + require.ErrorIs(t, err, tc.expectedErr) + } else { + require.NoError(t, err) + require.Equal(t, tc.want, got) + } + }) + } + } + + func parseAuthorLine(t *testing.T, committer string) *Signature { + t.Helper() + + sig, err := newSignatureFromCommitline([]byte(committer)) + if err != nil { + t.Fatalf("parse author line '%s': %v", committer, err) + } + + return sig + }
• modules/repository/repo.go
  • @@ -150,6 +150,9 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
    } if !opts.Releases { + // note: this will greatly improve release (tag) sync + // for pull-mirrors with many tags + repo.IsMirror = opts.Mirror if err = SyncReleasesWithTags(repo, gitRepo); err != nil { log.Error("Failed to synchronize tags to releases for repository: %v", err) }
  • @@ -254,6 +257,14 @@ func CleanUpMigrateInfo(ctx context.Context, repo *repo_model.Repository) (*repo
    // SyncReleasesWithTags synchronizes release table with repository tags func SyncReleasesWithTags(repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Debug("SyncReleasesWithTags: in Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) + + // optimized procedure for pull-mirrors which saves a lot of time (in + // particular for repos with many tags). + if repo.IsMirror { + return pullMirrorReleaseSync(repo, gitRepo) + } + existingRelTags := make(map[string]struct{}) opts := models.FindReleasesOptions{ IncludeDrafts: true,
  • @@ -450,3 +461,52 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re
    return nil } + + // pullMirrorReleaseSync is a pull-mirror specific tag<->release table + // synchronization which overwrites all Releases from the repository tags. This + // can be relied on since a pull-mirror is always identical to its + // upstream. Hence, after each sync we want the pull-mirror release set to be + // identical to the upstream tag set. This is much more efficient for + // repositories like https://github.com/vim/vim (with over 13000 tags). + func pullMirrorReleaseSync(repo *repo_model.Repository, gitRepo *git.Repository) error { + log.Trace("pullMirrorReleaseSync: rebuilding releases for pull-mirror Repo[%d:%s/%s]", repo.ID, repo.OwnerName, repo.Name) + tags, numTags, err := gitRepo.GetTagInfos(0, 0) + if err != nil { + return fmt.Errorf("unable to GetTagInfos in pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) + } + err = db.WithTx(func(ctx context.Context) error { + // + // clear out existing releases + // + if _, err := db.DeleteByBean(ctx, &models.Release{RepoID: repo.ID}); err != nil { + return fmt.Errorf("unable to clear releases for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) + } + // + // make release set identical to upstream tags + // + for _, tag := range tags { + release := models.Release{ + RepoID: repo.ID, + TagName: tag.Name, + LowerTagName: strings.ToLower(tag.Name), + Sha1: tag.Object.String(), + // NOTE: ignored, since NumCommits are unused + // for pull-mirrors (only relevant when + // displaying releases, IsTag: false) + NumCommits: -1, + CreatedUnix: timeutil.TimeStamp(tag.Tagger.When.Unix()), + IsTag: true, + } + if err := db.Insert(ctx, release); err != nil { + return fmt.Errorf("unable insert tag %s for pull-mirror Repo[%d:%s/%s]: %w", tag.Name, repo.ID, repo.OwnerName, repo.Name, err) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("unable to rebuild release table for pull-mirror Repo[%d:%s/%s]: %w", repo.ID, repo.OwnerName, repo.Name, err) + } + + log.Trace("pullMirrorReleaseSync: done rebuilding %d releases", numTags) + return nil + }