migrations: add test for importing pull requests in gitea uploader (#18752) * logs: add the buffer logger to inspect logs during testing Signed-off-by: Loïc Dachary <loic@dachary.org> * migrations: add test for importing pull requests in gitea uploader Signed-off-by: Loïc Dachary <loic@dachary.org> * for each git.OpenRepositoryCtx, call Close * Content is expected to return the content of the log * test for errors before defer Co-authored-by: Loïc Dachary <loic@dachary.org> Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>

Changes

12 changed files (+512/-20)

• integrations/testlogger.go
  • @@ -181,6 +181,11 @@ func (log *TestLogger) Init(config string) error {
    return nil } + // Content returns the content accumulated in the content provider + func (log *TestLogger) Content() (string, error) { + return "", fmt.Errorf("not supported") + } + // Flush when log should be flushed func (log *TestLogger) Flush() { }
• models/migrations/testlogger_test.go
  • @@ -166,6 +166,11 @@ func (log *TestLogger) Init(config string) error {
    return nil } + // Content returns the content accumulated in the content provider + func (log *TestLogger) Content() (string, error) { + return "", fmt.Errorf("not supported") + } + // Flush when log should be flushed func (log *TestLogger) Flush() { }
• modules/log/buffer.go (new)
  • @@ -0,0 +1,72 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package log + + import ( + "bytes" + "sync" + ) + + type bufferWriteCloser struct { + mu sync.Mutex + buffer bytes.Buffer + } + + func (b *bufferWriteCloser) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.Write(p) + } + + func (b *bufferWriteCloser) Close() error { + return nil + } + + func (b *bufferWriteCloser) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buffer.String() + } + + // BufferLogger implements LoggerProvider and writes messages in a buffer. + type BufferLogger struct { + WriterLogger + } + + // NewBufferLogger create BufferLogger returning as LoggerProvider. + func NewBufferLogger() LoggerProvider { + log := &BufferLogger{} + log.NewWriterLogger(&bufferWriteCloser{}) + return log + } + + // Init inits connection writer + func (log *BufferLogger) Init(string) error { + log.NewWriterLogger(log.out) + return nil + } + + // Content returns the content accumulated in the content provider + func (log *BufferLogger) Content() (string, error) { + return log.out.(*bufferWriteCloser).String(), nil + } + + // Flush when log should be flushed + func (log *BufferLogger) Flush() { + } + + // ReleaseReopen does nothing + func (log *BufferLogger) ReleaseReopen() error { + return nil + } + + // GetName returns the default name for this implementation + func (log *BufferLogger) GetName() string { + return "buffer" + } + + func init() { + Register("buffer", NewBufferLogger) + }
• modules/log/buffer_test.go (new)
  • @@ -0,0 +1,64 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package log + + import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + ) + + func TestBufferLogger(t *testing.T) { + logger := NewBufferLogger() + bufferLogger := logger.(*BufferLogger) + assert.NotNil(t, bufferLogger) + + err := logger.Init("") + assert.NoError(t, err) + + location, _ := time.LoadLocation("EST") + date := time.Date(2019, time.January, 13, 22, 3, 30, 15, location) + + msg := "TEST MSG" + event := Event{ + level: INFO, + msg: msg, + caller: "CALLER", + filename: "FULL/FILENAME", + line: 1, + time: date, + } + logger.LogEvent(&event) + content, err := bufferLogger.Content() + assert.NoError(t, err) + assert.Contains(t, content, msg) + logger.Close() + } + + func TestBufferLoggerContent(t *testing.T) { + level := INFO + logger := NewLogger(0, "console", "console", fmt.Sprintf(`{"level":"%s"}`, level.String())) + + logger.SetLogger("buffer", "buffer", "{}") + defer logger.DelLogger("buffer") + + msg := "A UNIQUE MESSAGE" + Error(msg) + + found := false + for i := 0; i < 30000; i++ { + content, err := logger.GetLoggerProviderContent("buffer") + assert.NoError(t, err) + if strings.Contains(content, msg) { + found = true + break + } + time.Sleep(1 * time.Millisecond) + } + assert.True(t, found) + }
• modules/log/conn.go
  • @@ -119,6 +119,11 @@ func (log *ConnLogger) Init(jsonconfig string) error {
    return nil } + // Content returns the content accumulated in the content provider + func (log *ConnLogger) Content() (string, error) { + return "", fmt.Errorf("not supported") + } + // Flush does nothing for this implementation func (log *ConnLogger) Flush() { }
• modules/log/console.go
  • @@ -66,6 +66,11 @@ func (log *ConsoleLogger) Init(config string) error {
    return nil } + // Content returns the content accumulated in the content provider + func (log *ConsoleLogger) Content() (string, error) { + return "", fmt.Errorf("not supported") + } + // Flush when log should be flushed func (log *ConsoleLogger) Flush() { }
• modules/log/event.go
  • @@ -216,6 +216,12 @@ func (m *MultiChannelledLog) GetEventLogger(name string) EventLogger {
    return m.loggers[name] } + // GetEventProvider returns a sub logger provider content from this MultiChannelledLog + func (m *MultiChannelledLog) GetLoggerProviderContent(name string) (string, error) { + channelledLogger := m.GetEventLogger(name).(*ChannelledLog) + return channelledLogger.loggerProvider.Content() + } + // GetEventLoggerNames returns a list of names func (m *MultiChannelledLog) GetEventLoggerNames() []string { m.rwmutex.RLock()
• modules/log/file.go
  • @@ -243,6 +243,15 @@ func (log *FileLogger) deleteOldLog() {
    }) } + // Content returns the content accumulated in the content provider + func (log *FileLogger) Content() (string, error) { + b, err := os.ReadFile(log.Filename) + if err != nil { + return "", err + } + return string(b), nil + } + // Flush flush file logger. // there are no buffering messages in file logger in memory. // flush file means sync file from disk.
• modules/log/provider.go
  • @@ -7,6 +7,7 @@ package log
    // LoggerProvider represents behaviors of a logger provider. type LoggerProvider interface { Init(config string) error + Content() (string, error) EventLogger }
• modules/log/smtp.go
  • @@ -95,6 +95,11 @@ func (log *SMTPLogger) sendMail(p []byte) (int, error) {
    ) } + // Content returns the content accumulated in the content provider + func (log *SMTPLogger) Content() (string, error) { + return "", fmt.Errorf("not supported") + } + // Flush when log should be flushed func (log *SMTPLogger) Flush() { }
• services/migrations/gitea_uploader.go
  • @@ -484,19 +484,9 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error
    return nil } - func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) { - var labels []*models.Label - for _, label := range pr.Labels { - lb, ok := g.labels[label.Name] - if ok { - labels = append(labels, lb) - } - } - - milestoneID := g.milestones[pr.Milestone] - + func (g *GiteaLocalUploader) updateGitForPullRequest(pr *base.PullRequest) (head string, err error) { // download patch file - err := func() error { + err = func() error { if pr.PatchURL == "" { return nil }
  • @@ -519,25 +509,25 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
    return err }() if err != nil { - return nil, err + return "", err } // set head information pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number)) if err := os.MkdirAll(pullHead, os.ModePerm); err != nil { - return nil, err + return "", err } p, err := os.Create(filepath.Join(pullHead, "head")) if err != nil { - return nil, err + return "", err } _, err = p.WriteString(pr.Head.SHA) p.Close() if err != nil { - return nil, err + return "", err } - head := "unknown repository" + head = "unknown repository" if pr.IsForkPullRequest() && pr.State != "closed" { if pr.Head.OwnerName != "" { remote := pr.Head.OwnerName
  • @@ -560,16 +550,16 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
    } else { headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref) if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil { - return nil, err + return "", err } b, err := os.Create(headBranch) if err != nil { - return nil, err + return "", err } _, err = b.WriteString(pr.Head.SHA) b.Close() if err != nil { - return nil, err + return "", err } head = pr.Head.OwnerName + "/" + pr.Head.Ref }
  • @@ -595,6 +585,25 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
    } } + return head, nil + } + + func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) { + var labels []*models.Label + for _, label := range pr.Labels { + lb, ok := g.labels[label.Name] + if ok { + labels = append(labels, lb) + } + } + + milestoneID := g.milestones[pr.Milestone] + + head, err := g.updateGitForPullRequest(pr) + if err != nil { + return nil, fmt.Errorf("updateGitForPullRequest: %w", err) + } + if pr.Created.IsZero() { if pr.Closed != nil { pr.Created = *pr.Closed
• services/migrations/gitea_uploader_test.go
  • @@ -7,7 +7,11 @@ package migrations
    import ( "context" + "fmt" + "os" + "path/filepath" "strconv" + "strings" "testing" "time"
  • @@ -16,7 +20,9 @@ import (
    repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util"
  • @@ -217,3 +223,303 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) {
    assert.NoError(t, err) assert.EqualValues(t, linkedUser.ID, target.GetUserID()) } + + func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { + unittest.PrepareTestEnv(t) + + // + // fromRepo master + // + fromRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}).(*repo_model.Repository) + baseRef := "master" + assert.NoError(t, git.InitRepository(git.DefaultContext, fromRepo.RepoPath(), false)) + _, err := git.NewCommand(git.DefaultContext, "symbolic-ref", "HEAD", git.BranchPrefix+baseRef).RunInDir(fromRepo.RepoPath()) + assert.NoError(t, err) + assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# Testing Repository\n\nOriginally created in: %s", fromRepo.RepoPath())), 0o644)) + assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true)) + signature := git.Signature{ + Email: "test@example.com", + Name: "test", + When: time.Now(), + } + assert.NoError(t, git.CommitChanges(fromRepo.RepoPath(), git.CommitChangesOptions{ + Committer: &signature, + Author: &signature, + Message: "Initial Commit", + })) + fromGitRepo, err := git.OpenRepositoryCtx(git.DefaultContext, fromRepo.RepoPath()) + assert.NoError(t, err) + defer fromGitRepo.Close() + baseSHA, err := fromGitRepo.GetBranchCommitID(baseRef) + assert.NoError(t, err) + + // + // fromRepo branch1 + // + headRef := "branch1" + _, err = git.NewCommand(git.DefaultContext, "checkout", "-b", headRef).RunInDir(fromRepo.RepoPath()) + assert.NoError(t, err) + assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte("SOMETHING"), 0o644)) + assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true)) + signature.When = time.Now() + assert.NoError(t, git.CommitChanges(fromRepo.RepoPath(), git.CommitChangesOptions{ + Committer: &signature, + Author: &signature, + Message: "Pull request", + })) + assert.NoError(t, err) + headSHA, err := fromGitRepo.GetBranchCommitID(headRef) + assert.NoError(t, err) + + fromRepoOwner := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: fromRepo.OwnerID}).(*user_model.User) + + // + // forkRepo branch2 + // + forkHeadRef := "branch2" + forkRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 8}).(*repo_model.Repository) + assert.NoError(t, git.CloneWithArgs(git.DefaultContext, fromRepo.RepoPath(), forkRepo.RepoPath(), []string{}, git.CloneRepoOptions{ + Branch: headRef, + })) + _, err = git.NewCommand(git.DefaultContext, "checkout", "-b", forkHeadRef).RunInDir(forkRepo.RepoPath()) + assert.NoError(t, err) + assert.NoError(t, os.WriteFile(filepath.Join(forkRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# branch2 %s", forkRepo.RepoPath())), 0o644)) + assert.NoError(t, git.AddChanges(forkRepo.RepoPath(), true)) + assert.NoError(t, git.CommitChanges(forkRepo.RepoPath(), git.CommitChangesOptions{ + Committer: &signature, + Author: &signature, + Message: "branch2 commit", + })) + forkGitRepo, err := git.OpenRepositoryCtx(git.DefaultContext, forkRepo.RepoPath()) + assert.NoError(t, err) + defer forkGitRepo.Close() + forkHeadSHA, err := forkGitRepo.GetBranchCommitID(forkHeadRef) + assert.NoError(t, err) + + toRepoName := "migrated" + uploader := NewGiteaLocalUploader(context.Background(), fromRepoOwner, fromRepoOwner.Name, toRepoName) + uploader.gitServiceType = structs.GiteaService + assert.NoError(t, uploader.CreateRepo(&base.Repository{ + Description: "description", + OriginalURL: fromRepo.RepoPath(), + CloneURL: fromRepo.RepoPath(), + IsPrivate: false, + IsMirror: true, + }, base.MigrateOptions{ + GitServiceType: structs.GiteaService, + Private: false, + Mirror: true, + })) + + for _, testCase := range []struct { + name string + head string + assertContent func(t *testing.T, content string) + pr base.PullRequest + }{ + { + name: "fork, good Head.SHA", + head: fmt.Sprintf("%s/%s", forkRepo.OwnerName, forkHeadRef), + pr: base.PullRequest{ + PatchURL: "", + Number: 1, + State: "open", + Base: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: baseRef, + SHA: baseSHA, + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + Head: base.PullRequestBranch{ + CloneURL: forkRepo.RepoPath(), + Ref: forkHeadRef, + SHA: forkHeadSHA, + RepoName: forkRepo.Name, + OwnerName: forkRepo.OwnerName, + }, + }, + }, + { + name: "fork, invalid Head.Ref", + head: "unknown repository", + pr: base.PullRequest{ + PatchURL: "", + Number: 1, + State: "open", + Base: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: baseRef, + SHA: baseSHA, + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + Head: base.PullRequestBranch{ + CloneURL: forkRepo.RepoPath(), + Ref: "INVALID", + SHA: forkHeadSHA, + RepoName: forkRepo.Name, + OwnerName: forkRepo.OwnerName, + }, + }, + assertContent: func(t *testing.T, content string) { + assert.Contains(t, content, "Fetch branch from") + }, + }, + { + name: "invalid fork CloneURL", + head: "unknown repository", + pr: base.PullRequest{ + PatchURL: "", + Number: 1, + State: "open", + Base: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: baseRef, + SHA: baseSHA, + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + Head: base.PullRequestBranch{ + CloneURL: "UNLIKELY", + Ref: forkHeadRef, + SHA: forkHeadSHA, + RepoName: forkRepo.Name, + OwnerName: "WRONG", + }, + }, + assertContent: func(t *testing.T, content string) { + assert.Contains(t, content, "AddRemote failed") + }, + }, + { + name: "no fork, good Head.SHA", + head: headRef, + pr: base.PullRequest{ + PatchURL: "", + Number: 1, + State: "open", + Base: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: baseRef, + SHA: baseSHA, + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + Head: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: headRef, + SHA: headSHA, + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + }, + }, + { + name: "no fork, empty Head.SHA", + head: headRef, + pr: base.PullRequest{ + PatchURL: "", + Number: 1, + State: "open", + Base: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: baseRef, + SHA: baseSHA, + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + Head: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: headRef, + SHA: "", + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + }, + assertContent: func(t *testing.T, content string) { + assert.Contains(t, content, "Empty reference, removing") + assert.NotContains(t, content, "Cannot remove local head") + }, + }, + { + name: "no fork, invalid Head.SHA", + head: headRef, + pr: base.PullRequest{ + PatchURL: "", + Number: 1, + State: "open", + Base: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: baseRef, + SHA: baseSHA, + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + Head: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: headRef, + SHA: "brokenSHA", + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + }, + assertContent: func(t *testing.T, content string) { + assert.Contains(t, content, "Deprecated local head") + assert.Contains(t, content, "Cannot remove local head") + }, + }, + { + name: "no fork, not found Head.SHA", + head: headRef, + pr: base.PullRequest{ + PatchURL: "", + Number: 1, + State: "open", + Base: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: baseRef, + SHA: baseSHA, + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + Head: base.PullRequestBranch{ + CloneURL: fromRepo.RepoPath(), + Ref: headRef, + SHA: "2697b352310fcd01cbd1f3dbd43b894080027f68", + RepoName: fromRepo.Name, + OwnerName: fromRepo.OwnerName, + }, + }, + assertContent: func(t *testing.T, content string) { + assert.Contains(t, content, "Deprecated local head") + assert.NotContains(t, content, "Cannot remove local head") + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + logger, ok := log.NamedLoggers.Load(log.DEFAULT) + assert.True(t, ok) + logger.SetLogger("buffer", "buffer", "{}") + defer logger.DelLogger("buffer") + + head, err := uploader.updateGitForPullRequest(&testCase.pr) + assert.NoError(t, err) + assert.EqualValues(t, testCase.head, head) + if testCase.assertContent != nil { + fence := fmt.Sprintf(">>>>>>>>>>>>>FENCE %s<<<<<<<<<<<<<<<", testCase.name) + log.Error(fence) + var content string + for i := 0; i < 5000; i++ { + content, err = logger.GetLoggerProviderContent("buffer") + assert.NoError(t, err) + if strings.Contains(content, fence) { + break + } + time.Sleep(1 * time.Millisecond) + } + testCase.assertContent(t, content) + } + }) + } + }