diff options
author | Anthony Wang | 2023-02-10 00:24:43 +0000 |
---|---|---|
committer | Anthony Wang | 2023-02-10 00:24:43 +0000 |
commit | 1a54d5e8970f2ff6ffe3aeaa19b3917f5e7dc9fd (patch) | |
tree | f8ff0e3f43a8d61879bb885e8f6248b95bf6ca57 /modules | |
parent | e44c986b86200eb862d1db9c10ff44602a638554 (diff) | |
parent | 8574a6433fab47b6f20997f024c176490dfad1c0 (diff) |
Merge remote-tracking branch 'origin/main' into forgejo-federation
Diffstat (limited to 'modules')
59 files changed, 1992 insertions, 190 deletions
diff --git a/modules/actions/log.go b/modules/actions/log.go new file mode 100644 index 000000000..386810199 --- /dev/null +++ b/modules/actions/log.go @@ -0,0 +1,163 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "strings" + "time" + + "code.gitea.io/gitea/models/dbfs" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/storage" + + runnerv1 "code.gitea.io/actions-proto-go/runner/v1" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + MaxLineSize = 64 * 1024 + DBFSPrefix = "actions_log/" + + timeFormat = "2006-01-02T15:04:05.0000000Z07:00" + defaultBufSize = MaxLineSize +) + +func WriteLogs(ctx context.Context, filename string, offset int64, rows []*runnerv1.LogRow) ([]int, error) { + name := DBFSPrefix + filename + f, err := dbfs.OpenFile(ctx, name, os.O_WRONLY|os.O_CREATE) + if err != nil { + return nil, fmt.Errorf("dbfs OpenFile %q: %w", name, err) + } + defer f.Close() + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return nil, fmt.Errorf("dbfs Seek %q: %w", name, err) + } + + writer := bufio.NewWriterSize(f, defaultBufSize) + + ns := make([]int, 0, len(rows)) + for _, row := range rows { + n, err := writer.WriteString(FormatLog(row.Time.AsTime(), row.Content) + "\n") + if err != nil { + return nil, err + } + ns = append(ns, n) + } + + if err := writer.Flush(); err != nil { + return nil, err + } + return ns, nil +} + +func ReadLogs(ctx context.Context, inStorage bool, filename string, offset, limit int64) ([]*runnerv1.LogRow, error) { + f, err := openLogs(ctx, inStorage, filename) + if err != nil { + return nil, err + } + defer f.Close() + + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return nil, fmt.Errorf("file seek: %w", err) + } + + scanner := bufio.NewScanner(f) + maxLineSize := len(timeFormat) + MaxLineSize + 1 + scanner.Buffer(make([]byte, maxLineSize), maxLineSize) + + var rows []*runnerv1.LogRow + for scanner.Scan() && (int64(len(rows)) < limit || limit < 0) { + t, c, err := ParseLog(scanner.Text()) + if err != nil { + return nil, fmt.Errorf("parse log %q: %w", scanner.Text(), err) + } + rows = append(rows, &runnerv1.LogRow{ + Time: timestamppb.New(t), + Content: c, + }) + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + + return rows, nil +} + +func TransferLogs(ctx context.Context, filename string) (func(), error) { + name := DBFSPrefix + filename + remove := func() { + if err := dbfs.Remove(ctx, name); err != nil { + log.Warn("dbfs remove %q: %v", name, err) + } + } + f, err := dbfs.Open(ctx, name) + if err != nil { + return nil, fmt.Errorf("dbfs open %q: %w", name, err) + } + defer f.Close() + + if _, err := storage.Actions.Save(filename, f, -1); err != nil { + return nil, fmt.Errorf("storage save %q: %w", filename, err) + } + return remove, nil +} + +func RemoveLogs(ctx context.Context, inStorage bool, filename string) error { + if !inStorage { + name := DBFSPrefix + filename + err := dbfs.Remove(ctx, name) + if err != nil { + return fmt.Errorf("dbfs remove %q: %w", name, err) + } + return nil + } + err := storage.Actions.Delete(filename) + if err != nil { + return fmt.Errorf("storage delete %q: %w", filename, err) + } + return nil +} + +func openLogs(ctx context.Context, inStorage bool, filename string) (io.ReadSeekCloser, error) { + if !inStorage { + name := DBFSPrefix + filename + f, err := dbfs.Open(ctx, name) + if err != nil { + return nil, fmt.Errorf("dbfs open %q: %w", name, err) + } + return f, nil + } + f, err := storage.Actions.Open(filename) + if err != nil { + return nil, fmt.Errorf("storage open %q: %w", filename, err) + } + return f, nil +} + +func FormatLog(timestamp time.Time, content string) string { + // Content shouldn't contain new line, it will break log indexes, other control chars are safe. + content = strings.ReplaceAll(content, "\n", `\n`) + if len(content) > MaxLineSize { + content = content[:MaxLineSize] + } + return fmt.Sprintf("%s %s", timestamp.UTC().Format(timeFormat), content) +} + +func ParseLog(in string) (time.Time, string, error) { + index := strings.IndexRune(in, ' ') + if index < 0 { + return time.Time{}, "", fmt.Errorf("invalid log: %q", in) + } + timestamp, err := time.Parse(timeFormat, in[:index]) + if err != nil { + return time.Time{}, "", err + } + return timestamp, in[index+1:], nil +} diff --git a/modules/actions/task_state.go b/modules/actions/task_state.go new file mode 100644 index 000000000..cbbc0b357 --- /dev/null +++ b/modules/actions/task_state.go @@ -0,0 +1,101 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + actions_model "code.gitea.io/gitea/models/actions" +) + +const ( + preStepName = "Set up job" + postStepName = "Complete job" +) + +// FullSteps returns steps with "Set up job" and "Complete job" +func FullSteps(task *actions_model.ActionTask) []*actions_model.ActionTaskStep { + if len(task.Steps) == 0 { + return fullStepsOfEmptySteps(task) + } + + firstStep := task.Steps[0] + var logIndex int64 + + preStep := &actions_model.ActionTaskStep{ + Name: preStepName, + LogLength: task.LogLength, + Started: task.Started, + Status: actions_model.StatusRunning, + } + + if firstStep.Status.HasRun() || firstStep.Status.IsRunning() { + preStep.LogLength = firstStep.LogIndex + preStep.Stopped = firstStep.Started + preStep.Status = actions_model.StatusSuccess + } else if task.Status.IsDone() { + preStep.Stopped = task.Stopped + preStep.Status = actions_model.StatusFailure + } + logIndex += preStep.LogLength + + var lastHasRunStep *actions_model.ActionTaskStep + for _, step := range task.Steps { + if step.Status.HasRun() { + lastHasRunStep = step + } + logIndex += step.LogLength + } + if lastHasRunStep == nil { + lastHasRunStep = preStep + } + + postStep := &actions_model.ActionTaskStep{ + Name: postStepName, + Status: actions_model.StatusWaiting, + } + if task.Status.IsDone() { + postStep.LogIndex = logIndex + postStep.LogLength = task.LogLength - postStep.LogIndex + postStep.Status = task.Status + postStep.Started = lastHasRunStep.Stopped + postStep.Stopped = task.Stopped + } + ret := make([]*actions_model.ActionTaskStep, 0, len(task.Steps)+2) + ret = append(ret, preStep) + ret = append(ret, task.Steps...) + ret = append(ret, postStep) + + return ret +} + +func fullStepsOfEmptySteps(task *actions_model.ActionTask) []*actions_model.ActionTaskStep { + preStep := &actions_model.ActionTaskStep{ + Name: preStepName, + LogLength: task.LogLength, + Started: task.Started, + Stopped: task.Stopped, + Status: actions_model.StatusRunning, + } + + postStep := &actions_model.ActionTaskStep{ + Name: postStepName, + LogIndex: task.LogLength, + Started: task.Stopped, + Stopped: task.Stopped, + Status: actions_model.StatusWaiting, + } + + if task.Status.IsDone() { + preStep.Status = task.Status + if preStep.Status.IsSuccess() { + postStep.Status = actions_model.StatusSuccess + } else { + postStep.Status = actions_model.StatusCancelled + } + } + + return []*actions_model.ActionTaskStep{ + preStep, + postStep, + } +} diff --git a/modules/actions/task_state_test.go b/modules/actions/task_state_test.go new file mode 100644 index 000000000..3a599fbcb --- /dev/null +++ b/modules/actions/task_state_test.go @@ -0,0 +1,112 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "testing" + + actions_model "code.gitea.io/gitea/models/actions" + + "github.com/stretchr/testify/assert" +) + +func TestFullSteps(t *testing.T) { + tests := []struct { + name string + task *actions_model.ActionTask + want []*actions_model.ActionTaskStep + }{ + { + name: "regular", + task: &actions_model.ActionTask{ + Steps: []*actions_model.ActionTaskStep{ + {Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090}, + }, + Status: actions_model.StatusSuccess, + Started: 10000, + Stopped: 10100, + LogLength: 100, + }, + want: []*actions_model.ActionTaskStep{ + {Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010}, + {Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 10090}, + {Name: postStepName, Status: actions_model.StatusSuccess, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 10100}, + }, + }, + { + name: "failed step", + task: &actions_model.ActionTask{ + Steps: []*actions_model.ActionTaskStep{ + {Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 20, Started: 10010, Stopped: 10020}, + {Status: actions_model.StatusFailure, LogIndex: 30, LogLength: 60, Started: 10020, Stopped: 10090}, + {Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0}, + }, + Status: actions_model.StatusFailure, + Started: 10000, + Stopped: 10100, + LogLength: 100, + }, + want: []*actions_model.ActionTaskStep{ + {Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010}, + {Status: actions_model.StatusSuccess, LogIndex: 10, LogLength: 20, Started: 10010, Stopped: 10020}, + {Status: actions_model.StatusFailure, LogIndex: 30, LogLength: 60, Started: 10020, Stopped: 10090}, + {Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0}, + {Name: postStepName, Status: actions_model.StatusFailure, LogIndex: 90, LogLength: 10, Started: 10090, Stopped: 10100}, + }, + }, + { + name: "first step is running", + task: &actions_model.ActionTask{ + Steps: []*actions_model.ActionTaskStep{ + {Status: actions_model.StatusRunning, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 0}, + }, + Status: actions_model.StatusRunning, + Started: 10000, + Stopped: 10100, + LogLength: 100, + }, + want: []*actions_model.ActionTaskStep{ + {Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 10, Started: 10000, Stopped: 10010}, + {Status: actions_model.StatusRunning, LogIndex: 10, LogLength: 80, Started: 10010, Stopped: 0}, + {Name: postStepName, Status: actions_model.StatusWaiting, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0}, + }, + }, + { + name: "first step has canceled", + task: &actions_model.ActionTask{ + Steps: []*actions_model.ActionTaskStep{ + {Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0}, + }, + Status: actions_model.StatusFailure, + Started: 10000, + Stopped: 10100, + LogLength: 100, + }, + want: []*actions_model.ActionTaskStep{ + {Name: preStepName, Status: actions_model.StatusFailure, LogIndex: 0, LogLength: 100, Started: 10000, Stopped: 10100}, + {Status: actions_model.StatusCancelled, LogIndex: 0, LogLength: 0, Started: 0, Stopped: 0}, + {Name: postStepName, Status: actions_model.StatusFailure, LogIndex: 100, LogLength: 0, Started: 10100, Stopped: 10100}, + }, + }, + { + name: "empty steps", + task: &actions_model.ActionTask{ + Steps: []*actions_model.ActionTaskStep{}, + Status: actions_model.StatusSuccess, + Started: 10000, + Stopped: 10100, + LogLength: 100, + }, + want: []*actions_model.ActionTaskStep{ + {Name: preStepName, Status: actions_model.StatusSuccess, LogIndex: 0, LogLength: 100, Started: 10000, Stopped: 10100}, + {Name: postStepName, Status: actions_model.StatusSuccess, LogIndex: 100, LogLength: 0, Started: 10100, Stopped: 10100}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, FullSteps(tt.task), "FullSteps(%v)", tt.task) + }) + } +} diff --git a/modules/actions/workflows.go b/modules/actions/workflows.go new file mode 100644 index 000000000..7f0e6e456 --- /dev/null +++ b/modules/actions/workflows.go @@ -0,0 +1,222 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package actions + +import ( + "bytes" + "io" + "strings" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" + api "code.gitea.io/gitea/modules/structs" + webhook_module "code.gitea.io/gitea/modules/webhook" + + "github.com/gobwas/glob" + "github.com/nektos/act/pkg/jobparser" + "github.com/nektos/act/pkg/model" +) + +func ListWorkflows(commit *git.Commit) (git.Entries, error) { + tree, err := commit.SubTree(".gitea/workflows") + if _, ok := err.(git.ErrNotExist); ok { + tree, err = commit.SubTree(".github/workflows") + } + if _, ok := err.(git.ErrNotExist); ok { + return nil, nil + } + if err != nil { + return nil, err + } + + entries, err := tree.ListEntriesRecursiveFast() + if err != nil { + return nil, err + } + + ret := make(git.Entries, 0, len(entries)) + for _, entry := range entries { + if strings.HasSuffix(entry.Name(), ".yml") || strings.HasSuffix(entry.Name(), ".yaml") { + ret = append(ret, entry) + } + } + return ret, nil +} + +func DetectWorkflows(commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader) (map[string][]byte, error) { + entries, err := ListWorkflows(commit) + if err != nil { + return nil, err + } + + workflows := make(map[string][]byte, len(entries)) + for _, entry := range entries { + f, err := entry.Blob().DataAsync() + if err != nil { + return nil, err + } + content, err := io.ReadAll(f) + _ = f.Close() + if err != nil { + return nil, err + } + workflow, err := model.ReadWorkflow(bytes.NewReader(content)) + if err != nil { + log.Warn("ignore invalid workflow %q: %v", entry.Name(), err) + continue + } + events, err := jobparser.ParseRawOn(&workflow.RawOn) + if err != nil { + log.Warn("ignore invalid workflow %q: %v", entry.Name(), err) + continue + } + for _, evt := range events { + if evt.Name != triggedEvent.Event() { + continue + } + if detectMatched(commit, triggedEvent, payload, evt) { + workflows[entry.Name()] = content + } + } + } + + return workflows, nil +} + +func detectMatched(commit *git.Commit, triggedEvent webhook_module.HookEventType, payload api.Payloader, evt *jobparser.Event) bool { + if len(evt.Acts) == 0 { + return true + } + + switch triggedEvent { + case webhook_module.HookEventCreate: + fallthrough + case webhook_module.HookEventDelete: + fallthrough + case webhook_module.HookEventFork: + log.Warn("unsupported event %q", triggedEvent.Event()) + return false + case webhook_module.HookEventPush: + pushPayload := payload.(*api.PushPayload) + matchTimes := 0 + // all acts conditions should be satisfied + for cond, vals := range evt.Acts { + switch cond { + case "branches", "tags": + refShortName := git.RefName(pushPayload.Ref).ShortName() + for _, val := range vals { + if glob.MustCompile(val, '/').Match(refShortName) { + matchTimes++ + break + } + } + case "paths": + filesChanged, err := commit.GetFilesChangedSinceCommit(pushPayload.Before) + if err != nil { + log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err) + } else { + for _, val := range vals { + matched := false + for _, file := range filesChanged { + if glob.MustCompile(val, '/').Match(file) { + matched = true + break + } + } + if matched { + matchTimes++ + break + } + } + } + default: + log.Warn("unsupported condition %q", cond) + } + } + return matchTimes == len(evt.Acts) + + case webhook_module.HookEventIssues: + fallthrough + case webhook_module.HookEventIssueAssign: + fallthrough + case webhook_module.HookEventIssueLabel: + fallthrough + case webhook_module.HookEventIssueMilestone: + fallthrough + case webhook_module.HookEventIssueComment: + fallthrough + case webhook_module.HookEventPullRequest: + prPayload := payload.(*api.PullRequestPayload) + matchTimes := 0 + // all acts conditions should be satisfied + for cond, vals := range evt.Acts { + switch cond { + case "types": + for _, val := range vals { + if glob.MustCompile(val, '/').Match(string(prPayload.Action)) { + matchTimes++ + break + } + } + case "branches": + refShortName := git.RefName(prPayload.PullRequest.Base.Ref).ShortName() + for _, val := range vals { + if glob.MustCompile(val, '/').Match(refShortName) { + matchTimes++ + break + } + } + case "paths": + filesChanged, err := commit.GetFilesChangedSinceCommit(prPayload.PullRequest.Base.Ref) + if err != nil { + log.Error("GetFilesChangedSinceCommit [commit_sha1: %s]: %v", commit.ID.String(), err) + } else { + for _, val := range vals { + matched := false + for _, file := range filesChanged { + if glob.MustCompile(val, '/').Match(file) { + matched = true + break + } + } + if matched { + matchTimes++ + break + } + } + } + default: + log.Warn("unsupported condition %q", cond) + } + } + return matchTimes == len(evt.Acts) + case webhook_module.HookEventPullRequestAssign: + fallthrough + case webhook_module.HookEventPullRequestLabel: + fallthrough + case webhook_module.HookEventPullRequestMilestone: + fallthrough + case webhook_module.HookEventPullRequestComment: + fallthrough + case webhook_module.HookEventPullRequestReviewApproved: + fallthrough + case webhook_module.HookEventPullRequestReviewRejected: + fallthrough + case webhook_module.HookEventPullRequestReviewComment: + fallthrough + case webhook_module.HookEventPullRequestSync: + fallthrough + case webhook_module.HookEventWiki: + fallthrough + case webhook_module.HookEventRepository: + fallthrough + case webhook_module.HookEventRelease: + fallthrough + case webhook_module.HookEventPackage: + fallthrough + default: + log.Warn("unsupported event %q", triggedEvent.Event()) + } + return false +} diff --git a/modules/auth/webauthn/webauthn.go b/modules/auth/webauthn/webauthn.go index d08f7bf7c..937da872c 100644 --- a/modules/auth/webauthn/webauthn.go +++ b/modules/auth/webauthn/webauthn.go @@ -28,7 +28,7 @@ func Init() { Config: &webauthn.Config{ RPDisplayName: setting.AppName, RPID: setting.Domain, - RPOrigin: appURL, + RPOrigins: []string{appURL}, AuthenticatorSelection: protocol.AuthenticatorSelection{ UserVerification: "discouraged", }, diff --git a/modules/auth/webauthn/webauthn_test.go b/modules/auth/webauthn/webauthn_test.go index 1beeb64cd..15a8d7182 100644 --- a/modules/auth/webauthn/webauthn_test.go +++ b/modules/auth/webauthn/webauthn_test.go @@ -15,11 +15,11 @@ func TestInit(t *testing.T) { setting.Domain = "domain" setting.AppName = "AppName" setting.AppURL = "https://domain/" - rpOrigin := "https://domain" + rpOrigin := []string{"https://domain"} Init() assert.Equal(t, setting.Domain, WebAuthn.Config.RPID) assert.Equal(t, setting.AppName, WebAuthn.Config.RPDisplayName) - assert.Equal(t, rpOrigin, WebAuthn.Config.RPOrigin) + assert.Equal(t, rpOrigin, WebAuthn.Config.RPOrigins) } diff --git a/modules/charset/escape.go b/modules/charset/escape.go index 3b1c20697..5608836a4 100644 --- a/modules/charset/escape.go +++ b/modules/charset/escape.go @@ -44,7 +44,7 @@ func EscapeControlReader(reader io.Reader, writer io.Writer, locale translation. return streamer.escaped, err } -// EscapeControlStringReader escapes the unicode control sequences in a provided reader of string content and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte +// EscapeControlStringReader escapes the unicode control sequences in a provided reader of string content and writer in a locale and returns the findings as an EscapeStatus and the escaped []byte. HTML line breaks are not inserted after every newline by this method. func EscapeControlStringReader(reader io.Reader, writer io.Writer, locale translation.Locale, allowed ...rune) (escaped *EscapeStatus, err error) { bufRd := bufio.NewReader(reader) outputStream := &HTMLStreamerWriter{Writer: writer} @@ -65,10 +65,6 @@ func EscapeControlStringReader(reader io.Reader, writer io.Writer, locale transl } break } - if err := streamer.SelfClosingTag("br"); err != nil { - streamer.escaped.HasError = true - return streamer.escaped, err - } } return streamer.escaped, err } diff --git a/modules/context/context.go b/modules/context/context.go index 627309306..84f40ce06 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -805,6 +805,7 @@ func Contexter(ctx context.Context) func(next http.Handler) http.Handler { ctx.Data["EnableOpenIDSignIn"] = setting.Service.EnableOpenIDSignIn ctx.Data["DisableMigrations"] = setting.Repository.DisableMigrations ctx.Data["DisableStars"] = setting.Repository.DisableStars + ctx.Data["EnableActions"] = setting.Actions.Enabled ctx.Data["ManifestData"] = setting.ManifestData @@ -812,6 +813,7 @@ func Contexter(ctx context.Context) func(next http.Handler) http.Handler { ctx.Data["UnitIssuesGlobalDisabled"] = unit.TypeIssues.UnitGlobalDisabled() ctx.Data["UnitPullsGlobalDisabled"] = unit.TypePullRequests.UnitGlobalDisabled() ctx.Data["UnitProjectsGlobalDisabled"] = unit.TypeProjects.UnitGlobalDisabled() + ctx.Data["UnitActionsGlobalDisabled"] = unit.TypeActions.UnitGlobalDisabled() ctx.Data["locale"] = locale ctx.Data["AllLangs"] = translation.AllLangs() diff --git a/modules/context/repo.go b/modules/context/repo.go index a5ade21e4..38c1d8454 100644 --- a/modules/context/repo.go +++ b/modules/context/repo.go @@ -1043,6 +1043,7 @@ func UnitTypes() func(ctx *Context) { ctx.Data["UnitTypeExternalTracker"] = unit_model.TypeExternalTracker ctx.Data["UnitTypeProjects"] = unit_model.TypeProjects ctx.Data["UnitTypePackages"] = unit_model.TypePackages + ctx.Data["UnitTypeActions"] = unit_model.TypeActions } } diff --git a/modules/git/command.go b/modules/git/command.go index d88fcd1a8..0bc810311 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -16,14 +16,20 @@ import ( "time" "unsafe" + "code.gitea.io/gitea/modules/git/internal" //nolint:depguard // only this file can use the internal type CmdArg, other files and packages should use AddXxx functions "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/util" ) +// TrustedCmdArgs returns the trusted arguments for git command. +// It's mainly for passing user-provided and trusted arguments to git command +// In most cases, it shouldn't be used. Use AddXxx function instead +type TrustedCmdArgs []internal.CmdArg + var ( // globalCommandArgs global command args for external package setting - globalCommandArgs []CmdArg + globalCommandArgs TrustedCmdArgs // defaultCommandExecutionTimeout default command execution timeout duration defaultCommandExecutionTimeout = 360 * time.Second @@ -42,8 +48,6 @@ type Command struct { brokenArgs []string } -type CmdArg string - func (c *Command) String() string { if len(c.args) == 0 { return c.name @@ -53,7 +57,7 @@ func (c *Command) String() string { // NewCommand creates and returns a new Git Command based on given command and arguments. // Each argument should be safe to be trusted. User-provided arguments should be passed to AddDynamicArguments instead. -func NewCommand(ctx context.Context, args ...CmdArg) *Command { +func NewCommand(ctx context.Context, args ...internal.CmdArg) *Command { // Make an explicit copy of globalCommandArgs, otherwise append might overwrite it cargs := make([]string, 0, len(globalCommandArgs)+len(args)) for _, arg := range globalCommandArgs { @@ -70,15 +74,9 @@ func NewCommand(ctx context.Context, args ...CmdArg) *Command { } } -// NewCommandNoGlobals creates and returns a new Git Command based on given command and arguments only with the specify args and don't care global command args -// Each argument should be safe to be trusted. User-provided arguments should be passed to AddDynamicArguments instead. -func NewCommandNoGlobals(args ...CmdArg) *Command { - return NewCommandContextNoGlobals(DefaultContext, args...) -} - // NewCommandContextNoGlobals creates and returns a new Git Command based on given command and arguments only with the specify args and don't care global command args // Each argument should be safe to be trusted. User-provided arguments should be passed to AddDynamicArguments instead. -func NewCommandContextNoGlobals(ctx context.Context, args ...CmdArg) *Command { +func NewCommandContextNoGlobals(ctx context.Context, args ...internal.CmdArg) *Command { cargs := make([]string, 0, len(args)) for _, arg := range args { cargs = append(cargs, string(arg)) @@ -96,27 +94,70 @@ func (c *Command) SetParentContext(ctx context.Context) *Command { return c } -// SetDescription sets the description for this command which be returned on -// c.String() +// SetDescription sets the description for this command which be returned on c.String() func (c *Command) SetDescription(desc string) *Command { c.desc = desc return c } -// AddArguments adds new git argument(s) to the command. Each argument must be safe to be trusted. -// User-provided arguments should be passed to AddDynamicArguments instead. -func (c *Command) AddArguments(args ...CmdArg) *Command { +// isSafeArgumentValue checks if the argument is safe to be used as a value (not an option) +func isSafeArgumentValue(s string) bool { + return s == "" || s[0] != '-' +} + +// isValidArgumentOption checks if the argument is a valid option (starting with '-'). +// It doesn't check whether the option is supported or not +func isValidArgumentOption(s string) bool { + return s != "" && s[0] == '-' +} + +// AddArguments adds new git arguments (option/value) to the command. It only accepts string literals, or trusted CmdArg. +// Type CmdArg is in the internal package, so it can not be used outside of this package directly, +// it makes sure that user-provided arguments won't cause RCE risks. +// User-provided arguments should be passed by other AddXxx functions +func (c *Command) AddArguments(args ...internal.CmdArg) *Command { for _, arg := range args { c.args = append(c.args, string(arg)) } return c } -// AddDynamicArguments adds new dynamic argument(s) to the command. -// The arguments may come from user input and can not be trusted, so no leading '-' is allowed to avoid passing options +// AddOptionValues adds a new option with a list of non-option values +// For example: AddOptionValues("--opt", val) means 2 arguments: {"--opt", val}. +// The values are treated as dynamic argument values. It equals to: AddArguments("--opt") then AddDynamicArguments(val). +func (c *Command) AddOptionValues(opt internal.CmdArg, args ...string) *Command { + if !isValidArgumentOption(string(opt)) { + c.brokenArgs = append(c.brokenArgs, string(opt)) + return c + } + c.args = append(c.args, string(opt)) + c.AddDynamicArguments(args...) + return c +} + +// AddOptionFormat adds a new option with a format string and arguments +// For example: AddOptionFormat("--opt=%s %s", val1, val2) means 1 argument: {"--opt=val1 val2"}. +func (c *Command) AddOptionFormat(opt string, args ...any) *Command { + if !isValidArgumentOption(opt) { + c.brokenArgs = append(c.brokenArgs, opt) + return c + } + // a quick check to make sure the format string matches the number of arguments, to find low-level mistakes ASAP + if strings.Count(strings.ReplaceAll(opt, "%%", ""), "%") != len(args) { + c.brokenArgs = append(c.brokenArgs, opt) + return c + } + s := fmt.Sprintf(opt, args...) + c.args = append(c.args, s) + return c +} + +// AddDynamicArguments adds new dynamic argument values to the command. +// The arguments may come from user input and can not be trusted, so no leading '-' is allowed to avoid passing options. +// TODO: in the future, this function can be renamed to AddArgumentValues func (c *Command) AddDynamicArguments(args ...string) *Command { for _, arg := range args { - if arg != "" && arg[0] == '-' { + if !isSafeArgumentValue(arg) { c.brokenArgs = append(c.brokenArgs, arg) } } @@ -137,14 +178,14 @@ func (c *Command) AddDashesAndList(list ...string) *Command { return c } -// CmdArgCheck checks whether the string is safe to be used as a dynamic argument. -// It panics if the check fails. Usually it should not be used, it's just for refactoring purpose -// deprecated -func CmdArgCheck(s string) CmdArg { - if s != "" && s[0] == '-' { - panic("invalid git cmd argument: " + s) +// ToTrustedCmdArgs converts a list of strings (trusted as argument) to TrustedCmdArgs +// In most cases, it shouldn't be used. Use AddXxx function instead +func ToTrustedCmdArgs(args []string) TrustedCmdArgs { + ret := make(TrustedCmdArgs, len(args)) + for i, arg := range args { + ret[i] = internal.CmdArg(arg) } - return CmdArg(s) + return ret } // RunOpts represents parameters to run the command. If UseContextTimeout is specified, then Timeout is ignored. @@ -364,9 +405,9 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS } // AllowLFSFiltersArgs return globalCommandArgs with lfs filter, it should only be used for tests -func AllowLFSFiltersArgs() []CmdArg { +func AllowLFSFiltersArgs() TrustedCmdArgs { // Now here we should explicitly allow lfs filters to run - filteredLFSGlobalArgs := make([]CmdArg, len(globalCommandArgs)) + filteredLFSGlobalArgs := make(TrustedCmdArgs, len(globalCommandArgs)) j := 0 for _, arg := range globalCommandArgs { if strings.Contains(string(arg), "lfs") { diff --git a/modules/git/command_test.go b/modules/git/command_test.go index 2dca2d0d3..4e5f991d3 100644 --- a/modules/git/command_test.go +++ b/modules/git/command_test.go @@ -41,3 +41,14 @@ func TestRunWithContextStd(t *testing.T) { assert.Empty(t, stderr) assert.Contains(t, stdout, "git version") } + +func TestGitArgument(t *testing.T) { + assert.True(t, isValidArgumentOption("-x")) + assert.True(t, isValidArgumentOption("--xx")) + assert.False(t, isValidArgumentOption("")) + assert.False(t, isValidArgumentOption("x")) + + assert.True(t, isSafeArgumentValue("")) + assert.True(t, isSafeArgumentValue("x")) + assert.False(t, isSafeArgumentValue("-x")) +} diff --git a/modules/git/commit.go b/modules/git/commit.go index 14710de61..1f6289ed0 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -9,7 +9,6 @@ import ( "bytes" "context" "errors" - "fmt" "io" "os/exec" "strconv" @@ -91,8 +90,8 @@ func AddChanges(repoPath string, all bool, files ...string) error { } // AddChangesWithArgs marks local changes to be ready for commit. -func AddChangesWithArgs(repoPath string, globalArgs []CmdArg, all bool, files ...string) error { - cmd := NewCommandNoGlobals(append(globalArgs, "add")...) +func AddChangesWithArgs(repoPath string, globalArgs TrustedCmdArgs, all bool, files ...string) error { + cmd := NewCommandContextNoGlobals(DefaultContext, globalArgs...).AddArguments("add") if all { cmd.AddArguments("--all") } @@ -111,17 +110,18 @@ type CommitChangesOptions struct { // CommitChanges commits local changes with given committer, author and message. // If author is nil, it will be the same as committer. func CommitChanges(repoPath string, opts CommitChangesOptions) error { - cargs := make([]CmdArg, len(globalCommandArgs)) + cargs := make(TrustedCmdArgs, len(globalCommandArgs)) copy(cargs, globalCommandArgs) return CommitChangesWithArgs(repoPath, cargs, opts) } // CommitChangesWithArgs commits local changes with given committer, author and message. // If author is nil, it will be the same as committer. -func CommitChangesWithArgs(repoPath string, args []CmdArg, opts CommitChangesOptions) error { - cmd := NewCommandNoGlobals(args...) +func CommitChangesWithArgs(repoPath string, args TrustedCmdArgs, opts CommitChangesOptions) error { + cmd := NewCommandContextNoGlobals(DefaultContext, args...) if opts.Committer != nil { - cmd.AddArguments("-c", CmdArg("user.name="+opts.Committer.Name), "-c", CmdArg("user.email="+opts.Committer.Email)) + cmd.AddOptionValues("-c", "user.name="+opts.Committer.Name) + cmd.AddOptionValues("-c", "user.email="+opts.Committer.Email) } cmd.AddArguments("commit") @@ -129,9 +129,9 @@ func CommitChangesWithArgs(repoPath string, args []CmdArg, opts CommitChangesOpt opts.Author = opts.Committer } if opts.Author != nil { - cmd.AddArguments(CmdArg(fmt.Sprintf("--author='%s <%s>'", opts.Author.Name, opts.Author.Email))) + cmd.AddOptionFormat("--author='%s <%s>'", opts.Author.Name, opts.Author.Email) } - cmd.AddArguments("-m").AddDynamicArguments(opts.Message) + cmd.AddOptionValues("-m", opts.Message) _, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) // No stderr but exit status 1 means nothing to commit. diff --git a/modules/git/git.go b/modules/git/git.go index f5919d82d..2feb242ac 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -383,6 +383,6 @@ func configUnsetAll(key, value string) error { } // Fsck verifies the connectivity and validity of the objects in the database -func Fsck(ctx context.Context, repoPath string, timeout time.Duration, args ...CmdArg) error { +func Fsck(ctx context.Context, repoPath string, timeout time.Duration, args TrustedCmdArgs) error { return NewCommand(ctx, "fsck").AddArguments(args...).Run(&RunOpts{Timeout: timeout, Dir: repoPath}) } diff --git a/modules/git/internal/cmdarg.go b/modules/git/internal/cmdarg.go new file mode 100644 index 000000000..f8f3c2011 --- /dev/null +++ b/modules/git/internal/cmdarg.go @@ -0,0 +1,9 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package internal + +// CmdArg represents a command argument for git command, and it will be used for the git command directly without any further processing. +// In most cases, you should use the "AddXxx" functions to add arguments, but not use this type directly. +// Casting a risky (user-provided) string to CmdArg would cause security issues if it's injected with a "--xxx" argument. +type CmdArg string diff --git a/modules/git/repo.go b/modules/git/repo.go index 4ba40d20a..e77a3a6ad 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -115,7 +115,7 @@ func Clone(ctx context.Context, from, to string, opts CloneRepoOptions) error { } // CloneWithArgs original repository to target path. -func CloneWithArgs(ctx context.Context, args []CmdArg, from, to string, opts CloneRepoOptions) (err error) { +func CloneWithArgs(ctx context.Context, args TrustedCmdArgs, from, to string, opts CloneRepoOptions) (err error) { toDir := path.Dir(to) if err = os.MkdirAll(toDir, os.ModePerm); err != nil { return err diff --git a/modules/git/repo_archive.go b/modules/git/repo_archive.go index cff9724f0..2b45a50f1 100644 --- a/modules/git/repo_archive.go +++ b/modules/git/repo_archive.go @@ -57,9 +57,9 @@ func (repo *Repository) CreateArchive(ctx context.Context, format ArchiveType, t cmd := NewCommand(ctx, "archive") if usePrefix { - cmd.AddArguments(CmdArg("--prefix=" + filepath.Base(strings.TrimSuffix(repo.Path, ".git")) + "/")) + cmd.AddOptionFormat("--prefix=%s", filepath.Base(strings.TrimSuffix(repo.Path, ".git"))+"/") } - cmd.AddArguments(CmdArg("--format=" + format.String())) + cmd.AddOptionFormat("--format=%s", format.String()) cmd.AddDynamicArguments(commitID) var stderr strings.Builder diff --git a/modules/git/repo_attribute.go b/modules/git/repo_attribute.go index 404d9e502..e7d5fb680 100644 --- a/modules/git/repo_attribute.go +++ b/modules/git/repo_attribute.go @@ -17,7 +17,7 @@ import ( type CheckAttributeOpts struct { CachedOnly bool AllAttributes bool - Attributes []CmdArg + Attributes []string Filenames []string IndexFile string WorkTree string @@ -48,7 +48,7 @@ func (repo *Repository) CheckAttribute(opts CheckAttributeOpts) (map[string]map[ } else { for _, attribute := range opts.Attributes { if attribute != "" { - cmd.AddArguments(attribute) + cmd.AddDynamicArguments(attribute) } } } @@ -95,7 +95,7 @@ func (repo *Repository) CheckAttribute(opts CheckAttributeOpts) (map[string]map[ // CheckAttributeReader provides a reader for check-attribute content that can be long running type CheckAttributeReader struct { // params - Attributes []CmdArg + Attributes []string Repo *Repository IndexFile string WorkTree string @@ -111,19 +111,6 @@ type CheckAttributeReader struct { // Init initializes the CheckAttributeReader func (c *CheckAttributeReader) Init(ctx context.Context) error { - cmdArgs := []CmdArg{"check-attr", "--stdin", "-z"} - - if len(c.IndexFile) > 0 { - cmdArgs = append(cmdArgs, "--cached") - c.env = append(c.env, "GIT_INDEX_FILE="+c.IndexFile) - } - - if len(c.WorkTree) > 0 { - c.env = append(c.env, "GIT_WORK_TREE="+c.WorkTree) - } - - c.env = append(c.env, "GIT_FLUSH=1") - if len(c.Attributes) == 0 { lw := new(nulSeparatedAttributeWriter) lw.attributes = make(chan attributeTriple) @@ -134,11 +121,22 @@ func (c *CheckAttributeReader) Init(ctx context.Context) error { return fmt.Errorf("no provided Attributes to check") } - cmdArgs = append(cmdArgs, c.Attributes...) - cmdArgs = append(cmdArgs, "--") - c.ctx, c.cancel = context.WithCancel(ctx) - c.cmd = NewCommand(c.ctx, cmdArgs...) + c.cmd = NewCommand(c.ctx, "check-attr", "--stdin", "-z") + + if len(c.IndexFile) > 0 { + c.cmd.AddArguments("--cached") + c.env = append(c.env, "GIT_INDEX_FILE="+c.IndexFile) + } + + if len(c.WorkTree) > 0 { + c.env = append(c.env, "GIT_WORK_TREE="+c.WorkTree) + } + + c.env = append(c.env, "GIT_FLUSH=1") + + // The empty "--" comes from #16773 , and it seems unnecessary because nothing else would be added later. + c.cmd.AddDynamicArguments(c.Attributes...).AddArguments("--") var err error @@ -294,7 +292,7 @@ func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeRe } checker := &CheckAttributeReader{ - Attributes: []CmdArg{"linguist-vendored", "linguist-generated", "linguist-language", "gitlab-language"}, + Attributes: []string{"linguist-vendored", "linguist-generated", "linguist-language", "gitlab-language"}, Repo: repo, IndexFile: indexFilename, WorkTree: worktree, diff --git a/modules/git/repo_blame.go b/modules/git/repo_blame.go index 7f44735f9..e25efa2d3 100644 --- a/modules/git/repo_blame.go +++ b/modules/git/repo_blame.go @@ -3,7 +3,9 @@ package git -import "fmt" +import ( + "fmt" +) // FileBlame return the Blame object of file func (repo *Repository) FileBlame(revision, path, file string) ([]byte, error) { @@ -14,8 +16,8 @@ func (repo *Repository) FileBlame(revision, path, file string) ([]byte, error) { // LineBlame returns the latest commit at the given line func (repo *Repository) LineBlame(revision, path, file string, line uint) (*Commit, error) { res, _, err := NewCommand(repo.Ctx, "blame"). - AddArguments(CmdArg(fmt.Sprintf("-L %d,%d", line, line))). - AddArguments("-p").AddDynamicArguments(revision). + AddOptionFormat("-L %d,%d", line, line). + AddOptionValues("-p", revision). AddDashesAndList(file).RunStdString(&RunOpts{Dir: path}) if err != nil { return nil, err diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go index ca19d3827..f9896a7a0 100644 --- a/modules/git/repo_branch_gogit.go +++ b/modules/git/repo_branch_gogit.go @@ -50,8 +50,8 @@ func (repo *Repository) IsBranchExist(name string) bool { return reference.Type() != plumbing.InvalidReference } -// GetBranches returns branches from the repository, skipping skip initial branches and -// returning at most limit branches, or all branches if limit is 0. +// GetBranches returns branches from the repository, skipping "skip" initial branches and +// returning at most "limit" branches, or all branches if "limit" is 0. func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) { var branchNames []string diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index 7559513c9..b1e7c8b73 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -59,10 +59,10 @@ func (repo *Repository) IsBranchExist(name string) bool { return repo.IsReferenceExist(BranchPrefix + name) } -// GetBranchNames returns branches from the repository, skipping skip initial branches and -// returning at most limit branches, or all branches if limit is 0. +// GetBranchNames returns branches from the repository, skipping "skip" initial branches and +// returning at most "limit" branches, or all branches if "limit" is 0. func (repo *Repository) GetBranchNames(skip, limit int) ([]string, int, error) { - return callShowRef(repo.Ctx, repo.Path, BranchPrefix, []CmdArg{BranchPrefix, "--sort=-committerdate"}, skip, limit) + return callShowRef(repo.Ctx, repo.Path, BranchPrefix, TrustedCmdArgs{BranchPrefix, "--sort=-committerdate"}, skip, limit) } // WalkReferences walks all the references from the repository @@ -73,19 +73,19 @@ func WalkReferences(ctx context.Context, repoPath string, walkfn func(sha1, refn // WalkReferences walks all the references from the repository // refType should be empty, ObjectTag or ObjectBranch. All other values are equivalent to empty. func (repo *Repository) WalkReferences(refType ObjectType, skip, limit int, walkfn func(sha1, refname string) error) (int, error) { - var args []CmdArg + var args TrustedCmdArgs switch refType { case ObjectTag: - args = []CmdArg{TagPrefix, "--sort=-taggerdate"} + args = TrustedCmdArgs{TagPrefix, "--sort=-taggerdate"} case ObjectBranch: - args = []CmdArg{BranchPrefix, "--sort=-committerdate"} + args = TrustedCmdArgs{BranchPrefix, "--sort=-committerdate"} } return walkShowRef(repo.Ctx, repo.Path, args, skip, limit, walkfn) } // callShowRef return refs, if limit = 0 it will not limit -func callShowRef(ctx context.Context, repoPath, trimPrefix string, extraArgs []CmdArg, skip, limit int) (branchNames []string, countAll int, err error) { +func callShowRef(ctx context.Context, repoPath, trimPrefix string, extraArgs TrustedCmdArgs, skip, limit int) (branchNames []string, countAll int, err error) { countAll, err = walkShowRef(ctx, repoPath, extraArgs, skip, limit, func(_, branchName string) error { branchName = strings.TrimPrefix(branchName, trimPrefix) branchNames = append(branchNames, branchName) @@ -95,7 +95,7 @@ func callShowRef(ctx context.Context, repoPath, trimPrefix string, extraArgs []C return branchNames, countAll, err } -func walkShowRef(ctx context.Context, repoPath string, extraArgs []CmdArg, skip, limit int, walkfn func(sha1, refname string) error) (countAll int, err error) { +func walkShowRef(ctx context.Context, repoPath string, extraArgs TrustedCmdArgs, skip, limit int, walkfn func(sha1, refname string) error) (countAll int, err error) { stdoutReader, stdoutWriter := io.Pipe() defer func() { _ = stdoutReader.Close() @@ -104,7 +104,7 @@ func walkShowRef(ctx context.Context, repoPath string, extraArgs []CmdArg, skip, go func() { stderrBuilder := &strings.Builder{} - args := []CmdArg{"for-each-ref", "--format=%(objectname) %(refname)"} + args := TrustedCmdArgs{"for-each-ref", "--format=%(objectname) %(refname)"} args = append(args, extraArgs...) err := NewCommand(ctx, args...).Run(&RunOpts{ Dir: repoPath, diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index 8343e3484..a62e0670f 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -89,7 +89,7 @@ func (repo *Repository) GetCommitByPath(relpath string) (*Commit, error) { func (repo *Repository) commitsByRange(id SHA1, page, pageSize int) ([]*Commit, error) { stdout, _, err := NewCommand(repo.Ctx, "log"). - AddArguments(CmdArg("--skip="+strconv.Itoa((page-1)*pageSize)), CmdArg("--max-count="+strconv.Itoa(pageSize)), prettyLogFormat). + AddOptionFormat("--skip=%d", (page-1)*pageSize).AddOptionFormat("--max-count=%d", pageSize).AddArguments(prettyLogFormat). AddDynamicArguments(id.String()). RunStdBytes(&RunOpts{Dir: repo.Path}) if err != nil { @@ -99,32 +99,36 @@ func (repo *Repository) commitsByRange(id SHA1, page, pageSize int) ([]*Commit, } func (repo *Repository) searchCommits(id SHA1, opts SearchCommitsOptions) ([]*Commit, error) { - // create new git log command with limit of 100 commis - cmd := NewCommand(repo.Ctx, "log", "-100", prettyLogFormat).AddDynamicArguments(id.String()) - // ignore case - args := []CmdArg{"-i"} + // add common arguments to git command + addCommonSearchArgs := func(c *Command) { + // ignore case + c.AddArguments("-i") + + // add authors if present in search query + if len(opts.Authors) > 0 { + for _, v := range opts.Authors { + c.AddOptionFormat("--author=%s", v) + } + } - // add authors if present in search query - if len(opts.Authors) > 0 { - for _, v := range opts.Authors { - args = append(args, CmdArg("--author="+v)) + // add committers if present in search query + if len(opts.Committers) > 0 { + for _, v := range opts.Committers { + c.AddOptionFormat("--committer=%s", v) + } } - } - // add committers if present in search query - if len(opts.Committers) > 0 { - for _, v := range opts.Committers { - args = append(args, CmdArg("--committer="+v)) + // add time constraints if present in search query + if len(opts.After) > 0 { + c.AddOptionFormat("--after=%s", opts.After) + } + if len(opts.Before) > 0 { + c.AddOptionFormat("--before=%s", opts.Before) } } - // add time constraints if present in search query - if len(opts.After) > 0 { - args = append(args, CmdArg("--after="+opts.After)) - } - if len(opts.Before) > 0 { - args = append(args, CmdArg("--before="+opts.Before)) - } + // create new git log command with limit of 100 commits + cmd := NewCommand(repo.Ctx, "log", "-100", prettyLogFormat).AddDynamicArguments(id.String()) // pretend that all refs along with HEAD were listed on command line as <commis> // https://git-scm.com/docs/git-log#Documentation/git-log.txt---all @@ -137,12 +141,12 @@ func (repo *Repository) searchCommits(id SHA1, opts SearchCommitsOptions) ([]*Co // note this is done only for command created above if len(opts.Keywords) > 0 { for _, v := range opts.Keywords { - cmd.AddArguments(CmdArg("--grep=" + v)) + cmd.AddOptionFormat("--grep=%s", v) } } // search for commits matching given constraints and keywords in commit msg - cmd.AddArguments(args...) + addCommonSearchArgs(cmd) stdout, _, err := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) if err != nil { return nil, err @@ -160,7 +164,7 @@ func (repo *Repository) searchCommits(id SHA1, opts SearchCommitsOptions) ([]*Co // create new git log command with 1 commit limit hashCmd := NewCommand(repo.Ctx, "log", "-1", prettyLogFormat) // add previous arguments except for --grep and --all - hashCmd.AddArguments(args...) + addCommonSearchArgs(hashCmd) // add keyword as <commit> hashCmd.AddDynamicArguments(v) @@ -213,8 +217,8 @@ func (repo *Repository) CommitsByFileAndRange(revision, file string, page int) ( go func() { stderr := strings.Builder{} gitCmd := NewCommand(repo.Ctx, "rev-list"). - AddArguments(CmdArg("--max-count=" + strconv.Itoa(setting.Git.CommitsRangeSize*page))). - AddArguments(CmdArg("--skip=" + strconv.Itoa(skip))) + AddOptionFormat("--max-count=%d", setting.Git.CommitsRangeSize*page). + AddOptionFormat("--skip=%d", skip) gitCmd.AddDynamicArguments(revision) gitCmd.AddDashesAndList(file) err := gitCmd.Run(&RunOpts{ @@ -295,21 +299,21 @@ func (repo *Repository) CommitsBetweenLimit(last, before *Commit, limit, skip in var stdout []byte var err error if before == nil { - stdout, _, err = NewCommand(repo.Ctx, "rev-list", - "--max-count", CmdArg(strconv.Itoa(limit)), - "--skip", CmdArg(strconv.Itoa(skip))). + stdout, _, err = NewCommand(repo.Ctx, "rev-list"). + AddOptionValues("--max-count", strconv.Itoa(limit)). + AddOptionValues("--skip", strconv.Itoa(skip)). AddDynamicArguments(last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) } else { - stdout, _, err = NewCommand(repo.Ctx, "rev-list", - "--max-count", CmdArg(strconv.Itoa(limit)), - "--skip", CmdArg(strconv.Itoa(skip))). + stdout, _, err = NewCommand(repo.Ctx, "rev-list"). + AddOptionValues("--max-count", strconv.Itoa(limit)). + AddOptionValues("--skip", strconv.Itoa(skip)). AddDynamicArguments(before.ID.String() + ".." + last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) if err != nil && strings.Contains(err.Error(), "no merge base") { // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. // previously it would return the results of git rev-list --max-count n before last so let's try that... - stdout, _, err = NewCommand(repo.Ctx, "rev-list", - "--max-count", CmdArg(strconv.Itoa(limit)), - "--skip", CmdArg(strconv.Itoa(skip))). + stdout, _, err = NewCommand(repo.Ctx, "rev-list"). + AddOptionValues("--max-count", strconv.Itoa(limit)). + AddOptionValues("--skip", strconv.Itoa(skip)). AddDynamicArguments(before.ID.String(), last.ID.String()).RunStdBytes(&RunOpts{Dir: repo.Path}) } } @@ -349,12 +353,11 @@ func (repo *Repository) CommitsCountBetween(start, end string) (int64, error) { // commitsBefore the limit is depth, not total number of returned commits. func (repo *Repository) commitsBefore(id SHA1, limit int) ([]*Commit, error) { - cmd := NewCommand(repo.Ctx, "log") + cmd := NewCommand(repo.Ctx, "log", prettyLogFormat) if limit > 0 { - cmd.AddArguments(CmdArg("-"+strconv.Itoa(limit)), prettyLogFormat).AddDynamicArguments(id.String()) - } else { - cmd.AddArguments(prettyLogFormat).AddDynamicArguments(id.String()) + cmd.AddOptionFormat("-%d", limit) } + cmd.AddDynamicArguments(id.String()) stdout, _, runErr := cmd.RunStdBytes(&RunOpts{Dir: repo.Path}) if runErr != nil { @@ -393,10 +396,9 @@ func (repo *Repository) getCommitsBeforeLimit(id SHA1, num int) ([]*Commit, erro func (repo *Repository) getBranches(commit *Commit, limit int) ([]string, error) { if CheckGitVersionAtLeast("2.7.0") == nil { - stdout, _, err := NewCommand(repo.Ctx, "for-each-ref", - CmdArg("--count="+strconv.Itoa(limit)), - "--format=%(refname:strip=2)", "--contains"). - AddDynamicArguments(commit.ID.String(), BranchPrefix). + stdout, _, err := NewCommand(repo.Ctx, "for-each-ref", "--format=%(refname:strip=2)"). + AddOptionFormat("--count=%d", limit). + AddOptionValues("--contains", commit.ID.String(), BranchPrefix). RunStdString(&RunOpts{Dir: repo.Path}) if err != nil { return nil, err @@ -406,7 +408,7 @@ func (repo *Repository) getBranches(commit *Commit, limit int) ([]string, error) return branches, nil } - stdout, _, err := NewCommand(repo.Ctx, "branch", "--contains").AddDynamicArguments(commit.ID.String()).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand(repo.Ctx, "branch").AddOptionValues("--contains", commit.ID.String()).RunStdString(&RunOpts{Dir: repo.Path}) if err != nil { return nil, err } diff --git a/modules/git/repo_compare.go b/modules/git/repo_compare.go index b1b55c88a..9a4d66f2f 100644 --- a/modules/git/repo_compare.go +++ b/modules/git/repo_compare.go @@ -172,25 +172,21 @@ func (repo *Repository) GetDiffNumChangedFiles(base, head string, directComparis // GetDiffShortStat counts number of changed files, number of additions and deletions func (repo *Repository) GetDiffShortStat(base, head string) (numFiles, totalAdditions, totalDeletions int, err error) { - numFiles, totalAdditions, totalDeletions, err = GetDiffShortStat(repo.Ctx, repo.Path, CmdArgCheck(base+"..."+head)) + numFiles, totalAdditions, totalDeletions, err = GetDiffShortStat(repo.Ctx, repo.Path, nil, base+"..."+head) if err != nil && strings.Contains(err.Error(), "no merge base") { - return GetDiffShortStat(repo.Ctx, repo.Path, CmdArgCheck(base), CmdArgCheck(head)) + return GetDiffShortStat(repo.Ctx, repo.Path, nil, base, head) } return numFiles, totalAdditions, totalDeletions, err } // GetDiffShortStat counts number of changed files, number of additions and deletions -func GetDiffShortStat(ctx context.Context, repoPath string, args ...CmdArg) (numFiles, totalAdditions, totalDeletions int, err error) { +func GetDiffShortStat(ctx context.Context, repoPath string, trustedArgs TrustedCmdArgs, dynamicArgs ...string) (numFiles, totalAdditions, totalDeletions int, err error) { // Now if we call: // $ git diff --shortstat 1ebb35b98889ff77299f24d82da426b434b0cca0...788b8b1440462d477f45b0088875 // we get: // " 9902 files changed, 2034198 insertions(+), 298800 deletions(-)\n" - args = append([]CmdArg{ - "diff", - "--shortstat", - }, args...) - - stdout, _, err := NewCommand(ctx, args...).RunStdString(&RunOpts{Dir: repoPath}) + cmd := NewCommand(ctx, "diff", "--shortstat").AddArguments(trustedArgs...).AddDynamicArguments(dynamicArgs...) + stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) if err != nil { return 0, 0, 0, err } diff --git a/modules/git/repo_stats.go b/modules/git/repo_stats.go index d6e91f25a..41f94e24f 100644 --- a/modules/git/repo_stats.go +++ b/modules/git/repo_stats.go @@ -40,7 +40,7 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) since := fromTime.Format(time.RFC3339) - stdout, _, runErr := NewCommand(repo.Ctx, "rev-list", "--count", "--no-merges", "--branches=*", "--date=iso", CmdArg(fmt.Sprintf("--since='%s'", since))).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, runErr := NewCommand(repo.Ctx, "rev-list", "--count", "--no-merges", "--branches=*", "--date=iso").AddOptionFormat("--since='%s'", since).RunStdString(&RunOpts{Dir: repo.Path}) if runErr != nil { return nil, runErr } @@ -60,7 +60,7 @@ func (repo *Repository) GetCodeActivityStats(fromTime time.Time, branch string) _ = stdoutWriter.Close() }() - gitCmd := NewCommand(repo.Ctx, "log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%aN%n%aE%n", "--date=iso", CmdArg(fmt.Sprintf("--since='%s'", since))) + gitCmd := NewCommand(repo.Ctx, "log", "--numstat", "--no-merges", "--pretty=format:---%n%h%n%aN%n%aE%n", "--date=iso").AddOptionFormat("--since='%s'", since) if len(branch) == 0 { gitCmd.AddArguments("--branches=*") } else { diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index 8aa06545d..ae877f021 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -121,7 +121,9 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, int, error) { rc := &RunOpts{Dir: repo.Path, Stdout: stdoutWriter, Stderr: &stderr} go func() { - err := NewCommand(repo.Ctx, "for-each-ref", CmdArg("--format="+forEachRefFmt.Flag()), "--sort", "-*creatordate", "refs/tags").Run(rc) + err := NewCommand(repo.Ctx, "for-each-ref"). + AddOptionFormat("--format=%s", forEachRefFmt.Flag()). + AddArguments("--sort", "-*creatordate", "refs/tags").Run(rc) if err != nil { _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String())) } else { diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go index d3331cf9b..9080ffcfd 100644 --- a/modules/git/repo_tag_nogogit.go +++ b/modules/git/repo_tag_nogogit.go @@ -25,7 +25,7 @@ func (repo *Repository) IsTagExist(name string) bool { // GetTags returns all tags of the repository. // returning at most limit tags, or all if limit is 0. func (repo *Repository) GetTags(skip, limit int) (tags []string, err error) { - tags, _, err = callShowRef(repo.Ctx, repo.Path, TagPrefix, []CmdArg{TagPrefix, "--sort=-taggerdate"}, skip, limit) + tags, _, err = callShowRef(repo.Ctx, repo.Path, TagPrefix, TrustedCmdArgs{TagPrefix, "--sort=-taggerdate"}, skip, limit) return tags, err } diff --git a/modules/git/repo_tree.go b/modules/git/repo_tree.go index 5fea5c0ae..63c33379b 100644 --- a/modules/git/repo_tree.go +++ b/modules/git/repo_tree.go @@ -6,7 +6,6 @@ package git import ( "bytes" - "fmt" "os" "strings" "time" @@ -45,7 +44,7 @@ func (repo *Repository) CommitTree(author, committer *Signature, tree *Tree, opt _, _ = messageBytes.WriteString("\n") if opts.KeyID != "" || opts.AlwaysSign { - cmd.AddArguments(CmdArg(fmt.Sprintf("-S%s", opts.KeyID))) + cmd.AddOptionFormat("-S%s", opts.KeyID) } if opts.NoGPGSign { diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 185317e7a..ef598d7e9 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -100,14 +100,15 @@ func (t *Tree) ListEntries() (Entries, error) { // listEntriesRecursive returns all entries of current tree recursively including all subtrees // extraArgs could be "-l" to get the size, which is slower -func (t *Tree) listEntriesRecursive(extraArgs ...CmdArg) (Entries, error) { +func (t *Tree) listEntriesRecursive(extraArgs TrustedCmdArgs) (Entries, error) { if t.entriesRecursiveParsed { return t.entriesRecursive, nil } - args := append([]CmdArg{"ls-tree", "-t", "-r"}, extraArgs...) - args = append(args, CmdArg(t.ID.String())) - stdout, _, runErr := NewCommand(t.repo.Ctx, args...).RunStdBytes(&RunOpts{Dir: t.repo.Path}) + stdout, _, runErr := NewCommand(t.repo.Ctx, "ls-tree", "-t", "-r"). + AddArguments(extraArgs...). + AddDynamicArguments(t.ID.String()). + RunStdBytes(&RunOpts{Dir: t.repo.Path}) if runErr != nil { return nil, runErr } @@ -123,10 +124,10 @@ func (t *Tree) listEntriesRecursive(extraArgs ...CmdArg) (Entries, error) { // ListEntriesRecursiveFast returns all entries of current tree recursively including all subtrees, no size func (t *Tree) ListEntriesRecursiveFast() (Entries, error) { - return t.listEntriesRecursive() + return t.listEntriesRecursive(nil) } // ListEntriesRecursiveWithSize returns all entries of current tree recursively including all subtrees, with size func (t *Tree) ListEntriesRecursiveWithSize() (Entries, error) { - return t.listEntriesRecursive("--long") + return t.listEntriesRecursive(TrustedCmdArgs{"--long"}) } diff --git a/modules/gitgraph/graph.go b/modules/gitgraph/graph.go index baedfe598..331ad6b21 100644 --- a/modules/gitgraph/graph.go +++ b/modules/gitgraph/graph.go @@ -7,7 +7,6 @@ import ( "bufio" "bytes" "context" - "fmt" "os" "strings" @@ -33,12 +32,9 @@ func GetCommitGraph(r *git.Repository, page, maxAllowedColors int, hidePRRefs bo graphCmd.AddArguments("--all") } - graphCmd.AddArguments( - "-C", - "-M", - git.CmdArg(fmt.Sprintf("-n %d", setting.UI.GraphMaxCommitNum*page)), - "--date=iso", - git.CmdArg(fmt.Sprintf("--pretty=format:%s", format))) + graphCmd.AddArguments("-C", "-M", "--date=iso"). + AddOptionFormat("-n %d", setting.UI.GraphMaxCommitNum*page). + AddOptionFormat("--pretty=format:%s", format) if len(branches) > 0 { graphCmd.AddDynamicArguments(branches...) diff --git a/modules/hcaptcha/error.go b/modules/hcaptcha/error.go new file mode 100644 index 000000000..7b68bf8eb --- /dev/null +++ b/modules/hcaptcha/error.go @@ -0,0 +1,47 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package hcaptcha + +const ( + ErrMissingInputSecret ErrorCode = "missing-input-secret" + ErrInvalidInputSecret ErrorCode = "invalid-input-secret" + ErrMissingInputResponse ErrorCode = "missing-input-response" + ErrInvalidInputResponse ErrorCode = "invalid-input-response" + ErrBadRequest ErrorCode = "bad-request" + ErrInvalidOrAlreadySeenResponse ErrorCode = "invalid-or-already-seen-response" + ErrNotUsingDummyPasscode ErrorCode = "not-using-dummy-passcode" + ErrSitekeySecretMismatch ErrorCode = "sitekey-secret-mismatch" +) + +// ErrorCode is any possible error from hCaptcha +type ErrorCode string + +// String fulfills the Stringer interface +func (err ErrorCode) String() string { + switch err { + case ErrMissingInputSecret: + return "Your secret key is missing." + case ErrInvalidInputSecret: + return "Your secret key is invalid or malformed." + case ErrMissingInputResponse: + return "The response parameter (verification token) is missing." + case ErrInvalidInputResponse: + return "The response parameter (verification token) is invalid or malformed." + case ErrBadRequest: + return "The request is invalid or malformed." + case ErrInvalidOrAlreadySeenResponse: + return "The response parameter has already been checked, or has another issue." + case ErrNotUsingDummyPasscode: + return "You have used a testing sitekey but have not used its matching secret." + case ErrSitekeySecretMismatch: + return "The sitekey is not registered with the provided secret." + default: + return "" + } +} + +// Error fulfills the error interface +func (err ErrorCode) Error() string { + return err.String() +} diff --git a/modules/hcaptcha/hcaptcha.go b/modules/hcaptcha/hcaptcha.go index 4d20cfd48..b970d491c 100644 --- a/modules/hcaptcha/hcaptcha.go +++ b/modules/hcaptcha/hcaptcha.go @@ -5,20 +5,127 @@ package hcaptcha import ( "context" + "io" + "net/http" + "net/url" + "strings" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/setting" - - "go.jolheiser.com/hcaptcha" ) +const verifyURL = "https://hcaptcha.com/siteverify" + +// Client is an hCaptcha client +type Client struct { + ctx context.Context + http *http.Client + + secret string +} + +// PostOptions are optional post form values +type PostOptions struct { + RemoteIP string + Sitekey string +} + +// ClientOption is a func to modify a new Client +type ClientOption func(*Client) + +// WithHTTP sets the http.Client of a Client +func WithHTTP(httpClient *http.Client) func(*Client) { + return func(hClient *Client) { + hClient.http = httpClient + } +} + +// WithContext sets the context.Context of a Client +func WithContext(ctx context.Context) func(*Client) { + return func(hClient *Client) { + hClient.ctx = ctx + } +} + +// New returns a new hCaptcha Client +func New(secret string, options ...ClientOption) (*Client, error) { + if strings.TrimSpace(secret) == "" { + return nil, ErrMissingInputSecret + } + + client := &Client{ + ctx: context.Background(), + http: http.DefaultClient, + secret: secret, + } + + for _, opt := range options { + opt(client) + } + + return client, nil +} + +// Response is an hCaptcha response +type Response struct { + Success bool `json:"success"` + ChallengeTS string `json:"challenge_ts"` + Hostname string `json:"hostname"` + Credit bool `json:"credit,omitempty"` + ErrorCodes []ErrorCode `json:"error-codes"` +} + +// Verify checks the response against the hCaptcha API +func (c *Client) Verify(token string, opts PostOptions) (*Response, error) { + if strings.TrimSpace(token) == "" { + return nil, ErrMissingInputResponse + } + + post := url.Values{ + "secret": []string{c.secret}, + "response": []string{token}, + } + if strings.TrimSpace(opts.RemoteIP) != "" { + post.Add("remoteip", opts.RemoteIP) + } + if strings.TrimSpace(opts.Sitekey) != "" { + post.Add("sitekey", opts.Sitekey) + } + + // Basically a copy of http.PostForm, but with a context + req, err := http.NewRequestWithContext(c.ctx, http.MethodPost, verifyURL, strings.NewReader(post.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + var response *Response + if err := json.Unmarshal(body, &response); err != nil { + return nil, err + } + + return response, nil +} + // Verify calls hCaptcha API to verify token func Verify(ctx context.Context, response string) (bool, error) { - client, err := hcaptcha.New(setting.Service.HcaptchaSecret, hcaptcha.WithContext(ctx)) + client, err := New(setting.Service.HcaptchaSecret, WithContext(ctx)) if err != nil { return false, err } - resp, err := client.Verify(response, hcaptcha.PostOptions{ + resp, err := client.Verify(response, PostOptions{ Sitekey: setting.Service.HcaptchaSitekey, }) if err != nil { diff --git a/modules/hcaptcha/hcaptcha_test.go b/modules/hcaptcha/hcaptcha_test.go new file mode 100644 index 000000000..55e01ec53 --- /dev/null +++ b/modules/hcaptcha/hcaptcha_test.go @@ -0,0 +1,106 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package hcaptcha + +import ( + "net/http" + "os" + "strings" + "testing" + "time" +) + +const ( + dummySiteKey = "10000000-ffff-ffff-ffff-000000000001" + dummySecret = "0x0000000000000000000000000000000000000000" + dummyToken = "10000000-aaaa-bbbb-cccc-000000000001" +) + +func TestMain(m *testing.M) { + os.Exit(m.Run()) +} + +func TestCaptcha(t *testing.T) { + tt := []struct { + Name string + Secret string + Token string + Error ErrorCode + }{ + { + Name: "Success", + Secret: dummySecret, + Token: dummyToken, + }, + { + Name: "Missing Secret", + Token: dummyToken, + Error: ErrMissingInputSecret, + }, + { + Name: "Missing Token", + Secret: dummySecret, + Error: ErrMissingInputResponse, + }, + { + Name: "Invalid Token", + Secret: dummySecret, + Token: "test", + Error: ErrInvalidInputResponse, + }, + } + + for _, tc := range tt { + t.Run(tc.Name, func(t *testing.T) { + client, err := New(tc.Secret, WithHTTP(&http.Client{ + Timeout: time.Second * 5, + })) + if err != nil { + // The only error that can be returned from creating a client + if tc.Error == ErrMissingInputSecret && err == ErrMissingInputSecret { + return + } + t.Log(err) + t.FailNow() + } + + resp, err := client.Verify(tc.Token, PostOptions{ + Sitekey: dummySiteKey, + }) + if err != nil { + // The only error that can be returned prior to the request + if tc.Error == ErrMissingInputResponse && err == ErrMissingInputResponse { + return + } + t.Log(err) + t.FailNow() + } + + if tc.Error.String() != "" { + if resp.Success { + t.Log("Verification should fail.") + t.Fail() + } + if len(resp.ErrorCodes) == 0 { + t.Log("hCaptcha should have returned an error.") + t.Fail() + } + var hasErr bool + for _, err := range resp.ErrorCodes { + if strings.EqualFold(err.String(), tc.Error.String()) { + hasErr = true + break + } + } + if !hasErr { + t.Log("hCaptcha did not return the error being tested") + t.Fail() + } + } else if !resp.Success { + t.Log("Verification should succeed.") + t.Fail() + } + }) + } +} diff --git a/modules/httpcache/httpcache.go b/modules/httpcache/httpcache.go index 1247a81fe..f0caa30eb 100644 --- a/modules/httpcache/httpcache.go +++ b/modules/httpcache/httpcache.go @@ -19,14 +19,16 @@ import ( func AddCacheControlToHeader(h http.Header, maxAge time.Duration, additionalDirectives ...string) { directives := make([]string, 0, 2+len(additionalDirectives)) + // "max-age=0 + must-revalidate" (aka "no-cache") is preferred instead of "no-store" + // because browsers may restore some input fields after navigate-back / reload a page. if setting.IsProd { if maxAge == 0 { - directives = append(directives, "no-store") + directives = append(directives, "max-age=0", "private", "must-revalidate") } else { directives = append(directives, "private", "max-age="+strconv.Itoa(int(maxAge.Seconds()))) } } else { - directives = append(directives, "no-store") + directives = append(directives, "max-age=0", "private", "must-revalidate") // to remind users they are using non-prod setting. h.Add("X-Gitea-Debug", "RUN_MODE="+setting.RunMode) diff --git a/modules/log/colors.go b/modules/log/colors.go index 02781afe8..85e205cb6 100644 --- a/modules/log/colors.go +++ b/modules/log/colors.go @@ -383,6 +383,13 @@ func (cv *ColoredValue) Format(s fmt.State, c rune) { s.Write(*cv.resetBytes) } +// ColorFormatAsString returns the result of the ColorFormat without the color +func ColorFormatAsString(colorVal ColorFormatted) string { + s := new(strings.Builder) + _, _ = ColorFprintf(&protectedANSIWriter{w: s, mode: removeColor}, "%-v", colorVal) + return s.String() +} + // SetColorBytes will allow a user to set the colorBytes of a colored value func (cv *ColoredValue) SetColorBytes(colorBytes []byte) { cv.colorBytes = &colorBytes diff --git a/modules/log/log.go b/modules/log/log.go index 73f908dfa..9057cda37 100644 --- a/modules/log/log.go +++ b/modules/log/log.go @@ -9,6 +9,8 @@ import ( "runtime" "strings" "sync" + + "code.gitea.io/gitea/modules/process" ) type loggerMap struct { @@ -285,6 +287,15 @@ func (l *LoggerAsWriter) Log(msg string) { } func init() { + process.Trace = func(start bool, pid process.IDType, description string, parentPID process.IDType, typ string) { + if start && parentPID != "" { + Log(1, TRACE, "Start %s: %s (from %s) (%s)", NewColoredValue(pid, FgHiYellow), description, NewColoredValue(parentPID, FgYellow), NewColoredValue(typ, Reset)) + } else if start { + Log(1, TRACE, "Start %s: %s (%s)", NewColoredValue(pid, FgHiYellow), description, NewColoredValue(typ, Reset)) + } else { + Log(1, TRACE, "Done %s: %s", NewColoredValue(pid, FgHiYellow), NewColoredValue(description, Reset)) + } + } _, filename, _, _ := runtime.Caller(0) prefix = strings.TrimSuffix(filename, "modules/log/log.go") if prefix == filename { diff --git a/modules/markup/html.go b/modules/markup/html.go index 6b5a8e32d..bcb38f99e 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -164,6 +164,7 @@ var defaultProcessors = []processor{ linkProcessor, mentionProcessor, issueIndexPatternProcessor, + commitCrossReferencePatternProcessor, sha1CurrentPatternProcessor, emailAddressProcessor, emojiProcessor, @@ -190,6 +191,7 @@ var commitMessageProcessors = []processor{ linkProcessor, mentionProcessor, issueIndexPatternProcessor, + commitCrossReferencePatternProcessor, sha1CurrentPatternProcessor, emailAddressProcessor, emojiProcessor, @@ -221,6 +223,7 @@ var commitMessageSubjectProcessors = []processor{ linkProcessor, mentionProcessor, issueIndexPatternProcessor, + commitCrossReferencePatternProcessor, sha1CurrentPatternProcessor, emojiShortCodeProcessor, emojiProcessor, @@ -257,6 +260,7 @@ func RenderIssueTitle( ) (string, error) { return renderProcessString(ctx, []processor{ issueIndexPatternProcessor, + commitCrossReferencePatternProcessor, sha1CurrentPatternProcessor, emojiShortCodeProcessor, emojiProcessor, @@ -354,12 +358,19 @@ func postProcess(ctx *RenderContext, procs []processor, input io.Reader, output } func visitNode(ctx *RenderContext, procs, textProcs []processor, node *html.Node) { - // Add user-content- to IDs if they don't already have them + // Add user-content- to IDs and "#" links if they don't already have them for idx, attr := range node.Attr { - if attr.Key == "id" && !(strings.HasPrefix(attr.Val, "user-content-") || blackfridayExtRegex.MatchString(attr.Val)) { + val := strings.TrimPrefix(attr.Val, "#") + notHasPrefix := !(strings.HasPrefix(val, "user-content-") || blackfridayExtRegex.MatchString(val)) + + if attr.Key == "id" && notHasPrefix { node.Attr[idx].Val = "user-content-" + attr.Val } + if attr.Key == "href" && strings.HasPrefix(attr.Val, "#") && notHasPrefix { + node.Attr[idx].Val = "#user-content-" + val + } + if attr.Key == "class" && attr.Val == "emoji" { textProcs = nil } @@ -907,6 +918,23 @@ func issueIndexPatternProcessor(ctx *RenderContext, node *html.Node) { } } +func commitCrossReferencePatternProcessor(ctx *RenderContext, node *html.Node) { + next := node.NextSibling + + for node != nil && node != next { + found, ref := references.FindRenderizableCommitCrossReference(node.Data) + if !found { + return + } + + reftext := ref.Owner + "/" + ref.Name + "@" + base.ShortSha(ref.CommitSha) + link := createLink(util.URLJoin(setting.AppSubURL, ref.Owner, ref.Name, "commit", ref.CommitSha), reftext, "commit") + + replaceContent(node, ref.RefLocation.Start, ref.RefLocation.End, link) + node = node.NextSibling.NextSibling + } +} + // fullSha1PatternProcessor renders SHA containing URLs func fullSha1PatternProcessor(ctx *RenderContext, node *html.Node) { if ctx.Metas == nil { diff --git a/modules/notification/notification.go b/modules/notification/notification.go index 2300b68f7..da2a7ab3a 100644 --- a/modules/notification/notification.go +++ b/modules/notification/notification.go @@ -10,6 +10,7 @@ import ( packages_model "code.gitea.io/gitea/models/packages" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/notification/action" "code.gitea.io/gitea/modules/notification/base" "code.gitea.io/gitea/modules/notification/indexer" @@ -106,6 +107,13 @@ func NotifyAutoMergePullRequest(ctx context.Context, doer *user_model.User, pr * // NotifyNewPullRequest notifies new pull request to notifiers func NotifyNewPullRequest(ctx context.Context, pr *issues_model.PullRequest, mentions []*user_model.User) { + if err := pr.LoadIssue(ctx); err != nil { + log.Error("%v", err) + return + } + if err := pr.Issue.LoadPoster(ctx); err != nil { + return + } for _, notifier := range notifiers { notifier.NotifyNewPullRequest(ctx, pr, mentions) } @@ -120,6 +128,10 @@ func NotifyPullRequestSynchronized(ctx context.Context, doer *user_model.User, p // NotifyPullRequestReview notifies new pull request review func NotifyPullRequestReview(ctx context.Context, pr *issues_model.PullRequest, review *issues_model.Review, comment *issues_model.Comment, mentions []*user_model.User) { + if err := review.LoadReviewer(ctx); err != nil { + log.Error("%v", err) + return + } for _, notifier := range notifiers { notifier.NotifyPullRequestReview(ctx, pr, review, comment, mentions) } @@ -127,6 +139,10 @@ func NotifyPullRequestReview(ctx context.Context, pr *issues_model.PullRequest, // NotifyPullRequestCodeComment notifies new pull request code comment func NotifyPullRequestCodeComment(ctx context.Context, pr *issues_model.PullRequest, comment *issues_model.Comment, mentions []*user_model.User) { + if err := comment.LoadPoster(ctx); err != nil { + log.Error("LoadPoster: %v", err) + return + } for _, notifier := range notifiers { notifier.NotifyPullRequestCodeComment(ctx, pr, comment, mentions) } @@ -169,6 +185,10 @@ func NotifyDeleteComment(ctx context.Context, doer *user_model.User, c *issues_m // NotifyNewRelease notifies new release to notifiers func NotifyNewRelease(ctx context.Context, rel *repo_model.Release) { + if err := rel.LoadAttributes(ctx); err != nil { + log.Error("LoadPublisher: %v", err) + return + } for _, notifier := range notifiers { notifier.NotifyNewRelease(ctx, rel) } diff --git a/modules/notification/ui/ui.go b/modules/notification/ui/ui.go index 4b85f17b6..73ea92274 100644 --- a/modules/notification/ui/ui.go +++ b/modules/notification/ui/ui.go @@ -97,6 +97,7 @@ func (ns *notificationService) NotifyIssueChangeStatus(ctx context.Context, doer _ = ns.issueQueue.Push(issueNotificationOpts{ IssueID: issue.ID, NotificationAuthorID: doer.ID, + CommentID: actionComment.ID, }) } diff --git a/modules/packages/conda/metadata.go b/modules/packages/conda/metadata.go new file mode 100644 index 000000000..02dbf313b --- /dev/null +++ b/modules/packages/conda/metadata.go @@ -0,0 +1,243 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conda + +import ( + "archive/tar" + "archive/zip" + "compress/bzip2" + "io" + "strings" + + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/validation" + + "github.com/klauspost/compress/zstd" +) + +var ( + ErrInvalidStructure = util.SilentWrap{Message: "package structure is invalid", Err: util.ErrInvalidArgument} + ErrInvalidName = util.SilentWrap{Message: "package name is invalid", Err: util.ErrInvalidArgument} + ErrInvalidVersion = util.SilentWrap{Message: "package version is invalid", Err: util.ErrInvalidArgument} +) + +const ( + PropertyName = "conda.name" + PropertyChannel = "conda.channel" + PropertySubdir = "conda.subdir" + PropertyMetadata = "conda.metdata" +) + +// Package represents a Conda package +type Package struct { + Name string + Version string + Subdir string + VersionMetadata *VersionMetadata + FileMetadata *FileMetadata +} + +// VersionMetadata represents the metadata of a Conda package +type VersionMetadata struct { + Description string `json:"description,omitempty"` + Summary string `json:"summary,omitempty"` + ProjectURL string `json:"project_url,omitempty"` + RepositoryURL string `json:"repository_url,omitempty"` + DocumentationURL string `json:"documentation_url,omitempty"` + License string `json:"license,omitempty"` + LicenseFamily string `json:"license_family,omitempty"` +} + +// FileMetadata represents the metadata of a Conda package file +type FileMetadata struct { + IsCondaPackage bool `json:"is_conda"` + Architecture string `json:"architecture,omitempty"` + NoArch string `json:"noarch,omitempty"` + Build string `json:"build,omitempty"` + BuildNumber int64 `json:"build_number,omitempty"` + Dependencies []string `json:"dependencies,omitempty"` + Platform string `json:"platform,omitempty"` + Timestamp int64 `json:"timestamp,omitempty"` +} + +type index struct { + Name string `json:"name"` + Version string `json:"version"` + Architecture string `json:"arch"` + NoArch string `json:"noarch"` + Build string `json:"build"` + BuildNumber int64 `json:"build_number"` + Dependencies []string `json:"depends"` + License string `json:"license"` + LicenseFamily string `json:"license_family"` + Platform string `json:"platform"` + Subdir string `json:"subdir"` + Timestamp int64 `json:"timestamp"` +} + +type about struct { + Description string `json:"description"` + Summary string `json:"summary"` + ProjectURL string `json:"home"` + RepositoryURL string `json:"dev_url"` + DocumentationURL string `json:"doc_url"` +} + +type ReaderAndReaderAt interface { + io.Reader + io.ReaderAt +} + +// ParsePackageBZ2 parses the Conda package file compressed with bzip2 +func ParsePackageBZ2(r io.Reader) (*Package, error) { + gzr := bzip2.NewReader(r) + + return parsePackageTar(gzr) +} + +// ParsePackageConda parses the Conda package file compressed with zip and zstd +func ParsePackageConda(r io.ReaderAt, size int64) (*Package, error) { + zr, err := zip.NewReader(r, size) + if err != nil { + return nil, err + } + + for _, file := range zr.File { + if strings.HasPrefix(file.Name, "info-") && strings.HasSuffix(file.Name, ".tar.zst") { + f, err := zr.Open(file.Name) + if err != nil { + return nil, err + } + defer f.Close() + + dec, err := zstd.NewReader(f) + if err != nil { + return nil, err + } + defer dec.Close() + + p, err := parsePackageTar(dec) + if p != nil { + p.FileMetadata.IsCondaPackage = true + } + return p, err + } + } + + return nil, ErrInvalidStructure +} + +func parsePackageTar(r io.Reader) (*Package, error) { + var i *index + var a *about + + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + + if hdr.Typeflag != tar.TypeReg { + continue + } + + if hdr.Name == "info/index.json" { + if err := json.NewDecoder(tr).Decode(&i); err != nil { + return nil, err + } + + if !checkName(i.Name) { + return nil, ErrInvalidName + } + + if !checkVersion(i.Version) { + return nil, ErrInvalidVersion + } + + if a != nil { + break // stop loop if both files were found + } + } else if hdr.Name == "info/about.json" { + if err := json.NewDecoder(tr).Decode(&a); err != nil { + return nil, err + } + + if !validation.IsValidURL(a.ProjectURL) { + a.ProjectURL = "" + } + if !validation.IsValidURL(a.RepositoryURL) { + a.RepositoryURL = "" + } + if !validation.IsValidURL(a.DocumentationURL) { + a.DocumentationURL = "" + } + + if i != nil { + break // stop loop if both files were found + } + } + } + + if i == nil { + return nil, ErrInvalidStructure + } + if a == nil { + a = &about{} + } + + return &Package{ + Name: i.Name, + Version: i.Version, + Subdir: i.Subdir, + VersionMetadata: &VersionMetadata{ + License: i.License, + LicenseFamily: i.LicenseFamily, + Description: a.Description, + Summary: a.Summary, + ProjectURL: a.ProjectURL, + RepositoryURL: a.RepositoryURL, + DocumentationURL: a.DocumentationURL, + }, + FileMetadata: &FileMetadata{ + Architecture: i.Architecture, + NoArch: i.NoArch, + Build: i.Build, + BuildNumber: i.BuildNumber, + Dependencies: i.Dependencies, + Platform: i.Platform, + Timestamp: i.Timestamp, + }, + }, nil +} + +// https://github.com/conda/conda-build/blob/db9a728a9e4e6cfc895637ca3221117970fc2663/conda_build/metadata.py#L1393 +func checkName(name string) bool { + if name == "" { + return false + } + if name != strings.ToLower(name) { + return false + } + return !checkBadCharacters(name, "!") +} + +// https://github.com/conda/conda-build/blob/db9a728a9e4e6cfc895637ca3221117970fc2663/conda_build/metadata.py#L1403 +func checkVersion(version string) bool { + if version == "" { + return false + } + return !checkBadCharacters(version, "-") +} + +func checkBadCharacters(s, additional string) bool { + if strings.ContainsAny(s, "=@#$%^&*:;\"'\\|<>?/ ") { + return true + } + return strings.ContainsAny(s, additional) +} diff --git a/modules/packages/conda/metadata_test.go b/modules/packages/conda/metadata_test.go new file mode 100644 index 000000000..2038ca370 --- /dev/null +++ b/modules/packages/conda/metadata_test.go @@ -0,0 +1,150 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package conda + +import ( + "archive/tar" + "archive/zip" + "bytes" + "io" + "testing" + + "github.com/dsnet/compress/bzip2" + "github.com/klauspost/compress/zstd" + "github.com/stretchr/testify/assert" +) + +const ( + packageName = "gitea" + packageVersion = "1.0.1" + description = "Package Description" + projectURL = "https://gitea.io" + repositoryURL = "https://gitea.io/gitea/gitea" + documentationURL = "https://docs.gitea.io" +) + +func TestParsePackage(t *testing.T) { + createArchive := func(files map[string][]byte) *bytes.Buffer { + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for filename, content := range files { + hdr := &tar.Header{ + Name: filename, + Mode: 0o600, + Size: int64(len(content)), + } + tw.WriteHeader(hdr) + tw.Write(content) + } + tw.Close() + return &buf + } + + t.Run("MissingIndexFile", func(t *testing.T) { + buf := createArchive(map[string][]byte{"dummy.txt": {}}) + + p, err := parsePackageTar(buf) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidStructure) + }) + + t.Run("MissingAboutFile", func(t *testing.T) { + buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"name","version":"1.0"}`)}) + + p, err := parsePackageTar(buf) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, "name", p.Name) + assert.Equal(t, "1.0", p.Version) + assert.Empty(t, p.VersionMetadata.ProjectURL) + }) + + t.Run("InvalidName", func(t *testing.T) { + for _, name := range []string{"", "name!", "nAMe"} { + buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"` + name + `","version":"1.0"}`)}) + + p, err := parsePackageTar(buf) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidName) + } + }) + + t.Run("InvalidVersion", func(t *testing.T) { + for _, version := range []string{"", "1.0-2"} { + buf := createArchive(map[string][]byte{"info/index.json": []byte(`{"name":"name","version":"` + version + `"}`)}) + + p, err := parsePackageTar(buf) + assert.Nil(t, p) + assert.ErrorIs(t, err, ErrInvalidVersion) + } + }) + + t.Run("Valid", func(t *testing.T) { + buf := createArchive(map[string][]byte{ + "info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `","subdir":"linux-64"}`), + "info/about.json": []byte(`{"description":"` + description + `","dev_url":"` + repositoryURL + `","doc_url":"` + documentationURL + `","home":"` + projectURL + `"}`), + }) + + p, err := parsePackageTar(buf) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.Equal(t, "linux-64", p.Subdir) + assert.Equal(t, description, p.VersionMetadata.Description) + assert.Equal(t, projectURL, p.VersionMetadata.ProjectURL) + assert.Equal(t, repositoryURL, p.VersionMetadata.RepositoryURL) + assert.Equal(t, documentationURL, p.VersionMetadata.DocumentationURL) + }) + + t.Run(".tar.bz2", func(t *testing.T) { + tarArchive := createArchive(map[string][]byte{ + "info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `"}`), + }) + + var buf bytes.Buffer + bw, _ := bzip2.NewWriter(&buf, nil) + io.Copy(bw, tarArchive) + bw.Close() + + br := bytes.NewReader(buf.Bytes()) + + p, err := ParsePackageBZ2(br) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.False(t, p.FileMetadata.IsCondaPackage) + }) + + t.Run(".conda", func(t *testing.T) { + tarArchive := createArchive(map[string][]byte{ + "info/index.json": []byte(`{"name":"` + packageName + `","version":"` + packageVersion + `"}`), + }) + + var infoBuf bytes.Buffer + zsw, _ := zstd.NewWriter(&infoBuf) + io.Copy(zsw, tarArchive) + zsw.Close() + + var buf bytes.Buffer + zpw := zip.NewWriter(&buf) + w, _ := zpw.Create("info-x.tar.zst") + w.Write(infoBuf.Bytes()) + zpw.Close() + + br := bytes.NewReader(buf.Bytes()) + + p, err := ParsePackageConda(br, int64(br.Len())) + assert.NotNil(t, p) + assert.NoError(t, err) + + assert.Equal(t, packageName, p.Name) + assert.Equal(t, packageVersion, p.Version) + assert.True(t, p.FileMetadata.IsCondaPackage) + }) +} diff --git a/modules/password/pwn.go b/modules/password/pwn.go index e8565e5bb..91bad0d25 100644 --- a/modules/password/pwn.go +++ b/modules/password/pwn.go @@ -6,9 +6,8 @@ package password import ( "context" + "code.gitea.io/gitea/modules/password/pwn" "code.gitea.io/gitea/modules/setting" - - "go.jolheiser.com/pwn" ) // IsPwned checks whether a password has been pwned diff --git a/modules/password/pwn/pwn.go b/modules/password/pwn/pwn.go new file mode 100644 index 000000000..b5a015fb9 --- /dev/null +++ b/modules/password/pwn/pwn.go @@ -0,0 +1,118 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package pwn + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + + "code.gitea.io/gitea/modules/setting" +) + +const passwordURL = "https://api.pwnedpasswords.com/range/" + +// ErrEmptyPassword is an empty password error +var ErrEmptyPassword = errors.New("password cannot be empty") + +// Client is a HaveIBeenPwned client +type Client struct { + ctx context.Context + http *http.Client +} + +// New returns a new HaveIBeenPwned Client +func New(options ...ClientOption) *Client { + client := &Client{ + ctx: context.Background(), + http: http.DefaultClient, + } + + for _, opt := range options { + opt(client) + } + + return client +} + +// ClientOption is a way to modify a new Client +type ClientOption func(*Client) + +// WithHTTP will set the http.Client of a Client +func WithHTTP(httpClient *http.Client) func(pwnClient *Client) { + return func(pwnClient *Client) { + pwnClient.http = httpClient + } +} + +// WithContext will set the context.Context of a Client +func WithContext(ctx context.Context) func(pwnClient *Client) { + return func(pwnClient *Client) { + pwnClient.ctx = ctx + } +} + +func newRequest(ctx context.Context, method, url string, body io.ReadCloser) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return nil, err + } + req.Header.Add("User-Agent", "Gitea "+setting.AppVer) + return req, nil +} + +// CheckPassword returns the number of times a password has been compromised +// Adding padding will make requests more secure, however is also slower +// because artificial responses will be added to the response +// For more information, see https://www.troyhunt.com/enhancing-pwned-passwords-privacy-with-padding/ +func (c *Client) CheckPassword(pw string, padding bool) (int, error) { + if strings.TrimSpace(pw) == "" { + return -1, ErrEmptyPassword + } + + sha := sha1.New() + sha.Write([]byte(pw)) + enc := hex.EncodeToString(sha.Sum(nil)) + prefix, suffix := enc[:5], enc[5:] + + req, err := newRequest(c.ctx, http.MethodGet, fmt.Sprintf("%s%s", passwordURL, prefix), nil) + if err != nil { + return -1, nil + } + if padding { + req.Header.Add("Add-Padding", "true") + } + + resp, err := c.http.Do(req) + if err != nil { + return -1, err + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return -1, err + } + defer resp.Body.Close() + + for _, pair := range strings.Split(string(body), "\n") { + parts := strings.Split(pair, ":") + if len(parts) != 2 { + continue + } + if strings.EqualFold(suffix, parts[0]) { + count, err := strconv.ParseInt(strings.TrimSpace(parts[1]), 10, 64) + if err != nil { + return -1, err + } + return int(count), nil + } + } + return 0, nil +} diff --git a/modules/password/pwn/pwn_test.go b/modules/password/pwn/pwn_test.go new file mode 100644 index 000000000..148208b96 --- /dev/null +++ b/modules/password/pwn/pwn_test.go @@ -0,0 +1,142 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package pwn + +import ( + "errors" + "math/rand" + "net/http" + "os" + "strings" + "testing" + "time" +) + +var client = New(WithHTTP(&http.Client{ + Timeout: time.Second * 2, +})) + +func TestMain(m *testing.M) { + rand.Seed(time.Now().Unix()) + os.Exit(m.Run()) +} + +func TestPassword(t *testing.T) { + // Check input error + _, err := client.CheckPassword("", false) + if err == nil { + t.Log("blank input should return an error") + t.Fail() + } + if !errors.Is(err, ErrEmptyPassword) { + t.Log("blank input should return ErrEmptyPassword") + t.Fail() + } + + // Should fail + fail := "password1234" + count, err := client.CheckPassword(fail, false) + if err != nil { + t.Log(err) + t.Fail() + } + if count == 0 { + t.Logf("%s should fail as a password\n", fail) + t.Fail() + } + + // Should fail (with padding) + failPad := "administrator" + count, err = client.CheckPassword(failPad, true) + if err != nil { + t.Log(err) + t.Fail() + } + if count == 0 { + t.Logf("%s should fail as a password\n", failPad) + t.Fail() + } + + // Checking for a "good" password isn't going to be perfect, but we can give it a good try + // with hopefully minimal error. Try five times? + var good bool + var pw string + for idx := 0; idx <= 5; idx++ { + pw = testPassword() + count, err = client.CheckPassword(pw, false) + if err != nil { + t.Log(err) + t.Fail() + } + if count == 0 { + good = true + break + } + } + if !good { + t.Log("no generated passwords passed. there is a chance this is a fluke") + t.Fail() + } + + // Again, but with padded responses + good = false + for idx := 0; idx <= 5; idx++ { + pw = testPassword() + count, err = client.CheckPassword(pw, true) + if err != nil { + t.Log(err) + t.Fail() + } + if count == 0 { + good = true + break + } + } + if !good { + t.Log("no generated passwords passed. there is a chance this is a fluke") + t.Fail() + } +} + +// Credit to https://golangbyexample.com/generate-random-password-golang/ +// DO NOT USE THIS FOR AN ACTUAL PASSWORD GENERATOR +var ( + lowerCharSet = "abcdedfghijklmnopqrst" + upperCharSet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + specialCharSet = "!@#$%&*" + numberSet = "0123456789" + allCharSet = lowerCharSet + upperCharSet + specialCharSet + numberSet +) + +func testPassword() string { + var password strings.Builder + + // Set special character + for i := 0; i < 5; i++ { + random := rand.Intn(len(specialCharSet)) + password.WriteString(string(specialCharSet[random])) + } + + // Set numeric + for i := 0; i < 5; i++ { + random := rand.Intn(len(numberSet)) + password.WriteString(string(numberSet[random])) + } + + // Set uppercase + for i := 0; i < 5; i++ { + random := rand.Intn(len(upperCharSet)) + password.WriteString(string(upperCharSet[random])) + } + + for i := 0; i < 5; i++ { + random := rand.Intn(len(allCharSet)) + password.WriteString(string(allCharSet[random])) + } + inRune := []rune(password.String()) + rand.Shuffle(len(inRune), func(i, j int) { + inRune[i], inRune[j] = inRune[j], inRune[i] + }) + return string(inRune) +} diff --git a/modules/private/hook.go b/modules/private/hook.go index 027014270..9533eaae5 100644 --- a/modules/private/hook.go +++ b/modules/private/hook.go @@ -57,6 +57,7 @@ type HookOptions struct { PullRequestID int64 DeployKeyID int64 // if the pusher is a DeployKey, then UserID is the repo's org user. IsWiki bool + ActionPerm int } // SSHLogOption ssh log options diff --git a/modules/process/manager.go b/modules/process/manager.go index 127251006..fdfca3db7 100644 --- a/modules/process/manager.go +++ b/modules/process/manager.go @@ -6,6 +6,7 @@ package process import ( "context" + "log" "runtime/pprof" "strconv" "sync" @@ -43,6 +44,18 @@ type IDType string // - it is simply an alias for context.CancelFunc and is only for documentary purposes type FinishedFunc = context.CancelFunc +var Trace = defaultTrace // this global can be overridden by particular logging packages - thus avoiding import cycles + +func defaultTrace(start bool, pid IDType, description string, parentPID IDType, typ string) { + if start && parentPID != "" { + log.Printf("start process %s: %s (from %s) (%s)", pid, description, parentPID, typ) + } else if start { + log.Printf("start process %s: %s (%s)", pid, description, typ) + } else { + log.Printf("end process %s: %s", pid, description) + } +} + // Manager manages all processes and counts PIDs. type Manager struct { mutex sync.Mutex @@ -154,6 +167,7 @@ func (pm *Manager) Add(ctx context.Context, description string, cancel context.C pm.processMap[pid] = process pm.mutex.Unlock() + Trace(true, pid, description, parentPID, processType) pprofCtx := pprof.WithLabels(ctx, pprof.Labels(DescriptionPProfLabel, description, PPIDPProfLabel, string(parentPID), PIDPProfLabel, string(pid), ProcessTypePProfLabel, processType)) if currentlyRunning { @@ -185,18 +199,12 @@ func (pm *Manager) nextPID() (start time.Time, pid IDType) { return start, pid } -// Remove a process from the ProcessManager. -func (pm *Manager) Remove(pid IDType) { - pm.mutex.Lock() - delete(pm.processMap, pid) - pm.mutex.Unlock() -} - func (pm *Manager) remove(process *process) { pm.mutex.Lock() defer pm.mutex.Unlock() if p := pm.processMap[process.PID]; p == process { delete(pm.processMap, process.PID) + Trace(false, process.PID, process.Description, process.ParentPID, process.Type) } } diff --git a/modules/process/manager_test.go b/modules/process/manager_test.go index 527072713..2e2e35d24 100644 --- a/modules/process/manager_test.go +++ b/modules/process/manager_test.go @@ -82,7 +82,7 @@ func TestManager_Remove(t *testing.T) { assert.NotEqual(t, GetContext(p1Ctx).GetPID(), GetContext(p2Ctx).GetPID(), "expected to get different pids got %s == %s", GetContext(p2Ctx).GetPID(), GetContext(p1Ctx).GetPID()) - pm.Remove(GetPID(p2Ctx)) + finished() _, exists := pm.processMap[GetPID(p2Ctx)] assert.False(t, exists, "PID %d is in the list but shouldn't", GetPID(p2Ctx)) diff --git a/modules/references/references.go b/modules/references/references.go index 5cbbf8313..68662425c 100644 --- a/modules/references/references.go +++ b/modules/references/references.go @@ -35,9 +35,12 @@ var ( // issueAlphanumericPattern matches string that references to an alphanumeric issue, e.g. ABC-1234 issueAlphanumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([A-Z]{1,10}-[1-9][0-9]*)(?:\s|$|\)|\]|:|\.(\s|$))`) // crossReferenceIssueNumericPattern matches string that references a numeric issue in a different repository - // e.g. gogits/gogs#12345 + // e.g. org/repo#12345 crossReferenceIssueNumericPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+/[0-9a-zA-Z-_\.]+[#!][0-9]+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`) - // spaceTrimmedPattern let's us find the trailing space + // crossReferenceCommitPattern matches a string that references a commit in a different repository + // e.g. go-gitea/gitea@d8a994ef, go-gitea/gitea@d8a994ef243349f321568f9e36d5c3f444b99cae (7-40 characters) + crossReferenceCommitPattern = regexp.MustCompile(`(?:\s|^|\(|\[)([0-9a-zA-Z-_\.]+)/([0-9a-zA-Z-_\.]+)@([0-9a-f]{7,40})(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`) + // spaceTrimmedPattern let's find the trailing space spaceTrimmedPattern = regexp.MustCompile(`(?:.*[0-9a-zA-Z-_])\s`) // timeLogPattern matches string for time tracking timeLogPattern = regexp.MustCompile(`(?:\s|^|\(|\[)(@([0-9]+([\.,][0-9]+)?(w|d|m|h))+)(?:\s|$|\)|\]|[:;,.?!]\s|[:;,.?!]$)`) @@ -92,6 +95,7 @@ type RenderizableReference struct { Issue string Owner string Name string + CommitSha string IsPull bool RefLocation *RefSpan Action XRefAction @@ -350,6 +354,21 @@ func FindRenderizableReferenceNumeric(content string, prOnly bool) (bool, *Rende } } +// FindRenderizableCommitCrossReference returns the first unvalidated commit cross reference found in a string. +func FindRenderizableCommitCrossReference(content string) (bool, *RenderizableReference) { + m := crossReferenceCommitPattern.FindStringSubmatchIndex(content) + if len(m) < 8 { + return false, nil + } + + return true, &RenderizableReference{ + Owner: content[m[2]:m[3]], + Name: content[m[4]:m[5]], + CommitSha: content[m[6]:m[7]], + RefLocation: &RefSpan{Start: m[2], End: m[7]}, + } +} + // FindRenderizableReferenceRegexp returns the first regexp unvalidated references found in a string. func FindRenderizableReferenceRegexp(content string, pattern *regexp.Regexp) (bool, *RenderizableReference) { match := pattern.FindStringSubmatchIndex(content) diff --git a/modules/references/references_test.go b/modules/references/references_test.go index 835cee3a3..75e002c99 100644 --- a/modules/references/references_test.go +++ b/modules/references/references_test.go @@ -303,6 +303,67 @@ func TestFindAllMentions(t *testing.T) { }, res) } +func TestFindRenderizableCommitCrossReference(t *testing.T) { + cases := []struct { + Input string + Expected *RenderizableReference + }{ + { + Input: "", + Expected: nil, + }, + { + Input: "test", + Expected: nil, + }, + { + Input: "go-gitea/gitea@test", + Expected: nil, + }, + { + Input: "go-gitea/gitea@ab1234", + Expected: nil, + }, + { + Input: "go-gitea/gitea@abcd1234", + Expected: &RenderizableReference{ + Owner: "go-gitea", + Name: "gitea", + CommitSha: "abcd1234", + RefLocation: &RefSpan{Start: 0, End: 23}, + }, + }, + { + Input: "go-gitea/gitea@abcd1234abcd1234abcd1234abcd1234abcd1234", + Expected: &RenderizableReference{ + Owner: "go-gitea", + Name: "gitea", + CommitSha: "abcd1234abcd1234abcd1234abcd1234abcd1234", + RefLocation: &RefSpan{Start: 0, End: 55}, + }, + }, + { + Input: "go-gitea/gitea@abcd1234abcd1234abcd1234abcd1234abcd12340", // longer than 40 characters + Expected: nil, + }, + { + Input: "test go-gitea/gitea@abcd1234 test", + Expected: &RenderizableReference{ + Owner: "go-gitea", + Name: "gitea", + CommitSha: "abcd1234", + RefLocation: &RefSpan{Start: 5, End: 28}, + }, + }, + } + + for _, c := range cases { + found, ref := FindRenderizableCommitCrossReference(c.Input) + assert.Equal(t, ref != nil, found) + assert.Equal(t, c.Expected, ref) + } +} + func TestRegExp_mentionPattern(t *testing.T) { trueTestCases := []struct { pat string diff --git a/modules/repository/create.go b/modules/repository/create.go index 454a192dd..7bcda0fe4 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -30,7 +30,7 @@ import ( ) // CreateRepositoryByExample creates a repository for the user/organization. -func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository, overwriteOrAdopt bool) (err error) { +func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, repo *repo_model.Repository, overwriteOrAdopt, isFork bool) (err error) { if err = repo_model.IsUsableRepoName(repo.Name); err != nil { return err } @@ -67,8 +67,12 @@ func CreateRepositoryByExample(ctx context.Context, doer, u *user_model.User, re } // insert units for repo - units := make([]repo_model.RepoUnit, 0, len(unit.DefaultRepoUnits)) - for _, tp := range unit.DefaultRepoUnits { + defaultUnits := unit.DefaultRepoUnits + if isFork { + defaultUnits = unit.DefaultForkRepoUnits + } + units := make([]repo_model.RepoUnit, 0, len(defaultUnits)) + for _, tp := range defaultUnits { if tp == unit.TypeIssues { units = append(units, repo_model.RepoUnit{ RepoID: repo.ID, @@ -212,7 +216,7 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m var rollbackRepo *repo_model.Repository if err := db.WithTx(db.DefaultContext, func(ctx context.Context) error { - if err := CreateRepositoryByExample(ctx, doer, u, repo, false); err != nil { + if err := CreateRepositoryByExample(ctx, doer, u, repo, false, false); err != nil { return err } diff --git a/modules/repository/env.go b/modules/repository/env.go index 646bf35cc..30edd1c9e 100644 --- a/modules/repository/env.go +++ b/modules/repository/env.go @@ -27,6 +27,7 @@ const ( EnvPRID = "GITEA_PR_ID" EnvIsInternal = "GITEA_INTERNAL_PUSH" EnvAppURL = "GITEA_ROOT_URL" + EnvActionPerm = "GITEA_ACTION_PERM" ) // InternalPushingEnvironment returns an os environment to switch off hooks on push diff --git a/modules/repository/generate.go b/modules/repository/generate.go index b6a1d7b43..31d5ebbb1 100644 --- a/modules/repository/generate.go +++ b/modules/repository/generate.go @@ -319,7 +319,7 @@ func GenerateRepository(ctx context.Context, doer, owner *user_model.User, templ TrustModel: templateRepo.TrustModel, } - if err = CreateRepositoryByExample(ctx, doer, owner, generateRepo, false); err != nil { + if err = CreateRepositoryByExample(ctx, doer, owner, generateRepo, false, false); err != nil { return nil, err } diff --git a/modules/repository/init.go b/modules/repository/init.go index 2b0d0be7b..5705fe5b9 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -316,14 +316,13 @@ func initRepoCommit(ctx context.Context, tmpPath string, repo *repo_model.Reposi return fmt.Errorf("git add --all: %w", err) } - cmd := git.NewCommand(ctx, - "commit", git.CmdArg(fmt.Sprintf("--author='%s <%s>'", sig.Name, sig.Email)), - "-m", "Initial commit", - ) + cmd := git.NewCommand(ctx, "commit"). + AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email). + AddOptionValues("-m", "Initial commit") sign, keyID, signer, _ := asymkey_service.SignInitialCommit(ctx, tmpPath, u) if sign { - cmd.AddArguments(git.CmdArg("-S" + keyID)) + cmd.AddOptionFormat("-S%s", keyID) if repo.GetTrustModel() == repo_model.CommitterTrustModel || repo.GetTrustModel() == repo_model.CollaboratorCommitterTrustModel { // need to set the committer to the KeyID owner diff --git a/modules/setting/actions.go b/modules/setting/actions.go new file mode 100644 index 000000000..5c83a73ae --- /dev/null +++ b/modules/setting/actions.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "code.gitea.io/gitea/modules/log" +) + +// Actions settings +var ( + Actions = struct { + Storage // how the created logs should be stored + Enabled bool + DefaultActionsURL string `ini:"DEFAULT_ACTIONS_URL"` + }{ + Enabled: false, + DefaultActionsURL: "https://gitea.com", + } +) + +func newActions() { + sec := Cfg.Section("actions") + if err := sec.MapTo(&Actions); err != nil { + log.Fatal("Failed to map Actions settings: %v", err) + } + + Actions.Storage = getStorage("actions_log", "", nil) +} diff --git a/modules/setting/packages.go b/modules/setting/packages.go index 120fbb5bd..d0cd80aa0 100644 --- a/modules/setting/packages.go +++ b/modules/setting/packages.go @@ -27,6 +27,7 @@ var ( LimitTotalOwnerSize int64 LimitSizeComposer int64 LimitSizeConan int64 + LimitSizeConda int64 LimitSizeContainer int64 LimitSizeGeneric int64 LimitSizeHelm int64 @@ -66,6 +67,7 @@ func newPackages() { Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE") Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER") Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN") + Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA") Packages.LimitSizeContainer = mustBytes(sec, "LIMIT_SIZE_CONTAINER") Packages.LimitSizeGeneric = mustBytes(sec, "LIMIT_SIZE_GENERIC") Packages.LimitSizeHelm = mustBytes(sec, "LIMIT_SIZE_HELM") diff --git a/modules/setting/repository.go b/modules/setting/repository.go index d78b63a1f..f53de17a4 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -41,6 +41,7 @@ var ( EnablePushCreateOrg bool DisabledRepoUnits []string DefaultRepoUnits []string + DefaultForkRepoUnits []string PrefixArchiveFiles bool DisableMigrations bool DisableStars bool `ini:"DISABLE_STARS"` @@ -157,6 +158,7 @@ var ( EnablePushCreateOrg: false, DisabledRepoUnits: []string{}, DefaultRepoUnits: []string{}, + DefaultForkRepoUnits: []string{}, PrefixArchiveFiles: true, DisableMigrations: false, DisableStars: false, diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 92861fb52..23cd90553 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -6,6 +6,7 @@ package setting import ( "encoding/base64" + "errors" "fmt" "math" "net" @@ -240,7 +241,6 @@ var ( CustomEmojisMap map[string]string `ini:"-"` SearchRepoDescription bool UseServiceWorker bool - OnlyShowRelevantRepos bool Notification struct { MinTimeout time.Duration @@ -466,8 +466,7 @@ func getAppPath() (string, error) { } if err != nil { - // FIXME: Once we switch to go 1.19 use !errors.Is(err, exec.ErrDot) - if !strings.Contains(err.Error(), "cannot run executable found relative to current directory") { + if !errors.Is(err, exec.ErrDot) { return "", err } appPath, err = filepath.Abs(os.Args[0]) @@ -1074,6 +1073,8 @@ func loadFromConf(allowEmpty bool, extraConfig string) { newPackages() + newActions() + if err = Cfg.Section("ui").MapTo(&UI); err != nil { log.Fatal("Failed to map UI settings: %v", err) } else if err = Cfg.Section("markdown").MapTo(&Markdown); err != nil { @@ -1121,7 +1122,6 @@ func loadFromConf(allowEmpty bool, extraConfig string) { UI.DefaultShowFullName = Cfg.Section("ui").Key("DEFAULT_SHOW_FULL_NAME").MustBool(false) UI.SearchRepoDescription = Cfg.Section("ui").Key("SEARCH_REPO_DESCRIPTION").MustBool(true) UI.UseServiceWorker = Cfg.Section("ui").Key("USE_SERVICE_WORKER").MustBool(false) - UI.OnlyShowRelevantRepos = Cfg.Section("ui").Key("ONLY_SHOW_RELEVANT_REPOS").MustBool(false) HasRobotsTxt, err = util.IsFile(path.Join(CustomPath, "robots.txt")) if err != nil { diff --git a/modules/storage/storage.go b/modules/storage/storage.go index 671e0ce56..d8998b192 100644 --- a/modules/storage/storage.go +++ b/modules/storage/storage.go @@ -125,6 +125,9 @@ var ( // Packages represents packages storage Packages ObjectStorage = uninitializedStorage + + // Actions represents actions storage + Actions ObjectStorage = uninitializedStorage ) // Init init the stoarge @@ -136,6 +139,7 @@ func Init() error { initLFS, initRepoArchives, initPackages, + initActions, } { if err := f(); err != nil { return err @@ -204,3 +208,13 @@ func initPackages() (err error) { Packages, err = NewStorage(setting.Packages.Storage.Type, &setting.Packages.Storage) return err } + +func initActions() (err error) { + if !setting.Actions.Enabled { + Actions = discardStorage("Actions isn't enabled") + return nil + } + log.Info("Initialising Actions storage with type: %s", setting.Actions.Storage.Type) + Actions, err = NewStorage(setting.Actions.Storage.Type, &setting.Actions.Storage) + return err +} diff --git a/modules/structs/commit_status.go b/modules/structs/commit_status.go index dfde79190..7e3b629b7 100644 --- a/modules/structs/commit_status.go +++ b/modules/structs/commit_status.go @@ -18,6 +18,8 @@ const ( CommitStatusFailure CommitStatusState = "failure" // CommitStatusWarning is for when the CommitStatus is Warning CommitStatusWarning CommitStatusState = "warning" + // CommitStatusRunning is for when the CommitStatus is Running + CommitStatusRunning CommitStatusState = "running" ) // NoBetterThan returns true if this State is no better than the given State diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 7b997b49d..a390d9459 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -472,6 +472,9 @@ func NewFuncMap() []template.FuncMap { curBranch, ) }, + "RefShortName": func(ref string) string { + return git.RefName(ref).ShortName() + }, }} } diff --git a/modules/util/io.go b/modules/util/io.go index e5d7561be..69b1d6314 100644 --- a/modules/util/io.go +++ b/modules/util/io.go @@ -4,6 +4,7 @@ package util import ( + "errors" "io" ) @@ -17,3 +18,24 @@ func ReadAtMost(r io.Reader, buf []byte) (n int, err error) { } return n, err } + +// ErrNotEmpty is an error reported when there is a non-empty reader +var ErrNotEmpty = errors.New("not-empty") + +// IsEmptyReader reads a reader and ensures it is empty +func IsEmptyReader(r io.Reader) (err error) { + var buf [1]byte + + for { + n, err := r.Read(buf[:]) + if err != nil { + if err == io.EOF { + return nil + } + return err + } + if n > 0 { + return ErrNotEmpty + } + } +} |