diff options
author | Anthony Wang | 2023-03-12 23:17:11 +0000 |
---|---|---|
committer | Anthony Wang | 2023-03-12 23:19:57 +0000 |
commit | dbc3eb7fa071307e623fde961bfdff0894856948 (patch) | |
tree | 8961dc6a935d9ee89f5adf680a142d36900aaebb /modules | |
parent | b4640101f3c8938ed689743606a79601201141ca (diff) | |
parent | e72290fd9aeb77a47311483d1d565e428ce40cd9 (diff) |
Merge remote-tracking branch 'origin/main' into forgejo-federation
Diffstat (limited to 'modules')
70 files changed, 1208 insertions, 471 deletions
diff --git a/modules/auth/password/hash/setting.go b/modules/auth/password/hash/setting.go index 701697430..f0715f31e 100644 --- a/modules/auth/password/hash/setting.go +++ b/modules/auth/password/hash/setting.go @@ -41,9 +41,8 @@ var RecommendedHashAlgorithms = []string{ "pbkdf2_hi", } -// SetDefaultPasswordHashAlgorithm will take a provided algorithmName and dealias it to -// a complete algorithm specification. -func SetDefaultPasswordHashAlgorithm(algorithmName string) (string, *PasswordHashAlgorithm) { +// hashAlgorithmToSpec converts an algorithm name or a specification to a full algorithm specification +func hashAlgorithmToSpec(algorithmName string) string { if algorithmName == "" { algorithmName = DefaultHashAlgorithmName } @@ -52,10 +51,26 @@ func SetDefaultPasswordHashAlgorithm(algorithmName string) (string, *PasswordHas algorithmName = alias alias, has = aliasAlgorithmNames[algorithmName] } + return algorithmName +} - // algorithmName should now be a full algorithm specification - // e.g. pbkdf2$50000$50 rather than pbdkf2 - DefaultHashAlgorithm = Parse(algorithmName) +// SetDefaultPasswordHashAlgorithm will take a provided algorithmName and de-alias it to +// a complete algorithm specification. +func SetDefaultPasswordHashAlgorithm(algorithmName string) (string, *PasswordHashAlgorithm) { + algoSpec := hashAlgorithmToSpec(algorithmName) + // now we get a full specification, e.g. pbkdf2$50000$50 rather than pbdkf2 + DefaultHashAlgorithm = Parse(algoSpec) + return algoSpec, DefaultHashAlgorithm +} - return algorithmName, DefaultHashAlgorithm +// ConfigHashAlgorithm will try to find a "recommended algorithm name" defined by RecommendedHashAlgorithms for config +// This function is not fast and is only used for the installation page +func ConfigHashAlgorithm(algorithm string) string { + algorithm = hashAlgorithmToSpec(algorithm) + for _, recommAlgo := range RecommendedHashAlgorithms { + if algorithm == hashAlgorithmToSpec(recommAlgo) { + return recommAlgo + } + } + return algorithm } diff --git a/modules/cache/context.go b/modules/cache/context.go index f741a8744..62bbf5dcb 100644 --- a/modules/cache/context.go +++ b/modules/cache/context.go @@ -6,6 +6,7 @@ package cache import ( "context" "sync" + "time" "code.gitea.io/gitea/modules/log" ) @@ -14,65 +15,151 @@ import ( // This is useful for caching data that is expensive to calculate and is likely to be // used multiple times in a request. type cacheContext struct { - ctx context.Context - data map[any]map[any]any - lock sync.RWMutex + data map[any]map[any]any + lock sync.RWMutex + created time.Time + discard bool } func (cc *cacheContext) Get(tp, key any) any { cc.lock.RLock() defer cc.lock.RUnlock() - if cc.data[tp] == nil { - return nil - } return cc.data[tp][key] } func (cc *cacheContext) Put(tp, key, value any) { cc.lock.Lock() defer cc.lock.Unlock() - if cc.data[tp] == nil { - cc.data[tp] = make(map[any]any) + + if cc.discard { + return + } + + d := cc.data[tp] + if d == nil { + d = make(map[any]any) + cc.data[tp] = d } - cc.data[tp][key] = value + d[key] = value } func (cc *cacheContext) Delete(tp, key any) { cc.lock.Lock() defer cc.lock.Unlock() - if cc.data[tp] == nil { - return - } delete(cc.data[tp], key) } +func (cc *cacheContext) Discard() { + cc.lock.Lock() + defer cc.lock.Unlock() + cc.data = nil + cc.discard = true +} + +func (cc *cacheContext) isDiscard() bool { + cc.lock.RLock() + defer cc.lock.RUnlock() + return cc.discard +} + +// cacheContextLifetime is the max lifetime of cacheContext. +// Since cacheContext is used to cache data in a request level context, 10s is enough. +// If a cacheContext is used more than 10s, it's probably misuse. +const cacheContextLifetime = 10 * time.Second + +var timeNow = time.Now + +func (cc *cacheContext) Expired() bool { + return timeNow().Sub(cc.created) > cacheContextLifetime +} + var cacheContextKey = struct{}{} +/* +Since there are both WithCacheContext and WithNoCacheContext, +it may be confusing when there is nesting. + +Some cases to explain the design: + +When: +- A, B or C means a cache context. +- A', B' or C' means a discard cache context. +- ctx means context.Backgrand(). +- A(ctx) means a cache context with ctx as the parent context. +- B(A(ctx)) means a cache context with A(ctx) as the parent context. +- With is alias of WithCacheContext. +- WithNo is alias of WithNoCacheContext. + +So: +- With(ctx) -> A(ctx) +- With(With(ctx)) -> A(ctx), not B(A(ctx)), always reuse parent cache context if possible. +- With(With(With(ctx))) -> A(ctx), not C(B(A(ctx))), ditto. +- WithNo(ctx) -> ctx, not A'(ctx), don't create new cache context if we don't have to. +- WithNo(With(ctx)) -> A'(ctx) +- WithNo(WithNo(With(ctx))) -> A'(ctx), not B'(A'(ctx)), don't create new cache context if we don't have to. +- With(WithNo(With(ctx))) -> B(A'(ctx)), not A(ctx), never reuse a discard cache context. +- WithNo(With(WithNo(With(ctx)))) -> B'(A'(ctx)) +- With(WithNo(With(WithNo(With(ctx))))) -> C(B'(A'(ctx))), so there's always only one not-discard cache context. +*/ + func WithCacheContext(ctx context.Context) context.Context { + if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { + if !c.isDiscard() { + // reuse parent context + return ctx + } + } return context.WithValue(ctx, cacheContextKey, &cacheContext{ - ctx: ctx, - data: make(map[any]map[any]any), + data: make(map[any]map[any]any), + created: timeNow(), }) } +func WithNoCacheContext(ctx context.Context) context.Context { + if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { + // The caller want to run long-life tasks, but the parent context is a cache context. + // So we should disable and clean the cache data, or it will be kept in memory for a long time. + c.Discard() + return ctx + } + + return ctx +} + func GetContextData(ctx context.Context, tp, key any) any { if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { + if c.Expired() { + // The warning means that the cache context is misused for long-life task, + // it can be resolved with WithNoCacheContext(ctx). + log.Warn("cache context is expired, may be misused for long-life tasks: %v", c) + return nil + } return c.Get(tp, key) } - log.Warn("cannot get cache context when getting data: %v", ctx) return nil } func SetContextData(ctx context.Context, tp, key, value any) { if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { + if c.Expired() { + // The warning means that the cache context is misused for long-life task, + // it can be resolved with WithNoCacheContext(ctx). + log.Warn("cache context is expired, may be misused for long-life tasks: %v", c) + return + } c.Put(tp, key, value) return } - log.Warn("cannot get cache context when setting data: %v", ctx) } func RemoveContextData(ctx context.Context, tp, key any) { if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok { + if c.Expired() { + // The warning means that the cache context is misused for long-life task, + // it can be resolved with WithNoCacheContext(ctx). + log.Warn("cache context is expired, may be misused for long-life tasks: %v", c) + return + } c.Delete(tp, key) } } diff --git a/modules/cache/context_test.go b/modules/cache/context_test.go index 77e3ecad2..531554786 100644 --- a/modules/cache/context_test.go +++ b/modules/cache/context_test.go @@ -6,6 +6,7 @@ package cache import ( "context" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -25,7 +26,7 @@ func TestWithCacheContext(t *testing.T) { assert.EqualValues(t, 1, v.(int)) RemoveContextData(ctx, field, "my_config1") - RemoveContextData(ctx, field, "my_config2") // remove an non-exist key + RemoveContextData(ctx, field, "my_config2") // remove a non-exist key v = GetContextData(ctx, field, "my_config1") assert.Nil(t, v) @@ -38,4 +39,40 @@ func TestWithCacheContext(t *testing.T) { v = GetContextData(ctx, field, "my_config1") assert.EqualValues(t, 1, v) + + now := timeNow + defer func() { + timeNow = now + }() + timeNow = func() time.Time { + return now().Add(10 * time.Second) + } + v = GetContextData(ctx, field, "my_config1") + assert.Nil(t, v) +} + +func TestWithNoCacheContext(t *testing.T) { + ctx := context.Background() + + const field = "system_setting" + + v := GetContextData(ctx, field, "my_config1") + assert.Nil(t, v) + SetContextData(ctx, field, "my_config1", 1) + v = GetContextData(ctx, field, "my_config1") + assert.Nil(t, v) // still no cache + + ctx = WithCacheContext(ctx) + v = GetContextData(ctx, field, "my_config1") + assert.Nil(t, v) + SetContextData(ctx, field, "my_config1", 1) + v = GetContextData(ctx, field, "my_config1") + assert.NotNil(t, v) + + ctx = WithNoCacheContext(ctx) + v = GetContextData(ctx, field, "my_config1") + assert.Nil(t, v) + SetContextData(ctx, field, "my_config1", 1) + v = GetContextData(ctx, field, "my_config1") + assert.Nil(t, v) // still no cache } diff --git a/modules/context/access_log.go b/modules/context/access_log.go index 1aaba9dc2..515682b64 100644 --- a/modules/context/access_log.go +++ b/modules/context/access_log.go @@ -6,7 +6,9 @@ package context import ( "bytes" "context" + "fmt" "net/http" + "strings" "text/template" "time" @@ -20,13 +22,39 @@ type routerLoggerOptions struct { Start *time.Time ResponseWriter http.ResponseWriter Ctx map[string]interface{} + RequestID *string } var signedUserNameStringPointerKey interface{} = "signedUserNameStringPointerKey" +const keyOfRequestIDInTemplate = ".RequestID" + +// According to: +// TraceId: A valid trace identifier is a 16-byte array with at least one non-zero byte +// MD5 output is 16 or 32 bytes: md5-bytes is 16, md5-hex is 32 +// SHA1: similar, SHA1-bytes is 20, SHA1-hex is 40. +// UUID is 128-bit, 32 hex chars, 36 ASCII chars with 4 dashes +// So, we accept a Request ID with a maximum character length of 40 +const maxRequestIDByteLength = 40 + +func parseRequestIDFromRequestHeader(req *http.Request) string { + requestID := "-" + for _, key := range setting.Log.RequestIDHeaders { + if req.Header.Get(key) != "" { + requestID = req.Header.Get(key) + break + } + } + if len(requestID) > maxRequestIDByteLength { + requestID = fmt.Sprintf("%s...", requestID[:maxRequestIDByteLength]) + } + return requestID +} + // AccessLogger returns a middleware to log access logger func AccessLogger() func(http.Handler) http.Handler { logger := log.GetLogger("access") + needRequestID := len(setting.Log.RequestIDHeaders) > 0 && strings.Contains(setting.Log.AccessLogTemplate, keyOfRequestIDInTemplate) logTemplate, _ := template.New("log").Parse(setting.Log.AccessLogTemplate) return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -34,6 +62,11 @@ func AccessLogger() func(http.Handler) http.Handler { identity := "-" r := req.WithContext(context.WithValue(req.Context(), signedUserNameStringPointerKey, &identity)) + var requestID string + if needRequestID { + requestID = parseRequestIDFromRequestHeader(req) + } + next.ServeHTTP(w, r) rw := w.(ResponseWriter) @@ -47,6 +80,7 @@ func AccessLogger() func(http.Handler) http.Handler { "RemoteAddr": req.RemoteAddr, "Req": req, }, + RequestID: &requestID, }) if err != nil { log.Error("Could not set up chi access logger: %v", err.Error()) diff --git a/modules/context/api.go b/modules/context/api.go index 3f938948a..f7a338469 100644 --- a/modules/context/api.go +++ b/modules/context/api.go @@ -244,7 +244,7 @@ func APIContexter() func(http.Handler) http.Handler { } } - httpcache.AddCacheControlToHeader(ctx.Resp.Header(), 0, "no-transform") + httpcache.SetCacheControlInHeader(ctx.Resp.Header(), 0, "no-transform") ctx.Resp.Header().Set(`X-Frame-Options`, setting.CORSConfig.XFrameOptions) ctx.Data["Context"] = &ctx diff --git a/modules/context/context.go b/modules/context/context.go index 0c8d7411e..50c34edae 100644 --- a/modules/context/context.go +++ b/modules/context/context.go @@ -388,7 +388,7 @@ func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) { if duration == 0 { duration = 5 * time.Minute } - httpcache.AddCacheControlToHeader(header, duration) + httpcache.SetCacheControlInHeader(header, duration) if !opts.LastModified.IsZero() { header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat)) @@ -753,7 +753,7 @@ func Contexter(ctx context.Context) func(next http.Handler) http.Handler { } } - httpcache.AddCacheControlToHeader(ctx.Resp.Header(), 0, "no-transform") + httpcache.SetCacheControlInHeader(ctx.Resp.Header(), 0, "no-transform") ctx.Resp.Header().Set(`X-Frame-Options`, setting.CORSConfig.XFrameOptions) ctx.Data["CsrfToken"] = ctx.csrf.GetToken() diff --git a/modules/context/org.go b/modules/context/org.go index 0add7f2c0..39a3038f9 100644 --- a/modules/context/org.go +++ b/modules/context/org.go @@ -11,7 +11,6 @@ import ( "code.gitea.io/gitea/models/perm" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" ) @@ -31,29 +30,34 @@ type Organization struct { } func (org *Organization) CanWriteUnit(ctx *Context, unitType unit.Type) bool { - if ctx.Doer == nil { - return false - } - return org.UnitPermission(ctx, ctx.Doer.ID, unitType) >= perm.AccessModeWrite + return org.Organization.UnitPermission(ctx, ctx.Doer, unitType) >= perm.AccessModeWrite } -func (org *Organization) UnitPermission(ctx *Context, doerID int64, unitType unit.Type) perm.AccessMode { - if doerID > 0 { - teams, err := organization.GetUserOrgTeams(ctx, org.Organization.ID, doerID) - if err != nil { - log.Error("GetUserOrgTeams: %v", err) - return perm.AccessModeNone - } - if len(teams) > 0 { - return teams.UnitMaxAccess(unitType) - } - } +func (org *Organization) CanReadUnit(ctx *Context, unitType unit.Type) bool { + return org.Organization.UnitPermission(ctx, ctx.Doer, unitType) >= perm.AccessModeRead +} - if org.Organization.Visibility == structs.VisibleTypePublic { - return perm.AccessModeRead - } +func GetOrganizationByParams(ctx *Context) { + orgName := ctx.Params(":org") + + var err error - return perm.AccessModeNone + ctx.Org.Organization, err = organization.GetOrgByName(ctx, orgName) + if err != nil { + if organization.IsErrOrgNotExist(err) { + redirectUserID, err := user_model.LookupUserRedirect(orgName) + if err == nil { + RedirectToUser(ctx, orgName, redirectUserID) + } else if user_model.IsErrUserRedirectNotExist(err) { + ctx.NotFound("GetUserByName", err) + } else { + ctx.ServerError("LookupUserRedirect", err) + } + } else { + ctx.ServerError("GetUserByName", err) + } + return + } } // HandleOrgAssignment handles organization assignment @@ -77,25 +81,26 @@ func HandleOrgAssignment(ctx *Context, args ...bool) { requireTeamAdmin = args[3] } - orgName := ctx.Params(":org") - var err error - ctx.Org.Organization, err = organization.GetOrgByName(ctx, orgName) - if err != nil { - if organization.IsErrOrgNotExist(err) { - redirectUserID, err := user_model.LookupUserRedirect(orgName) - if err == nil { - RedirectToUser(ctx, orgName, redirectUserID) - } else if user_model.IsErrUserRedirectNotExist(err) { - ctx.NotFound("GetUserByName", err) - } else { - ctx.ServerError("LookupUserRedirect", err) + + if ctx.ContextUser == nil { + // if Organization is not defined, get it from params + if ctx.Org.Organization == nil { + GetOrganizationByParams(ctx) + if ctx.Written() { + return } - } else { - ctx.ServerError("GetUserByName", err) } + } else if ctx.ContextUser.IsOrganization() { + if ctx.Org == nil { + ctx.Org = &Organization{} + } + ctx.Org.Organization = (*organization.Organization)(ctx.ContextUser) + } else { + // ContextUser is an individual User return } + org := ctx.Org.Organization // Handle Visibility @@ -156,6 +161,7 @@ func HandleOrgAssignment(ctx *Context, args ...bool) { } ctx.Data["IsOrganizationOwner"] = ctx.Org.IsOwner ctx.Data["IsOrganizationMember"] = ctx.Org.IsMember + ctx.Data["IsProjectEnabled"] = true ctx.Data["IsPackageEnabled"] = setting.Packages.Enabled ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled ctx.Data["IsPublicMember"] = func(uid int64) bool { @@ -231,6 +237,10 @@ func HandleOrgAssignment(ctx *Context, args ...bool) { return } } + + ctx.Data["CanReadProjects"] = ctx.Org.CanReadUnit(ctx, unit.TypeProjects) + ctx.Data["CanReadPackages"] = ctx.Org.CanReadUnit(ctx, unit.TypePackages) + ctx.Data["CanReadCode"] = ctx.Org.CanReadUnit(ctx, unit.TypeCode) } // OrgAssignment returns a middleware to handle organization assignment diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index c627597fa..6b91a81fc 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -11,6 +11,7 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/markup" "github.com/stretchr/testify/assert" @@ -229,7 +230,10 @@ John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimiters`, } for n, c := range cases { - delimiter := determineDelimiter(&markup.RenderContext{RelativePath: c.filename}, []byte(decodeSlashes(t, c.csv))) + delimiter := determineDelimiter(&markup.RenderContext{ + Ctx: git.DefaultContext, + RelativePath: c.filename, + }, []byte(decodeSlashes(t, c.csv))) assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } diff --git a/modules/doctor/dbconsistency.go b/modules/doctor/dbconsistency.go index bb560ac6a..541fc736f 100644 --- a/modules/doctor/dbconsistency.go +++ b/modules/doctor/dbconsistency.go @@ -155,7 +155,7 @@ func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) er // TODO: function to recalc all counters - if setting.Database.UsePostgreSQL { + if setting.Database.Type.IsPostgreSQL() { consistencyChecks = append(consistencyChecks, consistencyCheck{ Name: "Sequence values", Counter: db.CountBadSequences, diff --git a/modules/git/command.go b/modules/git/command.go index 0bc810311..9a65279a8 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -179,7 +179,7 @@ func (c *Command) AddDashesAndList(list ...string) *Command { } // ToTrustedCmdArgs converts a list of strings (trusted as argument) to TrustedCmdArgs -// In most cases, it shouldn't be used. Use AddXxx function instead +// In most cases, it shouldn't be used. Use NewCommand().AddXxx() function instead func ToTrustedCmdArgs(args []string) TrustedCmdArgs { ret := make(TrustedCmdArgs, len(args)) for i, arg := range args { diff --git a/modules/git/commit.go b/modules/git/commit.go index 4a55645d3..6e8fcb3e0 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -218,6 +218,19 @@ func (c *Commit) HasPreviousCommit(commitHash SHA1) (bool, error) { return false, err } +// IsForcePush returns true if a push from oldCommitHash to this is a force push +func (c *Commit) IsForcePush(oldCommitID string) (bool, error) { + if oldCommitID == EmptySHA { + return false, nil + } + oldCommit, err := c.repo.GetCommit(oldCommitID) + if err != nil { + return false, err + } + hasPreviousCommit, err := c.HasPreviousCommit(oldCommit.ID) + return !hasPreviousCommit, err +} + // CommitsBeforeLimit returns num commits before current revision func (c *Commit) CommitsBeforeLimit(num int) ([]*Commit, error) { return c.repo.getCommitsBeforeLimit(c.ID, num) diff --git a/modules/git/git.go b/modules/git/git.go index 2feb242ac..24cfea8c7 100644 --- a/modules/git/git.go +++ b/modules/git/git.go @@ -312,7 +312,7 @@ func CheckGitVersionAtLeast(atLeast string) error { } func configSet(key, value string) error { - stdout, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key).RunStdString(nil) + stdout, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil) if err != nil && !err.IsExitCode(1) { return fmt.Errorf("failed to get git config %s, err: %w", key, err) } @@ -331,7 +331,7 @@ func configSet(key, value string) error { } func configSetNonExist(key, value string) error { - _, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key).RunStdString(nil) + _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil) if err == nil { // already exist return nil @@ -349,7 +349,7 @@ func configSetNonExist(key, value string) error { } func configAddNonExist(key, value string) error { - _, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil) + _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil) if err == nil { // already exist return nil @@ -366,7 +366,7 @@ func configAddNonExist(key, value string) error { } func configUnsetAll(key, value string) error { - _, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key).RunStdString(nil) + _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil) if err == nil { // exist, need to remove _, _, err = NewCommand(DefaultContext, "config", "--global", "--unset-all").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil) diff --git a/modules/git/pipeline/revlist.go b/modules/git/pipeline/revlist.go index 09bb2c8b3..d88ebe78e 100644 --- a/modules/git/pipeline/revlist.go +++ b/modules/git/pipeline/revlist.go @@ -42,7 +42,10 @@ func RevListObjects(ctx context.Context, revListWriter *io.PipeWriter, wg *sync. defer revListWriter.Close() stderr := new(bytes.Buffer) var errbuf strings.Builder - cmd := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(headSHA).AddArguments("--not").AddDynamicArguments(baseSHA) + cmd := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(headSHA) + if baseSHA != "" { + cmd = cmd.AddArguments("--not").AddDynamicArguments(baseSHA) + } if err := cmd.Run(&git.RunOpts{ Dir: tmpBasePath, Stdout: revListWriter, diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index a62e0670f..0e1b00ce0 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -323,6 +323,27 @@ func (repo *Repository) CommitsBetweenLimit(last, before *Commit, limit, skip in return repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout)) } +// CommitsBetweenNotBase returns a list that contains commits between [before, last), excluding commits in baseBranch. +// If before is detached (removed by reset + push) it is not included. +func (repo *Repository) CommitsBetweenNotBase(last, before *Commit, baseBranch string) ([]*Commit, error) { + var stdout []byte + var err error + if before == nil { + stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path}) + } else { + stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String()+".."+last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path}) + if err != nil && strings.Contains(err.Error(), "no merge base") { + // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. + // previously it would return the results of git rev-list before last so let's try that... + stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String(), last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path}) + } + } + if err != nil { + return nil, err + } + return repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout)) +} + // CommitsBetweenIDs return commits between twoe commits func (repo *Repository) CommitsBetweenIDs(last, before string) ([]*Commit, error) { lastCommit, err := repo.GetCommit(last) diff --git a/modules/git/repo_commit_gogit.go b/modules/git/repo_commit_gogit.go index 72de158e6..ce0af9361 100644 --- a/modules/git/repo_commit_gogit.go +++ b/modules/git/repo_commit_gogit.go @@ -7,7 +7,6 @@ package git import ( - "fmt" "strings" "github.com/go-git/go-git/v5/plumbing" @@ -67,38 +66,6 @@ func (repo *Repository) IsCommitExist(name string) bool { return err == nil } -func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature { - if t.PGPSignature == "" { - return nil - } - - var w strings.Builder - var err error - - if _, err = fmt.Fprintf(&w, - "object %s\ntype %s\ntag %s\ntagger ", - t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil { - return nil - } - - if err = t.Tagger.Encode(&w); err != nil { - return nil - } - - if _, err = fmt.Fprintf(&w, "\n\n"); err != nil { - return nil - } - - if _, err = fmt.Fprintf(&w, t.Message); err != nil { - return nil - } - - return &CommitGPGSignature{ - Signature: t.PGPSignature, - Payload: strings.TrimSpace(w.String()) + "\n", - } -} - func (repo *Repository) getCommit(id SHA1) (*Commit, error) { var tagObject *object.Tag @@ -122,12 +89,6 @@ func (repo *Repository) getCommit(id SHA1) (*Commit, error) { commit := convertCommit(gogitCommit) commit.repo = repo - if tagObject != nil { - commit.CommitMessage = strings.TrimSpace(tagObject.Message) - commit.Author = &tagObject.Tagger - commit.Signature = convertPGPSignatureForTag(tagObject) - } - tree, err := gogitCommit.Tree() if err != nil { return nil, err diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go index 7373d01c8..d5eb72310 100644 --- a/modules/git/repo_commit_nogogit.go +++ b/modules/git/repo_commit_nogogit.go @@ -107,10 +107,6 @@ func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id SHA1) (*Co return nil, err } - commit.CommitMessage = strings.TrimSpace(tag.Message) - commit.Author = tag.Tagger - commit.Signature = tag.Signature - return commit, nil case "commit": commit, err := CommitFromReader(repo, id, io.LimitReader(rd, size)) diff --git a/modules/git/repo_commit_test.go b/modules/git/repo_commit_test.go index af8c0592f..729fb0ba1 100644 --- a/modules/git/repo_commit_test.go +++ b/modules/git/repo_commit_test.go @@ -43,12 +43,13 @@ func TestGetTagCommitWithSignature(t *testing.T) { assert.NoError(t, err) defer bareRepo1.Close() - commit, err := bareRepo1.GetCommit("3ad28a9149a2864384548f3d17ed7f38014c9e8a") + // both the tag and the commit are signed here, this validates only the commit signature + commit, err := bareRepo1.GetCommit("28b55526e7100924d864dd89e35c1ea62e7a5a32") assert.NoError(t, err) assert.NotNil(t, commit) assert.NotNil(t, commit.Signature) // test that signature is not in message - assert.Equal(t, "tag", commit.CommitMessage) + assert.Equal(t, "signed-commit\n", commit.CommitMessage) } func TestGetCommitWithBadCommitID(t *testing.T) { diff --git a/modules/git/repo_compare.go b/modules/git/repo_compare.go index 9a4d66f2f..439455e3c 100644 --- a/modules/git/repo_compare.go +++ b/modules/git/repo_compare.go @@ -277,11 +277,18 @@ func (repo *Repository) GetPatch(base, head string, w io.Writer) error { // GetFilesChangedBetween returns a list of all files that have been changed between the given commits func (repo *Repository) GetFilesChangedBetween(base, head string) ([]string, error) { - stdout, _, err := NewCommand(repo.Ctx, "diff", "--name-only").AddDynamicArguments(base + ".." + head).RunStdString(&RunOpts{Dir: repo.Path}) + stdout, _, err := NewCommand(repo.Ctx, "diff", "--name-only", "-z").AddDynamicArguments(base + ".." + head).RunStdString(&RunOpts{Dir: repo.Path}) if err != nil { return nil, err } - return strings.Split(stdout, "\n"), err + split := strings.Split(stdout, "\000") + + // Because Git will always emit filenames with a terminal NUL ignore the last entry in the split - which will always be empty. + if len(split) > 0 { + split = split[:len(split)-1] + } + + return split, err } // GetDiffFromMergeBase generates and return patch data from merge base to head diff --git a/modules/git/repo_ref_test.go b/modules/git/repo_ref_test.go index 776d7ce3e..c08ea1276 100644 --- a/modules/git/repo_ref_test.go +++ b/modules/git/repo_ref_test.go @@ -19,13 +19,14 @@ func TestRepository_GetRefs(t *testing.T) { refs, err := bareRepo1.GetRefs() assert.NoError(t, err) - assert.Len(t, refs, 5) + assert.Len(t, refs, 6) expectedRefs := []string{ BranchPrefix + "branch1", BranchPrefix + "branch2", BranchPrefix + "master", TagPrefix + "test", + TagPrefix + "signed-tag", NotesRef, } @@ -43,9 +44,12 @@ func TestRepository_GetRefsFiltered(t *testing.T) { refs, err := bareRepo1.GetRefsFiltered(TagPrefix) assert.NoError(t, err) - if assert.Len(t, refs, 1) { - assert.Equal(t, TagPrefix+"test", refs[0].Name) + if assert.Len(t, refs, 2) { + assert.Equal(t, TagPrefix+"signed-tag", refs[0].Name) assert.Equal(t, "tag", refs[0].Type) - assert.Equal(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", refs[0].Object.String()) + assert.Equal(t, "36f97d9a96457e2bab511db30fe2db03893ebc64", refs[0].Object.String()) + assert.Equal(t, TagPrefix+"test", refs[1].Name) + assert.Equal(t, "tag", refs[1].Type) + assert.Equal(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", refs[1].Object.String()) } } diff --git a/modules/git/repo_stats_test.go b/modules/git/repo_stats_test.go index 668ed6799..3d032385e 100644 --- a/modules/git/repo_stats_test.go +++ b/modules/git/repo_stats_test.go @@ -24,9 +24,9 @@ func TestRepository_GetCodeActivityStats(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, code) - assert.EqualValues(t, 9, code.CommitCount) + assert.EqualValues(t, 10, code.CommitCount) assert.EqualValues(t, 3, code.AuthorCount) - assert.EqualValues(t, 9, code.CommitCountInAllBranches) + assert.EqualValues(t, 10, code.CommitCountInAllBranches) assert.EqualValues(t, 10, code.Additions) assert.EqualValues(t, 1, code.Deletions) assert.Len(t, code.Authors, 3) diff --git a/modules/git/repo_tag_test.go b/modules/git/repo_tag_test.go index 589349a72..8ccfec3eb 100644 --- a/modules/git/repo_tag_test.go +++ b/modules/git/repo_tag_test.go @@ -25,11 +25,14 @@ func TestRepository_GetTags(t *testing.T) { assert.NoError(t, err) return } - assert.Len(t, tags, 1) + assert.Len(t, tags, 2) assert.Equal(t, len(tags), total) - assert.EqualValues(t, "test", tags[0].Name) - assert.EqualValues(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", tags[0].ID.String()) + assert.EqualValues(t, "signed-tag", tags[0].Name) + assert.EqualValues(t, "36f97d9a96457e2bab511db30fe2db03893ebc64", tags[0].ID.String()) assert.EqualValues(t, "tag", tags[0].Type) + assert.EqualValues(t, "test", tags[1].Name) + assert.EqualValues(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", tags[1].ID.String()) + assert.EqualValues(t, "tag", tags[1].Type) } func TestRepository_GetTag(t *testing.T) { diff --git a/modules/git/repo_test.go b/modules/git/repo_test.go index 2a3914819..044b9d406 100644 --- a/modules/git/repo_test.go +++ b/modules/git/repo_test.go @@ -14,10 +14,10 @@ func TestGetLatestCommitTime(t *testing.T) { bareRepo1Path := filepath.Join(testReposDir, "repo1_bare") lct, err := GetLatestCommitTime(DefaultContext, bareRepo1Path) assert.NoError(t, err) - // Time is Sun Jul 21 22:43:13 2019 +0200 + // Time is Sun Nov 13 16:40:14 2022 +0100 // which is the time of commit - // feaf4ba6bc635fec442f46ddd4512416ec43c2c2 (refs/heads/master) - assert.EqualValues(t, 1563741793, lct.Unix()) + // ce064814f4a0d337b333e646ece456cd39fab612 (refs/heads/master) + assert.EqualValues(t, 1668354014, lct.Unix()) } func TestRepoIsEmpty(t *testing.T) { diff --git a/modules/git/tests/repos/repo1_bare/index b/modules/git/tests/repos/repo1_bare/index Binary files differnew file mode 100644 index 000000000..65d675154 --- /dev/null +++ b/modules/git/tests/repos/repo1_bare/index diff --git a/modules/git/tests/repos/repo1_bare/logs/HEAD b/modules/git/tests/repos/repo1_bare/logs/HEAD index cef4ca2dc..46da5fe0b 100644 --- a/modules/git/tests/repos/repo1_bare/logs/HEAD +++ b/modules/git/tests/repos/repo1_bare/logs/HEAD @@ -1 +1,2 @@ 37991dec2c8e592043f47155ce4808d4580f9123 feaf4ba6bc635fec442f46ddd4512416ec43c2c2 silverwind <me@silverwind.io> 1563741799 +0200 push +feaf4ba6bc635fec442f46ddd4512416ec43c2c2 ce064814f4a0d337b333e646ece456cd39fab612 silverwind <me@silverwind.io> 1668354026 +0100 push diff --git a/modules/git/tests/repos/repo1_bare/logs/refs/heads/master b/modules/git/tests/repos/repo1_bare/logs/refs/heads/master index cef4ca2dc..46da5fe0b 100644 --- a/modules/git/tests/repos/repo1_bare/logs/refs/heads/master +++ b/modules/git/tests/repos/repo1_bare/logs/refs/heads/master @@ -1 +1,2 @@ 37991dec2c8e592043f47155ce4808d4580f9123 feaf4ba6bc635fec442f46ddd4512416ec43c2c2 silverwind <me@silverwind.io> 1563741799 +0200 push +feaf4ba6bc635fec442f46ddd4512416ec43c2c2 ce064814f4a0d337b333e646ece456cd39fab612 silverwind <me@silverwind.io> 1668354026 +0100 push diff --git a/modules/git/tests/repos/repo1_bare/objects/1c/91d130dc5fb75fd2d9f586a058650889cfe7fb b/modules/git/tests/repos/repo1_bare/objects/1c/91d130dc5fb75fd2d9f586a058650889cfe7fb Binary files differnew file mode 100644 index 000000000..fb50b65f9 --- /dev/null +++ b/modules/git/tests/repos/repo1_bare/objects/1c/91d130dc5fb75fd2d9f586a058650889cfe7fb diff --git a/modules/git/tests/repos/repo1_bare/objects/28/b55526e7100924d864dd89e35c1ea62e7a5a32 b/modules/git/tests/repos/repo1_bare/objects/28/b55526e7100924d864dd89e35c1ea62e7a5a32 Binary files differnew file mode 100644 index 000000000..7779599a0 --- /dev/null +++ b/modules/git/tests/repos/repo1_bare/objects/28/b55526e7100924d864dd89e35c1ea62e7a5a32 diff --git a/modules/git/tests/repos/repo1_bare/objects/36/f97d9a96457e2bab511db30fe2db03893ebc64 b/modules/git/tests/repos/repo1_bare/objects/36/f97d9a96457e2bab511db30fe2db03893ebc64 Binary files differnew file mode 100644 index 000000000..c96b84390 --- /dev/null +++ b/modules/git/tests/repos/repo1_bare/objects/36/f97d9a96457e2bab511db30fe2db03893ebc64 diff --git a/modules/git/tests/repos/repo1_bare/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 b/modules/git/tests/repos/repo1_bare/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 Binary files differnew file mode 100644 index 000000000..adf64119a --- /dev/null +++ b/modules/git/tests/repos/repo1_bare/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 diff --git a/modules/git/tests/repos/repo1_bare/objects/93/3305878a3c9ad485c29b87fb662a73a9675c4b b/modules/git/tests/repos/repo1_bare/objects/93/3305878a3c9ad485c29b87fb662a73a9675c4b Binary files differnew file mode 100644 index 000000000..e198e7658 --- /dev/null +++ b/modules/git/tests/repos/repo1_bare/objects/93/3305878a3c9ad485c29b87fb662a73a9675c4b diff --git a/modules/git/tests/repos/repo1_bare/objects/ce/064814f4a0d337b333e646ece456cd39fab612 b/modules/git/tests/repos/repo1_bare/objects/ce/064814f4a0d337b333e646ece456cd39fab612 Binary files differnew file mode 100644 index 000000000..93f1525e5 --- /dev/null +++ b/modules/git/tests/repos/repo1_bare/objects/ce/064814f4a0d337b333e646ece456cd39fab612 diff --git a/modules/git/tests/repos/repo1_bare/objects/cf/8b0b492a950b358a7ce7f9d01b18aef48a6b2d b/modules/git/tests/repos/repo1_bare/objects/cf/8b0b492a950b358a7ce7f9d01b18aef48a6b2d Binary files differnew file mode 100644 index 000000000..1152b25bb --- /dev/null +++ b/modules/git/tests/repos/repo1_bare/objects/cf/8b0b492a950b358a7ce7f9d01b18aef48a6b2d diff --git a/modules/git/tests/repos/repo1_bare/refs/heads/master b/modules/git/tests/repos/repo1_bare/refs/heads/master index c5e92eb22..9b0de2281 100644 --- a/modules/git/tests/repos/repo1_bare/refs/heads/master +++ b/modules/git/tests/repos/repo1_bare/refs/heads/master @@ -1 +1 @@ -feaf4ba6bc635fec442f46ddd4512416ec43c2c2 +ce064814f4a0d337b333e646ece456cd39fab612 diff --git a/modules/git/tests/repos/repo1_bare/refs/tags/signed-tag b/modules/git/tests/repos/repo1_bare/refs/tags/signed-tag new file mode 100644 index 000000000..3998a6850 --- /dev/null +++ b/modules/git/tests/repos/repo1_bare/refs/tags/signed-tag @@ -0,0 +1 @@ +36f97d9a96457e2bab511db30fe2db03893ebc64 diff --git a/modules/httpcache/httpcache.go b/modules/httpcache/httpcache.go index f0caa30eb..46e0152ef 100644 --- a/modules/httpcache/httpcache.go +++ b/modules/httpcache/httpcache.go @@ -15,8 +15,8 @@ import ( "code.gitea.io/gitea/modules/setting" ) -// AddCacheControlToHeader adds suitable cache-control headers to response -func AddCacheControlToHeader(h http.Header, maxAge time.Duration, additionalDirectives ...string) { +// SetCacheControlInHeader sets suitable cache-control headers in the response +func SetCacheControlInHeader(h http.Header, maxAge time.Duration, additionalDirectives ...string) { directives := make([]string, 0, 2+len(additionalDirectives)) // "max-age=0 + must-revalidate" (aka "no-cache") is preferred instead of "no-store" @@ -31,7 +31,7 @@ func AddCacheControlToHeader(h http.Header, maxAge time.Duration, additionalDire directives = append(directives, "max-age=0", "private", "must-revalidate") // to remind users they are using non-prod setting. - h.Add("X-Gitea-Debug", "RUN_MODE="+setting.RunMode) + h.Set("X-Gitea-Debug", "RUN_MODE="+setting.RunMode) } h.Set("Cache-Control", strings.Join(append(directives, additionalDirectives...), ", ")) @@ -50,7 +50,7 @@ func HandleTimeCache(req *http.Request, w http.ResponseWriter, fi os.FileInfo) ( // HandleGenericTimeCache handles time-based caching for a HTTP request func HandleGenericTimeCache(req *http.Request, w http.ResponseWriter, lastModified time.Time) (handled bool) { - AddCacheControlToHeader(w.Header(), setting.StaticCacheTime) + SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) ifModifiedSince := req.Header.Get("If-Modified-Since") if ifModifiedSince != "" { @@ -81,7 +81,7 @@ func HandleGenericETagCache(req *http.Request, w http.ResponseWriter, etag strin return true } } - AddCacheControlToHeader(w.Header(), setting.StaticCacheTime) + SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) return false } @@ -125,6 +125,6 @@ func HandleGenericETagTimeCache(req *http.Request, w http.ResponseWriter, etag s } } } - AddCacheControlToHeader(w.Header(), setting.StaticCacheTime) + SetCacheControlInHeader(w.Header(), setting.StaticCacheTime) return false } diff --git a/modules/label/label.go b/modules/label/label.go new file mode 100644 index 000000000..d3ef0e1dc --- /dev/null +++ b/modules/label/label.go @@ -0,0 +1,46 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package label + +import ( + "fmt" + "regexp" + "strings" +) + +// colorPattern is a regexp which can validate label color +var colorPattern = regexp.MustCompile("^#?(?:[0-9a-fA-F]{6}|[0-9a-fA-F]{3})$") + +// Label represents label information loaded from template +type Label struct { + Name string `yaml:"name"` + Color string `yaml:"color"` + Description string `yaml:"description,omitempty"` + Exclusive bool `yaml:"exclusive,omitempty"` +} + +// NormalizeColor normalizes a color string to a 6-character hex code +func NormalizeColor(color string) (string, error) { + // normalize case + color = strings.TrimSpace(strings.ToLower(color)) + + // add leading hash + if len(color) == 6 || len(color) == 3 { + color = "#" + color + } + + if !colorPattern.MatchString(color) { + return "", fmt.Errorf("bad color code: %s", color) + } + + // convert 3-character shorthand into 6-character version + if len(color) == 4 { + r := color[1] + g := color[2] + b := color[3] + color = fmt.Sprintf("#%c%c%c%c%c%c", r, r, g, g, b, b) + } + + return color, nil +} diff --git a/modules/label/parser.go b/modules/label/parser.go new file mode 100644 index 000000000..55bf570de --- /dev/null +++ b/modules/label/parser.go @@ -0,0 +1,126 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package label + +import ( + "errors" + "fmt" + "strings" + + "code.gitea.io/gitea/modules/options" + + "gopkg.in/yaml.v3" +) + +type labelFile struct { + Labels []*Label `yaml:"labels"` +} + +// ErrTemplateLoad represents a "ErrTemplateLoad" kind of error. +type ErrTemplateLoad struct { + TemplateFile string + OriginalError error +} + +// IsErrTemplateLoad checks if an error is a ErrTemplateLoad. +func IsErrTemplateLoad(err error) bool { + _, ok := err.(ErrTemplateLoad) + return ok +} + +func (err ErrTemplateLoad) Error() string { + return fmt.Sprintf("Failed to load label template file '%s': %v", err.TemplateFile, err.OriginalError) +} + +// GetTemplateFile loads the label template file by given name, +// then parses and returns a list of name-color pairs and optionally description. +func GetTemplateFile(name string) ([]*Label, error) { + data, err := options.Labels(name + ".yaml") + if err == nil && len(data) > 0 { + return parseYamlFormat(name+".yaml", data) + } + + data, err = options.Labels(name + ".yml") + if err == nil && len(data) > 0 { + return parseYamlFormat(name+".yml", data) + } + + data, err = options.Labels(name) + if err != nil { + return nil, ErrTemplateLoad{name, fmt.Errorf("GetRepoInitFile: %w", err)} + } + + return parseLegacyFormat(name, data) +} + +func parseYamlFormat(name string, data []byte) ([]*Label, error) { + lf := &labelFile{} + + if err := yaml.Unmarshal(data, lf); err != nil { + return nil, err + } + + // Validate label data and fix colors + for _, l := range lf.Labels { + l.Color = strings.TrimSpace(l.Color) + if len(l.Name) == 0 || len(l.Color) == 0 { + return nil, ErrTemplateLoad{name, errors.New("label name and color are required fields")} + } + color, err := NormalizeColor(l.Color) + if err != nil { + return nil, ErrTemplateLoad{name, fmt.Errorf("bad HTML color code '%s' in label: %s", l.Color, l.Name)} + } + l.Color = color + } + + return lf.Labels, nil +} + +func parseLegacyFormat(name string, data []byte) ([]*Label, error) { + lines := strings.Split(string(data), "\n") + list := make([]*Label, 0, len(lines)) + for i := 0; i < len(lines); i++ { + line := strings.TrimSpace(lines[i]) + if len(line) == 0 { + continue + } + + parts, description, _ := strings.Cut(line, ";") + + color, name, ok := strings.Cut(parts, " ") + if !ok { + return nil, ErrTemplateLoad{name, fmt.Errorf("line is malformed: %s", line)} + } + + color, err := NormalizeColor(color) + if err != nil { + return nil, ErrTemplateLoad{name, fmt.Errorf("bad HTML color code '%s' in line: %s", color, line)} + } + + list = append(list, &Label{ + Name: strings.TrimSpace(name), + Color: color, + Description: strings.TrimSpace(description), + }) + } + + return list, nil +} + +// LoadFormatted loads the labels' list of a template file as a string separated by comma +func LoadFormatted(name string) (string, error) { + var buf strings.Builder + list, err := GetTemplateFile(name) + if err != nil { + return "", err + } + + for i := 0; i < len(list); i++ { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(list[i].Name) + } + return buf.String(), nil +} diff --git a/modules/label/parser_test.go b/modules/label/parser_test.go new file mode 100644 index 000000000..5c8042f66 --- /dev/null +++ b/modules/label/parser_test.go @@ -0,0 +1,72 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package label + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestYamlParser(t *testing.T) { + data := []byte(`labels: + - name: priority/low + exclusive: true + color: "#0000ee" + description: "Low priority" + - name: priority/medium + exclusive: true + color: "0e0" + description: "Medium priority" + - name: priority/high + exclusive: true + color: "#ee0000" + description: "High priority" + - name: type/bug + color: "#f00" + description: "Bug"`) + + labels, err := parseYamlFormat("test", data) + require.NoError(t, err) + require.Len(t, labels, 4) + assert.Equal(t, "priority/low", labels[0].Name) + assert.True(t, labels[0].Exclusive) + assert.Equal(t, "#0000ee", labels[0].Color) + assert.Equal(t, "Low priority", labels[0].Description) + assert.Equal(t, "priority/medium", labels[1].Name) + assert.True(t, labels[1].Exclusive) + assert.Equal(t, "#00ee00", labels[1].Color) + assert.Equal(t, "Medium priority", labels[1].Description) + assert.Equal(t, "priority/high", labels[2].Name) + assert.True(t, labels[2].Exclusive) + assert.Equal(t, "#ee0000", labels[2].Color) + assert.Equal(t, "High priority", labels[2].Description) + assert.Equal(t, "type/bug", labels[3].Name) + assert.False(t, labels[3].Exclusive) + assert.Equal(t, "#ff0000", labels[3].Color) + assert.Equal(t, "Bug", labels[3].Description) +} + +func TestLegacyParser(t *testing.T) { + data := []byte(`#ee0701 bug ; Something is not working +#cccccc duplicate ; This issue or pull request already exists +#84b6eb enhancement`) + + labels, err := parseLegacyFormat("test", data) + require.NoError(t, err) + require.Len(t, labels, 3) + assert.Equal(t, "bug", labels[0].Name) + assert.False(t, labels[0].Exclusive) + assert.Equal(t, "#ee0701", labels[0].Color) + assert.Equal(t, "Something is not working", labels[0].Description) + assert.Equal(t, "duplicate", labels[1].Name) + assert.False(t, labels[1].Exclusive) + assert.Equal(t, "#cccccc", labels[1].Color) + assert.Equal(t, "This issue or pull request already exists", labels[1].Description) + assert.Equal(t, "enhancement", labels[2].Name) + assert.False(t, labels[2].Exclusive) + assert.Equal(t, "#84b6eb", labels[2].Color) + assert.Empty(t, labels[2].Description) +} diff --git a/modules/markup/console/console_test.go b/modules/markup/console/console_test.go index 506f86194..2337d91ac 100644 --- a/modules/markup/console/console_test.go +++ b/modules/markup/console/console_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/markup" "github.com/stretchr/testify/assert" @@ -23,7 +24,8 @@ func TestRenderConsole(t *testing.T) { canRender := render.CanRender("test", strings.NewReader(k)) assert.True(t, canRender) - err := render.Render(&markup.RenderContext{}, strings.NewReader(k), &buf) + err := render.Render(&markup.RenderContext{Ctx: git.DefaultContext}, + strings.NewReader(k), &buf) assert.NoError(t, err) assert.EqualValues(t, v, buf.String()) } diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go index b9f5769be..8c07184b2 100644 --- a/modules/markup/csv/csv_test.go +++ b/modules/markup/csv/csv_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/markup" "github.com/stretchr/testify/assert" @@ -23,7 +24,8 @@ func TestRenderCSV(t *testing.T) { for k, v := range kases { var buf strings.Builder - err := render.Render(&markup.RenderContext{}, strings.NewReader(k), &buf) + err := render.Render(&markup.RenderContext{Ctx: git.DefaultContext}, + strings.NewReader(k), &buf) assert.NoError(t, err) assert.EqualValues(t, v, buf.String()) } diff --git a/modules/markup/html.go b/modules/markup/html.go index bcb38f99e..76fc54cf4 100644 --- a/modules/markup/html.go +++ b/modules/markup/html.go @@ -291,9 +291,10 @@ func RenderDescriptionHTML( // RenderEmoji for when we want to just process emoji and shortcodes // in various places it isn't already run through the normal markdown processor func RenderEmoji( + ctx *RenderContext, content string, ) (string, error) { - return renderProcessString(&RenderContext{}, emojiProcessors, content) + return renderProcessString(ctx, emojiProcessors, content) } var ( diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go index 6746c40d1..a048f1f52 100644 --- a/modules/markup/html_internal_test.go +++ b/modules/markup/html_internal_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -70,8 +71,13 @@ var localMetas = map[string]string{ func TestRender_IssueIndexPattern(t *testing.T) { // numeric: render inputs without valid mentions test := func(s string) { - testRenderIssueIndexPattern(t, s, s, &RenderContext{}) - testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: numericMetas}) + testRenderIssueIndexPattern(t, s, s, &RenderContext{ + Ctx: git.DefaultContext, + }) + testRenderIssueIndexPattern(t, s, s, &RenderContext{ + Ctx: git.DefaultContext, + Metas: numericMetas, + }) } // should not render anything when there are no mentions @@ -119,7 +125,10 @@ func TestRender_IssueIndexPattern2(t *testing.T) { links[i] = numericIssueLink(util.URLJoin(TestRepoURL, path), "ref-issue", index, marker) } expectedNil := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expectedNil, &RenderContext{Metas: localMetas}) + testRenderIssueIndexPattern(t, s, expectedNil, &RenderContext{ + Ctx: git.DefaultContext, + Metas: localMetas, + }) class := "ref-issue" if isExternal { @@ -130,7 +139,10 @@ func TestRender_IssueIndexPattern2(t *testing.T) { links[i] = numericIssueLink(prefix, class, index, marker) } expectedNum := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expectedNum, &RenderContext{Metas: numericMetas}) + testRenderIssueIndexPattern(t, s, expectedNum, &RenderContext{ + Ctx: git.DefaultContext, + Metas: numericMetas, + }) } // should render freestanding mentions @@ -164,7 +176,10 @@ func TestRender_IssueIndexPattern3(t *testing.T) { // alphanumeric: render inputs without valid mentions test := func(s string) { - testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: alphanumericMetas}) + testRenderIssueIndexPattern(t, s, s, &RenderContext{ + Ctx: git.DefaultContext, + Metas: alphanumericMetas, + }) } test("") test("this is a test") @@ -194,7 +209,10 @@ func TestRender_IssueIndexPattern4(t *testing.T) { links[i] = externalIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name) } expected := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas}) + testRenderIssueIndexPattern(t, s, expected, &RenderContext{ + Ctx: git.DefaultContext, + Metas: alphanumericMetas, + }) } test("OTT-1234 test", "%s test", "OTT-1234") test("test T-12 issue", "test %s issue", "T-12") @@ -214,7 +232,10 @@ func TestRender_IssueIndexPattern5(t *testing.T) { } expected := fmt.Sprintf(expectedFmt, links...) - testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: metas}) + testRenderIssueIndexPattern(t, s, expected, &RenderContext{ + Ctx: git.DefaultContext, + Metas: metas, + }) } test("abc ISSUE-123 def", "abc %s def", @@ -235,7 +256,10 @@ func TestRender_IssueIndexPattern5(t *testing.T) { []string{"ISSUE-123"}, ) - testRenderIssueIndexPattern(t, "will not match", "will not match", &RenderContext{Metas: regexpMetas}) + testRenderIssueIndexPattern(t, "will not match", "will not match", &RenderContext{ + Ctx: git.DefaultContext, + Metas: regexpMetas, + }) } func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) { @@ -255,6 +279,7 @@ func TestRender_AutoLink(t *testing.T) { test := func(input, expected string) { var buffer strings.Builder err := PostProcess(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: TestRepoURL, Metas: localMetas, }, strings.NewReader(input), &buffer) @@ -263,6 +288,7 @@ func TestRender_AutoLink(t *testing.T) { buffer.Reset() err = PostProcess(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: TestRepoURL, Metas: localMetas, IsWiki: true, @@ -292,6 +318,7 @@ func TestRender_FullIssueURLs(t *testing.T) { test := func(input, expected string) { var result strings.Builder err := postProcess(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: TestRepoURL, Metas: localMetas, }, []processor{fullIssuePatternProcessor}, strings.NewReader(input), &result) diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index aea1d9267..f8b84485c 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -91,6 +91,7 @@ func TestRender_CrossReferences(t *testing.T) { test := func(input, expected string) { buffer, err := RenderString(&RenderContext{ + Ctx: git.DefaultContext, RelativePath: "a.md", URLPrefix: setting.AppSubURL, Metas: localMetas, @@ -135,6 +136,7 @@ func TestRender_links(t *testing.T) { test := func(input, expected string) { buffer, err := RenderString(&RenderContext{ + Ctx: git.DefaultContext, RelativePath: "a.md", URLPrefix: TestRepoURL, }, input) @@ -234,6 +236,7 @@ func TestRender_email(t *testing.T) { test := func(input, expected string) { res, err := RenderString(&RenderContext{ + Ctx: git.DefaultContext, RelativePath: "a.md", URLPrefix: TestRepoURL, }, input) @@ -292,6 +295,7 @@ func TestRender_emoji(t *testing.T) { test := func(input, expected string) { expected = strings.ReplaceAll(expected, "&", "&") buffer, err := RenderString(&RenderContext{ + Ctx: git.DefaultContext, RelativePath: "a.md", URLPrefix: TestRepoURL, }, input) @@ -355,11 +359,13 @@ func TestRender_ShortLinks(t *testing.T) { test := func(input, expected, expectedWiki string) { buffer, err := markdown.RenderString(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: tree, }, input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) buffer, err = markdown.RenderString(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: TestRepoURL, Metas: localMetas, IsWiki: true, @@ -461,12 +467,14 @@ func TestRender_RelativeImages(t *testing.T) { test := func(input, expected, expectedWiki string) { buffer, err := markdown.RenderString(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: tree, Metas: localMetas, }, input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) buffer, err = markdown.RenderString(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: TestRepoURL, Metas: localMetas, IsWiki: true, @@ -501,6 +509,7 @@ func Test_ParseClusterFuzz(t *testing.T) { var res strings.Builder err := PostProcess(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: "https://example.com", Metas: localMetas, }, strings.NewReader(data), &res) @@ -511,6 +520,7 @@ func Test_ParseClusterFuzz(t *testing.T) { res.Reset() err = PostProcess(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: "https://example.com", Metas: localMetas, }, strings.NewReader(data), &res) @@ -531,6 +541,7 @@ func TestIssue16020(t *testing.T) { var res strings.Builder err := PostProcess(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: "https://example.com", Metas: localMetas, }, strings.NewReader(data), &res) @@ -547,6 +558,7 @@ func BenchmarkEmojiPostprocess(b *testing.B) { for i := 0; i < b.N; i++ { var res strings.Builder err := PostProcess(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: "https://example.com", Metas: localMetas, }, strings.NewReader(data), &res) @@ -557,6 +569,7 @@ func BenchmarkEmojiPostprocess(b *testing.B) { func TestFuzz(t *testing.T) { s := "t/l/issues/8#/../../a" renderContext := RenderContext{ + Ctx: git.DefaultContext, URLPrefix: "https://example.com/go-gitea/gitea", Metas: map[string]string{ "user": "go-gitea", @@ -574,6 +587,7 @@ func TestIssue18471(t *testing.T) { var res strings.Builder err := PostProcess(&RenderContext{ + Ctx: git.DefaultContext, URLPrefix: "https://example.com", Metas: localMetas, }, strings.NewReader(data), &res) diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go index bb458a65c..0c7650a5f 100644 --- a/modules/markup/markdown/markdown_test.go +++ b/modules/markup/markdown/markdown_test.go @@ -52,12 +52,14 @@ func TestRender_StandardLinks(t *testing.T) { test := func(input, expected, expectedWiki string) { buffer, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, URLPrefix: setting.AppSubURL, }, input) assert.NoError(t, err) assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer)) buffer, err = RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, URLPrefix: setting.AppSubURL, IsWiki: true, }, input) @@ -81,6 +83,7 @@ func TestRender_Images(t *testing.T) { test := func(input, expected string) { buffer, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, URLPrefix: setting.AppSubURL, }, input) assert.NoError(t, err) @@ -311,6 +314,7 @@ func TestTotal_RenderWiki(t *testing.T) { for i := 0; i < len(testCases); i += 2 { line, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, URLPrefix: AppSubURL, IsWiki: true, }, testCases[i]) @@ -339,6 +343,7 @@ func TestTotal_RenderString(t *testing.T) { for i := 0; i < len(testCases); i += 2 { line, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, URLPrefix: AppSubURL, }, testCases[i]) assert.NoError(t, err) @@ -348,17 +353,17 @@ func TestTotal_RenderString(t *testing.T) { func TestRender_RenderParagraphs(t *testing.T) { test := func(t *testing.T, str string, cnt int) { - res, err := RenderRawString(&markup.RenderContext{}, str) + res, err := RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, str) assert.NoError(t, err) assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for unix should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) mac := strings.ReplaceAll(str, "\n", "\r") - res, err = RenderRawString(&markup.RenderContext{}, mac) + res, err = RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, mac) assert.NoError(t, err) assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for mac should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) dos := strings.ReplaceAll(str, "\n", "\r\n") - res, err = RenderRawString(&markup.RenderContext{}, dos) + res, err = RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, dos) assert.NoError(t, err) assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for windows should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res) } @@ -386,7 +391,7 @@ func TestMarkdownRenderRaw(t *testing.T) { for _, testcase := range testcases { log.Info("Test markdown render error with fuzzy data: %x, the following errors can be recovered", testcase) - _, err := RenderRawString(&markup.RenderContext{}, string(testcase)) + _, err := RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, string(testcase)) assert.NoError(t, err) } } @@ -398,7 +403,7 @@ func TestRenderSiblingImages_Issue12925(t *testing.T) { expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"></a><br> <a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"></a></p> ` - res, err := RenderRawString(&markup.RenderContext{}, testcase) + res, err := RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, testcase) assert.NoError(t, err) assert.Equal(t, expected, res) } @@ -407,7 +412,7 @@ func TestRenderEmojiInLinks_Issue12331(t *testing.T) { testcase := `[Link with emoji :moon: in text](https://gitea.io)` expected := `<p><a href="https://gitea.io" rel="nofollow">Link with emoji <span class="emoji" aria-label="waxing gibbous moon">🌔</span> in text</a></p> ` - res, err := RenderString(&markup.RenderContext{}, testcase) + res, err := RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, testcase) assert.NoError(t, err) assert.Equal(t, expected, res) } @@ -441,7 +446,7 @@ func TestColorPreview(t *testing.T) { } for _, test := range positiveTests { - res, err := RenderString(&markup.RenderContext{}, test.testcase) + res, err := RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test.testcase) assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) assert.Equal(t, test.expected, res, "Unexpected result in testcase %q", test.testcase) @@ -461,7 +466,7 @@ func TestColorPreview(t *testing.T) { } for _, test := range negativeTests { - res, err := RenderString(&markup.RenderContext{}, test) + res, err := RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test) assert.NoError(t, err, "Unexpected error in testcase: %q", test) assert.NotContains(t, res, `<span class="color-preview" style="background-color: `, "Unexpected result in testcase %q", test) } @@ -508,7 +513,7 @@ func TestMathBlock(t *testing.T) { } for _, test := range testcases { - res, err := RenderString(&markup.RenderContext{}, test.testcase) + res, err := RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test.testcase) assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase) assert.Equal(t, test.expected, res, "Unexpected result in testcase %q", test.testcase) diff --git a/modules/markup/orgmode/orgmode_test.go b/modules/markup/orgmode/orgmode_test.go index 4acb678cd..d6467c36f 100644 --- a/modules/markup/orgmode/orgmode_test.go +++ b/modules/markup/orgmode/orgmode_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" @@ -26,6 +27,7 @@ func TestRender_StandardLinks(t *testing.T) { test := func(input, expected string) { buffer, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, URLPrefix: setting.AppSubURL, }, input) assert.NoError(t, err) @@ -46,6 +48,7 @@ func TestRender_Images(t *testing.T) { test := func(input, expected string) { buffer, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, URLPrefix: setting.AppSubURL, }, input) assert.NoError(t, err) @@ -65,6 +68,7 @@ func TestRender_Source(t *testing.T) { test := func(input, expected string) { buffer, err := RenderString(&markup.RenderContext{ + Ctx: git.DefaultContext, URLPrefix: setting.AppSubURL, }, input) assert.NoError(t, err) diff --git a/modules/markup/sanitizer.go b/modules/markup/sanitizer.go index e59f6c7c8..600ccbf3c 100644 --- a/modules/markup/sanitizer.go +++ b/modules/markup/sanitizer.go @@ -132,6 +132,8 @@ func createDefaultPolicy() *bluemonday.Policy { policy.AllowAttrs(generalSafeAttrs...).OnElements(generalSafeElements...) + policy.AllowAttrs("src", "autoplay", "controls").OnElements("video") + policy.AllowAttrs("itemscope", "itemtype").OnElements("div") // FIXME: Need to handle longdesc in img but there is no easy way to do it diff --git a/modules/options/base.go b/modules/options/base.go index 039e934b3..e83e8df5d 100644 --- a/modules/options/base.go +++ b/modules/options/base.go @@ -7,11 +7,52 @@ import ( "fmt" "io/fs" "os" + "path" "path/filepath" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" ) +// Locale reads the content of a specific locale from static/bindata or custom path. +func Locale(name string) ([]byte, error) { + return fileFromDir(path.Join("locale", util.CleanPath(name))) +} + +// Readme reads the content of a specific readme from static/bindata or custom path. +func Readme(name string) ([]byte, error) { + return fileFromDir(path.Join("readme", util.CleanPath(name))) +} + +// Gitignore reads the content of a gitignore locale from static/bindata or custom path. +func Gitignore(name string) ([]byte, error) { + return fileFromDir(path.Join("gitignore", util.CleanPath(name))) +} + +// License reads the content of a specific license from static/bindata or custom path. +func License(name string) ([]byte, error) { + return fileFromDir(path.Join("license", util.CleanPath(name))) +} + +// Labels reads the content of a specific labels from static/bindata or custom path. +func Labels(name string) ([]byte, error) { + return fileFromDir(path.Join("label", util.CleanPath(name))) +} + +// WalkLocales reads the content of a specific locale +func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error { + if IsDynamic() { + if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to walk locales. Error: %w", err) + } + } + + if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to walk locales. Error: %w", err) + } + return nil +} + func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error { if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { // name is the path relative to the root @@ -37,3 +78,18 @@ func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, e } return nil } + +func statDirIfExist(dir string) ([]string, error) { + isDir, err := util.IsDir(dir) + if err != nil { + return nil, fmt.Errorf("unable to check if static directory %s is a directory. %w", dir, err) + } + if !isDir { + return nil, nil + } + files, err := util.StatDir(dir, true) + if err != nil { + return nil, fmt.Errorf("unable to read directory %q. %w", dir, err) + } + return files, nil +} diff --git a/modules/options/dynamic.go b/modules/options/dynamic.go index a20253676..8c954492a 100644 --- a/modules/options/dynamic.go +++ b/modules/options/dynamic.go @@ -7,10 +7,8 @@ package options import ( "fmt" - "io/fs" "os" "path" - "path/filepath" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -27,76 +25,20 @@ func Dir(name string) ([]string, error) { var result []string - customDir := path.Join(setting.CustomPath, "options", name) - - isDir, err := util.IsDir(customDir) - if err != nil { - return []string{}, fmt.Errorf("Unabe to check if custom directory %s is a directory. %w", customDir, err) - } - if isDir { - files, err := util.StatDir(customDir, true) - if err != nil { - return []string{}, fmt.Errorf("Failed to read custom directory. %w", err) - } - - result = append(result, files...) - } - - staticDir := path.Join(setting.StaticRootPath, "options", name) - - isDir, err = util.IsDir(staticDir) - if err != nil { - return []string{}, fmt.Errorf("unable to check if static directory %s is a directory. %w", staticDir, err) - } - if isDir { - files, err := util.StatDir(staticDir, true) + for _, dir := range []string{ + path.Join(setting.CustomPath, "options", name), // custom dir + path.Join(setting.StaticRootPath, "options", name), // static dir + } { + files, err := statDirIfExist(dir) if err != nil { - return []string{}, fmt.Errorf("Failed to read static directory. %w", err) + return nil, err } - result = append(result, files...) } return directories.AddAndGet(name, result), nil } -// Locale reads the content of a specific locale from static or custom path. -func Locale(name string) ([]byte, error) { - return fileFromDir(path.Join("locale", name)) -} - -// WalkLocales reads the content of a specific locale from static or custom path. -func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to walk locales. Error: %w", err) - } - - if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to walk locales. Error: %w", err) - } - return nil -} - -// Readme reads the content of a specific readme from static or custom path. -func Readme(name string) ([]byte, error) { - return fileFromDir(path.Join("readme", name)) -} - -// Gitignore reads the content of a specific gitignore from static or custom path. -func Gitignore(name string) ([]byte, error) { - return fileFromDir(path.Join("gitignore", name)) -} - -// License reads the content of a specific license from static or custom path. -func License(name string) ([]byte, error) { - return fileFromDir(path.Join("license", name)) -} - -// Labels reads the content of a specific labels from static or custom path. -func Labels(name string) ([]byte, error) { - return fileFromDir(path.Join("label", name)) -} - // fileFromDir is a helper to read files from static or custom path. func fileFromDir(name string) ([]byte, error) { customPath := path.Join(setting.CustomPath, "options", name) diff --git a/modules/options/static.go b/modules/options/static.go index ff3c86d3f..549f4e25b 100644 --- a/modules/options/static.go +++ b/modules/options/static.go @@ -8,10 +8,8 @@ package options import ( "fmt" "io" - "io/fs" "os" "path" - "path/filepath" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -28,17 +26,14 @@ func Dir(name string) ([]string, error) { var result []string - customDir := path.Join(setting.CustomPath, "options", name) - isDir, err := util.IsDir(customDir) - if err != nil { - return []string{}, fmt.Errorf("unable to check if custom directory %q is a directory. %w", customDir, err) - } - if isDir { - files, err := util.StatDir(customDir, true) + for _, dir := range []string{ + path.Join(setting.CustomPath, "options", name), // custom dir + // no static dir + } { + files, err := statDirIfExist(dir) if err != nil { - return []string{}, fmt.Errorf("unable to read custom directory %q. %w", customDir, err) + return nil, err } - result = append(result, files...) } @@ -69,39 +64,6 @@ func AssetDir(dirName string) ([]string, error) { return results, nil } -// Locale reads the content of a specific locale from bindata or custom path. -func Locale(name string) ([]byte, error) { - return fileFromDir(path.Join("locale", name)) -} - -// WalkLocales reads the content of a specific locale from static or custom path. -func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error { - if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) { - return fmt.Errorf("failed to walk locales. Error: %w", err) - } - return nil -} - -// Readme reads the content of a specific readme from bindata or custom path. -func Readme(name string) ([]byte, error) { - return fileFromDir(path.Join("readme", name)) -} - -// Gitignore reads the content of a gitignore locale from bindata or custom path. -func Gitignore(name string) ([]byte, error) { - return fileFromDir(path.Join("gitignore", name)) -} - -// License reads the content of a specific license from bindata or custom path. -func License(name string) ([]byte, error) { - return fileFromDir(path.Join("license", name)) -} - -// Labels reads the content of a specific labels from static or custom path. -func Labels(name string) ([]byte, error) { - return fileFromDir(path.Join("label", name)) -} - // fileFromDir is a helper to read files from bindata or custom path. func fileFromDir(name string) ([]byte, error) { customPath := path.Join(setting.CustomPath, "options", name) diff --git a/modules/public/public.go b/modules/public/public.go index 42026f9b1..e1d60d89e 100644 --- a/modules/public/public.go +++ b/modules/public/public.go @@ -6,7 +6,6 @@ package public import ( "net/http" "os" - "path" "path/filepath" "strings" @@ -14,6 +13,7 @@ import ( "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" ) // Options represents the available options to configure the handler. @@ -103,7 +103,7 @@ func setWellKnownContentType(w http.ResponseWriter, file string) { func (opts *Options) handle(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool { // use clean to keep the file is a valid path with no . or .. - f, err := fs.Open(path.Clean(file)) + f, err := fs.Open(util.CleanPath(file)) if err != nil { if os.IsNotExist(err) { return false diff --git a/modules/queue/queue_channel.go b/modules/queue/queue_channel.go index 6f75b8357..baac09739 100644 --- a/modules/queue/queue_channel.go +++ b/modules/queue/queue_channel.go @@ -124,7 +124,10 @@ func (q *ChannelQueue) Shutdown() { log.Trace("ChannelQueue: %s Flushing", q.name) // We can't use Cleanup here because that will close the channel if err := q.FlushWithContext(q.terminateCtx); err != nil { - log.Warn("ChannelQueue: %s Terminated before completed flushing", q.name) + count := atomic.LoadInt64(&q.numInQueue) + if count > 0 { + log.Warn("ChannelQueue: %s Terminated before completed flushing", q.name) + } return } log.Debug("ChannelQueue: %s Flushed", q.name) diff --git a/modules/queue/queue_disk_channel.go b/modules/queue/queue_disk_channel.go index c7526714c..91f91f0df 100644 --- a/modules/queue/queue_disk_channel.go +++ b/modules/queue/queue_disk_channel.go @@ -94,7 +94,8 @@ func NewPersistableChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) ( }, Workers: 0, }, - DataDir: config.DataDir, + DataDir: config.DataDir, + QueueName: config.Name + "-level", } levelQueue, err := NewLevelQueue(wrappedHandle, levelCfg, exemplar) @@ -172,16 +173,18 @@ func (q *PersistableChannelQueue) Run(atShutdown, atTerminate func(func())) { atShutdown(q.Shutdown) atTerminate(q.Terminate) - if lq, ok := q.internal.(*LevelQueue); ok && lq.byteFIFO.Len(lq.shutdownCtx) != 0 { + if lq, ok := q.internal.(*LevelQueue); ok && lq.byteFIFO.Len(lq.terminateCtx) != 0 { // Just run the level queue - we shut it down once it's flushed go q.internal.Run(func(_ func()) {}, func(_ func()) {}) go func() { - for !q.IsEmpty() { - _ = q.internal.Flush(0) + for !lq.IsEmpty() { + _ = lq.Flush(0) select { case <-time.After(100 * time.Millisecond): - case <-q.internal.(*LevelQueue).shutdownCtx.Done(): - log.Warn("LevelQueue: %s shut down before completely flushed", q.internal.(*LevelQueue).Name()) + case <-lq.shutdownCtx.Done(): + if lq.byteFIFO.Len(lq.terminateCtx) > 0 { + log.Warn("LevelQueue: %s shut down before completely flushed", q.internal.(*LevelQueue).Name()) + } return } } @@ -316,10 +319,22 @@ func (q *PersistableChannelQueue) Shutdown() { // Redirect all remaining data in the chan to the internal channel log.Trace("PersistableChannelQueue: %s Redirecting remaining data", q.delayedStarter.name) close(q.channelQueue.dataChan) + countOK, countLost := 0, 0 for data := range q.channelQueue.dataChan { - _ = q.internal.Push(data) + err := q.internal.Push(data) + if err != nil { + log.Error("PersistableChannelQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err) + countLost++ + } else { + countOK++ + } atomic.AddInt64(&q.channelQueue.numInQueue, -1) } + if countLost > 0 { + log.Warn("PersistableChannelQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost) + } else if countOK > 0 { + log.Warn("PersistableChannelQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK) + } log.Trace("PersistableChannelQueue: %s Done Redirecting remaining data", q.delayedStarter.name) log.Debug("PersistableChannelQueue: %s Shutdown", q.delayedStarter.name) diff --git a/modules/queue/queue_disk_channel_test.go b/modules/queue/queue_disk_channel_test.go index 318610355..4f14a5d79 100644 --- a/modules/queue/queue_disk_channel_test.go +++ b/modules/queue/queue_disk_channel_test.go @@ -39,7 +39,7 @@ func TestPersistableChannelQueue(t *testing.T) { Workers: 1, BoostWorkers: 0, MaxWorkers: 10, - Name: "first", + Name: "test-queue", }, &testData{}) assert.NoError(t, err) @@ -135,7 +135,7 @@ func TestPersistableChannelQueue(t *testing.T) { Workers: 1, BoostWorkers: 0, MaxWorkers: 10, - Name: "second", + Name: "test-queue", }, &testData{}) assert.NoError(t, err) @@ -227,7 +227,7 @@ func TestPersistableChannelQueue_Pause(t *testing.T) { Workers: 1, BoostWorkers: 0, MaxWorkers: 10, - Name: "first", + Name: "test-queue", }, &testData{}) assert.NoError(t, err) @@ -433,7 +433,7 @@ func TestPersistableChannelQueue_Pause(t *testing.T) { Workers: 1, BoostWorkers: 0, MaxWorkers: 10, - Name: "second", + Name: "test-queue", }, &testData{}) assert.NoError(t, err) pausable, ok = queue.(Pausable) diff --git a/modules/queue/unique_queue_channel.go b/modules/queue/unique_queue_channel.go index c43bd1db3..62c051aa3 100644 --- a/modules/queue/unique_queue_channel.go +++ b/modules/queue/unique_queue_channel.go @@ -177,7 +177,9 @@ func (q *ChannelUniqueQueue) Shutdown() { go func() { log.Trace("ChannelUniqueQueue: %s Flushing", q.name) if err := q.FlushWithContext(q.terminateCtx); err != nil { - log.Warn("ChannelUniqueQueue: %s Terminated before completed flushing", q.name) + if !q.IsEmpty() { + log.Warn("ChannelUniqueQueue: %s Terminated before completed flushing", q.name) + } return } log.Debug("ChannelUniqueQueue: %s Flushed", q.name) diff --git a/modules/queue/unique_queue_channel_test.go b/modules/queue/unique_queue_channel_test.go index 9372694b8..824015b83 100644 --- a/modules/queue/unique_queue_channel_test.go +++ b/modules/queue/unique_queue_channel_test.go @@ -8,10 +8,13 @@ import ( "testing" "time" + "code.gitea.io/gitea/modules/log" + "github.com/stretchr/testify/assert" ) func TestChannelUniqueQueue(t *testing.T) { + _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) handleChan := make(chan *testData) handle := func(data ...Data) []Data { for _, datum := range data { @@ -52,6 +55,8 @@ func TestChannelUniqueQueue(t *testing.T) { } func TestChannelUniqueQueue_Batch(t *testing.T) { + _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) + handleChan := make(chan *testData) handle := func(data ...Data) []Data { for _, datum := range data { @@ -98,6 +103,8 @@ func TestChannelUniqueQueue_Batch(t *testing.T) { } func TestChannelUniqueQueue_Pause(t *testing.T) { + _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) + lock := sync.Mutex{} var queue Queue var err error diff --git a/modules/queue/unique_queue_disk_channel.go b/modules/queue/unique_queue_disk_channel.go index 405726182..cc8a807c6 100644 --- a/modules/queue/unique_queue_disk_channel.go +++ b/modules/queue/unique_queue_disk_channel.go @@ -94,7 +94,8 @@ func NewPersistableChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interfac }, Workers: 0, }, - DataDir: config.DataDir, + DataDir: config.DataDir, + QueueName: config.Name + "-level", } queue.channelQueue = channelUniqueQueue.(*ChannelUniqueQueue) @@ -209,17 +210,29 @@ func (q *PersistableChannelUniqueQueue) Run(atShutdown, atTerminate func(func()) atTerminate(q.Terminate) _ = q.channelQueue.AddWorkers(q.channelQueue.workers, 0) - if luq, ok := q.internal.(*LevelUniqueQueue); ok && luq.ByteFIFOUniqueQueue.byteFIFO.Len(luq.shutdownCtx) != 0 { + if luq, ok := q.internal.(*LevelUniqueQueue); ok && !luq.IsEmpty() { // Just run the level queue - we shut it down once it's flushed - go q.internal.Run(func(_ func()) {}, func(_ func()) {}) + go luq.Run(func(_ func()) {}, func(_ func()) {}) go func() { - _ = q.internal.Flush(0) - log.Debug("LevelUniqueQueue: %s flushed so shutting down", q.internal.(*LevelUniqueQueue).Name()) - q.internal.(*LevelUniqueQueue).Shutdown() - GetManager().Remove(q.internal.(*LevelUniqueQueue).qid) + _ = luq.Flush(0) + for !luq.IsEmpty() { + _ = luq.Flush(0) + select { + case <-time.After(100 * time.Millisecond): + case <-luq.shutdownCtx.Done(): + if luq.byteFIFO.Len(luq.terminateCtx) > 0 { + log.Warn("LevelUniqueQueue: %s shut down before completely flushed", luq.Name()) + } + return + } + } + log.Debug("LevelUniqueQueue: %s flushed so shutting down", luq.Name()) + luq.Shutdown() + GetManager().Remove(luq.qid) }() } else { log.Debug("PersistableChannelUniqueQueue: %s Skipping running the empty level queue", q.delayedStarter.name) + _ = q.internal.Flush(0) q.internal.(*LevelUniqueQueue).Shutdown() GetManager().Remove(q.internal.(*LevelUniqueQueue).qid) } @@ -285,8 +298,20 @@ func (q *PersistableChannelUniqueQueue) Shutdown() { // Redirect all remaining data in the chan to the internal channel close(q.channelQueue.dataChan) log.Trace("PersistableChannelUniqueQueue: %s Redirecting remaining data", q.delayedStarter.name) + countOK, countLost := 0, 0 for data := range q.channelQueue.dataChan { - _ = q.internal.Push(data) + err := q.internal.(*LevelUniqueQueue).Push(data) + if err != nil { + log.Error("PersistableChannelUniqueQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err) + countLost++ + } else { + countOK++ + } + } + if countLost > 0 { + log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost) + } else if countOK > 0 { + log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK) } log.Trace("PersistableChannelUniqueQueue: %s Done Redirecting remaining data", q.delayedStarter.name) diff --git a/modules/queue/unique_queue_disk_channel_test.go b/modules/queue/unique_queue_disk_channel_test.go new file mode 100644 index 000000000..fd76163f4 --- /dev/null +++ b/modules/queue/unique_queue_disk_channel_test.go @@ -0,0 +1,259 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "fmt" + "strconv" + "sync" + "testing" + "time" + + "code.gitea.io/gitea/modules/log" + + "github.com/stretchr/testify/assert" +) + +func TestPersistableChannelUniqueQueue(t *testing.T) { + tmpDir := t.TempDir() + fmt.Printf("TempDir %s\n", tmpDir) + _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) + + // Common function to create the Queue + newQueue := func(name string, handle func(data ...Data) []Data) Queue { + q, err := NewPersistableChannelUniqueQueue(handle, + PersistableChannelUniqueQueueConfiguration{ + Name: name, + DataDir: tmpDir, + QueueLength: 200, + MaxWorkers: 1, + BlockTimeout: 1 * time.Second, + BoostTimeout: 5 * time.Minute, + BoostWorkers: 1, + Workers: 0, + }, "task-0") + assert.NoError(t, err) + return q + } + + // runs the provided queue and provides some timer function + type channels struct { + readyForShutdown chan struct{} // closed when shutdown functions have been assigned + readyForTerminate chan struct{} // closed when terminate functions have been assigned + signalShutdown chan struct{} // Should close to signal shutdown + doneShutdown chan struct{} // closed when shutdown function is done + queueTerminate []func() // list of atTerminate functions to call atTerminate - need to be accessed with lock + } + runQueue := func(q Queue, lock *sync.Mutex) *channels { + chans := &channels{ + readyForShutdown: make(chan struct{}), + readyForTerminate: make(chan struct{}), + signalShutdown: make(chan struct{}), + doneShutdown: make(chan struct{}), + } + go q.Run(func(atShutdown func()) { + go func() { + lock.Lock() + select { + case <-chans.readyForShutdown: + default: + close(chans.readyForShutdown) + } + lock.Unlock() + <-chans.signalShutdown + atShutdown() + close(chans.doneShutdown) + }() + }, func(atTerminate func()) { + lock.Lock() + defer lock.Unlock() + select { + case <-chans.readyForTerminate: + default: + close(chans.readyForTerminate) + } + chans.queueTerminate = append(chans.queueTerminate, atTerminate) + }) + + return chans + } + + // call to shutdown and terminate the queue associated with the channels + doTerminate := func(chans *channels, lock *sync.Mutex) { + <-chans.readyForTerminate + + lock.Lock() + callbacks := []func(){} + callbacks = append(callbacks, chans.queueTerminate...) + lock.Unlock() + + for _, callback := range callbacks { + callback() + } + } + + mapLock := sync.Mutex{} + executedInitial := map[string][]string{} + hasInitial := map[string][]string{} + + fillQueue := func(name string, done chan struct{}) { + t.Run("Initial Filling: "+name, func(t *testing.T) { + lock := sync.Mutex{} + + startAt100Queued := make(chan struct{}) + stopAt20Shutdown := make(chan struct{}) // stop and shutdown at the 20th item + + handle := func(data ...Data) []Data { + <-startAt100Queued + for _, datum := range data { + s := datum.(string) + mapLock.Lock() + executedInitial[name] = append(executedInitial[name], s) + mapLock.Unlock() + if s == "task-20" { + close(stopAt20Shutdown) + } + } + return nil + } + + q := newQueue(name, handle) + + // add 100 tasks to the queue + for i := 0; i < 100; i++ { + _ = q.Push("task-" + strconv.Itoa(i)) + } + close(startAt100Queued) + + chans := runQueue(q, &lock) + + <-chans.readyForShutdown + <-stopAt20Shutdown + close(chans.signalShutdown) + <-chans.doneShutdown + _ = q.Push("final") + + // check which tasks are still in the queue + for i := 0; i < 100; i++ { + if has, _ := q.(UniqueQueue).Has("task-" + strconv.Itoa(i)); has { + mapLock.Lock() + hasInitial[name] = append(hasInitial[name], "task-"+strconv.Itoa(i)) + mapLock.Unlock() + } + } + if has, _ := q.(UniqueQueue).Has("final"); has { + mapLock.Lock() + hasInitial[name] = append(hasInitial[name], "final") + mapLock.Unlock() + } else { + assert.Fail(t, "UnqueQueue %s should have \"final\"", name) + } + doTerminate(chans, &lock) + mapLock.Lock() + assert.Equal(t, 101, len(executedInitial[name])+len(hasInitial[name])) + mapLock.Unlock() + }) + close(done) + } + + doneA := make(chan struct{}) + doneB := make(chan struct{}) + + go fillQueue("QueueA", doneA) + go fillQueue("QueueB", doneB) + + <-doneA + <-doneB + + executedEmpty := map[string][]string{} + hasEmpty := map[string][]string{} + emptyQueue := func(name string, done chan struct{}) { + t.Run("Empty Queue: "+name, func(t *testing.T) { + lock := sync.Mutex{} + stop := make(chan struct{}) + + // collect the tasks that have been executed + handle := func(data ...Data) []Data { + lock.Lock() + for _, datum := range data { + mapLock.Lock() + executedEmpty[name] = append(executedEmpty[name], datum.(string)) + mapLock.Unlock() + if datum.(string) == "final" { + close(stop) + } + } + lock.Unlock() + return nil + } + + q := newQueue(name, handle) + chans := runQueue(q, &lock) + + <-chans.readyForShutdown + <-stop + close(chans.signalShutdown) + <-chans.doneShutdown + + // check which tasks are still in the queue + for i := 0; i < 100; i++ { + if has, _ := q.(UniqueQueue).Has("task-" + strconv.Itoa(i)); has { + mapLock.Lock() + hasEmpty[name] = append(hasEmpty[name], "task-"+strconv.Itoa(i)) + mapLock.Unlock() + } + } + doTerminate(chans, &lock) + + mapLock.Lock() + assert.Equal(t, 101, len(executedInitial[name])+len(executedEmpty[name])) + assert.Equal(t, 0, len(hasEmpty[name])) + mapLock.Unlock() + }) + close(done) + } + + doneA = make(chan struct{}) + doneB = make(chan struct{}) + + go emptyQueue("QueueA", doneA) + go emptyQueue("QueueB", doneB) + + <-doneA + <-doneB + + mapLock.Lock() + t.Logf("TestPersistableChannelUniqueQueue executedInitiallyA=%v, executedInitiallyB=%v, executedToEmptyA=%v, executedToEmptyB=%v", + len(executedInitial["QueueA"]), len(executedInitial["QueueB"]), len(executedEmpty["QueueA"]), len(executedEmpty["QueueB"])) + + // reset and rerun + executedInitial = map[string][]string{} + hasInitial = map[string][]string{} + executedEmpty = map[string][]string{} + hasEmpty = map[string][]string{} + mapLock.Unlock() + + doneA = make(chan struct{}) + doneB = make(chan struct{}) + + go fillQueue("QueueA", doneA) + go fillQueue("QueueB", doneB) + + <-doneA + <-doneB + + doneA = make(chan struct{}) + doneB = make(chan struct{}) + + go emptyQueue("QueueA", doneA) + go emptyQueue("QueueB", doneB) + + <-doneA + <-doneB + + mapLock.Lock() + t.Logf("TestPersistableChannelUniqueQueue executedInitiallyA=%v, executedInitiallyB=%v, executedToEmptyA=%v, executedToEmptyB=%v", + len(executedInitial["QueueA"]), len(executedInitial["QueueB"]), len(executedEmpty["QueueA"]), len(executedEmpty["QueueB"])) + mapLock.Unlock() +} diff --git a/modules/repository/create.go b/modules/repository/create.go index 1704ea792..6a1fa41b6 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -23,6 +23,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/models/webhook" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/label" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -189,7 +190,7 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m // Check if label template exist if len(opts.IssueLabels) > 0 { - if _, err := GetLabelTemplateFile(opts.IssueLabels); err != nil { + if _, err := label.GetTemplateFile(opts.IssueLabels); err != nil { return nil, err } } diff --git a/modules/repository/init.go b/modules/repository/init.go index 771b68a49..f9a33cd4f 100644 --- a/modules/repository/init.go +++ b/modules/repository/init.go @@ -18,6 +18,7 @@ import ( repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/label" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" "code.gitea.io/gitea/modules/setting" @@ -40,114 +41,6 @@ var ( LabelTemplates map[string]string ) -// ErrIssueLabelTemplateLoad represents a "ErrIssueLabelTemplateLoad" kind of error. -type ErrIssueLabelTemplateLoad struct { - TemplateFile string - OriginalError error -} - -// IsErrIssueLabelTemplateLoad checks if an error is a ErrIssueLabelTemplateLoad. -func IsErrIssueLabelTemplateLoad(err error) bool { - _, ok := err.(ErrIssueLabelTemplateLoad) - return ok -} - -func (err ErrIssueLabelTemplateLoad) Error() string { - return fmt.Sprintf("Failed to load label template file '%s': %v", err.TemplateFile, err.OriginalError) -} - -// GetRepoInitFile returns repository init files -func GetRepoInitFile(tp, name string) ([]byte, error) { - cleanedName := strings.TrimLeft(path.Clean("/"+name), "/") - relPath := path.Join("options", tp, cleanedName) - - // Use custom file when available. - customPath := path.Join(setting.CustomPath, relPath) - isFile, err := util.IsFile(customPath) - if err != nil { - log.Error("Unable to check if %s is a file. Error: %v", customPath, err) - } - if isFile { - return os.ReadFile(customPath) - } - - switch tp { - case "readme": - return options.Readme(cleanedName) - case "gitignore": - return options.Gitignore(cleanedName) - case "license": - return options.License(cleanedName) - case "label": - return options.Labels(cleanedName) - default: - return []byte{}, fmt.Errorf("Invalid init file type") - } -} - -// GetLabelTemplateFile loads the label template file by given name, -// then parses and returns a list of name-color pairs and optionally description. -func GetLabelTemplateFile(name string) ([][3]string, error) { - data, err := GetRepoInitFile("label", name) - if err != nil { - return nil, ErrIssueLabelTemplateLoad{name, fmt.Errorf("GetRepoInitFile: %w", err)} - } - - lines := strings.Split(string(data), "\n") - list := make([][3]string, 0, len(lines)) - for i := 0; i < len(lines); i++ { - line := strings.TrimSpace(lines[i]) - if len(line) == 0 { - continue - } - - parts := strings.SplitN(line, ";", 2) - - fields := strings.SplitN(parts[0], " ", 2) - if len(fields) != 2 { - return nil, ErrIssueLabelTemplateLoad{name, fmt.Errorf("line is malformed: %s", line)} - } - - color := strings.Trim(fields[0], " ") - if len(color) == 6 { - color = "#" + color - } - if !issues_model.LabelColorPattern.MatchString(color) { - return nil, ErrIssueLabelTemplateLoad{name, fmt.Errorf("bad HTML color code in line: %s", line)} - } - - var description string - - if len(parts) > 1 { - description = strings.TrimSpace(parts[1]) - } - - fields[1] = strings.TrimSpace(fields[1]) - list = append(list, [3]string{fields[1], color, description}) - } - - return list, nil -} - -func loadLabels(labelTemplate string) ([]string, error) { - list, err := GetLabelTemplateFile(labelTemplate) - if err != nil { - return nil, err - } - - labels := make([]string, len(list)) - for i := 0; i < len(list); i++ { - labels[i] = list[i][0] - } - return labels, nil -} - -// LoadLabelsFormatted loads the labels' list of a template file as a string separated by comma -func LoadLabelsFormatted(labelTemplate string) (string, error) { - labels, err := loadLabels(labelTemplate) - return strings.Join(labels, ", "), err -} - // LoadRepoConfig loads the repository config func LoadRepoConfig() { // Load .gitignore and license files and readme templates. @@ -158,6 +51,14 @@ func LoadRepoConfig() { if err != nil { log.Fatal("Failed to get %s files: %v", t, err) } + if t == "label" { + for i, f := range files { + ext := strings.ToLower(filepath.Ext(f)) + if ext == ".yaml" || ext == ".yml" { + files[i] = f[:len(f)-len(ext)] + } + } + } customPath := path.Join(setting.CustomPath, "options", t) isDir, err := util.IsDir(customPath) if err != nil { @@ -190,7 +91,7 @@ func LoadRepoConfig() { // Load label templates LabelTemplates = make(map[string]string) for _, templateFile := range LabelTemplatesFiles { - labels, err := LoadLabelsFormatted(templateFile) + labels, err := label.LoadFormatted(templateFile) if err != nil { log.Error("Failed to load labels: %v", err) } @@ -235,7 +136,7 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, } // README - data, err := GetRepoInitFile("readme", opts.Readme) + data, err := options.Readme(opts.Readme) if err != nil { return fmt.Errorf("GetRepoInitFile[%s]: %w", opts.Readme, err) } @@ -263,7 +164,7 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, var buf bytes.Buffer names := strings.Split(opts.Gitignores, ",") for _, name := range names { - data, err = GetRepoInitFile("gitignore", name) + data, err = options.Gitignore(name) if err != nil { return fmt.Errorf("GetRepoInitFile[%s]: %w", name, err) } @@ -281,7 +182,7 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, // LICENSE if len(opts.License) > 0 { - data, err = GetRepoInitFile("license", opts.License) + data, err = options.License(opts.License) if err != nil { return fmt.Errorf("GetRepoInitFile[%s]: %w", opts.License, err) } @@ -443,7 +344,7 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re // InitializeLabels adds a label set to a repository using a template func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg bool) error { - list, err := GetLabelTemplateFile(labelTemplate) + list, err := label.GetTemplateFile(labelTemplate) if err != nil { return err } @@ -451,9 +352,10 @@ func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg labels := make([]*issues_model.Label, len(list)) for i := 0; i < len(list); i++ { labels[i] = &issues_model.Label{ - Name: list[i][0], - Description: list[i][2], - Color: list[i][1], + Name: list[i].Name, + Exclusive: list[i].Exclusive, + Description: list[i].Description, + Color: list[i].Color, } if isOrg { labels[i].OrgID = id diff --git a/modules/repository/push.go b/modules/repository/push.go index 1fa711b35..aa1552351 100644 --- a/modules/repository/push.go +++ b/modules/repository/push.go @@ -4,10 +4,8 @@ package repository import ( - "context" "strings" - repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" ) @@ -96,19 +94,3 @@ func (opts *PushUpdateOptions) RefName() string { func (opts *PushUpdateOptions) RepoFullName() string { return opts.RepoUserName + "/" + opts.RepoName } - -// IsForcePush detect if a push is a force push -func IsForcePush(ctx context.Context, opts *PushUpdateOptions) (bool, error) { - if !opts.IsUpdateBranch() { - return false, nil - } - - output, _, err := git.NewCommand(ctx, "rev-list", "--max-count=1").AddDynamicArguments(opts.OldCommitID, "^"+opts.NewCommitID). - RunStdString(&git.RunOpts{Dir: repo_model.RepoPath(opts.RepoUserName, opts.RepoName)}) - if err != nil { - return false, err - } else if len(output) > 0 { - return true, nil - } - return false, nil -} diff --git a/modules/setting/database.go b/modules/setting/database.go index 49865a38a..d7a5078fe 100644 --- a/modules/setting/database.go +++ b/modules/setting/database.go @@ -27,7 +27,7 @@ var ( // Database holds the database settings Database = struct { - Type string + Type DatabaseType Host string Name string User string @@ -39,10 +39,6 @@ var ( Charset string Timeout int // seconds SQLiteJournalMode string - UseSQLite3 bool - UseMySQL bool - UseMSSQL bool - UsePostgreSQL bool DBConnectRetries int DBConnectBackoff time.Duration MaxIdleConns int @@ -59,24 +55,13 @@ var ( // LoadDBSetting loads the database settings func LoadDBSetting() { sec := CfgProvider.Section("database") - Database.Type = sec.Key("DB_TYPE").String() + Database.Type = DatabaseType(sec.Key("DB_TYPE").String()) defaultCharset := "utf8" - Database.UseMySQL = false - Database.UseSQLite3 = false - Database.UsePostgreSQL = false - Database.UseMSSQL = false - switch Database.Type { - case "sqlite3": - Database.UseSQLite3 = true - case "mysql": - Database.UseMySQL = true + if Database.Type.IsMySQL() { defaultCharset = "utf8mb4" - case "postgres": - Database.UsePostgreSQL = true - case "mssql": - Database.UseMSSQL = true } + Database.Host = sec.Key("HOST").String() Database.Name = sec.Key("NAME").String() Database.User = sec.Key("USER").String() @@ -86,7 +71,7 @@ func LoadDBSetting() { Database.Schema = sec.Key("SCHEMA").String() Database.SSLMode = sec.Key("SSL_MODE").MustString("disable") Database.Charset = sec.Key("CHARSET").In(defaultCharset, []string{"utf8", "utf8mb4"}) - if Database.UseMySQL && defaultCharset != "utf8mb4" { + if Database.Type.IsMySQL() && defaultCharset != "utf8mb4" { log.Error("Deprecated database mysql charset utf8 support, please use utf8mb4 or convert utf8 to utf8mb4.") } @@ -95,7 +80,7 @@ func LoadDBSetting() { Database.SQLiteJournalMode = sec.Key("SQLITE_JOURNAL_MODE").MustString("") Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2) - if Database.UseMySQL { + if Database.Type.IsMySQL() { Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFETIME").MustDuration(3 * time.Second) } else { Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFETIME").MustDuration(0) @@ -207,3 +192,25 @@ func ParseMSSQLHostPort(info string) (string, string) { } return host, port } + +type DatabaseType string + +func (t DatabaseType) String() string { + return string(t) +} + +func (t DatabaseType) IsSQLite3() bool { + return t == "sqlite3" +} + +func (t DatabaseType) IsMySQL() bool { + return t == "mysql" +} + +func (t DatabaseType) IsMSSQL() bool { + return t == "mssql" +} + +func (t DatabaseType) IsPostgreSQL() bool { + return t == "postgres" +} diff --git a/modules/setting/log.go b/modules/setting/log.go index 5448650aa..dabdb543a 100644 --- a/modules/setting/log.go +++ b/modules/setting/log.go @@ -38,6 +38,7 @@ var Log struct { EnableAccessLog bool AccessLogTemplate string BufferLength int64 + RequestIDHeaders []string } // GetLogDescriptions returns a race safe set of descriptions @@ -153,6 +154,7 @@ func loadLogFrom(rootCfg ConfigProvider) { Log.AccessLogTemplate = sec.Key("ACCESS_LOG_TEMPLATE").MustString( `{{.Ctx.RemoteAddr}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}\" \"{{.Ctx.Req.UserAgent}}"`, ) + Log.RequestIDHeaders = sec.Key("REQUEST_ID_HEADERS").Strings(",") // the `MustString` updates the default value, and `log.ACCESS` is used by `generateNamedLogger("access")` later _ = rootCfg.Section("log").Key("ACCESS").MustString("file") diff --git a/modules/setting/storage.go b/modules/setting/storage.go index 9197c5f8b..4d401614e 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -41,6 +41,7 @@ func getStorage(rootCfg ConfigProvider, name, typ string, targetSec *ini.Section sec.Key("MINIO_BUCKET").MustString("gitea") sec.Key("MINIO_LOCATION").MustString("us-east-1") sec.Key("MINIO_USE_SSL").MustBool(false) + sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false) if targetSec == nil { targetSec, _ = rootCfg.NewSection(name) diff --git a/modules/storage/local.go b/modules/storage/local.go index a6a9d54a8..05bf1fb28 100644 --- a/modules/storage/local.go +++ b/modules/storage/local.go @@ -8,7 +8,6 @@ import ( "io" "net/url" "os" - "path" "path/filepath" "strings" @@ -59,7 +58,7 @@ func NewLocalStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error } func (l *LocalStorage) buildLocalPath(p string) string { - return filepath.Join(l.dir, path.Clean("/" + strings.ReplaceAll(p, "\\", "/"))[1:]) + return filepath.Join(l.dir, util.CleanPath(strings.ReplaceAll(p, "\\", "/"))) } // Open a file diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 912f820ed..24da14b63 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -5,7 +5,9 @@ package storage import ( "context" + "crypto/tls" "io" + "net/http" "net/url" "os" "path" @@ -13,6 +15,7 @@ import ( "time" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/util" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -42,13 +45,14 @@ const MinioStorageType Type = "minio" // MinioStorageConfig represents the configuration for a minio storage type MinioStorageConfig struct { - Endpoint string `ini:"MINIO_ENDPOINT"` - AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"` - SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"` - Bucket string `ini:"MINIO_BUCKET"` - Location string `ini:"MINIO_LOCATION"` - BasePath string `ini:"MINIO_BASE_PATH"` - UseSSL bool `ini:"MINIO_USE_SSL"` + Endpoint string `ini:"MINIO_ENDPOINT"` + AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"` + SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"` + Bucket string `ini:"MINIO_BUCKET"` + Location string `ini:"MINIO_LOCATION"` + BasePath string `ini:"MINIO_BASE_PATH"` + UseSSL bool `ini:"MINIO_USE_SSL"` + InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"` } // MinioStorage returns a minio bucket storage @@ -90,8 +94,9 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath) minioClient, err := minio.New(config.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""), - Secure: config.UseSSL, + Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""), + Secure: config.UseSSL, + Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}}, }) if err != nil { return nil, convertMinioErr(err) @@ -116,7 +121,7 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error } func (m *MinioStorage) buildMinioPath(p string) string { - return strings.TrimPrefix(path.Join(m.basePath, path.Clean("/" + strings.ReplaceAll(p, "\\", "/"))[1:]), "/") + return strings.TrimPrefix(path.Join(m.basePath, util.CleanPath(strings.ReplaceAll(p, "\\", "/"))), "/") } // Open open a file diff --git a/modules/templates/helper.go b/modules/templates/helper.go index 17ac68dc6..19893c7c9 100644 --- a/modules/templates/helper.go +++ b/modules/templates/helper.go @@ -385,10 +385,10 @@ func NewFuncMap() []template.FuncMap { // the table is NOT sorted with this header return "" }, - "RenderLabel": func(label *issues_model.Label) template.HTML { - return template.HTML(RenderLabel(label)) + "RenderLabel": func(ctx context.Context, label *issues_model.Label) template.HTML { + return template.HTML(RenderLabel(ctx, label)) }, - "RenderLabels": func(labels []*issues_model.Label, repoLink string) template.HTML { + "RenderLabels": func(ctx context.Context, labels []*issues_model.Label, repoLink string) template.HTML { htmlCode := `<span class="labels-list">` for _, label := range labels { // Protect against nil value in labels - shouldn't happen but would cause a panic if so @@ -396,7 +396,7 @@ func NewFuncMap() []template.FuncMap { continue } htmlCode += fmt.Sprintf("<a href='%s/issues?labels=%d'>%s</a> ", - repoLink, label.ID, RenderLabel(label)) + repoLink, label.ID, RenderLabel(ctx, label)) } htmlCode += "</span>" return template.HTML(htmlCode) @@ -808,7 +808,7 @@ func RenderIssueTitle(ctx context.Context, text, urlPrefix string, metas map[str } // RenderLabel renders a label -func RenderLabel(label *issues_model.Label) string { +func RenderLabel(ctx context.Context, label *issues_model.Label) string { labelScope := label.ExclusiveScope() textColor := "#111" @@ -821,12 +821,12 @@ func RenderLabel(label *issues_model.Label) string { if labelScope == "" { // Regular label return fmt.Sprintf("<div class='ui label' style='color: %s !important; background-color: %s !important' title='%s'>%s</div>", - textColor, label.Color, description, RenderEmoji(label.Name)) + textColor, label.Color, description, RenderEmoji(ctx, label.Name)) } // Scoped label - scopeText := RenderEmoji(labelScope) - itemText := RenderEmoji(label.Name[len(labelScope)+1:]) + scopeText := RenderEmoji(ctx, labelScope) + itemText := RenderEmoji(ctx, label.Name[len(labelScope)+1:]) itemColor := label.Color scopeColor := label.Color @@ -869,8 +869,9 @@ func RenderLabel(label *issues_model.Label) string { } // RenderEmoji renders html text with emoji post processors -func RenderEmoji(text string) template.HTML { - renderedText, err := markup.RenderEmoji(template.HTMLEscapeString(text)) +func RenderEmoji(ctx context.Context, text string) template.HTML { + renderedText, err := markup.RenderEmoji(&markup.RenderContext{Ctx: ctx}, + template.HTMLEscapeString(text)) if err != nil { log.Error("RenderEmoji: %v", err) return template.HTML("") diff --git a/modules/typesniffer/typesniffer.go b/modules/typesniffer/typesniffer.go index c9fef953c..7887fd42b 100644 --- a/modules/typesniffer/typesniffer.go +++ b/modules/typesniffer/typesniffer.go @@ -4,6 +4,7 @@ package typesniffer import ( + "bytes" "fmt" "io" "net/http" @@ -24,8 +25,9 @@ const ( ) var ( - svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!--.*?-->|<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg[\s>\/]`) - svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!--.*?-->|<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg[\s>\/]`) + svgComment = regexp.MustCompile(`(?s)<!--.*?-->`) + svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) + svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`) ) // SniffedType contains information about a blobs type. @@ -91,10 +93,27 @@ func DetectContentType(data []byte) SniffedType { data = data[:sniffLen] } - if (strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html")) && svgTagRegex.Match(data) || - strings.Contains(ct, "text/xml") && svgTagInXMLRegex.Match(data) { - // SVG is unsupported. https://github.com/golang/go/issues/15888 - ct = SvgMimeType + // SVG is unsupported by http.DetectContentType, https://github.com/golang/go/issues/15888 + + detectByHTML := strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html") + detectByXML := strings.Contains(ct, "text/xml") + if detectByHTML || detectByXML { + dataProcessed := svgComment.ReplaceAll(data, nil) + dataProcessed = bytes.TrimSpace(dataProcessed) + if detectByHTML && svgTagRegex.Match(dataProcessed) || + detectByXML && svgTagInXMLRegex.Match(dataProcessed) { + ct = SvgMimeType + } + } + + if strings.HasPrefix(ct, "audio/") && bytes.HasPrefix(data, []byte("ID3")) { + // The MP3 detection is quite inaccurate, any content with "ID3" prefix will result in "audio/mpeg". + // So remove the "ID3" prefix and detect again, if result is text, then it must be text content. + // This works especially because audio files contain many unprintable/invalid characters like `0x00` + ct2 := http.DetectContentType(data[3:]) + if strings.HasPrefix(ct2, "text/") { + ct = ct2 + } } return SniffedType{ct} diff --git a/modules/typesniffer/typesniffer_test.go b/modules/typesniffer/typesniffer_test.go index dbce94fc3..6c6da34aa 100644 --- a/modules/typesniffer/typesniffer_test.go +++ b/modules/typesniffer/typesniffer_test.go @@ -28,7 +28,6 @@ func TestIsSvgImage(t *testing.T) { assert.True(t, DetectContentType([]byte("<svg></svg>")).IsSvgImage()) assert.True(t, DetectContentType([]byte(" <svg></svg>")).IsSvgImage()) assert.True(t, DetectContentType([]byte(`<svg width="100"></svg>`)).IsSvgImage()) - assert.True(t, DetectContentType([]byte("<svg/>")).IsSvgImage()) assert.True(t, DetectContentType([]byte(`<?xml version="1.0" encoding="UTF-8"?><svg></svg>`)).IsSvgImage()) assert.True(t, DetectContentType([]byte(`<!-- Comment --> <svg></svg>`)).IsSvgImage()) @@ -57,6 +56,10 @@ func TestIsSvgImage(t *testing.T) { <!-- Multline Comment --> <svg></svg>`)).IsSvgImage()) + + // the DetectContentType should work for incomplete data, because only beginning bytes are used for detection + assert.True(t, DetectContentType([]byte(`<svg>....`)).IsSvgImage()) + assert.False(t, DetectContentType([]byte{}).IsSvgImage()) assert.False(t, DetectContentType([]byte("svg")).IsSvgImage()) assert.False(t, DetectContentType([]byte("<svgfoo></svgfoo>")).IsSvgImage()) @@ -68,6 +71,26 @@ func TestIsSvgImage(t *testing.T) { assert.False(t, DetectContentType([]byte(`<?xml version="1.0" encoding="UTF-8"?> <!-- <svg></svg> inside comment --> <foo></foo>`)).IsSvgImage()) + + assert.False(t, DetectContentType([]byte(` +<!-- comment1 --> +<div> + <!-- comment2 --> + <svg></svg> +</div> +`)).IsSvgImage()) + + assert.False(t, DetectContentType([]byte(` +<!-- comment1 +--> +<div> + <!-- comment2 +--> + <svg></svg> +</div> +`)).IsSvgImage()) + assert.False(t, DetectContentType([]byte(`<html><body><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg></svg></body></html>`)).IsSvgImage()) + assert.False(t, DetectContentType([]byte(`<html><body><?xml version="1.0" encoding="UTF-8"?><svg></svg></body></html>`)).IsSvgImage()) } func TestIsPDF(t *testing.T) { @@ -86,6 +109,10 @@ func TestIsAudio(t *testing.T) { mp3, _ := base64.StdEncoding.DecodeString("SUQzBAAAAAABAFRYWFgAAAASAAADbWFqb3JfYnJhbmQAbXA0MgBUWFhYAAAAEQAAA21pbm9yX3Zl") assert.True(t, DetectContentType(mp3).IsAudio()) assert.False(t, DetectContentType([]byte("plain text")).IsAudio()) + + assert.True(t, DetectContentType([]byte("ID3Toy\000")).IsAudio()) + assert.True(t, DetectContentType([]byte("ID3Toy\n====\t* hi 🌞, ...")).IsText()) // test ID3 tag for plain text + assert.True(t, DetectContentType([]byte("ID3Toy\n====\t* hi 🌞, ..."+"🌛"[0:2])).IsText()) // test ID3 tag with incomplete UTF8 char } func TestDetectContentTypeFromReader(t *testing.T) { diff --git a/modules/util/path.go b/modules/util/path.go index 74acb7a85..5aa9e15f5 100644 --- a/modules/util/path.go +++ b/modules/util/path.go @@ -14,6 +14,14 @@ import ( "strings" ) +// CleanPath ensure to clean the path +func CleanPath(p string) string { + if strings.HasPrefix(p, "/") { + return path.Clean(p) + } + return path.Clean("/" + p)[1:] +} + // EnsureAbsolutePath ensure that a path is absolute, making it // relative to absoluteBase if necessary func EnsureAbsolutePath(path, absoluteBase string) string { diff --git a/modules/util/path_test.go b/modules/util/path_test.go index 93f4f67cf..2f020f924 100644 --- a/modules/util/path_test.go +++ b/modules/util/path_test.go @@ -136,3 +136,15 @@ func TestMisc_IsReadmeFileName(t *testing.T) { assert.Equal(t, testCase.idx, idx) } } + +func TestCleanPath(t *testing.T) { + cases := map[string]string{ + "../../test": "test", + "/test": "/test", + "/../test": "/test", + } + + for k, v := range cases { + assert.Equal(t, v, CleanPath(k)) + } +} |