aboutsummaryrefslogtreecommitdiff
path: root/modules
diff options
context:
space:
mode:
authorAnthony Wang2023-03-12 23:17:11 +0000
committerAnthony Wang2023-03-12 23:19:57 +0000
commitdbc3eb7fa071307e623fde961bfdff0894856948 (patch)
tree8961dc6a935d9ee89f5adf680a142d36900aaebb /modules
parentb4640101f3c8938ed689743606a79601201141ca (diff)
parente72290fd9aeb77a47311483d1d565e428ce40cd9 (diff)
Merge remote-tracking branch 'origin/main' into forgejo-federation
Diffstat (limited to 'modules')
-rw-r--r--modules/auth/password/hash/setting.go29
-rw-r--r--modules/cache/context.go119
-rw-r--r--modules/cache/context_test.go39
-rw-r--r--modules/context/access_log.go34
-rw-r--r--modules/context/api.go2
-rw-r--r--modules/context/context.go4
-rw-r--r--modules/context/org.go78
-rw-r--r--modules/csv/csv_test.go6
-rw-r--r--modules/doctor/dbconsistency.go2
-rw-r--r--modules/git/command.go2
-rw-r--r--modules/git/commit.go13
-rw-r--r--modules/git/git.go8
-rw-r--r--modules/git/pipeline/revlist.go5
-rw-r--r--modules/git/repo_commit.go21
-rw-r--r--modules/git/repo_commit_gogit.go39
-rw-r--r--modules/git/repo_commit_nogogit.go4
-rw-r--r--modules/git/repo_commit_test.go5
-rw-r--r--modules/git/repo_compare.go11
-rw-r--r--modules/git/repo_ref_test.go12
-rw-r--r--modules/git/repo_stats_test.go4
-rw-r--r--modules/git/repo_tag_test.go9
-rw-r--r--modules/git/repo_test.go6
-rw-r--r--modules/git/tests/repos/repo1_bare/indexbin0 -> 65 bytes
-rw-r--r--modules/git/tests/repos/repo1_bare/logs/HEAD1
-rw-r--r--modules/git/tests/repos/repo1_bare/logs/refs/heads/master1
-rw-r--r--modules/git/tests/repos/repo1_bare/objects/1c/91d130dc5fb75fd2d9f586a058650889cfe7fbbin0 -> 813 bytes
-rw-r--r--modules/git/tests/repos/repo1_bare/objects/28/b55526e7100924d864dd89e35c1ea62e7a5a32bin0 -> 818 bytes
-rw-r--r--modules/git/tests/repos/repo1_bare/objects/36/f97d9a96457e2bab511db30fe2db03893ebc64bin0 -> 770 bytes
-rw-r--r--modules/git/tests/repos/repo1_bare/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904bin0 -> 15 bytes
-rw-r--r--modules/git/tests/repos/repo1_bare/objects/93/3305878a3c9ad485c29b87fb662a73a9675c4bbin0 -> 770 bytes
-rw-r--r--modules/git/tests/repos/repo1_bare/objects/ce/064814f4a0d337b333e646ece456cd39fab612bin0 -> 815 bytes
-rw-r--r--modules/git/tests/repos/repo1_bare/objects/cf/8b0b492a950b358a7ce7f9d01b18aef48a6b2dbin0 -> 827 bytes
-rw-r--r--modules/git/tests/repos/repo1_bare/refs/heads/master2
-rw-r--r--modules/git/tests/repos/repo1_bare/refs/tags/signed-tag1
-rw-r--r--modules/httpcache/httpcache.go12
-rw-r--r--modules/label/label.go46
-rw-r--r--modules/label/parser.go126
-rw-r--r--modules/label/parser_test.go72
-rw-r--r--modules/markup/console/console_test.go4
-rw-r--r--modules/markup/csv/csv_test.go4
-rw-r--r--modules/markup/html.go3
-rw-r--r--modules/markup/html_internal_test.go43
-rw-r--r--modules/markup/html_test.go14
-rw-r--r--modules/markup/markdown/markdown_test.go23
-rw-r--r--modules/markup/orgmode/orgmode_test.go4
-rw-r--r--modules/markup/sanitizer.go2
-rw-r--r--modules/options/base.go56
-rw-r--r--modules/options/dynamic.go70
-rw-r--r--modules/options/static.go50
-rw-r--r--modules/public/public.go4
-rw-r--r--modules/queue/queue_channel.go5
-rw-r--r--modules/queue/queue_disk_channel.go29
-rw-r--r--modules/queue/queue_disk_channel_test.go8
-rw-r--r--modules/queue/unique_queue_channel.go4
-rw-r--r--modules/queue/unique_queue_channel_test.go7
-rw-r--r--modules/queue/unique_queue_disk_channel.go41
-rw-r--r--modules/queue/unique_queue_disk_channel_test.go259
-rw-r--r--modules/repository/create.go3
-rw-r--r--modules/repository/init.go134
-rw-r--r--modules/repository/push.go18
-rw-r--r--modules/setting/database.go49
-rw-r--r--modules/setting/log.go2
-rw-r--r--modules/setting/storage.go1
-rw-r--r--modules/storage/local.go3
-rw-r--r--modules/storage/minio.go25
-rw-r--r--modules/templates/helper.go21
-rw-r--r--modules/typesniffer/typesniffer.go31
-rw-r--r--modules/typesniffer/typesniffer_test.go29
-rw-r--r--modules/util/path.go8
-rw-r--r--modules/util/path_test.go12
70 files changed, 1208 insertions, 471 deletions
diff --git a/modules/auth/password/hash/setting.go b/modules/auth/password/hash/setting.go
index 701697430..f0715f31e 100644
--- a/modules/auth/password/hash/setting.go
+++ b/modules/auth/password/hash/setting.go
@@ -41,9 +41,8 @@ var RecommendedHashAlgorithms = []string{
"pbkdf2_hi",
}
-// SetDefaultPasswordHashAlgorithm will take a provided algorithmName and dealias it to
-// a complete algorithm specification.
-func SetDefaultPasswordHashAlgorithm(algorithmName string) (string, *PasswordHashAlgorithm) {
+// hashAlgorithmToSpec converts an algorithm name or a specification to a full algorithm specification
+func hashAlgorithmToSpec(algorithmName string) string {
if algorithmName == "" {
algorithmName = DefaultHashAlgorithmName
}
@@ -52,10 +51,26 @@ func SetDefaultPasswordHashAlgorithm(algorithmName string) (string, *PasswordHas
algorithmName = alias
alias, has = aliasAlgorithmNames[algorithmName]
}
+ return algorithmName
+}
- // algorithmName should now be a full algorithm specification
- // e.g. pbkdf2$50000$50 rather than pbdkf2
- DefaultHashAlgorithm = Parse(algorithmName)
+// SetDefaultPasswordHashAlgorithm will take a provided algorithmName and de-alias it to
+// a complete algorithm specification.
+func SetDefaultPasswordHashAlgorithm(algorithmName string) (string, *PasswordHashAlgorithm) {
+ algoSpec := hashAlgorithmToSpec(algorithmName)
+ // now we get a full specification, e.g. pbkdf2$50000$50 rather than pbdkf2
+ DefaultHashAlgorithm = Parse(algoSpec)
+ return algoSpec, DefaultHashAlgorithm
+}
- return algorithmName, DefaultHashAlgorithm
+// ConfigHashAlgorithm will try to find a "recommended algorithm name" defined by RecommendedHashAlgorithms for config
+// This function is not fast and is only used for the installation page
+func ConfigHashAlgorithm(algorithm string) string {
+ algorithm = hashAlgorithmToSpec(algorithm)
+ for _, recommAlgo := range RecommendedHashAlgorithms {
+ if algorithm == hashAlgorithmToSpec(recommAlgo) {
+ return recommAlgo
+ }
+ }
+ return algorithm
}
diff --git a/modules/cache/context.go b/modules/cache/context.go
index f741a8744..62bbf5dcb 100644
--- a/modules/cache/context.go
+++ b/modules/cache/context.go
@@ -6,6 +6,7 @@ package cache
import (
"context"
"sync"
+ "time"
"code.gitea.io/gitea/modules/log"
)
@@ -14,65 +15,151 @@ import (
// This is useful for caching data that is expensive to calculate and is likely to be
// used multiple times in a request.
type cacheContext struct {
- ctx context.Context
- data map[any]map[any]any
- lock sync.RWMutex
+ data map[any]map[any]any
+ lock sync.RWMutex
+ created time.Time
+ discard bool
}
func (cc *cacheContext) Get(tp, key any) any {
cc.lock.RLock()
defer cc.lock.RUnlock()
- if cc.data[tp] == nil {
- return nil
- }
return cc.data[tp][key]
}
func (cc *cacheContext) Put(tp, key, value any) {
cc.lock.Lock()
defer cc.lock.Unlock()
- if cc.data[tp] == nil {
- cc.data[tp] = make(map[any]any)
+
+ if cc.discard {
+ return
+ }
+
+ d := cc.data[tp]
+ if d == nil {
+ d = make(map[any]any)
+ cc.data[tp] = d
}
- cc.data[tp][key] = value
+ d[key] = value
}
func (cc *cacheContext) Delete(tp, key any) {
cc.lock.Lock()
defer cc.lock.Unlock()
- if cc.data[tp] == nil {
- return
- }
delete(cc.data[tp], key)
}
+func (cc *cacheContext) Discard() {
+ cc.lock.Lock()
+ defer cc.lock.Unlock()
+ cc.data = nil
+ cc.discard = true
+}
+
+func (cc *cacheContext) isDiscard() bool {
+ cc.lock.RLock()
+ defer cc.lock.RUnlock()
+ return cc.discard
+}
+
+// cacheContextLifetime is the max lifetime of cacheContext.
+// Since cacheContext is used to cache data in a request level context, 10s is enough.
+// If a cacheContext is used more than 10s, it's probably misuse.
+const cacheContextLifetime = 10 * time.Second
+
+var timeNow = time.Now
+
+func (cc *cacheContext) Expired() bool {
+ return timeNow().Sub(cc.created) > cacheContextLifetime
+}
+
var cacheContextKey = struct{}{}
+/*
+Since there are both WithCacheContext and WithNoCacheContext,
+it may be confusing when there is nesting.
+
+Some cases to explain the design:
+
+When:
+- A, B or C means a cache context.
+- A', B' or C' means a discard cache context.
+- ctx means context.Backgrand().
+- A(ctx) means a cache context with ctx as the parent context.
+- B(A(ctx)) means a cache context with A(ctx) as the parent context.
+- With is alias of WithCacheContext.
+- WithNo is alias of WithNoCacheContext.
+
+So:
+- With(ctx) -> A(ctx)
+- With(With(ctx)) -> A(ctx), not B(A(ctx)), always reuse parent cache context if possible.
+- With(With(With(ctx))) -> A(ctx), not C(B(A(ctx))), ditto.
+- WithNo(ctx) -> ctx, not A'(ctx), don't create new cache context if we don't have to.
+- WithNo(With(ctx)) -> A'(ctx)
+- WithNo(WithNo(With(ctx))) -> A'(ctx), not B'(A'(ctx)), don't create new cache context if we don't have to.
+- With(WithNo(With(ctx))) -> B(A'(ctx)), not A(ctx), never reuse a discard cache context.
+- WithNo(With(WithNo(With(ctx)))) -> B'(A'(ctx))
+- With(WithNo(With(WithNo(With(ctx))))) -> C(B'(A'(ctx))), so there's always only one not-discard cache context.
+*/
+
func WithCacheContext(ctx context.Context) context.Context {
+ if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok {
+ if !c.isDiscard() {
+ // reuse parent context
+ return ctx
+ }
+ }
return context.WithValue(ctx, cacheContextKey, &cacheContext{
- ctx: ctx,
- data: make(map[any]map[any]any),
+ data: make(map[any]map[any]any),
+ created: timeNow(),
})
}
+func WithNoCacheContext(ctx context.Context) context.Context {
+ if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok {
+ // The caller want to run long-life tasks, but the parent context is a cache context.
+ // So we should disable and clean the cache data, or it will be kept in memory for a long time.
+ c.Discard()
+ return ctx
+ }
+
+ return ctx
+}
+
func GetContextData(ctx context.Context, tp, key any) any {
if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok {
+ if c.Expired() {
+ // The warning means that the cache context is misused for long-life task,
+ // it can be resolved with WithNoCacheContext(ctx).
+ log.Warn("cache context is expired, may be misused for long-life tasks: %v", c)
+ return nil
+ }
return c.Get(tp, key)
}
- log.Warn("cannot get cache context when getting data: %v", ctx)
return nil
}
func SetContextData(ctx context.Context, tp, key, value any) {
if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok {
+ if c.Expired() {
+ // The warning means that the cache context is misused for long-life task,
+ // it can be resolved with WithNoCacheContext(ctx).
+ log.Warn("cache context is expired, may be misused for long-life tasks: %v", c)
+ return
+ }
c.Put(tp, key, value)
return
}
- log.Warn("cannot get cache context when setting data: %v", ctx)
}
func RemoveContextData(ctx context.Context, tp, key any) {
if c, ok := ctx.Value(cacheContextKey).(*cacheContext); ok {
+ if c.Expired() {
+ // The warning means that the cache context is misused for long-life task,
+ // it can be resolved with WithNoCacheContext(ctx).
+ log.Warn("cache context is expired, may be misused for long-life tasks: %v", c)
+ return
+ }
c.Delete(tp, key)
}
}
diff --git a/modules/cache/context_test.go b/modules/cache/context_test.go
index 77e3ecad2..531554786 100644
--- a/modules/cache/context_test.go
+++ b/modules/cache/context_test.go
@@ -6,6 +6,7 @@ package cache
import (
"context"
"testing"
+ "time"
"github.com/stretchr/testify/assert"
)
@@ -25,7 +26,7 @@ func TestWithCacheContext(t *testing.T) {
assert.EqualValues(t, 1, v.(int))
RemoveContextData(ctx, field, "my_config1")
- RemoveContextData(ctx, field, "my_config2") // remove an non-exist key
+ RemoveContextData(ctx, field, "my_config2") // remove a non-exist key
v = GetContextData(ctx, field, "my_config1")
assert.Nil(t, v)
@@ -38,4 +39,40 @@ func TestWithCacheContext(t *testing.T) {
v = GetContextData(ctx, field, "my_config1")
assert.EqualValues(t, 1, v)
+
+ now := timeNow
+ defer func() {
+ timeNow = now
+ }()
+ timeNow = func() time.Time {
+ return now().Add(10 * time.Second)
+ }
+ v = GetContextData(ctx, field, "my_config1")
+ assert.Nil(t, v)
+}
+
+func TestWithNoCacheContext(t *testing.T) {
+ ctx := context.Background()
+
+ const field = "system_setting"
+
+ v := GetContextData(ctx, field, "my_config1")
+ assert.Nil(t, v)
+ SetContextData(ctx, field, "my_config1", 1)
+ v = GetContextData(ctx, field, "my_config1")
+ assert.Nil(t, v) // still no cache
+
+ ctx = WithCacheContext(ctx)
+ v = GetContextData(ctx, field, "my_config1")
+ assert.Nil(t, v)
+ SetContextData(ctx, field, "my_config1", 1)
+ v = GetContextData(ctx, field, "my_config1")
+ assert.NotNil(t, v)
+
+ ctx = WithNoCacheContext(ctx)
+ v = GetContextData(ctx, field, "my_config1")
+ assert.Nil(t, v)
+ SetContextData(ctx, field, "my_config1", 1)
+ v = GetContextData(ctx, field, "my_config1")
+ assert.Nil(t, v) // still no cache
}
diff --git a/modules/context/access_log.go b/modules/context/access_log.go
index 1aaba9dc2..515682b64 100644
--- a/modules/context/access_log.go
+++ b/modules/context/access_log.go
@@ -6,7 +6,9 @@ package context
import (
"bytes"
"context"
+ "fmt"
"net/http"
+ "strings"
"text/template"
"time"
@@ -20,13 +22,39 @@ type routerLoggerOptions struct {
Start *time.Time
ResponseWriter http.ResponseWriter
Ctx map[string]interface{}
+ RequestID *string
}
var signedUserNameStringPointerKey interface{} = "signedUserNameStringPointerKey"
+const keyOfRequestIDInTemplate = ".RequestID"
+
+// According to:
+// TraceId: A valid trace identifier is a 16-byte array with at least one non-zero byte
+// MD5 output is 16 or 32 bytes: md5-bytes is 16, md5-hex is 32
+// SHA1: similar, SHA1-bytes is 20, SHA1-hex is 40.
+// UUID is 128-bit, 32 hex chars, 36 ASCII chars with 4 dashes
+// So, we accept a Request ID with a maximum character length of 40
+const maxRequestIDByteLength = 40
+
+func parseRequestIDFromRequestHeader(req *http.Request) string {
+ requestID := "-"
+ for _, key := range setting.Log.RequestIDHeaders {
+ if req.Header.Get(key) != "" {
+ requestID = req.Header.Get(key)
+ break
+ }
+ }
+ if len(requestID) > maxRequestIDByteLength {
+ requestID = fmt.Sprintf("%s...", requestID[:maxRequestIDByteLength])
+ }
+ return requestID
+}
+
// AccessLogger returns a middleware to log access logger
func AccessLogger() func(http.Handler) http.Handler {
logger := log.GetLogger("access")
+ needRequestID := len(setting.Log.RequestIDHeaders) > 0 && strings.Contains(setting.Log.AccessLogTemplate, keyOfRequestIDInTemplate)
logTemplate, _ := template.New("log").Parse(setting.Log.AccessLogTemplate)
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
@@ -34,6 +62,11 @@ func AccessLogger() func(http.Handler) http.Handler {
identity := "-"
r := req.WithContext(context.WithValue(req.Context(), signedUserNameStringPointerKey, &identity))
+ var requestID string
+ if needRequestID {
+ requestID = parseRequestIDFromRequestHeader(req)
+ }
+
next.ServeHTTP(w, r)
rw := w.(ResponseWriter)
@@ -47,6 +80,7 @@ func AccessLogger() func(http.Handler) http.Handler {
"RemoteAddr": req.RemoteAddr,
"Req": req,
},
+ RequestID: &requestID,
})
if err != nil {
log.Error("Could not set up chi access logger: %v", err.Error())
diff --git a/modules/context/api.go b/modules/context/api.go
index 3f938948a..f7a338469 100644
--- a/modules/context/api.go
+++ b/modules/context/api.go
@@ -244,7 +244,7 @@ func APIContexter() func(http.Handler) http.Handler {
}
}
- httpcache.AddCacheControlToHeader(ctx.Resp.Header(), 0, "no-transform")
+ httpcache.SetCacheControlInHeader(ctx.Resp.Header(), 0, "no-transform")
ctx.Resp.Header().Set(`X-Frame-Options`, setting.CORSConfig.XFrameOptions)
ctx.Data["Context"] = &ctx
diff --git a/modules/context/context.go b/modules/context/context.go
index 0c8d7411e..50c34edae 100644
--- a/modules/context/context.go
+++ b/modules/context/context.go
@@ -388,7 +388,7 @@ func (ctx *Context) SetServeHeaders(opts *ServeHeaderOptions) {
if duration == 0 {
duration = 5 * time.Minute
}
- httpcache.AddCacheControlToHeader(header, duration)
+ httpcache.SetCacheControlInHeader(header, duration)
if !opts.LastModified.IsZero() {
header.Set("Last-Modified", opts.LastModified.UTC().Format(http.TimeFormat))
@@ -753,7 +753,7 @@ func Contexter(ctx context.Context) func(next http.Handler) http.Handler {
}
}
- httpcache.AddCacheControlToHeader(ctx.Resp.Header(), 0, "no-transform")
+ httpcache.SetCacheControlInHeader(ctx.Resp.Header(), 0, "no-transform")
ctx.Resp.Header().Set(`X-Frame-Options`, setting.CORSConfig.XFrameOptions)
ctx.Data["CsrfToken"] = ctx.csrf.GetToken()
diff --git a/modules/context/org.go b/modules/context/org.go
index 0add7f2c0..39a3038f9 100644
--- a/modules/context/org.go
+++ b/modules/context/org.go
@@ -11,7 +11,6 @@ import (
"code.gitea.io/gitea/models/perm"
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
- "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
)
@@ -31,29 +30,34 @@ type Organization struct {
}
func (org *Organization) CanWriteUnit(ctx *Context, unitType unit.Type) bool {
- if ctx.Doer == nil {
- return false
- }
- return org.UnitPermission(ctx, ctx.Doer.ID, unitType) >= perm.AccessModeWrite
+ return org.Organization.UnitPermission(ctx, ctx.Doer, unitType) >= perm.AccessModeWrite
}
-func (org *Organization) UnitPermission(ctx *Context, doerID int64, unitType unit.Type) perm.AccessMode {
- if doerID > 0 {
- teams, err := organization.GetUserOrgTeams(ctx, org.Organization.ID, doerID)
- if err != nil {
- log.Error("GetUserOrgTeams: %v", err)
- return perm.AccessModeNone
- }
- if len(teams) > 0 {
- return teams.UnitMaxAccess(unitType)
- }
- }
+func (org *Organization) CanReadUnit(ctx *Context, unitType unit.Type) bool {
+ return org.Organization.UnitPermission(ctx, ctx.Doer, unitType) >= perm.AccessModeRead
+}
- if org.Organization.Visibility == structs.VisibleTypePublic {
- return perm.AccessModeRead
- }
+func GetOrganizationByParams(ctx *Context) {
+ orgName := ctx.Params(":org")
+
+ var err error
- return perm.AccessModeNone
+ ctx.Org.Organization, err = organization.GetOrgByName(ctx, orgName)
+ if err != nil {
+ if organization.IsErrOrgNotExist(err) {
+ redirectUserID, err := user_model.LookupUserRedirect(orgName)
+ if err == nil {
+ RedirectToUser(ctx, orgName, redirectUserID)
+ } else if user_model.IsErrUserRedirectNotExist(err) {
+ ctx.NotFound("GetUserByName", err)
+ } else {
+ ctx.ServerError("LookupUserRedirect", err)
+ }
+ } else {
+ ctx.ServerError("GetUserByName", err)
+ }
+ return
+ }
}
// HandleOrgAssignment handles organization assignment
@@ -77,25 +81,26 @@ func HandleOrgAssignment(ctx *Context, args ...bool) {
requireTeamAdmin = args[3]
}
- orgName := ctx.Params(":org")
-
var err error
- ctx.Org.Organization, err = organization.GetOrgByName(ctx, orgName)
- if err != nil {
- if organization.IsErrOrgNotExist(err) {
- redirectUserID, err := user_model.LookupUserRedirect(orgName)
- if err == nil {
- RedirectToUser(ctx, orgName, redirectUserID)
- } else if user_model.IsErrUserRedirectNotExist(err) {
- ctx.NotFound("GetUserByName", err)
- } else {
- ctx.ServerError("LookupUserRedirect", err)
+
+ if ctx.ContextUser == nil {
+ // if Organization is not defined, get it from params
+ if ctx.Org.Organization == nil {
+ GetOrganizationByParams(ctx)
+ if ctx.Written() {
+ return
}
- } else {
- ctx.ServerError("GetUserByName", err)
}
+ } else if ctx.ContextUser.IsOrganization() {
+ if ctx.Org == nil {
+ ctx.Org = &Organization{}
+ }
+ ctx.Org.Organization = (*organization.Organization)(ctx.ContextUser)
+ } else {
+ // ContextUser is an individual User
return
}
+
org := ctx.Org.Organization
// Handle Visibility
@@ -156,6 +161,7 @@ func HandleOrgAssignment(ctx *Context, args ...bool) {
}
ctx.Data["IsOrganizationOwner"] = ctx.Org.IsOwner
ctx.Data["IsOrganizationMember"] = ctx.Org.IsMember
+ ctx.Data["IsProjectEnabled"] = true
ctx.Data["IsPackageEnabled"] = setting.Packages.Enabled
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
ctx.Data["IsPublicMember"] = func(uid int64) bool {
@@ -231,6 +237,10 @@ func HandleOrgAssignment(ctx *Context, args ...bool) {
return
}
}
+
+ ctx.Data["CanReadProjects"] = ctx.Org.CanReadUnit(ctx, unit.TypeProjects)
+ ctx.Data["CanReadPackages"] = ctx.Org.CanReadUnit(ctx, unit.TypePackages)
+ ctx.Data["CanReadCode"] = ctx.Org.CanReadUnit(ctx, unit.TypeCode)
}
// OrgAssignment returns a middleware to handle organization assignment
diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go
index c627597fa..6b91a81fc 100644
--- a/modules/csv/csv_test.go
+++ b/modules/csv/csv_test.go
@@ -11,6 +11,7 @@ import (
"strings"
"testing"
+ "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/markup"
"github.com/stretchr/testify/assert"
@@ -229,7 +230,10 @@ John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimiters`,
}
for n, c := range cases {
- delimiter := determineDelimiter(&markup.RenderContext{RelativePath: c.filename}, []byte(decodeSlashes(t, c.csv)))
+ delimiter := determineDelimiter(&markup.RenderContext{
+ Ctx: git.DefaultContext,
+ RelativePath: c.filename,
+ }, []byte(decodeSlashes(t, c.csv)))
assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter)
}
}
diff --git a/modules/doctor/dbconsistency.go b/modules/doctor/dbconsistency.go
index bb560ac6a..541fc736f 100644
--- a/modules/doctor/dbconsistency.go
+++ b/modules/doctor/dbconsistency.go
@@ -155,7 +155,7 @@ func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) er
// TODO: function to recalc all counters
- if setting.Database.UsePostgreSQL {
+ if setting.Database.Type.IsPostgreSQL() {
consistencyChecks = append(consistencyChecks, consistencyCheck{
Name: "Sequence values",
Counter: db.CountBadSequences,
diff --git a/modules/git/command.go b/modules/git/command.go
index 0bc810311..9a65279a8 100644
--- a/modules/git/command.go
+++ b/modules/git/command.go
@@ -179,7 +179,7 @@ func (c *Command) AddDashesAndList(list ...string) *Command {
}
// ToTrustedCmdArgs converts a list of strings (trusted as argument) to TrustedCmdArgs
-// In most cases, it shouldn't be used. Use AddXxx function instead
+// In most cases, it shouldn't be used. Use NewCommand().AddXxx() function instead
func ToTrustedCmdArgs(args []string) TrustedCmdArgs {
ret := make(TrustedCmdArgs, len(args))
for i, arg := range args {
diff --git a/modules/git/commit.go b/modules/git/commit.go
index 4a55645d3..6e8fcb3e0 100644
--- a/modules/git/commit.go
+++ b/modules/git/commit.go
@@ -218,6 +218,19 @@ func (c *Commit) HasPreviousCommit(commitHash SHA1) (bool, error) {
return false, err
}
+// IsForcePush returns true if a push from oldCommitHash to this is a force push
+func (c *Commit) IsForcePush(oldCommitID string) (bool, error) {
+ if oldCommitID == EmptySHA {
+ return false, nil
+ }
+ oldCommit, err := c.repo.GetCommit(oldCommitID)
+ if err != nil {
+ return false, err
+ }
+ hasPreviousCommit, err := c.HasPreviousCommit(oldCommit.ID)
+ return !hasPreviousCommit, err
+}
+
// CommitsBeforeLimit returns num commits before current revision
func (c *Commit) CommitsBeforeLimit(num int) ([]*Commit, error) {
return c.repo.getCommitsBeforeLimit(c.ID, num)
diff --git a/modules/git/git.go b/modules/git/git.go
index 2feb242ac..24cfea8c7 100644
--- a/modules/git/git.go
+++ b/modules/git/git.go
@@ -312,7 +312,7 @@ func CheckGitVersionAtLeast(atLeast string) error {
}
func configSet(key, value string) error {
- stdout, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key).RunStdString(nil)
+ stdout, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil)
if err != nil && !err.IsExitCode(1) {
return fmt.Errorf("failed to get git config %s, err: %w", key, err)
}
@@ -331,7 +331,7 @@ func configSet(key, value string) error {
}
func configSetNonExist(key, value string) error {
- _, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key).RunStdString(nil)
+ _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil)
if err == nil {
// already exist
return nil
@@ -349,7 +349,7 @@ func configSetNonExist(key, value string) error {
}
func configAddNonExist(key, value string) error {
- _, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil)
+ _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil)
if err == nil {
// already exist
return nil
@@ -366,7 +366,7 @@ func configAddNonExist(key, value string) error {
}
func configUnsetAll(key, value string) error {
- _, _, err := NewCommand(DefaultContext, "config", "--get").AddDynamicArguments(key).RunStdString(nil)
+ _, _, err := NewCommand(DefaultContext, "config", "--global", "--get").AddDynamicArguments(key).RunStdString(nil)
if err == nil {
// exist, need to remove
_, _, err = NewCommand(DefaultContext, "config", "--global", "--unset-all").AddDynamicArguments(key, regexp.QuoteMeta(value)).RunStdString(nil)
diff --git a/modules/git/pipeline/revlist.go b/modules/git/pipeline/revlist.go
index 09bb2c8b3..d88ebe78e 100644
--- a/modules/git/pipeline/revlist.go
+++ b/modules/git/pipeline/revlist.go
@@ -42,7 +42,10 @@ func RevListObjects(ctx context.Context, revListWriter *io.PipeWriter, wg *sync.
defer revListWriter.Close()
stderr := new(bytes.Buffer)
var errbuf strings.Builder
- cmd := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(headSHA).AddArguments("--not").AddDynamicArguments(baseSHA)
+ cmd := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(headSHA)
+ if baseSHA != "" {
+ cmd = cmd.AddArguments("--not").AddDynamicArguments(baseSHA)
+ }
if err := cmd.Run(&git.RunOpts{
Dir: tmpBasePath,
Stdout: revListWriter,
diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go
index a62e0670f..0e1b00ce0 100644
--- a/modules/git/repo_commit.go
+++ b/modules/git/repo_commit.go
@@ -323,6 +323,27 @@ func (repo *Repository) CommitsBetweenLimit(last, before *Commit, limit, skip in
return repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout))
}
+// CommitsBetweenNotBase returns a list that contains commits between [before, last), excluding commits in baseBranch.
+// If before is detached (removed by reset + push) it is not included.
+func (repo *Repository) CommitsBetweenNotBase(last, before *Commit, baseBranch string) ([]*Commit, error) {
+ var stdout []byte
+ var err error
+ if before == nil {
+ stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path})
+ } else {
+ stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String()+".."+last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path})
+ if err != nil && strings.Contains(err.Error(), "no merge base") {
+ // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated.
+ // previously it would return the results of git rev-list before last so let's try that...
+ stdout, _, err = NewCommand(repo.Ctx, "rev-list").AddDynamicArguments(before.ID.String(), last.ID.String()).AddOptionValues("--not", baseBranch).RunStdBytes(&RunOpts{Dir: repo.Path})
+ }
+ }
+ if err != nil {
+ return nil, err
+ }
+ return repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout))
+}
+
// CommitsBetweenIDs return commits between twoe commits
func (repo *Repository) CommitsBetweenIDs(last, before string) ([]*Commit, error) {
lastCommit, err := repo.GetCommit(last)
diff --git a/modules/git/repo_commit_gogit.go b/modules/git/repo_commit_gogit.go
index 72de158e6..ce0af9361 100644
--- a/modules/git/repo_commit_gogit.go
+++ b/modules/git/repo_commit_gogit.go
@@ -7,7 +7,6 @@
package git
import (
- "fmt"
"strings"
"github.com/go-git/go-git/v5/plumbing"
@@ -67,38 +66,6 @@ func (repo *Repository) IsCommitExist(name string) bool {
return err == nil
}
-func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature {
- if t.PGPSignature == "" {
- return nil
- }
-
- var w strings.Builder
- var err error
-
- if _, err = fmt.Fprintf(&w,
- "object %s\ntype %s\ntag %s\ntagger ",
- t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil {
- return nil
- }
-
- if err = t.Tagger.Encode(&w); err != nil {
- return nil
- }
-
- if _, err = fmt.Fprintf(&w, "\n\n"); err != nil {
- return nil
- }
-
- if _, err = fmt.Fprintf(&w, t.Message); err != nil {
- return nil
- }
-
- return &CommitGPGSignature{
- Signature: t.PGPSignature,
- Payload: strings.TrimSpace(w.String()) + "\n",
- }
-}
-
func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
var tagObject *object.Tag
@@ -122,12 +89,6 @@ func (repo *Repository) getCommit(id SHA1) (*Commit, error) {
commit := convertCommit(gogitCommit)
commit.repo = repo
- if tagObject != nil {
- commit.CommitMessage = strings.TrimSpace(tagObject.Message)
- commit.Author = &tagObject.Tagger
- commit.Signature = convertPGPSignatureForTag(tagObject)
- }
-
tree, err := gogitCommit.Tree()
if err != nil {
return nil, err
diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go
index 7373d01c8..d5eb72310 100644
--- a/modules/git/repo_commit_nogogit.go
+++ b/modules/git/repo_commit_nogogit.go
@@ -107,10 +107,6 @@ func (repo *Repository) getCommitFromBatchReader(rd *bufio.Reader, id SHA1) (*Co
return nil, err
}
- commit.CommitMessage = strings.TrimSpace(tag.Message)
- commit.Author = tag.Tagger
- commit.Signature = tag.Signature
-
return commit, nil
case "commit":
commit, err := CommitFromReader(repo, id, io.LimitReader(rd, size))
diff --git a/modules/git/repo_commit_test.go b/modules/git/repo_commit_test.go
index af8c0592f..729fb0ba1 100644
--- a/modules/git/repo_commit_test.go
+++ b/modules/git/repo_commit_test.go
@@ -43,12 +43,13 @@ func TestGetTagCommitWithSignature(t *testing.T) {
assert.NoError(t, err)
defer bareRepo1.Close()
- commit, err := bareRepo1.GetCommit("3ad28a9149a2864384548f3d17ed7f38014c9e8a")
+ // both the tag and the commit are signed here, this validates only the commit signature
+ commit, err := bareRepo1.GetCommit("28b55526e7100924d864dd89e35c1ea62e7a5a32")
assert.NoError(t, err)
assert.NotNil(t, commit)
assert.NotNil(t, commit.Signature)
// test that signature is not in message
- assert.Equal(t, "tag", commit.CommitMessage)
+ assert.Equal(t, "signed-commit\n", commit.CommitMessage)
}
func TestGetCommitWithBadCommitID(t *testing.T) {
diff --git a/modules/git/repo_compare.go b/modules/git/repo_compare.go
index 9a4d66f2f..439455e3c 100644
--- a/modules/git/repo_compare.go
+++ b/modules/git/repo_compare.go
@@ -277,11 +277,18 @@ func (repo *Repository) GetPatch(base, head string, w io.Writer) error {
// GetFilesChangedBetween returns a list of all files that have been changed between the given commits
func (repo *Repository) GetFilesChangedBetween(base, head string) ([]string, error) {
- stdout, _, err := NewCommand(repo.Ctx, "diff", "--name-only").AddDynamicArguments(base + ".." + head).RunStdString(&RunOpts{Dir: repo.Path})
+ stdout, _, err := NewCommand(repo.Ctx, "diff", "--name-only", "-z").AddDynamicArguments(base + ".." + head).RunStdString(&RunOpts{Dir: repo.Path})
if err != nil {
return nil, err
}
- return strings.Split(stdout, "\n"), err
+ split := strings.Split(stdout, "\000")
+
+ // Because Git will always emit filenames with a terminal NUL ignore the last entry in the split - which will always be empty.
+ if len(split) > 0 {
+ split = split[:len(split)-1]
+ }
+
+ return split, err
}
// GetDiffFromMergeBase generates and return patch data from merge base to head
diff --git a/modules/git/repo_ref_test.go b/modules/git/repo_ref_test.go
index 776d7ce3e..c08ea1276 100644
--- a/modules/git/repo_ref_test.go
+++ b/modules/git/repo_ref_test.go
@@ -19,13 +19,14 @@ func TestRepository_GetRefs(t *testing.T) {
refs, err := bareRepo1.GetRefs()
assert.NoError(t, err)
- assert.Len(t, refs, 5)
+ assert.Len(t, refs, 6)
expectedRefs := []string{
BranchPrefix + "branch1",
BranchPrefix + "branch2",
BranchPrefix + "master",
TagPrefix + "test",
+ TagPrefix + "signed-tag",
NotesRef,
}
@@ -43,9 +44,12 @@ func TestRepository_GetRefsFiltered(t *testing.T) {
refs, err := bareRepo1.GetRefsFiltered(TagPrefix)
assert.NoError(t, err)
- if assert.Len(t, refs, 1) {
- assert.Equal(t, TagPrefix+"test", refs[0].Name)
+ if assert.Len(t, refs, 2) {
+ assert.Equal(t, TagPrefix+"signed-tag", refs[0].Name)
assert.Equal(t, "tag", refs[0].Type)
- assert.Equal(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", refs[0].Object.String())
+ assert.Equal(t, "36f97d9a96457e2bab511db30fe2db03893ebc64", refs[0].Object.String())
+ assert.Equal(t, TagPrefix+"test", refs[1].Name)
+ assert.Equal(t, "tag", refs[1].Type)
+ assert.Equal(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", refs[1].Object.String())
}
}
diff --git a/modules/git/repo_stats_test.go b/modules/git/repo_stats_test.go
index 668ed6799..3d032385e 100644
--- a/modules/git/repo_stats_test.go
+++ b/modules/git/repo_stats_test.go
@@ -24,9 +24,9 @@ func TestRepository_GetCodeActivityStats(t *testing.T) {
assert.NoError(t, err)
assert.NotNil(t, code)
- assert.EqualValues(t, 9, code.CommitCount)
+ assert.EqualValues(t, 10, code.CommitCount)
assert.EqualValues(t, 3, code.AuthorCount)
- assert.EqualValues(t, 9, code.CommitCountInAllBranches)
+ assert.EqualValues(t, 10, code.CommitCountInAllBranches)
assert.EqualValues(t, 10, code.Additions)
assert.EqualValues(t, 1, code.Deletions)
assert.Len(t, code.Authors, 3)
diff --git a/modules/git/repo_tag_test.go b/modules/git/repo_tag_test.go
index 589349a72..8ccfec3eb 100644
--- a/modules/git/repo_tag_test.go
+++ b/modules/git/repo_tag_test.go
@@ -25,11 +25,14 @@ func TestRepository_GetTags(t *testing.T) {
assert.NoError(t, err)
return
}
- assert.Len(t, tags, 1)
+ assert.Len(t, tags, 2)
assert.Equal(t, len(tags), total)
- assert.EqualValues(t, "test", tags[0].Name)
- assert.EqualValues(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", tags[0].ID.String())
+ assert.EqualValues(t, "signed-tag", tags[0].Name)
+ assert.EqualValues(t, "36f97d9a96457e2bab511db30fe2db03893ebc64", tags[0].ID.String())
assert.EqualValues(t, "tag", tags[0].Type)
+ assert.EqualValues(t, "test", tags[1].Name)
+ assert.EqualValues(t, "3ad28a9149a2864384548f3d17ed7f38014c9e8a", tags[1].ID.String())
+ assert.EqualValues(t, "tag", tags[1].Type)
}
func TestRepository_GetTag(t *testing.T) {
diff --git a/modules/git/repo_test.go b/modules/git/repo_test.go
index 2a3914819..044b9d406 100644
--- a/modules/git/repo_test.go
+++ b/modules/git/repo_test.go
@@ -14,10 +14,10 @@ func TestGetLatestCommitTime(t *testing.T) {
bareRepo1Path := filepath.Join(testReposDir, "repo1_bare")
lct, err := GetLatestCommitTime(DefaultContext, bareRepo1Path)
assert.NoError(t, err)
- // Time is Sun Jul 21 22:43:13 2019 +0200
+ // Time is Sun Nov 13 16:40:14 2022 +0100
// which is the time of commit
- // feaf4ba6bc635fec442f46ddd4512416ec43c2c2 (refs/heads/master)
- assert.EqualValues(t, 1563741793, lct.Unix())
+ // ce064814f4a0d337b333e646ece456cd39fab612 (refs/heads/master)
+ assert.EqualValues(t, 1668354014, lct.Unix())
}
func TestRepoIsEmpty(t *testing.T) {
diff --git a/modules/git/tests/repos/repo1_bare/index b/modules/git/tests/repos/repo1_bare/index
new file mode 100644
index 000000000..65d675154
--- /dev/null
+++ b/modules/git/tests/repos/repo1_bare/index
Binary files differ
diff --git a/modules/git/tests/repos/repo1_bare/logs/HEAD b/modules/git/tests/repos/repo1_bare/logs/HEAD
index cef4ca2dc..46da5fe0b 100644
--- a/modules/git/tests/repos/repo1_bare/logs/HEAD
+++ b/modules/git/tests/repos/repo1_bare/logs/HEAD
@@ -1 +1,2 @@
37991dec2c8e592043f47155ce4808d4580f9123 feaf4ba6bc635fec442f46ddd4512416ec43c2c2 silverwind <me@silverwind.io> 1563741799 +0200 push
+feaf4ba6bc635fec442f46ddd4512416ec43c2c2 ce064814f4a0d337b333e646ece456cd39fab612 silverwind <me@silverwind.io> 1668354026 +0100 push
diff --git a/modules/git/tests/repos/repo1_bare/logs/refs/heads/master b/modules/git/tests/repos/repo1_bare/logs/refs/heads/master
index cef4ca2dc..46da5fe0b 100644
--- a/modules/git/tests/repos/repo1_bare/logs/refs/heads/master
+++ b/modules/git/tests/repos/repo1_bare/logs/refs/heads/master
@@ -1 +1,2 @@
37991dec2c8e592043f47155ce4808d4580f9123 feaf4ba6bc635fec442f46ddd4512416ec43c2c2 silverwind <me@silverwind.io> 1563741799 +0200 push
+feaf4ba6bc635fec442f46ddd4512416ec43c2c2 ce064814f4a0d337b333e646ece456cd39fab612 silverwind <me@silverwind.io> 1668354026 +0100 push
diff --git a/modules/git/tests/repos/repo1_bare/objects/1c/91d130dc5fb75fd2d9f586a058650889cfe7fb b/modules/git/tests/repos/repo1_bare/objects/1c/91d130dc5fb75fd2d9f586a058650889cfe7fb
new file mode 100644
index 000000000..fb50b65f9
--- /dev/null
+++ b/modules/git/tests/repos/repo1_bare/objects/1c/91d130dc5fb75fd2d9f586a058650889cfe7fb
Binary files differ
diff --git a/modules/git/tests/repos/repo1_bare/objects/28/b55526e7100924d864dd89e35c1ea62e7a5a32 b/modules/git/tests/repos/repo1_bare/objects/28/b55526e7100924d864dd89e35c1ea62e7a5a32
new file mode 100644
index 000000000..7779599a0
--- /dev/null
+++ b/modules/git/tests/repos/repo1_bare/objects/28/b55526e7100924d864dd89e35c1ea62e7a5a32
Binary files differ
diff --git a/modules/git/tests/repos/repo1_bare/objects/36/f97d9a96457e2bab511db30fe2db03893ebc64 b/modules/git/tests/repos/repo1_bare/objects/36/f97d9a96457e2bab511db30fe2db03893ebc64
new file mode 100644
index 000000000..c96b84390
--- /dev/null
+++ b/modules/git/tests/repos/repo1_bare/objects/36/f97d9a96457e2bab511db30fe2db03893ebc64
Binary files differ
diff --git a/modules/git/tests/repos/repo1_bare/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904 b/modules/git/tests/repos/repo1_bare/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904
new file mode 100644
index 000000000..adf64119a
--- /dev/null
+++ b/modules/git/tests/repos/repo1_bare/objects/4b/825dc642cb6eb9a060e54bf8d69288fbee4904
Binary files differ
diff --git a/modules/git/tests/repos/repo1_bare/objects/93/3305878a3c9ad485c29b87fb662a73a9675c4b b/modules/git/tests/repos/repo1_bare/objects/93/3305878a3c9ad485c29b87fb662a73a9675c4b
new file mode 100644
index 000000000..e198e7658
--- /dev/null
+++ b/modules/git/tests/repos/repo1_bare/objects/93/3305878a3c9ad485c29b87fb662a73a9675c4b
Binary files differ
diff --git a/modules/git/tests/repos/repo1_bare/objects/ce/064814f4a0d337b333e646ece456cd39fab612 b/modules/git/tests/repos/repo1_bare/objects/ce/064814f4a0d337b333e646ece456cd39fab612
new file mode 100644
index 000000000..93f1525e5
--- /dev/null
+++ b/modules/git/tests/repos/repo1_bare/objects/ce/064814f4a0d337b333e646ece456cd39fab612
Binary files differ
diff --git a/modules/git/tests/repos/repo1_bare/objects/cf/8b0b492a950b358a7ce7f9d01b18aef48a6b2d b/modules/git/tests/repos/repo1_bare/objects/cf/8b0b492a950b358a7ce7f9d01b18aef48a6b2d
new file mode 100644
index 000000000..1152b25bb
--- /dev/null
+++ b/modules/git/tests/repos/repo1_bare/objects/cf/8b0b492a950b358a7ce7f9d01b18aef48a6b2d
Binary files differ
diff --git a/modules/git/tests/repos/repo1_bare/refs/heads/master b/modules/git/tests/repos/repo1_bare/refs/heads/master
index c5e92eb22..9b0de2281 100644
--- a/modules/git/tests/repos/repo1_bare/refs/heads/master
+++ b/modules/git/tests/repos/repo1_bare/refs/heads/master
@@ -1 +1 @@
-feaf4ba6bc635fec442f46ddd4512416ec43c2c2
+ce064814f4a0d337b333e646ece456cd39fab612
diff --git a/modules/git/tests/repos/repo1_bare/refs/tags/signed-tag b/modules/git/tests/repos/repo1_bare/refs/tags/signed-tag
new file mode 100644
index 000000000..3998a6850
--- /dev/null
+++ b/modules/git/tests/repos/repo1_bare/refs/tags/signed-tag
@@ -0,0 +1 @@
+36f97d9a96457e2bab511db30fe2db03893ebc64
diff --git a/modules/httpcache/httpcache.go b/modules/httpcache/httpcache.go
index f0caa30eb..46e0152ef 100644
--- a/modules/httpcache/httpcache.go
+++ b/modules/httpcache/httpcache.go
@@ -15,8 +15,8 @@ import (
"code.gitea.io/gitea/modules/setting"
)
-// AddCacheControlToHeader adds suitable cache-control headers to response
-func AddCacheControlToHeader(h http.Header, maxAge time.Duration, additionalDirectives ...string) {
+// SetCacheControlInHeader sets suitable cache-control headers in the response
+func SetCacheControlInHeader(h http.Header, maxAge time.Duration, additionalDirectives ...string) {
directives := make([]string, 0, 2+len(additionalDirectives))
// "max-age=0 + must-revalidate" (aka "no-cache") is preferred instead of "no-store"
@@ -31,7 +31,7 @@ func AddCacheControlToHeader(h http.Header, maxAge time.Duration, additionalDire
directives = append(directives, "max-age=0", "private", "must-revalidate")
// to remind users they are using non-prod setting.
- h.Add("X-Gitea-Debug", "RUN_MODE="+setting.RunMode)
+ h.Set("X-Gitea-Debug", "RUN_MODE="+setting.RunMode)
}
h.Set("Cache-Control", strings.Join(append(directives, additionalDirectives...), ", "))
@@ -50,7 +50,7 @@ func HandleTimeCache(req *http.Request, w http.ResponseWriter, fi os.FileInfo) (
// HandleGenericTimeCache handles time-based caching for a HTTP request
func HandleGenericTimeCache(req *http.Request, w http.ResponseWriter, lastModified time.Time) (handled bool) {
- AddCacheControlToHeader(w.Header(), setting.StaticCacheTime)
+ SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
ifModifiedSince := req.Header.Get("If-Modified-Since")
if ifModifiedSince != "" {
@@ -81,7 +81,7 @@ func HandleGenericETagCache(req *http.Request, w http.ResponseWriter, etag strin
return true
}
}
- AddCacheControlToHeader(w.Header(), setting.StaticCacheTime)
+ SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
return false
}
@@ -125,6 +125,6 @@ func HandleGenericETagTimeCache(req *http.Request, w http.ResponseWriter, etag s
}
}
}
- AddCacheControlToHeader(w.Header(), setting.StaticCacheTime)
+ SetCacheControlInHeader(w.Header(), setting.StaticCacheTime)
return false
}
diff --git a/modules/label/label.go b/modules/label/label.go
new file mode 100644
index 000000000..d3ef0e1dc
--- /dev/null
+++ b/modules/label/label.go
@@ -0,0 +1,46 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package label
+
+import (
+ "fmt"
+ "regexp"
+ "strings"
+)
+
+// colorPattern is a regexp which can validate label color
+var colorPattern = regexp.MustCompile("^#?(?:[0-9a-fA-F]{6}|[0-9a-fA-F]{3})$")
+
+// Label represents label information loaded from template
+type Label struct {
+ Name string `yaml:"name"`
+ Color string `yaml:"color"`
+ Description string `yaml:"description,omitempty"`
+ Exclusive bool `yaml:"exclusive,omitempty"`
+}
+
+// NormalizeColor normalizes a color string to a 6-character hex code
+func NormalizeColor(color string) (string, error) {
+ // normalize case
+ color = strings.TrimSpace(strings.ToLower(color))
+
+ // add leading hash
+ if len(color) == 6 || len(color) == 3 {
+ color = "#" + color
+ }
+
+ if !colorPattern.MatchString(color) {
+ return "", fmt.Errorf("bad color code: %s", color)
+ }
+
+ // convert 3-character shorthand into 6-character version
+ if len(color) == 4 {
+ r := color[1]
+ g := color[2]
+ b := color[3]
+ color = fmt.Sprintf("#%c%c%c%c%c%c", r, r, g, g, b, b)
+ }
+
+ return color, nil
+}
diff --git a/modules/label/parser.go b/modules/label/parser.go
new file mode 100644
index 000000000..55bf570de
--- /dev/null
+++ b/modules/label/parser.go
@@ -0,0 +1,126 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package label
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+
+ "code.gitea.io/gitea/modules/options"
+
+ "gopkg.in/yaml.v3"
+)
+
+type labelFile struct {
+ Labels []*Label `yaml:"labels"`
+}
+
+// ErrTemplateLoad represents a "ErrTemplateLoad" kind of error.
+type ErrTemplateLoad struct {
+ TemplateFile string
+ OriginalError error
+}
+
+// IsErrTemplateLoad checks if an error is a ErrTemplateLoad.
+func IsErrTemplateLoad(err error) bool {
+ _, ok := err.(ErrTemplateLoad)
+ return ok
+}
+
+func (err ErrTemplateLoad) Error() string {
+ return fmt.Sprintf("Failed to load label template file '%s': %v", err.TemplateFile, err.OriginalError)
+}
+
+// GetTemplateFile loads the label template file by given name,
+// then parses and returns a list of name-color pairs and optionally description.
+func GetTemplateFile(name string) ([]*Label, error) {
+ data, err := options.Labels(name + ".yaml")
+ if err == nil && len(data) > 0 {
+ return parseYamlFormat(name+".yaml", data)
+ }
+
+ data, err = options.Labels(name + ".yml")
+ if err == nil && len(data) > 0 {
+ return parseYamlFormat(name+".yml", data)
+ }
+
+ data, err = options.Labels(name)
+ if err != nil {
+ return nil, ErrTemplateLoad{name, fmt.Errorf("GetRepoInitFile: %w", err)}
+ }
+
+ return parseLegacyFormat(name, data)
+}
+
+func parseYamlFormat(name string, data []byte) ([]*Label, error) {
+ lf := &labelFile{}
+
+ if err := yaml.Unmarshal(data, lf); err != nil {
+ return nil, err
+ }
+
+ // Validate label data and fix colors
+ for _, l := range lf.Labels {
+ l.Color = strings.TrimSpace(l.Color)
+ if len(l.Name) == 0 || len(l.Color) == 0 {
+ return nil, ErrTemplateLoad{name, errors.New("label name and color are required fields")}
+ }
+ color, err := NormalizeColor(l.Color)
+ if err != nil {
+ return nil, ErrTemplateLoad{name, fmt.Errorf("bad HTML color code '%s' in label: %s", l.Color, l.Name)}
+ }
+ l.Color = color
+ }
+
+ return lf.Labels, nil
+}
+
+func parseLegacyFormat(name string, data []byte) ([]*Label, error) {
+ lines := strings.Split(string(data), "\n")
+ list := make([]*Label, 0, len(lines))
+ for i := 0; i < len(lines); i++ {
+ line := strings.TrimSpace(lines[i])
+ if len(line) == 0 {
+ continue
+ }
+
+ parts, description, _ := strings.Cut(line, ";")
+
+ color, name, ok := strings.Cut(parts, " ")
+ if !ok {
+ return nil, ErrTemplateLoad{name, fmt.Errorf("line is malformed: %s", line)}
+ }
+
+ color, err := NormalizeColor(color)
+ if err != nil {
+ return nil, ErrTemplateLoad{name, fmt.Errorf("bad HTML color code '%s' in line: %s", color, line)}
+ }
+
+ list = append(list, &Label{
+ Name: strings.TrimSpace(name),
+ Color: color,
+ Description: strings.TrimSpace(description),
+ })
+ }
+
+ return list, nil
+}
+
+// LoadFormatted loads the labels' list of a template file as a string separated by comma
+func LoadFormatted(name string) (string, error) {
+ var buf strings.Builder
+ list, err := GetTemplateFile(name)
+ if err != nil {
+ return "", err
+ }
+
+ for i := 0; i < len(list); i++ {
+ if i > 0 {
+ buf.WriteString(", ")
+ }
+ buf.WriteString(list[i].Name)
+ }
+ return buf.String(), nil
+}
diff --git a/modules/label/parser_test.go b/modules/label/parser_test.go
new file mode 100644
index 000000000..5c8042f66
--- /dev/null
+++ b/modules/label/parser_test.go
@@ -0,0 +1,72 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package label
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestYamlParser(t *testing.T) {
+ data := []byte(`labels:
+ - name: priority/low
+ exclusive: true
+ color: "#0000ee"
+ description: "Low priority"
+ - name: priority/medium
+ exclusive: true
+ color: "0e0"
+ description: "Medium priority"
+ - name: priority/high
+ exclusive: true
+ color: "#ee0000"
+ description: "High priority"
+ - name: type/bug
+ color: "#f00"
+ description: "Bug"`)
+
+ labels, err := parseYamlFormat("test", data)
+ require.NoError(t, err)
+ require.Len(t, labels, 4)
+ assert.Equal(t, "priority/low", labels[0].Name)
+ assert.True(t, labels[0].Exclusive)
+ assert.Equal(t, "#0000ee", labels[0].Color)
+ assert.Equal(t, "Low priority", labels[0].Description)
+ assert.Equal(t, "priority/medium", labels[1].Name)
+ assert.True(t, labels[1].Exclusive)
+ assert.Equal(t, "#00ee00", labels[1].Color)
+ assert.Equal(t, "Medium priority", labels[1].Description)
+ assert.Equal(t, "priority/high", labels[2].Name)
+ assert.True(t, labels[2].Exclusive)
+ assert.Equal(t, "#ee0000", labels[2].Color)
+ assert.Equal(t, "High priority", labels[2].Description)
+ assert.Equal(t, "type/bug", labels[3].Name)
+ assert.False(t, labels[3].Exclusive)
+ assert.Equal(t, "#ff0000", labels[3].Color)
+ assert.Equal(t, "Bug", labels[3].Description)
+}
+
+func TestLegacyParser(t *testing.T) {
+ data := []byte(`#ee0701 bug ; Something is not working
+#cccccc duplicate ; This issue or pull request already exists
+#84b6eb enhancement`)
+
+ labels, err := parseLegacyFormat("test", data)
+ require.NoError(t, err)
+ require.Len(t, labels, 3)
+ assert.Equal(t, "bug", labels[0].Name)
+ assert.False(t, labels[0].Exclusive)
+ assert.Equal(t, "#ee0701", labels[0].Color)
+ assert.Equal(t, "Something is not working", labels[0].Description)
+ assert.Equal(t, "duplicate", labels[1].Name)
+ assert.False(t, labels[1].Exclusive)
+ assert.Equal(t, "#cccccc", labels[1].Color)
+ assert.Equal(t, "This issue or pull request already exists", labels[1].Description)
+ assert.Equal(t, "enhancement", labels[2].Name)
+ assert.False(t, labels[2].Exclusive)
+ assert.Equal(t, "#84b6eb", labels[2].Color)
+ assert.Empty(t, labels[2].Description)
+}
diff --git a/modules/markup/console/console_test.go b/modules/markup/console/console_test.go
index 506f86194..2337d91ac 100644
--- a/modules/markup/console/console_test.go
+++ b/modules/markup/console/console_test.go
@@ -7,6 +7,7 @@ import (
"strings"
"testing"
+ "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/markup"
"github.com/stretchr/testify/assert"
@@ -23,7 +24,8 @@ func TestRenderConsole(t *testing.T) {
canRender := render.CanRender("test", strings.NewReader(k))
assert.True(t, canRender)
- err := render.Render(&markup.RenderContext{}, strings.NewReader(k), &buf)
+ err := render.Render(&markup.RenderContext{Ctx: git.DefaultContext},
+ strings.NewReader(k), &buf)
assert.NoError(t, err)
assert.EqualValues(t, v, buf.String())
}
diff --git a/modules/markup/csv/csv_test.go b/modules/markup/csv/csv_test.go
index b9f5769be..8c07184b2 100644
--- a/modules/markup/csv/csv_test.go
+++ b/modules/markup/csv/csv_test.go
@@ -7,6 +7,7 @@ import (
"strings"
"testing"
+ "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/markup"
"github.com/stretchr/testify/assert"
@@ -23,7 +24,8 @@ func TestRenderCSV(t *testing.T) {
for k, v := range kases {
var buf strings.Builder
- err := render.Render(&markup.RenderContext{}, strings.NewReader(k), &buf)
+ err := render.Render(&markup.RenderContext{Ctx: git.DefaultContext},
+ strings.NewReader(k), &buf)
assert.NoError(t, err)
assert.EqualValues(t, v, buf.String())
}
diff --git a/modules/markup/html.go b/modules/markup/html.go
index bcb38f99e..76fc54cf4 100644
--- a/modules/markup/html.go
+++ b/modules/markup/html.go
@@ -291,9 +291,10 @@ func RenderDescriptionHTML(
// RenderEmoji for when we want to just process emoji and shortcodes
// in various places it isn't already run through the normal markdown processor
func RenderEmoji(
+ ctx *RenderContext,
content string,
) (string, error) {
- return renderProcessString(&RenderContext{}, emojiProcessors, content)
+ return renderProcessString(ctx, emojiProcessors, content)
}
var (
diff --git a/modules/markup/html_internal_test.go b/modules/markup/html_internal_test.go
index 6746c40d1..a048f1f52 100644
--- a/modules/markup/html_internal_test.go
+++ b/modules/markup/html_internal_test.go
@@ -9,6 +9,7 @@ import (
"strings"
"testing"
+ "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@@ -70,8 +71,13 @@ var localMetas = map[string]string{
func TestRender_IssueIndexPattern(t *testing.T) {
// numeric: render inputs without valid mentions
test := func(s string) {
- testRenderIssueIndexPattern(t, s, s, &RenderContext{})
- testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: numericMetas})
+ testRenderIssueIndexPattern(t, s, s, &RenderContext{
+ Ctx: git.DefaultContext,
+ })
+ testRenderIssueIndexPattern(t, s, s, &RenderContext{
+ Ctx: git.DefaultContext,
+ Metas: numericMetas,
+ })
}
// should not render anything when there are no mentions
@@ -119,7 +125,10 @@ func TestRender_IssueIndexPattern2(t *testing.T) {
links[i] = numericIssueLink(util.URLJoin(TestRepoURL, path), "ref-issue", index, marker)
}
expectedNil := fmt.Sprintf(expectedFmt, links...)
- testRenderIssueIndexPattern(t, s, expectedNil, &RenderContext{Metas: localMetas})
+ testRenderIssueIndexPattern(t, s, expectedNil, &RenderContext{
+ Ctx: git.DefaultContext,
+ Metas: localMetas,
+ })
class := "ref-issue"
if isExternal {
@@ -130,7 +139,10 @@ func TestRender_IssueIndexPattern2(t *testing.T) {
links[i] = numericIssueLink(prefix, class, index, marker)
}
expectedNum := fmt.Sprintf(expectedFmt, links...)
- testRenderIssueIndexPattern(t, s, expectedNum, &RenderContext{Metas: numericMetas})
+ testRenderIssueIndexPattern(t, s, expectedNum, &RenderContext{
+ Ctx: git.DefaultContext,
+ Metas: numericMetas,
+ })
}
// should render freestanding mentions
@@ -164,7 +176,10 @@ func TestRender_IssueIndexPattern3(t *testing.T) {
// alphanumeric: render inputs without valid mentions
test := func(s string) {
- testRenderIssueIndexPattern(t, s, s, &RenderContext{Metas: alphanumericMetas})
+ testRenderIssueIndexPattern(t, s, s, &RenderContext{
+ Ctx: git.DefaultContext,
+ Metas: alphanumericMetas,
+ })
}
test("")
test("this is a test")
@@ -194,7 +209,10 @@ func TestRender_IssueIndexPattern4(t *testing.T) {
links[i] = externalIssueLink("https://someurl.com/someUser/someRepo/", "ref-issue ref-external-issue", name)
}
expected := fmt.Sprintf(expectedFmt, links...)
- testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: alphanumericMetas})
+ testRenderIssueIndexPattern(t, s, expected, &RenderContext{
+ Ctx: git.DefaultContext,
+ Metas: alphanumericMetas,
+ })
}
test("OTT-1234 test", "%s test", "OTT-1234")
test("test T-12 issue", "test %s issue", "T-12")
@@ -214,7 +232,10 @@ func TestRender_IssueIndexPattern5(t *testing.T) {
}
expected := fmt.Sprintf(expectedFmt, links...)
- testRenderIssueIndexPattern(t, s, expected, &RenderContext{Metas: metas})
+ testRenderIssueIndexPattern(t, s, expected, &RenderContext{
+ Ctx: git.DefaultContext,
+ Metas: metas,
+ })
}
test("abc ISSUE-123 def", "abc %s def",
@@ -235,7 +256,10 @@ func TestRender_IssueIndexPattern5(t *testing.T) {
[]string{"ISSUE-123"},
)
- testRenderIssueIndexPattern(t, "will not match", "will not match", &RenderContext{Metas: regexpMetas})
+ testRenderIssueIndexPattern(t, "will not match", "will not match", &RenderContext{
+ Ctx: git.DefaultContext,
+ Metas: regexpMetas,
+ })
}
func testRenderIssueIndexPattern(t *testing.T, input, expected string, ctx *RenderContext) {
@@ -255,6 +279,7 @@ func TestRender_AutoLink(t *testing.T) {
test := func(input, expected string) {
var buffer strings.Builder
err := PostProcess(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: TestRepoURL,
Metas: localMetas,
}, strings.NewReader(input), &buffer)
@@ -263,6 +288,7 @@ func TestRender_AutoLink(t *testing.T) {
buffer.Reset()
err = PostProcess(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: TestRepoURL,
Metas: localMetas,
IsWiki: true,
@@ -292,6 +318,7 @@ func TestRender_FullIssueURLs(t *testing.T) {
test := func(input, expected string) {
var result strings.Builder
err := postProcess(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: TestRepoURL,
Metas: localMetas,
}, []processor{fullIssuePatternProcessor}, strings.NewReader(input), &result)
diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go
index aea1d9267..f8b84485c 100644
--- a/modules/markup/html_test.go
+++ b/modules/markup/html_test.go
@@ -91,6 +91,7 @@ func TestRender_CrossReferences(t *testing.T) {
test := func(input, expected string) {
buffer, err := RenderString(&RenderContext{
+ Ctx: git.DefaultContext,
RelativePath: "a.md",
URLPrefix: setting.AppSubURL,
Metas: localMetas,
@@ -135,6 +136,7 @@ func TestRender_links(t *testing.T) {
test := func(input, expected string) {
buffer, err := RenderString(&RenderContext{
+ Ctx: git.DefaultContext,
RelativePath: "a.md",
URLPrefix: TestRepoURL,
}, input)
@@ -234,6 +236,7 @@ func TestRender_email(t *testing.T) {
test := func(input, expected string) {
res, err := RenderString(&RenderContext{
+ Ctx: git.DefaultContext,
RelativePath: "a.md",
URLPrefix: TestRepoURL,
}, input)
@@ -292,6 +295,7 @@ func TestRender_emoji(t *testing.T) {
test := func(input, expected string) {
expected = strings.ReplaceAll(expected, "&", "&amp;")
buffer, err := RenderString(&RenderContext{
+ Ctx: git.DefaultContext,
RelativePath: "a.md",
URLPrefix: TestRepoURL,
}, input)
@@ -355,11 +359,13 @@ func TestRender_ShortLinks(t *testing.T) {
test := func(input, expected, expectedWiki string) {
buffer, err := markdown.RenderString(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: tree,
}, input)
assert.NoError(t, err)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
buffer, err = markdown.RenderString(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: TestRepoURL,
Metas: localMetas,
IsWiki: true,
@@ -461,12 +467,14 @@ func TestRender_RelativeImages(t *testing.T) {
test := func(input, expected, expectedWiki string) {
buffer, err := markdown.RenderString(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: tree,
Metas: localMetas,
}, input)
assert.NoError(t, err)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
buffer, err = markdown.RenderString(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: TestRepoURL,
Metas: localMetas,
IsWiki: true,
@@ -501,6 +509,7 @@ func Test_ParseClusterFuzz(t *testing.T) {
var res strings.Builder
err := PostProcess(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: "https://example.com",
Metas: localMetas,
}, strings.NewReader(data), &res)
@@ -511,6 +520,7 @@ func Test_ParseClusterFuzz(t *testing.T) {
res.Reset()
err = PostProcess(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: "https://example.com",
Metas: localMetas,
}, strings.NewReader(data), &res)
@@ -531,6 +541,7 @@ func TestIssue16020(t *testing.T) {
var res strings.Builder
err := PostProcess(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: "https://example.com",
Metas: localMetas,
}, strings.NewReader(data), &res)
@@ -547,6 +558,7 @@ func BenchmarkEmojiPostprocess(b *testing.B) {
for i := 0; i < b.N; i++ {
var res strings.Builder
err := PostProcess(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: "https://example.com",
Metas: localMetas,
}, strings.NewReader(data), &res)
@@ -557,6 +569,7 @@ func BenchmarkEmojiPostprocess(b *testing.B) {
func TestFuzz(t *testing.T) {
s := "t/l/issues/8#/../../a"
renderContext := RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: "https://example.com/go-gitea/gitea",
Metas: map[string]string{
"user": "go-gitea",
@@ -574,6 +587,7 @@ func TestIssue18471(t *testing.T) {
var res strings.Builder
err := PostProcess(&RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: "https://example.com",
Metas: localMetas,
}, strings.NewReader(data), &res)
diff --git a/modules/markup/markdown/markdown_test.go b/modules/markup/markdown/markdown_test.go
index bb458a65c..0c7650a5f 100644
--- a/modules/markup/markdown/markdown_test.go
+++ b/modules/markup/markdown/markdown_test.go
@@ -52,12 +52,14 @@ func TestRender_StandardLinks(t *testing.T) {
test := func(input, expected, expectedWiki string) {
buffer, err := RenderString(&markup.RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: setting.AppSubURL,
}, input)
assert.NoError(t, err)
assert.Equal(t, strings.TrimSpace(expected), strings.TrimSpace(buffer))
buffer, err = RenderString(&markup.RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: setting.AppSubURL,
IsWiki: true,
}, input)
@@ -81,6 +83,7 @@ func TestRender_Images(t *testing.T) {
test := func(input, expected string) {
buffer, err := RenderString(&markup.RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: setting.AppSubURL,
}, input)
assert.NoError(t, err)
@@ -311,6 +314,7 @@ func TestTotal_RenderWiki(t *testing.T) {
for i := 0; i < len(testCases); i += 2 {
line, err := RenderString(&markup.RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: AppSubURL,
IsWiki: true,
}, testCases[i])
@@ -339,6 +343,7 @@ func TestTotal_RenderString(t *testing.T) {
for i := 0; i < len(testCases); i += 2 {
line, err := RenderString(&markup.RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: AppSubURL,
}, testCases[i])
assert.NoError(t, err)
@@ -348,17 +353,17 @@ func TestTotal_RenderString(t *testing.T) {
func TestRender_RenderParagraphs(t *testing.T) {
test := func(t *testing.T, str string, cnt int) {
- res, err := RenderRawString(&markup.RenderContext{}, str)
+ res, err := RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, str)
assert.NoError(t, err)
assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for unix should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
mac := strings.ReplaceAll(str, "\n", "\r")
- res, err = RenderRawString(&markup.RenderContext{}, mac)
+ res, err = RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, mac)
assert.NoError(t, err)
assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for mac should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
dos := strings.ReplaceAll(str, "\n", "\r\n")
- res, err = RenderRawString(&markup.RenderContext{}, dos)
+ res, err = RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, dos)
assert.NoError(t, err)
assert.Equal(t, cnt, strings.Count(res, "<p"), "Rendered result for windows should have %d paragraph(s) but has %d:\n%s\n", cnt, strings.Count(res, "<p"), res)
}
@@ -386,7 +391,7 @@ func TestMarkdownRenderRaw(t *testing.T) {
for _, testcase := range testcases {
log.Info("Test markdown render error with fuzzy data: %x, the following errors can be recovered", testcase)
- _, err := RenderRawString(&markup.RenderContext{}, string(testcase))
+ _, err := RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, string(testcase))
assert.NoError(t, err)
}
}
@@ -398,7 +403,7 @@ func TestRenderSiblingImages_Issue12925(t *testing.T) {
expected := `<p><a href="/image1" target="_blank" rel="nofollow noopener"><img src="/image1" alt="image1"></a><br>
<a href="/image2" target="_blank" rel="nofollow noopener"><img src="/image2" alt="image2"></a></p>
`
- res, err := RenderRawString(&markup.RenderContext{}, testcase)
+ res, err := RenderRawString(&markup.RenderContext{Ctx: git.DefaultContext}, testcase)
assert.NoError(t, err)
assert.Equal(t, expected, res)
}
@@ -407,7 +412,7 @@ func TestRenderEmojiInLinks_Issue12331(t *testing.T) {
testcase := `[Link with emoji :moon: in text](https://gitea.io)`
expected := `<p><a href="https://gitea.io" rel="nofollow">Link with emoji <span class="emoji" aria-label="waxing gibbous moon">🌔</span> in text</a></p>
`
- res, err := RenderString(&markup.RenderContext{}, testcase)
+ res, err := RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, testcase)
assert.NoError(t, err)
assert.Equal(t, expected, res)
}
@@ -441,7 +446,7 @@ func TestColorPreview(t *testing.T) {
}
for _, test := range positiveTests {
- res, err := RenderString(&markup.RenderContext{}, test.testcase)
+ res, err := RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test.testcase)
assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase)
assert.Equal(t, test.expected, res, "Unexpected result in testcase %q", test.testcase)
@@ -461,7 +466,7 @@ func TestColorPreview(t *testing.T) {
}
for _, test := range negativeTests {
- res, err := RenderString(&markup.RenderContext{}, test)
+ res, err := RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test)
assert.NoError(t, err, "Unexpected error in testcase: %q", test)
assert.NotContains(t, res, `<span class="color-preview" style="background-color: `, "Unexpected result in testcase %q", test)
}
@@ -508,7 +513,7 @@ func TestMathBlock(t *testing.T) {
}
for _, test := range testcases {
- res, err := RenderString(&markup.RenderContext{}, test.testcase)
+ res, err := RenderString(&markup.RenderContext{Ctx: git.DefaultContext}, test.testcase)
assert.NoError(t, err, "Unexpected error in testcase: %q", test.testcase)
assert.Equal(t, test.expected, res, "Unexpected result in testcase %q", test.testcase)
diff --git a/modules/markup/orgmode/orgmode_test.go b/modules/markup/orgmode/orgmode_test.go
index 4acb678cd..d6467c36f 100644
--- a/modules/markup/orgmode/orgmode_test.go
+++ b/modules/markup/orgmode/orgmode_test.go
@@ -7,6 +7,7 @@ import (
"strings"
"testing"
+ "code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@@ -26,6 +27,7 @@ func TestRender_StandardLinks(t *testing.T) {
test := func(input, expected string) {
buffer, err := RenderString(&markup.RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: setting.AppSubURL,
}, input)
assert.NoError(t, err)
@@ -46,6 +48,7 @@ func TestRender_Images(t *testing.T) {
test := func(input, expected string) {
buffer, err := RenderString(&markup.RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: setting.AppSubURL,
}, input)
assert.NoError(t, err)
@@ -65,6 +68,7 @@ func TestRender_Source(t *testing.T) {
test := func(input, expected string) {
buffer, err := RenderString(&markup.RenderContext{
+ Ctx: git.DefaultContext,
URLPrefix: setting.AppSubURL,
}, input)
assert.NoError(t, err)
diff --git a/modules/markup/sanitizer.go b/modules/markup/sanitizer.go
index e59f6c7c8..600ccbf3c 100644
--- a/modules/markup/sanitizer.go
+++ b/modules/markup/sanitizer.go
@@ -132,6 +132,8 @@ func createDefaultPolicy() *bluemonday.Policy {
policy.AllowAttrs(generalSafeAttrs...).OnElements(generalSafeElements...)
+ policy.AllowAttrs("src", "autoplay", "controls").OnElements("video")
+
policy.AllowAttrs("itemscope", "itemtype").OnElements("div")
// FIXME: Need to handle longdesc in img but there is no easy way to do it
diff --git a/modules/options/base.go b/modules/options/base.go
index 039e934b3..e83e8df5d 100644
--- a/modules/options/base.go
+++ b/modules/options/base.go
@@ -7,11 +7,52 @@ import (
"fmt"
"io/fs"
"os"
+ "path"
"path/filepath"
+ "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
)
+// Locale reads the content of a specific locale from static/bindata or custom path.
+func Locale(name string) ([]byte, error) {
+ return fileFromDir(path.Join("locale", util.CleanPath(name)))
+}
+
+// Readme reads the content of a specific readme from static/bindata or custom path.
+func Readme(name string) ([]byte, error) {
+ return fileFromDir(path.Join("readme", util.CleanPath(name)))
+}
+
+// Gitignore reads the content of a gitignore locale from static/bindata or custom path.
+func Gitignore(name string) ([]byte, error) {
+ return fileFromDir(path.Join("gitignore", util.CleanPath(name)))
+}
+
+// License reads the content of a specific license from static/bindata or custom path.
+func License(name string) ([]byte, error) {
+ return fileFromDir(path.Join("license", util.CleanPath(name)))
+}
+
+// Labels reads the content of a specific labels from static/bindata or custom path.
+func Labels(name string) ([]byte, error) {
+ return fileFromDir(path.Join("label", util.CleanPath(name)))
+}
+
+// WalkLocales reads the content of a specific locale
+func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error {
+ if IsDynamic() {
+ if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("failed to walk locales. Error: %w", err)
+ }
+ }
+
+ if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("failed to walk locales. Error: %w", err)
+ }
+ return nil
+}
+
func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error {
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
// name is the path relative to the root
@@ -37,3 +78,18 @@ func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, e
}
return nil
}
+
+func statDirIfExist(dir string) ([]string, error) {
+ isDir, err := util.IsDir(dir)
+ if err != nil {
+ return nil, fmt.Errorf("unable to check if static directory %s is a directory. %w", dir, err)
+ }
+ if !isDir {
+ return nil, nil
+ }
+ files, err := util.StatDir(dir, true)
+ if err != nil {
+ return nil, fmt.Errorf("unable to read directory %q. %w", dir, err)
+ }
+ return files, nil
+}
diff --git a/modules/options/dynamic.go b/modules/options/dynamic.go
index a20253676..8c954492a 100644
--- a/modules/options/dynamic.go
+++ b/modules/options/dynamic.go
@@ -7,10 +7,8 @@ package options
import (
"fmt"
- "io/fs"
"os"
"path"
- "path/filepath"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@@ -27,76 +25,20 @@ func Dir(name string) ([]string, error) {
var result []string
- customDir := path.Join(setting.CustomPath, "options", name)
-
- isDir, err := util.IsDir(customDir)
- if err != nil {
- return []string{}, fmt.Errorf("Unabe to check if custom directory %s is a directory. %w", customDir, err)
- }
- if isDir {
- files, err := util.StatDir(customDir, true)
- if err != nil {
- return []string{}, fmt.Errorf("Failed to read custom directory. %w", err)
- }
-
- result = append(result, files...)
- }
-
- staticDir := path.Join(setting.StaticRootPath, "options", name)
-
- isDir, err = util.IsDir(staticDir)
- if err != nil {
- return []string{}, fmt.Errorf("unable to check if static directory %s is a directory. %w", staticDir, err)
- }
- if isDir {
- files, err := util.StatDir(staticDir, true)
+ for _, dir := range []string{
+ path.Join(setting.CustomPath, "options", name), // custom dir
+ path.Join(setting.StaticRootPath, "options", name), // static dir
+ } {
+ files, err := statDirIfExist(dir)
if err != nil {
- return []string{}, fmt.Errorf("Failed to read static directory. %w", err)
+ return nil, err
}
-
result = append(result, files...)
}
return directories.AddAndGet(name, result), nil
}
-// Locale reads the content of a specific locale from static or custom path.
-func Locale(name string) ([]byte, error) {
- return fileFromDir(path.Join("locale", name))
-}
-
-// WalkLocales reads the content of a specific locale from static or custom path.
-func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error {
- if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
- return fmt.Errorf("failed to walk locales. Error: %w", err)
- }
-
- if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
- return fmt.Errorf("failed to walk locales. Error: %w", err)
- }
- return nil
-}
-
-// Readme reads the content of a specific readme from static or custom path.
-func Readme(name string) ([]byte, error) {
- return fileFromDir(path.Join("readme", name))
-}
-
-// Gitignore reads the content of a specific gitignore from static or custom path.
-func Gitignore(name string) ([]byte, error) {
- return fileFromDir(path.Join("gitignore", name))
-}
-
-// License reads the content of a specific license from static or custom path.
-func License(name string) ([]byte, error) {
- return fileFromDir(path.Join("license", name))
-}
-
-// Labels reads the content of a specific labels from static or custom path.
-func Labels(name string) ([]byte, error) {
- return fileFromDir(path.Join("label", name))
-}
-
// fileFromDir is a helper to read files from static or custom path.
func fileFromDir(name string) ([]byte, error) {
customPath := path.Join(setting.CustomPath, "options", name)
diff --git a/modules/options/static.go b/modules/options/static.go
index ff3c86d3f..549f4e25b 100644
--- a/modules/options/static.go
+++ b/modules/options/static.go
@@ -8,10 +8,8 @@ package options
import (
"fmt"
"io"
- "io/fs"
"os"
"path"
- "path/filepath"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
@@ -28,17 +26,14 @@ func Dir(name string) ([]string, error) {
var result []string
- customDir := path.Join(setting.CustomPath, "options", name)
- isDir, err := util.IsDir(customDir)
- if err != nil {
- return []string{}, fmt.Errorf("unable to check if custom directory %q is a directory. %w", customDir, err)
- }
- if isDir {
- files, err := util.StatDir(customDir, true)
+ for _, dir := range []string{
+ path.Join(setting.CustomPath, "options", name), // custom dir
+ // no static dir
+ } {
+ files, err := statDirIfExist(dir)
if err != nil {
- return []string{}, fmt.Errorf("unable to read custom directory %q. %w", customDir, err)
+ return nil, err
}
-
result = append(result, files...)
}
@@ -69,39 +64,6 @@ func AssetDir(dirName string) ([]string, error) {
return results, nil
}
-// Locale reads the content of a specific locale from bindata or custom path.
-func Locale(name string) ([]byte, error) {
- return fileFromDir(path.Join("locale", name))
-}
-
-// WalkLocales reads the content of a specific locale from static or custom path.
-func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error {
- if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
- return fmt.Errorf("failed to walk locales. Error: %w", err)
- }
- return nil
-}
-
-// Readme reads the content of a specific readme from bindata or custom path.
-func Readme(name string) ([]byte, error) {
- return fileFromDir(path.Join("readme", name))
-}
-
-// Gitignore reads the content of a gitignore locale from bindata or custom path.
-func Gitignore(name string) ([]byte, error) {
- return fileFromDir(path.Join("gitignore", name))
-}
-
-// License reads the content of a specific license from bindata or custom path.
-func License(name string) ([]byte, error) {
- return fileFromDir(path.Join("license", name))
-}
-
-// Labels reads the content of a specific labels from static or custom path.
-func Labels(name string) ([]byte, error) {
- return fileFromDir(path.Join("label", name))
-}
-
// fileFromDir is a helper to read files from bindata or custom path.
func fileFromDir(name string) ([]byte, error) {
customPath := path.Join(setting.CustomPath, "options", name)
diff --git a/modules/public/public.go b/modules/public/public.go
index 42026f9b1..e1d60d89e 100644
--- a/modules/public/public.go
+++ b/modules/public/public.go
@@ -6,7 +6,6 @@ package public
import (
"net/http"
"os"
- "path"
"path/filepath"
"strings"
@@ -14,6 +13,7 @@ import (
"code.gitea.io/gitea/modules/httpcache"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/util"
)
// Options represents the available options to configure the handler.
@@ -103,7 +103,7 @@ func setWellKnownContentType(w http.ResponseWriter, file string) {
func (opts *Options) handle(w http.ResponseWriter, req *http.Request, fs http.FileSystem, file string) bool {
// use clean to keep the file is a valid path with no . or ..
- f, err := fs.Open(path.Clean(file))
+ f, err := fs.Open(util.CleanPath(file))
if err != nil {
if os.IsNotExist(err) {
return false
diff --git a/modules/queue/queue_channel.go b/modules/queue/queue_channel.go
index 6f75b8357..baac09739 100644
--- a/modules/queue/queue_channel.go
+++ b/modules/queue/queue_channel.go
@@ -124,7 +124,10 @@ func (q *ChannelQueue) Shutdown() {
log.Trace("ChannelQueue: %s Flushing", q.name)
// We can't use Cleanup here because that will close the channel
if err := q.FlushWithContext(q.terminateCtx); err != nil {
- log.Warn("ChannelQueue: %s Terminated before completed flushing", q.name)
+ count := atomic.LoadInt64(&q.numInQueue)
+ if count > 0 {
+ log.Warn("ChannelQueue: %s Terminated before completed flushing", q.name)
+ }
return
}
log.Debug("ChannelQueue: %s Flushed", q.name)
diff --git a/modules/queue/queue_disk_channel.go b/modules/queue/queue_disk_channel.go
index c7526714c..91f91f0df 100644
--- a/modules/queue/queue_disk_channel.go
+++ b/modules/queue/queue_disk_channel.go
@@ -94,7 +94,8 @@ func NewPersistableChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) (
},
Workers: 0,
},
- DataDir: config.DataDir,
+ DataDir: config.DataDir,
+ QueueName: config.Name + "-level",
}
levelQueue, err := NewLevelQueue(wrappedHandle, levelCfg, exemplar)
@@ -172,16 +173,18 @@ func (q *PersistableChannelQueue) Run(atShutdown, atTerminate func(func())) {
atShutdown(q.Shutdown)
atTerminate(q.Terminate)
- if lq, ok := q.internal.(*LevelQueue); ok && lq.byteFIFO.Len(lq.shutdownCtx) != 0 {
+ if lq, ok := q.internal.(*LevelQueue); ok && lq.byteFIFO.Len(lq.terminateCtx) != 0 {
// Just run the level queue - we shut it down once it's flushed
go q.internal.Run(func(_ func()) {}, func(_ func()) {})
go func() {
- for !q.IsEmpty() {
- _ = q.internal.Flush(0)
+ for !lq.IsEmpty() {
+ _ = lq.Flush(0)
select {
case <-time.After(100 * time.Millisecond):
- case <-q.internal.(*LevelQueue).shutdownCtx.Done():
- log.Warn("LevelQueue: %s shut down before completely flushed", q.internal.(*LevelQueue).Name())
+ case <-lq.shutdownCtx.Done():
+ if lq.byteFIFO.Len(lq.terminateCtx) > 0 {
+ log.Warn("LevelQueue: %s shut down before completely flushed", q.internal.(*LevelQueue).Name())
+ }
return
}
}
@@ -316,10 +319,22 @@ func (q *PersistableChannelQueue) Shutdown() {
// Redirect all remaining data in the chan to the internal channel
log.Trace("PersistableChannelQueue: %s Redirecting remaining data", q.delayedStarter.name)
close(q.channelQueue.dataChan)
+ countOK, countLost := 0, 0
for data := range q.channelQueue.dataChan {
- _ = q.internal.Push(data)
+ err := q.internal.Push(data)
+ if err != nil {
+ log.Error("PersistableChannelQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err)
+ countLost++
+ } else {
+ countOK++
+ }
atomic.AddInt64(&q.channelQueue.numInQueue, -1)
}
+ if countLost > 0 {
+ log.Warn("PersistableChannelQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost)
+ } else if countOK > 0 {
+ log.Warn("PersistableChannelQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK)
+ }
log.Trace("PersistableChannelQueue: %s Done Redirecting remaining data", q.delayedStarter.name)
log.Debug("PersistableChannelQueue: %s Shutdown", q.delayedStarter.name)
diff --git a/modules/queue/queue_disk_channel_test.go b/modules/queue/queue_disk_channel_test.go
index 318610355..4f14a5d79 100644
--- a/modules/queue/queue_disk_channel_test.go
+++ b/modules/queue/queue_disk_channel_test.go
@@ -39,7 +39,7 @@ func TestPersistableChannelQueue(t *testing.T) {
Workers: 1,
BoostWorkers: 0,
MaxWorkers: 10,
- Name: "first",
+ Name: "test-queue",
}, &testData{})
assert.NoError(t, err)
@@ -135,7 +135,7 @@ func TestPersistableChannelQueue(t *testing.T) {
Workers: 1,
BoostWorkers: 0,
MaxWorkers: 10,
- Name: "second",
+ Name: "test-queue",
}, &testData{})
assert.NoError(t, err)
@@ -227,7 +227,7 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
Workers: 1,
BoostWorkers: 0,
MaxWorkers: 10,
- Name: "first",
+ Name: "test-queue",
}, &testData{})
assert.NoError(t, err)
@@ -433,7 +433,7 @@ func TestPersistableChannelQueue_Pause(t *testing.T) {
Workers: 1,
BoostWorkers: 0,
MaxWorkers: 10,
- Name: "second",
+ Name: "test-queue",
}, &testData{})
assert.NoError(t, err)
pausable, ok = queue.(Pausable)
diff --git a/modules/queue/unique_queue_channel.go b/modules/queue/unique_queue_channel.go
index c43bd1db3..62c051aa3 100644
--- a/modules/queue/unique_queue_channel.go
+++ b/modules/queue/unique_queue_channel.go
@@ -177,7 +177,9 @@ func (q *ChannelUniqueQueue) Shutdown() {
go func() {
log.Trace("ChannelUniqueQueue: %s Flushing", q.name)
if err := q.FlushWithContext(q.terminateCtx); err != nil {
- log.Warn("ChannelUniqueQueue: %s Terminated before completed flushing", q.name)
+ if !q.IsEmpty() {
+ log.Warn("ChannelUniqueQueue: %s Terminated before completed flushing", q.name)
+ }
return
}
log.Debug("ChannelUniqueQueue: %s Flushed", q.name)
diff --git a/modules/queue/unique_queue_channel_test.go b/modules/queue/unique_queue_channel_test.go
index 9372694b8..824015b83 100644
--- a/modules/queue/unique_queue_channel_test.go
+++ b/modules/queue/unique_queue_channel_test.go
@@ -8,10 +8,13 @@ import (
"testing"
"time"
+ "code.gitea.io/gitea/modules/log"
+
"github.com/stretchr/testify/assert"
)
func TestChannelUniqueQueue(t *testing.T) {
+ _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`)
handleChan := make(chan *testData)
handle := func(data ...Data) []Data {
for _, datum := range data {
@@ -52,6 +55,8 @@ func TestChannelUniqueQueue(t *testing.T) {
}
func TestChannelUniqueQueue_Batch(t *testing.T) {
+ _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`)
+
handleChan := make(chan *testData)
handle := func(data ...Data) []Data {
for _, datum := range data {
@@ -98,6 +103,8 @@ func TestChannelUniqueQueue_Batch(t *testing.T) {
}
func TestChannelUniqueQueue_Pause(t *testing.T) {
+ _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`)
+
lock := sync.Mutex{}
var queue Queue
var err error
diff --git a/modules/queue/unique_queue_disk_channel.go b/modules/queue/unique_queue_disk_channel.go
index 405726182..cc8a807c6 100644
--- a/modules/queue/unique_queue_disk_channel.go
+++ b/modules/queue/unique_queue_disk_channel.go
@@ -94,7 +94,8 @@ func NewPersistableChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interfac
},
Workers: 0,
},
- DataDir: config.DataDir,
+ DataDir: config.DataDir,
+ QueueName: config.Name + "-level",
}
queue.channelQueue = channelUniqueQueue.(*ChannelUniqueQueue)
@@ -209,17 +210,29 @@ func (q *PersistableChannelUniqueQueue) Run(atShutdown, atTerminate func(func())
atTerminate(q.Terminate)
_ = q.channelQueue.AddWorkers(q.channelQueue.workers, 0)
- if luq, ok := q.internal.(*LevelUniqueQueue); ok && luq.ByteFIFOUniqueQueue.byteFIFO.Len(luq.shutdownCtx) != 0 {
+ if luq, ok := q.internal.(*LevelUniqueQueue); ok && !luq.IsEmpty() {
// Just run the level queue - we shut it down once it's flushed
- go q.internal.Run(func(_ func()) {}, func(_ func()) {})
+ go luq.Run(func(_ func()) {}, func(_ func()) {})
go func() {
- _ = q.internal.Flush(0)
- log.Debug("LevelUniqueQueue: %s flushed so shutting down", q.internal.(*LevelUniqueQueue).Name())
- q.internal.(*LevelUniqueQueue).Shutdown()
- GetManager().Remove(q.internal.(*LevelUniqueQueue).qid)
+ _ = luq.Flush(0)
+ for !luq.IsEmpty() {
+ _ = luq.Flush(0)
+ select {
+ case <-time.After(100 * time.Millisecond):
+ case <-luq.shutdownCtx.Done():
+ if luq.byteFIFO.Len(luq.terminateCtx) > 0 {
+ log.Warn("LevelUniqueQueue: %s shut down before completely flushed", luq.Name())
+ }
+ return
+ }
+ }
+ log.Debug("LevelUniqueQueue: %s flushed so shutting down", luq.Name())
+ luq.Shutdown()
+ GetManager().Remove(luq.qid)
}()
} else {
log.Debug("PersistableChannelUniqueQueue: %s Skipping running the empty level queue", q.delayedStarter.name)
+ _ = q.internal.Flush(0)
q.internal.(*LevelUniqueQueue).Shutdown()
GetManager().Remove(q.internal.(*LevelUniqueQueue).qid)
}
@@ -285,8 +298,20 @@ func (q *PersistableChannelUniqueQueue) Shutdown() {
// Redirect all remaining data in the chan to the internal channel
close(q.channelQueue.dataChan)
log.Trace("PersistableChannelUniqueQueue: %s Redirecting remaining data", q.delayedStarter.name)
+ countOK, countLost := 0, 0
for data := range q.channelQueue.dataChan {
- _ = q.internal.Push(data)
+ err := q.internal.(*LevelUniqueQueue).Push(data)
+ if err != nil {
+ log.Error("PersistableChannelUniqueQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err)
+ countLost++
+ } else {
+ countOK++
+ }
+ }
+ if countLost > 0 {
+ log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost)
+ } else if countOK > 0 {
+ log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK)
}
log.Trace("PersistableChannelUniqueQueue: %s Done Redirecting remaining data", q.delayedStarter.name)
diff --git a/modules/queue/unique_queue_disk_channel_test.go b/modules/queue/unique_queue_disk_channel_test.go
new file mode 100644
index 000000000..fd76163f4
--- /dev/null
+++ b/modules/queue/unique_queue_disk_channel_test.go
@@ -0,0 +1,259 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package queue
+
+import (
+ "fmt"
+ "strconv"
+ "sync"
+ "testing"
+ "time"
+
+ "code.gitea.io/gitea/modules/log"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestPersistableChannelUniqueQueue(t *testing.T) {
+ tmpDir := t.TempDir()
+ fmt.Printf("TempDir %s\n", tmpDir)
+ _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`)
+
+ // Common function to create the Queue
+ newQueue := func(name string, handle func(data ...Data) []Data) Queue {
+ q, err := NewPersistableChannelUniqueQueue(handle,
+ PersistableChannelUniqueQueueConfiguration{
+ Name: name,
+ DataDir: tmpDir,
+ QueueLength: 200,
+ MaxWorkers: 1,
+ BlockTimeout: 1 * time.Second,
+ BoostTimeout: 5 * time.Minute,
+ BoostWorkers: 1,
+ Workers: 0,
+ }, "task-0")
+ assert.NoError(t, err)
+ return q
+ }
+
+ // runs the provided queue and provides some timer function
+ type channels struct {
+ readyForShutdown chan struct{} // closed when shutdown functions have been assigned
+ readyForTerminate chan struct{} // closed when terminate functions have been assigned
+ signalShutdown chan struct{} // Should close to signal shutdown
+ doneShutdown chan struct{} // closed when shutdown function is done
+ queueTerminate []func() // list of atTerminate functions to call atTerminate - need to be accessed with lock
+ }
+ runQueue := func(q Queue, lock *sync.Mutex) *channels {
+ chans := &channels{
+ readyForShutdown: make(chan struct{}),
+ readyForTerminate: make(chan struct{}),
+ signalShutdown: make(chan struct{}),
+ doneShutdown: make(chan struct{}),
+ }
+ go q.Run(func(atShutdown func()) {
+ go func() {
+ lock.Lock()
+ select {
+ case <-chans.readyForShutdown:
+ default:
+ close(chans.readyForShutdown)
+ }
+ lock.Unlock()
+ <-chans.signalShutdown
+ atShutdown()
+ close(chans.doneShutdown)
+ }()
+ }, func(atTerminate func()) {
+ lock.Lock()
+ defer lock.Unlock()
+ select {
+ case <-chans.readyForTerminate:
+ default:
+ close(chans.readyForTerminate)
+ }
+ chans.queueTerminate = append(chans.queueTerminate, atTerminate)
+ })
+
+ return chans
+ }
+
+ // call to shutdown and terminate the queue associated with the channels
+ doTerminate := func(chans *channels, lock *sync.Mutex) {
+ <-chans.readyForTerminate
+
+ lock.Lock()
+ callbacks := []func(){}
+ callbacks = append(callbacks, chans.queueTerminate...)
+ lock.Unlock()
+
+ for _, callback := range callbacks {
+ callback()
+ }
+ }
+
+ mapLock := sync.Mutex{}
+ executedInitial := map[string][]string{}
+ hasInitial := map[string][]string{}
+
+ fillQueue := func(name string, done chan struct{}) {
+ t.Run("Initial Filling: "+name, func(t *testing.T) {
+ lock := sync.Mutex{}
+
+ startAt100Queued := make(chan struct{})
+ stopAt20Shutdown := make(chan struct{}) // stop and shutdown at the 20th item
+
+ handle := func(data ...Data) []Data {
+ <-startAt100Queued
+ for _, datum := range data {
+ s := datum.(string)
+ mapLock.Lock()
+ executedInitial[name] = append(executedInitial[name], s)
+ mapLock.Unlock()
+ if s == "task-20" {
+ close(stopAt20Shutdown)
+ }
+ }
+ return nil
+ }
+
+ q := newQueue(name, handle)
+
+ // add 100 tasks to the queue
+ for i := 0; i < 100; i++ {
+ _ = q.Push("task-" + strconv.Itoa(i))
+ }
+ close(startAt100Queued)
+
+ chans := runQueue(q, &lock)
+
+ <-chans.readyForShutdown
+ <-stopAt20Shutdown
+ close(chans.signalShutdown)
+ <-chans.doneShutdown
+ _ = q.Push("final")
+
+ // check which tasks are still in the queue
+ for i := 0; i < 100; i++ {
+ if has, _ := q.(UniqueQueue).Has("task-" + strconv.Itoa(i)); has {
+ mapLock.Lock()
+ hasInitial[name] = append(hasInitial[name], "task-"+strconv.Itoa(i))
+ mapLock.Unlock()
+ }
+ }
+ if has, _ := q.(UniqueQueue).Has("final"); has {
+ mapLock.Lock()
+ hasInitial[name] = append(hasInitial[name], "final")
+ mapLock.Unlock()
+ } else {
+ assert.Fail(t, "UnqueQueue %s should have \"final\"", name)
+ }
+ doTerminate(chans, &lock)
+ mapLock.Lock()
+ assert.Equal(t, 101, len(executedInitial[name])+len(hasInitial[name]))
+ mapLock.Unlock()
+ })
+ close(done)
+ }
+
+ doneA := make(chan struct{})
+ doneB := make(chan struct{})
+
+ go fillQueue("QueueA", doneA)
+ go fillQueue("QueueB", doneB)
+
+ <-doneA
+ <-doneB
+
+ executedEmpty := map[string][]string{}
+ hasEmpty := map[string][]string{}
+ emptyQueue := func(name string, done chan struct{}) {
+ t.Run("Empty Queue: "+name, func(t *testing.T) {
+ lock := sync.Mutex{}
+ stop := make(chan struct{})
+
+ // collect the tasks that have been executed
+ handle := func(data ...Data) []Data {
+ lock.Lock()
+ for _, datum := range data {
+ mapLock.Lock()
+ executedEmpty[name] = append(executedEmpty[name], datum.(string))
+ mapLock.Unlock()
+ if datum.(string) == "final" {
+ close(stop)
+ }
+ }
+ lock.Unlock()
+ return nil
+ }
+
+ q := newQueue(name, handle)
+ chans := runQueue(q, &lock)
+
+ <-chans.readyForShutdown
+ <-stop
+ close(chans.signalShutdown)
+ <-chans.doneShutdown
+
+ // check which tasks are still in the queue
+ for i := 0; i < 100; i++ {
+ if has, _ := q.(UniqueQueue).Has("task-" + strconv.Itoa(i)); has {
+ mapLock.Lock()
+ hasEmpty[name] = append(hasEmpty[name], "task-"+strconv.Itoa(i))
+ mapLock.Unlock()
+ }
+ }
+ doTerminate(chans, &lock)
+
+ mapLock.Lock()
+ assert.Equal(t, 101, len(executedInitial[name])+len(executedEmpty[name]))
+ assert.Equal(t, 0, len(hasEmpty[name]))
+ mapLock.Unlock()
+ })
+ close(done)
+ }
+
+ doneA = make(chan struct{})
+ doneB = make(chan struct{})
+
+ go emptyQueue("QueueA", doneA)
+ go emptyQueue("QueueB", doneB)
+
+ <-doneA
+ <-doneB
+
+ mapLock.Lock()
+ t.Logf("TestPersistableChannelUniqueQueue executedInitiallyA=%v, executedInitiallyB=%v, executedToEmptyA=%v, executedToEmptyB=%v",
+ len(executedInitial["QueueA"]), len(executedInitial["QueueB"]), len(executedEmpty["QueueA"]), len(executedEmpty["QueueB"]))
+
+ // reset and rerun
+ executedInitial = map[string][]string{}
+ hasInitial = map[string][]string{}
+ executedEmpty = map[string][]string{}
+ hasEmpty = map[string][]string{}
+ mapLock.Unlock()
+
+ doneA = make(chan struct{})
+ doneB = make(chan struct{})
+
+ go fillQueue("QueueA", doneA)
+ go fillQueue("QueueB", doneB)
+
+ <-doneA
+ <-doneB
+
+ doneA = make(chan struct{})
+ doneB = make(chan struct{})
+
+ go emptyQueue("QueueA", doneA)
+ go emptyQueue("QueueB", doneB)
+
+ <-doneA
+ <-doneB
+
+ mapLock.Lock()
+ t.Logf("TestPersistableChannelUniqueQueue executedInitiallyA=%v, executedInitiallyB=%v, executedToEmptyA=%v, executedToEmptyB=%v",
+ len(executedInitial["QueueA"]), len(executedInitial["QueueB"]), len(executedEmpty["QueueA"]), len(executedEmpty["QueueB"]))
+ mapLock.Unlock()
+}
diff --git a/modules/repository/create.go b/modules/repository/create.go
index 1704ea792..6a1fa41b6 100644
--- a/modules/repository/create.go
+++ b/modules/repository/create.go
@@ -23,6 +23,7 @@ import (
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/models/webhook"
"code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/label"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
@@ -189,7 +190,7 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m
// Check if label template exist
if len(opts.IssueLabels) > 0 {
- if _, err := GetLabelTemplateFile(opts.IssueLabels); err != nil {
+ if _, err := label.GetTemplateFile(opts.IssueLabels); err != nil {
return nil, err
}
}
diff --git a/modules/repository/init.go b/modules/repository/init.go
index 771b68a49..f9a33cd4f 100644
--- a/modules/repository/init.go
+++ b/modules/repository/init.go
@@ -18,6 +18,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/label"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/setting"
@@ -40,114 +41,6 @@ var (
LabelTemplates map[string]string
)
-// ErrIssueLabelTemplateLoad represents a "ErrIssueLabelTemplateLoad" kind of error.
-type ErrIssueLabelTemplateLoad struct {
- TemplateFile string
- OriginalError error
-}
-
-// IsErrIssueLabelTemplateLoad checks if an error is a ErrIssueLabelTemplateLoad.
-func IsErrIssueLabelTemplateLoad(err error) bool {
- _, ok := err.(ErrIssueLabelTemplateLoad)
- return ok
-}
-
-func (err ErrIssueLabelTemplateLoad) Error() string {
- return fmt.Sprintf("Failed to load label template file '%s': %v", err.TemplateFile, err.OriginalError)
-}
-
-// GetRepoInitFile returns repository init files
-func GetRepoInitFile(tp, name string) ([]byte, error) {
- cleanedName := strings.TrimLeft(path.Clean("/"+name), "/")
- relPath := path.Join("options", tp, cleanedName)
-
- // Use custom file when available.
- customPath := path.Join(setting.CustomPath, relPath)
- isFile, err := util.IsFile(customPath)
- if err != nil {
- log.Error("Unable to check if %s is a file. Error: %v", customPath, err)
- }
- if isFile {
- return os.ReadFile(customPath)
- }
-
- switch tp {
- case "readme":
- return options.Readme(cleanedName)
- case "gitignore":
- return options.Gitignore(cleanedName)
- case "license":
- return options.License(cleanedName)
- case "label":
- return options.Labels(cleanedName)
- default:
- return []byte{}, fmt.Errorf("Invalid init file type")
- }
-}
-
-// GetLabelTemplateFile loads the label template file by given name,
-// then parses and returns a list of name-color pairs and optionally description.
-func GetLabelTemplateFile(name string) ([][3]string, error) {
- data, err := GetRepoInitFile("label", name)
- if err != nil {
- return nil, ErrIssueLabelTemplateLoad{name, fmt.Errorf("GetRepoInitFile: %w", err)}
- }
-
- lines := strings.Split(string(data), "\n")
- list := make([][3]string, 0, len(lines))
- for i := 0; i < len(lines); i++ {
- line := strings.TrimSpace(lines[i])
- if len(line) == 0 {
- continue
- }
-
- parts := strings.SplitN(line, ";", 2)
-
- fields := strings.SplitN(parts[0], " ", 2)
- if len(fields) != 2 {
- return nil, ErrIssueLabelTemplateLoad{name, fmt.Errorf("line is malformed: %s", line)}
- }
-
- color := strings.Trim(fields[0], " ")
- if len(color) == 6 {
- color = "#" + color
- }
- if !issues_model.LabelColorPattern.MatchString(color) {
- return nil, ErrIssueLabelTemplateLoad{name, fmt.Errorf("bad HTML color code in line: %s", line)}
- }
-
- var description string
-
- if len(parts) > 1 {
- description = strings.TrimSpace(parts[1])
- }
-
- fields[1] = strings.TrimSpace(fields[1])
- list = append(list, [3]string{fields[1], color, description})
- }
-
- return list, nil
-}
-
-func loadLabels(labelTemplate string) ([]string, error) {
- list, err := GetLabelTemplateFile(labelTemplate)
- if err != nil {
- return nil, err
- }
-
- labels := make([]string, len(list))
- for i := 0; i < len(list); i++ {
- labels[i] = list[i][0]
- }
- return labels, nil
-}
-
-// LoadLabelsFormatted loads the labels' list of a template file as a string separated by comma
-func LoadLabelsFormatted(labelTemplate string) (string, error) {
- labels, err := loadLabels(labelTemplate)
- return strings.Join(labels, ", "), err
-}
-
// LoadRepoConfig loads the repository config
func LoadRepoConfig() {
// Load .gitignore and license files and readme templates.
@@ -158,6 +51,14 @@ func LoadRepoConfig() {
if err != nil {
log.Fatal("Failed to get %s files: %v", t, err)
}
+ if t == "label" {
+ for i, f := range files {
+ ext := strings.ToLower(filepath.Ext(f))
+ if ext == ".yaml" || ext == ".yml" {
+ files[i] = f[:len(f)-len(ext)]
+ }
+ }
+ }
customPath := path.Join(setting.CustomPath, "options", t)
isDir, err := util.IsDir(customPath)
if err != nil {
@@ -190,7 +91,7 @@ func LoadRepoConfig() {
// Load label templates
LabelTemplates = make(map[string]string)
for _, templateFile := range LabelTemplatesFiles {
- labels, err := LoadLabelsFormatted(templateFile)
+ labels, err := label.LoadFormatted(templateFile)
if err != nil {
log.Error("Failed to load labels: %v", err)
}
@@ -235,7 +136,7 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir,
}
// README
- data, err := GetRepoInitFile("readme", opts.Readme)
+ data, err := options.Readme(opts.Readme)
if err != nil {
return fmt.Errorf("GetRepoInitFile[%s]: %w", opts.Readme, err)
}
@@ -263,7 +164,7 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir,
var buf bytes.Buffer
names := strings.Split(opts.Gitignores, ",")
for _, name := range names {
- data, err = GetRepoInitFile("gitignore", name)
+ data, err = options.Gitignore(name)
if err != nil {
return fmt.Errorf("GetRepoInitFile[%s]: %w", name, err)
}
@@ -281,7 +182,7 @@ func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir,
// LICENSE
if len(opts.License) > 0 {
- data, err = GetRepoInitFile("license", opts.License)
+ data, err = options.License(opts.License)
if err != nil {
return fmt.Errorf("GetRepoInitFile[%s]: %w", opts.License, err)
}
@@ -443,7 +344,7 @@ func initRepository(ctx context.Context, repoPath string, u *user_model.User, re
// InitializeLabels adds a label set to a repository using a template
func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg bool) error {
- list, err := GetLabelTemplateFile(labelTemplate)
+ list, err := label.GetTemplateFile(labelTemplate)
if err != nil {
return err
}
@@ -451,9 +352,10 @@ func InitializeLabels(ctx context.Context, id int64, labelTemplate string, isOrg
labels := make([]*issues_model.Label, len(list))
for i := 0; i < len(list); i++ {
labels[i] = &issues_model.Label{
- Name: list[i][0],
- Description: list[i][2],
- Color: list[i][1],
+ Name: list[i].Name,
+ Exclusive: list[i].Exclusive,
+ Description: list[i].Description,
+ Color: list[i].Color,
}
if isOrg {
labels[i].OrgID = id
diff --git a/modules/repository/push.go b/modules/repository/push.go
index 1fa711b35..aa1552351 100644
--- a/modules/repository/push.go
+++ b/modules/repository/push.go
@@ -4,10 +4,8 @@
package repository
import (
- "context"
"strings"
- repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
)
@@ -96,19 +94,3 @@ func (opts *PushUpdateOptions) RefName() string {
func (opts *PushUpdateOptions) RepoFullName() string {
return opts.RepoUserName + "/" + opts.RepoName
}
-
-// IsForcePush detect if a push is a force push
-func IsForcePush(ctx context.Context, opts *PushUpdateOptions) (bool, error) {
- if !opts.IsUpdateBranch() {
- return false, nil
- }
-
- output, _, err := git.NewCommand(ctx, "rev-list", "--max-count=1").AddDynamicArguments(opts.OldCommitID, "^"+opts.NewCommitID).
- RunStdString(&git.RunOpts{Dir: repo_model.RepoPath(opts.RepoUserName, opts.RepoName)})
- if err != nil {
- return false, err
- } else if len(output) > 0 {
- return true, nil
- }
- return false, nil
-}
diff --git a/modules/setting/database.go b/modules/setting/database.go
index 49865a38a..d7a5078fe 100644
--- a/modules/setting/database.go
+++ b/modules/setting/database.go
@@ -27,7 +27,7 @@ var (
// Database holds the database settings
Database = struct {
- Type string
+ Type DatabaseType
Host string
Name string
User string
@@ -39,10 +39,6 @@ var (
Charset string
Timeout int // seconds
SQLiteJournalMode string
- UseSQLite3 bool
- UseMySQL bool
- UseMSSQL bool
- UsePostgreSQL bool
DBConnectRetries int
DBConnectBackoff time.Duration
MaxIdleConns int
@@ -59,24 +55,13 @@ var (
// LoadDBSetting loads the database settings
func LoadDBSetting() {
sec := CfgProvider.Section("database")
- Database.Type = sec.Key("DB_TYPE").String()
+ Database.Type = DatabaseType(sec.Key("DB_TYPE").String())
defaultCharset := "utf8"
- Database.UseMySQL = false
- Database.UseSQLite3 = false
- Database.UsePostgreSQL = false
- Database.UseMSSQL = false
- switch Database.Type {
- case "sqlite3":
- Database.UseSQLite3 = true
- case "mysql":
- Database.UseMySQL = true
+ if Database.Type.IsMySQL() {
defaultCharset = "utf8mb4"
- case "postgres":
- Database.UsePostgreSQL = true
- case "mssql":
- Database.UseMSSQL = true
}
+
Database.Host = sec.Key("HOST").String()
Database.Name = sec.Key("NAME").String()
Database.User = sec.Key("USER").String()
@@ -86,7 +71,7 @@ func LoadDBSetting() {
Database.Schema = sec.Key("SCHEMA").String()
Database.SSLMode = sec.Key("SSL_MODE").MustString("disable")
Database.Charset = sec.Key("CHARSET").In(defaultCharset, []string{"utf8", "utf8mb4"})
- if Database.UseMySQL && defaultCharset != "utf8mb4" {
+ if Database.Type.IsMySQL() && defaultCharset != "utf8mb4" {
log.Error("Deprecated database mysql charset utf8 support, please use utf8mb4 or convert utf8 to utf8mb4.")
}
@@ -95,7 +80,7 @@ func LoadDBSetting() {
Database.SQLiteJournalMode = sec.Key("SQLITE_JOURNAL_MODE").MustString("")
Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2)
- if Database.UseMySQL {
+ if Database.Type.IsMySQL() {
Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFETIME").MustDuration(3 * time.Second)
} else {
Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFETIME").MustDuration(0)
@@ -207,3 +192,25 @@ func ParseMSSQLHostPort(info string) (string, string) {
}
return host, port
}
+
+type DatabaseType string
+
+func (t DatabaseType) String() string {
+ return string(t)
+}
+
+func (t DatabaseType) IsSQLite3() bool {
+ return t == "sqlite3"
+}
+
+func (t DatabaseType) IsMySQL() bool {
+ return t == "mysql"
+}
+
+func (t DatabaseType) IsMSSQL() bool {
+ return t == "mssql"
+}
+
+func (t DatabaseType) IsPostgreSQL() bool {
+ return t == "postgres"
+}
diff --git a/modules/setting/log.go b/modules/setting/log.go
index 5448650aa..dabdb543a 100644
--- a/modules/setting/log.go
+++ b/modules/setting/log.go
@@ -38,6 +38,7 @@ var Log struct {
EnableAccessLog bool
AccessLogTemplate string
BufferLength int64
+ RequestIDHeaders []string
}
// GetLogDescriptions returns a race safe set of descriptions
@@ -153,6 +154,7 @@ func loadLogFrom(rootCfg ConfigProvider) {
Log.AccessLogTemplate = sec.Key("ACCESS_LOG_TEMPLATE").MustString(
`{{.Ctx.RemoteAddr}} - {{.Identity}} {{.Start.Format "[02/Jan/2006:15:04:05 -0700]" }} "{{.Ctx.Req.Method}} {{.Ctx.Req.URL.RequestURI}} {{.Ctx.Req.Proto}}" {{.ResponseWriter.Status}} {{.ResponseWriter.Size}} "{{.Ctx.Req.Referer}}\" \"{{.Ctx.Req.UserAgent}}"`,
)
+ Log.RequestIDHeaders = sec.Key("REQUEST_ID_HEADERS").Strings(",")
// the `MustString` updates the default value, and `log.ACCESS` is used by `generateNamedLogger("access")` later
_ = rootCfg.Section("log").Key("ACCESS").MustString("file")
diff --git a/modules/setting/storage.go b/modules/setting/storage.go
index 9197c5f8b..4d401614e 100644
--- a/modules/setting/storage.go
+++ b/modules/setting/storage.go
@@ -41,6 +41,7 @@ func getStorage(rootCfg ConfigProvider, name, typ string, targetSec *ini.Section
sec.Key("MINIO_BUCKET").MustString("gitea")
sec.Key("MINIO_LOCATION").MustString("us-east-1")
sec.Key("MINIO_USE_SSL").MustBool(false)
+ sec.Key("MINIO_INSECURE_SKIP_VERIFY").MustBool(false)
if targetSec == nil {
targetSec, _ = rootCfg.NewSection(name)
diff --git a/modules/storage/local.go b/modules/storage/local.go
index a6a9d54a8..05bf1fb28 100644
--- a/modules/storage/local.go
+++ b/modules/storage/local.go
@@ -8,7 +8,6 @@ import (
"io"
"net/url"
"os"
- "path"
"path/filepath"
"strings"
@@ -59,7 +58,7 @@ func NewLocalStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
}
func (l *LocalStorage) buildLocalPath(p string) string {
- return filepath.Join(l.dir, path.Clean("/" + strings.ReplaceAll(p, "\\", "/"))[1:])
+ return filepath.Join(l.dir, util.CleanPath(strings.ReplaceAll(p, "\\", "/")))
}
// Open a file
diff --git a/modules/storage/minio.go b/modules/storage/minio.go
index 912f820ed..24da14b63 100644
--- a/modules/storage/minio.go
+++ b/modules/storage/minio.go
@@ -5,7 +5,9 @@ package storage
import (
"context"
+ "crypto/tls"
"io"
+ "net/http"
"net/url"
"os"
"path"
@@ -13,6 +15,7 @@ import (
"time"
"code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/util"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
@@ -42,13 +45,14 @@ const MinioStorageType Type = "minio"
// MinioStorageConfig represents the configuration for a minio storage
type MinioStorageConfig struct {
- Endpoint string `ini:"MINIO_ENDPOINT"`
- AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"`
- SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"`
- Bucket string `ini:"MINIO_BUCKET"`
- Location string `ini:"MINIO_LOCATION"`
- BasePath string `ini:"MINIO_BASE_PATH"`
- UseSSL bool `ini:"MINIO_USE_SSL"`
+ Endpoint string `ini:"MINIO_ENDPOINT"`
+ AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"`
+ SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"`
+ Bucket string `ini:"MINIO_BUCKET"`
+ Location string `ini:"MINIO_LOCATION"`
+ BasePath string `ini:"MINIO_BASE_PATH"`
+ UseSSL bool `ini:"MINIO_USE_SSL"`
+ InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
}
// MinioStorage returns a minio bucket storage
@@ -90,8 +94,9 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
minioClient, err := minio.New(config.Endpoint, &minio.Options{
- Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
- Secure: config.UseSSL,
+ Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""),
+ Secure: config.UseSSL,
+ Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}},
})
if err != nil {
return nil, convertMinioErr(err)
@@ -116,7 +121,7 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
}
func (m *MinioStorage) buildMinioPath(p string) string {
- return strings.TrimPrefix(path.Join(m.basePath, path.Clean("/" + strings.ReplaceAll(p, "\\", "/"))[1:]), "/")
+ return strings.TrimPrefix(path.Join(m.basePath, util.CleanPath(strings.ReplaceAll(p, "\\", "/"))), "/")
}
// Open open a file
diff --git a/modules/templates/helper.go b/modules/templates/helper.go
index 17ac68dc6..19893c7c9 100644
--- a/modules/templates/helper.go
+++ b/modules/templates/helper.go
@@ -385,10 +385,10 @@ func NewFuncMap() []template.FuncMap {
// the table is NOT sorted with this header
return ""
},
- "RenderLabel": func(label *issues_model.Label) template.HTML {
- return template.HTML(RenderLabel(label))
+ "RenderLabel": func(ctx context.Context, label *issues_model.Label) template.HTML {
+ return template.HTML(RenderLabel(ctx, label))
},
- "RenderLabels": func(labels []*issues_model.Label, repoLink string) template.HTML {
+ "RenderLabels": func(ctx context.Context, labels []*issues_model.Label, repoLink string) template.HTML {
htmlCode := `<span class="labels-list">`
for _, label := range labels {
// Protect against nil value in labels - shouldn't happen but would cause a panic if so
@@ -396,7 +396,7 @@ func NewFuncMap() []template.FuncMap {
continue
}
htmlCode += fmt.Sprintf("<a href='%s/issues?labels=%d'>%s</a> ",
- repoLink, label.ID, RenderLabel(label))
+ repoLink, label.ID, RenderLabel(ctx, label))
}
htmlCode += "</span>"
return template.HTML(htmlCode)
@@ -808,7 +808,7 @@ func RenderIssueTitle(ctx context.Context, text, urlPrefix string, metas map[str
}
// RenderLabel renders a label
-func RenderLabel(label *issues_model.Label) string {
+func RenderLabel(ctx context.Context, label *issues_model.Label) string {
labelScope := label.ExclusiveScope()
textColor := "#111"
@@ -821,12 +821,12 @@ func RenderLabel(label *issues_model.Label) string {
if labelScope == "" {
// Regular label
return fmt.Sprintf("<div class='ui label' style='color: %s !important; background-color: %s !important' title='%s'>%s</div>",
- textColor, label.Color, description, RenderEmoji(label.Name))
+ textColor, label.Color, description, RenderEmoji(ctx, label.Name))
}
// Scoped label
- scopeText := RenderEmoji(labelScope)
- itemText := RenderEmoji(label.Name[len(labelScope)+1:])
+ scopeText := RenderEmoji(ctx, labelScope)
+ itemText := RenderEmoji(ctx, label.Name[len(labelScope)+1:])
itemColor := label.Color
scopeColor := label.Color
@@ -869,8 +869,9 @@ func RenderLabel(label *issues_model.Label) string {
}
// RenderEmoji renders html text with emoji post processors
-func RenderEmoji(text string) template.HTML {
- renderedText, err := markup.RenderEmoji(template.HTMLEscapeString(text))
+func RenderEmoji(ctx context.Context, text string) template.HTML {
+ renderedText, err := markup.RenderEmoji(&markup.RenderContext{Ctx: ctx},
+ template.HTMLEscapeString(text))
if err != nil {
log.Error("RenderEmoji: %v", err)
return template.HTML("")
diff --git a/modules/typesniffer/typesniffer.go b/modules/typesniffer/typesniffer.go
index c9fef953c..7887fd42b 100644
--- a/modules/typesniffer/typesniffer.go
+++ b/modules/typesniffer/typesniffer.go
@@ -4,6 +4,7 @@
package typesniffer
import (
+ "bytes"
"fmt"
"io"
"net/http"
@@ -24,8 +25,9 @@ const (
)
var (
- svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!--.*?-->|<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg[\s>\/]`)
- svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!--.*?-->|<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg[\s>\/]`)
+ svgComment = regexp.MustCompile(`(?s)<!--.*?-->`)
+ svgTagRegex = regexp.MustCompile(`(?si)\A\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`)
+ svgTagInXMLRegex = regexp.MustCompile(`(?si)\A<\?xml\b.*?\?>\s*(?:(<!DOCTYPE\s+svg([\s:]+.*?>|>))\s*)*<svg\b`)
)
// SniffedType contains information about a blobs type.
@@ -91,10 +93,27 @@ func DetectContentType(data []byte) SniffedType {
data = data[:sniffLen]
}
- if (strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html")) && svgTagRegex.Match(data) ||
- strings.Contains(ct, "text/xml") && svgTagInXMLRegex.Match(data) {
- // SVG is unsupported. https://github.com/golang/go/issues/15888
- ct = SvgMimeType
+ // SVG is unsupported by http.DetectContentType, https://github.com/golang/go/issues/15888
+
+ detectByHTML := strings.Contains(ct, "text/plain") || strings.Contains(ct, "text/html")
+ detectByXML := strings.Contains(ct, "text/xml")
+ if detectByHTML || detectByXML {
+ dataProcessed := svgComment.ReplaceAll(data, nil)
+ dataProcessed = bytes.TrimSpace(dataProcessed)
+ if detectByHTML && svgTagRegex.Match(dataProcessed) ||
+ detectByXML && svgTagInXMLRegex.Match(dataProcessed) {
+ ct = SvgMimeType
+ }
+ }
+
+ if strings.HasPrefix(ct, "audio/") && bytes.HasPrefix(data, []byte("ID3")) {
+ // The MP3 detection is quite inaccurate, any content with "ID3" prefix will result in "audio/mpeg".
+ // So remove the "ID3" prefix and detect again, if result is text, then it must be text content.
+ // This works especially because audio files contain many unprintable/invalid characters like `0x00`
+ ct2 := http.DetectContentType(data[3:])
+ if strings.HasPrefix(ct2, "text/") {
+ ct = ct2
+ }
}
return SniffedType{ct}
diff --git a/modules/typesniffer/typesniffer_test.go b/modules/typesniffer/typesniffer_test.go
index dbce94fc3..6c6da34aa 100644
--- a/modules/typesniffer/typesniffer_test.go
+++ b/modules/typesniffer/typesniffer_test.go
@@ -28,7 +28,6 @@ func TestIsSvgImage(t *testing.T) {
assert.True(t, DetectContentType([]byte("<svg></svg>")).IsSvgImage())
assert.True(t, DetectContentType([]byte(" <svg></svg>")).IsSvgImage())
assert.True(t, DetectContentType([]byte(`<svg width="100"></svg>`)).IsSvgImage())
- assert.True(t, DetectContentType([]byte("<svg/>")).IsSvgImage())
assert.True(t, DetectContentType([]byte(`<?xml version="1.0" encoding="UTF-8"?><svg></svg>`)).IsSvgImage())
assert.True(t, DetectContentType([]byte(`<!-- Comment -->
<svg></svg>`)).IsSvgImage())
@@ -57,6 +56,10 @@ func TestIsSvgImage(t *testing.T) {
<!-- Multline
Comment -->
<svg></svg>`)).IsSvgImage())
+
+ // the DetectContentType should work for incomplete data, because only beginning bytes are used for detection
+ assert.True(t, DetectContentType([]byte(`<svg>....`)).IsSvgImage())
+
assert.False(t, DetectContentType([]byte{}).IsSvgImage())
assert.False(t, DetectContentType([]byte("svg")).IsSvgImage())
assert.False(t, DetectContentType([]byte("<svgfoo></svgfoo>")).IsSvgImage())
@@ -68,6 +71,26 @@ func TestIsSvgImage(t *testing.T) {
assert.False(t, DetectContentType([]byte(`<?xml version="1.0" encoding="UTF-8"?>
<!-- <svg></svg> inside comment -->
<foo></foo>`)).IsSvgImage())
+
+ assert.False(t, DetectContentType([]byte(`
+<!-- comment1 -->
+<div>
+ <!-- comment2 -->
+ <svg></svg>
+</div>
+`)).IsSvgImage())
+
+ assert.False(t, DetectContentType([]byte(`
+<!-- comment1
+-->
+<div>
+ <!-- comment2
+-->
+ <svg></svg>
+</div>
+`)).IsSvgImage())
+ assert.False(t, DetectContentType([]byte(`<html><body><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg></svg></body></html>`)).IsSvgImage())
+ assert.False(t, DetectContentType([]byte(`<html><body><?xml version="1.0" encoding="UTF-8"?><svg></svg></body></html>`)).IsSvgImage())
}
func TestIsPDF(t *testing.T) {
@@ -86,6 +109,10 @@ func TestIsAudio(t *testing.T) {
mp3, _ := base64.StdEncoding.DecodeString("SUQzBAAAAAABAFRYWFgAAAASAAADbWFqb3JfYnJhbmQAbXA0MgBUWFhYAAAAEQAAA21pbm9yX3Zl")
assert.True(t, DetectContentType(mp3).IsAudio())
assert.False(t, DetectContentType([]byte("plain text")).IsAudio())
+
+ assert.True(t, DetectContentType([]byte("ID3Toy\000")).IsAudio())
+ assert.True(t, DetectContentType([]byte("ID3Toy\n====\t* hi 🌞, ...")).IsText()) // test ID3 tag for plain text
+ assert.True(t, DetectContentType([]byte("ID3Toy\n====\t* hi 🌞, ..."+"🌛"[0:2])).IsText()) // test ID3 tag with incomplete UTF8 char
}
func TestDetectContentTypeFromReader(t *testing.T) {
diff --git a/modules/util/path.go b/modules/util/path.go
index 74acb7a85..5aa9e15f5 100644
--- a/modules/util/path.go
+++ b/modules/util/path.go
@@ -14,6 +14,14 @@ import (
"strings"
)
+// CleanPath ensure to clean the path
+func CleanPath(p string) string {
+ if strings.HasPrefix(p, "/") {
+ return path.Clean(p)
+ }
+ return path.Clean("/" + p)[1:]
+}
+
// EnsureAbsolutePath ensure that a path is absolute, making it
// relative to absoluteBase if necessary
func EnsureAbsolutePath(path, absoluteBase string) string {
diff --git a/modules/util/path_test.go b/modules/util/path_test.go
index 93f4f67cf..2f020f924 100644
--- a/modules/util/path_test.go
+++ b/modules/util/path_test.go
@@ -136,3 +136,15 @@ func TestMisc_IsReadmeFileName(t *testing.T) {
assert.Equal(t, testCase.idx, idx)
}
}
+
+func TestCleanPath(t *testing.T) {
+ cases := map[string]string{
+ "../../test": "test",
+ "/test": "/test",
+ "/../test": "/test",
+ }
+
+ for k, v := range cases {
+ assert.Equal(t, v, CleanPath(k))
+ }
+}