Add health check endpoint (#18465) * chore: add health check endpoint docs: update document about health check fix: fix up Sqlite3 ping. current ping will success even if the db file is missing fix: do not expose privacy information in output field * refactor: remove HealthChecker struct * Added `/api/healthz` to install routes. This was needed for using /api/healthz endpoint in Docker healthchecks, otherwise, Docker would never become healthy if using healthz endpoint and users would not be able to complete the installation of Gitea. * Update modules/cache/cache.go * fine tune * Remove unnecessary test code. Now there are 2 routes for installation (and maybe more in future) Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: wxiaoguang <wxiaoguang@gmail.com> Co-authored-by: 6543 <6543@obermui.de> Co-authored-by: Marcos de Oliveira <marcossantos@furb.br>

Changes

7 changed files (+261/-13)

• docs/content/doc/installation/on-kubernetes.en-us.md
  • @@ -25,3 +25,47 @@ helm install gitea gitea-charts/gitea
    ``` If you would like to customize your install, which includes kubernetes ingress, please refer to the complete [Gitea helm chart configuration details](https://gitea.com/gitea/helm-chart/) + + ## Health check endpoint + + Gitea comes with a health check endpoint `/api/healthz`, you can configure it in kubernetes like this: + + ```yaml + livenessProbe: + httpGet: + path: /api/healthz + port: http + initialDelaySeconds: 200 + timeoutSeconds: 5 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + ``` + + a successful health check response will respond with http code `200`, here's example: + + ``` + HTTP/1.1 200 OK + + + { + "status": "pass", + "description": "Gitea: Git with a cup of tea", + "checks": { + "cache:ping": [ + { + "status": "pass", + "time": "2022-02-19T09:16:08Z" + } + ], + "database:ping": [ + { + "status": "pass", + "time": "2022-02-19T09:16:08Z" + } + ] + } + } + ``` + + for more information, please reference to kubernetes documentation [Define a liveness HTTP request](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-liveness-http-request)
• docs/content/doc/installation/on-kubernetes.zh-tw.md
  • @@ -25,3 +25,47 @@ helm install gitea gitea-charts/gitea
    ``` 若您想自訂安裝（包括使用 kubernetes ingress），請前往完整的 [Gitea helm chart configuration details](https://gitea.com/gitea/helm-chart/) + + ##運行狀況檢查終端節點 + + Gitea 附帶了一個運行狀況檢查端點 `/api/healthz`，你可以像這樣在 kubernetes 中配置它: + + ```yaml + livenessProbe: + httpGet: + path: /api/healthz + port: http + initialDelaySeconds: 200 + timeoutSeconds: 5 + periodSeconds: 10 + successThreshold: 1 + failureThreshold: 10 + ``` + + 成功的運行狀況檢查回應將使用 HTTP 代碼 `200` 進行回應，下面是示例: + + ``` + HTTP/1.1 200 OK + + + { + "status": "pass", + "description": "Gitea: Git with a cup of tea", + "checks": { + "cache:ping": [ + { + "status": "pass", + "time": "2022-02-19T09:16:08Z" + } + ], + "database:ping": [ + { + "status": "pass", + "time": "2022-02-19T09:16:08Z" + } + ] + } + } + ``` + + 有關更多信息，請參考kubernetes文檔[定義一個存活態 HTTP請求接口]（https://kubernetes.io/zh/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/）
• modules/cache/cache.go
  • @@ -5,6 +5,7 @@
    package cache import ( + "errors" "fmt" "strconv"
  • @@ -34,25 +35,37 @@ func NewContext() error {
    if conn, err = newCache(setting.CacheService.Cache); err != nil { return err } - const testKey = "__gitea_cache_test" - const testVal = "test-value" - if err = conn.Put(testKey, testVal, 10); err != nil { + if err = Ping(); err != nil { return err } - val := conn.Get(testKey) - if valStr, ok := val.(string); !ok || valStr != testVal { - // If the cache is full, the Get may not read the expected value stored by Put. - // Since we have checked that Put can success, so we just show a warning here, do not return an error to panic. - log.Warn("cache (adapter:%s, config:%s) doesn't seem to work correctly, set test value '%v' but get '%v'", - setting.CacheService.Cache.Adapter, setting.CacheService.Cache.Conn, - testVal, val, - ) - } } return err } + // Ping checks if the cache service works or not, it not, it returns an error + func Ping() error { + if conn == nil { + return errors.New("cache not available") + } + var err error + const testKey = "__gitea_cache_test" + const testVal = "test-value" + if err = conn.Put(testKey, testVal, 10); err != nil { + return err + } + val := conn.Get(testKey) + if valStr, ok := val.(string); !ok || valStr != testVal { + // If the cache is full, the Get may not read the expected value stored by Put. + // Since we have checked that Put can success, so we just show a warning here, do not return an error to panic. + log.Warn("cache (adapter:%s, config:%s) doesn't seem to work correctly, set test value '%v' but get '%v'", + setting.CacheService.Cache.Adapter, setting.CacheService.Cache.Conn, + testVal, val, + ) + } + return nil + } + // GetCache returns the currently configured cache func GetCache() mc.Cache { return conn
• routers/install/routes.go
  • @@ -16,6 +16,7 @@ import (
    "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/modules/web/middleware" "code.gitea.io/gitea/routers/common" + "code.gitea.io/gitea/routers/web/healthcheck" "code.gitea.io/gitea/services/forms" "gitea.com/go-chi/session"
  • @@ -106,6 +107,7 @@ func Routes() *web.Route {
    r.Use(Init) r.Get("/", Install) r.Post("/", web.Bind(forms.InstallForm{}), SubmitInstall) + r.Get("/api/healthz", healthcheck.Check) r.NotFound(web.Wrap(installNotFound)) return r
• routers/install/routes_test.go
  • @@ -13,7 +13,6 @@ import (
    func TestRoutes(t *testing.T) { routes := Routes() assert.NotNil(t, routes) - assert.Len(t, routes.R.Routes(), 1) assert.EqualValues(t, "/", routes.R.Routes()[0].Pattern) assert.Nil(t, routes.R.Routes()[0].SubRoutes) assert.Len(t, routes.R.Routes()[0].Handlers, 2)
• routers/web/healthcheck/check.go (new)
  • @@ -0,0 +1,143 @@
    + // Copyright 2022 The Gitea Authors. All rights reserved. + // Use of this source code is governed by a MIT-style + // license that can be found in the LICENSE file. + + package healthcheck + + import ( + "net/http" + "os" + "time" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/cache" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + ) + + type status string + + const ( + // pass healthy (acceptable aliases: "ok" to support Node's Terminus and "up" for Java's SpringBoot) + // fail unhealthy (acceptable aliases: "error" to support Node's Terminus and "down" for Java's SpringBoot), and + // warn healthy, with some concerns. + // + // ref https://datatracker.ietf.org/doc/html/draft-inadarei-api-health-check#section-3.1 + // status: (required) indicates whether the service status is acceptable + // or not. API publishers SHOULD use following values for the field: + // The value of the status field is case-insensitive and is tightly + // related with the HTTP response code returned by the health endpoint. + // For "pass" status, HTTP response code in the 2xx-3xx range MUST be + // used. For "fail" status, HTTP response code in the 4xx-5xx range + // MUST be used. In case of the "warn" status, endpoints MUST return + // HTTP status in the 2xx-3xx range, and additional information SHOULD + // be provided, utilizing optional fields of the response. + pass status = "pass" + fail status = "fail" + warn status = "warn" + ) + + func (s status) ToHTTPStatus() int { + if s == pass || s == warn { + return http.StatusOK + } + return http.StatusFailedDependency + } + + type checks map[string][]componentStatus + + // response is the data returned by the health endpoint, which will be marshaled to JSON format + type response struct { + Status status `json:"status"` + Description string `json:"description"` // a human-friendly description of the service + Checks checks `json:"checks,omitempty"` // The Checks Object, should be omitted on installation route + } + + // componentStatus presents one status of a single check object + // an object that provides detailed health statuses of additional downstream systems and endpoints + // which can affect the overall health of the main API. + type componentStatus struct { + Status status `json:"status"` + Time string `json:"time"` // the date-time, in ISO8601 format + Output string `json:"output,omitempty"` // this field SHOULD be omitted for "pass" state. + } + + // Check is the health check API handler + func Check(w http.ResponseWriter, r *http.Request) { + rsp := response{ + Status: pass, + Description: setting.AppName, + Checks: make(checks), + } + + statuses := make([]status, 0) + if setting.InstallLock { + statuses = append(statuses, checkDatabase(rsp.Checks)) + statuses = append(statuses, checkCache(rsp.Checks)) + } + for _, s := range statuses { + if s != pass { + rsp.Status = fail + break + } + } + + data, _ := json.MarshalIndent(rsp, "", " ") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(rsp.Status.ToHTTPStatus()) + _, _ = w.Write(data) + } + + // database checks gitea database status + func checkDatabase(checks checks) status { + st := componentStatus{} + if err := db.GetEngine(db.DefaultContext).Ping(); err != nil { + st.Status = fail + st.Time = getCheckTime() + log.Error("database ping failed with error: %v", err) + } else { + st.Status = pass + st.Time = getCheckTime() + } + + if setting.Database.UseSQLite3 && st.Status == pass { + if !setting.EnableSQLite3 { + st.Status = fail + st.Time = getCheckTime() + log.Error("SQLite3 health check failed with error: %v", "this Gitea binary is built without SQLite3 enabled") + } else { + if _, err := os.Stat(setting.Database.Path); err != nil { + st.Status = fail + st.Time = getCheckTime() + log.Error("SQLite3 file exists check failed with error: %v", err) + } + } + } + + checks["database:ping"] = []componentStatus{st} + return st.Status + } + + // cache checks gitea cache status + func checkCache(checks checks) status { + if !setting.CacheService.Enabled { + return pass + } + + st := componentStatus{} + if err := cache.Ping(); err != nil { + st.Status = fail + st.Time = getCheckTime() + log.Error("cache ping failed with error: %v", err) + } else { + st.Status = pass + st.Time = getCheckTime() + } + checks["cache:ping"] = []componentStatus{st} + return st.Status + } + + func getCheckTime() string { + return time.Now().UTC().Format(time.RFC3339) + }
• routers/web/web.go
  • @@ -31,6 +31,7 @@ import (
    "code.gitea.io/gitea/routers/web/events" "code.gitea.io/gitea/routers/web/explore" "code.gitea.io/gitea/routers/web/feed" + "code.gitea.io/gitea/routers/web/healthcheck" "code.gitea.io/gitea/routers/web/misc" "code.gitea.io/gitea/routers/web/org" "code.gitea.io/gitea/routers/web/repo"
  • @@ -191,6 +192,8 @@ func Routes() *web.Route {
    rw.WriteHeader(http.StatusOK) }) + routes.Get("/api/healthz", healthcheck.Check) + // Removed: toolbox.Toolboxer middleware will provide debug information which seems unnecessary common = append(common, context.Contexter())