diff options
author | John Olheiser | 2023-03-10 00:13:17 -0600 |
---|---|---|
committer | GitHub | 2023-03-10 01:13:17 -0500 |
commit | f92e0a4018ca65936c95ac119c57d4b9ab62bc2d (patch) | |
tree | d2555dabb2742dafb5cc2590f67cf62c23948955 | |
parent | d647e74502fdf734c89b3e6592a9ad88c3005971 (diff) |
Split CI pipelines (#23385)
- This PR attempts to split our various DB tests into separate
pipelines.
- It splits up some of the extra feature-related tests rather than
having most of them in the MySQL test.
- It disables the race detector for some of the pipelines as well, as it
can cause slower runs and is mostly redundant when the pipelines just
swap DBs.
- It builds without SQLite support for any of the non-SQLite pipelines.
- It moves the e2e test to using SQLite rather than PG (partially
because I moved the minio tests to PG and that mucked up the test
config, and partially because it avoids another running service)
- It splits up the `go mod download` task in the Makefile from the tool
installation, as the tools are only needed in the compliance pipeline.
(Arguably even some of the tools aren't needed there, but that could be
a follow-up PR)
- SQLite is now the only arm64 pipeline, moving PG back to amd64 which
can leverage autoscaler
Should resolve #22010 - one thing that wasn't changed here but is
mentioned in that issue, unit tests are needed in the same pipeline as
an integration test in order to form a complete coverage report (at
least as far as I could tell), so for now it remains in a pipeline with
a DB integration test.
Please let me know if I've inadvertently changed something that was how
it was on purpose.
---
I will say sometimes it's hard to pin down the average time, as a
pipeline could be waiting for a runner for X minutes and that brings the
total up by X minutes as well, but overall this does seem to be faster
on average.
---------
Signed-off-by: jolheiser <john.olheiser@gmail.com>
Co-authored-by: techknowlogick <techknowlogick@gitea.io>
-rw-r--r-- | .drone.yml | 462 | ||||
-rw-r--r-- | Makefile | 6 | ||||
-rw-r--r-- | tests/mysql.ini.tmpl | 25 | ||||
-rw-r--r-- | tests/pgsql.ini.tmpl | 21 |
4 files changed, 364 insertions, 150 deletions
diff --git a/.drone.yml b/.drone.yml index d1ad625e3..0e1d79983 100644 --- a/.drone.yml +++ b/.drone.yml @@ -32,6 +32,7 @@ steps: pull: always commands: - make deps-backend + - make deps-tools volumes: - name: deps path: /go @@ -168,7 +169,7 @@ steps: --- kind: pipeline type: docker -name: testing-amd64 +name: testing-pgsql platform: os: linux @@ -191,51 +192,26 @@ volumes: temp: {} services: - - name: mysql - image: mysql:5.7 - pull: always - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_DATABASE: test - - - name: mysql8 - image: mysql:8 - pull: always - environment: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_DATABASE: testgitea - - - name: mssql - image: mcr.microsoft.com/mssql/server:latest - pull: always + - name: pgsql + pull: default + image: postgres:15 environment: - ACCEPT_EULA: Y - MSSQL_PID: Standard - SA_PASSWORD: MwantsaSecurePassword1 + POSTGRES_DB: test + POSTGRES_PASSWORD: postgres - name: ldap image: gitea/test-openldap:latest pull: always - - name: elasticsearch - image: elasticsearch:7.5.0 - pull: always - environment: - discovery.type: single-node - - name: minio image: minio/minio:RELEASE.2021-03-12T00-00-47Z pull: always commands: - - minio server /data + - minio server /data environment: MINIO_ACCESS_KEY: 123456 MINIO_SECRET_KEY: 12345678 - - name: smtpimap - image: tabascoterrier/docker-imap-devel:latest - pull: always - steps: - name: fetch-tags image: docker:git @@ -257,12 +233,6 @@ steps: - name: deps path: /go - - name: tag-pre-condition - image: drone/git - pull: always - commands: - - git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA} - - name: prepare-test-env image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env pull: always @@ -278,88 +248,157 @@ steps: environment: GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not GOSUMDB: sum.golang.org - TAGS: bindata sqlite sqlite_unlock_notify + TAGS: bindata depends_on: [deps-backend, prepare-test-env] volumes: - name: deps path: /go - - name: unit-test + - name: test-pgsql image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env user: gitea commands: - - make unit-test-coverage test-check + - timeout -s ABRT 50m make test-pgsql-migration test-pgsql environment: GOPROXY: https://goproxy.io - TAGS: bindata sqlite sqlite_unlock_notify + TAGS: bindata gogit RACE_ENABLED: true - GITHUB_READ_TOKEN: - from_secret: github_read_token - depends_on: [deps-backend, prepare-test-env] + TEST_TAGS: gogit + TEST_LDAP: 1 + USE_REPO_TEST_DIR: 1 + depends_on: [build] volumes: - name: deps path: /go - - name: unit-test-gogit +--- +kind: pipeline +type: docker +name: testing-mysql + +platform: + os: linux + arch: amd64 + +depends_on: + - compliance + +trigger: + event: + - push + - tag + - pull_request + paths: + exclude: + - docs/** + +volumes: + - name: deps + temp: {} + +services: + - name: mysql + image: mysql:5.7 + pull: always + environment: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: test + + - name: elasticsearch + image: elasticsearch:7.5.0 + pull: always + environment: + discovery.type: single-node + + - name: smtpimap + image: tabascoterrier/docker-imap-devel:latest + pull: always + +steps: + - name: fetch-tags + image: docker:git + pull: always + commands: + - git config --global --add safe.directory /drone/src + - git fetch --tags --force + when: + event: + exclude: + - pull_request + + - name: deps-backend + image: golang:1.20 + pull: always + commands: + - make deps-backend + volumes: + - name: deps + path: /go + + - name: prepare-test-env + image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env + pull: always + commands: + - ./build/test-env-prepare.sh + + - name: build image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env user: gitea commands: - - make unit-test-coverage test-check + - ./build/test-env-check.sh + - make backend environment: - GOPROXY: https://goproxy.io - TAGS: bindata gogit sqlite sqlite_unlock_notify - RACE_ENABLED: true - GITHUB_READ_TOKEN: - from_secret: github_read_token + GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not + GOSUMDB: sum.golang.org + TAGS: bindata depends_on: [deps-backend, prepare-test-env] volumes: - name: deps path: /go - - name: test-mysql + - name: unit-test image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env user: gitea commands: - - make test-mysql-migration integration-test-coverage + - make unit-test-coverage test-check environment: GOPROXY: https://goproxy.io TAGS: bindata RACE_ENABLED: true - TEST_LDAP: 1 - USE_REPO_TEST_DIR: 1 - TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" - depends_on: [build] + GITHUB_READ_TOKEN: + from_secret: github_read_token + depends_on: [deps-backend, prepare-test-env] volumes: - name: deps path: /go - - name: test-mysql8 + - name: unit-test-gogit image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env user: gitea commands: - - timeout -s ABRT 50m make test-mysql8-migration test-mysql8 + - make unit-test-coverage test-check environment: GOPROXY: https://goproxy.io - TAGS: bindata + TAGS: bindata gogit RACE_ENABLED: true - TEST_LDAP: 1 - USE_REPO_TEST_DIR: 1 - depends_on: [build] + GITHUB_READ_TOKEN: + from_secret: github_read_token + depends_on: [deps-backend, prepare-test-env] volumes: - name: deps path: /go - - name: test-mssql + - name: test-mysql image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env user: gitea commands: - - make test-mssql-migration test-mssql + - make test-mysql-migration integration-test-coverage environment: GOPROXY: https://goproxy.io TAGS: bindata RACE_ENABLED: true - TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 + TEST_INDEXER_CODE_ES_URL: "http://elastic:changeme@elasticsearch:9200" depends_on: [build] volumes: - name: deps @@ -398,11 +437,12 @@ steps: --- kind: pipeline -name: testing-arm64 +type: docker +name: testing-mysql8 platform: os: linux - arch: arm64 + arch: amd64 depends_on: - compliance @@ -421,16 +461,102 @@ volumes: temp: {} services: - - name: pgsql - pull: default - image: postgres:10 + - name: mysql8 + image: mysql:8 + pull: always environment: - POSTGRES_DB: test - POSTGRES_PASSWORD: postgres + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_DATABASE: testgitea - - name: ldap - pull: default - image: gitea/test-openldap:latest +steps: + - name: fetch-tags + image: docker:git + pull: always + commands: + - git config --global --add safe.directory /drone/src + - git fetch --tags --force + when: + event: + exclude: + - pull_request + + - name: deps-backend + image: golang:1.20 + pull: always + commands: + - make deps-backend + volumes: + - name: deps + path: /go + + - name: prepare-test-env + image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env + pull: always + commands: + - ./build/test-env-prepare.sh + + - name: build + image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env + user: gitea + commands: + - ./build/test-env-check.sh + - make backend + environment: + GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not + GOSUMDB: sum.golang.org + TAGS: bindata + depends_on: [deps-backend, prepare-test-env] + volumes: + - name: deps + path: /go + + - name: test-mysql8 + image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env + user: gitea + commands: + - timeout -s ABRT 50m make test-mysql8-migration test-mysql8 + environment: + GOPROXY: https://goproxy.io + TAGS: bindata + USE_REPO_TEST_DIR: 1 + depends_on: [build] + volumes: + - name: deps + path: /go + +--- +kind: pipeline +type: docker +name: testing-mssql + +platform: + os: linux + arch: amd64 + +depends_on: + - compliance + +trigger: + event: + - push + - tag + - pull_request + paths: + exclude: + - docs/** + +volumes: + - name: deps + temp: {} + +services: + - name: mssql + image: mcr.microsoft.com/mssql/server:latest + pull: always + environment: + ACCEPT_EULA: Y + MSSQL_PID: Standard + SA_PASSWORD: MwantsaSecurePassword1 steps: - name: fetch-tags @@ -454,13 +580,13 @@ steps: path: /go - name: prepare-test-env - image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env + image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env pull: always commands: - ./build/test-env-prepare.sh - name: build - image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env + image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env user: gitea commands: - ./build/test-env-check.sh @@ -468,39 +594,102 @@ steps: environment: GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not GOSUMDB: sum.golang.org - TAGS: bindata gogit sqlite sqlite_unlock_notify + TAGS: bindata depends_on: [deps-backend, prepare-test-env] volumes: - name: deps path: /go - - name: test-sqlite - image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env + - name: test-mssql + image: gitea/test_env:linux-amd64 # https://gitea.com/gitea/test-env user: gitea commands: - - timeout -s ABRT 50m make test-sqlite-migration test-sqlite + - make test-mssql-migration test-mssql environment: GOPROXY: https://goproxy.io - TAGS: bindata gogit sqlite sqlite_unlock_notify - RACE_ENABLED: true - TEST_TAGS: gogit sqlite sqlite_unlock_notify + TAGS: bindata USE_REPO_TEST_DIR: 1 depends_on: [build] volumes: - name: deps path: /go - - name: test-pgsql +--- +kind: pipeline +name: testing-sqlite + +platform: + os: linux + arch: arm64 + +depends_on: + - compliance + +trigger: + event: + - push + - tag + - pull_request + paths: + exclude: + - docs/** + +volumes: + - name: deps + temp: {} + +steps: + - name: fetch-tags + image: docker:git + pull: always + commands: + - git config --global --add safe.directory /drone/src + - git fetch --tags --force + when: + event: + exclude: + - pull_request + + - name: deps-backend + image: golang:1.20 + pull: always + commands: + - make deps-backend + volumes: + - name: deps + path: /go + + - name: prepare-test-env + image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env + pull: always + commands: + - ./build/test-env-prepare.sh + + - name: build image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env user: gitea commands: - - timeout -s ABRT 50m make test-pgsql-migration test-pgsql + - ./build/test-env-check.sh + - make backend + environment: + GOPROXY: https://goproxy.io # proxy.golang.org is blocked in China, this proxy is not + GOSUMDB: sum.golang.org + TAGS: bindata gogit sqlite sqlite_unlock_notify + depends_on: [deps-backend, prepare-test-env] + volumes: + - name: deps + path: /go + + - name: test-sqlite + image: gitea/test_env:linux-arm64 # https://gitea.com/gitea/test-env + user: gitea + commands: + - timeout -s ABRT 50m make test-sqlite-migration test-sqlite environment: GOPROXY: https://goproxy.io - TAGS: bindata gogit + TAGS: bindata gogit sqlite sqlite_unlock_notify RACE_ENABLED: true - TEST_TAGS: gogit - TEST_LDAP: 1 + TEST_TAGS: gogit sqlite sqlite_unlock_notify USE_REPO_TEST_DIR: 1 depends_on: [build] volumes: @@ -530,15 +719,6 @@ volumes: - name: deps temp: {} -services: - - name: pgsql - pull: default - image: postgres:10 - environment: - POSTGRES_DB: testgitea-e2e - POSTGRES_PASSWORD: postgres - POSTGRES_INITDB_ARGS: --encoding=UTF8 --lc-collate='en_US.UTF-8' --lc-ctype='en_US.UTF-8' - steps: - name: deps-frontend image: node:18 @@ -568,14 +748,12 @@ steps: - curl -sLO https://go.dev/dl/go1.20.linux-amd64.tar.gz && tar -C /usr/local -xzf go1.20.linux-amd64.tar.gz - groupadd --gid 1001 gitea && useradd -m --gid 1001 --uid 1001 gitea - apt-get -qq update && apt-get -qqy install build-essential - - export TEST_PGSQL_SCHEMA='' - ./build/test-env-prepare.sh - - su gitea bash -c "export PATH=$PATH:/usr/local/go/bin && timeout -s ABRT 40m make test-e2e-pgsql" + - su gitea bash -c "export PATH=$PATH:/usr/local/go/bin && timeout -s ABRT 40m make test-e2e-sqlite" environment: GOPROXY: https://goproxy.io GOSUMDB: sum.golang.org USE_REPO_TEST_DIR: 1 - TEST_PGSQL_DBNAME: 'testgitea-e2e' DEBIAN_FRONTEND: noninteractive depends_on: [build-frontend, deps-backend] volumes: @@ -709,8 +887,11 @@ trigger: - docs/** depends_on: - - testing-amd64 - - testing-arm64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite volumes: - name: deps @@ -842,8 +1023,11 @@ trigger: - tag depends_on: - - testing-arm64 - - testing-amd64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite volumes: - name: deps @@ -994,8 +1178,11 @@ platform: arch: amd64 depends_on: - - testing-amd64 - - testing-arm64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite trigger: ref: @@ -1064,8 +1251,11 @@ platform: arch: amd64 depends_on: - - testing-amd64 - - testing-arm64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite trigger: ref: @@ -1129,8 +1319,11 @@ platform: arch: amd64 depends_on: - - testing-amd64 - - testing-arm64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite trigger: ref: @@ -1192,8 +1385,11 @@ platform: arch: amd64 depends_on: - - testing-amd64 - - testing-arm64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite trigger: ref: @@ -1292,8 +1488,11 @@ platform: arch: arm64 depends_on: - - testing-amd64 - - testing-arm64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite trigger: ref: @@ -1362,8 +1561,11 @@ platform: arch: arm64 depends_on: - - testing-amd64 - - testing-arm64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite trigger: ref: @@ -1427,8 +1629,11 @@ platform: arch: arm64 depends_on: - - testing-amd64 - - testing-arm64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite trigger: ref: @@ -1493,8 +1698,11 @@ platform: arch: arm64 depends_on: - - testing-amd64 - - testing-arm64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite trigger: ref: @@ -1607,7 +1815,6 @@ platform: steps: - name: manifest-rootless - pull: always image: plugins/manifest pull: always settings: @@ -1671,8 +1878,11 @@ trigger: - failure depends_on: - - testing-amd64 - - testing-arm64 + - testing-mysql + - testing-mysql8 + - testing-mssql + - testing-pgsql + - testing-sqlite - release-version - release-latest - docker-linux-amd64-release @@ -190,6 +190,7 @@ help: @echo " - deps install dependencies" @echo " - deps-frontend install frontend dependencies" @echo " - deps-backend install backend dependencies" + @echo " - deps-tools install tool dependencies" @echo " - lint lint everything" @echo " - lint-frontend lint frontend files" @echo " - lint-backend lint backend files" @@ -821,7 +822,7 @@ docs: cd docs; make trans-copy clean build-offline; .PHONY: deps -deps: deps-frontend deps-backend +deps: deps-frontend deps-backend deps-tools .PHONY: deps-frontend deps-frontend: node_modules @@ -829,6 +830,9 @@ deps-frontend: node_modules .PHONY: deps-backend deps-backend: $(GO) mod download + +.PHONY: deps-tools +deps-tools: $(GO) install $(AIR_PACKAGE) $(GO) install $(EDITORCONFIG_CHECKER_PACKAGE) $(GO) install $(ERRCHECK_PACKAGE) diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index 1dd7bfab2..b286f37bf 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -55,28 +55,6 @@ LFS_START_SERVER = true LFS_JWT_SECRET = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w SSH_TRUSTED_USER_CA_KEYS = ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCb4DC1dMFnJ6pXWo7GMxTchtzmJHYzfN6sZ9FAPFR4ijMLfGki+olvOMO5Fql1/yGnGfbELQa1S6y4shSvj/5K+zUFScmEXYf3Gcr87RqilLkyk16RS+cHNB1u87xTHbETaa3nyCJeGQRpd4IQ4NKob745mwDZ7jQBH8AZEng50Oh8y8fi8skBBBzaYp1ilgvzG740L7uex6fHV62myq0SXeCa+oJUjq326FU8y+Vsa32H8A3e7tOgXZPdt2TVNltx2S9H2WO8RMi7LfaSwARNfy1zu+bfR50r6ef8Yx5YKCMz4wWb1SHU1GS800mjOjlInLQORYRNMlSwR1+vLlVDciOqFapDSbj+YOVOawR0R1aqlSKpZkt33DuOBPx9qe6CVnIi7Z+Px/KqM+OLCzlLY/RS+LbxQpDWcfTVRiP+S5qRTcE3M3UioN/e0BE/1+MpX90IGpvVkA63ILYbKEa4bM3ASL7ChTCr6xN5XT+GpVJveFKK1cfNx9ExHI4rzYE= -[lfs] -MINIO_BASE_PATH = lfs/ - -[attachment] -MINIO_BASE_PATH = attachments/ - -[avatars] -MINIO_BASE_PATH = avatars/ - -[repo-avatars] -MINIO_BASE_PATH = repo-avatars/ - -[storage] -STORAGE_TYPE = minio -SERVE_DIRECT = false -MINIO_ENDPOINT = minio:9000 -MINIO_ACCESS_KEY_ID = 123456 -MINIO_SECRET_ACCESS_KEY = 12345678 -MINIO_BUCKET = gitea -MINIO_LOCATION = us-east-1 -MINIO_USE_SSL = false - [mailer] ENABLED = true MAILER_TYPE = dummy @@ -122,6 +100,9 @@ INSTALL_LOCK = true SECRET_KEY = 9pCviYTWSb INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ +[lfs] +PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql/data/lfs + [packages] ENABLED = true diff --git a/tests/pgsql.ini.tmpl b/tests/pgsql.ini.tmpl index c39b6a79c..fbfbae7c6 100644 --- a/tests/pgsql.ini.tmpl +++ b/tests/pgsql.ini.tmpl @@ -105,7 +105,26 @@ SECRET_KEY = 9pCviYTWSb INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.hhSVGOANkaKk3vfCd2jDOIww4pUk0xtg9JRde5UogyQ [lfs] -PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-pgsql/data/lfs +MINIO_BASE_PATH = lfs/ + +[attachment] +MINIO_BASE_PATH = attachments/ + +[avatars] +MINIO_BASE_PATH = avatars/ + +[repo-avatars] +MINIO_BASE_PATH = repo-avatars/ + +[storage] +STORAGE_TYPE = minio +SERVE_DIRECT = false +MINIO_ENDPOINT = minio:9000 +MINIO_ACCESS_KEY_ID = 123456 +MINIO_SECRET_ACCESS_KEY = 12345678 +MINIO_BUCKET = gitea +MINIO_LOCATION = us-east-1 +MINIO_USE_SSL = false [packages] ENABLED = true |