diff --git a/.github/actions/restore-go-cache/action.yml b/.github/actions/restore-go-cache/action.yml new file mode 100644 index 00000000000..c4bec5e46c4 --- /dev/null +++ b/.github/actions/restore-go-cache/action.yml @@ -0,0 +1,27 @@ +name: Restore Go Cache +description: > + Read-only restore of the Go module cache, build cache, and CI tool binaries + warmed by the daily server-ci-cache-warm workflow. Never writes back — + only the warm job is allowed to save. +runs: + using: composite + steps: + - name: Export Go cache paths + shell: bash + run: | + echo "GOMODCACHE=${{ github.workspace }}/.go-mod-cache" >> "$GITHUB_ENV" + echo "GOCACHE=${{ github.workspace }}/.go-build-cache" >> "$GITHUB_ENV" + echo "GOBIN=${{ github.workspace }}/server/bin" >> "$GITHUB_ENV" + - name: Create Go cache directories + shell: bash + run: mkdir -p "$GOMODCACHE" "$GOCACHE" "$GOBIN" + - name: Restore Go module and build cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: | + .go-mod-cache + .go-build-cache + server/bin + key: gomod-v1-${{ hashFiles('server/.go-version') }}-${{ hashFiles('server/go.mod', 'server/go.sum', 'server/public/go.mod', 'server/public/go.sum') }} + restore-keys: | + gomod-v1-${{ hashFiles('server/.go-version') }}- # fall back to most recent cache for this Go version if go.mod changed diff --git a/.github/workflows/mmctl-test-template.yml b/.github/workflows/mmctl-test-template.yml index 5415e25558f..367ce1bc3b8 100644 --- a/.github/workflows/mmctl-test-template.yml +++ b/.github/workflows/mmctl-test-template.yml @@ -39,6 +39,10 @@ jobs: - name: Checkout mattermost project uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache + - name: Setup BUILD_IMAGE id: build run: | @@ -75,6 +79,9 @@ jobs: else export TESTFLAGS="-timeout 30m" fi + # GOBIN is not mounted separately: server/bin is inside the workspace + # mount below, and the Makefile sets GOBIN=$(PWD)/bin which resolves + # to /mattermost/server/bin — exactly where the test runner expects it. docker run --net ghactions_mm-test \ --ulimit nofile=8096:8096 \ --env-file=server/build/dotenv/test.env \ @@ -82,6 +89,10 @@ jobs: --env MM_SQLSETTINGS_DATASOURCE="${{ inputs.datasource }}" \ --env MMCTL_TESTFLAGS="$TESTFLAGS" \ --env FIPS_ENABLED="${{ inputs.fips-enabled }}" \ + --env GOMODCACHE \ + --env GOCACHE \ + -v "$GOMODCACHE":"$GOMODCACHE" \ + -v "$GOCACHE":"$GOCACHE" \ -v $PWD:/mattermost \ -w /mattermost/server \ $BUILD_IMAGE \ diff --git a/.github/workflows/server-ci-cache-warm.yml b/.github/workflows/server-ci-cache-warm.yml new file mode 100644 index 00000000000..0a54ea19c99 --- /dev/null +++ b/.github/workflows/server-ci-cache-warm.yml @@ -0,0 +1,91 @@ +# Warms the Go module cache, build cache, and CI tool binaries used by all +# server CI jobs. Runs once per day on master; all other CI jobs restore from +# this cache (read-only) and never write back to it. +# +# The cache is keyed on the Go version and a hash of all go.mod/go.sum files. +# PRs that change go.mod will miss the exact key and fall back to the most +# recent warm entry for the same Go version via restore-keys, then download +# only the delta. +# +# Run manually via workflow_dispatch to force a refresh (e.g. after a Go +# version bump before the next scheduled run). +name: Server CI Cache Warm + +on: + schedule: + - cron: "0 2 * * *" # Daily 2am UTC — 4 h before the nightly race run at 6am UTC + workflow_dispatch: + +permissions: + contents: read + +jobs: + warm: + name: Warm Go Module and Build Cache + runs-on: ubuntu-22.04 + steps: + - name: Checkout mattermost project + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # Note: this only warms the open-source platform cache. Enterprise + # builds have the same cold-start gap and can be addressed as a follow-up. + + - name: Compute Go version + id: go + working-directory: server + run: echo "version=$(cat .go-version)" >> "$GITHUB_OUTPUT" + + - name: Check if cache already warm + id: cache-check + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: | + .go-mod-cache + .go-build-cache + server/bin + key: gomod-v1-${{ hashFiles('server/.go-version') }}-${{ hashFiles('server/go.mod', 'server/go.sum', 'server/public/go.mod', 'server/public/go.sum') }} + lookup-only: true + + - name: Export Go cache paths + if: steps.cache-check.outputs.cache-hit != 'true' + run: | + echo "GOMODCACHE=$GITHUB_WORKSPACE/.go-mod-cache" >> "$GITHUB_ENV" + echo "GOCACHE=$GITHUB_WORKSPACE/.go-build-cache" >> "$GITHUB_ENV" + + - name: Create Go cache directories + if: steps.cache-check.outputs.cache-hit != 'true' + run: | + mkdir -p "$GOMODCACHE" "$GOCACHE" server/bin + chmod -R a+rwX "$GOMODCACHE" "$GOCACHE" server/bin + + - name: Warm cache inside build container + if: steps.cache-check.outputs.cache-hit != 'true' + run: | + # GOBIN is not mounted separately: server/bin is inside the workspace + # mount below, and the Makefile sets GOBIN=$(PWD)/bin which resolves + # to /mattermost/server/bin — the correct location for tool binaries. + docker run --rm \ + --env GOMODCACHE \ + --env GOCACHE \ + -v "$GOMODCACHE":"$GOMODCACHE" \ + -v "$GOCACHE":"$GOCACHE" \ + -v "$PWD:/mattermost" \ + -w /mattermost/server \ + "mattermost/mattermost-build-server:${{ steps.go.outputs.version }}" \ + make go-cache # cache warming logic lives in the Makefile, not buried in CI + du -sh "$GOMODCACHE" "$GOCACHE" server/bin + + - name: Fix file ownership (Docker writes files as root) + if: steps.cache-check.outputs.cache-hit != 'true' + run: sudo chown -R "$USER:$USER" .go-mod-cache .go-build-cache server/bin + + - name: Save cache + if: steps.cache-check.outputs.cache-hit != 'true' + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: | + .go-mod-cache + .go-build-cache + server/bin + key: gomod-v1-${{ hashFiles('server/.go-version') }}-${{ hashFiles('server/go.mod', 'server/go.sum', 'server/public/go.mod', 'server/public/go.sum') }} diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index c7537dad254..d196a2fb2b6 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -59,6 +59,8 @@ jobs: steps: - name: Checkout mattermost project uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache - name: Run setup-go-work run: make setup-go-work - name: Generate mocks @@ -76,6 +78,8 @@ jobs: steps: - name: Checkout mattermost project uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache - name: Run setup-go-work run: make setup-go-work - name: Run go mod tidy @@ -93,6 +97,8 @@ jobs: steps: - name: Checkout mattermost project uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache - name: Run setup-go-work run: make setup-go-work - name: Run go fix @@ -110,6 +116,8 @@ jobs: steps: - name: Checkout mattermost project uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache - name: Run setup-go-work run: make setup-go-work - name: Run golangci @@ -125,6 +133,8 @@ jobs: steps: - name: Checkout mattermost project uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache - name: Run setup-go-work run: make setup-go-work - name: Run make-gen-serialized @@ -142,6 +152,8 @@ jobs: steps: - name: Checkout mattermost project uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache - name: Run setup-go-work run: make setup-go-work - name: Run mattermost-vet-api @@ -189,6 +201,8 @@ jobs: steps: - name: Checkout mattermost project uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache - name: Run setup-go-work run: make setup-go-work - name: Generate store layers @@ -206,6 +220,8 @@ jobs: steps: - name: Checkout mattermost-server uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache - name: Run setup-go-work run: make setup-go-work - name: Check docs @@ -350,6 +366,8 @@ jobs: steps: - name: Checkout mattermost project uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache - name: ci/setup-node uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: diff --git a/.github/workflows/server-test-template.yml b/.github/workflows/server-test-template.yml index a8168fa2f47..be5118d08a4 100644 --- a/.github/workflows/server-test-template.yml +++ b/.github/workflows/server-test-template.yml @@ -87,6 +87,9 @@ jobs: - name: Checkout mattermost project uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Restore Go cache + uses: ./.github/actions/restore-go-cache + - name: Restore test timing data if: inputs.shard-total > 1 id: timing-cache @@ -220,6 +223,9 @@ jobs: DOCKER_CMD="/mattermost/server/run-shard-tests.sh" fi + # GOBIN is not mounted separately: server/bin is inside the workspace + # mount below, and the Makefile sets GOBIN=$(PWD)/bin which resolves + # to /mattermost/server/bin — exactly where run-shard-tests.sh expects it. docker run --net ghactions_mm-test \ --ulimit nofile=8096:8096 \ --env-file=server/build/dotenv/test.env \ @@ -232,6 +238,10 @@ jobs: --env RACE_MODE \ --env TEST_TARGET \ --env BUILD_NUMBER \ + --env GOMODCACHE \ + --env GOCACHE \ + -v "$GOMODCACHE":"$GOMODCACHE" \ + -v "$GOCACHE":"$GOCACHE" \ -v $PWD:/mattermost \ -w /mattermost/server \ $BUILD_IMAGE \ diff --git a/server/Makefile b/server/Makefile index 3ec286eaa70..5039e618b01 100644 --- a/server/Makefile +++ b/server/Makefile @@ -327,8 +327,10 @@ golang-versions: ## Install Golang versions used for compatibility testing (e.g. done export GO_COMPATIBILITY_TEST_VERSIONS="${GO_COMPATIBILITY_TEST_VERSIONS}" -golangci-lint: setup-go-work ## Run golangci-lint on codebase +install-golangci-lint: $(GO) install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2 + +golangci-lint: setup-go-work install-golangci-lint ## Run golangci-lint on codebase ifeq ($(BUILD_ENTERPRISE_READY),true) $(GOBIN)/golangci-lint run ./... ./public/... $(BUILD_ENTERPRISE_DIR)/... else @@ -344,61 +346,57 @@ i18n-check: ## Exit on empty translation strings and translation source strings $(GOBIN)/mmgotool i18n clean-empty --portal-dir="" --check $(GOBIN)/mmgotool i18n check-empty-src --portal-dir="" -store-mocks: setup-go-work ## Creates mock files. +install-mockery: $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 + +store-mocks: setup-go-work install-mockery ## Creates mock files. $(GOBIN)/mockery --config channels/store/.mockery.yaml -cache-mocks: setup-go-work - $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 +cache-mocks: setup-go-work install-mockery $(GOBIN)/mockery --config platform/services/cache/.mockery.yaml store-layers: setup-go-work ## Generate layers for the store $(GO) generate $(GOFLAGS) ./channels/store -new-migration: ## Creates a new migration. Run with make new-migration name=<> +install-morph: $(GO) install github.com/mattermost/morph/cmd/morph@1e0640c + +new-migration: install-morph ## Creates a new migration. Run with make new-migration name=<> @echo "Generating new migration for postgres" $(GOBIN)/morph new script $(name) --driver postgres --dir channels/db/migrations --sequence -filestore-mocks: setup-go-work ## Creates mock files. - $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 +filestore-mocks: setup-go-work install-mockery ## Creates mock files. $(GOBIN)/mockery --config platform/shared/filestore/.mockery.yaml -ldap-mocks: setup-go-work ## Creates mock files for ldap. - $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 +ldap-mocks: setup-go-work install-mockery ## Creates mock files for ldap. $(GOBIN)/mockery --dir $(BUILD_ENTERPRISE_DIR)/ldap --all --inpackage --note 'Regenerate this file using `make ldap-mocks`.' -plugin-mocks: setup-go-work ## Creates mock files for plugins. - $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 +plugin-mocks: setup-go-work install-mockery ## Creates mock files for plugins. $(GOBIN)/mockery --config public/plugin/.mockery.yaml -einterfaces-mocks: setup-go-work ## Creates mock files for einterfaces. - $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 +einterfaces-mocks: setup-go-work install-mockery ## Creates mock files for einterfaces. $(GOBIN)/mockery --config einterfaces/.mockery.yaml -searchengine-mocks: setup-go-work ## Creates mock files for searchengines. - $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 +searchengine-mocks: setup-go-work install-mockery ## Creates mock files for searchengines. $(GOBIN)/mockery --config platform/services/searchengine/.mockery.yaml -sharedchannel-mocks: setup-go-work ## Creates mock files for shared channels. - $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 +sharedchannel-mocks: setup-go-work install-mockery ## Creates mock files for shared channels. $(GOBIN)/mockery --config platform/services/sharedchannel/.mockery.yaml -misc-mocks: setup-go-work ## Creates mocks for misc interfaces. - $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 +misc-mocks: setup-go-work install-mockery ## Creates mocks for misc interfaces. $(GOBIN)/mockery --config channels/utils/.mockery.yaml -email-mocks: setup-go-work ## Creates mocks for misc interfaces. - $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 +email-mocks: setup-go-work install-mockery ## Creates mocks for misc interfaces. $(GOBIN)/mockery --config channels/app/email/.mockery.yaml -platform-mocks: setup-go-work ## Creates mocks for platform interfaces. - $(GO) install github.com/vektra/mockery/v2/...@v2.53.4 +platform-mocks: setup-go-work install-mockery ## Creates mocks for platform interfaces. $(GOBIN)/mockery --config channels/app/platform/.mockery.yaml -mmctl-mocks: setup-go-work ## Creates mocks for mmctl +install-mockgen: $(GO) install github.com/golang/mock/mockgen@v1.6.0 + +mmctl-mocks: setup-go-work install-mockgen ## Creates mocks for mmctl $(GOBIN)/mockgen -destination=cmd/mmctl/mocks/client_mock.go -copyright_file=cmd/mmctl/mocks/copyright.txt -package=mocks github.com/mattermost/mattermost/server/v8/cmd/mmctl/client Client pluginapi: setup-go-work ## Generates api and hooks glue code for plugins @@ -440,6 +438,16 @@ check-style: plugin-checker vet golangci-lint ## Runs style/lint checks gotestsum: $(GO) install gotest.tools/gotestsum@v1.13.0 +install-ci-tools: gotestsum install-golangci-lint install-govet install-mockery install-mockgen install-morph install-msgp ## Install all tools required by CI + +go-cache: setup-go-work install-ci-tools ## Pre-warm Go module and build cache for CI + $(GO) mod download all + @(cd public && $(GO) mod download all) + $(GO) build ./... + @(cd public && $(GO) build ./...) + $(GO) test -run=^$$ -count=1 ./... 2>&1 | tail -20 + @(cd public && $(GO) test -run=^$$ -count=1 ./... 2>&1 | tail -20) + test-compile: setup-go-work gotestsum ## Compile tests. @echo COMPILE TESTS @@ -842,8 +850,10 @@ ifeq ($(BUILD_ENTERPRISE_READY),true) mv enterprise/external_imports.go.orig enterprise/external_imports.go endif -vet: setup-go-work ## Run mattermost go vet specific checks +install-govet: cd ../tools/mattermost-govet && $(GO) install . + +vet: setup-go-work install-govet ## Run mattermost go vet specific checks $(GO) vet -vettool=$(GOBIN)/mattermost-govet \ -structuredLogging \ -inconsistentReceiverName \ @@ -868,13 +878,15 @@ endif vet-api: export GO := $(GO) vet-api: export GOBIN := $(GOBIN) vet-api: export ROOT := $(ROOT) -vet-api: ## Run mattermost go vet to verify api4 documentation, currently not passing - cd ../tools/mattermost-govet && $(GO) install . +vet-api: install-govet ## Run mattermost go vet to verify api4 documentation, currently not passing make -C ../api build ./scripts/vet-api-check.sh +install-msgp: + $(GO) install github.com/tinylib/msgp@v1.1.6 + gen-serialized: export LICENSE_HEADER:=$(LICENSE_HEADER) -gen-serialized: setup-go-work ## Generates serialization methods for hot structs +gen-serialized: setup-go-work install-msgp ## Generates serialization methods for hot structs # This tool only works at a file level, not at a package level. # There will be some warnings about "unresolved identifiers", # but that is because of the above problem. Since we are generating @@ -882,7 +894,6 @@ gen-serialized: setup-go-work ## Generates serialization methods for hot structs # identifiers will be resolved. An alternative to remove the warnings # would be to temporarily move all the structs to the same file, # but that involves a lot of manual work. - $(GO) install github.com/tinylib/msgp@v1.1.6 $(GOBIN)/msgp -file=./public/model/utils.go -tests=false -o=./public/model/utils_serial_gen.go @echo "$$LICENSE_HEADER" > tmp.go @cat ./public/model/utils_serial_gen.go >> tmp.go