2019-02-12 13:19:01 -05:00
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
2019-11-29 06:59:40 -05:00
// See LICENSE.txt for license information.
2019-02-12 13:19:01 -05:00
package config
import (
2021-05-19 07:30:26 -04:00
"bytes"
2020-07-15 14:40:36 -04:00
"encoding/json"
2021-05-19 07:30:26 -04:00
"fmt"
2020-09-21 03:28:46 -04:00
"reflect"
2019-02-12 13:19:01 -05:00
"strings"
2023-06-11 01:24:35 -04:00
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/i18n"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/v8/channels/utils"
2019-02-12 13:19:01 -05:00
)
2021-05-19 07:30:26 -04:00
// marshalConfig converts the given configuration into JSON bytes for persistence.
func marshalConfig ( cfg * model . Config ) ( [ ] byte , error ) {
return json . MarshalIndent ( cfg , "" , " " )
}
2019-02-12 13:19:01 -05:00
// desanitize replaces fake settings with their actual values.
func desanitize ( actual , target * model . Config ) {
2021-07-12 14:05:36 -04:00
if target . LdapSettings . BindPassword != nil && * target . LdapSettings . BindPassword == model . FakeSetting {
2019-02-12 13:19:01 -05:00
* target . LdapSettings . BindPassword = * actual . LdapSettings . BindPassword
}
2021-07-12 14:05:36 -04:00
if * target . FileSettings . PublicLinkSalt == model . FakeSetting {
2019-02-12 13:19:01 -05:00
* target . FileSettings . PublicLinkSalt = * actual . FileSettings . PublicLinkSalt
}
2021-07-12 14:05:36 -04:00
if * target . FileSettings . AmazonS3SecretAccessKey == model . FakeSetting {
2019-02-12 13:19:01 -05:00
target . FileSettings . AmazonS3SecretAccessKey = actual . FileSettings . AmazonS3SecretAccessKey
}
2026-05-20 06:04:21 -04:00
if target . FileSettings . ExportAmazonS3SecretAccessKey != nil && * target . FileSettings . ExportAmazonS3SecretAccessKey == model . FakeSetting {
target . FileSettings . ExportAmazonS3SecretAccessKey = actual . FileSettings . ExportAmazonS3SecretAccessKey
}
MM-68662: Add Azure Blob Storage filestore backend (#36498)
* Generalize file backend error types
Replace S3FileBackendAuthError and S3FileBackendNoBucketError with
backend-agnostic FileBackendAuthError and FileBackendNoBucketError so
non-S3 drivers can return them and the admin "Test Connection" flow
keeps surfacing useful messages.
The old S3-prefixed names are kept as type aliases of the generic
types so external code (plugins, historical consumers) continues to
compile, and so existing S3 construction sites stay untouched.
The type switch in connectionTestErrorToAppError now matches the
generic types, with new i18n keys (test_connection_auth.app_error
and test_connection_no_bucket.app_error) whose wording does not name
S3. The old S3-specific i18n keys are dropped via `make i18n-extract`
since they are no longer referenced from code; the api4 test that
asserted on those keys is updated, and the Cypress
`MM-T996 Amazon S3 connection error messaging` spec that asserted
on the old user-facing string is updated to the new wording.
------
AI assisted commit
* Pull in Azure SDK and uuid dependencies
Bring in github.com/Azure/azure-sdk-for-go/sdk/azcore and
.../sdk/storage/azblob (with .../sdk/internal as their indirect
dependency). The two are needed by the upcoming Azure Blob Storage
filestore backend and its lazy-Range-backed reader. The bump of
golang.org/x/{crypto,net,sys,term,text} comes transitively from
azblob's minimum versions.
Also promotes github.com/google/uuid from indirect to direct,
since the Azure backend uses it to generate block IDs that share
the same wire format the SDK itself produces in UploadStream.
------
AI assisted commit
* Add azureRangeReader, a seekable Range-backed blob reader
A small standalone type that satisfies the FileBackend interface's
ReadCloseSeeker + the broader io.ReaderAt contract on top of Azure
Blob Storage HTTP Range requests. Lands as its own commit because
the upcoming Azure FileBackend driver builds on it, and the reader
itself is independently useful — and independently testable against
a fake downloader without standing up an Azure client.
Design notes:
* Read opens an HTTP Range stream lazily at the current offset and
reuses it for sequential reads. Seek to a different offset closes
the open stream; the next Read re-opens it.
* Seek to the same offset is a no-op and does not close the open
stream, so callers like zip.NewReader that probe with redundant
seeks don't kick off a fresh download.
* ReadAt issues a dedicated ranged DownloadStream per call and does
not touch the streaming cursor — matches the io.ReaderAt contract
the bulk-import worker's zip.NewReader path relies on.
* Close cancels the context (which any in-flight Azure call will
observe and abort), stops the deadline timer, and closes the
current body if any. It is safe to call when no body was ever
opened.
* CancelTimeout lets long-running consumers like the import worker
opt out of the per-operation deadline that would otherwise kill
multi-minute downloads partway through.
The implementation talks to a small blobDownloader interface rather
than *blob.Client directly so the unit tests can substitute a fake
downloader that records every requested Range and tracks Close
calls on the bodies it hands out.
------
AI assisted commit
* Add Azure Blob Storage filestore driver
Implements the FileBackend interface against Azure Blob Storage in
a new azurestore.go (~520 LOC). The driver is not yet selectable
via NewFileBackend's switch — that wiring lands in the next commit
together with the admin config surface — but the driver itself is
complete and self-contained behind the FileBackendSettings struct.
Filesstore.go grows three pieces of supporting infrastructure that
the driver consumes:
* a `driverAzure = "azureblob"` constant alongside the existing
driverS3 and driverLocal,
* an Azure-specific block on FileBackendSettings (storage account,
access key, container, path prefix, endpoint, SSL flag, request
timeout),
* a CheckMandatoryAzureFields validator that mirrors
CheckMandatoryS3Fields.
Behavioural notes that warrant calling out:
* Reader returns the previously-added azureRangeReader, so reads
stream lazily over HTTP Range and ReadAt is available for the
bulk-import worker's zip.NewReader path. The deadline timer is
armed before the initial GetProperties call so the HEAD itself
is bounded.
* WriteFile and AppendFile both go through StageBlock +
CommitBlockList via a shared stageBlocks helper, never the SDK's
UploadStream. UploadStream's small-payload fast path falls back
to single-shot PutBlob, which leaves the resulting blob with no
committed block list; a subsequent AppendFile that calls
CommitBlockList on that blob would then clobber its content.
Routing every write through the block-list mechanism keeps
AppendFile correct regardless of payload size.
* AppendFile stages the new chunk as one or more blocks and commits
the existing committed block list plus the newly staged IDs.
The new bytes go up exactly once — no re-download, no
re-concatenate, no re-upload of the prior contents.
* WriteFileContext does not wrap the caller-supplied context with
its own timeout — that timeout is applied in WriteFile only,
matching the S3 driver, so long-running TryWriteFileContext
callers (like message-export bulk writes) opt out of the
per-operation timeout the way the abstraction documents.
Authentication is shared-key only for this drop; Microsoft Entra
ID / managed identity is deferred to a follow-up. The endpoint is
configurable so the same code targets the production Azure host
(vhost style — {account}.blob.core.windows.net) or Azurite /
Azure Government / sovereign clouds (path style —
host[:port]/{account}).
------
AI assisted commit
* Wire Azure backend into config, validation, and driver selection
This commit registers the previously-added AzureFileBackend driver
with the rest of the system. Until now the driver was usable only
via direct construction; after this commit, `DriverName: "azureblob"`
in config.json is a fully-supported deployment configuration.
Five integration sites are touched:
* `newFileBackend` in filesstore.go now dispatches `driverAzure` to
NewAzureFileBackend, alongside the existing s3 and local cases.
NewFileBackendSettingsFromConfig (and its export counterpart) gain
an Azure branch that maps the model.FileSettings fields onto the
Azure-specific FileBackendSettings fields.
* `model.FileSettings` grows the user-facing Azure config schema:
storage account, access key, container, path prefix, endpoint,
SSL flag, request timeout, plus matching Export* fields for the
dedicated export store. SetDefaults populates them so deployments
that never opted into Azure don't carry nil pointers. `isValid`
accepts the new ImageDriverAzure constant.
* `Config.Sanitize()` masks AzureAccessKey and ExportAzureAccessKey
the same way it masks AmazonS3SecretAccessKey, so the shared key
never reaches an API consumer in plain text.
* `desanitize()` restores the masked keys on a config write so a
PATCH that doesn't touch the key doesn't clobber it with the
FakeSetting placeholder.
* `configSensitivePaths` covers both Azure key paths so audit
diffs don't include them either.
* `ConfigToFileBackendSettings` in the `mattermost db` CLI helper
gets the Azure branch its production counterpart already has —
without it, `mattermost db migrate` / `db downgrade` would fail
on Azure-configured deployments with "missing azure storage
account setting".
Finally, the shared FileBackendTestSuite is now wired against
Azurite via TestAzureFileBackendTestSuite, which skips when
CI_AZURITE_HOST is unreachable. The test-infra wiring (the docker
service, the env vars, the start_dependencies entry) landed in a
previous PR; this commit is what makes the suite actually exercise
the Azure driver end to end.
------
AI assisted commit
* Validate Azure timeout and path prefix in Config.IsValid
Parity with the S3-side checks that already cover
AmazonS3RequestTimeoutMilliseconds and AmazonS3PathPrefix. Without
these, a zero/negative AzureRequestTimeoutMilliseconds passes
validation and later creates immediately-expired request contexts,
and leading/trailing whitespace in AzurePathPrefix produces blob
keys that don't match what the admin configured.
Same checks added for the Export* counterparts. The
file_driver.app_error translation is updated to mention the new
'azureblob' option alongside 'local' and 'amazons3'.
------
AI assisted commit
* Stream zip entries from the Azure backend
writeZipEntry was calling ReadFile, which loads the entire blob
into memory before writing it to the archive. For large blobs or
deep directories this spikes RSS or OOMs the goroutine. Switch to
Reader (the streaming azureRangeReader) and io.Copy into the zip
entry so memory stays bounded regardless of blob size.
------
AI assisted commit
* Use a backend-agnostic fallback for FileBackendNoBucketError
The fallback Error() message was "no such bucket", which leaks S3
terminology when an Azure caller returns the type with no wrapped
Err. Use "no such bucket or container" so logs and external error
handling stay neutral across backends.
------
AI assisted commit
* Defend Azure path prefix against directory traversal
Reject ".." in AzurePathPrefix and ExportAzurePathPrefix at config
validation time, since path.Join collapses traversal segments and a
prefix like "../other-tenant" would otherwise escape the configured
isolation boundary.
Harden the prefix helper as a second line of defense: if the joined
path no longer sits inside pathPrefix, fall back to joining the prefix
with the base name of the caller-supplied path. That preserves the
prefix invariant for plugin and import paths that the upload code does
not sanitize uniformly.
------
AI assisted commit
* Honor SkipVerify when constructing the Azure client
FileBackendSettings.SkipVerify is plumbed through from the System Console
the same way it is for S3, so admins toggling the flag for self-signed
endpoints (Azurite, sovereign clouds) get the behavior they expect
without having to drop SSL entirely and send the shared key in clear
text.
------
AI assisted commit
* Warn when the Azure request timeout falls back to its default
Config.IsValid already rejects non-positive AzureRequestTimeoutMilliseconds
for any path that goes through config validation, so this warn only fires
for direct callers that bypass validation (tests, helpers). Logging the
substitution turns a silent coercion into something an operator can
correlate against unexpected request behavior.
------
AI assisted commit
* Cap Azure request timeout at 10 minutes
Reject AzureRequestTimeoutMilliseconds values above the ceiling so an
operator (or someone who has admin access) cannot effectively disable
timeouts by setting the value to math.MaxInt64. A hung Azure call then
holds a goroutine open until the OS gives up.
Applies the same bound to ExportAzureRequestTimeoutMilliseconds. S3 has
the same gap; treating it is out of scope here but worth a follow-up.
------
AI assisted commit
* Refuse AppendFile on blobs without a committed block list
A blob written by another tool (Azure portal, azcopy, a migration script,
a plugin using Put Blob) has its content in the blob but an empty
committed-block list. Committing a new block list against such a blob
silently replaces the existing content with only the appended bytes.
Check the blob's properties before staging when the committed-block list
is empty, and refuse with a clear error if the blob has content. Same
hazard for an admin pointing the backend at an existing container with
pre-existing files.
Adds an integration test against Azurite to lock the behavior in.
------
AI assisted commit
* Surface truncated reads from azureRangeReader
Read closed the body cleanly and returned io.EOF even when the remote
stream terminated before the blob's content length. Callers (and any
retry layer above) then accepted a partial blob as complete.
ReadAt unconditionally rewrote io.ErrUnexpectedEOF to io.EOF, which made
truncated downloads indistinguishable from clean reads. That is exactly
what zip.NewReader consumes for archive readers, so the bulk-import
worker would silently import partial archives.
Read now closes the body, nils it, and returns io.ErrUnexpectedEOF when
EOF arrives before offset reaches size. ReadAt only collapses
ErrUnexpectedEOF to EOF when the full count was delivered and the stream
was consumed to the end of the blob. Otherwise the truncation
propagates with context.
Both code paths are exercised by new fakeDownloader-backed tests.
------
AI assisted commit
* Move container provisioning out of Azure TestConnection
Auto-creating the container inside TestConnection meant a typo in the
System Console (mattermosst instead of mattermost) silently provisioned
an unwanted container in the admin's Azure subscription, with no audit
log and no warning. They'd discover it later when uploads landed
somewhere unexpected.
TestConnection now returns FileBackendNoBucketError when the container
is missing, mirroring the S3 contract. A new MakeContainer method
mirrors S3FileBackend.MakeBucket, and Server.Start dispatches via two
capability interfaces (bucketMaker / containerMaker) instead of a hard
S3 type assertion — so the NoBucket error is no longer silently
swallowed for backends Server.Start has not been taught about.
------
AI assisted commit
* Carry file backend auth detail through to AppError
The Test Connection button collapsed every typed backend failure into
the same generic i18n message. Operators trying to debug bad credentials
or a missing bucket only saw "Unable to authenticate against the file
storage backend" with no SDK code to grep for in their logs.
Use errors.As so the typed checks survive future wrapping, and pass the
underlying error string through the NewAppError details argument. The
AppError serializer surfaces that detail to the admin console alongside
the translated message, so a bad S3 InvalidAccessKeyId or an Azure
AuthenticationFailed shows up in the toast without an i18n schema
change.
------
AI assisted commit
* Remove non-ascii characters from comments
------
AI assisted commit
* Make linter happy
------
AI assisted commit
* Harden Azure prefix boundary check
strings.HasPrefix on the joined path is a string-level check, not a
path-level one, so a configured prefix of "mattermost" accepts a joined
result of "mattermost-evil/...". A crafted caller path like
"../mattermost-evil/secrets" would collapse via path.Join to that exact
sibling and slip through the boundary check, escaping the configured
prefix scope.
Require the joined path to be the cleaned prefix itself or to start with
the prefix followed by a path separator. The fallback path.Join uses the
same cleaned prefix for consistency.
------
AI assisted commit
* Provision Azurite container in standalone test setup
The shared FileBackendTestSuite's SetupTest already handles a missing
container by detecting FileBackendNoBucketError from TestConnection and
calling MakeContainer, but TestAzureFileBackendAppendRefusesNonBlockBlob
bypasses SetupTest and calls TestConnection directly. On a fresh Azurite
instance the test would fail before exercising the append-refusal logic.
Extract a newAzuriteBackend(t) helper alongside azuriteSettings(t) that
builds the backend and ensures the container exists, mirroring the
suite's setup. Use errors.As for forward compatibility with future
wrapping.
------
AI assisted commit
* Fix grammar in email-settings i18n string
"Email settings has unset values." -> "Email settings have unset values."
------
AI assisted commit
* Make Azure MakeContainer idempotent
Treat a ContainerAlreadyExists response as success so that two nodes
racing through TestConnection plus MakeContainer at boot both converge
instead of having the loser fail. Mirrors how the S3 backend handles
the equivalent BucketAlreadyOwnedByYou case.
------
AI assisted commit
* Narrow AzureEndpoint comment to path-style only
The setting only builds path-style URLs, so it cannot reach sovereign
clouds like Azure Government or Azure China, which require vhost-style
endpoints. Update the comment to reflect what the code actually does
and document that sovereign-cloud support is out of scope.
------
AI assisted commit
2026-05-14 12:59:18 -04:00
if target . FileSettings . AzureAccessKey != nil && * target . FileSettings . AzureAccessKey == model . FakeSetting {
target . FileSettings . AzureAccessKey = actual . FileSettings . AzureAccessKey
}
if target . FileSettings . ExportAzureAccessKey != nil && * target . FileSettings . ExportAzureAccessKey == model . FakeSetting {
target . FileSettings . ExportAzureAccessKey = actual . FileSettings . ExportAzureAccessKey
}
2019-02-12 13:19:01 -05:00
2021-07-12 14:05:36 -04:00
if * target . EmailSettings . SMTPPassword == model . FakeSetting {
2019-02-12 13:19:01 -05:00
target . EmailSettings . SMTPPassword = actual . EmailSettings . SMTPPassword
}
2021-07-12 14:05:36 -04:00
if * target . GitLabSettings . Secret == model . FakeSetting {
2019-02-12 13:19:01 -05:00
target . GitLabSettings . Secret = actual . GitLabSettings . Secret
}
2021-07-12 14:05:36 -04:00
if target . GoogleSettings . Secret != nil && * target . GoogleSettings . Secret == model . FakeSetting {
2020-08-31 02:18:39 -04:00
target . GoogleSettings . Secret = actual . GoogleSettings . Secret
}
2021-07-12 14:05:36 -04:00
if target . Office365Settings . Secret != nil && * target . Office365Settings . Secret == model . FakeSetting {
2020-08-31 02:18:39 -04:00
target . Office365Settings . Secret = actual . Office365Settings . Secret
}
2021-07-12 14:05:36 -04:00
if target . OpenIdSettings . Secret != nil && * target . OpenIdSettings . Secret == model . FakeSetting {
2020-12-08 21:58:37 -05:00
target . OpenIdSettings . Secret = actual . OpenIdSettings . Secret
}
2021-07-12 14:05:36 -04:00
if * target . SqlSettings . DataSource == model . FakeSetting {
2019-02-12 13:19:01 -05:00
* target . SqlSettings . DataSource = * actual . SqlSettings . DataSource
}
2021-07-12 14:05:36 -04:00
if * target . SqlSettings . AtRestEncryptKey == model . FakeSetting {
2020-05-07 08:11:05 -04:00
target . SqlSettings . AtRestEncryptKey = actual . SqlSettings . AtRestEncryptKey
2019-02-12 13:19:01 -05:00
}
2026-05-26 03:36:03 -04:00
if target . AuditStorageSettings . DataSource != nil &&
* target . AuditStorageSettings . DataSource == model . FakeSetting &&
actual . AuditStorageSettings . DataSource != nil {
* target . AuditStorageSettings . DataSource = * actual . AuditStorageSettings . DataSource
}
2021-07-12 14:05:36 -04:00
if * target . ElasticsearchSettings . Password == model . FakeSetting {
2020-05-07 08:11:05 -04:00
* target . ElasticsearchSettings . Password = * actual . ElasticsearchSettings . Password
2019-02-12 13:19:01 -05:00
}
2020-05-27 09:42:48 -04:00
2020-09-17 14:16:59 -04:00
if len ( target . SqlSettings . DataSourceReplicas ) == len ( actual . SqlSettings . DataSourceReplicas ) {
for i , value := range target . SqlSettings . DataSourceReplicas {
2021-07-12 14:05:36 -04:00
if value == model . FakeSetting {
2020-09-17 14:16:59 -04:00
target . SqlSettings . DataSourceReplicas [ i ] = actual . SqlSettings . DataSourceReplicas [ i ]
}
}
2020-05-27 09:42:48 -04:00
}
2020-09-17 14:16:59 -04:00
if len ( target . SqlSettings . DataSourceSearchReplicas ) == len ( actual . SqlSettings . DataSourceSearchReplicas ) {
for i , value := range target . SqlSettings . DataSourceSearchReplicas {
2021-07-12 14:05:36 -04:00
if value == model . FakeSetting {
2020-09-17 14:16:59 -04:00
target . SqlSettings . DataSourceSearchReplicas [ i ] = actual . SqlSettings . DataSourceSearchReplicas [ i ]
}
}
2020-05-27 09:42:48 -04:00
}
2020-08-20 09:01:22 -04:00
2021-08-12 05:49:16 -04:00
if * target . MessageExportSettings . GlobalRelaySettings . SMTPPassword == model . FakeSetting {
* target . MessageExportSettings . GlobalRelaySettings . SMTPPassword = * actual . MessageExportSettings . GlobalRelaySettings . SMTPPassword
2020-08-20 09:01:22 -04:00
}
2020-08-31 02:18:39 -04:00
2021-07-12 14:05:36 -04:00
if * target . ServiceSettings . SplitKey == model . FakeSetting {
2020-10-29 18:54:39 -04:00
* target . ServiceSettings . SplitKey = * actual . ServiceSettings . SplitKey
}
2024-11-11 05:43:26 -05:00
2026-05-20 06:04:21 -04:00
if target . ServiceSettings . GoogleDeveloperKey != nil && * target . ServiceSettings . GoogleDeveloperKey == model . FakeSetting {
target . ServiceSettings . GoogleDeveloperKey = actual . ServiceSettings . GoogleDeveloperKey
}
if target . ServiceSettings . GiphySdkKey != nil && * target . ServiceSettings . GiphySdkKey == model . FakeSetting {
target . ServiceSettings . GiphySdkKey = actual . ServiceSettings . GiphySdkKey
}
if target . CacheSettings . RedisPassword != nil && * target . CacheSettings . RedisPassword == model . FakeSetting {
target . CacheSettings . RedisPassword = actual . CacheSettings . RedisPassword
}
if target . AutoTranslationSettings . LibreTranslate != nil &&
target . AutoTranslationSettings . LibreTranslate . APIKey != nil &&
* target . AutoTranslationSettings . LibreTranslate . APIKey == model . FakeSetting {
target . AutoTranslationSettings . LibreTranslate . APIKey = actual . AutoTranslationSettings . LibreTranslate . APIKey
}
2024-11-11 05:43:26 -05:00
for id , settings := range target . PluginSettings . Plugins {
for k , v := range settings {
if v == model . FakeSetting {
settings [ k ] = actual . PluginSettings . Plugins [ id ] [ k ]
}
}
}
2019-02-12 13:19:01 -05:00
}
2021-02-12 04:22:27 -05:00
// fixConfig patches invalid or missing data in the configuration.
func fixConfig ( cfg * model . Config ) {
2019-02-12 13:19:01 -05:00
// Ensure SiteURL has no trailing slash.
if strings . HasSuffix ( * cfg . ServiceSettings . SiteURL , "/" ) {
* cfg . ServiceSettings . SiteURL = strings . TrimRight ( * cfg . ServiceSettings . SiteURL , "/" )
}
// Ensure the directory for a local file store has a trailing slash.
2021-07-12 14:05:36 -04:00
if * cfg . FileSettings . DriverName == model . ImageDriverLocal {
2020-08-04 07:58:48 -04:00
if * cfg . FileSettings . Directory != "" && ! strings . HasSuffix ( * cfg . FileSettings . Directory , "/" ) {
2019-02-12 13:19:01 -05:00
* cfg . FileSettings . Directory += "/"
}
}
2024-04-29 05:23:01 -04:00
fixInvalidLocales ( cfg )
2019-02-12 13:19:01 -05:00
}
2024-04-29 05:23:01 -04:00
// fixInvalidLocales checks and corrects the given config for invalid locale-related settings.
func fixInvalidLocales ( cfg * model . Config ) bool {
2019-02-12 13:19:01 -05:00
var changed bool
2021-02-26 02:12:49 -05:00
locales := i18n . GetSupportedLocales ( )
2019-02-12 13:19:01 -05:00
if _ , ok := locales [ * cfg . LocalizationSettings . DefaultServerLocale ] ; ! ok {
2023-09-26 12:49:27 -04:00
mlog . Warn ( "DefaultServerLocale must be one of the supported locales. Setting DefaultServerLocale to en as default value." , mlog . String ( "locale" , * cfg . LocalizationSettings . DefaultServerLocale ) )
2021-07-12 14:05:36 -04:00
* cfg . LocalizationSettings . DefaultServerLocale = model . DefaultLocale
2019-02-12 13:19:01 -05:00
changed = true
}
if _ , ok := locales [ * cfg . LocalizationSettings . DefaultClientLocale ] ; ! ok {
2023-09-26 12:49:27 -04:00
mlog . Warn ( "DefaultClientLocale must be one of the supported locales. Setting DefaultClientLocale to en as default value." , mlog . String ( "locale" , * cfg . LocalizationSettings . DefaultClientLocale ) )
2021-07-12 14:05:36 -04:00
* cfg . LocalizationSettings . DefaultClientLocale = model . DefaultLocale
2019-02-12 13:19:01 -05:00
changed = true
}
2021-01-25 05:15:17 -05:00
if * cfg . LocalizationSettings . AvailableLocales != "" {
2019-02-12 13:19:01 -05:00
isDefaultClientLocaleInAvailableLocales := false
2025-07-18 06:54:51 -04:00
for word := range strings . SplitSeq ( * cfg . LocalizationSettings . AvailableLocales , "," ) {
2019-02-12 13:19:01 -05:00
if _ , ok := locales [ word ] ; ! ok {
* cfg . LocalizationSettings . AvailableLocales = ""
isDefaultClientLocaleInAvailableLocales = true
mlog . Warn ( "AvailableLocales must include DefaultClientLocale. Setting AvailableLocales to all locales as default value." )
changed = true
break
}
if word == * cfg . LocalizationSettings . DefaultClientLocale {
isDefaultClientLocaleInAvailableLocales = true
}
}
availableLocales := * cfg . LocalizationSettings . AvailableLocales
if ! isDefaultClientLocaleInAvailableLocales {
availableLocales += "," + * cfg . LocalizationSettings . DefaultClientLocale
mlog . Warn ( "Adding DefaultClientLocale to AvailableLocales." )
changed = true
}
* cfg . LocalizationSettings . AvailableLocales = strings . Join ( utils . RemoveDuplicatesFromStringArray ( strings . Split ( availableLocales , "," ) ) , "," )
}
return changed
}
2019-03-21 15:46:38 -04:00
// Merge merges two configs together. The receiver's values are overwritten with the patch's
// values except when the patch's values are nil.
func Merge ( cfg * model . Config , patch * model . Config , mergeConfig * utils . MergeConfig ) ( * model . Config , error ) {
2023-08-21 05:29:30 -04:00
return utils . Merge ( cfg , patch , mergeConfig )
2019-03-21 15:46:38 -04:00
}
2019-09-26 01:17:39 -04:00
2020-12-16 14:45:17 -05:00
func IsDatabaseDSN ( dsn string ) bool {
Remove vestigial MySQL support (#34865)
* Remove legacy quoteColumnName() utility
Since Mattermost only supports PostgreSQL, the quoteColumnName() helper
that was designed to handle database-specific column quoting is no longer
needed. The function was a no-op that simply returned the column name
unchanged.
Remove the function from utils.go and update status_store.go to use
the "Manual" column name directly.
* Remove legacy driver checks from store.go
Since Mattermost only supports PostgreSQL, remove conditional checks
for different database drivers:
- Simplify specialSearchChars() to always return PostgreSQL-compatible chars
- Remove driver check from computeBinaryParam()
- Remove driver check from computeDefaultTextSearchConfig()
- Simplify GetDbVersion() to use PostgreSQL syntax directly
- Remove switch statement from ensureMinimumDBVersion()
- Remove unused driver parameter from versionString()
* Remove MySQL alternatives for batch delete operations
Since Mattermost only supports PostgreSQL, remove the MySQL-specific
DELETE...LIMIT syntax and keep only the PostgreSQL array-based approach:
- reaction_store.go: Use PostgreSQL array syntax for PermanentDeleteBatch
- file_info_store.go: Use PostgreSQL array syntax for PermanentDeleteBatch
- preference_store.go: Use PostgreSQL tuple IN subquery for DeleteInvalidVisibleDmsGms
* Remove MySQL alternatives for UPDATE...FROM syntax
Since Mattermost only supports PostgreSQL, remove the MySQL-specific
UPDATE syntax that joins tables differently:
- thread_store.go: Use PostgreSQL UPDATE...FROM syntax in
MarkAllAsReadByChannels and MarkAllAsReadByTeam
- post_store.go: Use PostgreSQL UPDATE...FROM syntax in deleteThreadFiles
* Remove MySQL alternatives for JSON and subquery operations
Since Mattermost only supports PostgreSQL, remove the MySQL-specific
JSON and subquery syntax:
- thread_store.go: Use PostgreSQL JSONB operators for updating participants
- access_control_policy_store.go: Use PostgreSQL JSONB @> operator for
querying JSON imports
- session_store.go: Use PostgreSQL subquery syntax for Cleanup
- job_store.go: Use PostgreSQL subquery syntax for Cleanup
* Remove MySQL alternatives for CTE queries
Since Mattermost only supports PostgreSQL, simplify code that
uses CTEs (Common Table Expressions):
- channel_store.go: Remove MySQL CASE-based fallback in
UpdateLastViewedAt and use PostgreSQL CTE exclusively
- draft_store.go: Remove driver checks in DeleteEmptyDraftsByCreateAtAndUserId,
DeleteOrphanDraftsByCreateAtAndUserId, and determineMaxDraftSize
* Remove driver checks in migrate.go and schema_dump.go
Simplify migration code to use PostgreSQL driver directly since
PostgreSQL is the only supported database.
* Remove driver checks in sqlx_wrapper.go
Always apply lowercase named parameter transformation since PostgreSQL
is the only supported database.
* Remove driver checks in user_store.go
Simplify user store functions to use PostgreSQL-only code paths:
- Remove isPostgreSQL parameter from helper functions
- Use LEFT JOIN pattern instead of subqueries for bot filtering
- Always use case-insensitive LIKE with lower() for search
- Remove MySQL-specific role filtering alternatives
* Remove driver checks in post_store.go
Simplify post_store.go to use PostgreSQL-only code paths:
- Inline getParentsPostsPostgreSQL into getParentsPosts
- Use PostgreSQL TO_CHAR/TO_TIMESTAMP for date formatting in analytics
- Use PostgreSQL array syntax for batch deletes
- Simplify determineMaxPostSize to always use information_schema
- Use PostgreSQL jsonb subtraction for thread participants
- Always execute RefreshPostStats (PostgreSQL materialized views)
- Use materialized views for AnalyticsPostCountsByDay
- Simplify AnalyticsPostCountByTeam to always use countByTeam
* Remove driver checks in channel_store.go
Simplify channel_store.go to use PostgreSQL-only code paths:
- Always use sq.Dollar.ReplacePlaceholders for UNION queries
- Use PostgreSQL LEFT JOIN for retention policy exclusion
- Use PostgreSQL jsonb @> operator for access control policy imports
- Simplify buildLIKEClause to always use LOWER() for case-insensitive search
- Simplify buildFulltextClauseX to always use PostgreSQL to_tsvector/to_tsquery
- Simplify searchGroupChannelsQuery to use ARRAY_TO_STRING/ARRAY_AGG
* Remove driver checks in file_info_store.go
Simplify file_info_store.go to use PostgreSQL-only code paths:
- Always use PostgreSQL to_tsvector/to_tsquery for file search
- Use file_stats materialized view for CountAll()
- Use file_stats materialized view for GetStorageUsage() when not including deleted
- Always execute RefreshFileStats() for materialized view refresh
* Remove driver checks in attributes_store.go
Simplify attributes_store.go to use PostgreSQL-only code paths:
- Always execute RefreshAttributes() for materialized view refresh
- Remove isPostgreSQL parameter from generateSearchQueryForExpression
- Always use PostgreSQL LOWER() LIKE LOWER() syntax for case-insensitive search
* Remove driver checks in retention_policy_store.go
Simplify retention_policy_store.go to use PostgreSQL-only code paths:
- Remove isPostgres parameter from scanRetentionIdsForDeletion
- Always use pq.Array for scanning retention IDs
- Always use pq.Array for inserting retention IDs
- Remove unused json import
* Remove driver checks in property stores
Simplify property_field_store.go and property_value_store.go to use
PostgreSQL-only code paths:
- Always use PostgreSQL type casts (::text, ::jsonb, ::bigint, etc.)
- Remove isPostgres variable and conditionals
* Remove driver checks in channel_member_history_store.go
Simplify PermanentDeleteBatch to use PostgreSQL-only code path:
- Always use ctid-based subquery for DELETE with LIMIT
* Remove remaining driver checks in user_store.go
Simplify user_store.go to use PostgreSQL-only code paths:
- Use LEFT JOIN for bot exclusion in AnalyticsActiveCountForPeriod
- Use LEFT JOIN for bot exclusion in IsEmpty
* Simplify fulltext search by consolidating buildFulltextClause functions
Remove convertMySQLFullTextColumnsToPostgres and consolidate
buildFulltextClause and buildFulltextClauseX into a single function
that takes variadic column arguments and returns sq.Sqlizer.
* Simplify SQL stores leveraging PostgreSQL-only support
- Simplify UpdateMembersRole in channel_store.go and team_store.go
to use UPDATE...RETURNING instead of SELECT + UPDATE
- Simplify GetPostReminders in post_store.go to use DELETE...RETURNING
- Simplify DeleteOrphanedRows queries by removing MySQL workarounds
for subquery locking issues
- Simplify UpdateUserLastSyncAt to use UPDATE...FROM...RETURNING
instead of fetching user first then updating
- Remove MySQL index hint workarounds in ORDER BY clauses
- Update outdated comments referencing MySQL
- Consolidate buildFulltextClause and remove convertMySQLFullTextColumnsToPostgres
* Remove MySQL-specific test artifacts
- Delete unused MySQLStopWords variable and stop_word.go file
- Remove redundant testSearchEmailAddressesWithQuotes test
(already covered by testSearchEmailAddresses)
- Update comment that referenced MySQL query planning
* Remove MySQL references from server code outside sqlstore
- Update config example and DSN parsing docs to reflect PostgreSQL-only support
- Remove mysql:// scheme check from IsDatabaseDSN
- Simplify SanitizeDataSource to only handle PostgreSQL
- Remove outdated MySQL comments from model and plugin code
* Remove MySQL references from test files
- Update test DSNs to use PostgreSQL format
- Remove dead mysql-replica flag and replicaFlag variable
- Simplify tests that had MySQL/PostgreSQL branches
* Update docs and test config to use PostgreSQL
- Update mmctl config set example to use postgres driver
- Update test-config.json to use PostgreSQL DSN format
* Remove MySQL migration scripts, test data, and docker image
Delete MySQL-related files that are no longer needed:
- ESR upgrade scripts (esr.*.mysql.*.sql)
- MySQL schema dumps (mattermost-mysql-*.sql)
- MySQL replication test scripts (replica-*.sh, mysql-migration-test.sh)
- MySQL test warmup data (mysql_migration_warmup.sql)
- MySQL docker image reference from mirror-docker-images.json
* Remove MySQL references from webapp
- Simplify minimumHashtagLength description to remove MySQL-specific configuration note
- Remove unused HIDE_MYSQL_STATS_NOTIFICATION preference constant
- Update en.json i18n source file
* clean up e2e-tests
* rm server/tests/template.load
* Use teamMemberSliceColumns() in UpdateMembersRole RETURNING clause
Refactor to use the existing helper function instead of hardcoding
the column names, ensuring consistency if the columns are updated.
* u.id -> u.Id
* address code review feedback
---------
Co-authored-by: Mattermost Build <build@mattermost.com>
2026-01-20 16:01:59 -05:00
return strings . HasPrefix ( dsn , "postgres://" ) ||
2022-01-12 07:23:56 -05:00
strings . HasPrefix ( dsn , "postgresql://" )
2020-12-16 14:45:17 -05:00
}
2023-05-15 10:37:48 -04:00
func isJSONMap ( data [ ] byte ) bool {
2022-07-05 02:46:50 -04:00
var m map [ string ] any
2023-05-15 10:37:48 -04:00
err := json . Unmarshal ( data , & m )
return err == nil
2020-07-15 14:40:36 -04:00
}
2022-07-05 02:46:50 -04:00
func GetValueByPath ( path [ ] string , obj any ) ( any , bool ) {
2020-09-21 03:28:46 -04:00
r := reflect . ValueOf ( obj )
var val reflect . Value
if r . Kind ( ) == reflect . Map {
val = r . MapIndex ( reflect . ValueOf ( path [ 0 ] ) )
if val . IsValid ( ) {
val = val . Elem ( )
}
} else {
val = r . FieldByName ( path [ 0 ] )
}
if ! val . IsValid ( ) {
return nil , false
}
switch {
case len ( path ) == 1 :
return val . Interface ( ) , true
case val . Kind ( ) == reflect . Struct :
return GetValueByPath ( path [ 1 : ] , val . Interface ( ) )
case val . Kind ( ) == reflect . Map :
remainingPath := strings . Join ( path [ 1 : ] , "." )
mapIter := val . MapRange ( )
for mapIter . Next ( ) {
key := mapIter . Key ( ) . String ( )
if strings . HasPrefix ( remainingPath , key ) {
i := strings . Count ( key , "." ) + 2 // number of dots + a dot on each side
mapVal := mapIter . Value ( )
// if no sub field path specified, return the object
if len ( path [ i : ] ) == 0 {
return mapVal . Interface ( ) , true
}
data := mapVal . Interface ( )
2026-05-12 11:59:12 -04:00
if mapVal . Kind ( ) == reflect . Pointer {
2020-09-21 03:28:46 -04:00
data = mapVal . Elem ( ) . Interface ( ) // if value is a pointer, dereference it
}
// pass subpath
return GetValueByPath ( path [ i : ] , data )
}
}
}
return nil , false
}
2021-05-19 07:30:26 -04:00
func equal ( oldCfg , newCfg * model . Config ) ( bool , error ) {
oldCfgBytes , err := json . Marshal ( oldCfg )
if err != nil {
return false , fmt . Errorf ( "failed to marshal old config: %w" , err )
}
newCfgBytes , err := json . Marshal ( newCfg )
if err != nil {
return false , fmt . Errorf ( "failed to marshal new config: %w" , err )
}
return ! bytes . Equal ( oldCfgBytes , newCfgBytes ) , nil
}