From bf0e0ed45a35c82de1fa00cbdc99798f461b0b57 Mon Sep 17 00:00:00 2001 From: Ben Cooke Date: Wed, 5 Jul 2023 11:09:03 -0400 Subject: [PATCH] [MM-53466] Remove concurrency from data retention delete batching (#23927) * Add TimeBetweenBatches config value for data retention * adding feature flag for data retention concurrency --------- Co-authored-by: Julien Tant Co-authored-by: Mattermost Build --- .../platform/services/telemetry/telemetry.go | 1 + server/public/model/config.go | 32 +++++++++++-------- server/public/model/feature_flags.go | 3 ++ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/server/platform/services/telemetry/telemetry.go b/server/platform/services/telemetry/telemetry.go index b609f7892a0..ad5b7d0bebc 100644 --- a/server/platform/services/telemetry/telemetry.go +++ b/server/platform/services/telemetry/telemetry.go @@ -835,6 +835,7 @@ func (ts *TelemetryService) trackConfig() { "boards_retention_days": *cfg.DataRetentionSettings.BoardsRetentionDays, "deletion_job_start_time": *cfg.DataRetentionSettings.DeletionJobStartTime, "batch_size": *cfg.DataRetentionSettings.BatchSize, + "time_between_batches": *cfg.DataRetentionSettings.TimeBetweenBatchesMilliseconds, "cleanup_jobs_threshold_days": *cfg.JobSettings.CleanupJobsThresholdDays, "cleanup_config_threshold_days": *cfg.JobSettings.CleanupConfigThresholdDays, }) diff --git a/server/public/model/config.go b/server/public/model/config.go index d663826574c..cdb9ab8c645 100644 --- a/server/public/model/config.go +++ b/server/public/model/config.go @@ -205,11 +205,12 @@ const ( BleveSettingsDefaultIndexDir = "" BleveSettingsDefaultBatchSize = 10000 - DataRetentionSettingsDefaultMessageRetentionDays = 365 - DataRetentionSettingsDefaultFileRetentionDays = 365 - DataRetentionSettingsDefaultBoardsRetentionDays = 365 - DataRetentionSettingsDefaultDeletionJobStartTime = "02:00" - DataRetentionSettingsDefaultBatchSize = 3000 + DataRetentionSettingsDefaultMessageRetentionDays = 365 + DataRetentionSettingsDefaultFileRetentionDays = 365 + DataRetentionSettingsDefaultBoardsRetentionDays = 365 + DataRetentionSettingsDefaultDeletionJobStartTime = "02:00" + DataRetentionSettingsDefaultBatchSize = 3000 + DataRetentionSettingsDefaultTimeBetweenBatchesMilliseconds = 100 PluginSettingsDefaultDirectory = "./plugins" PluginSettingsDefaultClientDirectory = "./client/plugins" @@ -2765,14 +2766,15 @@ func (bs *BleveSettings) SetDefaults() { } type DataRetentionSettings struct { - EnableMessageDeletion *bool `access:"compliance_data_retention_policy"` - EnableFileDeletion *bool `access:"compliance_data_retention_policy"` - EnableBoardsDeletion *bool `access:"compliance_data_retention_policy"` - MessageRetentionDays *int `access:"compliance_data_retention_policy"` - FileRetentionDays *int `access:"compliance_data_retention_policy"` - BoardsRetentionDays *int `access:"compliance_data_retention_policy"` - DeletionJobStartTime *string `access:"compliance_data_retention_policy"` - BatchSize *int `access:"compliance_data_retention_policy"` + EnableMessageDeletion *bool `access:"compliance_data_retention_policy"` + EnableFileDeletion *bool `access:"compliance_data_retention_policy"` + EnableBoardsDeletion *bool `access:"compliance_data_retention_policy"` + MessageRetentionDays *int `access:"compliance_data_retention_policy"` + FileRetentionDays *int `access:"compliance_data_retention_policy"` + BoardsRetentionDays *int `access:"compliance_data_retention_policy"` + DeletionJobStartTime *string `access:"compliance_data_retention_policy"` + BatchSize *int `access:"compliance_data_retention_policy"` + TimeBetweenBatchesMilliseconds *int `access:"compliance_data_retention_policy"` } func (s *DataRetentionSettings) SetDefaults() { @@ -2807,6 +2809,10 @@ func (s *DataRetentionSettings) SetDefaults() { if s.BatchSize == nil { s.BatchSize = NewInt(DataRetentionSettingsDefaultBatchSize) } + + if s.TimeBetweenBatchesMilliseconds == nil { + s.TimeBetweenBatchesMilliseconds = NewInt(DataRetentionSettingsDefaultTimeBetweenBatchesMilliseconds) + } } type JobSettings struct { diff --git a/server/public/model/feature_flags.go b/server/public/model/feature_flags.go index ee50c2524eb..9db09788d92 100644 --- a/server/public/model/feature_flags.go +++ b/server/public/model/feature_flags.go @@ -62,6 +62,8 @@ type FeatureFlags struct { DeprecateCloudFree bool CloudReverseTrial bool + + DataRetentionConcurrencyEnabled bool } func (f *FeatureFlags) SetDefaults() { @@ -85,6 +87,7 @@ func (f *FeatureFlags) SetDefaults() { f.WysiwygEditor = false f.OnboardingTourTips = true f.CloudReverseTrial = false + f.DataRetentionConcurrencyEnabled = true } func (f *FeatureFlags) Plugins() map[string]string {