From 956e21cfa24dc41fe5e07ef6ad548707beb36e91 Mon Sep 17 00:00:00 2001 From: Scott Bishel Date: Thu, 20 Jan 2022 17:46:03 -0700 Subject: [PATCH] Boards data retention (#19262) * add Boards to DataRetention, add hook for data retention * remove replaces * update hook to remove parameter * add boards data retention to telemetry * fix unit test * update test, update hooks * update RunDataRetention server version * put behind a feature flag Co-authored-by: Mattermod --- app/plugin_hooks_test.go | 44 +++++++++++++++++++++++++++ config/client.go | 4 +++ jobs/workers.go | 4 +-- model/config.go | 11 +++++++ model/data_retention_policy.go | 2 ++ model/feature_flags.go | 4 +++ plugin/client_rpc_generated.go | 37 ++++++++++++++++++++++ plugin/hooks.go | 6 ++++ plugin/hooks_timer_layer_generated.go | 7 +++++ plugin/plugintest/hooks.go | 21 +++++++++++++ services/telemetry/telemetry.go | 2 ++ 11 files changed, 140 insertions(+), 2 deletions(-) diff --git a/app/plugin_hooks_test.go b/app/plugin_hooks_test.go index 581d583aa8d..f2518226858 100644 --- a/app/plugin_hooks_test.go +++ b/app/plugin_hooks_test.go @@ -1215,3 +1215,47 @@ func TestHookReactionHasBeenRemoved(t *testing.T) { require.Nil(t, err) } + +func TestHookRunDataRetention(t *testing.T) { + th := Setup(t).InitBasic() + defer th.TearDown() + + tearDown, pluginIDs, _ := SetAppEnvironmentWithPlugins(t, + []string{ + ` + package main + + import ( + "github.com/mattermost/mattermost-server/v6/plugin" + ) + + type MyPlugin struct { + plugin.MattermostPlugin + } + + func (p *MyPlugin) RunDataRetention(nowMillis, batchSize int64) (int64, error){ + return 100, nil + } + + func main() { + plugin.ClientMain(&MyPlugin{}) + } + `}, th.App, th.NewPluginAPI) + defer tearDown() + + require.Len(t, pluginIDs, 1) + pluginID := pluginIDs[0] + + require.True(t, th.App.GetPluginsEnvironment().IsActive(pluginID)) + + hookCalled := false + th.App.GetPluginsEnvironment().RunMultiPluginHook(func(hooks plugin.Hooks) bool { + n, _ := hooks.RunDataRetention(0, 0) + // Ensure return it correct + assert.Equal(t, int64(100), n) + hookCalled = true + return hookCalled + }, plugin.RunDataRetentionID) + + require.True(t, hookCalled) +} diff --git a/config/client.go b/config/client.go index 9c7cdfd5866..a4877670161 100644 --- a/config/client.go +++ b/config/client.go @@ -126,6 +126,8 @@ func GenerateClientConfig(c *model.Config, telemetryID string, license *model.Li props["DataRetentionMessageRetentionDays"] = "0" props["DataRetentionEnableFileDeletion"] = "false" props["DataRetentionFileRetentionDays"] = "0" + props["DataRetentionEnableBoardsDeletion"] = "false" + props["DataRetentionBoardsRetentionDays"] = "0" props["CWSURL"] = "" props["CustomUrlSchemes"] = strings.Join(c.DisplaySettings.CustomURLSchemes, ",") @@ -190,6 +192,8 @@ func GenerateClientConfig(c *model.Config, telemetryID string, license *model.Li props["DataRetentionMessageRetentionDays"] = strconv.FormatInt(int64(*c.DataRetentionSettings.MessageRetentionDays), 10) props["DataRetentionEnableFileDeletion"] = strconv.FormatBool(*c.DataRetentionSettings.EnableFileDeletion) props["DataRetentionFileRetentionDays"] = strconv.FormatInt(int64(*c.DataRetentionSettings.FileRetentionDays), 10) + props["DataRetentionEnableBoardsDeletion"] = strconv.FormatBool(*c.DataRetentionSettings.EnableBoardsDeletion) + props["DataRetentionBoardsRetentionDays"] = strconv.FormatInt(int64(*c.DataRetentionSettings.BoardsRetentionDays), 10) } if *license.Features.Cloud { diff --git a/jobs/workers.go b/jobs/workers.go index bb34ed4b972..ec2b94f71f4 100644 --- a/jobs/workers.go +++ b/jobs/workers.go @@ -221,9 +221,9 @@ func (workers *Workers) handleConfigChange(oldConfig *model.Config, newConfig *m mlog.Debug("Workers received config change.") if workers.DataRetention != nil { - if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion) { + if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion && !*oldConfig.DataRetentionSettings.EnableBoardsDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion || *newConfig.DataRetentionSettings.EnableBoardsDeletion) { go workers.DataRetention.Run() - } else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion) { + } else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion || *oldConfig.DataRetentionSettings.EnableBoardsDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion && !*newConfig.DataRetentionSettings.EnableBoardsDeletion) { workers.DataRetention.Stop() } } diff --git a/model/config.go b/model/config.go index 54ed41c68f2..22efd28997f 100644 --- a/model/config.go +++ b/model/config.go @@ -202,6 +202,7 @@ const ( DataRetentionSettingsDefaultMessageRetentionDays = 365 DataRetentionSettingsDefaultFileRetentionDays = 365 + DataRetentionSettingsDefaultBoardsRetentionDays = 365 DataRetentionSettingsDefaultDeletionJobStartTime = "02:00" DataRetentionSettingsDefaultBatchSize = 3000 @@ -2586,8 +2587,10 @@ func (bs *BleveSettings) SetDefaults() { type DataRetentionSettings struct { EnableMessageDeletion *bool `access:"compliance_data_retention_policy"` EnableFileDeletion *bool `access:"compliance_data_retention_policy"` + EnableBoardsDeletion *bool `access:"compliance_data_retention_policy"` MessageRetentionDays *int `access:"compliance_data_retention_policy"` FileRetentionDays *int `access:"compliance_data_retention_policy"` + BoardsRetentionDays *int `access:"compliance_data_retention_policy"` DeletionJobStartTime *string `access:"compliance_data_retention_policy"` BatchSize *int `access:"compliance_data_retention_policy"` } @@ -2601,6 +2604,10 @@ func (s *DataRetentionSettings) SetDefaults() { s.EnableFileDeletion = NewBool(false) } + if s.EnableBoardsDeletion == nil { + s.EnableBoardsDeletion = NewBool(false) + } + if s.MessageRetentionDays == nil { s.MessageRetentionDays = NewInt(DataRetentionSettingsDefaultMessageRetentionDays) } @@ -2609,6 +2616,10 @@ func (s *DataRetentionSettings) SetDefaults() { s.FileRetentionDays = NewInt(DataRetentionSettingsDefaultFileRetentionDays) } + if s.BoardsRetentionDays == nil { + s.BoardsRetentionDays = NewInt(DataRetentionSettingsDefaultBoardsRetentionDays) + } + if s.DeletionJobStartTime == nil { s.DeletionJobStartTime = NewString(DataRetentionSettingsDefaultDeletionJobStartTime) } diff --git a/model/data_retention_policy.go b/model/data_retention_policy.go index 0a1d318e6e9..3210251700c 100644 --- a/model/data_retention_policy.go +++ b/model/data_retention_policy.go @@ -6,8 +6,10 @@ package model type GlobalRetentionPolicy struct { MessageDeletionEnabled bool `json:"message_deletion_enabled"` FileDeletionEnabled bool `json:"file_deletion_enabled"` + BoardsDeletionEnabled bool `json:"boards_deletion_enabled"` MessageRetentionCutoff int64 `json:"message_retention_cutoff"` FileRetentionCutoff int64 `json:"file_retention_cutoff"` + BoardsRetentionCutoff int64 `json:"boards_retention_cutoff"` } type RetentionPolicy struct { diff --git a/model/feature_flags.go b/model/feature_flags.go index 37f044ea9ee..6998a03af79 100644 --- a/model/feature_flags.go +++ b/model/feature_flags.go @@ -71,6 +71,9 @@ type FeatureFlags struct { // Enable inline post editing InlinePostEditing bool + // Enable DataRetention for Boards + BoardsDataRetention bool + NormalizeLdapDNs bool } @@ -96,6 +99,7 @@ func (f *FeatureFlags) SetDefaults() { f.ResendInviteEmailInterval = "" f.InviteToTeam = "none" f.InlinePostEditing = false + f.BoardsDataRetention = false f.NormalizeLdapDNs = false } diff --git a/plugin/client_rpc_generated.go b/plugin/client_rpc_generated.go index 7c05e90c850..11382384a7d 100644 --- a/plugin/client_rpc_generated.go +++ b/plugin/client_rpc_generated.go @@ -669,6 +669,43 @@ func (s *hooksRPCServer) WebSocketMessageHasBeenPosted(args *Z_WebSocketMessageH return nil } +func init() { + hookNameToId["RunDataRetention"] = RunDataRetentionID +} + +type Z_RunDataRetentionArgs struct { + A int64 + B int64 +} + +type Z_RunDataRetentionReturns struct { + A int64 + B error +} + +func (g *hooksRPCClient) RunDataRetention(nowTime, batchSize int64) (int64, error) { + _args := &Z_RunDataRetentionArgs{nowTime, batchSize} + _returns := &Z_RunDataRetentionReturns{} + if g.implemented[RunDataRetentionID] { + if err := g.client.Call("Plugin.RunDataRetention", _args, _returns); err != nil { + g.log.Error("RPC call RunDataRetention to plugin failed.", mlog.Err(err)) + } + } + return _returns.A, _returns.B +} + +func (s *hooksRPCServer) RunDataRetention(args *Z_RunDataRetentionArgs, returns *Z_RunDataRetentionReturns) error { + if hook, ok := s.impl.(interface { + RunDataRetention(nowTime, batchSize int64) (int64, error) + }); ok { + returns.A, returns.B = hook.RunDataRetention(args.A, args.B) + returns.B = encodableError(returns.B) + } else { + return encodableError(fmt.Errorf("Hook RunDataRetention called but not implemented.")) + } + return nil +} + type Z_RegisterCommandArgs struct { A *model.Command } diff --git a/plugin/hooks.go b/plugin/hooks.go index 1a3fdd4ff7b..ff646c9248d 100644 --- a/plugin/hooks.go +++ b/plugin/hooks.go @@ -39,6 +39,7 @@ const ( OnWebSocketConnectID = 21 OnWebSocketDisconnectID = 22 WebSocketMessageHasBeenPostedID = 23 + RunDataRetentionID = 24 TotalHooksID = iota ) @@ -243,4 +244,9 @@ type Hooks interface { // // Minimum server version: 6.0 WebSocketMessageHasBeenPosted(webConnID, userID string, req *model.WebSocketRequest) + + // RunDataRetention is invoked during a DataRetentionJob + // + // Minimum server version: 6.4 + RunDataRetention(nowTime, batchSize int64) (int64, error) } diff --git a/plugin/hooks_timer_layer_generated.go b/plugin/hooks_timer_layer_generated.go index 1d757d65c94..1567cff7a0f 100644 --- a/plugin/hooks_timer_layer_generated.go +++ b/plugin/hooks_timer_layer_generated.go @@ -186,3 +186,10 @@ func (hooks *hooksTimerLayer) WebSocketMessageHasBeenPosted(webConnID, userID st hooks.hooksImpl.WebSocketMessageHasBeenPosted(webConnID, userID, req) hooks.recordTime(startTime, "WebSocketMessageHasBeenPosted", true) } + +func (hooks *hooksTimerLayer) RunDataRetention(nowTime, batchSize int64) (int64, error) { + startTime := timePkg.Now() + _returnsA, _returnsB := hooks.hooksImpl.RunDataRetention(nowTime, batchSize) + hooks.recordTime(startTime, "RunDataRetention", _returnsB == nil) + return _returnsA, _returnsB +} diff --git a/plugin/plugintest/hooks.go b/plugin/plugintest/hooks.go index daab4a0aa84..e2ceb616f7e 100644 --- a/plugin/plugintest/hooks.go +++ b/plugin/plugintest/hooks.go @@ -219,6 +219,27 @@ func (_m *Hooks) ReactionHasBeenRemoved(c *plugin.Context, reaction *model.React _m.Called(c, reaction) } +// RunDataRetention provides a mock function with given fields: nowTime, batchSize +func (_m *Hooks) RunDataRetention(nowTime int64, batchSize int64) (int64, error) { + ret := _m.Called(nowTime, batchSize) + + var r0 int64 + if rf, ok := ret.Get(0).(func(int64, int64) int64); ok { + r0 = rf(nowTime, batchSize) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(int64, int64) error); ok { + r1 = rf(nowTime, batchSize) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // ServeHTTP provides a mock function with given fields: c, w, r func (_m *Hooks) ServeHTTP(c *plugin.Context, w http.ResponseWriter, r *http.Request) { _m.Called(c, w, r) diff --git a/services/telemetry/telemetry.go b/services/telemetry/telemetry.go index ba5dfbbeb9b..dad61b6dea7 100644 --- a/services/telemetry/telemetry.go +++ b/services/telemetry/telemetry.go @@ -758,8 +758,10 @@ func (ts *TelemetryService) trackConfig() { ts.SendTelemetry(TrackConfigDataRetention, map[string]interface{}{ "enable_message_deletion": *cfg.DataRetentionSettings.EnableMessageDeletion, "enable_file_deletion": *cfg.DataRetentionSettings.EnableFileDeletion, + "enable_boards_deletion": *cfg.DataRetentionSettings.EnableBoardsDeletion, "message_retention_days": *cfg.DataRetentionSettings.MessageRetentionDays, "file_retention_days": *cfg.DataRetentionSettings.FileRetentionDays, + "boards_retention_days": *cfg.DataRetentionSettings.BoardsRetentionDays, "deletion_job_start_time": *cfg.DataRetentionSettings.DeletionJobStartTime, "batch_size": *cfg.DataRetentionSettings.BatchSize, "cleanup_jobs_threshold_days": *cfg.JobSettings.CleanupJobsThresholdDays,