upstream/mattermost
1
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2026-05-28 04:35:04 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Boards data retention (#19262)

Browse source 
* add Boards to DataRetention, add hook for data retention

* remove replaces

* update hook to remove parameter

* add boards data retention to telemetry

* fix unit test

* update test, update hooks

* update RunDataRetention server version

* put behind a feature flag

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
This commit is contained in:
Scott Bishel 2022-01-20 17:46:03 -07:00 committed by GitHub
parent 294bd44971
commit 956e21cfa2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 140 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
app/plugin_hooks_test.go
View file

@ -1215,3 +1215,47 @@ func TestHookReactionHasBeenRemoved(t *testing.T) {
require.Nil(t, err)
}
func TestHookRunDataRetention(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
tearDown, pluginIDs, _ := SetAppEnvironmentWithPlugins(t,
[]string{
`
package main
import (
"github.com/mattermost/mattermost-server/v6/plugin"
)
type MyPlugin struct {
plugin.MattermostPlugin
}
func (p *MyPlugin) RunDataRetention(nowMillis, batchSize int64) (int64, error){
return 100, nil
}
func main() {
plugin.ClientMain(&MyPlugin{})
}
`}, th.App, th.NewPluginAPI)
defer tearDown()
require.Len(t, pluginIDs, 1)
pluginID := pluginIDs[0]
require.True(t, th.App.GetPluginsEnvironment().IsActive(pluginID))
hookCalled := false
th.App.GetPluginsEnvironment().RunMultiPluginHook(func(hooks plugin.Hooks) bool {
n, _ := hooks.RunDataRetention(0, 0)
// Ensure return it correct
assert.Equal(t, int64(100), n)
hookCalled = true
return hookCalled
}, plugin.RunDataRetentionID)
require.True(t, hookCalled)
}

4
config/client.go
View file

@ -126,6 +126,8 @@ func GenerateClientConfig(c *model.Config, telemetryID string, license *model.Li
props["DataRetentionMessageRetentionDays"] = "0"
props["DataRetentionEnableFileDeletion"] = "false"
props["DataRetentionFileRetentionDays"] = "0"
props["DataRetentionEnableBoardsDeletion"] = "false"
props["DataRetentionBoardsRetentionDays"] = "0"
props["CWSURL"] = ""
props["CustomUrlSchemes"] = strings.Join(c.DisplaySettings.CustomURLSchemes, ",")
@ -190,6 +192,8 @@ func GenerateClientConfig(c *model.Config, telemetryID string, license *model.Li
props["DataRetentionMessageRetentionDays"] = strconv.FormatInt(int64(*c.DataRetentionSettings.MessageRetentionDays), 10)
props["DataRetentionEnableFileDeletion"] = strconv.FormatBool(*c.DataRetentionSettings.EnableFileDeletion)
props["DataRetentionFileRetentionDays"] = strconv.FormatInt(int64(*c.DataRetentionSettings.FileRetentionDays), 10)
props["DataRetentionEnableBoardsDeletion"] = strconv.FormatBool(*c.DataRetentionSettings.EnableBoardsDeletion)
props["DataRetentionBoardsRetentionDays"] = strconv.FormatInt(int64(*c.DataRetentionSettings.BoardsRetentionDays), 10)
}
if *license.Features.Cloud {

4
jobs/workers.go
View file

@ -221,9 +221,9 @@ func (workers *Workers) handleConfigChange(oldConfig *model.Config, newConfig *m
mlog.Debug("Workers received config change.")
if workers.DataRetention != nil {
if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion) {
if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion && !*oldConfig.DataRetentionSettings.EnableBoardsDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion || *newConfig.DataRetentionSettings.EnableBoardsDeletion) {
go workers.DataRetention.Run()
} else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion) {
} else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion || *oldConfig.DataRetentionSettings.EnableBoardsDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion && !*newConfig.DataRetentionSettings.EnableBoardsDeletion) {
workers.DataRetention.Stop()
}
}

11
model/config.go
View file

@ -202,6 +202,7 @@ const (
DataRetentionSettingsDefaultMessageRetentionDays = 365
DataRetentionSettingsDefaultFileRetentionDays = 365
DataRetentionSettingsDefaultBoardsRetentionDays = 365
DataRetentionSettingsDefaultDeletionJobStartTime = "02:00"
DataRetentionSettingsDefaultBatchSize = 3000
@ -2586,8 +2587,10 @@ func (bs *BleveSettings) SetDefaults() {
type DataRetentionSettings struct {
EnableMessageDeletion *bool `access:"compliance_data_retention_policy"`
EnableFileDeletion *bool `access:"compliance_data_retention_policy"`
EnableBoardsDeletion *bool `access:"compliance_data_retention_policy"`
MessageRetentionDays *int `access:"compliance_data_retention_policy"`
FileRetentionDays *int `access:"compliance_data_retention_policy"`
BoardsRetentionDays *int `access:"compliance_data_retention_policy"`
DeletionJobStartTime *string `access:"compliance_data_retention_policy"`
BatchSize *int `access:"compliance_data_retention_policy"`
}
@ -2601,6 +2604,10 @@ func (s *DataRetentionSettings) SetDefaults() {
s.EnableFileDeletion = NewBool(false)
}
if s.EnableBoardsDeletion == nil {
s.EnableBoardsDeletion = NewBool(false)
}
if s.MessageRetentionDays == nil {
s.MessageRetentionDays = NewInt(DataRetentionSettingsDefaultMessageRetentionDays)
}
@ -2609,6 +2616,10 @@ func (s *DataRetentionSettings) SetDefaults() {
s.FileRetentionDays = NewInt(DataRetentionSettingsDefaultFileRetentionDays)
}
if s.BoardsRetentionDays == nil {
s.BoardsRetentionDays = NewInt(DataRetentionSettingsDefaultBoardsRetentionDays)
}
if s.DeletionJobStartTime == nil {
s.DeletionJobStartTime = NewString(DataRetentionSettingsDefaultDeletionJobStartTime)
}

2
model/data_retention_policy.go
View file

@ -6,8 +6,10 @@ package model
type GlobalRetentionPolicy struct {
MessageDeletionEnabled bool `json:"message_deletion_enabled"`
FileDeletionEnabled bool `json:"file_deletion_enabled"`
BoardsDeletionEnabled bool `json:"boards_deletion_enabled"`
MessageRetentionCutoff int64 `json:"message_retention_cutoff"`
FileRetentionCutoff int64 `json:"file_retention_cutoff"`
BoardsRetentionCutoff int64 `json:"boards_retention_cutoff"`
}
type RetentionPolicy struct {

4
model/feature_flags.go
View file

@ -71,6 +71,9 @@ type FeatureFlags struct {
// Enable inline post editing
InlinePostEditing bool
// Enable DataRetention for Boards
BoardsDataRetention bool
NormalizeLdapDNs bool
}
@ -96,6 +99,7 @@ func (f *FeatureFlags) SetDefaults() {
f.ResendInviteEmailInterval = ""
f.InviteToTeam = "none"
f.InlinePostEditing = false
f.BoardsDataRetention = false
f.NormalizeLdapDNs = false
}

37
plugin/client_rpc_generated.go
View file

@ -669,6 +669,43 @@ func (s *hooksRPCServer) WebSocketMessageHasBeenPosted(args *Z_WebSocketMessageH
return nil
}
func init() {
hookNameToId["RunDataRetention"] = RunDataRetentionID
}
type Z_RunDataRetentionArgs struct {
A int64
B int64
}
type Z_RunDataRetentionReturns struct {
A int64
B error
}
func (g *hooksRPCClient) RunDataRetention(nowTime, batchSize int64) (int64, error) {
_args := &Z_RunDataRetentionArgs{nowTime, batchSize}
_returns := &Z_RunDataRetentionReturns{}
if g.implemented[RunDataRetentionID] {
if err := g.client.Call("Plugin.RunDataRetention", _args, _returns); err != nil {
g.log.Error("RPC call RunDataRetention to plugin failed.", mlog.Err(err))
}
}
return _returns.A, _returns.B
}
func (s *hooksRPCServer) RunDataRetention(args *Z_RunDataRetentionArgs, returns *Z_RunDataRetentionReturns) error {
if hook, ok := s.impl.(interface {
RunDataRetention(nowTime, batchSize int64) (int64, error)
}); ok {
returns.A, returns.B = hook.RunDataRetention(args.A, args.B)
returns.B = encodableError(returns.B)
} else {
return encodableError(fmt.Errorf("Hook RunDataRetention called but not implemented."))
}
return nil
}
type Z_RegisterCommandArgs struct {
A *model.Command
}

6
plugin/hooks.go
View file

@ -39,6 +39,7 @@ const (
OnWebSocketConnectID = 21
OnWebSocketDisconnectID = 22
WebSocketMessageHasBeenPostedID = 23
RunDataRetentionID = 24
TotalHooksID = iota
)
@ -243,4 +244,9 @@ type Hooks interface {
//
// Minimum server version: 6.0
WebSocketMessageHasBeenPosted(webConnID, userID string, req *model.WebSocketRequest)
// RunDataRetention is invoked during a DataRetentionJob
//
// Minimum server version: 6.4
RunDataRetention(nowTime, batchSize int64) (int64, error)
}

7
plugin/hooks_timer_layer_generated.go
View file

@ -186,3 +186,10 @@ func (hooks *hooksTimerLayer) WebSocketMessageHasBeenPosted(webConnID, userID st
hooks.hooksImpl.WebSocketMessageHasBeenPosted(webConnID, userID, req)
hooks.recordTime(startTime, "WebSocketMessageHasBeenPosted", true)
}
func (hooks *hooksTimerLayer) RunDataRetention(nowTime, batchSize int64) (int64, error) {
startTime := timePkg.Now()
_returnsA, _returnsB := hooks.hooksImpl.RunDataRetention(nowTime, batchSize)
hooks.recordTime(startTime, "RunDataRetention", _returnsB == nil)
return _returnsA, _returnsB
}

21
plugin/plugintest/hooks.go
View file

@ -219,6 +219,27 @@ func (_m *Hooks) ReactionHasBeenRemoved(c *plugin.Context, reaction *model.React
_m.Called(c, reaction)
}
// RunDataRetention provides a mock function with given fields: nowTime, batchSize
func (_m *Hooks) RunDataRetention(nowTime int64, batchSize int64) (int64, error) {
ret := _m.Called(nowTime, batchSize)
var r0 int64
if rf, ok := ret.Get(0).(func(int64, int64) int64); ok {
r0 = rf(nowTime, batchSize)
} else {
r0 = ret.Get(0).(int64)
}
var r1 error
if rf, ok := ret.Get(1).(func(int64, int64) error); ok {
r1 = rf(nowTime, batchSize)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// ServeHTTP provides a mock function with given fields: c, w, r
func (_m *Hooks) ServeHTTP(c *plugin.Context, w http.ResponseWriter, r *http.Request) {
_m.Called(c, w, r)

2
services/telemetry/telemetry.go
View file

@ -758,8 +758,10 @@ func (ts *TelemetryService) trackConfig() {
ts.SendTelemetry(TrackConfigDataRetention, map[string]interface{}{
"enable_message_deletion": *cfg.DataRetentionSettings.EnableMessageDeletion,
"enable_file_deletion": *cfg.DataRetentionSettings.EnableFileDeletion,
"enable_boards_deletion": *cfg.DataRetentionSettings.EnableBoardsDeletion,
"message_retention_days": *cfg.DataRetentionSettings.MessageRetentionDays,
"file_retention_days": *cfg.DataRetentionSettings.FileRetentionDays,
"boards_retention_days": *cfg.DataRetentionSettings.BoardsRetentionDays,
"deletion_job_start_time": *cfg.DataRetentionSettings.DeletionJobStartTime,
"batch_size": *cfg.DataRetentionSettings.BatchSize,
"cleanup_jobs_threshold_days": *cfg.JobSettings.CleanupJobsThresholdDays,