diff --git a/server/channels/app/post.go b/server/channels/app/post.go index caee12dc5db..ba3243ee554 100644 --- a/server/channels/app/post.go +++ b/server/channels/app/post.go @@ -1207,7 +1207,17 @@ func (a *App) GetPosts(rctx request.CTX, channelID string, offset int, limit int } func (a *App) GetPostsEtag(channelID string, collapsedThreads bool) string { - return a.Srv().Store().Post().GetEtag(channelID, true, collapsedThreads) + if a.AutoTranslation() == nil || !a.AutoTranslation().IsFeatureAvailable() { + return a.Srv().Store().Post().GetEtag(channelID, true, collapsedThreads, false) + } + + channelEnabled, err := a.AutoTranslation().IsChannelEnabled(channelID) + if err != nil || !channelEnabled { + return a.Srv().Store().Post().GetEtag(channelID, true, collapsedThreads, false) + } + + // Channel has auto-translation enabled - include translation etag + return a.Srv().Store().Post().GetEtag(channelID, true, collapsedThreads, true) } func (a *App) GetPostsSince(rctx request.CTX, options model.GetPostsSinceOptions) (*model.PostList, *model.AppError) { diff --git a/server/channels/db/migrations/migrations.list b/server/channels/db/migrations/migrations.list index b98968bda45..66fdb3db6e5 100644 --- a/server/channels/db/migrations/migrations.list +++ b/server/channels/db/migrations/migrations.list @@ -301,3 +301,9 @@ channels/db/migrations/postgres/000151_add_autotranslationdisabled_to_channelmem channels/db/migrations/postgres/000151_add_autotranslationdisabled_to_channelmembers.up.sql channels/db/migrations/postgres/000152_translations_primary_key_change.down.sql channels/db/migrations/postgres/000152_translations_primary_key_change.up.sql +channels/db/migrations/postgres/000153_add_translation_channel_id.down.sql +channels/db/migrations/postgres/000153_add_translation_channel_id.up.sql +channels/db/migrations/postgres/000154_drop_translation_updateat_index.down.sql +channels/db/migrations/postgres/000154_drop_translation_updateat_index.up.sql +channels/db/migrations/postgres/000155_create_translation_channel_updateat_index.down.sql +channels/db/migrations/postgres/000155_create_translation_channel_updateat_index.up.sql diff --git a/server/channels/db/migrations/postgres/000153_add_translation_channel_id.down.sql b/server/channels/db/migrations/postgres/000153_add_translation_channel_id.down.sql new file mode 100644 index 00000000000..77d45ea05f5 --- /dev/null +++ b/server/channels/db/migrations/postgres/000153_add_translation_channel_id.down.sql @@ -0,0 +1 @@ +ALTER TABLE translations DROP COLUMN IF EXISTS channelid; diff --git a/server/channels/db/migrations/postgres/000153_add_translation_channel_id.up.sql b/server/channels/db/migrations/postgres/000153_add_translation_channel_id.up.sql new file mode 100644 index 00000000000..2e1ec16e4f4 --- /dev/null +++ b/server/channels/db/migrations/postgres/000153_add_translation_channel_id.up.sql @@ -0,0 +1 @@ +ALTER TABLE translations ADD COLUMN IF NOT EXISTS channelid varchar(26); diff --git a/server/channels/db/migrations/postgres/000154_drop_translation_updateat_index.down.sql b/server/channels/db/migrations/postgres/000154_drop_translation_updateat_index.down.sql new file mode 100644 index 00000000000..2c37d175781 --- /dev/null +++ b/server/channels/db/migrations/postgres/000154_drop_translation_updateat_index.down.sql @@ -0,0 +1,3 @@ +-- morph:nontransactional +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_translations_updateat + ON translations (updateAt DESC); diff --git a/server/channels/db/migrations/postgres/000154_drop_translation_updateat_index.up.sql b/server/channels/db/migrations/postgres/000154_drop_translation_updateat_index.up.sql new file mode 100644 index 00000000000..6892f855366 --- /dev/null +++ b/server/channels/db/migrations/postgres/000154_drop_translation_updateat_index.up.sql @@ -0,0 +1,2 @@ +-- morph:nontransactional +DROP INDEX CONCURRENTLY IF EXISTS idx_translations_updateat; diff --git a/server/channels/db/migrations/postgres/000155_create_translation_channel_updateat_index.down.sql b/server/channels/db/migrations/postgres/000155_create_translation_channel_updateat_index.down.sql new file mode 100644 index 00000000000..6f07e977db3 --- /dev/null +++ b/server/channels/db/migrations/postgres/000155_create_translation_channel_updateat_index.down.sql @@ -0,0 +1,2 @@ +-- morph:nontransactional +DROP INDEX CONCURRENTLY IF EXISTS idx_translations_channel_updateat; diff --git a/server/channels/db/migrations/postgres/000155_create_translation_channel_updateat_index.up.sql b/server/channels/db/migrations/postgres/000155_create_translation_channel_updateat_index.up.sql new file mode 100644 index 00000000000..ccbfe29dd48 --- /dev/null +++ b/server/channels/db/migrations/postgres/000155_create_translation_channel_updateat_index.up.sql @@ -0,0 +1,3 @@ +-- morph:nontransactional +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_translations_channel_updateat + ON translations(channelid, objectType, updateAt DESC, dstlang); diff --git a/server/channels/store/localcachelayer/autotranslation_layer.go b/server/channels/store/localcachelayer/autotranslation_layer.go index 0e7fd7673a9..1a718793468 100644 --- a/server/channels/store/localcachelayer/autotranslation_layer.go +++ b/server/channels/store/localcachelayer/autotranslation_layer.go @@ -25,7 +25,11 @@ func userLanguageKey(userID, channelID string) string { return fmt.Sprintf("lang:%s:%s", userID, channelID) } -// Cluster invalidation handler +func postTranslationEtagKey(channelID string) string { + return fmt.Sprintf("etag:%s", channelID) +} + +// Cluster invalidation handler for user auto-translation cache func (s *LocalCacheAutoTranslationStore) handleClusterInvalidateUserAutoTranslation(msg *model.ClusterMessage) { if bytes.Equal(msg.Data, clearCacheMessageData) { s.rootStore.userAutoTranslationCache.Purge() @@ -34,12 +38,23 @@ func (s *LocalCacheAutoTranslationStore) handleClusterInvalidateUserAutoTranslat } } +// Cluster invalidation handler for post translation etag cache +func (s *LocalCacheAutoTranslationStore) handleClusterInvalidatePostTranslationEtag(msg *model.ClusterMessage) { + if bytes.Equal(msg.Data, clearCacheMessageData) { + s.rootStore.postTranslationEtagCache.Purge() + } else { + s.rootStore.postTranslationEtagCache.Remove(string(msg.Data)) + } +} + // ClearCaches purges all auto-translation caches func (s LocalCacheAutoTranslationStore) ClearCaches() { s.rootStore.doClearCacheCluster(s.rootStore.userAutoTranslationCache) + s.rootStore.doClearCacheCluster(s.rootStore.postTranslationEtagCache) if s.rootStore.metrics != nil { s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.userAutoTranslationCache.Name()) + s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.postTranslationEtagCache.Name()) } } @@ -133,3 +148,48 @@ func (s LocalCacheAutoTranslationStore) InvalidateUserLocaleCache(userID string) s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.userAutoTranslationCache.Name()) } } + +// GetLatestPostUpdateAtForChannel returns the most recent updateAt timestamp for post translations +// in the given channel (across all locales, with caching) +func (s LocalCacheAutoTranslationStore) GetLatestPostUpdateAtForChannel(channelID string) (int64, error) { + key := postTranslationEtagKey(channelID) + + var updateAt int64 + if err := s.rootStore.doStandardReadCache(s.rootStore.postTranslationEtagCache, key, &updateAt); err == nil { + return updateAt, nil + } + + updateAt, err := s.AutoTranslationStore.GetLatestPostUpdateAtForChannel(channelID) + if err != nil { + return 0, err + } + + s.rootStore.doStandardAddToCache(s.rootStore.postTranslationEtagCache, key, updateAt) + return updateAt, nil +} + +// InvalidatePostTranslationEtag invalidates the cached post translation etag for a channel +// This should be called after saving a new post translation +func (s LocalCacheAutoTranslationStore) InvalidatePostTranslationEtag(channelID string) { + key := postTranslationEtagKey(channelID) + s.rootStore.doInvalidateCacheCluster(s.rootStore.postTranslationEtagCache, key, nil) + + if s.rootStore.metrics != nil { + s.rootStore.metrics.IncrementMemCacheInvalidationCounter(s.rootStore.postTranslationEtagCache.Name()) + } +} + +// Save wraps the underlying Save and invalidates the post translation etag cache for post translations +func (s LocalCacheAutoTranslationStore) Save(translation *model.Translation) error { + err := s.AutoTranslationStore.Save(translation) + if err != nil { + return err + } + + // Invalidate post translation etag cache only for post translations + if translation.ChannelID != "" && translation.ObjectType == model.TranslationObjectTypePost { + s.InvalidatePostTranslationEtag(translation.ChannelID) + } + + return nil +} diff --git a/server/channels/store/localcachelayer/layer.go b/server/channels/store/localcachelayer/layer.go index 136548c18a7..6de4e9f4a33 100644 --- a/server/channels/store/localcachelayer/layer.go +++ b/server/channels/store/localcachelayer/layer.go @@ -73,6 +73,8 @@ const ( // Auto-translation caches UserAutoTranslationCacheSize = 50000 // User+channel combos UserAutoTranslationCacheSec = 15 * 60 + PostTranslationEtagCacheSize = 25000 // Channel etags for post translations + PostTranslationEtagCacheSec = 15 * 60 ContentFlaggingCacheSize = 100 @@ -138,6 +140,7 @@ type LocalCacheStore struct { autotranslation LocalCacheAutoTranslationStore userAutoTranslationCache cache.Cache + postTranslationEtagCache cache.Cache contentFlagging LocalCacheContentFlaggingStore contentFlaggingCache cache.Cache @@ -400,6 +403,14 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf }); err != nil { return } + if localCacheStore.postTranslationEtagCache, err = cacheProvider.NewCache(&cache.CacheOptions{ + Size: PostTranslationEtagCacheSize, + Name: "PostTranslationEtag", + DefaultExpiry: PostTranslationEtagCacheSec * time.Second, + InvalidateClusterEvent: model.ClusterEventInvalidateCacheForPostTranslationEtag, + }); err != nil { + return + } localCacheStore.autotranslation = LocalCacheAutoTranslationStore{AutoTranslationStore: baseStore.AutoTranslation(), rootStore: &localCacheStore} if localCacheStore.contentFlaggingCache, err = cacheProvider.NewCache(&cache.CacheOptions{ Size: ContentFlaggingCacheSize, @@ -470,6 +481,7 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForAllProfiles, localCacheStore.user.handleClusterInvalidateAllProfiles) cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForTeams, localCacheStore.team.handleClusterInvalidateTeam) cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForUserAutoTranslation, localCacheStore.autotranslation.handleClusterInvalidateUserAutoTranslation) + cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForPostTranslationEtag, localCacheStore.autotranslation.handleClusterInvalidatePostTranslationEtag) cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForContentFlagging, localCacheStore.contentFlagging.handleClusterInvalidateContentFlagging) cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForReadReceipts, localCacheStore.readReceipt.handleClusterInvalidateReadReceipts) cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForTemporaryPosts, localCacheStore.temporaryPost.handleClusterInvalidateTemporaryPosts) @@ -671,6 +683,7 @@ func (s *LocalCacheStore) Invalidate() { s.doClearCacheCluster(s.teamAllTeamIdsForUserCache) s.doClearCacheCluster(s.rolePermissionsCache) s.doClearCacheCluster(s.userAutoTranslationCache) + s.doClearCacheCluster(s.postTranslationEtagCache) s.doClearCacheCluster(s.readReceiptCache) s.doClearCacheCluster(s.readReceiptPostReadersCache) s.doClearCacheCluster(s.temporaryPostCache) diff --git a/server/channels/store/localcachelayer/main_test.go b/server/channels/store/localcachelayer/main_test.go index 02b9be0e43b..417a0c99a0e 100644 --- a/server/channels/store/localcachelayer/main_test.go +++ b/server/channels/store/localcachelayer/main_test.go @@ -42,6 +42,8 @@ func getMockStore(t *testing.T) *mocks.Store { mockStore.On("Reaction").Return(&mockReactionsStore) mockAutoTranslationStore := mocks.AutoTranslationStore{} + // GetLatestPostUpdateAtForChannel now takes only channelID (no locale) since caching is per-channel + mockAutoTranslationStore.On("GetLatestPostUpdateAtForChannel", "channelId").Return(int64(5000), nil) mockStore.On("AutoTranslation").Return(&mockAutoTranslationStore) fakeRole := model.Role{Id: "123", Name: "role-name"} @@ -140,8 +142,10 @@ func getMockStore(t *testing.T) *mocks.Store { mockPostStoreEtagResult := fmt.Sprintf("%v.%v", model.CurrentVersion, 1) mockPostStore.On("ClearCaches") mockPostStore.On("InvalidateLastPostTimeCache", "channelId") - mockPostStore.On("GetEtag", "channelId", true, false).Return(mockPostStoreEtagResult) - mockPostStore.On("GetEtag", "channelId", false, false).Return(mockPostStoreEtagResult) + mockPostStore.On("GetEtag", "channelId", true, false, false).Return(mockPostStoreEtagResult) + mockPostStore.On("GetEtag", "channelId", false, false, false).Return(mockPostStoreEtagResult) + mockPostStore.On("GetEtag", "channelId", true, false, true).Return(mockPostStoreEtagResult) + mockPostStore.On("GetEtag", "channelId", false, false, true).Return(mockPostStoreEtagResult) mockPostStore.On("GetPostsSince", mock.AnythingOfType("*request.Context"), mockPostStoreOptions, true, map[string]bool{}).Return(model.NewPostList(), nil) mockPostStore.On("GetPostsSince", mock.AnythingOfType("*request.Context"), mockPostStoreOptions, false, map[string]bool{}).Return(model.NewPostList(), nil) mockStore.On("Post").Return(&mockPostStore) diff --git a/server/channels/store/localcachelayer/post_layer.go b/server/channels/store/localcachelayer/post_layer.go index c22ec3410f9..4cf54025751 100644 --- a/server/channels/store/localcachelayer/post_layer.go +++ b/server/channels/store/localcachelayer/post_layer.go @@ -71,23 +71,35 @@ func (s LocalCachePostStore) InvalidateLastPostTimeCache(channelId string) { } } -func (s LocalCachePostStore) GetEtag(channelId string, allowFromCache, collapsedThreads bool) string { +func (s LocalCachePostStore) GetEtag(channelId string, allowFromCache, collapsedThreads bool, includeTranslations bool) string { + var baseEtag string + if allowFromCache { var lastTime int64 if err := s.rootStore.doStandardReadCache(s.rootStore.lastPostTimeCache, channelId, &lastTime); err == nil { - return fmt.Sprintf("%v.%v", model.CurrentVersion, lastTime) + baseEtag = fmt.Sprintf("%v.%v", model.CurrentVersion, lastTime) } } - result := s.PostStore.GetEtag(channelId, allowFromCache, collapsedThreads) + if baseEtag == "" { + baseEtag = s.PostStore.GetEtag(channelId, allowFromCache, collapsedThreads, includeTranslations) - splittedResult := strings.Split(result, ".") + splittedResult := strings.Split(baseEtag, ".") - lastTime, _ := strconv.ParseInt((splittedResult[len(splittedResult)-1]), 10, 64) + lastTime, _ := strconv.ParseInt((splittedResult[len(splittedResult)-1]), 10, 64) - s.rootStore.doStandardAddToCache(s.rootStore.lastPostTimeCache, channelId, lastTime) + s.rootStore.doStandardAddToCache(s.rootStore.lastPostTimeCache, channelId, lastTime) + } - return result + // If translations should be included, append translation time + if includeTranslations { + translationTime, err := s.rootStore.AutoTranslation().GetLatestPostUpdateAtForChannel(channelId) + if err == nil && translationTime > 0 { + return fmt.Sprintf("%s_%d", baseEtag, translationTime) + } + } + + return baseEtag } func (s LocalCachePostStore) GetPostsSince(rctx request.CTX, options model.GetPostsSinceOptions, allowFromCache bool, sanitizeOptions map[string]bool) (*model.PostList, error) { diff --git a/server/channels/store/localcachelayer/post_layer_test.go b/server/channels/store/localcachelayer/post_layer_test.go index e950aea6250..1b2f3470aeb 100644 --- a/server/channels/store/localcachelayer/post_layer_test.go +++ b/server/channels/store/localcachelayer/post_layer_test.go @@ -41,11 +41,11 @@ func TestPostStoreLastPostTimeCache(t *testing.T) { expectedResult := fmt.Sprintf("%v.%v", model.CurrentVersion, fakeLastTime) - etag := cachedStore.Post().GetEtag(channelId, true, false) + etag := cachedStore.Post().GetEtag(channelId, true, false, false) assert.Equal(t, etag, expectedResult) mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1) - etag = cachedStore.Post().GetEtag(channelId, true, false) + etag = cachedStore.Post().GetEtag(channelId, true, false, false) assert.Equal(t, etag, expectedResult) mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1) }) @@ -56,9 +56,9 @@ func TestPostStoreLastPostTimeCache(t *testing.T) { cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger) require.NoError(t, err) - cachedStore.Post().GetEtag(channelId, true, false) + cachedStore.Post().GetEtag(channelId, true, false, false) mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1) - cachedStore.Post().GetEtag(channelId, false, false) + cachedStore.Post().GetEtag(channelId, false, false, false) mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 2) }) @@ -68,10 +68,10 @@ func TestPostStoreLastPostTimeCache(t *testing.T) { cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger) require.NoError(t, err) - cachedStore.Post().GetEtag(channelId, true, false) + cachedStore.Post().GetEtag(channelId, true, false, false) mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1) cachedStore.Post().InvalidateLastPostTimeCache(channelId) - cachedStore.Post().GetEtag(channelId, true, false) + cachedStore.Post().GetEtag(channelId, true, false, false) mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 2) }) @@ -81,10 +81,10 @@ func TestPostStoreLastPostTimeCache(t *testing.T) { cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger) require.NoError(t, err) - cachedStore.Post().GetEtag(channelId, true, false) + cachedStore.Post().GetEtag(channelId, true, false, false) mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1) cachedStore.Post().ClearCaches() - cachedStore.Post().GetEtag(channelId, true, false) + cachedStore.Post().GetEtag(channelId, true, false, false) mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 2) }) @@ -144,6 +144,60 @@ func TestPostStoreLastPostTimeCache(t *testing.T) { cachedStore.Post().GetPostsSince(rctx, fakeOptions, true, map[string]bool{}) mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetPostsSince", 2) }) + + t.Run("GetEtag: with includeTranslations appends translation time", func(t *testing.T) { + mockStore := getMockStore(t) + mockCacheProvider := getMockCacheProvider() + cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger) + require.NoError(t, err) + + // Expected: "{version}.{lastTime}.{translationTime}" + expectedResult := fmt.Sprintf("%v.%v_%v", model.CurrentVersion, fakeLastTime, 5000) + + etag := cachedStore.Post().GetEtag(channelId, true, false, true) + assert.Equal(t, expectedResult, etag) + mockStore.Post().(*mocks.PostStore).AssertNumberOfCalls(t, "GetEtag", 1) + mockStore.AutoTranslation().(*mocks.AutoTranslationStore).AssertNumberOfCalls(t, "GetLatestPostUpdateAtForChannel", 1) + }) + + t.Run("GetEtag: translation etag is cached per-channel", func(t *testing.T) { + mockStore := getMockStore(t) + mockCacheProvider := getMockCacheProvider() + cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger) + require.NoError(t, err) + + // First call with includeTranslations=true - should call GetLatestPostUpdateAtForChannel + expected := fmt.Sprintf("%v.%v_%v", model.CurrentVersion, fakeLastTime, 5000) + etag := cachedStore.Post().GetEtag(channelId, true, false, true) + assert.Equal(t, expected, etag) + mockStore.AutoTranslation().(*mocks.AutoTranslationStore).AssertNumberOfCalls(t, "GetLatestPostUpdateAtForChannel", 1) + + // Second call - should use cached translation time + etag = cachedStore.Post().GetEtag(channelId, true, false, true) + assert.Equal(t, expected, etag) + mockStore.AutoTranslation().(*mocks.AutoTranslationStore).AssertNumberOfCalls(t, "GetLatestPostUpdateAtForChannel", 1) + }) + + t.Run("GetEtag: invalidate translation etag clears the channel cache", func(t *testing.T) { + mockStore := getMockStore(t) + mockCacheProvider := getMockCacheProvider() + cachedStore, err := NewLocalCacheLayer(mockStore, nil, nil, mockCacheProvider, logger) + require.NoError(t, err) + + // Call GetEtag with includeTranslations=true + expected := fmt.Sprintf("%v.%v_%v", model.CurrentVersion, fakeLastTime, 5000) + etag := cachedStore.Post().GetEtag(channelId, true, false, true) + assert.Equal(t, expected, etag) + mockStore.AutoTranslation().(*mocks.AutoTranslationStore).AssertNumberOfCalls(t, "GetLatestPostUpdateAtForChannel", 1) + + // Invalidate the channel's post translation etag + cachedStore.AutoTranslation().InvalidatePostTranslationEtag(channelId) + + // Call GetEtag again - should re-fetch + etag = cachedStore.Post().GetEtag(channelId, true, false, true) + assert.Equal(t, expected, etag) + mockStore.AutoTranslation().(*mocks.AutoTranslationStore).AssertNumberOfCalls(t, "GetLatestPostUpdateAtForChannel", 2) + }) } func TestPostStoreCache(t *testing.T) { diff --git a/server/channels/store/retrylayer/retrylayer.go b/server/channels/store/retrylayer/retrylayer.go index 228b545cdac..4ce37da0c8c 100644 --- a/server/channels/store/retrylayer/retrylayer.go +++ b/server/channels/store/retrylayer/retrylayer.go @@ -914,27 +914,6 @@ func (s *RetryLayerAutoTranslationStore) GetActiveDestinationLanguages(channelID } -func (s *RetryLayerAutoTranslationStore) GetAllByStatePage(state model.TranslationState, offset int, limit int) ([]*model.Translation, error) { - - tries := 0 - for { - result, err := s.AutoTranslationStore.GetAllByStatePage(state, offset, limit) - if err == nil { - return result, nil - } - if !isRepeatableError(err) { - return result, err - } - tries++ - if tries >= 3 { - err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") - return result, err - } - timepkg.Sleep(100 * timepkg.Millisecond) - } - -} - func (s *RetryLayerAutoTranslationStore) GetAllForObject(objectType string, objectID string) ([]*model.Translation, error) { tries := 0 @@ -998,6 +977,27 @@ func (s *RetryLayerAutoTranslationStore) GetByStateOlderThan(state model.Transla } +func (s *RetryLayerAutoTranslationStore) GetLatestPostUpdateAtForChannel(channelID string) (int64, error) { + + tries := 0 + for { + result, err := s.AutoTranslationStore.GetLatestPostUpdateAtForChannel(channelID) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerAutoTranslationStore) GetUserLanguage(userID string, channelID string) (string, error) { tries := 0 @@ -1019,6 +1019,12 @@ func (s *RetryLayerAutoTranslationStore) GetUserLanguage(userID string, channelI } +func (s *RetryLayerAutoTranslationStore) InvalidatePostTranslationEtag(channelID string) { + + s.AutoTranslationStore.InvalidatePostTranslationEtag(channelID) + +} + func (s *RetryLayerAutoTranslationStore) InvalidateUserAutoTranslation(userID string, channelID string) { s.AutoTranslationStore.InvalidateUserAutoTranslation(userID, channelID) @@ -8114,9 +8120,9 @@ func (s *RetryLayerPostStore) GetEditHistoryForPost(postID string) ([]*model.Pos } -func (s *RetryLayerPostStore) GetEtag(channelID string, allowFromCache bool, collapsedThreads bool) string { +func (s *RetryLayerPostStore) GetEtag(channelID string, allowFromCache bool, collapsedThreads bool, includeTranslations bool) string { - return s.PostStore.GetEtag(channelID, allowFromCache, collapsedThreads) + return s.PostStore.GetEtag(channelID, allowFromCache, collapsedThreads, includeTranslations) } diff --git a/server/channels/store/sqlstore/autotranslation_store.go b/server/channels/store/sqlstore/autotranslation_store.go index 22e9fbba013..1739bcbdcc5 100644 --- a/server/channels/store/sqlstore/autotranslation_store.go +++ b/server/channels/store/sqlstore/autotranslation_store.go @@ -24,6 +24,7 @@ type TranslationMeta json.RawMessage type Translation struct { ObjectType string ObjectID string + ChannelID *string DstLang string ProviderID string NormHash string @@ -294,6 +295,11 @@ func (s *SqlAutoTranslationStore) Save(translation *model.Translation) error { objectType = model.TranslationObjectTypePost } + var channelID *string + if translation.ChannelID != "" { + channelID = &translation.ChannelID + } + objectID := translation.ObjectID // Preserve existing Meta fields and add/override "type" @@ -321,10 +327,11 @@ func (s *SqlAutoTranslationStore) Save(translation *model.Translation) error { query := s.getQueryBuilder(). Insert("Translations"). - Columns("ObjectId", "DstLang", "ObjectType", "ProviderId", "NormHash", "Text", "Confidence", "Meta", "State", "UpdateAt"). - Values(objectID, dstLang, objectType, providerID, translation.NormHash, text, confidence, metaBytes, string(translation.State), now). + Columns("ObjectId", "DstLang", "ObjectType", "ChannelId", "ProviderId", "NormHash", "Text", "Confidence", "Meta", "State", "UpdateAt"). + Values(objectID, dstLang, objectType, channelID, providerID, translation.NormHash, text, confidence, metaBytes, string(translation.State), now). Suffix(`ON CONFLICT (ObjectId, ObjectType, dstLang) DO UPDATE SET + ChannelId = EXCLUDED.ChannelId, ProviderId = EXCLUDED.ProviderId, NormHash = EXCLUDED.NormHash, Text = EXCLUDED.Text, @@ -342,66 +349,6 @@ func (s *SqlAutoTranslationStore) Save(translation *model.Translation) error { return nil } -func (s *SqlAutoTranslationStore) GetAllByStatePage(state model.TranslationState, offset int, limit int) ([]*model.Translation, error) { - query := s.getQueryBuilder(). - Select("ObjectType", "ObjectId", "DstLang", "ProviderId", "NormHash", "Text", "Confidence", "Meta", "State", "UpdateAt"). - From("Translations"). - Where(sq.Eq{"State": string(state)}). - OrderBy("UpdateAt ASC"). - Limit(uint64(limit)). - Offset(uint64(offset)) - - var translations []Translation - if err := s.GetReplica().SelectBuilder(&translations, query); err != nil { - return nil, errors.Wrapf(err, "failed to get translations by state=%s", state) - } - - result := make([]*model.Translation, 0, len(translations)) - for _, t := range translations { - var translationTypeStr string - - meta, err := t.Meta.ToMap() - if err != nil { - // Log error but continue with other translations - continue - } - - if v, ok := meta["type"]; ok { - if s, ok := v.(string); ok { - translationTypeStr = s - } - } - - // Default objectType to "post" if not set - objectType := t.ObjectType - if objectType == "" { - objectType = model.TranslationObjectTypePost - } - - modelT := &model.Translation{ - ObjectID: t.ObjectID, - ObjectType: objectType, - Lang: t.DstLang, - Type: model.TranslationType(translationTypeStr), - Confidence: t.Confidence, - State: model.TranslationState(t.State), - NormHash: t.NormHash, - Meta: meta, - UpdateAt: t.UpdateAt, - } - - if modelT.Type == model.TranslationTypeObject { - modelT.ObjectJSON = json.RawMessage(t.Text) - } else { - modelT.Text = t.Text - } - - result = append(result, modelT) - } - - return result, nil -} - func (s *SqlAutoTranslationStore) GetByStateOlderThan(state model.TranslationState, olderThanMillis int64, limit int) ([]*model.Translation, error) { query := s.getQueryBuilder(). Select("ObjectType", "ObjectId", "DstLang", "ProviderId", "NormHash", "Text", "Confidence", "Meta", "State", "UpdateAt"). @@ -467,3 +414,23 @@ func (s *SqlAutoTranslationStore) ClearCaches() {} func (s *SqlAutoTranslationStore) InvalidateUserAutoTranslation(userID, channelID string) {} func (s *SqlAutoTranslationStore) InvalidateUserLocaleCache(userID string) {} + +// GetLatestPostUpdateAtForChannel returns the most recent updateAt timestamp for post translations +// in the given channel (across all locales). Uses a direct lookup on the channelid column +// for O(1) performance. Returns 0 if no translations exist. +func (s *SqlAutoTranslationStore) GetLatestPostUpdateAtForChannel(channelID string) (int64, error) { + query := s.getQueryBuilder(). + Select("COALESCE(MAX(updateAt), 0)"). + From("translations"). + Where(sq.Eq{"channelid": channelID}). + Where(sq.Eq{"objectType": model.TranslationObjectTypePost}) + + var updateAt int64 + if err := s.GetReplica().GetBuilder(&updateAt, query); err != nil { + return 0, errors.Wrapf(err, "failed to get latest translation updateAt for channel_id=%s", channelID) + } + + return updateAt, nil +} + +func (s *SqlAutoTranslationStore) InvalidatePostTranslationEtag(channelID string) {} diff --git a/server/channels/store/sqlstore/post_store.go b/server/channels/store/sqlstore/post_store.go index 0ea20a0cfc5..a9faf82b51d 100644 --- a/server/channels/store/sqlstore/post_store.go +++ b/server/channels/store/sqlstore/post_store.go @@ -938,7 +938,7 @@ func (s *SqlPostStore) InvalidateLastPostTimeCache(channelId string) { } //nolint:unparam -func (s *SqlPostStore) GetEtag(channelId string, allowFromCache, collapsedThreads bool) string { +func (s *SqlPostStore) GetEtag(channelId string, allowFromCache, collapsedThreads bool, includeTranslations bool) string { q := s.getQueryBuilder().Select("Id", "UpdateAt").From("Posts").Where(sq.Eq{"ChannelId": channelId}).OrderBy("UpdateAt DESC").Limit(1) if collapsedThreads { q.Where(sq.Eq{"RootId": ""}) diff --git a/server/channels/store/store.go b/server/channels/store/store.go index 9c0fa01ae72..ea6bade46ec 100644 --- a/server/channels/store/store.go +++ b/server/channels/store/store.go @@ -391,7 +391,7 @@ type PostStore interface { GetPostAfterTime(channelID string, timestamp int64, collapsedThreads bool) (*model.Post, error) GetPostIdAfterTime(channelID string, timestamp int64, collapsedThreads bool) (string, error) GetPostIdBeforeTime(channelID string, timestamp int64, collapsedThreads bool) (string, error) - GetEtag(channelID string, allowFromCache bool, collapsedThreads bool) string + GetEtag(channelID string, allowFromCache bool, collapsedThreads bool, includeTranslations bool) string Search(teamID string, userID string, params *model.SearchParams) (*model.PostList, error) AnalyticsUserCountsWithPostsByDay(teamID string) (model.AnalyticsRows, error) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) @@ -1166,7 +1166,6 @@ type AutoTranslationStore interface { GetBatch(objectType string, objectIDs []string, dstLang string) (map[string]*model.Translation, error) GetAllForObject(objectType, objectID string) ([]*model.Translation, error) Save(translation *model.Translation) error - GetAllByStatePage(state model.TranslationState, offset, limit int) ([]*model.Translation, error) GetByStateOlderThan(state model.TranslationState, olderThanMillis int64, limit int) ([]*model.Translation, error) ClearCaches() @@ -1176,6 +1175,12 @@ type AutoTranslationStore interface { // InvalidateUserLocaleCache invalidates all language caches for a user across all channels. // This is called when a user changes their locale preference. InvalidateUserLocaleCache(userID string) + // GetLatestPostUpdateAtForChannel returns the most recent updateAt timestamp for post translations + // in the given channel (across all locales). Returns 0 if no translations exist. + GetLatestPostUpdateAtForChannel(channelID string) (int64, error) + // InvalidatePostTranslationEtag invalidates the cached post translation etag for a channel. + // This should be called after saving a new post translation. + InvalidatePostTranslationEtag(channelID string) } type ContentFlaggingStore interface { diff --git a/server/channels/store/storetest/mocks/AutoTranslationStore.go b/server/channels/store/storetest/mocks/AutoTranslationStore.go index 10ea7a1b94a..ae6ea31b774 100644 --- a/server/channels/store/storetest/mocks/AutoTranslationStore.go +++ b/server/channels/store/storetest/mocks/AutoTranslationStore.go @@ -79,36 +79,6 @@ func (_m *AutoTranslationStore) GetActiveDestinationLanguages(channelID string, return r0, r1 } -// GetAllByStatePage provides a mock function with given fields: state, offset, limit -func (_m *AutoTranslationStore) GetAllByStatePage(state model.TranslationState, offset int, limit int) ([]*model.Translation, error) { - ret := _m.Called(state, offset, limit) - - if len(ret) == 0 { - panic("no return value specified for GetAllByStatePage") - } - - var r0 []*model.Translation - var r1 error - if rf, ok := ret.Get(0).(func(model.TranslationState, int, int) ([]*model.Translation, error)); ok { - return rf(state, offset, limit) - } - if rf, ok := ret.Get(0).(func(model.TranslationState, int, int) []*model.Translation); ok { - r0 = rf(state, offset, limit) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*model.Translation) - } - } - - if rf, ok := ret.Get(1).(func(model.TranslationState, int, int) error); ok { - r1 = rf(state, offset, limit) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // GetAllForObject provides a mock function with given fields: objectType, objectID func (_m *AutoTranslationStore) GetAllForObject(objectType string, objectID string) ([]*model.Translation, error) { ret := _m.Called(objectType, objectID) @@ -199,6 +169,34 @@ func (_m *AutoTranslationStore) GetByStateOlderThan(state model.TranslationState return r0, r1 } +// GetLatestPostUpdateAtForChannel provides a mock function with given fields: channelID +func (_m *AutoTranslationStore) GetLatestPostUpdateAtForChannel(channelID string) (int64, error) { + ret := _m.Called(channelID) + + if len(ret) == 0 { + panic("no return value specified for GetLatestPostUpdateAtForChannel") + } + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(string) (int64, error)); ok { + return rf(channelID) + } + if rf, ok := ret.Get(0).(func(string) int64); ok { + r0 = rf(channelID) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(channelID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetUserLanguage provides a mock function with given fields: userID, channelID func (_m *AutoTranslationStore) GetUserLanguage(userID string, channelID string) (string, error) { ret := _m.Called(userID, channelID) @@ -227,6 +225,11 @@ func (_m *AutoTranslationStore) GetUserLanguage(userID string, channelID string) return r0, r1 } +// InvalidatePostTranslationEtag provides a mock function with given fields: channelID +func (_m *AutoTranslationStore) InvalidatePostTranslationEtag(channelID string) { + _m.Called(channelID) +} + // InvalidateUserAutoTranslation provides a mock function with given fields: userID, channelID func (_m *AutoTranslationStore) InvalidateUserAutoTranslation(userID string, channelID string) { _m.Called(userID, channelID) @@ -283,24 +286,6 @@ func (_m *AutoTranslationStore) Save(translation *model.Translation) error { return r0 } -// SetUserEnabled provides a mock function with given fields: userID, channelID, enabled -func (_m *AutoTranslationStore) SetUserEnabled(userID string, channelID string, enabled bool) error { - ret := _m.Called(userID, channelID, enabled) - - if len(ret) == 0 { - panic("no return value specified for SetUserEnabled") - } - - var r0 error - if rf, ok := ret.Get(0).(func(string, string, bool) error); ok { - r0 = rf(userID, channelID, enabled) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // NewAutoTranslationStore creates a new instance of AutoTranslationStore. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func NewAutoTranslationStore(t interface { diff --git a/server/channels/store/storetest/mocks/PostStore.go b/server/channels/store/storetest/mocks/PostStore.go index a18333c7d45..6f959b7e3a8 100644 --- a/server/channels/store/storetest/mocks/PostStore.go +++ b/server/channels/store/storetest/mocks/PostStore.go @@ -264,17 +264,17 @@ func (_m *PostStore) GetEditHistoryForPost(postID string) ([]*model.Post, error) return r0, r1 } -// GetEtag provides a mock function with given fields: channelID, allowFromCache, collapsedThreads -func (_m *PostStore) GetEtag(channelID string, allowFromCache bool, collapsedThreads bool) string { - ret := _m.Called(channelID, allowFromCache, collapsedThreads) +// GetEtag provides a mock function with given fields: channelID, allowFromCache, collapsedThreads, includeTranslations +func (_m *PostStore) GetEtag(channelID string, allowFromCache bool, collapsedThreads bool, includeTranslations bool) string { + ret := _m.Called(channelID, allowFromCache, collapsedThreads, includeTranslations) if len(ret) == 0 { panic("no return value specified for GetEtag") } var r0 string - if rf, ok := ret.Get(0).(func(string, bool, bool) string); ok { - r0 = rf(channelID, allowFromCache, collapsedThreads) + if rf, ok := ret.Get(0).(func(string, bool, bool, bool) string); ok { + r0 = rf(channelID, allowFromCache, collapsedThreads, includeTranslations) } else { r0 = ret.Get(0).(string) } diff --git a/server/channels/store/storetest/post_store.go b/server/channels/store/storetest/post_store.go index 4596dddca72..9fd11e44a2d 100644 --- a/server/channels/store/storetest/post_store.go +++ b/server/channels/store/storetest/post_store.go @@ -644,13 +644,13 @@ func testPostStoreGet(t *testing.T, rctx request.CTX, ss store.Store) { o1.UserId = model.NewId() o1.Message = NewTestID() - etag1 := ss.Post().GetEtag(o1.ChannelId, false, false) + etag1 := ss.Post().GetEtag(o1.ChannelId, false, false, false) require.Equal(t, 0, strings.Index(etag1, model.CurrentVersion+"."), "Invalid Etag") o1, err = ss.Post().Save(rctx, o1) require.NoError(t, err) - etag2 := ss.Post().GetEtag(o1.ChannelId, false, false) + etag2 := ss.Post().GetEtag(o1.ChannelId, false, false, false) require.Equal(t, 0, strings.Index(etag2, fmt.Sprintf("%v.%v", model.CurrentVersion, o1.UpdateAt)), "Invalid Etag") r1, err := ss.Post().Get(rctx, o1.Id, model.GetPostsOptions{}, "", map[string]bool{}) @@ -1224,7 +1224,7 @@ func testPostStoreDelete(t *testing.T, rctx request.CTX, ss store.Store) { require.NoError(t, err) // Verify etag generation for the channel containing the post. - etag1 := ss.Post().GetEtag(rootPost.ChannelId, false, false) + etag1 := ss.Post().GetEtag(rootPost.ChannelId, false, false, false) require.Equal(t, 0, strings.Index(etag1, model.CurrentVersion+"."), "Invalid Etag") // Verify the created post. @@ -1250,7 +1250,7 @@ func testPostStoreDelete(t *testing.T, rctx request.CTX, ss store.Store) { require.IsType(t, &store.ErrNotFound{}, err) // Verify etag generation for the channel containing the now deleted post. - etag2 := ss.Post().GetEtag(rootPost.ChannelId, false, false) + etag2 := ss.Post().GetEtag(rootPost.ChannelId, false, false, false) require.Equal(t, 0, strings.Index(etag2, model.CurrentVersion+"."), "Invalid Etag") }) diff --git a/server/channels/store/timerlayer/timerlayer.go b/server/channels/store/timerlayer/timerlayer.go index f642f16bb54..e925aee6627 100644 --- a/server/channels/store/timerlayer/timerlayer.go +++ b/server/channels/store/timerlayer/timerlayer.go @@ -836,22 +836,6 @@ func (s *TimerLayerAutoTranslationStore) GetActiveDestinationLanguages(channelID return result, err } -func (s *TimerLayerAutoTranslationStore) GetAllByStatePage(state model.TranslationState, offset int, limit int) ([]*model.Translation, error) { - start := time.Now() - - result, err := s.AutoTranslationStore.GetAllByStatePage(state, offset, limit) - - elapsed := float64(time.Since(start)) / float64(time.Second) - if s.Root.Metrics != nil { - success := "false" - if err == nil { - success = "true" - } - s.Root.Metrics.ObserveStoreMethodDuration("AutoTranslationStore.GetAllByStatePage", success, elapsed) - } - return result, err -} - func (s *TimerLayerAutoTranslationStore) GetAllForObject(objectType string, objectID string) ([]*model.Translation, error) { start := time.Now() @@ -900,6 +884,22 @@ func (s *TimerLayerAutoTranslationStore) GetByStateOlderThan(state model.Transla return result, err } +func (s *TimerLayerAutoTranslationStore) GetLatestPostUpdateAtForChannel(channelID string) (int64, error) { + start := time.Now() + + result, err := s.AutoTranslationStore.GetLatestPostUpdateAtForChannel(channelID) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("AutoTranslationStore.GetLatestPostUpdateAtForChannel", success, elapsed) + } + return result, err +} + func (s *TimerLayerAutoTranslationStore) GetUserLanguage(userID string, channelID string) (string, error) { start := time.Now() @@ -916,6 +916,21 @@ func (s *TimerLayerAutoTranslationStore) GetUserLanguage(userID string, channelI return result, err } +func (s *TimerLayerAutoTranslationStore) InvalidatePostTranslationEtag(channelID string) { + start := time.Now() + + s.AutoTranslationStore.InvalidatePostTranslationEtag(channelID) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if true { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("AutoTranslationStore.InvalidatePostTranslationEtag", success, elapsed) + } +} + func (s *TimerLayerAutoTranslationStore) InvalidateUserAutoTranslation(userID string, channelID string) { start := time.Now() @@ -6500,10 +6515,10 @@ func (s *TimerLayerPostStore) GetEditHistoryForPost(postID string) ([]*model.Pos return result, err } -func (s *TimerLayerPostStore) GetEtag(channelID string, allowFromCache bool, collapsedThreads bool) string { +func (s *TimerLayerPostStore) GetEtag(channelID string, allowFromCache bool, collapsedThreads bool, includeTranslations bool) string { start := time.Now() - result := s.PostStore.GetEtag(channelID, allowFromCache, collapsedThreads) + result := s.PostStore.GetEtag(channelID, allowFromCache, collapsedThreads, includeTranslations) elapsed := float64(time.Since(start)) / float64(time.Second) if s.Root.Metrics != nil { diff --git a/server/einterfaces/mocks/AutoTranslationInterface.go b/server/einterfaces/mocks/AutoTranslationInterface.go index b0d41bf50b1..7cedb088561 100644 --- a/server/einterfaces/mocks/AutoTranslationInterface.go +++ b/server/einterfaces/mocks/AutoTranslationInterface.go @@ -256,26 +256,6 @@ func (_m *AutoTranslationInterface) MakeWorker() model.Worker { return r0 } -// SetUserEnabled provides a mock function with given fields: channelID, userID, enabled -func (_m *AutoTranslationInterface) SetUserEnabled(channelID string, userID string, enabled bool) *model.AppError { - ret := _m.Called(channelID, userID, enabled) - - if len(ret) == 0 { - panic("no return value specified for SetUserEnabled") - } - - var r0 *model.AppError - if rf, ok := ret.Get(0).(func(string, string, bool) *model.AppError); ok { - r0 = rf(channelID, userID, enabled) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*model.AppError) - } - } - - return r0 -} - // Shutdown provides a mock function with no fields func (_m *AutoTranslationInterface) Shutdown() error { ret := _m.Called() diff --git a/server/public/model/autotranslation.go b/server/public/model/autotranslation.go index d6f4166bf92..8b94d6dd9b1 100644 --- a/server/public/model/autotranslation.go +++ b/server/public/model/autotranslation.go @@ -36,6 +36,7 @@ const ( type Translation struct { ObjectID string `json:"object_id"` ObjectType string `json:"object_type"` + ChannelID string `json:"channel_id,omitempty"` // Channel ID for efficient queries Lang string `json:"lang"` Provider string `json:"provider"` Type TranslationType `json:"type"` @@ -71,6 +72,7 @@ func (t *Translation) Clone() *Translation { return &Translation{ ObjectID: t.ObjectID, ObjectType: t.ObjectType, + ChannelID: t.ChannelID, Lang: t.Lang, Provider: t.Provider, Type: t.Type, diff --git a/server/public/model/cluster_message.go b/server/public/model/cluster_message.go index ffc8d425106..4b9c17162a7 100644 --- a/server/public/model/cluster_message.go +++ b/server/public/model/cluster_message.go @@ -46,6 +46,7 @@ const ( ClusterEventPluginEvent ClusterEvent = "plugin_event" ClusterEventInvalidateCacheForTermsOfService ClusterEvent = "inv_terms_of_service" ClusterEventInvalidateCacheForUserAutoTranslation ClusterEvent = "inv_user_autotranslation" + ClusterEventInvalidateCacheForPostTranslationEtag ClusterEvent = "inv_post_translation_etag" ClusterEventAutoTranslationTask ClusterEvent = "autotranslation_task" ClusterEventBusyStateChanged ClusterEvent = "busy_state_change" // Note: if you are adding a new event, please also add it in the slice of