mattermost/server/channels/app/import_bulk_insert_test.go

// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.

package app

import (
	"fmt"
	"os"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/mattermost/mattermost/server/public/model"
	"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
)

// TestImportBulkInsertChunking reads a pre-generated JSONL fixture through
// BulkImport that exceeds the default INSERT parameter threshold (50,000
// params — 76% of PostgreSQL's 65,535 hard limit). A single worker is used
// so that the import batches are large enough to trigger multi-chunk INSERTs.
//
// The fixture contains placeholder tokens that are replaced with unique
// names here so parallel test runs do not collide.
//
// Overflow thresholds exercised:
//
//	Posts (replies):      18 cols → chunk at 2,777 rows  (2,778 generated)
//	Thread memberships:   6 cols → chunk at 8,333 rows  (9,000 generated)
func TestImportBulkInsertChunking(t *testing.T) {
	mainHelper.Parallel(t)
	th := Setup(t)

	// server/tests/bulk_insert_chunk_test.jsonl — 1,015-line, 535 KB file containing:
	// - 1 team, 2 channels, 10 users
	// - 1,000 root posts each with 9 thread followers (9,000 memberships; threshold is 8,333)
	// - 1 root post with 2,778 replies (threshold is 2,777)
	testsDir, _ := fileutils.FindDir("tests")
	raw, err := os.ReadFile(testsDir + "/bulk_insert_chunk_test.jsonl")
	require.NoError(t, err)

	// Replace placeholders with unique names for this test run.
	teamName := model.NewRandomTeamName()
	chanThreads := NewTestId()
	chanReplies := NewTestId()

	usernames := make([]string, 10)
	oldNew := []string{
		"__TEAM__", teamName,
		"__CHAN_THREADS__", chanThreads,
		"__CHAN_REPLIES__", chanReplies,
	}
	for i := range 10 {
		usernames[i] = model.NewUsername()
		oldNew = append(oldNew,
			fmt.Sprintf("__USER%d__", i), usernames[i],
		)
	}
	jsonl := strings.NewReplacer(oldNew...).Replace(string(raw))

	line, appErr := th.App.BulkImport(th.Context, strings.NewReader(jsonl), nil, false, 1)
	require.Nil(t, appErr, "BulkImport failed at line %d", line)
	require.Equal(t, 0, line)

	team, gErr := th.App.GetTeamByName(teamName)
	require.Nil(t, gErr)

	t.Run("thread memberships were chunked across parameter limit", func(t *testing.T) {
		channel, cErr := th.App.GetChannelByName(th.Context, chanThreads, team.Id, false)
		require.Nil(t, cErr)

		followerIDs := make([]string, 9)
		for i := range 9 {
			u, uErr := th.App.GetUserByUsername(usernames[i+1])
			require.Nil(t, uErr)
			followerIDs[i] = u.Id
		}

		// Spot-check thread followers on the first and last posts.
		const (
			baseTime     = 1700000000000
			numRootPosts = 1000
		)

		firstPosts, nErr := th.App.Srv().Store().Post().GetPostsCreatedAt(channel.Id, baseTime)
		require.NoError(t, nErr)
		require.Len(t, firstPosts, 1)
		firstFollowers, nErr := th.App.Srv().Store().Thread().GetThreadFollowers(firstPosts[0].Id, true)
		require.NoError(t, nErr)
		assert.ElementsMatch(t, followerIDs, firstFollowers)

		lastPosts, nErr := th.App.Srv().Store().Post().GetPostsCreatedAt(channel.Id, baseTime+numRootPosts-1)
		require.NoError(t, nErr)
		require.Len(t, lastPosts, 1)
		lastFollowers, nErr := th.App.Srv().Store().Thread().GetThreadFollowers(lastPosts[0].Id, true)
		require.NoError(t, nErr)
		assert.ElementsMatch(t, followerIDs, lastFollowers)
	})

	t.Run("replies were chunked across parameter limit", func(t *testing.T) {
		channel, cErr := th.App.GetChannelByName(th.Context, chanReplies, team.Id, false)
		require.Nil(t, cErr)

		const (
			replyBaseTime = 1700001000000
			numReplies    = 2778
		)

		rootPosts, nErr := th.App.Srv().Store().Post().GetPostsCreatedAt(channel.Id, replyBaseTime)
		require.NoError(t, nErr)
		require.Len(t, rootPosts, 1)

		thread, nErr := th.App.Srv().Store().Thread().Get(rootPosts[0].Id)
		require.NoError(t, nErr)
		require.NotNil(t, thread)
		assert.Equal(t, int64(numReplies), thread.ReplyCount)
	})
}