[MM-62775] Fix: Bulk export not completing (#30044)

* do not error on exportFile error

* add tests for local and s3 storage exporting with missing file

* linting

* fix attachment path validation in mmctl

server/channels/app/export.go

@@ -207,9 +207,10 @@ func (a *App) BulkExport(ctx request.CTX, writer io.Writer, outPath string, job
 		ctx.Logger().Info("Bulk export: exporting custom emojis")
 		for _, emojiPath := range emojiPaths {
 			if err := a.exportFile(ctx, outPath, emojiPath, zipWr); err != nil {
-				return err
+				ctx.Logger().Warn("Unable to export emoji", mlog.String("emoji_path", emojiPath), mlog.Err(err))
+			} else {
+				totalExportedEmojis++
 			}
-			totalExportedEmojis++
 			if totalExportedEmojis%10 == 0 {
 				ctx.Logger().Info("Bulk export: exporting emojis progress", mlog.Int("total_successfully_exported_emojis", totalExportedEmojis), mlog.Int("total_emojis_to_export", emojisLen))
 			}
@@ -236,9 +237,10 @@ func (a *App) exportAttachments(ctx request.CTX, attachments []imports.Attachmen
 	attachmentsLen := len(attachments)
 	for _, attachment := range attachments {
 		if err := a.exportFile(ctx, outPath, *attachment.Path, zipWr); err != nil {
-			return err
+			ctx.Logger().Warn("Unable to export file attachment", mlog.String("attachment_path", *attachment.Path), mlog.Err(err))
+		} else {
+			totalExportedFiles++
 		}
-		totalExportedFiles++
 		if totalExportedFiles%10 == 0 {
 			ctx.Logger().Info("Bulk export: exporting file attachments progress", mlog.Int("total_successfully_exported_files", totalExportedFiles), mlog.Int("total_files_to_export", attachmentsLen))
 		}

server/channels/app/export_test.go

@@ -14,6 +14,10 @@ import (
 	"testing"
 	"time"
 
+	"github.com/mattermost/mattermost/server/v8/channels/testlib"
+
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -843,6 +847,113 @@ func TestExportPostsWithThread(t *testing.T) {
 	})
 }
 
+func TestExportFileWarnings(t *testing.T) {
+	testCases := []struct {
+		Description string
+		ConfigFunc  func(cfg *model.Config)
+	}{
+		{
+			"local",
+			func(cfg *model.Config) {
+				cfg.FileSettings.DriverName = model.NewPointer(model.ImageDriverLocal)
+			},
+		},
+		{
+			"s3",
+			func(cfg *model.Config) {
+				s3Host := os.Getenv("CI_MINIO_HOST")
+				if s3Host == "" {
+					s3Host = "localhost"
+				}
+
+				s3Port := os.Getenv("CI_MINIO_PORT")
+				if s3Port == "" {
+					s3Port = "9000"
+				}
+
+				s3Endpoint := fmt.Sprintf("%s:%s", s3Host, s3Port)
+				cfg.FileSettings.DriverName = model.NewPointer(model.ImageDriverS3)
+				cfg.FileSettings.AmazonS3AccessKeyId = model.NewPointer(model.MinioAccessKey)
+				cfg.FileSettings.AmazonS3SecretAccessKey = model.NewPointer(model.MinioSecretKey)
+				cfg.FileSettings.AmazonS3Bucket = model.NewPointer(model.MinioBucket)
+				cfg.FileSettings.AmazonS3PathPrefix = model.NewPointer("")
+				cfg.FileSettings.AmazonS3Endpoint = model.NewPointer(s3Endpoint)
+				cfg.FileSettings.AmazonS3Region = model.NewPointer("")
+				cfg.FileSettings.AmazonS3SSL = model.NewPointer(false)
+			},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.Description, func(t *testing.T) {
+			th := Setup(t)
+			defer th.TearDown()
+
+			th.App.UpdateConfig(testCase.ConfigFunc)
+
+			// Create a buffer to capture logs
+			buffer := &mlog.Buffer{}
+			err := mlog.AddWriterTarget(th.TestLogger, buffer, true, mlog.StdAll...)
+			require.NoError(t, err)
+
+			testsDir, _ := fileutils.FindDir("tests")
+			dir, err := os.MkdirTemp("", "import_test")
+			require.NoError(t, err)
+			defer os.RemoveAll(dir)
+
+			extractImportFile := func(filePath string) *os.File {
+				importFile, err2 := os.Open(filePath)
+				require.NoError(t, err2)
+				defer importFile.Close()
+
+				info, err2 := importFile.Stat()
+				require.NoError(t, err2)
+
+				paths, err2 := utils.UnzipToPath(importFile, info.Size(), dir)
+				require.NoError(t, err2)
+				require.NotEmpty(t, paths)
+
+				jsonFile, err2 := os.Open(filepath.Join(dir, "import.jsonl"))
+				require.NoError(t, err2)
+
+				return jsonFile
+			}
+
+			jsonFile := extractImportFile(filepath.Join(testsDir, "import_test.zip"))
+			defer jsonFile.Close()
+
+			appErr, _ := th.App.BulkImportWithPath(th.Context, jsonFile, nil, false, true, 1, dir)
+			require.Nil(t, appErr)
+
+			// delete one of the files
+			params := &model.SearchParams{Terms: "test3.png", SearchWithoutUserId: true}
+			results, err := th.App.Srv().Store().FileInfo().Search(th.Context, []*model.SearchParams{params}, "", "", 0, 20)
+			require.NoError(t, err)
+			require.Len(t, results.FileInfos, 1)
+
+			for _, info2 := range results.FileInfos {
+				err2 := th.App.RemoveFile(info2.Path)
+				require.Nil(t, err2)
+			}
+
+			exportFile, err := os.Create(filepath.Join(dir, "export.zip"))
+			require.NoError(t, err)
+			defer exportFile.Close()
+
+			opts := model.BulkExportOpts{
+				IncludeAttachments: true,
+				CreateArchive:      true,
+			}
+			appErr = th.App.BulkExport(th.Context, exportFile, dir, nil, opts)
+			// should not get an error for the missing file
+			require.Nil(t, appErr)
+
+			// should get a warning instead:
+			testlib.AssertLog(t, buffer, mlog.LvlWarn.Name, "Unable to export file attachment")
+		})
+	}
+}
+
 func TestBulkExport(t *testing.T) {
 	th := Setup(t)
 	testsDir, _ := fileutils.FindDir("tests")

server/cmd/mmctl/commands/importer/validate.go

@@ -815,7 +815,10 @@ func (v *Validator) validatePost(info ImportFileInfo, line imports.LineImportDat
 				continue
 			}
 
-			attachmentPath := path.Join("data", *attachment.Path)
+			attachmentPath := *attachment.Path
+			if _, ok := v.attachments[attachmentPath]; !ok {
+				attachmentPath = path.Join("data", *attachment.Path)
+			}
 
 			if _, ok := v.attachments[attachmentPath]; !ok {
 				helpful := ""
@@ -949,7 +952,10 @@ func (v *Validator) validateDirectPost(info ImportFileInfo, line imports.LineImp
 				continue
 			}
 
-			attachmentPath := path.Join("data", *attachment.Path)
+			attachmentPath := *attachment.Path
+			if _, ok := v.attachments[attachmentPath]; !ok {
+				attachmentPath = path.Join("data", *attachment.Path)
+			}
 
 			if _, ok := v.attachments[attachmentPath]; !ok {
 				helpful := ""