feat(preview): Implement scanning for previews

This work similarly to the move preview job to migrate the previews to
the new DB table and also reuse some code.

So when we are finding files in appdata/preview, try adding them to the
oc_previews table and delete them from the oc_filecache table.

Signed-off-by: Carl Schwan <carl.schwan@nextcloud.com>

apps/files/lib/Command/ScanAppData.php

@@ -12,6 +12,7 @@ use OC\DB\Connection;
 use OC\DB\ConnectionAdapter;
 use OC\Files\Utils\Scanner;
 use OC\ForbiddenException;
+use OC\Preview\Storage\StorageFactory;
 use OCP\EventDispatcher\IEventDispatcher;
 use OCP\Files\Folder;
 use OCP\Files\IRootFolder;
@@ -32,10 +33,12 @@ class ScanAppData extends Base {
 	protected int $foldersCounter = 0;
 
 	protected int $filesCounter = 0;
+	protected int $previewsCounter = -1;
 
 	public function __construct(
 		protected IRootFolder $rootFolder,
 		protected IConfig $config,
+		private StorageFactory $previewStorage,
 	) {
 		parent::__construct();
 	}
@@ -51,9 +54,12 @@ class ScanAppData extends Base {
 	}
 
 	protected function scanFiles(OutputInterface $output, string $folder): int {
-		if ($folder === 'preview') {
-			$output->writeln('<error>Scanning the preview folder is not supported.</error>');
-			return self::FAILURE;
+		if ($folder === 'preview' || $folder === '') {
+			$this->previewsCounter = $this->previewStorage->scan();
+
+			if ($folder === 'preview') {
+				return self::SUCCESS;
+			}
 		}
 
 		try {
@@ -139,7 +145,7 @@ class ScanAppData extends Base {
 		$this->initTools();
 
 		$exitCode = $this->scanFiles($output, $folder);
-		if ($exitCode === 0) {
+		if ($exitCode === self::SUCCESS) {
 			$this->presentStats($output);
 		}
 		return $exitCode;
@@ -167,7 +173,7 @@ class ScanAppData extends Base {
 	 *
 	 * @throws \ErrorException
 	 */
-	public function exceptionErrorHandler($severity, $message, $file, $line) {
+	public function exceptionErrorHandler(int $severity, string $message, string $file, int $line): void {
 		if (!(error_reporting() & $severity)) {
 			// This error code is not included in error_reporting
 			return;
@@ -178,10 +184,12 @@ class ScanAppData extends Base {
 	protected function presentStats(OutputInterface $output): void {
 		// Stop the timer
 		$this->execTime += microtime(true);
-
-		$headers = [
-			'Folders', 'Files', 'Elapsed time'
-		];
+		if ($this->previewsCounter !== -1) {
+			$headers[] = 'Previews';
+		}
+		$headers[] = 'Folders';
+		$headers[] = 'Files';
+		$headers[] = 'Elapsed time';
 
 		$this->showSummary($headers, null, $output);
 	}
@@ -192,14 +200,15 @@ class ScanAppData extends Base {
 	 * @param string[] $headers
 	 * @param string[] $rows
 	 */
-	protected function showSummary($headers, $rows, OutputInterface $output): void {
+	protected function showSummary(array $headers, ?array $rows, OutputInterface $output): void {
 		$niceDate = $this->formatExecTime();
 		if (!$rows) {
-			$rows = [
-				$this->foldersCounter,
-				$this->filesCounter,
-				$niceDate,
-			];
+			if ($this->previewsCounter !== -1) {
+				$rows[] = $this->previewsCounter;
+			}
+			$rows[] = $this->foldersCounter;
+			$rows[] = $this->filesCounter;
+			$rows[] = $niceDate;
 		}
 		$table = new Table($output);
 		$table

build/psalm-baseline.xml

@@ -1240,9 +1240,11 @@
       <code><![CDATA[listen]]></code>
       <code><![CDATA[listen]]></code>
     </DeprecatedMethod>
+    <InvalidArgument>
+      <code><![CDATA[[$this, 'exceptionErrorHandler']]]></code>
+    </InvalidArgument>
     <NullArgument>
       <code><![CDATA[null]]></code>
-      <code><![CDATA[null]]></code>
     </NullArgument>
   </file>
   <file src="apps/files/lib/Controller/DirectEditingController.php">

core/BackgroundJobs/MovePreviewJob.php

@@ -82,13 +82,18 @@ class MovePreviewJob extends TimedJob {
 				->setMaxResults(100);
 
 			$result = $qb->executeQuery();
+			$foundOldPreview = false;
 			while ($row = $result->fetch()) {
 				$pathSplit = explode('/', $row['path']);
 				assert(count($pathSplit) >= 2);
 				$fileId = $pathSplit[count($pathSplit) - 2];
 				array_pop($pathSplit);
-				$path = implode('/', $pathSplit);
 				$this->processPreviews($fileId, true);
+				$foundOldPreview = true;
+			}
+
+			if (!$foundOldPreview) {
+				break;
 			}
 
 			// Stop if execution time is more than one hour.
@@ -114,44 +119,21 @@ class MovePreviewJob extends TimedJob {
 		$folder = $this->appData->getFolder($internalPath);
 
 		/**
-		 * @var list<array{
-		 *     file: SimpleFile, width: int, height: int, crop: bool, max: bool, extension: string, mtime: int, size: int, version: ?int
-		 * }> $previewFiles
+		 * @var list<array{file: SimpleFile, preview: Preview}> $previewFiles
 		 */
 		$previewFiles = [];
 
 		foreach ($folder->getDirectoryListing() as $previewFile) {
 			/** @var SimpleFile $previewFile */
-			[0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName());
-			$nameSplit = explode('-', $baseName);
-
-			$offset = 0;
-			$version = null;
-			if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) {
-				$offset = 1;
-				$version = (int)$nameSplit[0];
-			}
-
-			$width = (int)$nameSplit[$offset + 0];
-			$height = (int)$nameSplit[$offset + 1];
-
-			$crop = false;
-			$max = false;
-			if (isset($nameSplit[$offset + 2])) {
-				$crop = $nameSplit[$offset + 2] === 'crop';
-				$max = $nameSplit[$offset + 2] === 'max';
-			}
+			$preview = Preview::fromPath($fileId . '/' . $previewFile->getName());
+			$preview->setSize($previewFile->getSize());
+			$preview->setMtime($previewFile->getMtime());
+			$preview->setOldFileId($previewFile->getId());
+			$preview->setEncrypted(false);
 
 			$previewFiles[] = [
 				'file' => $previewFile,
-				'width' => $width,
-				'height' => $height,
-				'crop' => $crop,
-				'version' => $version,
-				'max' => $max,
-				'extension' => $extension,
-				'size' => $previewFile->getSize(),
-				'mtime' => $previewFile->getMTime(),
+				'preview' => $preview,
 			];
 		}
 
@@ -166,27 +148,11 @@ class MovePreviewJob extends TimedJob {
 
 		if (count($result) > 0) {
 			foreach ($previewFiles as $previewFile) {
-				$preview = new Preview();
-				$preview->setFileId((int)$fileId);
+				$preview = $previewFile['preview'];
 				/** @var SimpleFile $file */
 				$file = $previewFile['file'];
-				$preview->setOldFileId($file->getId());
 				$preview->setStorageId($result[0]['storage']);
 				$preview->setEtag($result[0]['etag']);
-				$preview->setMtime($previewFile['mtime']);
-				$preview->setWidth($previewFile['width']);
-				$preview->setHeight($previewFile['height']);
-				$preview->setCropped($previewFile['crop']);
-				$preview->setVersion($previewFile['version']);
-				$preview->setMax($previewFile['max']);
-				$preview->setEncrypted(false);
-				$preview->setMimetype(match ($previewFile['extension']) {
-					'png' => IPreview::MIMETYPE_PNG,
-					'webp' => IPreview::MIMETYPE_WEBP,
-					'gif' => IPreview::MIMETYPE_GIF,
-					default => IPreview::MIMETYPE_JPEG,
-				});
-				$preview->setSize($previewFile['size']);
 				try {
 					$preview = $this->previewMapper->insert($preview);
 				} catch (Exception $e) {

lib/private/Files/Cache/Scanner.php

@@ -65,6 +65,8 @@ class Scanner extends BasicEmitter implements IScanner {
 
 	protected IDBConnection $connection;
 
+	private string $previewFolder;
+
 	public function __construct(\OC\Files\Storage\Storage $storage) {
 		$this->storage = $storage;
 		$this->storageId = $this->storage->getId();
@@ -75,6 +77,7 @@ class Scanner extends BasicEmitter implements IScanner {
 		$this->useTransactions = !$config->getValue('filescanner_no_transactions', false);
 		$this->lockingProvider = \OC::$server->get(ILockingProvider::class);
 		$this->connection = \OC::$server->get(IDBConnection::class);
+		$this->previewFolder = 'appdata_' . $config->getValue('instanceid', '') . '/preview';
 	}
 
 	/**
@@ -318,7 +321,6 @@ class Scanner extends BasicEmitter implements IScanner {
 
 		try {
 			$data = $this->scanFile($path, $reuse, -1, lock: $lock);
-
 			if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') {
 				$size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']);
 				$data['size'] = $size;
@@ -413,6 +415,11 @@ class Scanner extends BasicEmitter implements IScanner {
 		$size = 0;
 		$childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged);
 
+		if (str_starts_with($path, $this->previewFolder)) {
+			// Preview scanning is handled in LocalPreviewStorage
+			return 0;
+		}
+
 		foreach ($childQueue as $child => [$childId, $childSize]) {
 			// "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed
 			$childEtagChanged = false;

lib/private/Preview/Db/Preview.php

@@ -86,6 +86,41 @@ class Preview extends Entity {
 		$this->addType('version', Types::BIGINT);
 	}
 
+	public static function fromPath(string $path): Preview {
+		$preview = new self();
+		$preview->setFileId((int)basename(dirname($path)));
+
+		$fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION);
+
+		[0 => $baseName, 1 => $extension] = explode('.', $fileName);
+		$preview->setMimetype(match ($extension) {
+			'jpg' | 'jpeg' => IPreview::MIMETYPE_JPEG,
+			'png' => IPreview::MIMETYPE_PNG,
+			'gif' => IPreview::MIMETYPE_GIF,
+			'webp' => IPreview::MIMETYPE_WEBP,
+			default => IPreview::MIMETYPE_JPEG,
+		});
+		$nameSplit = explode('-', $baseName);
+
+		$offset = 0;
+		$preview->setVersion(null);
+		if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) {
+			$offset = 1;
+			$preview->setVersion((int)$nameSplit[0]);
+		}
+
+		$preview->setWidth((int)$nameSplit[$offset + 0]);
+		$preview->setHeight((int)$nameSplit[$offset + 1]);
+
+		$preview->setCropped(false);
+		$preview->setMax(false);
+		if (isset($nameSplit[$offset + 2])) {
+			$preview->setCropped($nameSplit[$offset + 2] === 'crop');
+			$preview->setMax($nameSplit[$offset + 2] === 'max');
+		}
+		return $preview;
+	}
+
 	public function getName(): string {
 		$path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight();
 		if ($this->isCropped()) {

lib/private/Preview/Storage/IPreviewStorage.php

@@ -36,4 +36,6 @@ interface IPreviewStorage {
 	 * @throw \Exception
 	 */
 	public function migratePreview(Preview $preview, SimpleFile $file): void;
+
+	public function scan(): int;
 }

lib/private/Preview/Storage/LocalPreviewStorage.php

@@ -14,7 +14,13 @@ use LogicException;
 use OC;
 use OC\Files\SimpleFS\SimpleFile;
 use OC\Preview\Db\Preview;
+use OC\Preview\Db\PreviewMapper;
+use OCP\DB\Exception;
+use OCP\IAppConfig;
 use OCP\IConfig;
+use OCP\IDBConnection;
+use RecursiveDirectoryIterator;
+use RecursiveIteratorIterator;
 
 class LocalPreviewStorage implements IPreviewStorage {
 	private readonly string $rootFolder;
@@ -22,6 +28,10 @@ class LocalPreviewStorage implements IPreviewStorage {
 
 	public function __construct(
 		private readonly IConfig $config,
+		private readonly PreviewMapper $previewMapper,
+		private readonly StorageFactory $previewStorage,
+		private readonly IAppConfig $appConfig,
+		private readonly IDBConnection $connection,
 	) {
 		$this->instanceId = $this->config->getSystemValueString('instanceid');
 		$this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data');
@@ -52,8 +62,9 @@ class LocalPreviewStorage implements IPreviewStorage {
 	}
 
 	private function createParentFiles(string $path): bool {
-		['dirname' => $dirname] = pathinfo($path);
-		return mkdir($dirname, recursive: true);
+		$dirname = dirname($path);
+		@mkdir($dirname, recursive: true);
+		return is_dir($dirname);
 	}
 
 	public function migratePreview(Preview $preview, SimpleFile $file): void {
@@ -75,4 +86,124 @@ class LocalPreviewStorage implements IPreviewStorage {
 			throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath);
 		}
 	}
+
+	public function scan(): int {
+		$checkForFileCache = !$this->appConfig->getValueBool('core', 'previewMovedDone');
+
+		$scanner = new RecursiveDirectoryIterator($this->getPreviewRootFolder());
+		$previewsFound = 0;
+		foreach (new RecursiveIteratorIterator($scanner) as $file) {
+			if ($file->isFile()) {
+				$preview = Preview::fromPath((string)$file);
+				try {
+					$preview->setSize($file->getSize());
+					$preview->setMtime($file->getMtime());
+					$preview->setEncrypted(false);
+
+					$qb = $this->connection->getQueryBuilder();
+					$result = $qb->select('*')
+						->from('filecache')
+						->where($qb->expr()->eq('fileid', $qb->createNamedParameter($preview->getFileId())))
+						->setMaxResults(1)
+						->runAcrossAllShards() // Unavoidable because we can't extract the storage_id from the preview name
+						->executeQuery()
+						->fetchAll();
+
+					if (empty($result)) {
+						// original file is deleted
+						@unlink($file->getRealPath());
+						continue;
+					}
+
+					if ($checkForFileCache) {
+						$relativePath = str_replace($this->rootFolder . '/', '', $file->getRealPath());
+						$rowAffected = $qb->delete('filecache')
+							->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($relativePath))))
+							->executeStatement();
+						if ($rowAffected > 0) {
+							$this->deleteParentsFromFileCache(dirname($relativePath));
+						}
+					}
+
+					$preview->setStorageId($result[0]['storage']);
+					$preview->setEtag($result[0]['etag']);
+
+					// try to insert, if that fails the preview is already in the DB
+					$this->previewMapper->insert($preview);
+
+					// Move old flat preview to new format
+					$this->previewStorage->migratePreview($preview, $file);
+				} catch (Exception $e) {
+					if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
+						throw $e;
+					}
+				}
+				$previewsFound++;
+			}
+		}
+
+		return $previewsFound;
+	}
+
+	private function deleteParentsFromFileCache(string $dirname): void {
+		$qb = $this->connection->getQueryBuilder();
+
+		$result = $qb->select('fileid', 'path', 'storage', 'parent')
+			->from('filecache')
+			->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($dirname))))
+			->setMaxResults(1)
+			->runAcrossAllShards()
+			->executeQuery()
+			->fetchAll();
+
+		if (empty($result)) {
+			return;
+		}
+
+		$this->connection->beginTransaction();
+
+		$parentId = $result[0]['parent'];
+		$fileId = $result[0]['fileid'];
+		$storage = $result[0]['storage'];
+
+		try {
+			while (true) {
+				$qb = $this->connection->getQueryBuilder();
+				$childs = $qb->select('fileid', 'path', 'storage')
+					->from('filecache')
+					->where($qb->expr()->eq('parent', $qb->createNamedParameter($fileId)))
+					->hintShardKey('storage', $storage)
+					->executeQuery()
+					->fetchAll();
+
+				if (!empty($childs)) {
+					break;
+				}
+
+				$qb = $this->connection->getQueryBuilder();
+				$qb->delete('filecache')
+					->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId)))
+					->hintShardKey('storage', $result[0]['storage'])
+					->executeStatement();
+
+				$qb = $this->connection->getQueryBuilder();
+				$result = $qb->select('fileid', 'path', 'storage', 'parent')
+					->from('filecache')
+					->where($qb->expr()->eq('fileid', $qb->createNamedParameter($parentId)))
+					->setMaxResults(1)
+					->hintShardKey('storage', $storage)
+					->executeQuery()
+					->fetchAll();
+
+				if (empty($result)) {
+					break;
+				}
+
+				$fileId = $parentId;
+				$parentId = $result[0]['parent'];
+			}
+		} finally {
+			$this->connection->commit();
+		}
+	}
 }

lib/private/Preview/Storage/ObjectStorePreviewStorage.php

@@ -149,4 +149,8 @@ class ObjectStorePreviewStorage implements IPreviewStorage {
 			return 'uri:oid:preview:';
 		}
 	}
+
+	public function scan(): int {
+		return 0;
+	}
 }

lib/private/Preview/Storage/StorageFactory.php

@@ -15,6 +15,7 @@ use OC\Files\SimpleFS\SimpleFile;
 use OC\Preview\Db\Preview;
 use OC\Preview\Db\PreviewMapper;
 use OCP\IConfig;
+use OCP\Server;
 
 class StorageFactory implements IPreviewStorage {
 	private ?IPreviewStorage $backend = null;
@@ -46,7 +47,7 @@ class StorageFactory implements IPreviewStorage {
 		if ($this->objectStoreConfig->hasObjectStore()) {
 			$this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config, $this->previewMapper);
 		} else {
-			$this->backend = new LocalPreviewStorage($this->config);
+			$this->backend = Server::get(LocalPreviewStorage::class);
 		}
 
 		return $this->backend;
@@ -55,4 +56,8 @@ class StorageFactory implements IPreviewStorage {
 	public function migratePreview(Preview $preview, SimpleFile $file): void {
 		$this->getBackend()->migratePreview($preview, $file);
 	}
+
+	public function scan(): int {
+		return $this->getBackend()->scan();
+	}
 }

lib/public/Preview/IVersionedPreviewFile.php

@@ -11,15 +11,15 @@ namespace OCP\Preview;
  * Marks files that should keep multiple preview "versions" for the same file id
  *
  * Examples of this are files where the storage backend provides versioning, for those
- * files, we dont have fileids for the different versions but still need to be able to generate
+ * files, we don't have fileIds for the different versions but still need to be able to generate
  * previews for all versions
  *
  * @since 17.0.0
  */
 interface IVersionedPreviewFile {
 	/**
-	 * @return string
+	 * @return numeric
 	 * @since 17.0.0
 	 */
-	public function getPreviewVersion(): string;
+	public function getPreviewVersion();
 }