From bfc7d5dd9fac04124db1b65e9f22d9f33a5e5d12 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 25 Sep 2025 14:25:47 +0200 Subject: [PATCH] feat(preview): Implement scanning for previews This work similarly to the move preview job to migrate the previews to the new DB table and also reuse some code. So when we are finding files in appdata/preview, try adding them to the oc_previews table and delete them from the oc_filecache table. Signed-off-by: Carl Schwan --- apps/files/lib/Command/ScanAppData.php | 39 +++-- build/psalm-baseline.xml | 4 +- core/BackgroundJobs/MovePreviewJob.php | 62 ++------ lib/private/Files/Cache/Scanner.php | 9 +- lib/private/Preview/Db/Preview.php | 35 +++++ .../Preview/Storage/IPreviewStorage.php | 2 + .../Preview/Storage/LocalPreviewStorage.php | 135 +++++++++++++++++- .../Storage/ObjectStorePreviewStorage.php | 4 + .../Preview/Storage/StorageFactory.php | 7 +- lib/public/Preview/IVersionedPreviewFile.php | 6 +- 10 files changed, 232 insertions(+), 71 deletions(-) diff --git a/apps/files/lib/Command/ScanAppData.php b/apps/files/lib/Command/ScanAppData.php index 385e0624b3a..23604a82df9 100644 --- a/apps/files/lib/Command/ScanAppData.php +++ b/apps/files/lib/Command/ScanAppData.php @@ -12,6 +12,7 @@ use OC\DB\Connection; use OC\DB\ConnectionAdapter; use OC\Files\Utils\Scanner; use OC\ForbiddenException; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Folder; use OCP\Files\IRootFolder; @@ -32,10 +33,12 @@ class ScanAppData extends Base { protected int $foldersCounter = 0; protected int $filesCounter = 0; + protected int $previewsCounter = -1; public function __construct( protected IRootFolder $rootFolder, protected IConfig $config, + private StorageFactory $previewStorage, ) { parent::__construct(); } @@ -51,9 +54,12 @@ class ScanAppData extends Base { } protected function scanFiles(OutputInterface $output, string $folder): int { - if ($folder === 'preview') { - $output->writeln('Scanning the preview folder is not supported.'); - return self::FAILURE; + if ($folder === 'preview' || $folder === '') { + $this->previewsCounter = $this->previewStorage->scan(); + + if ($folder === 'preview') { + return self::SUCCESS; + } } try { @@ -139,7 +145,7 @@ class ScanAppData extends Base { $this->initTools(); $exitCode = $this->scanFiles($output, $folder); - if ($exitCode === 0) { + if ($exitCode === self::SUCCESS) { $this->presentStats($output); } return $exitCode; @@ -167,7 +173,7 @@ class ScanAppData extends Base { * * @throws \ErrorException */ - public function exceptionErrorHandler($severity, $message, $file, $line) { + public function exceptionErrorHandler(int $severity, string $message, string $file, int $line): void { if (!(error_reporting() & $severity)) { // This error code is not included in error_reporting return; @@ -178,10 +184,12 @@ class ScanAppData extends Base { protected function presentStats(OutputInterface $output): void { // Stop the timer $this->execTime += microtime(true); - - $headers = [ - 'Folders', 'Files', 'Elapsed time' - ]; + if ($this->previewsCounter !== -1) { + $headers[] = 'Previews'; + } + $headers[] = 'Folders'; + $headers[] = 'Files'; + $headers[] = 'Elapsed time'; $this->showSummary($headers, null, $output); } @@ -192,14 +200,15 @@ class ScanAppData extends Base { * @param string[] $headers * @param string[] $rows */ - protected function showSummary($headers, $rows, OutputInterface $output): void { + protected function showSummary(array $headers, ?array $rows, OutputInterface $output): void { $niceDate = $this->formatExecTime(); if (!$rows) { - $rows = [ - $this->foldersCounter, - $this->filesCounter, - $niceDate, - ]; + if ($this->previewsCounter !== -1) { + $rows[] = $this->previewsCounter; + } + $rows[] = $this->foldersCounter; + $rows[] = $this->filesCounter; + $rows[] = $niceDate; } $table = new Table($output); $table diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 21baeb9a33b..8be3f1e9c1d 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -1240,9 +1240,11 @@ + + + - diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 22db7aea90d..764561a1ba8 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -82,13 +82,18 @@ class MovePreviewJob extends TimedJob { ->setMaxResults(100); $result = $qb->executeQuery(); + $foundOldPreview = false; while ($row = $result->fetch()) { $pathSplit = explode('/', $row['path']); assert(count($pathSplit) >= 2); $fileId = $pathSplit[count($pathSplit) - 2]; array_pop($pathSplit); - $path = implode('/', $pathSplit); $this->processPreviews($fileId, true); + $foundOldPreview = true; + } + + if (!$foundOldPreview) { + break; } // Stop if execution time is more than one hour. @@ -114,44 +119,21 @@ class MovePreviewJob extends TimedJob { $folder = $this->appData->getFolder($internalPath); /** - * @var list $previewFiles + * @var list $previewFiles */ $previewFiles = []; foreach ($folder->getDirectoryListing() as $previewFile) { /** @var SimpleFile $previewFile */ - [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); - $nameSplit = explode('-', $baseName); - - $offset = 0; - $version = null; - if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { - $offset = 1; - $version = (int)$nameSplit[0]; - } - - $width = (int)$nameSplit[$offset + 0]; - $height = (int)$nameSplit[$offset + 1]; - - $crop = false; - $max = false; - if (isset($nameSplit[$offset + 2])) { - $crop = $nameSplit[$offset + 2] === 'crop'; - $max = $nameSplit[$offset + 2] === 'max'; - } + $preview = Preview::fromPath($fileId . '/' . $previewFile->getName()); + $preview->setSize($previewFile->getSize()); + $preview->setMtime($previewFile->getMtime()); + $preview->setOldFileId($previewFile->getId()); + $preview->setEncrypted(false); $previewFiles[] = [ 'file' => $previewFile, - 'width' => $width, - 'height' => $height, - 'crop' => $crop, - 'version' => $version, - 'max' => $max, - 'extension' => $extension, - 'size' => $previewFile->getSize(), - 'mtime' => $previewFile->getMTime(), + 'preview' => $preview, ]; } @@ -166,27 +148,11 @@ class MovePreviewJob extends TimedJob { if (count($result) > 0) { foreach ($previewFiles as $previewFile) { - $preview = new Preview(); - $preview->setFileId((int)$fileId); + $preview = $previewFile['preview']; /** @var SimpleFile $file */ $file = $previewFile['file']; - $preview->setOldFileId($file->getId()); $preview->setStorageId($result[0]['storage']); $preview->setEtag($result[0]['etag']); - $preview->setMtime($previewFile['mtime']); - $preview->setWidth($previewFile['width']); - $preview->setHeight($previewFile['height']); - $preview->setCropped($previewFile['crop']); - $preview->setVersion($previewFile['version']); - $preview->setMax($previewFile['max']); - $preview->setEncrypted(false); - $preview->setMimetype(match ($previewFile['extension']) { - 'png' => IPreview::MIMETYPE_PNG, - 'webp' => IPreview::MIMETYPE_WEBP, - 'gif' => IPreview::MIMETYPE_GIF, - default => IPreview::MIMETYPE_JPEG, - }); - $preview->setSize($previewFile['size']); try { $preview = $this->previewMapper->insert($preview); } catch (Exception $e) { diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index b067f70b8cb..75ca9da0abe 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -65,6 +65,8 @@ class Scanner extends BasicEmitter implements IScanner { protected IDBConnection $connection; + private string $previewFolder; + public function __construct(\OC\Files\Storage\Storage $storage) { $this->storage = $storage; $this->storageId = $this->storage->getId(); @@ -75,6 +77,7 @@ class Scanner extends BasicEmitter implements IScanner { $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); $this->lockingProvider = \OC::$server->get(ILockingProvider::class); $this->connection = \OC::$server->get(IDBConnection::class); + $this->previewFolder = 'appdata_' . $config->getValue('instanceid', '') . '/preview'; } /** @@ -318,7 +321,6 @@ class Scanner extends BasicEmitter implements IScanner { try { $data = $this->scanFile($path, $reuse, -1, lock: $lock); - if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); $data['size'] = $size; @@ -413,6 +415,11 @@ class Scanner extends BasicEmitter implements IScanner { $size = 0; $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); + if (str_starts_with($path, $this->previewFolder)) { + // Preview scanning is handled in LocalPreviewStorage + return 0; + } + foreach ($childQueue as $child => [$childId, $childSize]) { // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed $childEtagChanged = false; diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index b6f2ef2942e..3f10328fd8e 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -86,6 +86,41 @@ class Preview extends Entity { $this->addType('version', Types::BIGINT); } + public static function fromPath(string $path): Preview { + $preview = new self(); + $preview->setFileId((int)basename(dirname($path))); + + $fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION); + + [0 => $baseName, 1 => $extension] = explode('.', $fileName); + $preview->setMimetype(match ($extension) { + 'jpg' | 'jpeg' => IPreview::MIMETYPE_JPEG, + 'png' => IPreview::MIMETYPE_PNG, + 'gif' => IPreview::MIMETYPE_GIF, + 'webp' => IPreview::MIMETYPE_WEBP, + default => IPreview::MIMETYPE_JPEG, + }); + $nameSplit = explode('-', $baseName); + + $offset = 0; + $preview->setVersion(null); + if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { + $offset = 1; + $preview->setVersion((int)$nameSplit[0]); + } + + $preview->setWidth((int)$nameSplit[$offset + 0]); + $preview->setHeight((int)$nameSplit[$offset + 1]); + + $preview->setCropped(false); + $preview->setMax(false); + if (isset($nameSplit[$offset + 2])) { + $preview->setCropped($nameSplit[$offset + 2] === 'crop'); + $preview->setMax($nameSplit[$offset + 2] === 'max'); + } + return $preview; + } + public function getName(): string { $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); if ($this->isCropped()) { diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index 933fc2850b0..56464326838 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -36,4 +36,6 @@ interface IPreviewStorage { * @throw \Exception */ public function migratePreview(Preview $preview, SimpleFile $file): void; + + public function scan(): int; } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 59a4248e92b..3bd1fe2ccb9 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -14,7 +14,13 @@ use LogicException; use OC; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; +use OCP\DB\Exception; +use OCP\IAppConfig; use OCP\IConfig; +use OCP\IDBConnection; +use RecursiveDirectoryIterator; +use RecursiveIteratorIterator; class LocalPreviewStorage implements IPreviewStorage { private readonly string $rootFolder; @@ -22,6 +28,10 @@ class LocalPreviewStorage implements IPreviewStorage { public function __construct( private readonly IConfig $config, + private readonly PreviewMapper $previewMapper, + private readonly StorageFactory $previewStorage, + private readonly IAppConfig $appConfig, + private readonly IDBConnection $connection, ) { $this->instanceId = $this->config->getSystemValueString('instanceid'); $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); @@ -52,8 +62,9 @@ class LocalPreviewStorage implements IPreviewStorage { } private function createParentFiles(string $path): bool { - ['dirname' => $dirname] = pathinfo($path); - return mkdir($dirname, recursive: true); + $dirname = dirname($path); + @mkdir($dirname, recursive: true); + return is_dir($dirname); } public function migratePreview(Preview $preview, SimpleFile $file): void { @@ -75,4 +86,124 @@ class LocalPreviewStorage implements IPreviewStorage { throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); } } + + public function scan(): int { + $checkForFileCache = !$this->appConfig->getValueBool('core', 'previewMovedDone'); + + $scanner = new RecursiveDirectoryIterator($this->getPreviewRootFolder()); + $previewsFound = 0; + foreach (new RecursiveIteratorIterator($scanner) as $file) { + if ($file->isFile()) { + $preview = Preview::fromPath((string)$file); + try { + $preview->setSize($file->getSize()); + $preview->setMtime($file->getMtime()); + $preview->setEncrypted(false); + + $qb = $this->connection->getQueryBuilder(); + $result = $qb->select('*') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($preview->getFileId()))) + ->setMaxResults(1) + ->runAcrossAllShards() // Unavoidable because we can't extract the storage_id from the preview name + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + // original file is deleted + @unlink($file->getRealPath()); + continue; + } + + if ($checkForFileCache) { + $relativePath = str_replace($this->rootFolder . '/', '', $file->getRealPath()); + $rowAffected = $qb->delete('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($relativePath)))) + ->executeStatement(); + if ($rowAffected > 0) { + $this->deleteParentsFromFileCache(dirname($relativePath)); + } + } + + $preview->setStorageId($result[0]['storage']); + $preview->setEtag($result[0]['etag']); + + // try to insert, if that fails the preview is already in the DB + $this->previewMapper->insert($preview); + + // Move old flat preview to new format + $this->previewStorage->migratePreview($preview, $file); + } catch (Exception $e) { + if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + throw $e; + } + } + $previewsFound++; + } + } + + return $previewsFound; + } + + private function deleteParentsFromFileCache(string $dirname): void { + $qb = $this->connection->getQueryBuilder(); + + $result = $qb->select('fileid', 'path', 'storage', 'parent') + ->from('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($dirname)))) + ->setMaxResults(1) + ->runAcrossAllShards() + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + return; + } + + $this->connection->beginTransaction(); + + $parentId = $result[0]['parent']; + $fileId = $result[0]['fileid']; + $storage = $result[0]['storage']; + + try { + while (true) { + $qb = $this->connection->getQueryBuilder(); + $childs = $qb->select('fileid', 'path', 'storage') + ->from('filecache') + ->where($qb->expr()->eq('parent', $qb->createNamedParameter($fileId))) + ->hintShardKey('storage', $storage) + ->executeQuery() + ->fetchAll(); + + if (!empty($childs)) { + break; + } + + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))) + ->hintShardKey('storage', $result[0]['storage']) + ->executeStatement(); + + $qb = $this->connection->getQueryBuilder(); + $result = $qb->select('fileid', 'path', 'storage', 'parent') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($parentId))) + ->setMaxResults(1) + ->hintShardKey('storage', $storage) + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + break; + } + + $fileId = $parentId; + $parentId = $result[0]['parent']; + } + } finally { + $this->connection->commit(); + } + } } diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 0a735e5a912..faa63763779 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -149,4 +149,8 @@ class ObjectStorePreviewStorage implements IPreviewStorage { return 'uri:oid:preview:'; } } + + public function scan(): int { + return 0; + } } diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 3051ad5869f..3438534d07e 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -15,6 +15,7 @@ use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\IConfig; +use OCP\Server; class StorageFactory implements IPreviewStorage { private ?IPreviewStorage $backend = null; @@ -46,7 +47,7 @@ class StorageFactory implements IPreviewStorage { if ($this->objectStoreConfig->hasObjectStore()) { $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config, $this->previewMapper); } else { - $this->backend = new LocalPreviewStorage($this->config); + $this->backend = Server::get(LocalPreviewStorage::class); } return $this->backend; @@ -55,4 +56,8 @@ class StorageFactory implements IPreviewStorage { public function migratePreview(Preview $preview, SimpleFile $file): void { $this->getBackend()->migratePreview($preview, $file); } + + public function scan(): int { + return $this->getBackend()->scan(); + } } diff --git a/lib/public/Preview/IVersionedPreviewFile.php b/lib/public/Preview/IVersionedPreviewFile.php index 9266b1ac067..7d68fe8d15e 100644 --- a/lib/public/Preview/IVersionedPreviewFile.php +++ b/lib/public/Preview/IVersionedPreviewFile.php @@ -11,15 +11,15 @@ namespace OCP\Preview; * Marks files that should keep multiple preview "versions" for the same file id * * Examples of this are files where the storage backend provides versioning, for those - * files, we dont have fileids for the different versions but still need to be able to generate + * files, we don't have fileIds for the different versions but still need to be able to generate * previews for all versions * * @since 17.0.0 */ interface IVersionedPreviewFile { /** - * @return string + * @return numeric * @since 17.0.0 */ - public function getPreviewVersion(): string; + public function getPreviewVersion(); }