diff --git a/apps/files/lib/Command/ScanAppData.php b/apps/files/lib/Command/ScanAppData.php index 385e0624b3a..23604a82df9 100644 --- a/apps/files/lib/Command/ScanAppData.php +++ b/apps/files/lib/Command/ScanAppData.php @@ -12,6 +12,7 @@ use OC\DB\Connection; use OC\DB\ConnectionAdapter; use OC\Files\Utils\Scanner; use OC\ForbiddenException; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Folder; use OCP\Files\IRootFolder; @@ -32,10 +33,12 @@ class ScanAppData extends Base { protected int $foldersCounter = 0; protected int $filesCounter = 0; + protected int $previewsCounter = -1; public function __construct( protected IRootFolder $rootFolder, protected IConfig $config, + private StorageFactory $previewStorage, ) { parent::__construct(); } @@ -51,9 +54,12 @@ class ScanAppData extends Base { } protected function scanFiles(OutputInterface $output, string $folder): int { - if ($folder === 'preview') { - $output->writeln('Scanning the preview folder is not supported.'); - return self::FAILURE; + if ($folder === 'preview' || $folder === '') { + $this->previewsCounter = $this->previewStorage->scan(); + + if ($folder === 'preview') { + return self::SUCCESS; + } } try { @@ -139,7 +145,7 @@ class ScanAppData extends Base { $this->initTools(); $exitCode = $this->scanFiles($output, $folder); - if ($exitCode === 0) { + if ($exitCode === self::SUCCESS) { $this->presentStats($output); } return $exitCode; @@ -167,7 +173,7 @@ class ScanAppData extends Base { * * @throws \ErrorException */ - public function exceptionErrorHandler($severity, $message, $file, $line) { + public function exceptionErrorHandler(int $severity, string $message, string $file, int $line): void { if (!(error_reporting() & $severity)) { // This error code is not included in error_reporting return; @@ -178,10 +184,12 @@ class ScanAppData extends Base { protected function presentStats(OutputInterface $output): void { // Stop the timer $this->execTime += microtime(true); - - $headers = [ - 'Folders', 'Files', 'Elapsed time' - ]; + if ($this->previewsCounter !== -1) { + $headers[] = 'Previews'; + } + $headers[] = 'Folders'; + $headers[] = 'Files'; + $headers[] = 'Elapsed time'; $this->showSummary($headers, null, $output); } @@ -192,14 +200,15 @@ class ScanAppData extends Base { * @param string[] $headers * @param string[] $rows */ - protected function showSummary($headers, $rows, OutputInterface $output): void { + protected function showSummary(array $headers, ?array $rows, OutputInterface $output): void { $niceDate = $this->formatExecTime(); if (!$rows) { - $rows = [ - $this->foldersCounter, - $this->filesCounter, - $niceDate, - ]; + if ($this->previewsCounter !== -1) { + $rows[] = $this->previewsCounter; + } + $rows[] = $this->foldersCounter; + $rows[] = $this->filesCounter; + $rows[] = $niceDate; } $table = new Table($output); $table diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 21baeb9a33b..8be3f1e9c1d 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -1240,9 +1240,11 @@ + + + - diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 22db7aea90d..764561a1ba8 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -82,13 +82,18 @@ class MovePreviewJob extends TimedJob { ->setMaxResults(100); $result = $qb->executeQuery(); + $foundOldPreview = false; while ($row = $result->fetch()) { $pathSplit = explode('/', $row['path']); assert(count($pathSplit) >= 2); $fileId = $pathSplit[count($pathSplit) - 2]; array_pop($pathSplit); - $path = implode('/', $pathSplit); $this->processPreviews($fileId, true); + $foundOldPreview = true; + } + + if (!$foundOldPreview) { + break; } // Stop if execution time is more than one hour. @@ -114,44 +119,21 @@ class MovePreviewJob extends TimedJob { $folder = $this->appData->getFolder($internalPath); /** - * @var list $previewFiles + * @var list $previewFiles */ $previewFiles = []; foreach ($folder->getDirectoryListing() as $previewFile) { /** @var SimpleFile $previewFile */ - [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); - $nameSplit = explode('-', $baseName); - - $offset = 0; - $version = null; - if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { - $offset = 1; - $version = (int)$nameSplit[0]; - } - - $width = (int)$nameSplit[$offset + 0]; - $height = (int)$nameSplit[$offset + 1]; - - $crop = false; - $max = false; - if (isset($nameSplit[$offset + 2])) { - $crop = $nameSplit[$offset + 2] === 'crop'; - $max = $nameSplit[$offset + 2] === 'max'; - } + $preview = Preview::fromPath($fileId . '/' . $previewFile->getName()); + $preview->setSize($previewFile->getSize()); + $preview->setMtime($previewFile->getMtime()); + $preview->setOldFileId($previewFile->getId()); + $preview->setEncrypted(false); $previewFiles[] = [ 'file' => $previewFile, - 'width' => $width, - 'height' => $height, - 'crop' => $crop, - 'version' => $version, - 'max' => $max, - 'extension' => $extension, - 'size' => $previewFile->getSize(), - 'mtime' => $previewFile->getMTime(), + 'preview' => $preview, ]; } @@ -166,27 +148,11 @@ class MovePreviewJob extends TimedJob { if (count($result) > 0) { foreach ($previewFiles as $previewFile) { - $preview = new Preview(); - $preview->setFileId((int)$fileId); + $preview = $previewFile['preview']; /** @var SimpleFile $file */ $file = $previewFile['file']; - $preview->setOldFileId($file->getId()); $preview->setStorageId($result[0]['storage']); $preview->setEtag($result[0]['etag']); - $preview->setMtime($previewFile['mtime']); - $preview->setWidth($previewFile['width']); - $preview->setHeight($previewFile['height']); - $preview->setCropped($previewFile['crop']); - $preview->setVersion($previewFile['version']); - $preview->setMax($previewFile['max']); - $preview->setEncrypted(false); - $preview->setMimetype(match ($previewFile['extension']) { - 'png' => IPreview::MIMETYPE_PNG, - 'webp' => IPreview::MIMETYPE_WEBP, - 'gif' => IPreview::MIMETYPE_GIF, - default => IPreview::MIMETYPE_JPEG, - }); - $preview->setSize($previewFile['size']); try { $preview = $this->previewMapper->insert($preview); } catch (Exception $e) { diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index b067f70b8cb..75ca9da0abe 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -65,6 +65,8 @@ class Scanner extends BasicEmitter implements IScanner { protected IDBConnection $connection; + private string $previewFolder; + public function __construct(\OC\Files\Storage\Storage $storage) { $this->storage = $storage; $this->storageId = $this->storage->getId(); @@ -75,6 +77,7 @@ class Scanner extends BasicEmitter implements IScanner { $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); $this->lockingProvider = \OC::$server->get(ILockingProvider::class); $this->connection = \OC::$server->get(IDBConnection::class); + $this->previewFolder = 'appdata_' . $config->getValue('instanceid', '') . '/preview'; } /** @@ -318,7 +321,6 @@ class Scanner extends BasicEmitter implements IScanner { try { $data = $this->scanFile($path, $reuse, -1, lock: $lock); - if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); $data['size'] = $size; @@ -413,6 +415,11 @@ class Scanner extends BasicEmitter implements IScanner { $size = 0; $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); + if (str_starts_with($path, $this->previewFolder)) { + // Preview scanning is handled in LocalPreviewStorage + return 0; + } + foreach ($childQueue as $child => [$childId, $childSize]) { // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed $childEtagChanged = false; diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index b6f2ef2942e..3f10328fd8e 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -86,6 +86,41 @@ class Preview extends Entity { $this->addType('version', Types::BIGINT); } + public static function fromPath(string $path): Preview { + $preview = new self(); + $preview->setFileId((int)basename(dirname($path))); + + $fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION); + + [0 => $baseName, 1 => $extension] = explode('.', $fileName); + $preview->setMimetype(match ($extension) { + 'jpg' | 'jpeg' => IPreview::MIMETYPE_JPEG, + 'png' => IPreview::MIMETYPE_PNG, + 'gif' => IPreview::MIMETYPE_GIF, + 'webp' => IPreview::MIMETYPE_WEBP, + default => IPreview::MIMETYPE_JPEG, + }); + $nameSplit = explode('-', $baseName); + + $offset = 0; + $preview->setVersion(null); + if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { + $offset = 1; + $preview->setVersion((int)$nameSplit[0]); + } + + $preview->setWidth((int)$nameSplit[$offset + 0]); + $preview->setHeight((int)$nameSplit[$offset + 1]); + + $preview->setCropped(false); + $preview->setMax(false); + if (isset($nameSplit[$offset + 2])) { + $preview->setCropped($nameSplit[$offset + 2] === 'crop'); + $preview->setMax($nameSplit[$offset + 2] === 'max'); + } + return $preview; + } + public function getName(): string { $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); if ($this->isCropped()) { diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index 933fc2850b0..56464326838 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -36,4 +36,6 @@ interface IPreviewStorage { * @throw \Exception */ public function migratePreview(Preview $preview, SimpleFile $file): void; + + public function scan(): int; } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 59a4248e92b..3bd1fe2ccb9 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -14,7 +14,13 @@ use LogicException; use OC; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; +use OCP\DB\Exception; +use OCP\IAppConfig; use OCP\IConfig; +use OCP\IDBConnection; +use RecursiveDirectoryIterator; +use RecursiveIteratorIterator; class LocalPreviewStorage implements IPreviewStorage { private readonly string $rootFolder; @@ -22,6 +28,10 @@ class LocalPreviewStorage implements IPreviewStorage { public function __construct( private readonly IConfig $config, + private readonly PreviewMapper $previewMapper, + private readonly StorageFactory $previewStorage, + private readonly IAppConfig $appConfig, + private readonly IDBConnection $connection, ) { $this->instanceId = $this->config->getSystemValueString('instanceid'); $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); @@ -52,8 +62,9 @@ class LocalPreviewStorage implements IPreviewStorage { } private function createParentFiles(string $path): bool { - ['dirname' => $dirname] = pathinfo($path); - return mkdir($dirname, recursive: true); + $dirname = dirname($path); + @mkdir($dirname, recursive: true); + return is_dir($dirname); } public function migratePreview(Preview $preview, SimpleFile $file): void { @@ -75,4 +86,124 @@ class LocalPreviewStorage implements IPreviewStorage { throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); } } + + public function scan(): int { + $checkForFileCache = !$this->appConfig->getValueBool('core', 'previewMovedDone'); + + $scanner = new RecursiveDirectoryIterator($this->getPreviewRootFolder()); + $previewsFound = 0; + foreach (new RecursiveIteratorIterator($scanner) as $file) { + if ($file->isFile()) { + $preview = Preview::fromPath((string)$file); + try { + $preview->setSize($file->getSize()); + $preview->setMtime($file->getMtime()); + $preview->setEncrypted(false); + + $qb = $this->connection->getQueryBuilder(); + $result = $qb->select('*') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($preview->getFileId()))) + ->setMaxResults(1) + ->runAcrossAllShards() // Unavoidable because we can't extract the storage_id from the preview name + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + // original file is deleted + @unlink($file->getRealPath()); + continue; + } + + if ($checkForFileCache) { + $relativePath = str_replace($this->rootFolder . '/', '', $file->getRealPath()); + $rowAffected = $qb->delete('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($relativePath)))) + ->executeStatement(); + if ($rowAffected > 0) { + $this->deleteParentsFromFileCache(dirname($relativePath)); + } + } + + $preview->setStorageId($result[0]['storage']); + $preview->setEtag($result[0]['etag']); + + // try to insert, if that fails the preview is already in the DB + $this->previewMapper->insert($preview); + + // Move old flat preview to new format + $this->previewStorage->migratePreview($preview, $file); + } catch (Exception $e) { + if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + throw $e; + } + } + $previewsFound++; + } + } + + return $previewsFound; + } + + private function deleteParentsFromFileCache(string $dirname): void { + $qb = $this->connection->getQueryBuilder(); + + $result = $qb->select('fileid', 'path', 'storage', 'parent') + ->from('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($dirname)))) + ->setMaxResults(1) + ->runAcrossAllShards() + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + return; + } + + $this->connection->beginTransaction(); + + $parentId = $result[0]['parent']; + $fileId = $result[0]['fileid']; + $storage = $result[0]['storage']; + + try { + while (true) { + $qb = $this->connection->getQueryBuilder(); + $childs = $qb->select('fileid', 'path', 'storage') + ->from('filecache') + ->where($qb->expr()->eq('parent', $qb->createNamedParameter($fileId))) + ->hintShardKey('storage', $storage) + ->executeQuery() + ->fetchAll(); + + if (!empty($childs)) { + break; + } + + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))) + ->hintShardKey('storage', $result[0]['storage']) + ->executeStatement(); + + $qb = $this->connection->getQueryBuilder(); + $result = $qb->select('fileid', 'path', 'storage', 'parent') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($parentId))) + ->setMaxResults(1) + ->hintShardKey('storage', $storage) + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + break; + } + + $fileId = $parentId; + $parentId = $result[0]['parent']; + } + } finally { + $this->connection->commit(); + } + } } diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 0a735e5a912..faa63763779 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -149,4 +149,8 @@ class ObjectStorePreviewStorage implements IPreviewStorage { return 'uri:oid:preview:'; } } + + public function scan(): int { + return 0; + } } diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 3051ad5869f..3438534d07e 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -15,6 +15,7 @@ use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\IConfig; +use OCP\Server; class StorageFactory implements IPreviewStorage { private ?IPreviewStorage $backend = null; @@ -46,7 +47,7 @@ class StorageFactory implements IPreviewStorage { if ($this->objectStoreConfig->hasObjectStore()) { $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config, $this->previewMapper); } else { - $this->backend = new LocalPreviewStorage($this->config); + $this->backend = Server::get(LocalPreviewStorage::class); } return $this->backend; @@ -55,4 +56,8 @@ class StorageFactory implements IPreviewStorage { public function migratePreview(Preview $preview, SimpleFile $file): void { $this->getBackend()->migratePreview($preview, $file); } + + public function scan(): int { + return $this->getBackend()->scan(); + } } diff --git a/lib/public/Preview/IVersionedPreviewFile.php b/lib/public/Preview/IVersionedPreviewFile.php index 9266b1ac067..7d68fe8d15e 100644 --- a/lib/public/Preview/IVersionedPreviewFile.php +++ b/lib/public/Preview/IVersionedPreviewFile.php @@ -11,15 +11,15 @@ namespace OCP\Preview; * Marks files that should keep multiple preview "versions" for the same file id * * Examples of this are files where the storage backend provides versioning, for those - * files, we dont have fileids for the different versions but still need to be able to generate + * files, we don't have fileIds for the different versions but still need to be able to generate * previews for all versions * * @since 17.0.0 */ interface IVersionedPreviewFile { /** - * @return string + * @return numeric * @since 17.0.0 */ - public function getPreviewVersion(): string; + public function getPreviewVersion(); }