From 18fbacdd8d519e88e8cc53438a6209428f90085d Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Wed, 20 Aug 2025 17:34:07 +0200 Subject: [PATCH 01/14] perf(preview): Split preview data to new table The new oc_previews table is optimized for storing previews and should decrease significantly the space taken by previews in the filecache table. This attend to reuse the IObjectStore abstraction over S3/Swift/Azure but currently only support one single bucket configuration. Signed-off-by: Carl Schwan --- .../Version33000Date20250819110529.php | 48 ++++ lib/composer/composer/LICENSE | 2 - lib/composer/composer/autoload_classmap.php | 8 + lib/composer/composer/autoload_static.php | 8 + .../ObjectStore/PrimaryObjectStoreConfig.php | 2 +- lib/private/Preview/Db/Preview.php | 105 ++++++++ lib/private/Preview/Db/PreviewMapper.php | 66 +++++ lib/private/Preview/Generator.php | 226 ++++++++---------- .../Preview/Storage/IPreviewStorage.php | 22 ++ .../Preview/Storage/LocalPreviewStorage.php | 43 ++++ .../Storage/ObjectStorePreviewStorage.php | 50 ++++ lib/private/Preview/Storage/PreviewFile.php | 90 +++++++ .../Preview/Storage/StorageFactory.php | 44 ++++ lib/private/PreviewManager.php | 4 + lib/public/IPreview.php | 5 + 15 files changed, 596 insertions(+), 127 deletions(-) create mode 100644 core/Migrations/Version33000Date20250819110529.php create mode 100644 lib/private/Preview/Db/Preview.php create mode 100644 lib/private/Preview/Db/PreviewMapper.php create mode 100644 lib/private/Preview/Storage/IPreviewStorage.php create mode 100644 lib/private/Preview/Storage/LocalPreviewStorage.php create mode 100644 lib/private/Preview/Storage/ObjectStorePreviewStorage.php create mode 100644 lib/private/Preview/Storage/PreviewFile.php create mode 100644 lib/private/Preview/Storage/StorageFactory.php diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php new file mode 100644 index 00000000000..fcc008b4283 --- /dev/null +++ b/core/Migrations/Version33000Date20250819110529.php @@ -0,0 +1,48 @@ +hasTable('previews')) { + $table = $schema->createTable('previews'); + $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]); + $table->addColumn('is_max', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('crop', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('version', Types::BIGINT, ['notnull' => false, 'unsigned' => true]); + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx'); + } + + return $schema; + } +} diff --git a/lib/composer/composer/LICENSE b/lib/composer/composer/LICENSE index f27399a042d..62ecfd8d004 100644 --- a/lib/composer/composer/LICENSE +++ b/lib/composer/composer/LICENSE @@ -1,4 +1,3 @@ - Copyright (c) Nils Adermann, Jordi Boggiano Permission is hereby granted, free of charge, to any person obtaining a copy @@ -18,4 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index ee77fbd4cda..80191ae4b7f 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1528,6 +1528,7 @@ return array( 'OC\\Core\\Migrations\\Version32000Date20250620081925' => $baseDir . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110523' => $baseDir . '/core/Migrations/Version33000Date20250819110523.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', @@ -1880,6 +1881,8 @@ return array( 'OC\\Preview\\BackgroundCleanupJob' => $baseDir . '/lib/private/Preview/BackgroundCleanupJob.php', 'OC\\Preview\\Bitmap' => $baseDir . '/lib/private/Preview/Bitmap.php', 'OC\\Preview\\Bundled' => $baseDir . '/lib/private/Preview/Bundled.php', + 'OC\\Preview\\Db\\Preview' => $baseDir . '/lib/private/Preview/Db/Preview.php', + 'OC\\Preview\\Db\\PreviewMapper' => $baseDir . '/lib/private/Preview/Db/PreviewMapper.php', 'OC\\Preview\\EMF' => $baseDir . '/lib/private/Preview/EMF.php', 'OC\\Preview\\Font' => $baseDir . '/lib/private/Preview/Font.php', 'OC\\Preview\\GIF' => $baseDir . '/lib/private/Preview/GIF.php', @@ -1912,7 +1915,12 @@ return array( 'OC\\Preview\\SGI' => $baseDir . '/lib/private/Preview/SGI.php', 'OC\\Preview\\SVG' => $baseDir . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => $baseDir . '/lib/private/Preview/StarOffice.php', + 'OC\\Preview\\Storage\\IPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/IPreviewStorage.php', + 'OC\\Preview\\Storage\\LocalPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/LocalPreviewStorage.php', + 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => $baseDir . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', + 'OC\\Preview\\Storage\\PreviewFile' => $baseDir . '/lib/private/Preview/Storage/PreviewFile.php', 'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php', + 'OC\\Preview\\Storage\\StorageFactory' => $baseDir . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => $baseDir . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => $baseDir . '/lib/private/Preview/TXT.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 3b18f00da96..923cf1d5f0b 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1569,6 +1569,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version32000Date20250620081925' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110523' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110523.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', @@ -1921,6 +1922,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\BackgroundCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Preview/BackgroundCleanupJob.php', 'OC\\Preview\\Bitmap' => __DIR__ . '/../../..' . '/lib/private/Preview/Bitmap.php', 'OC\\Preview\\Bundled' => __DIR__ . '/../../..' . '/lib/private/Preview/Bundled.php', + 'OC\\Preview\\Db\\Preview' => __DIR__ . '/../../..' . '/lib/private/Preview/Db/Preview.php', + 'OC\\Preview\\Db\\PreviewMapper' => __DIR__ . '/../../..' . '/lib/private/Preview/Db/PreviewMapper.php', 'OC\\Preview\\EMF' => __DIR__ . '/../../..' . '/lib/private/Preview/EMF.php', 'OC\\Preview\\Font' => __DIR__ . '/../../..' . '/lib/private/Preview/Font.php', 'OC\\Preview\\GIF' => __DIR__ . '/../../..' . '/lib/private/Preview/GIF.php', @@ -1953,7 +1956,12 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\SGI' => __DIR__ . '/../../..' . '/lib/private/Preview/SGI.php', 'OC\\Preview\\SVG' => __DIR__ . '/../../..' . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => __DIR__ . '/../../..' . '/lib/private/Preview/StarOffice.php', + 'OC\\Preview\\Storage\\IPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/IPreviewStorage.php', + 'OC\\Preview\\Storage\\LocalPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/LocalPreviewStorage.php', + 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', + 'OC\\Preview\\Storage\\PreviewFile' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/PreviewFile.php', 'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php', + 'OC\\Preview\\Storage\\StorageFactory' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => __DIR__ . '/../../..' . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => __DIR__ . '/../../..' . '/lib/private/Preview/TXT.php', diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php index 008431b3fbf..4a31d2e51f5 100644 --- a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -14,7 +14,7 @@ use OCP\IConfig; use OCP\IUser; /** - * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, ...}} + * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, objectPrefix: ?string, ...}} */ class PrimaryObjectStoreConfig { public function __construct( diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php new file mode 100644 index 00000000000..f2f05871082 --- /dev/null +++ b/lib/private/Preview/Db/Preview.php @@ -0,0 +1,105 @@ +addType('fileId', Types::INTEGER); + $this->addType('width', Types::INTEGER); + $this->addType('height', Types::INTEGER); + $this->addType('mimetype', Types::INTEGER); + $this->addType('mtime', Types::INTEGER); + $this->addType('size', Types::INTEGER); + $this->addType('isMax', Types::BOOLEAN); + $this->addType('crop', Types::BOOLEAN); + $this->addType('etag', Types::STRING); + $this->addType('version', Types::INTEGER); + } + + public function getName(): string { + $path = ($this->getVersion() ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); + if ($this->getCrop()) { + $path .= '-crop'; + } + if ($this->getIsMax()) { + $path .= '-max'; + } + + $ext = $this->getExtension(); + $path .= '.' . $ext; + return $path; + } + + public function getMimetypeValue(): string { + return match ($this->mimetype) { + IPreview::MIMETYPE_JPEG => 'image/jpeg', + IPreview::MIMETYPE_PNG => 'image/png', + IPreview::MIMETYPE_WEBP => 'image/webp', + IPreview::MIMETYPE_GIF => 'image/gif', + }; + } + + public function getExtension(): string { + return match ($this->mimetype) { + IPreview::MIMETYPE_JPEG => 'jpeg', + IPreview::MIMETYPE_PNG => 'png', + IPreview::MIMETYPE_WEBP => 'webp', + IPreview::MIMETYPE_GIF => 'gif', + }; + } +} diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php new file mode 100644 index 00000000000..5faa508c8e0 --- /dev/null +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -0,0 +1,66 @@ + + */ +class PreviewMapper extends QBMapper { + + private const TABLE_NAME = 'previews'; + + public function __construct(IDBConnection $db) { + parent::__construct($db, self::TABLE_NAME, Preview::class); + } + + /** + * @param int[] $fileIds + * @return array + * @throws Exception + */ + public function getAvailablePreviews(array $fileIds): array { + $selectQb = $this->db->getQueryBuilder(); + $selectQb->select('*') + ->from(self::TABLE_NAME) + ->where( + $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), + ); + $previews = array_fill_keys($fileIds, []); + foreach ($this->yieldEntities($selectQb) as $preview) { + $previews[$preview->getFileId()][] = $preview; + } + return $previews; + } + + public function getPreview(int $fileId, int $width, int $height, string $mode, int $mimetype = IPreview::MIMETYPE_JPEG): ?Preview { + $selectQb = $this->db->getQueryBuilder(); + $selectQb->select('*') + ->from(self::TABLE_NAME) + ->where( + $selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId)), + $selectQb->expr()->eq('width', $selectQb->createNamedParameter($width)), + $selectQb->expr()->eq('height', $selectQb->createNamedParameter($height)), + $selectQb->expr()->eq('mode', $selectQb->createNamedParameter($mode)), + $selectQb->expr()->eq('mimetype', $selectQb->createNamedParameter($mimetype)), + ); + try { + return $this->findEntity($selectQb); + } catch (DoesNotExistException) { + return null; + } + } +} diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 4a7341896ef..39e1de54e69 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -6,6 +6,10 @@ */ namespace OC\Preview; +use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\PreviewFile; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; use OCP\Files\IAppData; @@ -23,6 +27,7 @@ use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IProviderV2; use OCP\Preview\IVersionedPreviewFile; use Psr\Log\LoggerInterface; +use function Symfony\Component\Translation\t; class Generator { public const SEMAPHORE_ID_ALL = 0x0a11; @@ -35,6 +40,8 @@ class Generator { private GeneratorHelper $helper, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, + private PreviewMapper $previewMapper, + private StorageFactory $storageFactory, ) { } @@ -104,25 +111,25 @@ class Generator { $mimeType = $file->getMimeType(); } - $previewFolder = $this->getPreviewFolder($file); - // List every existing preview first instead of trying to find them one by one - $previewFiles = $previewFolder->getDirectoryListing(); + [$file->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$file->getId()]); - $previewVersion = ''; + $previewVersion = null; if ($file instanceof IVersionedPreviewFile) { - $previewVersion = $file->getPreviewVersion() . '-'; + $previewVersion = (int)$file->getPreviewVersion(); } // Get the max preview and infer the max preview sizes from that - $maxPreview = $this->getMaxPreview($previewFolder, $previewFiles, $file, $mimeType, $previewVersion); + $maxPreview = $this->getMaxPreview($previews, $file, $mimeType, $previewVersion); $maxPreviewImage = null; // only load the image when we need it if ($maxPreview->getSize() === 0) { - $maxPreview->delete(); + $this->storageFactory->deletePreview($maxPreview); + $this->previewMapper->delete($maxPreview); $this->logger->error('Max preview generated for file {path} has size 0, deleting and throwing exception.', ['path' => $file->getPath()]); throw new NotFoundException('Max preview size 0, invalid!'); } - [$maxWidth, $maxHeight] = $this->getPreviewSize($maxPreview, $previewVersion); + $maxWidth = $maxPreview->getWidth(); + $maxHeight = $maxPreview->getHeight(); if ($maxWidth <= 0 || $maxHeight <= 0) { throw new NotFoundException('The maximum preview sizes are zero or less pixels'); @@ -154,32 +161,40 @@ class Generator { // Try to get a cached preview. Else generate (and store) one try { - try { - $preview = $this->getCachedPreview($previewFiles, $width, $height, $crop, $maxPreview->getMimeType(), $previewVersion); - } catch (NotFoundException $e) { + /** @var ISimpleFile $previewFile */ + $previewFile = null; + // TODO(php8.4) replace by array_find + foreach ($previews as $p) { + if ($p->getWidth() === $width && $p->getHeight() === $height && $p->getMimetype() === $maxPreview->getMimetype() && $p->getVersion() === $previewVersion && $p->getCrop() === $crop) { + $previewFile = new PreviewFile($p, $this->storageFactory, $this->previewMapper); + break; + } + } + + if ($previewFile === null) { if (!$this->previewManager->isMimeSupported($mimeType)) { throw new NotFoundException(); } if ($maxPreviewImage === null) { - $maxPreviewImage = $this->helper->getImage($maxPreview); + $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); } $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); - $preview = $this->generatePreview($previewFolder, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); + $preview = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); // New file, augment our array - $previewFiles[] = $preview; + //$previews[] = $preview; } } catch (\InvalidArgumentException $e) { throw new NotFoundException('', 0, $e); } - if ($preview->getSize() === 0) { - $preview->delete(); + if ($previewFile->getSize() === 0) { + $previewFile->delete(); throw new NotFoundException('Cached preview size 0, invalid!'); } } - assert($preview !== null); + assert($previewFile !== null); // Free memory being used by the embedded image resource. Without this the image is kept in memory indefinitely. // Garbage Collection does NOT free this memory. We have to do it ourselves. @@ -187,7 +202,7 @@ class Generator { $maxPreviewImage->destroy(); } - return $preview; + return $previewFile; } /** @@ -289,31 +304,25 @@ class Generator { } /** - * @param ISimpleFolder $previewFolder - * @param ISimpleFile[] $previewFiles - * @param File $file - * @param string $mimeType - * @param string $prefix - * @return ISimpleFile + * @param Preview[] $previews * @throws NotFoundException */ - private function getMaxPreview(ISimpleFolder $previewFolder, array $previewFiles, File $file, $mimeType, $prefix) { + private function getMaxPreview(array $previews, File $file, string $mimeType, ?int $version): Preview { // We don't know the max preview size, so we can't use getCachedPreview. // It might have been generated with a higher resolution than the current value. - foreach ($previewFiles as $node) { - $name = $node->getName(); - if (($prefix === '' || str_starts_with($name, $prefix)) && strpos($name, 'max')) { - return $node; + foreach ($previews as $preview) { + if ($preview->getIsMax() && ($version == $preview->getVersion())) { + return $preview; } } $maxWidth = $this->config->getSystemValueInt('preview_max_x', 4096); $maxHeight = $this->config->getSystemValueInt('preview_max_y', 4096); - return $this->generateProviderPreview($previewFolder, $file, $maxWidth, $maxHeight, false, true, $mimeType, $prefix); + return $this->generateProviderPreview($file, $maxWidth, $maxHeight, false, true, $mimeType, $version); } - private function generateProviderPreview(ISimpleFolder $previewFolder, File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, string $prefix) { + private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): Preview { $previewProviders = $this->previewManager->getProviders(); foreach ($previewProviders as $supportedMimeType => $providers) { // Filter out providers that does not support this mime @@ -348,45 +357,19 @@ class Generator { continue; } - $path = $this->generatePath($preview->width(), $preview->height(), $crop, $max, $preview->dataMimeType(), $prefix); try { - if ($preview instanceof IStreamImage) { - return $previewFolder->newFile($path, $preview->resource()); - } else { - return $previewFolder->newFile($path, $preview->data()); - } + return $this->savePreview($file, $width, $height, $crop, $max, $preview, $version); } catch (NotPermittedException $e) { throw new NotFoundException(); } - - return $file; } } throw new NotFoundException('No provider successfully handled the preview generation'); } - /** - * @param ISimpleFile $file - * @param string $prefix - * @return int[] - */ - private function getPreviewSize(ISimpleFile $file, string $prefix = '') { - $size = explode('-', substr($file->getName(), strlen($prefix))); - return [(int)$size[0], (int)$size[1]]; - } - - /** - * @param int $width - * @param int $height - * @param bool $crop - * @param bool $max - * @param string $mimeType - * @param string $prefix - * @return string - */ - private function generatePath($width, $height, $crop, $max, $mimeType, $prefix) { - $path = $prefix . (string)$width . '-' . (string)$height; + private function generatePath(int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): string { + $path = ($version ? $version . '-' : '') . $width . '-' . $height; if ($crop) { $path .= '-crop'; } @@ -401,15 +384,10 @@ class Generator { /** - * @param int $width - * @param int $height - * @param bool $crop - * @param string $mode - * @param int $maxWidth - * @param int $maxHeight + * @psalm-param IPreview::MODE_* $mode * @return int[] */ - private function calculateSize($width, $height, $crop, $mode, $maxWidth, $maxHeight) { + private function calculateSize(int $width, int $height, bool $crop, string $mode, int $maxWidth, int $maxHeight): array { /* * If we are not cropping we have to make sure the requested image * respects the aspect ratio of the original. @@ -492,14 +470,14 @@ class Generator { * @throws \InvalidArgumentException if the preview would be invalid (in case the original image is invalid) */ private function generatePreview( - ISimpleFolder $previewFolder, + File $file, IImage $maxPreview, int $width, int $height, bool $crop, int $maxWidth, int $maxHeight, - string $prefix, + ?int $version, bool $cacheResult, ): ISimpleFile { $preview = $maxPreview; @@ -536,62 +514,13 @@ class Generator { } - $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $prefix); - try { - if ($cacheResult) { - return $previewFolder->newFile($path, $preview->data()); - } else { - return new InMemoryFile($path, $preview->data()); - } - } catch (NotPermittedException $e) { - throw new NotFoundException(); + $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $version); + if ($cacheResult) { + $previewEntry = $this->savePreview($file, $width, $height, $crop, false, $preview, $version); + return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper); + } else { + return new InMemoryFile($path, $preview->data()); } - return $file; - } - - /** - * @param ISimpleFile[] $files Array of FileInfo, as the result of getDirectoryListing() - * @param int $width - * @param int $height - * @param bool $crop - * @param string $mimeType - * @param string $prefix - * @return ISimpleFile - * - * @throws NotFoundException - */ - private function getCachedPreview($files, $width, $height, $crop, $mimeType, $prefix) { - $path = $this->generatePath($width, $height, $crop, false, $mimeType, $prefix); - foreach ($files as $file) { - if ($file->getName() === $path) { - $this->logger->debug('Found cached preview: {path}', ['path' => $path]); - return $file; - } - } - throw new NotFoundException(); - } - - /** - * Get the specific preview folder for this file - * - * @param File $file - * @return ISimpleFolder - * - * @throws InvalidPathException - * @throws NotFoundException - * @throws NotPermittedException - */ - private function getPreviewFolder(File $file) { - // Obtain file id outside of try catch block to prevent the creation of an existing folder - $fileId = (string)$file->getId(); - - try { - $folder = $this->appData->getFolder($fileId); - } catch (NotFoundException $e) { - $folder = $this->appData->newFolder($fileId); - } - - return $folder; } /** @@ -613,4 +542,53 @@ class Generator { throw new \InvalidArgumentException('Not a valid mimetype: "' . $mimeType . '"'); } } + + /** + * @throws InvalidPathException + * @throws NotFoundException + * @throws NotPermittedException + * @throws \OCP\DB\Exception + */ + public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, ?int $version): Preview { + $previewEntry = new Preview(); + $previewEntry->setFileId($file->getId()); + $previewEntry->setWidth($width); + $previewEntry->setHeight($height); + $previewEntry->setVersion($version); + $previewEntry->setIsMax($max); + $previewEntry->setCrop($crop); + switch ($preview->dataMimeType()) { + case 'image/jpeg': + $previewEntry->setMimetype(IPreview::MIMETYPE_JPEG); + break; + case 'image/gif': + $previewEntry->setMimetype(IPreview::MIMETYPE_GIF); + break; + case 'image/webp': + $previewEntry->setMimetype(IPreview::MIMETYPE_WEBP); + break; + default: + $previewEntry->setMimetype(IPreview::MIMETYPE_PNG); + break; + } + $previewEntry->setEtag($file->getEtag()); + $previewEntry->setMtime((new \DateTime())->getTimestamp()); + $previewEntry->setSize(0); + + $previewEntry = $this->previewMapper->insert($previewEntry); + + // we need to save to DB first + try { + if ($preview instanceof IStreamImage) { + $size = $this->storageFactory->writePreview($previewEntry, $preview->resource()); + } else { + $size = $this->storageFactory->writePreview($previewEntry, $preview->data()); + } + } catch (\Exception $e) { + $this->previewMapper->delete($previewEntry); + throw $e; + } + $previewEntry->setSize($size); + return $this->previewMapper->update($previewEntry); + } } diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php new file mode 100644 index 00000000000..a989c492c76 --- /dev/null +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -0,0 +1,22 @@ +constructPath($preview); + ['basename' => $basename, 'dirname' => $dirname] = pathinfo($previewPath); + $currentDir = $this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY; + mkdir($currentDir); + foreach (explode('/', $dirname) as $suffix) { + $currentDir .= "/$suffix"; + mkdir($currentDir); + } + $file = @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "w"); + return fwrite($file, $stream); + } + + public function readPreview(Preview $preview) { + $previewPath = $this->constructPath($preview); + return @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "r"); + } + + public function deletePreview(Preview $preview) { + $previewPath = $this->constructPath($preview); + @unlink($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath); + } + + private function constructPath(Preview $preview): string { + return implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); + } +} diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php new file mode 100644 index 00000000000..fc9a64ed761 --- /dev/null +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -0,0 +1,50 @@ +objectPrefix = $parameters['objectPrefix'] . 'preview:'; + } + } + + public function writePreview(Preview $preview, $stream): false|int { + if (!is_resource($stream)) { + $fh = fopen('php://temp', 'w+'); + fwrite($fh, $stream); + rewind($fh); + + $stream = $fh; + } + + $size = 0; + $countStream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size): void { + $size = $writtenSize; + }); + + $this->objectStore->writeObject($this->constructUrn($preview), $countStream); + return $size; + } + + public function readPreview(Preview $preview) { + return $this->objectStore->readObject($this->constructUrn($preview)); + } + + public function deletePreview(Preview $preview) { + return $this->objectStore->deleteObject($this->constructUrn($preview)); + } + + private function constructUrn(Preview $preview): string { + return $this->objectPrefix . $preview->getId(); + } +} diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php new file mode 100644 index 00000000000..f9e4cc0f59a --- /dev/null +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -0,0 +1,90 @@ +preview->getName(); + } + + /** + * @inheritDoc + */ + public function getSize(): int|float { + return $this->preview->getSize(); + } + + /** + * @inheritDoc + */ + public function getETag(): string { + return $this->preview->getEtag(); + } + + /** + * @inheritDoc + */ + public function getMTime(): int { + return $this->preview->getMtime(); + } + + /** + * @inheritDoc + */ + public function getContent(): string { + $stream = $this->storage->readPreview($this->preview); + return stream_get_contents($stream); + } + + /** + * @inheritDoc + */ + public function putContent($data): void { + } + + /** + * @inheritDoc + */ + public function delete(): void { + $this->storage->deletePreview($this->preview); + $this->previewMapper->delete($this->preview); + } + + /** + * @inheritDoc + */ + public function getMimeType(): string { + return $this->preview->getMimetypeValue(); + } + + /** + * @inheritDoc + */ + public function getExtension(): string { + return $this->preview->getExtension(); + } + + /** + * @inheritDoc + */ + public function read() { + return $this->storage->readPreview($this->preview); + } + + /** + * @inheritDoc + */ + public function write() { + return false; + } +} diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php new file mode 100644 index 00000000000..15866276774 --- /dev/null +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -0,0 +1,44 @@ +getBackend()->writePreview($preview, $stream); + } + + public function readPreview(Preview $preview) { + return $this->getBackend()->readPreview($preview); + } + + public function deletePreview(Preview $preview) { + $this->getBackend()->deletePreview($preview); + } + + private function getBackend(): IPreviewStorage { + if ($this->backend) { + return $this->backend; + } + + $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); + + if ($objectStoreConfig) { + $objectStore = $this->objectStoreConfig->buildObjectStore($objectStoreConfig); + $this->backend = new ObjectStorePreviewStorage($objectStore, $objectStoreConfig['arguments']); + } else { + $configDataDirectory = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); + $this->backend = new LocalPreviewStorage($configDataDirectory); + } + + return $this->backend; + } +} diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 97e9b5e313c..7c8e1b13eb9 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -8,9 +8,11 @@ namespace OC; use OC\AppFramework\Bootstrap\Coordinator; +use OC\Preview\Db\PreviewMapper; use OC\Preview\Generator; use OC\Preview\GeneratorHelper; use OC\Preview\IMagickSupport; +use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\QueryException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; @@ -140,6 +142,8 @@ class PreviewManager implements IPreview { ), $this->eventDispatcher, $this->container->get(LoggerInterface::class), + $this->container->get(PreviewMapper::class), + $this->container->get(StorageFactory::class), ); } return $this->generator; diff --git a/lib/public/IPreview.php b/lib/public/IPreview.php index 3c9eadd4577..cbd0e0ae525 100644 --- a/lib/public/IPreview.php +++ b/lib/public/IPreview.php @@ -29,6 +29,11 @@ interface IPreview { */ public const MODE_COVER = 'cover'; + public const MIMETYPE_JPEG = 0; + public const MIMETYPE_WEBP = 1; + public const MIMETYPE_PNG = 2; + public const MIMETYPE_GIF = 3; + /** * In order to improve lazy loading a closure can be registered which will be * called in case preview providers are actually requested From 656e33e8daf2be3a51285715ea9a87bf462a6c94 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 21 Aug 2025 11:18:22 +0200 Subject: [PATCH 02/14] perf(preview): Add support for multibucket storage Signed-off-by: Carl Schwan --- lib/private/Preview/Db/Preview.php | 10 ++++++++-- lib/private/Preview/Generator.php | 1 + 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index f2f05871082..fbecd485421 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -17,6 +17,8 @@ use OCP\IPreview; /** * @method \int getFileId() * @method void setFileId(int $fileId) + * @method \int getStorageId() + * @method void setStorageId(\int $fileId) * @method \int getWidth() * @method void setWidth(int $width) * @method \int getHeight() @@ -41,6 +43,8 @@ use OCP\IPreview; class Preview extends Entity { protected ?int $fileId = null; + protected ?int $storageId = null; + protected ?int $width = null; protected ?int $height = null; @@ -56,10 +60,12 @@ class Preview extends Entity { protected ?bool $crop = null; protected ?string $etag = null; + protected ?int $version = null; public function __construct() { - $this->addType('fileId', Types::INTEGER); + $this->addType('fileId', Types::BIGINT); + $this->addType('storageId', Types::BIGINT); $this->addType('width', Types::INTEGER); $this->addType('height', Types::INTEGER); $this->addType('mimetype', Types::INTEGER); @@ -68,7 +74,7 @@ class Preview extends Entity { $this->addType('isMax', Types::BOOLEAN); $this->addType('crop', Types::BOOLEAN); $this->addType('etag', Types::STRING); - $this->addType('version', Types::INTEGER); + $this->addType('version', Types::BIGINT); } public function getName(): string { diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 39e1de54e69..ed4e5d32177 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -552,6 +552,7 @@ class Generator { public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, ?int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); + $previewEntry->setStorageId((int)$file->getStorage()->getId()); $previewEntry->setWidth($width); $previewEntry->setHeight($height); $previewEntry->setVersion($version); From 13c35c0f17d41a0c6e567c345021ee1c0ae9c4e6 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 21 Aug 2025 16:42:34 +0200 Subject: [PATCH 03/14] perf(preview): Migrate previews to the new optimized table Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 238 ++++++++++++++++++ .../Version33000Date20250819110529.php | 3 +- lib/composer/composer/autoload_classmap.php | 4 +- lib/composer/composer/autoload_static.php | 4 +- lib/private/BackgroundJob/JobList.php | 1 + .../ObjectStore/PrimaryObjectStoreConfig.php | 4 +- lib/private/Preview/Db/Preview.php | 4 +- lib/private/Preview/Generator.php | 25 +- .../Preview/Storage/IPreviewStorage.php | 8 + .../Preview/Storage/LocalPreviewStorage.php | 69 +++-- .../Storage/ObjectStorePreviewStorage.php | 26 +- lib/private/Preview/Storage/PreviewFile.php | 14 +- .../Preview/Storage/StorageFactory.php | 17 +- lib/private/PreviewManager.php | 1 - lib/private/Repair.php | 2 + lib/private/Repair/AddMovePreviewJob.php | 27 ++ lib/private/Setup.php | 2 + 17 files changed, 401 insertions(+), 48 deletions(-) create mode 100644 core/BackgroundJobs/MovePreviewJob.php create mode 100644 lib/private/Repair/AddMovePreviewJob.php diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php new file mode 100644 index 00000000000..d75673e6d2d --- /dev/null +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -0,0 +1,238 @@ +appData = $appDataFactory->get('preview'); + $this->setTimeSensitivity(self::TIME_INSENSITIVE); + $this->setInterval(24 * 60 * 60); + } + + protected function run(mixed $argument): void { + try { + $this->doRun($argument); + } catch (\Throwable $exception) { + echo $exception->getMessage(); + throw $exception; + } + } + + private function doRun($argument): void { + if ($this->appConfig->getValueBool('core', 'previewMovedDone')) { + //return; + } + + $emptyHierarchicalPreviewFolders = false; + + $startTime = time(); + while (true) { + $previewFolders = []; + + // Check new hierarchical preview folders first + if (!$emptyHierarchicalPreviewFolders) { + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('filecache') + ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%/%/%/%/%/%/%/%'))) + ->setMaxResults(100); + + $result = $qb->executeQuery(); + while ($row = $result->fetch()) { + $pathSplit = explode('/', $row['path']); + assert(count($pathSplit) >= 2); + $fileId = $pathSplit[count($pathSplit) - 2]; + $previewFolders[$fileId][] = $row['path']; + } + + if (!empty($previewFolders)) { + $this->processPreviews($previewFolders, false); + continue; + } + } + + // And then the flat preview folder (legacy) + $emptyHierarchicalPreviewFolders = true; + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('filecache') + ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.jpg'))) + ->setMaxResults(100); + + $result = $qb->executeQuery(); + while ($row = $result->fetch()) { + $pathSplit = explode('/', $row['path']); + assert(count($pathSplit) >= 2); + $fileId = $pathSplit[count($pathSplit) - 2]; + array_pop($pathSplit); + $path = implode('/', $pathSplit); + if (!isset($previewFolders[$fileId])) { + $previewFolders[$fileId] = []; + } + if (!in_array($path, $previewFolders[$fileId])) { + $previewFolders[$fileId][] = $path; + } + } + + if (empty($previewFolders)) { + break; + } else { + $this->processPreviews($previewFolders, true); + } + + // Stop if execution time is more than one hour. + if (time() - $startTime > 3600) { + return; + } + } + + // Delete any left over preview directory + $this->appData->getFolder('.')->delete(); + $this->appConfig->setValueBool('core', 'previewMovedDone', true); + } + + /** + * @param array $previewFolders + */ + private function processPreviews(array $previewFolders, bool $simplePaths): void { + foreach ($previewFolders as $fileId => $previewFolder) { + $internalPath = $this->getInternalFolder((string)$fileId, $simplePaths); + $folder = $this->appData->getFolder($internalPath); + + /** + * @var list $previewFiles + */ + $previewFiles = []; + + foreach ($folder->getDirectoryListing() as $previewFile) { + [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); + $nameSplit = explode('-', $baseName); + + // TODO VERSION/PREFIX extraction + + $width = $nameSplit[0]; + $height = $nameSplit[1]; + + if (isset($nameSplit[2])) { + $crop = $nameSplit[2] === 'crop'; + $max = $nameSplit[2] === 'max'; + } + + $previewFiles[] = [ + 'file' => $previewFile, + 'width' => $width, + 'height' => $height, + 'crop' => $crop, + 'max' => $max, + 'extension' => $extension, + 'size' => $previewFile->getSize(), + 'mtime' => $previewFile->getMTime(), + ]; + } + + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('filecache') + ->where($qb->expr()->like('fileid', $qb->createNamedParameter($fileId))); + + $result = $qb->executeQuery(); + $result = $result->fetchAll(); + + if (count($result) > 0) { + foreach ($previewFiles as $previewFile) { + $preview = new Preview(); + $preview->setFileId((int)$fileId); + $preview->setStorageId($result[0]['storage']); + $preview->setEtag($result[0]['etag']); + $preview->setMtime($previewFile['mtime']); + $preview->setWidth($previewFile['width']); + $preview->setHeight($previewFile['height']); + $preview->setCrop($previewFile['crop']); + $preview->setIsMax($previewFile['max']); + $preview->setMimetype(match ($previewFile['extension']) { + 'png' => IPreview::MIMETYPE_PNG, + 'webp' => IPreview::MIMETYPE_WEBP, + 'gif' => IPreview::MIMETYPE_GIF, + default => IPreview::MIMETYPE_JPEG, + }); + $preview->setSize($previewFile['size']); + try { + $preview = $this->previewMapper->insert($preview); + } catch (Exception $e) { + // We already have this preview in the preview table, skip + continue; + } + + try { + $this->storageFactory->migratePreview($preview, $previewFile['file']); + $previewFile['file']->delete(); + } catch (Exception $e) { + $this->previewMapper->delete($preview); + throw $e; + } + + } + } + + $this->deleteFolder($internalPath, $folder); + } + } + + public static function getInternalFolder(string $name, bool $simplePaths): string { + if ($simplePaths) { + return '/' . $name; + } + return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; + } + + private function deleteFolder(string $path, ISimpleFolder $folder): void { + $folder->delete(); + + $current = $path; + + while (true) { + $current = dirname($current); + if ($current === '/' || $current === '.' || $current === '') { + break; + } + + + $folder = $this->appData->getFolder($current); + if (count($folder->getDirectoryListing()) !== 0) { + break; + } + $folder->delete(); + } + } +} diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php index fcc008b4283..496ee849beb 100644 --- a/core/Migrations/Version33000Date20250819110529.php +++ b/core/Migrations/Version33000Date20250819110529.php @@ -30,6 +30,7 @@ class Version33000Date20250819110529 extends SimpleMigrationStep { $table = $schema->createTable('previews'); $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('storage_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]); @@ -38,7 +39,7 @@ class Version33000Date20250819110529 extends SimpleMigrationStep { $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40]); $table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); - $table->addColumn('version', Types::BIGINT, ['notnull' => false, 'unsigned' => true]); + $table->addColumn('version', Types::BIGINT, ['notnull' => true, 'default' => -1]); // can not be null otherwise unique index doesn't work $table->setPrimaryKey(['id']); $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx'); } diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 80191ae4b7f..d3eb6553b8c 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1253,6 +1253,7 @@ return array( 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php', 'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', + 'OC\\Core\\BackgroundJobs\\MovePreviewJob' => $baseDir . '/core/BackgroundJobs/MovePreviewJob.php', 'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => $baseDir . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => $baseDir . '/core/Command/App/GetPath.php', @@ -1528,7 +1529,7 @@ return array( 'OC\\Core\\Migrations\\Version32000Date20250620081925' => $baseDir . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php', - 'OC\\Core\\Migrations\\Version33000Date20250819110523' => $baseDir . '/core/Migrations/Version33000Date20250819110523.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110529' => $baseDir . '/core/Migrations/Version33000Date20250819110529.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', @@ -1958,6 +1959,7 @@ return array( 'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => $baseDir . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', 'OC\\Repair\\AddMetadataGenerationJob' => $baseDir . '/lib/private/Repair/AddMetadataGenerationJob.php', + 'OC\\Repair\\AddMovePreviewJob' => $baseDir . '/lib/private/Repair/AddMovePreviewJob.php', 'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => $baseDir . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php', 'OC\\Repair\\CleanTags' => $baseDir . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => $baseDir . '/lib/private/Repair/CleanUpAbandonedApps.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 923cf1d5f0b..d14e3989a6a 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1294,6 +1294,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php', 'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/GenerateMetadataJob.php', 'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php', + 'OC\\Core\\BackgroundJobs\\MovePreviewJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/MovePreviewJob.php', 'OC\\Core\\Command\\App\\Disable' => __DIR__ . '/../../..' . '/core/Command/App/Disable.php', 'OC\\Core\\Command\\App\\Enable' => __DIR__ . '/../../..' . '/core/Command/App/Enable.php', 'OC\\Core\\Command\\App\\GetPath' => __DIR__ . '/../../..' . '/core/Command/App/GetPath.php', @@ -1569,7 +1570,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version32000Date20250620081925' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php', - 'OC\\Core\\Migrations\\Version33000Date20250819110523' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110523.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110529' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110529.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', @@ -1999,6 +2000,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Repair\\AddCleanupDeletedUsersBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupDeletedUsersBackgroundJob.php', 'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php', 'OC\\Repair\\AddMetadataGenerationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataGenerationJob.php', + 'OC\\Repair\\AddMovePreviewJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMovePreviewJob.php', 'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php', 'OC\\Repair\\CleanTags' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanTags.php', 'OC\\Repair\\CleanUpAbandonedApps' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanUpAbandonedApps.php', diff --git a/lib/private/BackgroundJob/JobList.php b/lib/private/BackgroundJob/JobList.php index c00a51e3851..302bee22cd5 100644 --- a/lib/private/BackgroundJob/JobList.php +++ b/lib/private/BackgroundJob/JobList.php @@ -321,6 +321,7 @@ class JobList implements IJobList { /** @var IJob $job */ $job = \OCP\Server::get($row['class']); } catch (QueryException $e) { + echo $e->getMessage(); if (class_exists($row['class'])) { $class = $row['class']; $job = new $class(); diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php index 4a31d2e51f5..cd2983d385c 100644 --- a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -14,7 +14,7 @@ use OCP\IConfig; use OCP\IUser; /** - * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, objectPrefix: ?string, ...}} + * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, objectPrefix?: string, ...}} */ class PrimaryObjectStoreConfig { public function __construct( @@ -155,7 +155,7 @@ class PrimaryObjectStoreConfig { } /** - * @param array|string $config + * @param array{multibucket?: bool, objectPrefix?: string, ...}|string $config * @return string|ObjectStoreConfig */ private function validateObjectStoreConfig(array|string $config): array|string { diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index fbecd485421..c88cebedc91 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -78,7 +78,7 @@ class Preview extends Entity { } public function getName(): string { - $path = ($this->getVersion() ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); + $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); if ($this->getCrop()) { $path .= '-crop'; } @@ -102,7 +102,7 @@ class Preview extends Entity { public function getExtension(): string { return match ($this->mimetype) { - IPreview::MIMETYPE_JPEG => 'jpeg', + IPreview::MIMETYPE_JPEG => 'jpg', IPreview::MIMETYPE_PNG => 'png', IPreview::MIMETYPE_WEBP => 'webp', IPreview::MIMETYPE_GIF => 'gif', diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index ed4e5d32177..29590d6fa93 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -12,13 +12,11 @@ use OC\Preview\Storage\PreviewFile; use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IAppData; use OCP\Files\InvalidPathException; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\Files\SimpleFS\InMemoryFile; use OCP\Files\SimpleFS\ISimpleFile; -use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IConfig; use OCP\IImage; use OCP\IPreview; @@ -27,7 +25,6 @@ use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IProviderV2; use OCP\Preview\IVersionedPreviewFile; use Psr\Log\LoggerInterface; -use function Symfony\Component\Translation\t; class Generator { public const SEMAPHORE_ID_ALL = 0x0a11; @@ -36,7 +33,6 @@ class Generator { public function __construct( private IConfig $config, private IPreview $previewManager, - private IAppData $appData, private GeneratorHelper $helper, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, @@ -113,7 +109,7 @@ class Generator { [$file->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$file->getId()]); - $previewVersion = null; + $previewVersion = -1; if ($file instanceof IVersionedPreviewFile) { $previewVersion = (int)$file->getPreviewVersion(); } @@ -135,8 +131,7 @@ class Generator { throw new NotFoundException('The maximum preview sizes are zero or less pixels'); } - $preview = null; - + $previewFile = null; foreach ($specifications as $specification) { $width = $specification['width'] ?? -1; $height = $specification['height'] ?? -1; @@ -155,7 +150,7 @@ class Generator { // No need to generate a preview that is just the max preview if ($width === $maxWidth && $height === $maxHeight) { // ensure correct return value if this was the last one - $preview = $maxPreview; + $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper); continue; } @@ -181,9 +176,7 @@ class Generator { } $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); - $preview = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); - // New file, augment our array - //$previews[] = $preview; + $previewFile = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); } } catch (\InvalidArgumentException $e) { throw new NotFoundException('', 0, $e); @@ -307,7 +300,7 @@ class Generator { * @param Preview[] $previews * @throws NotFoundException */ - private function getMaxPreview(array $previews, File $file, string $mimeType, ?int $version): Preview { + private function getMaxPreview(array $previews, File $file, string $mimeType, int $version): Preview { // We don't know the max preview size, so we can't use getCachedPreview. // It might have been generated with a higher resolution than the current value. foreach ($previews as $preview) { @@ -322,7 +315,7 @@ class Generator { return $this->generateProviderPreview($file, $maxWidth, $maxHeight, false, true, $mimeType, $version); } - private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): Preview { + private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, int $version): Preview { $previewProviders = $this->previewManager->getProviders(); foreach ($previewProviders as $supportedMimeType => $providers) { // Filter out providers that does not support this mime @@ -368,8 +361,8 @@ class Generator { throw new NotFoundException('No provider successfully handled the preview generation'); } - private function generatePath(int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): string { - $path = ($version ? $version . '-' : '') . $width . '-' . $height; + private function generatePath(int $width, int $height, bool $crop, bool $max, string $mimeType, int $version): string { + $path = ($version !== -1 ? $version . '-' : '') . $width . '-' . $height; if ($crop) { $path .= '-crop'; } @@ -549,7 +542,7 @@ class Generator { * @throws NotPermittedException * @throws \OCP\DB\Exception */ - public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, ?int $version): Preview { + public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId((int)$file->getStorage()->getId()); diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index a989c492c76..8ebf98b7493 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -2,6 +2,7 @@ namespace OC\Preview\Storage; +use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OCP\Files\NotPermittedException; @@ -19,4 +20,11 @@ interface IPreviewStorage { public function readPreview(Preview $preview); public function deletePreview(Preview $preview); + + /** + * Migration helper + * + * To remove at some point + */ + public function migratePreview(Preview $preview, SimpleFile $file): void; } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 8df75f317ff..5971113cb53 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -1,43 +1,80 @@ rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); } public function writePreview(Preview $preview, $stream): false|int { $previewPath = $this->constructPath($preview); - ['basename' => $basename, 'dirname' => $dirname] = pathinfo($previewPath); - $currentDir = $this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY; - mkdir($currentDir); - foreach (explode('/', $dirname) as $suffix) { - $currentDir .= "/$suffix"; - mkdir($currentDir); - } - $file = @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "w"); + $this->createParentFiles($previewPath); + $file = @fopen($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath, 'w'); return fwrite($file, $stream); } public function readPreview(Preview $preview) { $previewPath = $this->constructPath($preview); - return @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "r"); + return @fopen($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath, 'r'); } public function deletePreview(Preview $preview) { $previewPath = $this->constructPath($preview); - @unlink($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath); + @unlink($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath); } private function constructPath(Preview $preview): string { return implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); } + + private function createParentFiles($path) { + ['basename' => $basename, 'dirname' => $dirname] = pathinfo($path); + $currentDir = $this->rootFolder . '/' . self::PREVIEW_DIRECTORY; + mkdir($currentDir); + foreach (explode('/', $dirname) as $suffix) { + $currentDir .= "/$suffix"; + mkdir($currentDir); + } + } + + public function migratePreview(Preview $preview, SimpleFile $file): void { + $instanceId = $this->config->getSystemValueString('instanceid'); + $previewPath = $this->constructPath($preview); + $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $previewPath; + $destinationPath = $this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath; + if (!file_exists($sourcePath)) { + // legacy flat directory + $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $preview->getFileId() . '/' . $preview->getName(); + } + if (file_exists($destinationPath)) { + return; + } + $this->createParentFiles($previewPath); + echo 'Copying ' . $sourcePath . ' to ' . $destinationPath . PHP_EOL; + $ok = copy($sourcePath, $destinationPath); + if (!$ok) { + throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); + } + } } diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index fc9a64ed761..11944dbf846 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -1,8 +1,17 @@ objectPrefix = $parameters['objectPrefix'] . 'preview:'; } @@ -47,4 +59,14 @@ class ObjectStorePreviewStorage implements IPreviewStorage { private function constructUrn(Preview $preview): string { return $this->objectPrefix . $preview->getId(); } + + public function migratePreview(Preview $preview, SimpleFile $file): void { + if (isset($this->parameters['objectPrefix'])) { + $objectPrefix = $this->parameters['objectPrefix']; + } else { + $objectPrefix = 'urn:oid:'; + } + + $this->objectStore->copyObject($objectPrefix . $file->getId(), $this->constructUrn($preview)); + } } diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php index f9e4cc0f59a..c9381ce933c 100644 --- a/lib/private/Preview/Storage/PreviewFile.php +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -1,5 +1,13 @@ getBackend()->writePreview($preview, $stream); @@ -20,7 +24,7 @@ class StorageFactory implements IPreviewStorage { return $this->getBackend()->readPreview($preview); } - public function deletePreview(Preview $preview) { + public function deletePreview(Preview $preview): void { $this->getBackend()->deletePreview($preview); } @@ -35,10 +39,13 @@ class StorageFactory implements IPreviewStorage { $objectStore = $this->objectStoreConfig->buildObjectStore($objectStoreConfig); $this->backend = new ObjectStorePreviewStorage($objectStore, $objectStoreConfig['arguments']); } else { - $configDataDirectory = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); - $this->backend = new LocalPreviewStorage($configDataDirectory); + $this->backend = new LocalPreviewStorage($this->config); } return $this->backend; } + + public function migratePreview(Preview $preview, SimpleFile $file): void { + $this->getBackend()->migratePreview($preview, $file); + } } diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 7c8e1b13eb9..3bc63a55adb 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -135,7 +135,6 @@ class PreviewManager implements IPreview { $this->generator = new Generator( $this->config, $this, - $this->appData, new GeneratorHelper( $this->rootFolder, $this->config diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 583604515af..6a123cc56dd 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -13,6 +13,7 @@ use OC\Repair\AddBruteForceCleanupJob; use OC\Repair\AddCleanupDeletedUsersBackgroundJob; use OC\Repair\AddCleanupUpdaterBackupsJob; use OC\Repair\AddMetadataGenerationJob; +use OC\Repair\AddMovePreviewJob; use OC\Repair\AddRemoveOldTasksBackgroundJob; use OC\Repair\CleanTags; use OC\Repair\CleanUpAbandonedApps; @@ -199,6 +200,7 @@ class Repair implements IOutput { \OCP\Server::get(RemoveLegacyDatadirFile::class), \OCP\Server::get(AddCleanupDeletedUsersBackgroundJob::class), \OCP\Server::get(SanitizeAccountProperties::class), + \OCP\Server::get(AddMovePreviewJob::class), ]; } diff --git a/lib/private/Repair/AddMovePreviewJob.php b/lib/private/Repair/AddMovePreviewJob.php new file mode 100644 index 00000000000..5edcd64d46e --- /dev/null +++ b/lib/private/Repair/AddMovePreviewJob.php @@ -0,0 +1,27 @@ +jobList->add(MovePreviewJob::class); + } +} diff --git a/lib/private/Setup.php b/lib/private/Setup.php index 15f0a4eb617..8781127b30a 100644 --- a/lib/private/Setup.php +++ b/lib/private/Setup.php @@ -15,6 +15,7 @@ use InvalidArgumentException; use OC\Authentication\Token\PublicKeyTokenProvider; use OC\Authentication\Token\TokenCleanupJob; use OC\Core\BackgroundJobs\GenerateMetadataJob; +use OC\Core\BackgroundJobs\MovePreviewJob; use OC\Log\Rotate; use OC\Preview\BackgroundCleanupJob; use OC\TextProcessing\RemoveOldTasksBackgroundJob; @@ -505,6 +506,7 @@ class Setup { $jobList->add(RemoveOldTasksBackgroundJob::class); $jobList->add(CleanupDeletedUsers::class); $jobList->add(GenerateMetadataJob::class); + $jobList->add(MovePreviewJob::class); } /** From 6008852232205a02df68ca4290c65c1f6b81ec17 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 28 Aug 2025 14:25:55 +0200 Subject: [PATCH 04/14] feat(preview): Support multibucket storage Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 2 +- lib/private/Preview/Generator.php | 6 +- lib/private/Preview/GeneratorHelper.php | 6 +- .../Preview/Storage/IPreviewStorage.php | 1 + .../Preview/Storage/LocalPreviewStorage.php | 2 +- .../Storage/ObjectStorePreviewStorage.php | 146 +++++++- .../Preview/Storage/StorageFactory.php | 7 +- tests/lib/Preview/GeneratorTest.php | 330 ++++++++---------- 8 files changed, 290 insertions(+), 210 deletions(-) diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index d75673e6d2d..d326fab88d3 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -197,7 +197,7 @@ class MovePreviewJob extends TimedJob { try { $this->storageFactory->migratePreview($preview, $previewFile['file']); $previewFile['file']->delete(); - } catch (Exception $e) { + } catch (\Exception $e) { $this->previewMapper->delete($preview); throw $e; } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 29590d6fa93..e846a8221ab 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -174,6 +174,7 @@ class Generator { if ($maxPreviewImage === null) { $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); } + assert($maxPreviewImage); $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); $previewFile = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); @@ -324,6 +325,7 @@ class Generator { } foreach ($providers as $providerClosure) { + $provider = $this->helper->getProvider($providerClosure); if (!($provider instanceof IProviderV2)) { continue; @@ -351,7 +353,7 @@ class Generator { } try { - return $this->savePreview($file, $width, $height, $crop, $max, $preview, $version); + return $this->savePreview($file, $preview->width(), $preview->height(), $crop, $max, $preview, $version); } catch (NotPermittedException $e) { throw new NotFoundException(); } @@ -545,7 +547,7 @@ class Generator { public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); - $previewEntry->setStorageId((int)$file->getStorage()->getId()); + $previewEntry->setStorageId((int)$file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); $previewEntry->setHeight($height); $previewEntry->setVersion($version); diff --git a/lib/private/Preview/GeneratorHelper.php b/lib/private/Preview/GeneratorHelper.php index e914dcc2002..7114a412e36 100644 --- a/lib/private/Preview/GeneratorHelper.php +++ b/lib/private/Preview/GeneratorHelper.php @@ -45,11 +45,7 @@ class GeneratorHelper { return $provider->getThumbnail($file, $maxWidth, $maxHeight) ?? false; } - /** - * @param ISimpleFile $maxPreview - * @return IImage - */ - public function getImage(ISimpleFile $maxPreview) { + public function getImage(ISimpleFile $maxPreview): IImage { $image = new OCPImage(); $image->loadFromData($maxPreview->getContent()); return $image; diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index 8ebf98b7493..787518e53a4 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -25,6 +25,7 @@ interface IPreviewStorage { * Migration helper * * To remove at some point + * @throw \Exception */ public function migratePreview(Preview $preview, SimpleFile $file): void; } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 5971113cb53..2c176948c79 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -72,7 +72,7 @@ class LocalPreviewStorage implements IPreviewStorage { } $this->createParentFiles($previewPath); echo 'Copying ' . $sourcePath . ' to ' . $destinationPath . PHP_EOL; - $ok = copy($sourcePath, $destinationPath); + $ok = rename($sourcePath, $destinationPath); if (!$ok) { throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); } diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 11944dbf846..a8fe79d4732 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -11,23 +11,32 @@ declare(strict_types=1); namespace OC\Preview\Storage; use Icewind\Streams\CountWrapper; +use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; +use OCP\IConfig; +/** + * @psalm-type ObjectStoreDefinition = array{store: IObjectStore, objectPrefix: string, config?: array} + */ class ObjectStorePreviewStorage implements IPreviewStorage { - private string $objectPrefix = 'urn:oid:preview:'; /** - * @param array{objectPrefix?: string, ...} $parameters + * @var array<'root'|int, ObjectStoreDefinition> */ + private array $objectStoreCache = []; + + private bool $isMultibucketEnabled; + private bool $isMultibucketPreviewDistributionEnabled; + public function __construct( - private readonly IObjectStore $objectStore, - private readonly array $parameters, + private readonly PrimaryObjectStoreConfig $objectStoreConfig, + readonly private IConfig $config, ) { - if (isset($parameters['objectPrefix'])) { - $this->objectPrefix = $parameters['objectPrefix'] . 'preview:'; - } + $this->isMultibucketEnabled = is_array($config->getSystemValue('objectstore_multibucket')); + $this->isMultibucketPreviewDistributionEnabled = $config->getSystemValueBool('objectstore.multibucket.preview-distribution'); } public function writePreview(Preview $preview, $stream): false|int { @@ -44,29 +53,128 @@ class ObjectStorePreviewStorage implements IPreviewStorage { $size = $writtenSize; }); - $this->objectStore->writeObject($this->constructUrn($preview), $countStream); + [ + 'objectPrefix' => $objectPrefix, + 'store' => $store, + ] = $this->getObjectStoreForPreview($preview); + + $store->writeObject($this->constructUrn($objectPrefix, $preview->getId()), $countStream); return $size; } public function readPreview(Preview $preview) { - return $this->objectStore->readObject($this->constructUrn($preview)); + [ + 'objectPrefix' => $objectPrefix, + 'store' => $store, + ] = $this->getObjectStoreForPreview($preview); + return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); } public function deletePreview(Preview $preview) { - return $this->objectStore->deleteObject($this->constructUrn($preview)); - } - - private function constructUrn(Preview $preview): string { - return $this->objectPrefix . $preview->getId(); + [ + 'objectPrefix' => $objectPrefix, + 'store' => $store, + ] = $this->getObjectStoreForPreview($preview); + return $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); } public function migratePreview(Preview $preview, SimpleFile $file): void { - if (isset($this->parameters['objectPrefix'])) { - $objectPrefix = $this->parameters['objectPrefix']; - } else { - $objectPrefix = 'urn:oid:'; + foreach ([false, true] as $fallback) { + [ + 'objectPrefix' => $objectPrefix, + 'store' => $store, + 'config' => $config, + ] = $this->getObjectStoreForPreview($preview, $fallback); + + $oldObjectPrefix = 'urn:oid:'; + if (isset($config['objectPrefix'])) { + $oldObjectPrefix = $config['objectPrefix']; + } + + try { + $store->copyObject($this->constructUrn($oldObjectPrefix, $file->getId()), $this->constructUrn($objectPrefix, $preview->getId())); + break; + } catch (NotFoundException $e) { + if (!$fallback && $this->isMultibucketPreviewDistributionEnabled) { + continue; + } + throw $e; + } + } + } + + /** + * @return ObjectStoreDefinition + */ + private function getMultiBucketObjectStore(int $number): array { + /** + * @var array{class: class-string, ...} $config + */ + $config = $this->config->getSystemValue('objectstore_multibucket'); + + if (!isset($config['arguments'])) { + $config['arguments'] = []; } - $this->objectStore->copyObject($objectPrefix . $file->getId(), $this->constructUrn($preview)); + /* + * Use any provided bucket argument as prefix + * and add the mapping from parent/child => bucket + */ + if (!isset($config['arguments']['bucket'])) { + $config['arguments']['bucket'] = ''; + } + + $config['arguments']['bucket'] .= "-preview-$number"; + + $objectPrefix = 'urn:oid:preview:'; + if (isset($config['objectPrefix'])) { + $objectPrefix = $config['objectPrefix'] . 'preview:'; + } + + return [ + 'store' => new $config['class']($config['arguments']), + 'objectPrefix' => $objectPrefix, + 'config' => $config, + ]; + } + + /** + * @return ObjectStoreDefinition + */ + private function getRootObjectStore(): array { + if (!isset($this->objectStoreCache['root'])) { + $rootConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); + $objectPrefix = 'urn:oid:preview:'; + if (isset($rootConfig['arguments']['objectPrefix'])) { + $objectPrefix = $rootConfig['arguments']['objectPrefix'] . 'preview:'; + } + $this->objectStoreCache['root'] = [ + 'store' => $this->objectStoreConfig->buildObjectStore($rootConfig), + 'objectPrefix' => $objectPrefix, + ]; + } + return $this->objectStoreCache['root']; + } + + /** + * @return ObjectStoreDefinition + */ + private function getObjectStoreForPreview(Preview $preview, bool $oldFallback = false): array { + if (!$this->isMultibucketEnabled || !$this->isMultibucketPreviewDistributionEnabled || $oldFallback) { + return $this->getRootObjectStore(); + } + + $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); + $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); + + if (!isset($this->objectStoreCache[$bucketNumber])) { + $this->objectStoreCache[$bucketNumber] = $this->getMultiBucketObjectStore($bucketNumber); + } + + return $this->objectStoreCache[$bucketNumber]; + } + + private function constructUrn(string $objectPrefix, int $id): string { + return $objectPrefix . $id; } } diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index a8ba57e8e44..077395c8b93 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -33,11 +33,8 @@ class StorageFactory implements IPreviewStorage { return $this->backend; } - $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); - - if ($objectStoreConfig) { - $objectStore = $this->objectStoreConfig->buildObjectStore($objectStoreConfig); - $this->backend = new ObjectStorePreviewStorage($objectStore, $objectStoreConfig['arguments']); + if ($this->objectStoreConfig->hasObjectStore()) { + $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config); } else { $this->backend = new LocalPreviewStorage($this->config); } diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index edf5418da6e..91fa3537468 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -7,11 +7,14 @@ namespace Test\Preview; +use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; use OC\Preview\Generator; use OC\Preview\GeneratorHelper; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IAppData; +use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFile; use OCP\Files\SimpleFS\ISimpleFolder; @@ -20,110 +23,108 @@ use OCP\IImage; use OCP\IPreview; use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IProviderV2; +use PHPUnit\Framework\MockObject\MockObject; use Psr\Log\LoggerInterface; +use Test\TestCase; -class GeneratorTest extends \Test\TestCase { - /** @var IConfig&\PHPUnit\Framework\MockObject\MockObject */ - private $config; - - /** @var IPreview&\PHPUnit\Framework\MockObject\MockObject */ - private $previewManager; - - /** @var IAppData&\PHPUnit\Framework\MockObject\MockObject */ - private $appData; - - /** @var GeneratorHelper&\PHPUnit\Framework\MockObject\MockObject */ - private $helper; - - /** @var IEventDispatcher&\PHPUnit\Framework\MockObject\MockObject */ - private $eventDispatcher; - - /** @var Generator */ - private $generator; - - private LoggerInterface&\PHPUnit\Framework\MockObject\MockObject $logger; +class GeneratorTest extends TestCase { + private IConfig&MockObject $config; + private IPreview&MockObject $previewManager; + private GeneratorHelper&MockObject $helper; + private IEventDispatcher&MockObject $eventDispatcher; + private Generator $generator; + private LoggerInterface&MockObject $logger; + private StorageFactory&MockObject $storageFactory; + private PreviewMapper&MockObject $previewMapper; protected function setUp(): void { parent::setUp(); $this->config = $this->createMock(IConfig::class); $this->previewManager = $this->createMock(IPreview::class); - $this->appData = $this->createMock(IAppData::class); $this->helper = $this->createMock(GeneratorHelper::class); $this->eventDispatcher = $this->createMock(IEventDispatcher::class); $this->logger = $this->createMock(LoggerInterface::class); + $this->previewMapper = $this->createMock(PreviewMapper::class); + $this->storageFactory = $this->createMock(StorageFactory::class); $this->generator = new Generator( $this->config, $this->previewManager, - $this->appData, $this->helper, $this->eventDispatcher, $this->logger, + $this->previewMapper, + $this->storageFactory ); } - public function testGetCachedPreview(): void { + private function getFile(int $fileId, string $mimeType): File { + $mountPoint = $this->createMock(IMountPoint::class); + $mountPoint->method('getNumericStorageId')->willReturn(42); $file = $this->createMock(File::class); $file->method('isReadable') ->willReturn(true); $file->method('getMimeType') - ->willReturn('myMimeType'); + ->willReturn($mimeType); $file->method('getId') - ->willReturn(42); + ->willReturn($fileId); + $file->method('getMountPoint') + ->willReturn($mountPoint); + return $file; + } + + public function testGetCachedPreview(): void { + $file = $this->getFile(42, 'myMimeType'); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) ->willReturn(true); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); + $maxPreview = new Preview(); + $maxPreview->setWidth(1000); + $maxPreview->setHeight(1000); + $maxPreview->setIsMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(-1); + $maxPreview->setCrop(false); + $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn('1000-1000-max.png'); - $maxPreview->method('getSize')->willReturn(1000); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); + $previewFile = new Preview(); + $previewFile->setWidth(256); + $previewFile->setHeight(256); + $previewFile->setIsMax(false); + $previewFile->setSize(1000); + $previewFile->setVersion(-1); + $previewFile->setCrop(false); + $previewFile->setMimetype(IPreview::MIMETYPE_PNG); - $previewFile = $this->createMock(ISimpleFile::class); - $previewFile->method('getSize')->willReturn(1000); - $previewFile->method('getName')->willReturn('256-256.png'); - - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview, $previewFile]); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + $previewFile, + ]]); $this->eventDispatcher->expects($this->once()) ->method('dispatchTyped') ->with(new BeforePreviewFetchedEvent($file, 100, 100, false, IPreview::MODE_FILL, null)); $result = $this->generator->getPreview($file, 100, 100); - $this->assertSame($previewFile, $result); + $this->assertSame('256-256.png', $result->getName()); + $this->assertSame(1000, $result->getSize()); } public function testGetNewPreview(): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getMimeType') - ->willReturn('myMimeType'); - $file->method('getId') - ->willReturn(42); + $file = $this->getFile(42, 'myMimeType'); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) ->willReturn(true); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willThrowException(new NotFoundException()); - - $this->appData->method('newFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => []]); $this->config->method('getSystemValue') ->willReturnCallback(function ($key, $default) { @@ -175,7 +176,7 @@ class GeneratorTest extends \Test\TestCase { $image->method('dataMimeType')->willReturn('image/png'); $this->helper->method('getThumbnail') - ->willReturnCallback(function ($provider, $file, $x, $y) use ($invalidProvider, $validProvider, $image) { + ->willReturnCallback(function ($provider, $file, $x, $y) use ($invalidProvider, $validProvider, $image): false|IImage { if ($provider === $validProvider) { return $image; } else { @@ -186,29 +187,34 @@ class GeneratorTest extends \Test\TestCase { $image->method('data') ->willReturn('my data'); - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName')->willReturn('2048-2048-max.png'); - $maxPreview->method('getMimeType')->willReturn('image/png'); - $maxPreview->method('getSize')->willReturn(1000); + $maxPreview = new Preview(); + $maxPreview->setWidth(2048); + $maxPreview->setHeight(2048); + $maxPreview->setIsMax(true); + $maxPreview->setSize(1000); + $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); - $previewFile = $this->createMock(ISimpleFile::class); - $previewFile->method('getSize')->willReturn(1000); + $this->previewMapper->method('insert') + ->willReturnCallback(fn (Preview $preview): Preview => $preview); - $previewFolder->method('getDirectoryListing') - ->willReturn([]); - $previewFolder->method('newFile') - ->willReturnMap([ - ['2048-2048-max.png', 'my data', $maxPreview], - ['256-256.png', 'my resized data', $previewFile], - ]); + $this->previewMapper->method('update') + ->willReturnCallback(fn (Preview $preview): Preview => $preview); - $previewFolder->method('getFile') - ->with($this->equalTo('256-256.png')) - ->willThrowException(new NotFoundException()); + $this->storageFactory->method('writePreview') + ->willReturnCallback(function (Preview $preview, string $data): int { + switch ($preview->getName()) { + case '2048-2048-max.png': + $this->assertSame('my data', $data); + return 1000; + case '256-256.png': + $this->assertSame('my resized data', $data); + return 1000; + } + $this->fail("file name is wrong:". $preview->getName()); + }); $image = $this->getMockImage(2048, 2048, 'my resized data'); $this->helper->method('getImage') - ->with($this->equalTo($maxPreview)) ->willReturn($image); $this->eventDispatcher->expects($this->once()) @@ -216,39 +222,32 @@ class GeneratorTest extends \Test\TestCase { ->with(new BeforePreviewFetchedEvent($file, 100, 100, false, IPreview::MODE_FILL, null)); $result = $this->generator->getPreview($file, 100, 100); - $this->assertSame($previewFile, $result); + $this->assertSame('256-256.png', $result->getName()); + $this->assertSame(1000, $result->getSize()); } public function testInvalidMimeType(): void { $this->expectException(NotFoundException::class); - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getId') - ->willReturn(42); + $file = $this->getFile(42, 'invalidType'); $this->previewManager->method('isMimeSupported') ->with('invalidType') ->willReturn(false); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); + $maxPreview = new Preview(); + $maxPreview->setWidth(2048); + $maxPreview->setHeight(2048); + $maxPreview->setIsMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(-1); + $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn('2048-2048-max.png'); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview]); - - $previewFolder->method('getFile') - ->with($this->equalTo('1024-512-crop.png')) - ->willThrowException(new NotFoundException()); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + ]]); $this->eventDispatcher->expects($this->once()) ->method('dispatchTyped') @@ -258,31 +257,31 @@ class GeneratorTest extends \Test\TestCase { } public function testReturnCachedPreviewsWithoutCheckingSupportedMimetype(): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getId') - ->willReturn(42); + $file = $this->getFile(42, 'myMimeType'); + $maxPreview = new Preview(); + $maxPreview->setWidth(2048); + $maxPreview->setHeight(2048); + $maxPreview->setIsMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(-1); + $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); + $previewFile = new Preview(); + $previewFile->setWidth(1024); + $previewFile->setHeight(512); + $previewFile->setIsMax(false); + $previewFile->setSize(1000); + $previewFile->setCrop(true); + $previewFile->setVersion(-1); + $previewFile->setMimetype(IPreview::MIMETYPE_PNG); - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn('2048-2048-max.png'); - $maxPreview->method('getSize')->willReturn(1000); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - - $preview = $this->createMock(ISimpleFile::class); - $preview->method('getSize')->willReturn(1000); - $preview->method('getName')->willReturn('1024-512-crop.png'); - - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview, $preview]); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + $previewFile, + ]]); $this->previewManager->expects($this->never()) ->method('isMimeSupported'); @@ -292,25 +291,15 @@ class GeneratorTest extends \Test\TestCase { ->with(new BeforePreviewFetchedEvent($file, 1024, 512, true, IPreview::MODE_COVER, 'invalidType')); $result = $this->generator->getPreview($file, 1024, 512, true, IPreview::MODE_COVER, 'invalidType'); - $this->assertSame($preview, $result); + $this->assertSame('1024-512-crop.png', $result->getName()); } public function testNoProvider(): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getMimeType') - ->willReturn('myMimeType'); - $file->method('getId') - ->willReturn(42); + $file = $this->getFile(42, 'myMimeType'); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); - - $previewFolder->method('getDirectoryListing') - ->willReturn([]); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => []]); $this->previewManager->method('getProviders') ->willReturn([]); @@ -380,65 +369,52 @@ class GeneratorTest extends \Test\TestCase { ]; } - /** - * - * @param int $maxX - * @param int $maxY - * @param int $reqX - * @param int $reqY - * @param bool $crop - * @param string $mode - * @param int $expectedX - * @param int $expectedY - */ #[\PHPUnit\Framework\Attributes\DataProvider('dataSize')] - public function testCorrectSize($maxX, $maxY, $reqX, $reqY, $crop, $mode, $expectedX, $expectedY): void { - $file = $this->createMock(File::class); - $file->method('isReadable') - ->willReturn(true); - $file->method('getMimeType') - ->willReturn('myMimeType'); - $file->method('getId') - ->willReturn(42); + public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $crop, string $mode, int $expectedX, int $expectedY): void { + $file = $this->getFile(42, 'myMimeType'); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) ->willReturn(true); - $previewFolder = $this->createMock(ISimpleFolder::class); - $this->appData->method('getFolder') - ->with($this->equalTo(42)) - ->willReturn($previewFolder); + $maxPreview = new Preview(); + $maxPreview->setWidth($maxX); + $maxPreview->setHeight($maxY); + $maxPreview->setIsMax(true); + $maxPreview->setSize(1000); + $maxPreview->setVersion(-1); + $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); - $maxPreview = $this->createMock(ISimpleFile::class); - $maxPreview->method('getName') - ->willReturn($maxX . '-' . $maxY . '-max.png'); - $maxPreview->method('getMimeType') - ->willReturn('image/png'); - $maxPreview->method('getSize')->willReturn(1000); + $this->assertSame($maxPreview->getName(), $maxX . '-' . $maxY . '-max.png'); + $this->assertSame($maxPreview->getMimetypeValue(), 'image/png'); - $previewFolder->method('getDirectoryListing') - ->willReturn([$maxPreview]); + $this->previewMapper->method('getAvailablePreviews') + ->with($this->equalTo([42])) + ->willReturn([42 => [ + $maxPreview, + ]]); $filename = $expectedX . '-' . $expectedY; if ($crop) { $filename .= '-crop'; } $filename .= '.png'; - $previewFolder->method('getFile') - ->with($this->equalTo($filename)) - ->willThrowException(new NotFoundException()); $image = $this->getMockImage($maxX, $maxY); $this->helper->method('getImage') - ->with($this->equalTo($maxPreview)) ->willReturn($image); - $preview = $this->createMock(ISimpleFile::class); - $preview->method('getSize')->willReturn(1000); - $previewFolder->method('newFile') - ->with($this->equalTo($filename)) - ->willReturn($preview); + $this->previewMapper->method('insert') + ->willReturnCallback(function (Preview $preview) use ($filename): Preview { + $this->assertSame($preview->getName(), $filename); + return $preview; + }); + + $this->previewMapper->method('update') + ->willReturnCallback(fn (Preview $preview): Preview => $preview); + + $this->storageFactory->method('writePreview') + ->willReturn(1000); $this->eventDispatcher->expects($this->once()) ->method('dispatchTyped') @@ -446,9 +422,9 @@ class GeneratorTest extends \Test\TestCase { $result = $this->generator->getPreview($file, $reqX, $reqY, $crop, $mode); if ($expectedX === $maxX && $expectedY === $maxY) { - $this->assertSame($maxPreview, $result); + $this->assertSame($maxPreview->getName(), $result->getName()); } else { - $this->assertSame($preview, $result); + $this->assertSame($filename, $result->getName()); } } From bba96678826ba9b6726484ff05a1d7fd48cfa8b7 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Fri, 29 Aug 2025 14:14:07 +0200 Subject: [PATCH 05/14] perf(preview): Adapt BackgroundCleanupJob to new previews table Signed-off-by: Carl Schwan --- build/psalm-baseline.xml | 18 -- lib/private/Preview/BackgroundCleanupJob.php | 185 +++++++----------- lib/private/Preview/Db/PreviewMapper.php | 30 +++ lib/private/Preview/Generator.php | 6 +- lib/public/AppFramework/Db/QBMapper.php | 1 - .../Preview/BeforePreviewFetchedEvent.php | 1 + .../lib/Preview/BackgroundCleanupJobTest.php | 122 ++---------- 7 files changed, 128 insertions(+), 235 deletions(-) diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 621e368f0a3..606cb3b5bb1 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -3721,16 +3721,6 @@ - - - - - - - - - - @@ -3991,14 +3981,6 @@ - - - mode]]> - - - - - diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index 3138abb1bf9..62e3303dc4e 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -8,23 +8,25 @@ declare(strict_types=1); */ namespace OC\Preview; -use OC\Preview\Storage\Root; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; use OCP\DB\QueryBuilder\IQueryBuilder; -use OCP\Files\IMimeTypeLoader; -use OCP\Files\NotFoundException; -use OCP\Files\NotPermittedException; use OCP\IDBConnection; +/** + * @psalm-type FileId int + * @psalm-type StorageId int + */ class BackgroundCleanupJob extends TimedJob { public function __construct( ITimeFactory $timeFactory, - private IDBConnection $connection, - private Root $previewFolder, - private IMimeTypeLoader $mimeTypeLoader, - private bool $isCLI, + readonly private IDBConnection $connection, + readonly private PreviewMapper $previewMapper, + readonly private StorageFactory $storageFactory, + readonly private bool $isCLI, ) { parent::__construct($timeFactory); // Run at most once an hour @@ -32,88 +34,37 @@ class BackgroundCleanupJob extends TimedJob { $this->setTimeSensitivity(self::TIME_INSENSITIVE); } - public function run($argument) { - foreach ($this->getDeletedFiles() as $fileId) { - try { - $preview = $this->previewFolder->getFolder((string)$fileId); - $preview->delete(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { - // continue + public function run($argument): void { + foreach ($this->getDeletedFiles() as $chunk) { + foreach ($chunk as $storage => $fileIds) { + foreach ($this->previewMapper->getByFileIds($storage, $fileIds) as $previews) { + $previewIds = []; + foreach ($previews as $preview) { + $previewIds[] = $preview->getId(); + $this->storageFactory->deletePreview($preview); + } + + $this->previewMapper->deleteByIds($storage, $previewIds); + }; } } } + /** + * @return \Iterator> + */ private function getDeletedFiles(): \Iterator { - yield from $this->getOldPreviewLocations(); - yield from $this->getNewPreviewLocations(); - } - - private function getOldPreviewLocations(): \Iterator { if ($this->connection->getShardDefinition('filecache')) { - // sharding is new enough that we don't need to support this - return; - } - - $qb = $this->connection->getQueryBuilder(); - $qb->select('a.name') - ->from('filecache', 'a') - ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( - $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' - )) - ->where( - $qb->expr()->andX( - $qb->expr()->isNull('b.fileid'), - $qb->expr()->eq('a.storage', $qb->createNamedParameter($this->previewFolder->getStorageId())), - $qb->expr()->eq('a.parent', $qb->createNamedParameter($this->previewFolder->getId())), - $qb->expr()->like('a.name', $qb->createNamedParameter('__%')), - $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))) - ) - ); - - if (!$this->isCLI) { - $qb->setMaxResults(10); - } - - $cursor = $qb->executeQuery(); - - while ($row = $cursor->fetch()) { - yield $row['name']; - } - - $cursor->closeCursor(); - } - - private function getNewPreviewLocations(): \Iterator { - $qb = $this->connection->getQueryBuilder(); - $qb->select('path', 'mimetype') - ->from('filecache') - ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId()))); - $cursor = $qb->executeQuery(); - $data = $cursor->fetch(); - $cursor->closeCursor(); - - if ($data === null) { - return []; - } - - if ($this->connection->getShardDefinition('filecache')) { - $chunks = $this->getAllPreviewIds($data['path'], 1000); + $chunks = $this->getAllPreviewIds(1000); foreach ($chunks as $chunk) { - yield from $this->findMissingSources($chunk); + foreach ($chunk as $storage => $preview) { + yield [$storage => $this->findMissingSources($storage, $preview)]; + } } return; } - /* - * This lovely like is the result of the way the new previews are stored - * We take the md5 of the name (fileid) and split the first 7 chars. That way - * there are not a gazillion files in the root of the preview appdata. - */ - $like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%'; - /* * Deleting a file will not delete related previews right away. * @@ -130,19 +81,12 @@ class BackgroundCleanupJob extends TimedJob { * If the related file is deleted, b.fileid will be null and the preview folder can be deleted. */ $qb = $this->connection->getQueryBuilder(); - $qb->select('a.name') - ->from('filecache', 'a') - ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( - $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' + $qb->select('p.storage_id', 'p.file_id') + ->from('previews', 'p') + ->leftJoin('p', 'filecache', 'f', $qb->expr()->eq( + 'p.file_id', 'f.fileid' )) - ->where( - $qb->expr()->andX( - $qb->expr()->eq('a.storage', $qb->createNamedParameter($this->previewFolder->getStorageId())), - $qb->expr()->isNull('b.fileid'), - $qb->expr()->like('a.path', $qb->createNamedParameter($like)), - $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))) - ) - ); + ->where($qb->expr()->isNull('f.fileid')); if (!$this->isCLI) { $qb->setMaxResults(10); @@ -150,29 +94,38 @@ class BackgroundCleanupJob extends TimedJob { $cursor = $qb->executeQuery(); + $lastStorageId = null; + /** @var FileId[] $tmpResult */ + $tmpResult = []; while ($row = $cursor->fetch()) { - yield $row['name']; + if ($lastStorageId === null) { + $lastStorageId = $row['storage_id']; + } else if ($lastStorageId !== $row['storage_id']) { + yield [$lastStorageId => $tmpResult]; + $tmpResult = []; + $lastStorageId = $row['storage_id']; + } + $tmpResult[] = $row['file_id']; + } + + if (!empty($tmpResult)) { + yield [$lastStorageId => $tmpResult]; } $cursor->closeCursor(); } - private function getAllPreviewIds(string $previewRoot, int $chunkSize): \Iterator { - // See `getNewPreviewLocations` for some more info about the logic here - $like = $this->connection->escapeLikeParameter($previewRoot) . '/_/_/_/_/_/_/_/%'; - + /** + * @return \Iterator> + */ + private function getAllPreviewIds(int $chunkSize): \Iterator { $qb = $this->connection->getQueryBuilder(); - $qb->select('name', 'fileid') - ->from('filecache') + $qb->select('id', 'file_id', 'storage_id') + ->from('previews') ->where( - $qb->expr()->andX( - $qb->expr()->eq('storage', $qb->createNamedParameter($this->previewFolder->getStorageId())), - $qb->expr()->like('path', $qb->createNamedParameter($like)), - $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))), - $qb->expr()->gt('fileid', $qb->createParameter('min_id')), - ) + $qb->expr()->gt('id', $qb->createParameter('min_id')), ) - ->orderBy('fileid', 'ASC') + ->orderBy('id', 'ASC') ->setMaxResults($chunkSize); $minId = 0; @@ -180,21 +133,33 @@ class BackgroundCleanupJob extends TimedJob { $qb->setParameter('min_id', $minId); $rows = $qb->executeQuery()->fetchAll(); if (count($rows) > 0) { - $minId = $rows[count($rows) - 1]['fileid']; - yield array_map(function ($row) { - return (int)$row['name']; - }, $rows); + $minId = $rows[count($rows) - 1]['id']; + $result = []; + foreach ($rows as $row) { + if (!isset($result[$row['storage_id']])) { + $result[$row['storage_id']] = []; + } + $result[$row['storage_id']][] = $row['file_id']; + } + yield $result; } else { break; } } } - private function findMissingSources(array $ids): array { + /** + * @param FileId[] $ids + * @return FileId[] + */ + private function findMissingSources(int $storage, array $ids): array { $qb = $this->connection->getQueryBuilder(); $qb->select('fileid') ->from('filecache') - ->where($qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY))); + ->where($qb->expr()->andX( + $qb->expr()->in('fileid', $qb->createNamedParameter($ids, IQueryBuilder::PARAM_INT_ARRAY)), + $qb->expr()->eq('storage', $qb->createNamedParameter($storage, IQueryBuilder::PARAM_INT)), + )); $found = $qb->executeQuery()->fetchAll(\PDO::FETCH_COLUMN); return array_diff($ids, $found); } diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index 5faa508c8e0..62023638150 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -63,4 +63,34 @@ class PreviewMapper extends QBMapper { return null; } } + + /** + * @param int[] $fileIds + * @return array + */ + public function getByFileIds(int $storageId, array $fileIds): array { + $selectQb = $this->db->getQueryBuilder(); + $selectQb->select('*') + ->from(self::TABLE_NAME) + ->where($selectQb->expr()->andX( + $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), + )); + $previews = array_fill_keys($fileIds, []); + foreach ($this->yieldEntities($selectQb) as $preview) { + $previews[$preview->getFileId()][] = $preview; + } + return $previews; + } + + /** + * @param int[] $previewIds + */ + public function deleteByIds(int $storageId, array $previewIds): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete(self::TABLE_NAME) + ->where($qb->expr()->andX( + $qb->expr()->eq('storage_id', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)), + $qb->expr()->in('id', $qb->createNamedParameter($previewIds, IQueryBuilder::PARAM_INT_ARRAY)) + ))->executeStatement(); + } } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index e846a8221ab..2dfbfa6db62 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -174,7 +174,6 @@ class Generator { if ($maxPreviewImage === null) { $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); } - assert($maxPreviewImage); $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); $previewFile = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); @@ -508,7 +507,6 @@ class Generator { self::unguardWithSemaphore($sem); } - $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $version); if ($cacheResult) { $previewEntry = $this->savePreview($file, $width, $height, $crop, false, $preview, $version); @@ -519,11 +517,9 @@ class Generator { } /** - * @param string $mimeType - * @return null|string * @throws \InvalidArgumentException */ - private function getExtension($mimeType) { + private function getExtension(string $mimeType): string { switch ($mimeType) { case 'image/png': return 'png'; diff --git a/lib/public/AppFramework/Db/QBMapper.php b/lib/public/AppFramework/Db/QBMapper.php index 7fb5b2a9afd..d80bb5aec8b 100644 --- a/lib/public/AppFramework/Db/QBMapper.php +++ b/lib/public/AppFramework/Db/QBMapper.php @@ -84,7 +84,6 @@ abstract class QBMapper { return $entity; } - /** * Creates a new entry in the db from an entity * diff --git a/lib/public/Preview/BeforePreviewFetchedEvent.php b/lib/public/Preview/BeforePreviewFetchedEvent.php index 8ab875070d9..69cd281ac02 100644 --- a/lib/public/Preview/BeforePreviewFetchedEvent.php +++ b/lib/public/Preview/BeforePreviewFetchedEvent.php @@ -21,6 +21,7 @@ use OCP\IPreview; */ class BeforePreviewFetchedEvent extends \OCP\EventDispatcher\Event { /** + * @param null|IPreview::MODE_FILL|IPreview::MODE_COVER $mode * @since 25.0.1 */ public function __construct( diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index ab904f2b499..89503085fa4 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -9,7 +9,10 @@ namespace Test\Preview; use OC\Files\Storage\Temporary; use OC\Preview\BackgroundCleanupJob; +use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; use OC\Preview\Storage\Root; +use OC\Preview\Storage\StorageFactory; use OC\PreviewManager; use OC\SystemConfig; use OCP\App\IAppManager; @@ -42,6 +45,8 @@ class BackgroundCleanupJobTest extends \Test\TestCase { private IRootFolder $rootFolder; private IMimeTypeLoader $mimeTypeLoader; private ITimeFactory $timeFactory; + private PreviewMapper $previewMapper; + private StorageFactory $previewStorageFactory; protected function setUp(): void { parent::setUp(); @@ -65,6 +70,8 @@ class BackgroundCleanupJobTest extends \Test\TestCase { $this->rootFolder = Server::get(IRootFolder::class); $this->mimeTypeLoader = Server::get(IMimeTypeLoader::class); $this->timeFactory = Server::get(ITimeFactory::class); + $this->previewMapper = Server::get(PreviewMapper::class); + $this->previewStorageFactory = Server::get(StorageFactory::class); } protected function tearDown(): void { @@ -78,13 +85,6 @@ class BackgroundCleanupJobTest extends \Test\TestCase { parent::tearDown(); } - private function getRoot(): Root { - return new Root( - Server::get(IRootFolder::class), - Server::get(SystemConfig::class) - ); - } - private function setup11Previews(): array { $userFolder = $this->rootFolder->getUserFolder($this->userId); @@ -99,130 +99,50 @@ class BackgroundCleanupJobTest extends \Test\TestCase { return $files; } - private function countPreviews(Root $previewRoot, array $fileIds): int { - $i = 0; - - foreach ($fileIds as $fileId) { - try { - $previewRoot->getFolder((string)$fileId); - } catch (NotFoundException $e) { - continue; - } - - $i++; - } - - return $i; + private function countPreviews(PreviewMapper $previewMapper, array $fileIds): int { + $previews = $previewMapper->getAvailablePreviews($fileIds); + return array_reduce($previews, fn (int $result, array $previews) => $result + count($previews), 0); } public function testCleanupSystemCron(): void { $files = $this->setup11Previews(); - $fileIds = array_map(function (File $f) { - return $f->getId(); - }, $files); + $fileIds = array_map(fn (File $f): int => $f->getId(), $files); - $root = $this->getRoot(); - - $this->assertSame(11, $this->countPreviews($root, $fileIds)); - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $root, $this->mimeTypeLoader, true); + $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); + $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewMapper, $this->previewStorageFactory, true); $job->run([]); foreach ($files as $file) { $file->delete(); } - $root = $this->getRoot(); - $this->assertSame(11, $this->countPreviews($root, $fileIds)); + $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); $job->run([]); - $root = $this->getRoot(); - $this->assertSame(0, $this->countPreviews($root, $fileIds)); + $this->assertSame(0, $this->countPreviews($this->previewMapper, $fileIds)); } public function testCleanupAjax(): void { if ($this->connection->getShardDefinition('filecache')) { $this->markTestSkipped('ajax cron is not supported for sharded setups'); - return; } $files = $this->setup11Previews(); - $fileIds = array_map(function (File $f) { - return $f->getId(); - }, $files); + $fileIds = array_map(fn (File $f): int => $f->getId(), $files); - $root = $this->getRoot(); - - $this->assertSame(11, $this->countPreviews($root, $fileIds)); - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $root, $this->mimeTypeLoader, false); + $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); + $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewMapper, $this->previewStorageFactory, false); $job->run([]); foreach ($files as $file) { $file->delete(); } - $root = $this->getRoot(); - $this->assertSame(11, $this->countPreviews($root, $fileIds)); + $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); $job->run([]); - $root = $this->getRoot(); - $this->assertSame(1, $this->countPreviews($root, $fileIds)); + $this->assertSame(1, $this->countPreviews($this->previewMapper, $fileIds)); $job->run([]); - $root = $this->getRoot(); - $this->assertSame(0, $this->countPreviews($root, $fileIds)); - } - - public function testOldPreviews(): void { - if ($this->connection->getShardDefinition('filecache')) { - $this->markTestSkipped('old previews are not supported for sharded setups'); - return; - } - $appdata = Server::get(IAppDataFactory::class)->get('preview'); - - $f1 = $appdata->newFolder('123456781'); - $f1->newFile('foo.jpg', 'foo'); - $f2 = $appdata->newFolder('123456782'); - $f2->newFile('foo.jpg', 'foo'); - $f2 = $appdata->newFolder((string)PHP_INT_MAX - 1); - $f2->newFile('foo.jpg', 'foo'); - - /* - * Cleanup of OldPreviewLocations should only remove numeric folders on AppData level, - * therefore these files should stay untouched. - */ - $appdata->getFolder('/')->newFile('not-a-directory', 'foo'); - $appdata->getFolder('/')->newFile('133742', 'bar'); - - $appdata = Server::get(IAppDataFactory::class)->get('preview'); - // AppData::getDirectoryListing filters all non-folders - $this->assertSame(3, count($appdata->getDirectoryListing())); - try { - $appdata->getFolder('/')->getFile('not-a-directory'); - } catch (NotFoundException) { - $this->fail('Could not find file \'not-a-directory\''); - } - try { - $appdata->getFolder('/')->getFile('133742'); - } catch (NotFoundException) { - $this->fail('Could not find file \'133742\''); - } - - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->getRoot(), $this->mimeTypeLoader, true); - $job->run([]); - - $appdata = Server::get(IAppDataFactory::class)->get('preview'); - - // Check if the files created above are still present - // Remember: AppData::getDirectoryListing filters all non-folders - $this->assertSame(0, count($appdata->getDirectoryListing())); - try { - $appdata->getFolder('/')->getFile('not-a-directory'); - } catch (NotFoundException) { - $this->fail('Could not find file \'not-a-directory\''); - } - try { - $appdata->getFolder('/')->getFile('133742'); - } catch (NotFoundException) { - $this->fail('Could not find file \'133742\''); - } + $this->assertSame(0, $this->countPreviews($this->previewMapper, $fileIds)); } } From b0357663b96cf4a3a9bd34194466a4ce76e4f41d Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Wed, 10 Sep 2025 10:51:41 +0200 Subject: [PATCH 06/14] perf(preview): Optimize migration and simplify DB layout * Simplify migration by not moving the actual files and just updating the DB * Don't store the storageid in the preview table as it is not needed * Start adding tests Signed-off-by: Carl Schwan --- apps/files/lib/Command/ScanAppData.php | 5 + core/BackgroundJobs/MovePreviewJob.php | 6 +- .../Version33000Date20250819110529.php | 23 ++- lib/composer/composer/LICENSE | 2 + lib/composer/composer/autoload_classmap.php | 1 - lib/composer/composer/autoload_static.php | 13 +- .../ObjectStorePreviewCacheMountProvider.php | 138 ----------------- .../ObjectStore/PrimaryObjectStoreConfig.php | 6 + lib/private/Preview/Db/Preview.php | 25 ++- lib/private/Preview/Db/PreviewMapper.php | 43 ++++-- lib/private/Preview/Generator.php | 17 +- .../Preview/Storage/LocalPreviewStorage.php | 22 ++- .../Storage/ObjectStorePreviewStorage.php | 146 +++++++----------- .../Preview/Storage/StorageFactory.php | 4 +- lib/private/Server.php | 2 - ...jectStorePreviewCacheMountProviderTest.php | 95 ------------ tests/lib/Files/Storage/Storage.php | 1 + tests/lib/Preview/MovePreviewJobTest.php | 32 ++++ tests/lib/Preview/PreviewMapperTest.php | 79 ++++++++++ 19 files changed, 292 insertions(+), 368 deletions(-) delete mode 100644 lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php delete mode 100644 tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php create mode 100644 tests/lib/Preview/MovePreviewJobTest.php create mode 100644 tests/lib/Preview/PreviewMapperTest.php diff --git a/apps/files/lib/Command/ScanAppData.php b/apps/files/lib/Command/ScanAppData.php index 0e08c6a8cfe..385e0624b3a 100644 --- a/apps/files/lib/Command/ScanAppData.php +++ b/apps/files/lib/Command/ScanAppData.php @@ -51,6 +51,11 @@ class ScanAppData extends Base { } protected function scanFiles(OutputInterface $output, string $folder): int { + if ($folder === 'preview') { + $output->writeln('Scanning the preview folder is not supported.'); + return self::FAILURE; + } + try { /** @var Folder $appData */ $appData = $this->getAppDataFolder(); diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index d326fab88d3..3541a128a46 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -8,6 +8,7 @@ declare(strict_types=1); namespace OC\Core\BackgroundJobs; +use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OC\Preview\Storage\StorageFactory; @@ -131,11 +132,12 @@ class MovePreviewJob extends TimedJob { $folder = $this->appData->getFolder($internalPath); /** - * @var list $previewFiles + * @var list $previewFiles */ $previewFiles = []; foreach ($folder->getDirectoryListing() as $previewFile) { + /** @var SimpleFile $previewFile */ [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); $nameSplit = explode('-', $baseName); @@ -173,7 +175,7 @@ class MovePreviewJob extends TimedJob { foreach ($previewFiles as $previewFile) { $preview = new Preview(); $preview->setFileId((int)$fileId); - $preview->setStorageId($result[0]['storage']); + $preview->setOldFileId($previewFile['file']->getId()); $preview->setEtag($result[0]['etag']); $preview->setMtime($previewFile['mtime']); $preview->setWidth($previewFile['width']); diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php index 496ee849beb..5e607024ec7 100644 --- a/core/Migrations/Version33000Date20250819110529.php +++ b/core/Migrations/Version33000Date20250819110529.php @@ -11,12 +11,15 @@ namespace OC\Core\Migrations; use Closure; use OCP\DB\ISchemaWrapper; use OCP\DB\Types; +use OCP\Migration\Attributes\CreateTable; use OCP\Migration\IOutput; use OCP\Migration\SimpleMigrationStep; /** * */ +#[CreateTable(table: 'preview', description: 'Holds the preview data')] +#[CreateTable(table: 'preview_locations', description: 'Holds the preview location in an object store')] class Version33000Date20250819110529 extends SimpleMigrationStep { /** @@ -26,21 +29,33 @@ class Version33000Date20250819110529 extends SimpleMigrationStep { /** @var ISchemaWrapper $schema */ $schema = $schemaClosure(); + if (!$schema->hasTable('preview_locations')) { + $table = $schema->createTable('preview_locations'); + $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('bucket_name', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->addColumn('object_store_name', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->setPrimaryKey(['id']); + } + if (!$schema->hasTable('previews')) { $table = $schema->createTable('previews'); $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); - $table->addColumn('storage_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('old_file_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); + $table->addColumn('location_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]); - $table->addColumn('is_max', Types::BOOLEAN, ['notnull' => true, 'default' => false]); - $table->addColumn('crop', Types::BOOLEAN, ['notnull' => true, 'default' => false]); - $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->addColumn('source_mimetype', Types::INTEGER, ['notnull' => true]); + $table->addColumn('max', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('cropped', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('encrypted', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40, 'fixed' => true]); $table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('version', Types::BIGINT, ['notnull' => true, 'default' => -1]); // can not be null otherwise unique index doesn't work $table->setPrimaryKey(['id']); + $table->addIndex(['file_id']); $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx'); } diff --git a/lib/composer/composer/LICENSE b/lib/composer/composer/LICENSE index 62ecfd8d004..f27399a042d 100644 --- a/lib/composer/composer/LICENSE +++ b/lib/composer/composer/LICENSE @@ -1,3 +1,4 @@ + Copyright (c) Nils Adermann, Jordi Boggiano Permission is hereby granted, free of charge, to any person obtaining a copy @@ -17,3 +18,4 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index d3eb6553b8c..5c168a25de3 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1687,7 +1687,6 @@ return array( 'OC\\Files\\Mount\\MountPoint' => $baseDir . '/lib/private/Files/Mount/MountPoint.php', 'OC\\Files\\Mount\\MoveableMount' => $baseDir . '/lib/private/Files/Mount/MoveableMount.php', 'OC\\Files\\Mount\\ObjectHomeMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectHomeMountProvider.php', - 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => $baseDir . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php', 'OC\\Files\\Mount\\RootMountProvider' => $baseDir . '/lib/private/Files/Mount/RootMountProvider.php', 'OC\\Files\\Node\\File' => $baseDir . '/lib/private/Files/Node/File.php', 'OC\\Files\\Node\\Folder' => $baseDir . '/lib/private/Files/Node/Folder.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index d14e3989a6a..423b4cc6b57 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -11,32 +11,32 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 ); public static $prefixLengthsPsr4 = array ( - 'O' => + 'O' => array ( 'OC\\Core\\' => 8, 'OC\\' => 3, 'OCP\\' => 4, ), - 'N' => + 'N' => array ( 'NCU\\' => 4, ), ); public static $prefixDirsPsr4 = array ( - 'OC\\Core\\' => + 'OC\\Core\\' => array ( 0 => __DIR__ . '/../../..' . '/core', ), - 'OC\\' => + 'OC\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/private', ), - 'OCP\\' => + 'OCP\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/public', ), - 'NCU\\' => + 'NCU\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/unstable', ), @@ -1728,7 +1728,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Files\\Mount\\MountPoint' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MountPoint.php', 'OC\\Files\\Mount\\MoveableMount' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/MoveableMount.php', 'OC\\Files\\Mount\\ObjectHomeMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectHomeMountProvider.php', - 'OC\\Files\\Mount\\ObjectStorePreviewCacheMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php', 'OC\\Files\\Mount\\RootMountProvider' => __DIR__ . '/../../..' . '/lib/private/Files/Mount/RootMountProvider.php', 'OC\\Files\\Node\\File' => __DIR__ . '/../../..' . '/lib/private/Files/Node/File.php', 'OC\\Files\\Node\\Folder' => __DIR__ . '/../../..' . '/lib/private/Files/Node/Folder.php', diff --git a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php b/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php deleted file mode 100644 index 1546ef98f50..00000000000 --- a/lib/private/Files/Mount/ObjectStorePreviewCacheMountProvider.php +++ /dev/null @@ -1,138 +0,0 @@ -logger = $logger; - $this->config = $config; - } - - /** - * @return MountPoint[] - * @throws \Exception - */ - public function getRootMounts(IStorageFactory $loader): array { - if (!is_array($this->config->getSystemValue('objectstore_multibucket'))) { - return []; - } - if ($this->config->getSystemValue('objectstore.multibucket.preview-distribution', false) !== true) { - return []; - } - - $instanceId = $this->config->getSystemValueString('instanceid', ''); - $mountPoints = []; - $directoryRange = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']; - $i = 0; - foreach ($directoryRange as $parent) { - foreach ($directoryRange as $child) { - $mountPoints[] = new MountPoint( - AppdataPreviewObjectStoreStorage::class, - '/appdata_' . $instanceId . '/preview/' . $parent . '/' . $child, - $this->getMultiBucketObjectStore($i), - $loader, - null, - null, - self::class - ); - $i++; - } - } - - $rootStorageArguments = $this->getMultiBucketObjectStoreForRoot(); - $fakeRootStorage = new ObjectStoreStorage($rootStorageArguments); - $fakeRootStorageJail = new Jail([ - 'storage' => $fakeRootStorage, - 'root' => '/appdata_' . $instanceId . '/preview', - ]); - - // add a fallback location to be able to fetch existing previews from the old bucket - $mountPoints[] = new MountPoint( - $fakeRootStorageJail, - '/appdata_' . $instanceId . '/preview/old-multibucket', - null, - $loader, - null, - null, - self::class - ); - - return $mountPoints; - } - - protected function getMultiBucketObjectStore(int $number): array { - $config = $this->config->getSystemValue('objectstore_multibucket'); - - // sanity checks - if (empty($config['class'])) { - $this->logger->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - /* - * Use any provided bucket argument as prefix - * and add the mapping from parent/child => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - - $config['arguments']['bucket'] .= "-preview-$number"; - - // instantiate object store implementation - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - - $config['arguments']['internal-id'] = $number; - - return $config['arguments']; - } - - protected function getMultiBucketObjectStoreForRoot(): array { - $config = $this->config->getSystemValue('objectstore_multibucket'); - - // sanity checks - if (empty($config['class'])) { - $this->logger->error('No class given for objectstore', ['app' => 'files']); - } - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - /* - * Use any provided bucket argument as prefix - * and add the mapping from parent/child => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - $config['arguments']['bucket'] .= '0'; - - // instantiate object store implementation - $config['arguments']['objectstore'] = new $config['class']($config['arguments']); - - return $config['arguments']; - } -} diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php index cd2983d385c..02bc28f376e 100644 --- a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -119,12 +119,14 @@ class PrimaryObjectStoreConfig { 'default' => 'server1', 'server1' => $this->validateObjectStoreConfig($objectStoreMultiBucket), 'root' => 'server1', + 'preview' => 'server1', ]; } elseif ($objectStore) { if (!isset($objectStore['default'])) { $objectStore = [ 'default' => 'server1', 'root' => 'server1', + 'preview' => 'server1', 'server1' => $objectStore, ]; } @@ -132,6 +134,10 @@ class PrimaryObjectStoreConfig { $objectStore['root'] = 'default'; } + if (!isset($objectStore['preview'])) { + $objectStore['preview'] = 'default'; + } + if (!is_string($objectStore['default'])) { throw new InvalidObjectStoreConfigurationException('The \'default\' object storage configuration is required to be a reference to another configuration.'); } diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index c88cebedc91..b4ebe8b4805 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -17,8 +17,12 @@ use OCP\IPreview; /** * @method \int getFileId() * @method void setFileId(int $fileId) - * @method \int getStorageId() - * @method void setStorageId(\int $fileId) + * @method \int getOldFileId() // Old location in the file-cache table, for legacy compatibility + * @method void setOldFileId(int $fileId) + * @method \int getLocationId() + * @method void setLocationId(int $locationId) + * @method \string getBucketName() + * @method \string getObjectStoreName() * @method \int getWidth() * @method void setWidth(int $width) * @method \int getHeight() @@ -43,7 +47,11 @@ use OCP\IPreview; class Preview extends Entity { protected ?int $fileId = null; - protected ?int $storageId = null; + protected ?int $oldFileId = null; + + protected ?int $locationId = null; + protected ?string $bucketName = null; + protected ?string $objectStoreName = null; protected ?int $width = null; @@ -65,7 +73,8 @@ class Preview extends Entity { public function __construct() { $this->addType('fileId', Types::BIGINT); - $this->addType('storageId', Types::BIGINT); + $this->addType('oldFileId', Types::BIGINT); + $this->addType('locationId', Types::BIGINT); $this->addType('width', Types::INTEGER); $this->addType('height', Types::INTEGER); $this->addType('mimetype', Types::INTEGER); @@ -108,4 +117,12 @@ class Preview extends Entity { IPreview::MIMETYPE_GIF => 'gif', }; } + + public function setBucketName(string $bucketName): void { + $this->bucketName = $bucketName; + } + + public function setObjectStoreName(string $objectStoreName): void { + $this->objectStoreName = $objectStoreName; + } } diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index 62023638150..e0ec05ef88f 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -22,6 +22,7 @@ use OCP\IPreview; class PreviewMapper extends QBMapper { private const TABLE_NAME = 'previews'; + private const LOCATION_TABLE_NAME = 'preview_locations'; public function __construct(IDBConnection $db) { parent::__construct($db, self::TABLE_NAME, Preview::class); @@ -34,8 +35,7 @@ class PreviewMapper extends QBMapper { */ public function getAvailablePreviews(array $fileIds): array { $selectQb = $this->db->getQueryBuilder(); - $selectQb->select('*') - ->from(self::TABLE_NAME) + $this->joinLocation($selectQb) ->where( $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), ); @@ -48,8 +48,7 @@ class PreviewMapper extends QBMapper { public function getPreview(int $fileId, int $width, int $height, string $mode, int $mimetype = IPreview::MIMETYPE_JPEG): ?Preview { $selectQb = $this->db->getQueryBuilder(); - $selectQb->select('*') - ->from(self::TABLE_NAME) + $this->joinLocation($selectQb) ->where( $selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId)), $selectQb->expr()->eq('width', $selectQb->createNamedParameter($width)), @@ -68,10 +67,9 @@ class PreviewMapper extends QBMapper { * @param int[] $fileIds * @return array */ - public function getByFileIds(int $storageId, array $fileIds): array { + public function getByFileIds(array $fileIds): array { $selectQb = $this->db->getQueryBuilder(); - $selectQb->select('*') - ->from(self::TABLE_NAME) + $this->joinLocation($selectQb) ->where($selectQb->expr()->andX( $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), )); @@ -85,12 +83,39 @@ class PreviewMapper extends QBMapper { /** * @param int[] $previewIds */ - public function deleteByIds(int $storageId, array $previewIds): void { + public function deleteByIds(array $previewIds): void { $qb = $this->db->getQueryBuilder(); $qb->delete(self::TABLE_NAME) ->where($qb->expr()->andX( - $qb->expr()->eq('storage_id', $qb->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)), $qb->expr()->in('id', $qb->createNamedParameter($previewIds, IQueryBuilder::PARAM_INT_ARRAY)) ))->executeStatement(); } + + protected function joinLocation(IQueryBuilder $qb): IQueryBuilder { + return $qb->select('p.*', 'l.bucket_name', 'l.object_store_name') + ->from(self::TABLE_NAME, 'p') + ->join('p', 'preview_locations', 'l', $qb->expr()->eq( + 'p.location_id', 'l.id' + )); + } + + public function getLocationId(string $bucket, string $objectStore): int { + $qb = $this->db->getQueryBuilder(); + $result = $qb->select('id') + ->from(self::LOCATION_TABLE_NAME) + ->where($qb->expr()->eq('bucket_name', $qb->createNamedParameter($bucket))) + ->andWhere($qb->expr()->eq('object_store_name', $qb->createNamedParameter($objectStore))) + ->executeQuery(); + $data = $result->fetchOne(); + if ($data) { + return $data; + } else { + $qb->insert(self::LOCATION_TABLE_NAME) + ->values([ + 'bucket_name' => $qb->createNamedParameter($bucket), + 'object_store_name' => $qb->createNamedParameter($objectStore), + ])->executeStatement(); + return $qb->getLastInsertId(); + } + } } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 2dfbfa6db62..44a12624c3e 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -156,17 +156,13 @@ class Generator { // Try to get a cached preview. Else generate (and store) one try { - /** @var ISimpleFile $previewFile */ - $previewFile = null; - // TODO(php8.4) replace by array_find - foreach ($previews as $p) { - if ($p->getWidth() === $width && $p->getHeight() === $height && $p->getMimetype() === $maxPreview->getMimetype() && $p->getVersion() === $previewVersion && $p->getCrop() === $crop) { - $previewFile = new PreviewFile($p, $this->storageFactory, $this->previewMapper); - break; - } - } + $preview = array_find($previews, fn (Preview $preview): bool => $preview->getWidth() === $width + && $preview->getHeight() === $height && $preview->getMimetype() === $maxPreview->getMimetype() + && $preview->getVersion() === $previewVersion && $preview->getCrop() === $crop); - if ($previewFile === null) { + if ($preview) { + $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper); + } else { if (!$this->previewManager->isMimeSupported($mimeType)) { throw new NotFoundException(); } @@ -543,7 +539,6 @@ class Generator { public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); - $previewEntry->setStorageId((int)$file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); $previewEntry->setHeight($height); $previewEntry->setVersion($version); diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 2c176948c79..d4d7e37c0e5 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -18,30 +18,35 @@ use OCP\IConfig; class LocalPreviewStorage implements IPreviewStorage { private const PREVIEW_DIRECTORY = '__preview'; - private readonly string $rootFolder; + private readonly string $instanceId; public function __construct( private readonly IConfig $config, ) { + $this->instanceId = $this->config->getSystemValueString('instanceid'); $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); } public function writePreview(Preview $preview, $stream): false|int { $previewPath = $this->constructPath($preview); $this->createParentFiles($previewPath); - $file = @fopen($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath, 'w'); + $file = @fopen($this->getPreviewRootFolder() . $previewPath, 'w'); return fwrite($file, $stream); } public function readPreview(Preview $preview) { $previewPath = $this->constructPath($preview); - return @fopen($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath, 'r'); + return @fopen($this->getPreviewRootFolder() . $previewPath, 'r'); } public function deletePreview(Preview $preview) { $previewPath = $this->constructPath($preview); - @unlink($this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath); + @unlink($this->getPreviewRootFolder() . $previewPath); + } + + public function getPreviewRootFolder(): string { + return $this->rootFolder . '/appdata_' . $this->instanceId . '/preview/'; } private function constructPath(Preview $preview): string { @@ -63,11 +68,14 @@ class LocalPreviewStorage implements IPreviewStorage { $previewPath = $this->constructPath($preview); $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $previewPath; $destinationPath = $this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath; - if (!file_exists($sourcePath)) { - // legacy flat directory - $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $preview->getFileId() . '/' . $preview->getName(); + if (file_exists($sourcePath)) { + return; // No need to migrate } + + // legacy flat directory + $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $preview->getFileId() . '/' . $preview->getName(); if (file_exists($destinationPath)) { + @unlink($sourcePath); // We already have a new preview, just delete the old one return; } $this->createParentFiles($previewPath); diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index a8fe79d4732..97313fd306f 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -14,28 +14,29 @@ use Icewind\Streams\CountWrapper; use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; /** - * @psalm-type ObjectStoreDefinition = array{store: IObjectStore, objectPrefix: string, config?: array} + * @psalm-import-type ObjectStoreConfig from PrimaryObjectStoreConfig + * @psalm-type ObjectStoreDefinition = array{store: IObjectStore, objectPrefix: string, config?: ObjectStoreConfig} */ class ObjectStorePreviewStorage implements IPreviewStorage { /** - * @var array<'root'|int, ObjectStoreDefinition> + * @var array> */ private array $objectStoreCache = []; - private bool $isMultibucketEnabled; private bool $isMultibucketPreviewDistributionEnabled; public function __construct( private readonly PrimaryObjectStoreConfig $objectStoreConfig, - readonly private IConfig $config, + IConfig $config, + readonly private PreviewMapper $previewMapper, ) { - $this->isMultibucketEnabled = is_array($config->getSystemValue('objectstore_multibucket')); $this->isMultibucketPreviewDistributionEnabled = $config->getSystemValueBool('objectstore.multibucket.preview-distribution'); } @@ -56,6 +57,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { [ 'objectPrefix' => $objectPrefix, 'store' => $store, + 'config' => $config, ] = $this->getObjectStoreForPreview($preview); $store->writeObject($this->constructUrn($objectPrefix, $preview->getId()), $countStream); @@ -79,102 +81,72 @@ class ObjectStorePreviewStorage implements IPreviewStorage { } public function migratePreview(Preview $preview, SimpleFile $file): void { - foreach ([false, true] as $fallback) { - [ - 'objectPrefix' => $objectPrefix, - 'store' => $store, - 'config' => $config, - ] = $this->getObjectStoreForPreview($preview, $fallback); - - $oldObjectPrefix = 'urn:oid:'; - if (isset($config['objectPrefix'])) { - $oldObjectPrefix = $config['objectPrefix']; - } - - try { - $store->copyObject($this->constructUrn($oldObjectPrefix, $file->getId()), $this->constructUrn($objectPrefix, $preview->getId())); - break; - } catch (NotFoundException $e) { - if (!$fallback && $this->isMultibucketPreviewDistributionEnabled) { - continue; - } - throw $e; - } - } - } - - /** - * @return ObjectStoreDefinition - */ - private function getMultiBucketObjectStore(int $number): array { - /** - * @var array{class: class-string, ...} $config - */ - $config = $this->config->getSystemValue('objectstore_multibucket'); - - if (!isset($config['arguments'])) { - $config['arguments'] = []; - } - - /* - * Use any provided bucket argument as prefix - * and add the mapping from parent/child => bucket - */ - if (!isset($config['arguments']['bucket'])) { - $config['arguments']['bucket'] = ''; - } - - $config['arguments']['bucket'] .= "-preview-$number"; - - $objectPrefix = 'urn:oid:preview:'; - if (isset($config['objectPrefix'])) { - $objectPrefix = $config['objectPrefix'] . 'preview:'; - } - - return [ - 'store' => new $config['class']($config['arguments']), - 'objectPrefix' => $objectPrefix, - 'config' => $config, - ]; - } - - /** - * @return ObjectStoreDefinition - */ - private function getRootObjectStore(): array { - if (!isset($this->objectStoreCache['root'])) { - $rootConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); - $objectPrefix = 'urn:oid:preview:'; - if (isset($rootConfig['arguments']['objectPrefix'])) { - $objectPrefix = $rootConfig['arguments']['objectPrefix'] . 'preview:'; - } - $this->objectStoreCache['root'] = [ - 'store' => $this->objectStoreConfig->buildObjectStore($rootConfig), - 'objectPrefix' => $objectPrefix, - ]; - } - return $this->objectStoreCache['root']; + // Just set the Preview::bucket and Preview::objectStore + $this->getObjectStoreForPreview($preview, true); } /** * @return ObjectStoreDefinition */ private function getObjectStoreForPreview(Preview $preview, bool $oldFallback = false): array { - if (!$this->isMultibucketEnabled || !$this->isMultibucketPreviewDistributionEnabled || $oldFallback) { - return $this->getRootObjectStore(); + if ($preview->getObjectStoreName() === null) { + $config = $this->objectStoreConfig->getObjectStoreConfiguration($oldFallback ? 'root' : 'preview'); + $objectStoreName = $this->objectStoreConfig->resolveAlias($oldFallback ? 'root' : 'preview'); + + $bucketName = $config['arguments']['bucket']; + if ($config['arguments']['multibucket']) { + if ($this->isMultibucketPreviewDistributionEnabled) { + $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); + $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); + $bucketName .= '-preview-' . $bucketNumber; + } else { + $bucketName .= '0'; + } + } + $config['arguments']['bucket'] = $bucketName; + + $locationId = $this->previewMapper->getLocationId($bucketName, $objectStoreName); + $preview->setLocationId($locationId); + $preview->setObjectStoreName($objectStoreName); + $preview->setBucketName($bucketName); + } else { + $config = $this->objectStoreConfig->getObjectStoreConfiguration($preview->getObjectStoreName()); + $config['arguments']['bucket'] = $bucketName = $preview->getBucketName(); + $objectStoreName = $preview->getObjectStoreName(); } - $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); - $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); + $objectPrefix = $this->getObjectPrefix($preview, $config); - if (!isset($this->objectStoreCache[$bucketNumber])) { - $this->objectStoreCache[$bucketNumber] = $this->getMultiBucketObjectStore($bucketNumber); + if (!isset($this->objectStoreCache[$objectStoreName])) { + $this->objectStoreCache[$objectStoreName] = []; + $this->objectStoreCache[$objectStoreName][$bucketName] = [ + 'store' => $this->objectStoreConfig->buildObjectStore($config), + 'objectPrefix' => $objectPrefix, + 'config' => $config, + ]; + } elseif (!isset($this->objectStoreCache[$objectStoreName][$bucketName])) { + $this->objectStoreCache[$objectStoreName][$bucketName] = [ + 'store' => $this->objectStoreConfig->buildObjectStore($config), + 'objectPrefix' => $objectPrefix, + 'config' => $config, + ]; } - return $this->objectStoreCache[$bucketNumber]; + return $this->objectStoreCache[$objectStoreName][$bucketName]; } private function constructUrn(string $objectPrefix, int $id): string { return $objectPrefix . $id; } + + public function getObjectPrefix(Preview $preview, array $config): string { + if ($preview->getOldFileId()) { + return $config['arguments']['objectPrefix'] ?? 'uri:oid:'; + } + if (isset($config['arguments']['objectPrefix'])) { + return $config['arguments']['objectPrefix'] . 'preview:'; + } else { + return 'uri:oid:preview:'; + } + } } diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 077395c8b93..0755bacd0b9 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -5,6 +5,7 @@ namespace OC\Preview\Storage; use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; use OCP\IConfig; class StorageFactory implements IPreviewStorage { @@ -13,6 +14,7 @@ class StorageFactory implements IPreviewStorage { public function __construct( private readonly PrimaryObjectStoreConfig $objectStoreConfig, private readonly IConfig $config, + private readonly PreviewMapper $previewMapper, ) { } @@ -34,7 +36,7 @@ class StorageFactory implements IPreviewStorage { } if ($this->objectStoreConfig->hasObjectStore()) { - $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config); + $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config, $this->previewMapper); } else { $this->backend = new LocalPreviewStorage($this->config); } diff --git a/lib/private/Server.php b/lib/private/Server.php index 4445788ec4e..fa88d8353e3 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -49,7 +49,6 @@ use OC\Files\Lock\LockManager; use OC\Files\Mount\CacheMountProvider; use OC\Files\Mount\LocalHomeMountProvider; use OC\Files\Mount\ObjectHomeMountProvider; -use OC\Files\Mount\ObjectStorePreviewCacheMountProvider; use OC\Files\Mount\RootMountProvider; use OC\Files\Node\HookConnector; use OC\Files\Node\LazyRoot; @@ -789,7 +788,6 @@ class Server extends ServerContainer implements IServerContainer { $manager->registerHomeProvider(new LocalHomeMountProvider()); $manager->registerHomeProvider(new ObjectHomeMountProvider($objectStoreConfig)); $manager->registerRootProvider(new RootMountProvider($objectStoreConfig, $config)); - $manager->registerRootProvider(new ObjectStorePreviewCacheMountProvider($logger, $config)); return $manager; }); diff --git a/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php b/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php deleted file mode 100644 index 9060bf0d5f5..00000000000 --- a/tests/lib/Files/Mount/ObjectStorePreviewCacheMountProviderTest.php +++ /dev/null @@ -1,95 +0,0 @@ -logger = $this->createMock(LoggerInterface::class); - $this->config = $this->createMock(IConfig::class); - $this->loader = $this->createMock(StorageFactory::class); - - $this->provider = new ObjectStorePreviewCacheMountProvider($this->logger, $this->config); - } - - public function testNoMultibucketObjectStorage(): void { - $this->config->expects($this->once()) - ->method('getSystemValue') - ->with('objectstore_multibucket') - ->willReturn(null); - - $this->assertEquals([], $this->provider->getRootMounts($this->loader)); - } - - public function testMultibucketObjectStorage(): void { - $objectstoreConfig = [ - 'class' => S3::class, - 'arguments' => [ - 'bucket' => 'abc', - 'num_buckets' => 64, - 'key' => 'KEY', - 'secret' => 'SECRET', - 'hostname' => 'IP', - 'port' => 'PORT', - 'use_ssl' => false, - 'use_path_style' => true, - ], - ]; - $this->config->expects($this->any()) - ->method('getSystemValue') - ->willReturnCallback(function ($config) use ($objectstoreConfig) { - if ($config === 'objectstore_multibucket') { - return $objectstoreConfig; - } elseif ($config === 'objectstore.multibucket.preview-distribution') { - return true; - } - return null; - }); - $this->config->expects($this->once()) - ->method('getSystemValueString') - ->with('instanceid') - ->willReturn('INSTANCEID'); - - $mounts = $this->provider->getRootMounts($this->loader); - - // 256 mounts for the subfolders and 1 for the fake root - $this->assertCount(257, $mounts); - - // do some sanity checks if they have correct mount point paths - $this->assertEquals('/appdata_INSTANCEID/preview/0/0/', $mounts[0]->getMountPoint()); - $this->assertEquals('/appdata_INSTANCEID/preview/2/5/', $mounts[37]->getMountPoint()); - // also test the path of the fake bucket - $this->assertEquals('/appdata_INSTANCEID/preview/old-multibucket/', $mounts[256]->getMountPoint()); - } -} diff --git a/tests/lib/Files/Storage/Storage.php b/tests/lib/Files/Storage/Storage.php index 8110ab6e8cc..60e696739ee 100644 --- a/tests/lib/Files/Storage/Storage.php +++ b/tests/lib/Files/Storage/Storage.php @@ -358,6 +358,7 @@ abstract class Storage extends \Test\TestCase { $this->assertTrue($this->instance->file_exists($fileName)); $fh = $this->instance->fopen($fileName, 'r'); + $this->assertTrue(is_resource($fh)); $content = stream_get_contents($fh); $this->assertEquals(file_get_contents($textFile), $content); } diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php new file mode 100644 index 00000000000..418f28faff7 --- /dev/null +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -0,0 +1,32 @@ +previewAppData = Server::get(IAppDataFactory::class)->get('preview'); + } + + #[TestDox("Test the migration from the legacy flat hierarchy to the new one")] + function testMigrationLegacyPath(): void { + $folder = $this->previewAppData->newFolder(5); + $file = $folder->newFile('64-64-crop.png', 'abcdefg'); + $job = Server::get(MovePreviewJob::class); + $this->invokePrivate($job, 'run', []); + } +} diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php new file mode 100644 index 00000000000..68aea1bfcc1 --- /dev/null +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -0,0 +1,79 @@ +previewMapper = Server::get(PreviewMapper::class); + $this->connection = Server::get(IDBConnection::class); + } + + public function testGetAvailablePreviews() { + // Empty + $this->assertEquals([], $this->previewMapper->getAvailablePreviews([])); + + // No preview available + $this->assertEquals([42 => []], $this->previewMapper->getAvailablePreviews([42])); + + $this->createPreviewForFileId(42); + $previews = $this->previewMapper->getAvailablePreviews([42]); + $this->assertNotEmpty($previews[42]); + $this->assertNull($previews[42][0]->getLocationId()); + $this->assertNull($previews[42][0]->getBucketName()); + $this->assertNull($previews[42][0]->getObjectStoreName()); + + $this->createPreviewForFileId(43, 2); + $previews = $this->previewMapper->getAvailablePreviews([43]); + $this->assertNotEmpty($previews[43]); + $this->assertEquals('preview-2', $previews[43][0]->getBucketName()); + $this->assertEquals('default', $previews[43][0]->getObjectStoreName()); + } + + private function createPreviewForFileId(int $fileId, ?int $bucket = null) { + if ($bucket) { + $qb = $this->connection->getQueryBuilder(); + $qb->insert('preview_locations') + ->values([ + 'bucket' => $qb->createNamedParameter('preview-' . $bucket), + 'object_store' => $qb->createNamedParameter('default'), + ]); + $locationId = $qb->executeStatement(); + } + $preview = new Preview(); + $preview->setFileId($fileId); + $preview->setCrop(true); + $preview->setIsMax(true); + $preview->setWidth(100); + $preview->setHeight(100); + $preview->setSize(100); + $preview->setMtime(time()); + $preview->setMimetype(IPreview::MIMETYPE_PNG); + $preview->setEtag("abcdefg"); + + if ($locationId) { + $preview->setLocationId($locationId); + } + $this->previewMapper->insert($preview); + } +} From 6f56dcf73eb1c54920dbf81ae9bb87f1c1e8ed00 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 11 Sep 2025 14:52:34 +0200 Subject: [PATCH 07/14] fix(preview): Fix some tests Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 8 +- lib/private/Preview/BackgroundCleanupJob.php | 58 ++++---------- lib/private/Preview/Db/PreviewMapper.php | 35 +++++---- lib/private/Preview/Generator.php | 5 +- .../Preview/Storage/LocalPreviewStorage.php | 44 +++++------ .../Storage/ObjectStorePreviewStorage.php | 1 + lib/private/Preview/Storage/Root.php | 74 ------------------ lib/private/Preview/Watcher.php | 36 ++++----- lib/private/PreviewManager.php | 3 - lib/private/Server.php | 16 ++-- .../lib/Preview/BackgroundCleanupJobTest.php | 7 +- tests/lib/Preview/MovePreviewJobTest.php | 75 ++++++++++++++++++- tests/lib/Preview/PreviewMapperTest.php | 4 +- version.php | 2 +- 14 files changed, 162 insertions(+), 206 deletions(-) delete mode 100644 lib/private/Preview/Storage/Root.php diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 3541a128a46..67f60c900ee 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -52,7 +52,7 @@ class MovePreviewJob extends TimedJob { private function doRun($argument): void { if ($this->appConfig->getValueBool('core', 'previewMovedDone')) { - //return; + return; } $emptyHierarchicalPreviewFolders = false; @@ -88,7 +88,7 @@ class MovePreviewJob extends TimedJob { $qb = $this->connection->getQueryBuilder(); $qb->select('*') ->from('filecache') - ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.jpg'))) + ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.%'))) ->setMaxResults(100); $result = $qb->executeQuery(); @@ -118,13 +118,13 @@ class MovePreviewJob extends TimedJob { } } - // Delete any left over preview directory + // Delete any leftover preview directory $this->appData->getFolder('.')->delete(); $this->appConfig->setValueBool('core', 'previewMovedDone', true); } /** - * @param array $previewFolders + * @param array $previewFolders */ private function processPreviews(array $previewFolders, bool $simplePaths): void { foreach ($previewFolders as $fileId => $previewFolder) { diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index 62e3303dc4e..d65b740745d 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -35,23 +35,18 @@ class BackgroundCleanupJob extends TimedJob { } public function run($argument): void { - foreach ($this->getDeletedFiles() as $chunk) { - foreach ($chunk as $storage => $fileIds) { - foreach ($this->previewMapper->getByFileIds($storage, $fileIds) as $previews) { - $previewIds = []; - foreach ($previews as $preview) { - $previewIds[] = $preview->getId(); - $this->storageFactory->deletePreview($preview); - } - - $this->previewMapper->deleteByIds($storage, $previewIds); - }; + foreach ($this->getDeletedFiles() as $fileId) { + $previewIds = []; + foreach ($this->previewMapper->getByFileId($fileId) as $preview) { + $previewIds[] = $preview->getId(); + $this->storageFactory->deletePreview($preview); } + $this->previewMapper->deleteByIds($previewIds); } } /** - * @return \Iterator> + * @return \Iterator */ private function getDeletedFiles(): \Iterator { if ($this->connection->getShardDefinition('filecache')) { @@ -81,7 +76,7 @@ class BackgroundCleanupJob extends TimedJob { * If the related file is deleted, b.fileid will be null and the preview folder can be deleted. */ $qb = $this->connection->getQueryBuilder(); - $qb->select('p.storage_id', 'p.file_id') + $qb->select('p.file_id') ->from('previews', 'p') ->leftJoin('p', 'filecache', 'f', $qb->expr()->eq( 'p.file_id', 'f.fileid' @@ -93,30 +88,14 @@ class BackgroundCleanupJob extends TimedJob { } $cursor = $qb->executeQuery(); - - $lastStorageId = null; - /** @var FileId[] $tmpResult */ - $tmpResult = []; while ($row = $cursor->fetch()) { - if ($lastStorageId === null) { - $lastStorageId = $row['storage_id']; - } else if ($lastStorageId !== $row['storage_id']) { - yield [$lastStorageId => $tmpResult]; - $tmpResult = []; - $lastStorageId = $row['storage_id']; - } - $tmpResult[] = $row['file_id']; + yield $row['file_id']; } - - if (!empty($tmpResult)) { - yield [$lastStorageId => $tmpResult]; - } - $cursor->closeCursor(); } /** - * @return \Iterator> + * @return \Iterator */ private function getAllPreviewIds(int $chunkSize): \Iterator { $qb = $this->connection->getQueryBuilder(); @@ -131,20 +110,11 @@ class BackgroundCleanupJob extends TimedJob { $minId = 0; while (true) { $qb->setParameter('min_id', $minId); - $rows = $qb->executeQuery()->fetchAll(); - if (count($rows) > 0) { - $minId = $rows[count($rows) - 1]['id']; - $result = []; - foreach ($rows as $row) { - if (!isset($result[$row['storage_id']])) { - $result[$row['storage_id']] = []; - } - $result[$row['storage_id']][] = $row['file_id']; - } - yield $result; - } else { - break; + $cursor = $qb->executeQuery(); + while ($row = $cursor->fetch()) { + yield $row['file_id']; } + $cursor->closeCursor(); } } diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index e0ec05ef88f..dba62c5a163 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -9,6 +9,7 @@ declare(strict_types=1); namespace OC\Preview\Db; +use OC\Preview\Generator; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\QBMapper; use OCP\DB\Exception; @@ -28,6 +29,17 @@ class PreviewMapper extends QBMapper { parent::__construct($db, self::TABLE_NAME, Preview::class); } + /** + * @return \Generator + * @throws Exception + */ + public function getAvailablePreviewForFile(int $fileId): \Generator { + $selectQb = $this->db->getQueryBuilder(); + $this->joinLocation($selectQb) + ->where($selectQb->expr()->eq('p.file_id', $selectQb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + yield from $this->yieldEntities($selectQb); + } + /** * @param int[] $fileIds * @return array @@ -37,7 +49,7 @@ class PreviewMapper extends QBMapper { $selectQb = $this->db->getQueryBuilder(); $this->joinLocation($selectQb) ->where( - $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), + $selectQb->expr()->in('p.file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), ); $previews = array_fill_keys($fileIds, []); foreach ($this->yieldEntities($selectQb) as $preview) { @@ -64,20 +76,13 @@ class PreviewMapper extends QBMapper { } /** - * @param int[] $fileIds - * @return array + * @return \Generator */ - public function getByFileIds(array $fileIds): array { + public function getByFileId(int $fileId): \Generator { $selectQb = $this->db->getQueryBuilder(); $this->joinLocation($selectQb) - ->where($selectQb->expr()->andX( - $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), - )); - $previews = array_fill_keys($fileIds, []); - foreach ($this->yieldEntities($selectQb) as $preview) { - $previews[$preview->getFileId()][] = $preview; - } - return $previews; + ->where($selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); + yield from $this->yieldEntities($selectQb); } /** @@ -94,9 +99,9 @@ class PreviewMapper extends QBMapper { protected function joinLocation(IQueryBuilder $qb): IQueryBuilder { return $qb->select('p.*', 'l.bucket_name', 'l.object_store_name') ->from(self::TABLE_NAME, 'p') - ->join('p', 'preview_locations', 'l', $qb->expr()->eq( - 'p.location_id', 'l.id' - )); + ->leftJoin('p', 'preview_locations', 'l', $qb->expr()->eq( + 'p.location_id', 'l.id' + )); } public function getLocationId(string $bucket, string $objectStore): int { diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 44a12624c3e..6d3626bb2fc 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -349,7 +349,7 @@ class Generator { try { return $this->savePreview($file, $preview->width(), $preview->height(), $crop, $max, $preview, $version); - } catch (NotPermittedException $e) { + } catch (NotPermittedException) { throw new NotFoundException(); } } @@ -571,6 +571,9 @@ class Generator { } else { $size = $this->storageFactory->writePreview($previewEntry, $preview->data()); } + if (!$size) { + throw new \RuntimeException('Unable to write preview file'); + } } catch (\Exception $e) { $this->previewMapper->delete($previewEntry); throw $e; diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index d4d7e37c0e5..c4577cac8bd 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -17,7 +17,6 @@ use OC\Preview\Db\Preview; use OCP\IConfig; class LocalPreviewStorage implements IPreviewStorage { - private const PREVIEW_DIRECTORY = '__preview'; private readonly string $rootFolder; private readonly string $instanceId; @@ -30,19 +29,18 @@ class LocalPreviewStorage implements IPreviewStorage { public function writePreview(Preview $preview, $stream): false|int { $previewPath = $this->constructPath($preview); - $this->createParentFiles($previewPath); - $file = @fopen($this->getPreviewRootFolder() . $previewPath, 'w'); - return fwrite($file, $stream); + if (!$this->createParentFiles($previewPath)) { + return false; + } + return file_put_contents($previewPath, $stream); } public function readPreview(Preview $preview) { - $previewPath = $this->constructPath($preview); - return @fopen($this->getPreviewRootFolder() . $previewPath, 'r'); + return @fopen($this->constructPath($preview), 'r'); } public function deletePreview(Preview $preview) { - $previewPath = $this->constructPath($preview); - @unlink($this->getPreviewRootFolder() . $previewPath); + @unlink($this->constructPath($preview)); } public function getPreviewRootFolder(): string { @@ -50,36 +48,28 @@ class LocalPreviewStorage implements IPreviewStorage { } private function constructPath(Preview $preview): string { - return implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); + return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); } - private function createParentFiles($path) { - ['basename' => $basename, 'dirname' => $dirname] = pathinfo($path); - $currentDir = $this->rootFolder . '/' . self::PREVIEW_DIRECTORY; - mkdir($currentDir); - foreach (explode('/', $dirname) as $suffix) { - $currentDir .= "/$suffix"; - mkdir($currentDir); - } + private function createParentFiles(string $path): bool { + ['dirname' => $dirname] = pathinfo($path); + return mkdir($dirname, recursive: true); } public function migratePreview(Preview $preview, SimpleFile $file): void { - $instanceId = $this->config->getSystemValueString('instanceid'); - $previewPath = $this->constructPath($preview); - $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $previewPath; - $destinationPath = $this->rootFolder . '/' . self::PREVIEW_DIRECTORY . '/' . $previewPath; - if (file_exists($sourcePath)) { - return; // No need to migrate + // legacy flat directory + $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName(); + if (!file_exists($sourcePath)) { + return; } - // legacy flat directory - $sourcePath = $this->rootFolder . '/appdata_' . $instanceId . '/preview/' . $preview->getFileId() . '/' . $preview->getName(); + $destinationPath = $this->constructPath($preview); if (file_exists($destinationPath)) { @unlink($sourcePath); // We already have a new preview, just delete the old one return; } - $this->createParentFiles($previewPath); - echo 'Copying ' . $sourcePath . ' to ' . $destinationPath . PHP_EOL; + + $this->createParentFiles($destinationPath); $ok = rename($sourcePath, $destinationPath); if (!$ok) { throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 97313fd306f..7963c870dd8 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -83,6 +83,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { public function migratePreview(Preview $preview, SimpleFile $file): void { // Just set the Preview::bucket and Preview::objectStore $this->getObjectStoreForPreview($preview, true); + $this->previewMapper->update($preview); } /** diff --git a/lib/private/Preview/Storage/Root.php b/lib/private/Preview/Storage/Root.php deleted file mode 100644 index 41378653962..00000000000 --- a/lib/private/Preview/Storage/Root.php +++ /dev/null @@ -1,74 +0,0 @@ -isMultibucketPreviewDistributionEnabled = $systemConfig->getValue('objectstore.multibucket.preview-distribution', false) === true; - } - - - public function getFolder(string $name): ISimpleFolder { - $internalFolder = self::getInternalFolder($name); - - try { - return parent::getFolder($internalFolder); - } catch (NotFoundException $e) { - /* - * The new folder structure is not found. - * Lets try the old one - */ - } - - try { - return parent::getFolder($name); - } catch (NotFoundException $e) { - /* - * The old folder structure is not found. - * Lets try the multibucket fallback if available - */ - if ($this->isMultibucketPreviewDistributionEnabled) { - return parent::getFolder('old-multibucket/' . $internalFolder); - } - - // when there is no further fallback just throw the exception - throw $e; - } - } - - public function newFolder(string $name): ISimpleFolder { - $internalFolder = self::getInternalFolder($name); - return parent::newFolder($internalFolder); - } - - /* - * Do not allow directory listing on this special root - * since it gets to big and time consuming - */ - public function getDirectoryListing(): array { - return []; - } - - public static function getInternalFolder(string $name): string { - return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; - } - - public function getStorageId(): int { - return $this->getAppDataRootFolder()->getStorage()->getCache()->getNumericStorageId(); - } -} diff --git a/lib/private/Preview/Watcher.php b/lib/private/Preview/Watcher.php index 21f040d8342..bbbbcb835fa 100644 --- a/lib/private/Preview/Watcher.php +++ b/lib/private/Preview/Watcher.php @@ -8,11 +8,11 @@ declare(strict_types=1); */ namespace OC\Preview; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\StorageFactory; use OCP\Files\FileInfo; use OCP\Files\Folder; -use OCP\Files\IAppData; use OCP\Files\Node; -use OCP\Files\NotFoundException; /** * Class Watcher @@ -22,40 +22,36 @@ use OCP\Files\NotFoundException; * Class that will watch filesystem activity and remove previews as needed. */ class Watcher { - /** @var IAppData */ - private $appData; - /** * Watcher constructor. - * - * @param IAppData $appData */ - public function __construct(IAppData $appData) { - $this->appData = $appData; + public function __construct( + readonly private StorageFactory $storageFactory, + readonly private PreviewMapper $previewMapper, + ) { } - public function postWrite(Node $node) { + public function postWrite(Node $node): void { $this->deleteNode($node); } - protected function deleteNode(FileInfo $node) { + protected function deleteNode(FileInfo $node): void { // We only handle files if ($node instanceof Folder) { return; } - try { - if (is_null($node->getId())) { - return; - } - $folder = $this->appData->getFolder((string)$node->getId()); - $folder->delete(); - } catch (NotFoundException $e) { - //Nothing to do + if (is_null($node->getId())) { + return; + } + + [$node->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$node->getId()]); + foreach ($previews as $preview) { + $this->storageFactory->deletePreview($preview); } } - public function versionRollback(array $data) { + public function versionRollback(array $data): void { if (isset($data['node'])) { $this->deleteNode($data['node']); } diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 3bc63a55adb..912d2b3fe5b 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -32,7 +32,6 @@ use function array_key_exists; class PreviewManager implements IPreview { protected IConfig $config; protected IRootFolder $rootFolder; - protected IAppData $appData; protected IEventDispatcher $eventDispatcher; private ?Generator $generator = null; private GeneratorHelper $helper; @@ -59,7 +58,6 @@ class PreviewManager implements IPreview { public function __construct( IConfig $config, IRootFolder $rootFolder, - IAppData $appData, IEventDispatcher $eventDispatcher, GeneratorHelper $helper, ?string $userId, @@ -70,7 +68,6 @@ class PreviewManager implements IPreview { ) { $this->config = $config; $this->rootFolder = $rootFolder; - $this->appData = $appData; $this->eventDispatcher = $eventDispatcher; $this->helper = $helper; $this->userId = $userId; diff --git a/lib/private/Server.php b/lib/private/Server.php index fa88d8353e3..97483844333 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -82,9 +82,11 @@ use OC\Notification\Manager; use OC\OCM\Model\OCMProvider; use OC\OCM\OCMDiscoveryService; use OC\OCS\DiscoveryService; +use OC\Preview\Db\PreviewMapper; use OC\Preview\GeneratorHelper; use OC\Preview\IMagickSupport; use OC\Preview\MimeIconProvider; +use OC\Preview\Watcher; use OC\Profile\ProfileManager; use OC\Profiler\Profiler; use OC\Remote\Api\ApiFactory; @@ -291,10 +293,6 @@ class Server extends ServerContainer implements IServerContainer { return new PreviewManager( $c->get(\OCP\IConfig::class), $c->get(IRootFolder::class), - new \OC\Preview\Storage\Root( - $c->get(IRootFolder::class), - $c->get(SystemConfig::class) - ), $c->get(IEventDispatcher::class), $c->get(GeneratorHelper::class), $c->get(ISession::class)->get('user_id'), @@ -306,12 +304,10 @@ class Server extends ServerContainer implements IServerContainer { }); $this->registerAlias(IMimeIconProvider::class, MimeIconProvider::class); - $this->registerService(\OC\Preview\Watcher::class, function (ContainerInterface $c) { - return new \OC\Preview\Watcher( - new \OC\Preview\Storage\Root( - $c->get(IRootFolder::class), - $c->get(SystemConfig::class) - ) + $this->registerService(Watcher::class, function (ContainerInterface $c): Watcher { + return new Watcher( + $c->get(\OC\Preview\Storage\StorageFactory::class), + $c->get(PreviewMapper::class), ); }); diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index 89503085fa4..ea08b58955d 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -82,6 +82,11 @@ class BackgroundCleanupJobTest extends \Test\TestCase { $this->logout(); + foreach ($this->previewMapper->getAvailablePreviews(5) as $preview) { + $this->previewStorageFactory->deletePreview($preview); + $this->previewMapper->delete($preview); + } + parent::tearDown(); } @@ -89,7 +94,7 @@ class BackgroundCleanupJobTest extends \Test\TestCase { $userFolder = $this->rootFolder->getUserFolder($this->userId); $files = []; - for ($i = 0; $i < 11; $i++) { + foreach (range(0, 10) as $i) { $file = $userFolder->newFile($i . '.txt'); $file->putContent('hello world!'); $this->previewManager->getPreview($file); diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php index 418f28faff7..a226b611c5a 100644 --- a/tests/lib/Preview/MovePreviewJobTest.php +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -3,11 +3,17 @@ namespace lib\Preview; use OC\Core\BackgroundJobs\MovePreviewJob; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\StorageFactory; +use OCP\AppFramework\Utility\ITimeFactory; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\IAppData; +use OCP\IAppConfig; +use OCP\IDBConnection; use OCP\Server; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\Attributes\TestDox; +use PHPUnit\Framework\MockObject\MockObject; use Test\TestCase; /** @@ -16,17 +22,78 @@ use Test\TestCase; #[CoversClass(MovePreviewJob::class)] class MovePreviewJobTest extends TestCase { private IAppData $previewAppData; + private PreviewMapper $previewMapper; + private IAppConfig&MockObject $appConfig; + private StorageFactory $storageFactory; public function setUp(): void { parent::setUp(); $this->previewAppData = Server::get(IAppDataFactory::class)->get('preview'); + $this->previewMapper = Server::get(PreviewMapper::class); + $this->appConfig = $this->createMock(IAppConfig::class); + $this->appConfig->expects($this->any()) + ->method('getValueBool') + ->willReturn(false); + $this->storageFactory = Server::get(StorageFactory::class); } - #[TestDox("Test the migration from the legacy flat hierarchy to the new one")] + public function tearDown(): void { + foreach ($this->previewMapper->getAvailablePreviewForFile(5) as $preview) { + $this->storageFactory->deletePreview($preview); + $this->previewMapper->delete($preview); + } + + foreach ($this->previewAppData->getDirectoryListing() as $folder) { + $folder->delete(); + } + } + + #[TestDox("Test the migration from the legacy flat hierarchy to the new database format")] function testMigrationLegacyPath(): void { $folder = $this->previewAppData->newFolder(5); - $file = $folder->newFile('64-64-crop.png', 'abcdefg'); - $job = Server::get(MovePreviewJob::class); - $this->invokePrivate($job, 'run', []); + $folder->newFile('64-64-crop.jpg', 'abcdefg'); + $folder->newFile('128-128-crop.png', 'abcdefg'); + $this->assertEquals(1, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(2, count($folder->getDirectoryListing())); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + + $job = new MovePreviewJob( + Server::get(ITimeFactory::class), + $this->appConfig, + $this->previewMapper, + $this->storageFactory, + Server::get(IDBConnection::class), + Server::get(IAppDataFactory::class) + ); + $this->invokePrivate($job, 'run', [[]]); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + } + + private static function getInternalFolder(string $name): string { + return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; + } + + #[TestDox("Test the migration from the 'new' nested hierarchy to the database format")] + function testMigrationPath(): void { + $folder = $this->previewAppData->newFolder(self::getInternalFolder(5)); + $folder->newFile('64-64-crop.jpg', 'abcdefg'); + $folder->newFile('128-128-crop.png', 'abcdefg'); + + $folder = $this->previewAppData->getFolder(self::getInternalFolder(5)); + $this->assertEquals(2, count($folder->getDirectoryListing())); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + + $job = new MovePreviewJob( + Server::get(ITimeFactory::class), + $this->appConfig, + $this->previewMapper, + $this->storageFactory, + Server::get(IDBConnection::class), + Server::get(IAppDataFactory::class) + ); + $this->invokePrivate($job, 'run', [[]]); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); + $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); } } diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index 68aea1bfcc1..d6925641d4e 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -55,8 +55,8 @@ class PreviewMapperTest extends TestCase { $qb = $this->connection->getQueryBuilder(); $qb->insert('preview_locations') ->values([ - 'bucket' => $qb->createNamedParameter('preview-' . $bucket), - 'object_store' => $qb->createNamedParameter('default'), + 'bucket_name' => $qb->createNamedParameter('preview-' . $bucket), + 'object_store_name' => $qb->createNamedParameter('default'), ]); $locationId = $qb->executeStatement(); } diff --git a/version.php b/version.php index d244b03ab24..b4a6c283655 100644 --- a/version.php +++ b/version.php @@ -9,7 +9,7 @@ // between betas, final and RCs. This is _not_ the public version number. Reset minor/patch level // when updating major/minor version number. -$OC_Version = [33, 0, 0, 0]; +$OC_Version = [33, 0, 0, 1]; // The human-readable string $OC_VersionString = '33.0.0 dev'; From 324b54b863ac07f93e0553462608089b7240c618 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Tue, 16 Sep 2025 11:34:41 +0200 Subject: [PATCH 08/14] refactor(preview): Cleanup the implementation of the new preview backend Signed-off-by: Carl Schwan --- build/psalm-baseline.xml | 11 - build/stubs/php-polyfill.php | 9 + core/BackgroundJobs/MovePreviewJob.php | 223 +++++++------ core/Command/Preview/Repair.php | 293 ------------------ core/Command/Preview/ResetRenderedTexts.php | 72 ++--- .../Version33000Date20250819110529.php | 3 +- core/register_command.php | 1 - lib/composer/composer/autoload_classmap.php | 3 +- lib/composer/composer/autoload_static.php | 15 +- lib/private/BackgroundJob/JobList.php | 1 - lib/private/Preview/BackgroundCleanupJob.php | 47 +-- lib/private/Preview/Db/Preview.php | 74 ++--- lib/private/Preview/Db/PreviewMapper.php | 18 +- lib/private/Preview/Generator.php | 10 +- lib/private/Preview/PreviewService.php | 101 ++++++ .../Preview/Storage/IPreviewStorage.php | 14 +- .../Preview/Storage/LocalPreviewStorage.php | 6 +- .../Storage/ObjectStorePreviewStorage.php | 11 +- .../Preview/Storage/StorageFactory.php | 12 +- lib/private/Preview/Watcher.php | 5 +- lib/private/PreviewManager.php | 1 - psalm.xml | 1 + tests/Core/Command/Preview/RepairTest.php | 153 --------- .../PrimaryObjectStoreConfigTest.php | 3 + .../lib/Preview/BackgroundCleanupJobTest.php | 41 +-- tests/lib/Preview/GeneratorTest.php | 26 +- tests/lib/Preview/MovePreviewJobTest.php | 123 +++++++- tests/lib/Preview/PreviewMapperTest.php | 17 +- tests/lib/Preview/PreviewServiceTest.php | 59 ++++ 29 files changed, 565 insertions(+), 788 deletions(-) create mode 100644 build/stubs/php-polyfill.php delete mode 100644 core/Command/Preview/Repair.php create mode 100644 lib/private/Preview/PreviewService.php delete mode 100644 tests/Core/Command/Preview/RepairTest.php create mode 100644 tests/lib/Preview/PreviewServiceTest.php diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 606cb3b5bb1..21baeb9a33b 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -2708,17 +2708,6 @@ timeFactory->getTime()]]> - - - - - - - - - - - diff --git a/build/stubs/php-polyfill.php b/build/stubs/php-polyfill.php new file mode 100644 index 00000000000..606a21f8dfe --- /dev/null +++ b/build/stubs/php-polyfill.php @@ -0,0 +1,9 @@ +doRun($argument); - } catch (\Throwable $exception) { - echo $exception->getMessage(); - throw $exception; - } - } - - private function doRun($argument): void { if ($this->appConfig->getValueBool('core', 'previewMovedDone')) { return; } @@ -59,14 +54,13 @@ class MovePreviewJob extends TimedJob { $startTime = time(); while (true) { - $previewFolders = []; - // Check new hierarchical preview folders first if (!$emptyHierarchicalPreviewFolders) { $qb = $this->connection->getQueryBuilder(); $qb->select('*') ->from('filecache') ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%/%/%/%/%/%/%/%'))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) ->setMaxResults(100); $result = $qb->executeQuery(); @@ -74,12 +68,7 @@ class MovePreviewJob extends TimedJob { $pathSplit = explode('/', $row['path']); assert(count($pathSplit) >= 2); $fileId = $pathSplit[count($pathSplit) - 2]; - $previewFolders[$fileId][] = $row['path']; - } - - if (!empty($previewFolders)) { - $this->processPreviews($previewFolders, false); - continue; + $this->processPreviews($fileId, false); } } @@ -89,6 +78,7 @@ class MovePreviewJob extends TimedJob { $qb->select('*') ->from('filecache') ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.%'))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) ->setMaxResults(100); $result = $qb->executeQuery(); @@ -98,18 +88,7 @@ class MovePreviewJob extends TimedJob { $fileId = $pathSplit[count($pathSplit) - 2]; array_pop($pathSplit); $path = implode('/', $pathSplit); - if (!isset($previewFolders[$fileId])) { - $previewFolders[$fileId] = []; - } - if (!in_array($path, $previewFolders[$fileId])) { - $previewFolders[$fileId][] = $path; - } - } - - if (empty($previewFolders)) { - break; - } else { - $this->processPreviews($previewFolders, true); + $this->processPreviews($fileId, true); } // Stop if execution time is more than one hour. @@ -118,97 +97,117 @@ class MovePreviewJob extends TimedJob { } } - // Delete any leftover preview directory - $this->appData->getFolder('.')->delete(); + try { + // Delete any leftover preview directory + $this->appData->getFolder('.')->delete(); + } catch (NotFoundException) { + // ignore + } $this->appConfig->setValueBool('core', 'previewMovedDone', true); } /** * @param array $previewFolders */ - private function processPreviews(array $previewFolders, bool $simplePaths): void { - foreach ($previewFolders as $fileId => $previewFolder) { - $internalPath = $this->getInternalFolder((string)$fileId, $simplePaths); - $folder = $this->appData->getFolder($internalPath); + private function processPreviews(int|string $fileId, bool $simplePaths): void { + $internalPath = $this->getInternalFolder((string)$fileId, $simplePaths); + $folder = $this->appData->getFolder($internalPath); - /** - * @var list $previewFiles - */ - $previewFiles = []; + /** + * @var list $previewFiles + */ + $previewFiles = []; - foreach ($folder->getDirectoryListing() as $previewFile) { - /** @var SimpleFile $previewFile */ - [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); - $nameSplit = explode('-', $baseName); + foreach ($folder->getDirectoryListing() as $previewFile) { + /** @var SimpleFile $previewFile */ + [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); + $nameSplit = explode('-', $baseName); - // TODO VERSION/PREFIX extraction - - $width = $nameSplit[0]; - $height = $nameSplit[1]; - - if (isset($nameSplit[2])) { - $crop = $nameSplit[2] === 'crop'; - $max = $nameSplit[2] === 'max'; - } - - $previewFiles[] = [ - 'file' => $previewFile, - 'width' => $width, - 'height' => $height, - 'crop' => $crop, - 'max' => $max, - 'extension' => $extension, - 'size' => $previewFile->getSize(), - 'mtime' => $previewFile->getMTime(), - ]; + $offset = 0; + $version = null; + if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { + $offset = 1; + $version = (int)$nameSplit[0]; } - $qb = $this->connection->getQueryBuilder(); - $qb->select('*') - ->from('filecache') - ->where($qb->expr()->like('fileid', $qb->createNamedParameter($fileId))); + $width = (int)$nameSplit[$offset + 0]; + $height = (int)$nameSplit[$offset + 1]; - $result = $qb->executeQuery(); - $result = $result->fetchAll(); - - if (count($result) > 0) { - foreach ($previewFiles as $previewFile) { - $preview = new Preview(); - $preview->setFileId((int)$fileId); - $preview->setOldFileId($previewFile['file']->getId()); - $preview->setEtag($result[0]['etag']); - $preview->setMtime($previewFile['mtime']); - $preview->setWidth($previewFile['width']); - $preview->setHeight($previewFile['height']); - $preview->setCrop($previewFile['crop']); - $preview->setIsMax($previewFile['max']); - $preview->setMimetype(match ($previewFile['extension']) { - 'png' => IPreview::MIMETYPE_PNG, - 'webp' => IPreview::MIMETYPE_WEBP, - 'gif' => IPreview::MIMETYPE_GIF, - default => IPreview::MIMETYPE_JPEG, - }); - $preview->setSize($previewFile['size']); - try { - $preview = $this->previewMapper->insert($preview); - } catch (Exception $e) { - // We already have this preview in the preview table, skip - continue; - } - - try { - $this->storageFactory->migratePreview($preview, $previewFile['file']); - $previewFile['file']->delete(); - } catch (\Exception $e) { - $this->previewMapper->delete($preview); - throw $e; - } - - } + $crop = false; + $max = false; + if (isset($nameSplit[$offset + 2])) { + $crop = $nameSplit[$offset + 2] === 'crop'; + $max = $nameSplit[$offset + 2] === 'max'; } - $this->deleteFolder($internalPath, $folder); + $previewFiles[] = [ + 'file' => $previewFile, + 'width' => $width, + 'height' => $height, + 'crop' => $crop, + 'version' => $version, + 'max' => $max, + 'extension' => $extension, + 'size' => $previewFile->getSize(), + 'mtime' => $previewFile->getMTime(), + ]; } + + $qb = $this->connection->getQueryBuilder(); + $qb->select('*') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))) + ->setMaxResults(1); + + $result = $qb->executeQuery(); + $result = $result->fetchAll(); + + if (count($result) > 0) { + foreach ($previewFiles as $previewFile) { + $preview = new Preview(); + $preview->setFileId((int)$fileId); + /** @var SimpleFile $file */ + $file = $previewFile['file']; + $preview->setOldFileId($file->getId()); + $preview->setStorageId($result[0]['storage']); + $preview->setEtag($result[0]['etag']); + $preview->setMtime($previewFile['mtime']); + $preview->setWidth($previewFile['width']); + $preview->setHeight($previewFile['height']); + $preview->setCropped($previewFile['crop']); + $preview->setVersion($previewFile['version']); + $preview->setMax($previewFile['max']); + $preview->setEncrypted(false); + $preview->setMimetype(match ($previewFile['extension']) { + 'png' => IPreview::MIMETYPE_PNG, + 'webp' => IPreview::MIMETYPE_WEBP, + 'gif' => IPreview::MIMETYPE_GIF, + default => IPreview::MIMETYPE_JPEG, + }); + $preview->setSize($previewFile['size']); + try { + $preview = $this->previewMapper->insert($preview); + } catch (Exception $e) { + // We already have this preview in the preview table, skip + continue; + } + + try { + $this->storageFactory->migratePreview($preview, $file); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId()))) + ->executeStatement(); + // Do not call $file->delete() as this will also delete the file from the file system + } catch (\Exception $e) { + $this->previewMapper->delete($preview); + throw $e; + } + } + } + + $this->deleteFolder($internalPath, $folder); } public static function getInternalFolder(string $name, bool $simplePaths): string { diff --git a/core/Command/Preview/Repair.php b/core/Command/Preview/Repair.php deleted file mode 100644 index a92a4cf8ed0..00000000000 --- a/core/Command/Preview/Repair.php +++ /dev/null @@ -1,293 +0,0 @@ -memoryLimit = (int)$phpIni->getBytes('memory_limit'); - $this->memoryTreshold = $this->memoryLimit - 25 * 1024 * 1024; - - parent::__construct(); - } - - protected function configure() { - $this - ->setName('preview:repair') - ->setDescription('distributes the existing previews into subfolders') - ->addOption('batch', 'b', InputOption::VALUE_NONE, 'Batch mode - will not ask to start the migration and start it right away.') - ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not create, move or delete any files - in combination with the verbose mode one could check the operations.') - ->addOption('delete', null, InputOption::VALUE_NONE, 'Delete instead of migrating them. Usefull if too many entries to migrate.'); - } - - protected function execute(InputInterface $input, OutputInterface $output): int { - if ($this->memoryLimit !== -1) { - $limitInMiB = round($this->memoryLimit / 1024 / 1024, 1); - $thresholdInMiB = round($this->memoryTreshold / 1024 / 1024, 1); - $output->writeln("Memory limit is $limitInMiB MiB"); - $output->writeln("Memory threshold is $thresholdInMiB MiB"); - $output->writeln(''); - $memoryCheckEnabled = true; - } else { - $output->writeln('No memory limit in place - disabled memory check. Set a PHP memory limit to automatically stop the execution of this migration script once memory consumption is close to this limit.'); - $output->writeln(''); - $memoryCheckEnabled = false; - } - - $dryMode = $input->getOption('dry'); - $deleteMode = $input->getOption('delete'); - - - if ($dryMode) { - $output->writeln('INFO: The migration is run in dry mode and will not modify anything.'); - $output->writeln(''); - } elseif ($deleteMode) { - $output->writeln('WARN: The migration will _DELETE_ old previews.'); - $output->writeln(''); - } - - $instanceId = $this->config->getSystemValueString('instanceid'); - - $output->writeln('This will migrate all previews from the old preview location to the new one.'); - $output->writeln(''); - - $output->writeln('Fetching previews that need to be migrated …'); - /** @var Folder $currentPreviewFolder */ - $currentPreviewFolder = $this->rootFolder->get("appdata_$instanceId/preview"); - - $directoryListing = $currentPreviewFolder->getDirectoryListing(); - - $total = count($directoryListing); - /** - * by default there could be 0-9 a-f and the old-multibucket folder which are all fine - */ - if ($total < 18) { - $directoryListing = array_filter($directoryListing, function ($dir) { - if ($dir->getName() === 'old-multibucket') { - return false; - } - - // a-f can't be a file ID -> removing from migration - if (preg_match('!^[a-f]$!', $dir->getName())) { - return false; - } - - if (preg_match('!^[0-9]$!', $dir->getName())) { - // ignore folders that only has folders in them - if ($dir instanceof Folder) { - foreach ($dir->getDirectoryListing() as $entry) { - if (!$entry instanceof Folder) { - return true; - } - } - return false; - } - } - return true; - }); - $total = count($directoryListing); - } - - if ($total === 0) { - $output->writeln('All previews are already migrated.'); - return 0; - } - - $output->writeln("A total of $total preview files need to be migrated."); - $output->writeln(''); - $output->writeln('The migration will always migrate all previews of a single file in a batch. After each batch the process can be canceled by pressing CTRL-C. This will finish the current batch and then stop the migration. This migration can then just be started and it will continue.'); - - if ($input->getOption('batch')) { - $output->writeln('Batch mode active: migration is started right away.'); - } else { - /** @var QuestionHelper $helper */ - $helper = $this->getHelper('question'); - $question = new ConfirmationQuestion('Should the migration be started? (y/[n]) ', false); - - if (!$helper->ask($input, $output, $question)) { - return 0; - } - } - - // register the SIGINT listener late in here to be able to exit in the early process of this command - pcntl_signal(SIGINT, [$this, 'sigIntHandler']); - - $output->writeln(''); - $output->writeln(''); - $section1 = $output->section(); - $section2 = $output->section(); - $progressBar = new ProgressBar($section2, $total); - $progressBar->setFormat('%current%/%max% [%bar%] %percent:3s%% %elapsed:6s%/%estimated:-6s% Used Memory: %memory:6s%'); - $time = (new \DateTime())->format('H:i:s'); - $progressBar->setMessage("$time Starting …"); - $progressBar->maxSecondsBetweenRedraws(0.2); - $progressBar->start(); - - foreach ($directoryListing as $oldPreviewFolder) { - pcntl_signal_dispatch(); - $name = $oldPreviewFolder->getName(); - $time = (new \DateTime())->format('H:i:s'); - $section1->writeln("$time Migrating previews of file with fileId $name …"); - $progressBar->display(); - - if ($this->stopSignalReceived) { - $section1->writeln("$time Stopping migration …"); - return 0; - } - if (!$oldPreviewFolder instanceof Folder) { - $section1->writeln(" Skipping non-folder $name …"); - $progressBar->advance(); - continue; - } - if ($name === 'old-multibucket') { - $section1->writeln(" Skipping fallback mount point $name …"); - $progressBar->advance(); - continue; - } - if (in_array($name, ['a', 'b', 'c', 'd', 'e', 'f'])) { - $section1->writeln(" Skipping hex-digit folder $name …"); - $progressBar->advance(); - continue; - } - if (!preg_match('!^\d+$!', $name)) { - $section1->writeln(" Skipping non-numeric folder $name …"); - $progressBar->advance(); - continue; - } - - $newFoldername = Root::getInternalFolder($name); - - $memoryUsage = memory_get_usage(); - if ($memoryCheckEnabled && $memoryUsage > $this->memoryTreshold) { - $section1->writeln(''); - $section1->writeln(''); - $section1->writeln(''); - $section1->writeln(' Stopped process 25 MB before reaching the memory limit to avoid a hard crash.'); - $time = (new \DateTime())->format('H:i:s'); - $section1->writeln("$time Reached memory limit and stopped to avoid hard crash."); - return 1; - } - - $lockName = 'occ preview:repair lock ' . $oldPreviewFolder->getId(); - try { - $section1->writeln(" Locking \"$lockName\" …", OutputInterface::VERBOSITY_VERBOSE); - $this->lockingProvider->acquireLock($lockName, ILockingProvider::LOCK_EXCLUSIVE); - } catch (LockedException $e) { - $section1->writeln(' Skipping because it is locked - another process seems to work on this …'); - continue; - } - - $previews = $oldPreviewFolder->getDirectoryListing(); - if ($previews !== []) { - try { - $this->rootFolder->get("appdata_$instanceId/preview/$newFoldername"); - } catch (NotFoundException $e) { - $section1->writeln(" Create folder preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE); - if (!$dryMode) { - $this->rootFolder->newFolder("appdata_$instanceId/preview/$newFoldername"); - } - } - - foreach ($previews as $preview) { - pcntl_signal_dispatch(); - $previewName = $preview->getName(); - - if ($preview instanceof Folder) { - $section1->writeln(" Skipping folder $name/$previewName …"); - $progressBar->advance(); - continue; - } - - // Execute process - if (!$dryMode) { - // Delete preview instead of moving - if ($deleteMode) { - try { - $section1->writeln(" Delete preview/$name/$previewName", OutputInterface::VERBOSITY_VERBOSE); - $preview->delete(); - } catch (\Exception $e) { - $this->logger->error("Failed to delete preview at preview/$name/$previewName", [ - 'app' => 'core', - 'exception' => $e, - ]); - } - } else { - try { - $section1->writeln(" Move preview/$name/$previewName to preview/$newFoldername", OutputInterface::VERBOSITY_VERBOSE); - $preview->move("appdata_$instanceId/preview/$newFoldername/$previewName"); - } catch (\Exception $e) { - $this->logger->error("Failed to move preview from preview/$name/$previewName to preview/$newFoldername", [ - 'app' => 'core', - 'exception' => $e, - ]); - } - } - } - } - } - - if ($oldPreviewFolder->getDirectoryListing() === []) { - $section1->writeln(" Delete empty folder preview/$name", OutputInterface::VERBOSITY_VERBOSE); - if (!$dryMode) { - try { - $oldPreviewFolder->delete(); - } catch (\Exception $e) { - $this->logger->error("Failed to delete empty folder preview/$name", [ - 'app' => 'core', - 'exception' => $e, - ]); - } - } - } - - $this->lockingProvider->releaseLock($lockName, ILockingProvider::LOCK_EXCLUSIVE); - $section1->writeln(' Unlocked', OutputInterface::VERBOSITY_VERBOSE); - - $section1->writeln(" Finished migrating previews of file with fileId $name …"); - $progressBar->advance(); - } - - $progressBar->finish(); - $output->writeln(''); - return 0; - } - - protected function sigIntHandler() { - echo "\nSignal received - will finish the step and then stop the migration.\n\n\n"; - $this->stopSignalReceived = true; - } -} diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index ce874e15406..fb7fb9fa620 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -8,8 +8,8 @@ declare(strict_types=1); */ namespace OC\Core\Command\Preview; -use OC\Preview\Storage\Root; -use OCP\DB\QueryBuilder\IQueryBuilder; +use OC\Preview\Db\Preview; +use OC\Preview\PreviewService; use OCP\Files\IMimeTypeLoader; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; @@ -23,16 +23,16 @@ use Symfony\Component\Console\Output\OutputInterface; class ResetRenderedTexts extends Command { public function __construct( - protected IDBConnection $connection, - protected IUserManager $userManager, - protected IAvatarManager $avatarManager, - private Root $previewFolder, - private IMimeTypeLoader $mimeTypeLoader, + protected readonly IDBConnection $connection, + protected readonly IUserManager $userManager, + protected readonly IAvatarManager $avatarManager, + private readonly PreviewService $previewService, + private readonly IMimeTypeLoader $mimeTypeLoader, ) { parent::__construct(); } - protected function configure() { + protected function configure(): void { $this ->setName('preview:reset-rendered-texts') ->setDescription('Deletes all generated avatars and previews of text and md files') @@ -91,7 +91,7 @@ class ResetRenderedTexts extends Command { private function deletePreviews(OutputInterface $output, bool $dryMode): void { $previewsToDeleteCount = 0; - foreach ($this->getPreviewsToDelete() as ['name' => $previewFileId, 'path' => $filePath]) { + foreach ($this->getPreviewsToDelete() as ['path' => $filePath, 'preview' => $preview]) { $output->writeln('Deleting previews for ' . $filePath, OutputInterface::VERBOSITY_VERBOSE); $previewsToDeleteCount++; @@ -100,63 +100,33 @@ class ResetRenderedTexts extends Command { continue; } - try { - $preview = $this->previewFolder->getFolder((string)$previewFileId); - $preview->delete(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { - // continue - } + $this->previewService->deletePreview($preview); } $output->writeln('Deleted ' . $previewsToDeleteCount . ' previews'); } - // Copy pasted and adjusted from - // "lib/private/Preview/BackgroundCleanupJob.php". + /** + * @return \Iterator + */ private function getPreviewsToDelete(): \Iterator { $qb = $this->connection->getQueryBuilder(); - $qb->select('path', 'mimetype') + $qb->select('fileid', 'path') ->from('filecache') - ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId()))); - $cursor = $qb->executeQuery(); - $data = $cursor->fetch(); - $cursor->closeCursor(); - - if ($data === null) { - return []; - } - - /* - * This lovely like is the result of the way the new previews are stored - * We take the md5 of the name (fileid) and split the first 7 chars. That way - * there are not a gazillion files in the root of the preview appdata. - */ - $like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%'; - - $qb = $this->connection->getQueryBuilder(); - $qb->select('a.name', 'b.path') - ->from('filecache', 'a') - ->leftJoin('a', 'filecache', 'b', $qb->expr()->eq( - $qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid' - )) ->where( - $qb->expr()->andX( - $qb->expr()->like('a.path', $qb->createNamedParameter($like)), - $qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))), - $qb->expr()->orX( - $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))), - $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))), - $qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown'))) - ) + $qb->expr()->orX( + $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))), + $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))), + $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown'))) ) ); $cursor = $qb->executeQuery(); while ($row = $cursor->fetch()) { - yield $row; + foreach ($this->previewService->getAvailablePreviewForFile($row['fileid']) as $preview) { + yield ['path' => $row['path'], 'preview' => $preview]; + } } $cursor->closeCursor(); diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php index 5e607024ec7..27bf9ab89b7 100644 --- a/core/Migrations/Version33000Date20250819110529.php +++ b/core/Migrations/Version33000Date20250819110529.php @@ -41,6 +41,7 @@ class Version33000Date20250819110529 extends SimpleMigrationStep { $table = $schema->createTable('previews'); $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('storage_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); $table->addColumn('old_file_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); $table->addColumn('location_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); @@ -56,7 +57,7 @@ class Version33000Date20250819110529 extends SimpleMigrationStep { $table->addColumn('version', Types::BIGINT, ['notnull' => true, 'default' => -1]); // can not be null otherwise unique index doesn't work $table->setPrimaryKey(['id']); $table->addIndex(['file_id']); - $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx'); + $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'cropped', 'version'], 'previews_file_uniq_idx'); } return $schema; diff --git a/core/register_command.php b/core/register_command.php index 9fd5b9b611e..f6c0b9466b5 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -206,7 +206,6 @@ if ($config->getSystemValueBool('installed', false)) { $application->add(Server::get(Command\Preview\Cleanup::class)); $application->add(Server::get(Generate::class)); - $application->add(Server::get(Command\Preview\Repair::class)); $application->add(Server::get(ResetRenderedTexts::class)); $application->add(Server::get(Add::class)); diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 5c168a25de3..4c2e473240b 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1344,7 +1344,6 @@ return array( 'OC\\Core\\Command\\Memcache\\RedisCommand' => $baseDir . '/core/Command/Memcache/RedisCommand.php', 'OC\\Core\\Command\\Preview\\Cleanup' => $baseDir . '/core/Command/Preview/Cleanup.php', 'OC\\Core\\Command\\Preview\\Generate' => $baseDir . '/core/Command/Preview/Generate.php', - 'OC\\Core\\Command\\Preview\\Repair' => $baseDir . '/core/Command/Preview/Repair.php', 'OC\\Core\\Command\\Preview\\ResetRenderedTexts' => $baseDir . '/core/Command/Preview/ResetRenderedTexts.php', 'OC\\Core\\Command\\Router\\ListRoutes' => $baseDir . '/core/Command/Router/ListRoutes.php', 'OC\\Core\\Command\\Router\\MatchRoute' => $baseDir . '/core/Command/Router/MatchRoute.php', @@ -1909,6 +1908,7 @@ return array( 'OC\\Preview\\PNG' => $baseDir . '/lib/private/Preview/PNG.php', 'OC\\Preview\\Photoshop' => $baseDir . '/lib/private/Preview/Photoshop.php', 'OC\\Preview\\Postscript' => $baseDir . '/lib/private/Preview/Postscript.php', + 'OC\\Preview\\PreviewService' => $baseDir . '/lib/private/Preview/PreviewService.php', 'OC\\Preview\\Provider' => $baseDir . '/lib/private/Preview/Provider.php', 'OC\\Preview\\ProviderV1Adapter' => $baseDir . '/lib/private/Preview/ProviderV1Adapter.php', 'OC\\Preview\\ProviderV2' => $baseDir . '/lib/private/Preview/ProviderV2.php', @@ -1919,7 +1919,6 @@ return array( 'OC\\Preview\\Storage\\LocalPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/LocalPreviewStorage.php', 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => $baseDir . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', 'OC\\Preview\\Storage\\PreviewFile' => $baseDir . '/lib/private/Preview/Storage/PreviewFile.php', - 'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php', 'OC\\Preview\\Storage\\StorageFactory' => $baseDir . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => $baseDir . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 423b4cc6b57..2cb14099c01 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -11,32 +11,32 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 ); public static $prefixLengthsPsr4 = array ( - 'O' => + 'O' => array ( 'OC\\Core\\' => 8, 'OC\\' => 3, 'OCP\\' => 4, ), - 'N' => + 'N' => array ( 'NCU\\' => 4, ), ); public static $prefixDirsPsr4 = array ( - 'OC\\Core\\' => + 'OC\\Core\\' => array ( 0 => __DIR__ . '/../../..' . '/core', ), - 'OC\\' => + 'OC\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/private', ), - 'OCP\\' => + 'OCP\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/public', ), - 'NCU\\' => + 'NCU\\' => array ( 0 => __DIR__ . '/../../..' . '/lib/unstable', ), @@ -1385,7 +1385,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Command\\Memcache\\RedisCommand' => __DIR__ . '/../../..' . '/core/Command/Memcache/RedisCommand.php', 'OC\\Core\\Command\\Preview\\Cleanup' => __DIR__ . '/../../..' . '/core/Command/Preview/Cleanup.php', 'OC\\Core\\Command\\Preview\\Generate' => __DIR__ . '/../../..' . '/core/Command/Preview/Generate.php', - 'OC\\Core\\Command\\Preview\\Repair' => __DIR__ . '/../../..' . '/core/Command/Preview/Repair.php', 'OC\\Core\\Command\\Preview\\ResetRenderedTexts' => __DIR__ . '/../../..' . '/core/Command/Preview/ResetRenderedTexts.php', 'OC\\Core\\Command\\Router\\ListRoutes' => __DIR__ . '/../../..' . '/core/Command/Router/ListRoutes.php', 'OC\\Core\\Command\\Router\\MatchRoute' => __DIR__ . '/../../..' . '/core/Command/Router/MatchRoute.php', @@ -1950,6 +1949,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\PNG' => __DIR__ . '/../../..' . '/lib/private/Preview/PNG.php', 'OC\\Preview\\Photoshop' => __DIR__ . '/../../..' . '/lib/private/Preview/Photoshop.php', 'OC\\Preview\\Postscript' => __DIR__ . '/../../..' . '/lib/private/Preview/Postscript.php', + 'OC\\Preview\\PreviewService' => __DIR__ . '/../../..' . '/lib/private/Preview/PreviewService.php', 'OC\\Preview\\Provider' => __DIR__ . '/../../..' . '/lib/private/Preview/Provider.php', 'OC\\Preview\\ProviderV1Adapter' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV1Adapter.php', 'OC\\Preview\\ProviderV2' => __DIR__ . '/../../..' . '/lib/private/Preview/ProviderV2.php', @@ -1960,7 +1960,6 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\Storage\\LocalPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/LocalPreviewStorage.php', 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', 'OC\\Preview\\Storage\\PreviewFile' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/PreviewFile.php', - 'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php', 'OC\\Preview\\Storage\\StorageFactory' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => __DIR__ . '/../../..' . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php', diff --git a/lib/private/BackgroundJob/JobList.php b/lib/private/BackgroundJob/JobList.php index 302bee22cd5..c00a51e3851 100644 --- a/lib/private/BackgroundJob/JobList.php +++ b/lib/private/BackgroundJob/JobList.php @@ -321,7 +321,6 @@ class JobList implements IJobList { /** @var IJob $job */ $job = \OCP\Server::get($row['class']); } catch (QueryException $e) { - echo $e->getMessage(); if (class_exists($row['class'])) { $class = $row['class']; $job = new $class(); diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index d65b740745d..257bebb1d5d 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -8,8 +8,7 @@ declare(strict_types=1); */ namespace OC\Preview; -use OC\Preview\Db\PreviewMapper; -use OC\Preview\Storage\StorageFactory; +use OC\Preview\Db\Preview; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; use OCP\DB\QueryBuilder\IQueryBuilder; @@ -24,8 +23,7 @@ class BackgroundCleanupJob extends TimedJob { public function __construct( ITimeFactory $timeFactory, readonly private IDBConnection $connection, - readonly private PreviewMapper $previewMapper, - readonly private StorageFactory $storageFactory, + readonly private PreviewService $previewService, readonly private bool $isCLI, ) { parent::__construct($timeFactory); @@ -37,11 +35,9 @@ class BackgroundCleanupJob extends TimedJob { public function run($argument): void { foreach ($this->getDeletedFiles() as $fileId) { $previewIds = []; - foreach ($this->previewMapper->getByFileId($fileId) as $preview) { - $previewIds[] = $preview->getId(); - $this->storageFactory->deletePreview($preview); + foreach ($this->previewService->getAvailablePreviewForFile($fileId) as $preview) { + $this->previewService->deletePreview($preview); } - $this->previewMapper->deleteByIds($previewIds); } } @@ -50,13 +46,12 @@ class BackgroundCleanupJob extends TimedJob { */ private function getDeletedFiles(): \Iterator { if ($this->connection->getShardDefinition('filecache')) { - $chunks = $this->getAllPreviewIds(1000); - foreach ($chunks as $chunk) { - foreach ($chunk as $storage => $preview) { - yield [$storage => $this->findMissingSources($storage, $preview)]; + foreach ($this->previewService->getAvailableFileIds() as $availableFileIdGroup) { + $fileIds = $this->findMissingSources($availableFileIdGroup['storageId'], $availableFileIdGroup['fileIds']); + foreach ($fileIds as $fileId) { + yield $fileId; } } - return; } @@ -89,35 +84,11 @@ class BackgroundCleanupJob extends TimedJob { $cursor = $qb->executeQuery(); while ($row = $cursor->fetch()) { - yield $row['file_id']; + yield (int)$row['file_id']; } $cursor->closeCursor(); } - /** - * @return \Iterator - */ - private function getAllPreviewIds(int $chunkSize): \Iterator { - $qb = $this->connection->getQueryBuilder(); - $qb->select('id', 'file_id', 'storage_id') - ->from('previews') - ->where( - $qb->expr()->gt('id', $qb->createParameter('min_id')), - ) - ->orderBy('id', 'ASC') - ->setMaxResults($chunkSize); - - $minId = 0; - while (true) { - $qb->setParameter('min_id', $minId); - $cursor = $qb->executeQuery(); - while ($row = $cursor->fetch()) { - yield $row['file_id']; - } - $cursor->closeCursor(); - } - } - /** * @param FileId[] $ids * @return FileId[] diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index b4ebe8b4805..b6f2ef2942e 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -15,64 +15,63 @@ use OCP\DB\Types; use OCP\IPreview; /** - * @method \int getFileId() + * Preview entity mapped to the oc_previews and oc_preview_locations table. + * + * @method int getFileId() Get the file id of the original file. * @method void setFileId(int $fileId) - * @method \int getOldFileId() // Old location in the file-cache table, for legacy compatibility - * @method void setOldFileId(int $fileId) - * @method \int getLocationId() + * @method int getStorageId() Get the storage id of the original file. + * @method void setStorageId(int $fileId) + * @method int getOldFileId() Get the old location in the file-cache table, for legacy compatibility. + * @method void setOldFileId(int $oldFileId) + * @method int getLocationId() Get the location id in the preview_locations table. Only set when using an object store as primary storage. * @method void setLocationId(int $locationId) - * @method \string getBucketName() - * @method \string getObjectStoreName() - * @method \int getWidth() + * @method string getBucketName() Get the bucket name where the preview is stored. This is stored in the preview_locations table. + * @method string getObjectStoreName() Get the object store name where the preview is stored. This is stored in the preview_locations table. + * @method int getWidth() Get the width of the preview. * @method void setWidth(int $width) - * @method \int getHeight() + * @method int getHeight() Get the height of the preview. * @method void setHeight(int $height) - * @method \int getMode() - * @method void setMode(int $mode) - * @method \bool getCrop() - * @method void setCrop(bool $crop) - * @method void setMimetype(int $mimetype) - * @method IPreview::MIMETYPE_* getMimetype() - * @method \int getMtime() + * @method bool isCropped() Get whether the preview is cropped or not. + * @method void setCropped(bool $cropped) + * @method void setMimetype(int $mimetype) Set the mimetype of the preview. + * @method int getMimetype() Get the mimetype of the preview. + * @method int getMtime() Get the modification time of the preview. * @method void setMtime(int $mtime) - * @method \int getSize() + * @method int getSize() Get the size of the preview. * @method void setSize(int $size) - * @method \bool getIsMax() - * @method void setIsMax(bool $max) - * @method \string getEtag() + * @method bool isMax() Get whether the preview is the biggest one which is then used to generate the smaller previews. + * @method void setMax(bool $max) + * @method string getEtag() Get the etag of the preview. * @method void setEtag(string $etag) - * @method ?\int getVersion() + * @method int|null getVersion() Get the version for files_versions_s3 * @method void setVersion(?int $version) + * @method bool|null getIs() Get the version for files_versions_s3 + * @method bool isEncrypted() Get whether the preview is encrypted. At the moment every preview is unencrypted. + * @method void setEncrypted(bool $encrypted) + * + * @see PreviewMapper */ class Preview extends Entity { protected ?int $fileId = null; - protected ?int $oldFileId = null; - + protected ?int $storageId = null; protected ?int $locationId = null; protected ?string $bucketName = null; protected ?string $objectStoreName = null; - protected ?int $width = null; - protected ?int $height = null; - protected ?int $mimetype = null; - protected ?int $mtime = null; - protected ?int $size = null; - - protected ?bool $isMax = null; - - protected ?bool $crop = null; - + protected ?bool $max = null; + protected ?bool $cropped = null; protected ?string $etag = null; - protected ?int $version = null; + protected ?bool $encrypted = null; public function __construct() { $this->addType('fileId', Types::BIGINT); + $this->addType('storageId', Types::BIGINT); $this->addType('oldFileId', Types::BIGINT); $this->addType('locationId', Types::BIGINT); $this->addType('width', Types::INTEGER); @@ -80,18 +79,19 @@ class Preview extends Entity { $this->addType('mimetype', Types::INTEGER); $this->addType('mtime', Types::INTEGER); $this->addType('size', Types::INTEGER); - $this->addType('isMax', Types::BOOLEAN); - $this->addType('crop', Types::BOOLEAN); + $this->addType('max', Types::BOOLEAN); + $this->addType('cropped', Types::BOOLEAN); + $this->addType('encrypted', Types::BOOLEAN); $this->addType('etag', Types::STRING); $this->addType('version', Types::BIGINT); } public function getName(): string { $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); - if ($this->getCrop()) { + if ($this->isCropped()) { $path .= '-crop'; } - if ($this->getIsMax()) { + if ($this->isMax()) { $path .= '-max'; } diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index dba62c5a163..ec3a0fa7e1f 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -9,7 +9,6 @@ declare(strict_types=1); namespace OC\Preview\Db; -use OC\Preview\Generator; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\QBMapper; use OCP\DB\Exception; @@ -123,4 +122,21 @@ class PreviewMapper extends QBMapper { return $qb->getLastInsertId(); } } + + public function deleteAll(): void { + $delete = $this->db->getQueryBuilder(); + $delete->delete($this->getTableName()); + } + + /** + * @return \Generator + */ + public function getPreviews(int $lastId, int $limit = 1000): \Generator { + $qb = $this->db->getQueryBuilder(); + $this->joinLocation($qb) + ->where($qb->expr()->gt('p.id', $qb->createNamedParameter($lastId, IQueryBuilder::PARAM_INT))) + ->setMaxResults($limit); + return $this->yieldEntities($qb); + + } } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 6d3626bb2fc..c6e49a1c6ce 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -158,7 +158,7 @@ class Generator { try { $preview = array_find($previews, fn (Preview $preview): bool => $preview->getWidth() === $width && $preview->getHeight() === $height && $preview->getMimetype() === $maxPreview->getMimetype() - && $preview->getVersion() === $previewVersion && $preview->getCrop() === $crop); + && $preview->getVersion() === $previewVersion && $preview->isCropped() === $crop); if ($preview) { $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper); @@ -300,7 +300,7 @@ class Generator { // We don't know the max preview size, so we can't use getCachedPreview. // It might have been generated with a higher resolution than the current value. foreach ($previews as $preview) { - if ($preview->getIsMax() && ($version == $preview->getVersion())) { + if ($preview->isMax() && ($version === $preview->getVersion())) { return $preview; } } @@ -539,11 +539,13 @@ class Generator { public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, int $version): Preview { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); + $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); $previewEntry->setHeight($height); $previewEntry->setVersion($version); - $previewEntry->setIsMax($max); - $previewEntry->setCrop($crop); + $previewEntry->setMax($max); + $previewEntry->setCropped($crop); + $previewEntry->setEncrypted(false); switch ($preview->dataMimeType()) { case 'image/jpeg': $previewEntry->setMimetype(IPreview::MIMETYPE_JPEG); diff --git a/lib/private/Preview/PreviewService.php b/lib/private/Preview/PreviewService.php new file mode 100644 index 00000000000..67d6b011416 --- /dev/null +++ b/lib/private/Preview/PreviewService.php @@ -0,0 +1,101 @@ +storageFactory->deletePreview($preview); + $this->previewMapper->delete($preview); + } + + /** + * Get storageId and fileIds for which we have at least one preview. + * + * @return \Generator + */ + public function getAvailableFileIds(): \Generator { + $maxQb = $this->connection->getQueryBuilder(); + $maxQb->select($maxQb->func()->max('id')) + ->from($this->previewMapper->getTableName()) + ->groupBy('file_id'); + + $qb = $this->connection->getQueryBuilder(); + $qb->select('file_id', 'storage_id') + ->from($this->previewMapper->getTableName()) + ->where($qb->expr()->in('id', $qb->createFunction($maxQb->getSQL()))); + + $result = $qb->executeQuery(); + + $lastStorageId = -1; + /** @var int[] $fileIds */ + $fileIds = []; + + // Previews next to each others in the database are likely in the same storage, so group them + while ($row = $result->fetch()) { + if ($lastStorageId !== $row['storage_id']) { + if ($lastStorageId !== -1) { + yield ['storageId' => $lastStorageId, 'fileIds' => $fileIds]; + $fileIds = []; + } + $lastStorageId = (int)$row['storage_id']; + } + $fileIds[] = (int)$row['file_id']; + } + + if (count($fileIds) > 0) { + yield ['storageId' => $lastStorageId, 'fileIds' => $fileIds]; + } + } + + /** + * @return \Generator + */ + public function getAvailablePreviewForFile(int $fileId): \Generator { + yield from $this->previewMapper->getAvailablePreviewForFile($fileId); + } + + public function deleteAll(): void { + $lastId = 0; + while (true) { + $previews = $this->previewMapper->getPreviews($lastId, 1000); + $i = 0; + foreach ($previews as $preview) { + $this->deletePreview($preview); + $i++; + $lastId = $preview->getId(); + } + + if ($i !== 1000) { + break; + } + } + } + + /** + * @param int[] $fileIds + * @return array + */ + public function getAvailablePreviews(array $fileIds): array { + return $this->previewMapper->getAvailablePreviews($fileIds); + } +} diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index 787518e53a4..933fc2850b0 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -1,5 +1,13 @@ rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); } - public function writePreview(Preview $preview, $stream): false|int { + public function writePreview(Preview $preview, mixed $stream): false|int { $previewPath = $this->constructPath($preview); if (!$this->createParentFiles($previewPath)) { return false; @@ -35,11 +35,11 @@ class LocalPreviewStorage implements IPreviewStorage { return file_put_contents($previewPath, $stream); } - public function readPreview(Preview $preview) { + public function readPreview(Preview $preview): mixed { return @fopen($this->constructPath($preview), 'r'); } - public function deletePreview(Preview $preview) { + public function deletePreview(Preview $preview): void { @unlink($this->constructPath($preview)); } diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 7963c870dd8..0a735e5a912 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -15,7 +15,6 @@ use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; -use OCP\Files\NotFoundException; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; @@ -26,7 +25,7 @@ use OCP\IConfig; class ObjectStorePreviewStorage implements IPreviewStorage { /** - * @var array> + * @var array> */ private array $objectStoreCache = []; @@ -40,7 +39,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { $this->isMultibucketPreviewDistributionEnabled = $config->getSystemValueBool('objectstore.multibucket.preview-distribution'); } - public function writePreview(Preview $preview, $stream): false|int { + public function writePreview(Preview $preview, mixed $stream): false|int { if (!is_resource($stream)) { $fh = fopen('php://temp', 'w+'); fwrite($fh, $stream); @@ -64,7 +63,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { return $size; } - public function readPreview(Preview $preview) { + public function readPreview(Preview $preview): mixed { [ 'objectPrefix' => $objectPrefix, 'store' => $store, @@ -72,12 +71,12 @@ class ObjectStorePreviewStorage implements IPreviewStorage { return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); } - public function deletePreview(Preview $preview) { + public function deletePreview(Preview $preview): void { [ 'objectPrefix' => $objectPrefix, 'store' => $store, ] = $this->getObjectStoreForPreview($preview); - return $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); + $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); } public function migratePreview(Preview $preview, SimpleFile $file): void { diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 0755bacd0b9..3051ad5869f 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -1,5 +1,13 @@ getBackend()->writePreview($preview, $stream); } - public function readPreview(Preview $preview) { + public function readPreview(Preview $preview): mixed { return $this->getBackend()->readPreview($preview); } diff --git a/lib/private/Preview/Watcher.php b/lib/private/Preview/Watcher.php index bbbbcb835fa..ea0f72796ae 100644 --- a/lib/private/Preview/Watcher.php +++ b/lib/private/Preview/Watcher.php @@ -41,11 +41,12 @@ class Watcher { return; } - if (is_null($node->getId())) { + $nodeId = $node->getId(); + if (is_null($nodeId)) { return; } - [$node->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$node->getId()]); + [$node->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$nodeId]); foreach ($previews as $preview) { $this->storageFactory->deletePreview($preview); } diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 912d2b3fe5b..fb884092872 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -16,7 +16,6 @@ use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\QueryException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IAppData; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFile; diff --git a/psalm.xml b/psalm.xml index 648b250c38b..abea7c5a443 100644 --- a/psalm.xml +++ b/psalm.xml @@ -92,6 +92,7 @@ + diff --git a/tests/Core/Command/Preview/RepairTest.php b/tests/Core/Command/Preview/RepairTest.php deleted file mode 100644 index 9b9cde6dd95..00000000000 --- a/tests/Core/Command/Preview/RepairTest.php +++ /dev/null @@ -1,153 +0,0 @@ -config = $this->getMockBuilder(IConfig::class) - ->getMock(); - $this->rootFolder = $this->getMockBuilder(IRootFolder::class) - ->getMock(); - $this->logger = $this->getMockBuilder(LoggerInterface::class) - ->getMock(); - $this->iniGetWrapper = $this->getMockBuilder(IniGetWrapper::class) - ->getMock(); - $this->repair = new Repair( - $this->config, - $this->rootFolder, - $this->logger, - $this->iniGetWrapper, - $this->createMock(ILockingProvider::class) - ); - $this->input = $this->createMock(InputInterface::class); - $this->input->expects($this->any()) - ->method('getOption') - ->willReturnCallback(function ($parameter) { - if ($parameter === 'batch') { - return true; - } - return null; - }); - $this->output = $this->getMockBuilder(ConsoleOutput::class) - ->onlyMethods(['section', 'writeln', 'getFormatter']) - ->getMock(); - $self = $this; - - /* We need format method to return a string */ - $outputFormatter = $this->createMock(OutputFormatterInterface::class); - $outputFormatter->method('isDecorated')->willReturn(false); - $outputFormatter->method('format')->willReturnArgument(0); - - $this->output->expects($this->any()) - ->method('getFormatter') - ->willReturn($outputFormatter); - $this->output->expects($this->any()) - ->method('writeln') - ->willReturnCallback(function ($line) use ($self): void { - $self->outputLines .= $line . "\n"; - }); - } - - public static function dataEmptyTest(): array { - /** directoryNames, expectedOutput */ - return [ - [ - [], - 'All previews are already migrated.' - ], - [ - [['name' => 'a'], ['name' => 'b'], ['name' => 'c']], - 'All previews are already migrated.' - ], - [ - [['name' => '0', 'content' => ['folder', 'folder']], ['name' => 'b'], ['name' => 'c']], - 'All previews are already migrated.' - ], - [ - [['name' => '0', 'content' => ['file', 'folder', 'folder']], ['name' => 'b'], ['name' => 'c']], - 'A total of 1 preview files need to be migrated.' - ], - [ - [['name' => '23'], ['name' => 'b'], ['name' => 'c']], - 'A total of 1 preview files need to be migrated.' - ], - ]; - } - - #[\PHPUnit\Framework\Attributes\DataProvider('dataEmptyTest')] - public function testEmptyExecute($directoryNames, $expectedOutput): void { - $previewFolder = $this->getMockBuilder(Folder::class) - ->getMock(); - $directories = array_map(function ($element) { - $dir = $this->getMockBuilder(Folder::class) - ->getMock(); - $dir->expects($this->any()) - ->method('getName') - ->willReturn($element['name']); - if (isset($element['content'])) { - $list = []; - foreach ($element['content'] as $item) { - if ($item === 'file') { - $list[] = $this->getMockBuilder(Node::class) - ->getMock(); - } elseif ($item === 'folder') { - $list[] = $this->getMockBuilder(Folder::class) - ->getMock(); - } - } - $dir->expects($this->once()) - ->method('getDirectoryListing') - ->willReturn($list); - } - return $dir; - }, $directoryNames); - $previewFolder->expects($this->once()) - ->method('getDirectoryListing') - ->willReturn($directories); - $this->rootFolder->expects($this->once()) - ->method('get') - ->with('appdata_/preview') - ->willReturn($previewFolder); - - $this->repair->run($this->input, $this->output); - - $this->assertStringContainsString($expectedOutput, $this->outputLines); - } -} diff --git a/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php b/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php index c13b170ca18..07e58d56f11 100644 --- a/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php +++ b/tests/lib/Files/ObjectStore/PrimaryObjectStoreConfigTest.php @@ -212,6 +212,7 @@ class PrimaryObjectStoreConfigTest extends TestCase { $this->assertEquals([ 'default' => 'server1', 'root' => 'server1', + 'preview' => 'server1', 'server1' => [ 'class' => StorageObjectStore::class, 'arguments' => [ @@ -235,6 +236,7 @@ class PrimaryObjectStoreConfigTest extends TestCase { $this->assertEquals([ 'default' => 'server1', 'root' => 'server1', + 'preview' => 'server1', 'server1' => [ 'class' => StorageObjectStore::class, 'arguments' => [ @@ -270,6 +272,7 @@ class PrimaryObjectStoreConfigTest extends TestCase { $this->setConfig('objectstore', [ 'default' => 'server1', 'root' => 'server2', + 'preview' => 'server1', 'server1' => [ 'class' => StorageObjectStore::class, 'arguments' => [ diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index ea08b58955d..a2c72cbad57 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -9,19 +9,13 @@ namespace Test\Preview; use OC\Files\Storage\Temporary; use OC\Preview\BackgroundCleanupJob; -use OC\Preview\Db\Preview; -use OC\Preview\Db\PreviewMapper; -use OC\Preview\Storage\Root; -use OC\Preview\Storage\StorageFactory; +use OC\Preview\PreviewService; use OC\PreviewManager; -use OC\SystemConfig; use OCP\App\IAppManager; use OCP\AppFramework\Utility\ITimeFactory; -use OCP\Files\AppData\IAppDataFactory; use OCP\Files\File; use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; -use OCP\Files\NotFoundException; use OCP\IDBConnection; use OCP\IPreview; use OCP\Server; @@ -45,8 +39,7 @@ class BackgroundCleanupJobTest extends \Test\TestCase { private IRootFolder $rootFolder; private IMimeTypeLoader $mimeTypeLoader; private ITimeFactory $timeFactory; - private PreviewMapper $previewMapper; - private StorageFactory $previewStorageFactory; + private PreviewService $previewService; protected function setUp(): void { parent::setUp(); @@ -70,8 +63,7 @@ class BackgroundCleanupJobTest extends \Test\TestCase { $this->rootFolder = Server::get(IRootFolder::class); $this->mimeTypeLoader = Server::get(IMimeTypeLoader::class); $this->timeFactory = Server::get(ITimeFactory::class); - $this->previewMapper = Server::get(PreviewMapper::class); - $this->previewStorageFactory = Server::get(StorageFactory::class); + $this->previewService = Server::get(PreviewService::class); } protected function tearDown(): void { @@ -82,9 +74,8 @@ class BackgroundCleanupJobTest extends \Test\TestCase { $this->logout(); - foreach ($this->previewMapper->getAvailablePreviews(5) as $preview) { - $this->previewStorageFactory->deletePreview($preview); - $this->previewMapper->delete($preview); + foreach ($this->previewService->getAvailablePreviewForFile(5) as $preview) { + $this->previewService->deletePreview($preview); } parent::tearDown(); @@ -104,8 +95,8 @@ class BackgroundCleanupJobTest extends \Test\TestCase { return $files; } - private function countPreviews(PreviewMapper $previewMapper, array $fileIds): int { - $previews = $previewMapper->getAvailablePreviews($fileIds); + private function countPreviews(PreviewService $previewService, array $fileIds): int { + $previews = $previewService->getAvailablePreviews($fileIds); return array_reduce($previews, fn (int $result, array $previews) => $result + count($previews), 0); } @@ -113,18 +104,18 @@ class BackgroundCleanupJobTest extends \Test\TestCase { $files = $this->setup11Previews(); $fileIds = array_map(fn (File $f): int => $f->getId(), $files); - $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewMapper, $this->previewStorageFactory, true); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); + $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewService, true); $job->run([]); foreach ($files as $file) { $file->delete(); } - $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); $job->run([]); - $this->assertSame(0, $this->countPreviews($this->previewMapper, $fileIds)); + $this->assertSame(0, $this->countPreviews($this->previewService, $fileIds)); } public function testCleanupAjax(): void { @@ -134,20 +125,20 @@ class BackgroundCleanupJobTest extends \Test\TestCase { $files = $this->setup11Previews(); $fileIds = array_map(fn (File $f): int => $f->getId(), $files); - $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); - $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewMapper, $this->previewStorageFactory, false); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); + $job = new BackgroundCleanupJob($this->timeFactory, $this->connection, $this->previewService, false); $job->run([]); foreach ($files as $file) { $file->delete(); } - $this->assertSame(11, $this->countPreviews($this->previewMapper, $fileIds)); + $this->assertSame(11, $this->countPreviews($this->previewService, $fileIds)); $job->run([]); - $this->assertSame(1, $this->countPreviews($this->previewMapper, $fileIds)); + $this->assertSame(1, $this->countPreviews($this->previewService, $fileIds)); $job->run([]); - $this->assertSame(0, $this->countPreviews($this->previewMapper, $fileIds)); + $this->assertSame(0, $this->countPreviews($this->previewService, $fileIds)); } } diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index 91fa3537468..54d28747cf3 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -16,8 +16,6 @@ use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; -use OCP\Files\SimpleFS\ISimpleFile; -use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IConfig; use OCP\IImage; use OCP\IPreview; @@ -84,19 +82,21 @@ class GeneratorTest extends TestCase { $maxPreview = new Preview(); $maxPreview->setWidth(1000); $maxPreview->setHeight(1000); - $maxPreview->setIsMax(true); + $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); - $maxPreview->setCrop(false); + $maxPreview->setCropped(false); + $maxPreview->setStorageId(1); $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); $previewFile = new Preview(); $previewFile->setWidth(256); $previewFile->setHeight(256); - $previewFile->setIsMax(false); + $previewFile->setMax(false); $previewFile->setSize(1000); $previewFile->setVersion(-1); - $previewFile->setCrop(false); + $previewFile->setCropped(false); + $previewFile->setStorageId(1); $previewFile->setMimetype(IPreview::MIMETYPE_PNG); $this->previewMapper->method('getAvailablePreviews') @@ -190,7 +190,7 @@ class GeneratorTest extends TestCase { $maxPreview = new Preview(); $maxPreview->setWidth(2048); $maxPreview->setHeight(2048); - $maxPreview->setIsMax(true); + $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); @@ -210,7 +210,7 @@ class GeneratorTest extends TestCase { $this->assertSame('my resized data', $data); return 1000; } - $this->fail("file name is wrong:". $preview->getName()); + $this->fail('file name is wrong:' . $preview->getName()); }); $image = $this->getMockImage(2048, 2048, 'my resized data'); @@ -238,7 +238,7 @@ class GeneratorTest extends TestCase { $maxPreview = new Preview(); $maxPreview->setWidth(2048); $maxPreview->setHeight(2048); - $maxPreview->setIsMax(true); + $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); @@ -262,7 +262,7 @@ class GeneratorTest extends TestCase { $maxPreview = new Preview(); $maxPreview->setWidth(2048); $maxPreview->setHeight(2048); - $maxPreview->setIsMax(true); + $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); @@ -270,9 +270,9 @@ class GeneratorTest extends TestCase { $previewFile = new Preview(); $previewFile->setWidth(1024); $previewFile->setHeight(512); - $previewFile->setIsMax(false); + $previewFile->setMax(false); $previewFile->setSize(1000); - $previewFile->setCrop(true); + $previewFile->setCropped(true); $previewFile->setVersion(-1); $previewFile->setMimetype(IPreview::MIMETYPE_PNG); @@ -380,7 +380,7 @@ class GeneratorTest extends TestCase { $maxPreview = new Preview(); $maxPreview->setWidth($maxX); $maxPreview->setHeight($maxY); - $maxPreview->setIsMax(true); + $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php index a226b611c5a..cd9f3253654 100644 --- a/tests/lib/Preview/MovePreviewJobTest.php +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -1,17 +1,26 @@ appConfig->expects($this->any()) ->method('getValueBool') ->willReturn(false); + $this->appConfig->expects($this->any()) + ->method('setValueBool') + ->willReturn(true); $this->storageFactory = Server::get(StorageFactory::class); + $this->previewService = Server::get(PreviewService::class); + $this->db = Server::get(IDBConnection::class); + + $qb = $this->db->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter(5))) + ->executeStatement(); + + $qb = $this->db->getQueryBuilder(); + $qb->insert('filecache') + ->values([ + 'fileid' => $qb->createNamedParameter(5), + 'storage' => $qb->createNamedParameter(1), + 'path' => $qb->createNamedParameter('test/abc'), + 'path_hash' => $qb->createNamedParameter(md5('test')), + 'parent' => $qb->createNamedParameter(0), + 'name' => $qb->createNamedParameter('abc'), + 'mimetype' => $qb->createNamedParameter(0), + 'size' => $qb->createNamedParameter(1000), + 'mtime' => $qb->createNamedParameter(1000), + 'storage_mtime' => $qb->createNamedParameter(1000), + 'encrypted' => $qb->createNamedParameter(0), + 'unencrypted_size' => $qb->createNamedParameter(0), + 'etag' => $qb->createNamedParameter('abcdefg'), + 'permissions' => $qb->createNamedParameter(0), + 'checksum' => $qb->createNamedParameter('abcdefg'), + ])->executeStatement(); } public function tearDown(): void { - foreach ($this->previewMapper->getAvailablePreviewForFile(5) as $preview) { - $this->storageFactory->deletePreview($preview); - $this->previewMapper->delete($preview); - } - foreach ($this->previewAppData->getDirectoryListing() as $folder) { $folder->delete(); } + $this->previewService->deleteAll(); + + $qb = $this->db->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter(5))) + ->executeStatement(); } - #[TestDox("Test the migration from the legacy flat hierarchy to the new database format")] - function testMigrationLegacyPath(): void { - $folder = $this->previewAppData->newFolder(5); + #[TestDox('Test the migration from the legacy flat hierarchy to the new database format')] + public function testMigrationLegacyPath(): void { + $folder = $this->previewAppData->newFolder('5'); $folder->newFile('64-64-crop.jpg', 'abcdefg'); $folder->newFile('128-128-crop.png', 'abcdefg'); $this->assertEquals(1, count($this->previewAppData->getDirectoryListing())); @@ -63,6 +104,7 @@ class MovePreviewJobTest extends TestCase { $this->previewMapper, $this->storageFactory, Server::get(IDBConnection::class), + Server::get(IRootFolder::class), Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); @@ -75,12 +117,12 @@ class MovePreviewJobTest extends TestCase { } #[TestDox("Test the migration from the 'new' nested hierarchy to the database format")] - function testMigrationPath(): void { - $folder = $this->previewAppData->newFolder(self::getInternalFolder(5)); + public function testMigrationPath(): void { + $folder = $this->previewAppData->newFolder(self::getInternalFolder((string)5)); $folder->newFile('64-64-crop.jpg', 'abcdefg'); $folder->newFile('128-128-crop.png', 'abcdefg'); - $folder = $this->previewAppData->getFolder(self::getInternalFolder(5)); + $folder = $this->previewAppData->getFolder(self::getInternalFolder((string)5)); $this->assertEquals(2, count($folder->getDirectoryListing())); $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); @@ -90,10 +132,65 @@ class MovePreviewJobTest extends TestCase { $this->previewMapper, $this->storageFactory, Server::get(IDBConnection::class), + Server::get(IRootFolder::class), Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); } + + #[TestDox("Test the migration from the 'new' nested hierarchy to the database format")] + public function testMigrationPathWithVersion(): void { + $folder = $this->previewAppData->newFolder(self::getInternalFolder((string)5)); + // No version + $folder->newFile('128-128-crop.png', 'abcdefg'); + $folder->newFile('256-256-max.png', 'abcdefg'); + $folder->newFile('128-128.png', 'abcdefg'); + + // Version 1000 + $folder->newFile('1000-128-128-crop.png', 'abcdefg'); + $folder->newFile('1000-256-256-max.png', 'abcdefg'); + $folder->newFile('1000-128-128.png', 'abcdefg'); + + // Version 1001 + $folder->newFile('1001-128-128-crop.png', 'abcdefg'); + $folder->newFile('1001-256-256-max.png', 'abcdefg'); + $folder->newFile('1001-128-128.png', 'abcdefg'); + + $folder = $this->previewAppData->getFolder(self::getInternalFolder((string)5)); + $this->assertEquals(9, count($folder->getDirectoryListing())); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + + $job = new MovePreviewJob( + Server::get(ITimeFactory::class), + $this->appConfig, + $this->previewMapper, + $this->storageFactory, + Server::get(IDBConnection::class), + Server::get(IRootFolder::class), + Server::get(IAppDataFactory::class) + ); + $this->invokePrivate($job, 'run', [[]]); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); + $previews = iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)); + $this->assertEquals(9, count($previews)); + + $nameVersionMapping = []; + foreach ($previews as $preview) { + $nameVersionMapping[$preview->getName()] = $preview->getVersion(); + } + + $this->assertEquals([ + '1000-128-128.png' => 1000, + '1000-128-128-crop.png' => 1000, + '1000-256-256-max.png' => 1000, + '1001-128-128.png' => 1001, + '1001-128-128-crop.png' => 1001, + '1001-256-256-max.png' => 1001, + '128-128.png' => -1, + '128-128-crop.png' => -1, + '256-256-max.png' => -1, + ], $nameVersionMapping); + } } diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index d6925641d4e..6f59c6f8802 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -29,7 +29,7 @@ class PreviewMapperTest extends TestCase { $this->connection = Server::get(IDBConnection::class); } - public function testGetAvailablePreviews() { + public function testGetAvailablePreviews(): void { // Empty $this->assertEquals([], $this->previewMapper->getAvailablePreviews([])); @@ -50,7 +50,8 @@ class PreviewMapperTest extends TestCase { $this->assertEquals('default', $previews[43][0]->getObjectStoreName()); } - private function createPreviewForFileId(int $fileId, ?int $bucket = null) { + private function createPreviewForFileId(int $fileId, ?int $bucket = null): void { + $locationId = null; if ($bucket) { $qb = $this->connection->getQueryBuilder(); $qb->insert('preview_locations') @@ -58,20 +59,22 @@ class PreviewMapperTest extends TestCase { 'bucket_name' => $qb->createNamedParameter('preview-' . $bucket), 'object_store_name' => $qb->createNamedParameter('default'), ]); - $locationId = $qb->executeStatement(); + $qb->executeStatement(); + $locationId = $qb->getLastInsertId(); } $preview = new Preview(); $preview->setFileId($fileId); - $preview->setCrop(true); - $preview->setIsMax(true); + $preview->setStorageId(1); + $preview->setCropped(true); + $preview->setMax(true); $preview->setWidth(100); $preview->setHeight(100); $preview->setSize(100); $preview->setMtime(time()); $preview->setMimetype(IPreview::MIMETYPE_PNG); - $preview->setEtag("abcdefg"); + $preview->setEtag('abcdefg'); - if ($locationId) { + if ($locationId !== null) { $preview->setLocationId($locationId); } $this->previewMapper->insert($preview); diff --git a/tests/lib/Preview/PreviewServiceTest.php b/tests/lib/Preview/PreviewServiceTest.php new file mode 100644 index 00000000000..01bcff29a5a --- /dev/null +++ b/tests/lib/Preview/PreviewServiceTest.php @@ -0,0 +1,59 @@ +previewService = Server::get(PreviewService::class); + $this->previewMapper = Server::get(PreviewMapper::class); + $this->previewService->deleteAll(); + } + + public function tearDown(): void { + $this->previewService->deleteAll(); + } + + public function testGetAvailableFileIds(): void { + foreach (range(1, 20) as $i) { + $preview = new Preview(); + $preview->setFileId($i % 10); + $preview->setStorageId(1); + $preview->setWidth($i); + $preview->setHeight($i); + $preview->setMax(true); + $preview->setCropped(true); + $preview->setEncrypted(false); + $preview->setMimetype(IPreview::MIMETYPE_JPEG); + $preview->setEtag('abc'); + $preview->setMtime((new \DateTime())->getTimestamp()); + $preview->setSize(0); + $this->previewMapper->insert($preview); + } + + $files = iterator_to_array($this->previewService->getAvailableFileIds()); + $this->assertCount(1, $files); + $this->assertCount(10, $files[0]['fileIds']); + } +} From bfc7d5dd9fac04124db1b65e9f22d9f33a5e5d12 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Thu, 25 Sep 2025 14:25:47 +0200 Subject: [PATCH 09/14] feat(preview): Implement scanning for previews This work similarly to the move preview job to migrate the previews to the new DB table and also reuse some code. So when we are finding files in appdata/preview, try adding them to the oc_previews table and delete them from the oc_filecache table. Signed-off-by: Carl Schwan --- apps/files/lib/Command/ScanAppData.php | 39 +++-- build/psalm-baseline.xml | 4 +- core/BackgroundJobs/MovePreviewJob.php | 62 ++------ lib/private/Files/Cache/Scanner.php | 9 +- lib/private/Preview/Db/Preview.php | 35 +++++ .../Preview/Storage/IPreviewStorage.php | 2 + .../Preview/Storage/LocalPreviewStorage.php | 135 +++++++++++++++++- .../Storage/ObjectStorePreviewStorage.php | 4 + .../Preview/Storage/StorageFactory.php | 7 +- lib/public/Preview/IVersionedPreviewFile.php | 6 +- 10 files changed, 232 insertions(+), 71 deletions(-) diff --git a/apps/files/lib/Command/ScanAppData.php b/apps/files/lib/Command/ScanAppData.php index 385e0624b3a..23604a82df9 100644 --- a/apps/files/lib/Command/ScanAppData.php +++ b/apps/files/lib/Command/ScanAppData.php @@ -12,6 +12,7 @@ use OC\DB\Connection; use OC\DB\ConnectionAdapter; use OC\Files\Utils\Scanner; use OC\ForbiddenException; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\Folder; use OCP\Files\IRootFolder; @@ -32,10 +33,12 @@ class ScanAppData extends Base { protected int $foldersCounter = 0; protected int $filesCounter = 0; + protected int $previewsCounter = -1; public function __construct( protected IRootFolder $rootFolder, protected IConfig $config, + private StorageFactory $previewStorage, ) { parent::__construct(); } @@ -51,9 +54,12 @@ class ScanAppData extends Base { } protected function scanFiles(OutputInterface $output, string $folder): int { - if ($folder === 'preview') { - $output->writeln('Scanning the preview folder is not supported.'); - return self::FAILURE; + if ($folder === 'preview' || $folder === '') { + $this->previewsCounter = $this->previewStorage->scan(); + + if ($folder === 'preview') { + return self::SUCCESS; + } } try { @@ -139,7 +145,7 @@ class ScanAppData extends Base { $this->initTools(); $exitCode = $this->scanFiles($output, $folder); - if ($exitCode === 0) { + if ($exitCode === self::SUCCESS) { $this->presentStats($output); } return $exitCode; @@ -167,7 +173,7 @@ class ScanAppData extends Base { * * @throws \ErrorException */ - public function exceptionErrorHandler($severity, $message, $file, $line) { + public function exceptionErrorHandler(int $severity, string $message, string $file, int $line): void { if (!(error_reporting() & $severity)) { // This error code is not included in error_reporting return; @@ -178,10 +184,12 @@ class ScanAppData extends Base { protected function presentStats(OutputInterface $output): void { // Stop the timer $this->execTime += microtime(true); - - $headers = [ - 'Folders', 'Files', 'Elapsed time' - ]; + if ($this->previewsCounter !== -1) { + $headers[] = 'Previews'; + } + $headers[] = 'Folders'; + $headers[] = 'Files'; + $headers[] = 'Elapsed time'; $this->showSummary($headers, null, $output); } @@ -192,14 +200,15 @@ class ScanAppData extends Base { * @param string[] $headers * @param string[] $rows */ - protected function showSummary($headers, $rows, OutputInterface $output): void { + protected function showSummary(array $headers, ?array $rows, OutputInterface $output): void { $niceDate = $this->formatExecTime(); if (!$rows) { - $rows = [ - $this->foldersCounter, - $this->filesCounter, - $niceDate, - ]; + if ($this->previewsCounter !== -1) { + $rows[] = $this->previewsCounter; + } + $rows[] = $this->foldersCounter; + $rows[] = $this->filesCounter; + $rows[] = $niceDate; } $table = new Table($output); $table diff --git a/build/psalm-baseline.xml b/build/psalm-baseline.xml index 21baeb9a33b..8be3f1e9c1d 100644 --- a/build/psalm-baseline.xml +++ b/build/psalm-baseline.xml @@ -1240,9 +1240,11 @@ + + + - diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 22db7aea90d..764561a1ba8 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -82,13 +82,18 @@ class MovePreviewJob extends TimedJob { ->setMaxResults(100); $result = $qb->executeQuery(); + $foundOldPreview = false; while ($row = $result->fetch()) { $pathSplit = explode('/', $row['path']); assert(count($pathSplit) >= 2); $fileId = $pathSplit[count($pathSplit) - 2]; array_pop($pathSplit); - $path = implode('/', $pathSplit); $this->processPreviews($fileId, true); + $foundOldPreview = true; + } + + if (!$foundOldPreview) { + break; } // Stop if execution time is more than one hour. @@ -114,44 +119,21 @@ class MovePreviewJob extends TimedJob { $folder = $this->appData->getFolder($internalPath); /** - * @var list $previewFiles + * @var list $previewFiles */ $previewFiles = []; foreach ($folder->getDirectoryListing() as $previewFile) { /** @var SimpleFile $previewFile */ - [0 => $baseName, 1 => $extension] = explode('.', $previewFile->getName()); - $nameSplit = explode('-', $baseName); - - $offset = 0; - $version = null; - if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { - $offset = 1; - $version = (int)$nameSplit[0]; - } - - $width = (int)$nameSplit[$offset + 0]; - $height = (int)$nameSplit[$offset + 1]; - - $crop = false; - $max = false; - if (isset($nameSplit[$offset + 2])) { - $crop = $nameSplit[$offset + 2] === 'crop'; - $max = $nameSplit[$offset + 2] === 'max'; - } + $preview = Preview::fromPath($fileId . '/' . $previewFile->getName()); + $preview->setSize($previewFile->getSize()); + $preview->setMtime($previewFile->getMtime()); + $preview->setOldFileId($previewFile->getId()); + $preview->setEncrypted(false); $previewFiles[] = [ 'file' => $previewFile, - 'width' => $width, - 'height' => $height, - 'crop' => $crop, - 'version' => $version, - 'max' => $max, - 'extension' => $extension, - 'size' => $previewFile->getSize(), - 'mtime' => $previewFile->getMTime(), + 'preview' => $preview, ]; } @@ -166,27 +148,11 @@ class MovePreviewJob extends TimedJob { if (count($result) > 0) { foreach ($previewFiles as $previewFile) { - $preview = new Preview(); - $preview->setFileId((int)$fileId); + $preview = $previewFile['preview']; /** @var SimpleFile $file */ $file = $previewFile['file']; - $preview->setOldFileId($file->getId()); $preview->setStorageId($result[0]['storage']); $preview->setEtag($result[0]['etag']); - $preview->setMtime($previewFile['mtime']); - $preview->setWidth($previewFile['width']); - $preview->setHeight($previewFile['height']); - $preview->setCropped($previewFile['crop']); - $preview->setVersion($previewFile['version']); - $preview->setMax($previewFile['max']); - $preview->setEncrypted(false); - $preview->setMimetype(match ($previewFile['extension']) { - 'png' => IPreview::MIMETYPE_PNG, - 'webp' => IPreview::MIMETYPE_WEBP, - 'gif' => IPreview::MIMETYPE_GIF, - default => IPreview::MIMETYPE_JPEG, - }); - $preview->setSize($previewFile['size']); try { $preview = $this->previewMapper->insert($preview); } catch (Exception $e) { diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index b067f70b8cb..75ca9da0abe 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -65,6 +65,8 @@ class Scanner extends BasicEmitter implements IScanner { protected IDBConnection $connection; + private string $previewFolder; + public function __construct(\OC\Files\Storage\Storage $storage) { $this->storage = $storage; $this->storageId = $this->storage->getId(); @@ -75,6 +77,7 @@ class Scanner extends BasicEmitter implements IScanner { $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); $this->lockingProvider = \OC::$server->get(ILockingProvider::class); $this->connection = \OC::$server->get(IDBConnection::class); + $this->previewFolder = 'appdata_' . $config->getValue('instanceid', '') . '/preview'; } /** @@ -318,7 +321,6 @@ class Scanner extends BasicEmitter implements IScanner { try { $data = $this->scanFile($path, $reuse, -1, lock: $lock); - if ($data !== null && $data['mimetype'] === 'httpd/unix-directory') { $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data['size']); $data['size'] = $size; @@ -413,6 +415,11 @@ class Scanner extends BasicEmitter implements IScanner { $size = 0; $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); + if (str_starts_with($path, $this->previewFolder)) { + // Preview scanning is handled in LocalPreviewStorage + return 0; + } + foreach ($childQueue as $child => [$childId, $childSize]) { // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed $childEtagChanged = false; diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index b6f2ef2942e..3f10328fd8e 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -86,6 +86,41 @@ class Preview extends Entity { $this->addType('version', Types::BIGINT); } + public static function fromPath(string $path): Preview { + $preview = new self(); + $preview->setFileId((int)basename(dirname($path))); + + $fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION); + + [0 => $baseName, 1 => $extension] = explode('.', $fileName); + $preview->setMimetype(match ($extension) { + 'jpg' | 'jpeg' => IPreview::MIMETYPE_JPEG, + 'png' => IPreview::MIMETYPE_PNG, + 'gif' => IPreview::MIMETYPE_GIF, + 'webp' => IPreview::MIMETYPE_WEBP, + default => IPreview::MIMETYPE_JPEG, + }); + $nameSplit = explode('-', $baseName); + + $offset = 0; + $preview->setVersion(null); + if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { + $offset = 1; + $preview->setVersion((int)$nameSplit[0]); + } + + $preview->setWidth((int)$nameSplit[$offset + 0]); + $preview->setHeight((int)$nameSplit[$offset + 1]); + + $preview->setCropped(false); + $preview->setMax(false); + if (isset($nameSplit[$offset + 2])) { + $preview->setCropped($nameSplit[$offset + 2] === 'crop'); + $preview->setMax($nameSplit[$offset + 2] === 'max'); + } + return $preview; + } + public function getName(): string { $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); if ($this->isCropped()) { diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index 933fc2850b0..56464326838 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -36,4 +36,6 @@ interface IPreviewStorage { * @throw \Exception */ public function migratePreview(Preview $preview, SimpleFile $file): void; + + public function scan(): int; } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 59a4248e92b..3bd1fe2ccb9 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -14,7 +14,13 @@ use LogicException; use OC; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; +use OCP\DB\Exception; +use OCP\IAppConfig; use OCP\IConfig; +use OCP\IDBConnection; +use RecursiveDirectoryIterator; +use RecursiveIteratorIterator; class LocalPreviewStorage implements IPreviewStorage { private readonly string $rootFolder; @@ -22,6 +28,10 @@ class LocalPreviewStorage implements IPreviewStorage { public function __construct( private readonly IConfig $config, + private readonly PreviewMapper $previewMapper, + private readonly StorageFactory $previewStorage, + private readonly IAppConfig $appConfig, + private readonly IDBConnection $connection, ) { $this->instanceId = $this->config->getSystemValueString('instanceid'); $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); @@ -52,8 +62,9 @@ class LocalPreviewStorage implements IPreviewStorage { } private function createParentFiles(string $path): bool { - ['dirname' => $dirname] = pathinfo($path); - return mkdir($dirname, recursive: true); + $dirname = dirname($path); + @mkdir($dirname, recursive: true); + return is_dir($dirname); } public function migratePreview(Preview $preview, SimpleFile $file): void { @@ -75,4 +86,124 @@ class LocalPreviewStorage implements IPreviewStorage { throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); } } + + public function scan(): int { + $checkForFileCache = !$this->appConfig->getValueBool('core', 'previewMovedDone'); + + $scanner = new RecursiveDirectoryIterator($this->getPreviewRootFolder()); + $previewsFound = 0; + foreach (new RecursiveIteratorIterator($scanner) as $file) { + if ($file->isFile()) { + $preview = Preview::fromPath((string)$file); + try { + $preview->setSize($file->getSize()); + $preview->setMtime($file->getMtime()); + $preview->setEncrypted(false); + + $qb = $this->connection->getQueryBuilder(); + $result = $qb->select('*') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($preview->getFileId()))) + ->setMaxResults(1) + ->runAcrossAllShards() // Unavoidable because we can't extract the storage_id from the preview name + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + // original file is deleted + @unlink($file->getRealPath()); + continue; + } + + if ($checkForFileCache) { + $relativePath = str_replace($this->rootFolder . '/', '', $file->getRealPath()); + $rowAffected = $qb->delete('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($relativePath)))) + ->executeStatement(); + if ($rowAffected > 0) { + $this->deleteParentsFromFileCache(dirname($relativePath)); + } + } + + $preview->setStorageId($result[0]['storage']); + $preview->setEtag($result[0]['etag']); + + // try to insert, if that fails the preview is already in the DB + $this->previewMapper->insert($preview); + + // Move old flat preview to new format + $this->previewStorage->migratePreview($preview, $file); + } catch (Exception $e) { + if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + throw $e; + } + } + $previewsFound++; + } + } + + return $previewsFound; + } + + private function deleteParentsFromFileCache(string $dirname): void { + $qb = $this->connection->getQueryBuilder(); + + $result = $qb->select('fileid', 'path', 'storage', 'parent') + ->from('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($dirname)))) + ->setMaxResults(1) + ->runAcrossAllShards() + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + return; + } + + $this->connection->beginTransaction(); + + $parentId = $result[0]['parent']; + $fileId = $result[0]['fileid']; + $storage = $result[0]['storage']; + + try { + while (true) { + $qb = $this->connection->getQueryBuilder(); + $childs = $qb->select('fileid', 'path', 'storage') + ->from('filecache') + ->where($qb->expr()->eq('parent', $qb->createNamedParameter($fileId))) + ->hintShardKey('storage', $storage) + ->executeQuery() + ->fetchAll(); + + if (!empty($childs)) { + break; + } + + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))) + ->hintShardKey('storage', $result[0]['storage']) + ->executeStatement(); + + $qb = $this->connection->getQueryBuilder(); + $result = $qb->select('fileid', 'path', 'storage', 'parent') + ->from('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($parentId))) + ->setMaxResults(1) + ->hintShardKey('storage', $storage) + ->executeQuery() + ->fetchAll(); + + if (empty($result)) { + break; + } + + $fileId = $parentId; + $parentId = $result[0]['parent']; + } + } finally { + $this->connection->commit(); + } + } } diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 0a735e5a912..faa63763779 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -149,4 +149,8 @@ class ObjectStorePreviewStorage implements IPreviewStorage { return 'uri:oid:preview:'; } } + + public function scan(): int { + return 0; + } } diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 3051ad5869f..3438534d07e 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -15,6 +15,7 @@ use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\IConfig; +use OCP\Server; class StorageFactory implements IPreviewStorage { private ?IPreviewStorage $backend = null; @@ -46,7 +47,7 @@ class StorageFactory implements IPreviewStorage { if ($this->objectStoreConfig->hasObjectStore()) { $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config, $this->previewMapper); } else { - $this->backend = new LocalPreviewStorage($this->config); + $this->backend = Server::get(LocalPreviewStorage::class); } return $this->backend; @@ -55,4 +56,8 @@ class StorageFactory implements IPreviewStorage { public function migratePreview(Preview $preview, SimpleFile $file): void { $this->getBackend()->migratePreview($preview, $file); } + + public function scan(): int { + return $this->getBackend()->scan(); + } } diff --git a/lib/public/Preview/IVersionedPreviewFile.php b/lib/public/Preview/IVersionedPreviewFile.php index 9266b1ac067..7d68fe8d15e 100644 --- a/lib/public/Preview/IVersionedPreviewFile.php +++ b/lib/public/Preview/IVersionedPreviewFile.php @@ -11,15 +11,15 @@ namespace OCP\Preview; * Marks files that should keep multiple preview "versions" for the same file id * * Examples of this are files where the storage backend provides versioning, for those - * files, we dont have fileids for the different versions but still need to be able to generate + * files, we don't have fileIds for the different versions but still need to be able to generate * previews for all versions * * @since 17.0.0 */ interface IVersionedPreviewFile { /** - * @return string + * @return numeric * @since 17.0.0 */ - public function getPreviewVersion(): string; + public function getPreviewVersion(); } From 58023782b69637221272718699eb2519c530279b Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Fri, 26 Sep 2025 12:26:43 +0200 Subject: [PATCH 10/14] feat(preview): Store original file mimetype in preview table Allow to quickly query all the files from a specific mimetype like in the ResetRenderedTexts command. Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 4 +- core/Command/Preview/ResetRenderedTexts.php | 37 +++++-------------- lib/private/Preview/Db/Preview.php | 5 +++ lib/private/Preview/Db/PreviewMapper.php | 15 ++++++++ lib/private/Preview/Generator.php | 3 ++ lib/private/Preview/PreviewService.php | 10 ++++- .../Preview/Storage/LocalPreviewStorage.php | 3 +- lib/private/PreviewManager.php | 2 + tests/lib/Preview/GeneratorTest.php | 6 ++- tests/lib/Preview/PreviewMapperTest.php | 1 + tests/lib/Preview/PreviewServiceTest.php | 1 + 11 files changed, 55 insertions(+), 32 deletions(-) diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 764561a1ba8..82acea82c7a 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -24,7 +24,6 @@ use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IAppConfig; use OCP\IDBConnection; -use OCP\IPreview; class MovePreviewJob extends TimedJob { private IAppData $appData; @@ -138,7 +137,7 @@ class MovePreviewJob extends TimedJob { } $qb = $this->connection->getQueryBuilder(); - $qb->select('*') + $qb->select('storage', 'etag', 'mimetype') ->from('filecache') ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($fileId))) ->setMaxResults(1); @@ -153,6 +152,7 @@ class MovePreviewJob extends TimedJob { $file = $previewFile['file']; $preview->setStorageId($result[0]['storage']); $preview->setEtag($result[0]['etag']); + $preview->setSourceMimetype($result[0]['mimetype']); try { $preview = $this->previewMapper->insert($preview); } catch (Exception $e) { diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index fb7fb9fa620..b80f0f6ffce 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -67,9 +67,7 @@ class ResetRenderedTexts extends Command { try { $avatar->remove(); - } catch (NotFoundException $e) { - // continue - } catch (NotPermittedException $e) { + } catch (NotFoundException|NotPermittedException) { // continue } } @@ -91,8 +89,8 @@ class ResetRenderedTexts extends Command { private function deletePreviews(OutputInterface $output, bool $dryMode): void { $previewsToDeleteCount = 0; - foreach ($this->getPreviewsToDelete() as ['path' => $filePath, 'preview' => $preview]) { - $output->writeln('Deleting previews for ' . $filePath, OutputInterface::VERBOSITY_VERBOSE); + foreach ($this->getPreviewsToDelete() as $preview) { + $output->writeln('Deleting preview ' . $preview->getName() . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); $previewsToDeleteCount++; @@ -107,28 +105,13 @@ class ResetRenderedTexts extends Command { } /** - * @return \Iterator + * @return \Generator */ - private function getPreviewsToDelete(): \Iterator { - $qb = $this->connection->getQueryBuilder(); - $qb->select('fileid', 'path') - ->from('filecache') - ->where( - $qb->expr()->orX( - $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))), - $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))), - $qb->expr()->eq('mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown'))) - ) - ); - - $cursor = $qb->executeQuery(); - - while ($row = $cursor->fetch()) { - foreach ($this->previewService->getAvailablePreviewForFile($row['fileid']) as $preview) { - yield ['path' => $row['path'], 'preview' => $preview]; - } - } - - $cursor->closeCursor(); + private function getPreviewsToDelete(): \Generator { + return $this->previewService->getPreviewsForMimeTypes([ + $this->mimeTypeLoader->getId('text/plain'), + $this->mimeTypeLoader->getId('text/markdown'), + $this->mimeTypeLoader->getId('text/x-markdown'), + ]); } } diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index 3f10328fd8e..fe30da6589c 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -35,6 +35,8 @@ use OCP\IPreview; * @method void setCropped(bool $cropped) * @method void setMimetype(int $mimetype) Set the mimetype of the preview. * @method int getMimetype() Get the mimetype of the preview. + * @method void setSourceMimetype(int $sourceMimetype) Set the mimetype of the source file. + * @method int getSourceMimetype() Get the mimetype of the source file. * @method int getMtime() Get the modification time of the preview. * @method void setMtime(int $mtime) * @method int getSize() Get the size of the preview. @@ -61,6 +63,8 @@ class Preview extends Entity { protected ?int $width = null; protected ?int $height = null; protected ?int $mimetype = null; + + protected ?int $sourceMimetype = null; protected ?int $mtime = null; protected ?int $size = null; protected ?bool $max = null; @@ -77,6 +81,7 @@ class Preview extends Entity { $this->addType('width', Types::INTEGER); $this->addType('height', Types::INTEGER); $this->addType('mimetype', Types::INTEGER); + $this->addType('sourceMimetype', Types::INTEGER); $this->addType('mtime', Types::INTEGER); $this->addType('size', Types::INTEGER); $this->addType('max', Types::BOOLEAN); diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index ec3a0fa7e1f..d4ef242eb21 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -139,4 +139,19 @@ class PreviewMapper extends QBMapper { return $this->yieldEntities($qb); } + + /** + * @param int[] $mimeTypes + * @return \Generator + */ + public function getPreviewsForMimeTypes(array $mimeTypes): \Generator { + $qb = $this->db->getQueryBuilder(); + $this->joinLocation($qb) + ->where($qb->expr()->orX( + ...array_map(function (int $mimeType) use ($qb) { + return $qb->expr()->eq('source_mimetype', $qb->createNamedParameter($mimeType, IQueryBuilder::PARAM_INT)); + }, $mimeTypes) + )); + return $this->yieldEntities($qb); + } } diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index c6e49a1c6ce..1743ca97e0d 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -12,6 +12,7 @@ use OC\Preview\Storage\PreviewFile; use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; +use OCP\Files\IMimeTypeLoader; use OCP\Files\InvalidPathException; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; @@ -38,6 +39,7 @@ class Generator { private LoggerInterface $logger, private PreviewMapper $previewMapper, private StorageFactory $storageFactory, + private IMimeTypeLoader $mimeTypeLoader, ) { } @@ -541,6 +543,7 @@ class Generator { $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); + $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); $previewEntry->setHeight($height); $previewEntry->setVersion($version); $previewEntry->setMax($max); diff --git a/lib/private/Preview/PreviewService.php b/lib/private/Preview/PreviewService.php index 67d6b011416..0e7c66dc12d 100644 --- a/lib/private/Preview/PreviewService.php +++ b/lib/private/Preview/PreviewService.php @@ -71,7 +71,15 @@ class PreviewService { * @return \Generator */ public function getAvailablePreviewForFile(int $fileId): \Generator { - yield from $this->previewMapper->getAvailablePreviewForFile($fileId); + return $this->previewMapper->getAvailablePreviewForFile($fileId); + } + + /** + * @param int[] $mimeTypes + * @return \Generator + */ + public function getPreviewsForMimeTypes(array $mimeTypes): \Generator { + return $this->previewMapper->getPreviewsForMimeTypes($mimeTypes); } public function deleteAll(): void { diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 3bd1fe2ccb9..9af1b594175 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -101,7 +101,7 @@ class LocalPreviewStorage implements IPreviewStorage { $preview->setEncrypted(false); $qb = $this->connection->getQueryBuilder(); - $result = $qb->select('*') + $result = $qb->select('storage', 'etag', 'mimetype') ->from('filecache') ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($preview->getFileId()))) ->setMaxResults(1) @@ -127,6 +127,7 @@ class LocalPreviewStorage implements IPreviewStorage { $preview->setStorageId($result[0]['storage']); $preview->setEtag($result[0]['etag']); + $preview->setSourceMimetype($result[0]['mimetype']); // try to insert, if that fails the preview is already in the DB $this->previewMapper->insert($preview); diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index fb884092872..480408d7724 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -16,6 +16,7 @@ use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\QueryException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; +use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFile; @@ -139,6 +140,7 @@ class PreviewManager implements IPreview { $this->container->get(LoggerInterface::class), $this->container->get(PreviewMapper::class), $this->container->get(StorageFactory::class), + $this->container->get(IMimeTypeLoader::class), ); } return $this->generator; diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index 54d28747cf3..640c2ce8b36 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -14,6 +14,7 @@ use OC\Preview\GeneratorHelper; use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; +use OCP\Files\IMimeTypeLoader; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\IConfig; @@ -34,6 +35,7 @@ class GeneratorTest extends TestCase { private LoggerInterface&MockObject $logger; private StorageFactory&MockObject $storageFactory; private PreviewMapper&MockObject $previewMapper; + private IMimeTypeLoader&MockObject $mimeTypeLoader; protected function setUp(): void { parent::setUp(); @@ -45,6 +47,7 @@ class GeneratorTest extends TestCase { $this->logger = $this->createMock(LoggerInterface::class); $this->previewMapper = $this->createMock(PreviewMapper::class); $this->storageFactory = $this->createMock(StorageFactory::class); + $this->mimetypeLoader = $this->createMock(IMimeTypeLoader::class); $this->generator = new Generator( $this->config, @@ -53,7 +56,8 @@ class GeneratorTest extends TestCase { $this->eventDispatcher, $this->logger, $this->previewMapper, - $this->storageFactory + $this->storageFactory, + $this->mimetypeLoader, ); } diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index 6f59c6f8802..e4b2b1ab21a 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -68,6 +68,7 @@ class PreviewMapperTest extends TestCase { $preview->setCropped(true); $preview->setMax(true); $preview->setWidth(100); + $preview->setSourceMimetype(1); $preview->setHeight(100); $preview->setSize(100); $preview->setMtime(time()); diff --git a/tests/lib/Preview/PreviewServiceTest.php b/tests/lib/Preview/PreviewServiceTest.php index 01bcff29a5a..f9a8d88f664 100644 --- a/tests/lib/Preview/PreviewServiceTest.php +++ b/tests/lib/Preview/PreviewServiceTest.php @@ -43,6 +43,7 @@ class PreviewServiceTest extends TestCase { $preview->setWidth($i); $preview->setHeight($i); $preview->setMax(true); + $preview->setSourceMimetype(1); $preview->setCropped(true); $preview->setEncrypted(false); $preview->setMimetype(IPreview::MIMETYPE_JPEG); From bd001c9524ffc0ffd81a0b62571678784af00148 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Fri, 26 Sep 2025 14:22:38 +0200 Subject: [PATCH 11/14] refactor: Use Override annotation in new preview code Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 2 + core/Command/Preview/ResetRenderedTexts.php | 3 ++ .../Preview/Storage/LocalPreviewStorage.php | 6 +++ .../Storage/ObjectStorePreviewStorage.php | 6 +++ lib/private/Preview/Storage/PreviewFile.php | 45 +++++-------------- .../Preview/Storage/StorageFactory.php | 6 +++ 6 files changed, 35 insertions(+), 33 deletions(-) diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 82acea82c7a..0eaf409e433 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -24,6 +24,7 @@ use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IAppConfig; use OCP\IDBConnection; +use Override; class MovePreviewJob extends TimedJob { private IAppData $appData; @@ -44,6 +45,7 @@ class MovePreviewJob extends TimedJob { $this->setInterval(24 * 60 * 60); } + #[Override] protected function run(mixed $argument): void { if ($this->appConfig->getValueBool('core', 'previewMovedDone')) { return; diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index b80f0f6ffce..6ec26ad6ac3 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -16,6 +16,7 @@ use OCP\Files\NotPermittedException; use OCP\IAvatarManager; use OCP\IDBConnection; use OCP\IUserManager; +use Override; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; @@ -32,6 +33,7 @@ class ResetRenderedTexts extends Command { parent::__construct(); } + #[Override] protected function configure(): void { $this ->setName('preview:reset-rendered-texts') @@ -39,6 +41,7 @@ class ResetRenderedTexts extends Command { ->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not delete any files - in combination with the verbose mode one could check the operations.'); } + #[Override] protected function execute(InputInterface $input, OutputInterface $output): int { $dryMode = $input->getOption('dry'); diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 9af1b594175..8d85897115b 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -19,6 +19,7 @@ use OCP\DB\Exception; use OCP\IAppConfig; use OCP\IConfig; use OCP\IDBConnection; +use Override; use RecursiveDirectoryIterator; use RecursiveIteratorIterator; @@ -37,6 +38,7 @@ class LocalPreviewStorage implements IPreviewStorage { $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); } + #[Override] public function writePreview(Preview $preview, mixed $stream): false|int { $previewPath = $this->constructPath($preview); if (!$this->createParentFiles($previewPath)) { @@ -45,10 +47,12 @@ class LocalPreviewStorage implements IPreviewStorage { return file_put_contents($previewPath, $stream); } + #[Override] public function readPreview(Preview $preview): mixed { return @fopen($this->constructPath($preview), 'r'); } + #[Override] public function deletePreview(Preview $preview): void { @unlink($this->constructPath($preview)); } @@ -67,6 +71,7 @@ class LocalPreviewStorage implements IPreviewStorage { return is_dir($dirname); } + #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { // legacy flat directory $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName(); @@ -87,6 +92,7 @@ class LocalPreviewStorage implements IPreviewStorage { } } + #[Override] public function scan(): int { $checkForFileCache = !$this->appConfig->getValueBool('core', 'previewMovedDone'); diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index faa63763779..88470802667 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -17,6 +17,7 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; +use Override; /** * @psalm-import-type ObjectStoreConfig from PrimaryObjectStoreConfig @@ -39,6 +40,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { $this->isMultibucketPreviewDistributionEnabled = $config->getSystemValueBool('objectstore.multibucket.preview-distribution'); } + #[Override] public function writePreview(Preview $preview, mixed $stream): false|int { if (!is_resource($stream)) { $fh = fopen('php://temp', 'w+'); @@ -63,6 +65,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { return $size; } + #[Override] public function readPreview(Preview $preview): mixed { [ 'objectPrefix' => $objectPrefix, @@ -71,6 +74,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); } + #[Override] public function deletePreview(Preview $preview): void { [ 'objectPrefix' => $objectPrefix, @@ -79,6 +83,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); } + #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { // Just set the Preview::bucket and Preview::objectStore $this->getObjectStoreForPreview($preview, true); @@ -150,6 +155,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { } } + #[Override] public function scan(): int { return 0; } diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php index c9381ce933c..8c9852673cc 100644 --- a/lib/private/Preview/Storage/PreviewFile.php +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -13,6 +13,7 @@ namespace OC\Preview\Storage; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\Files\SimpleFS\ISimpleFile; +use Override; class PreviewFile implements ISimpleFile { public function __construct( @@ -22,80 +23,58 @@ class PreviewFile implements ISimpleFile { ) { } - /** - * @inheritDoc - */ + #[Override] public function getName(): string { return $this->preview->getName(); } - /** - * @inheritDoc - */ + #[Override] public function getSize(): int|float { return $this->preview->getSize(); } - /** - * @inheritDoc - */ + #[Override] public function getETag(): string { return $this->preview->getEtag(); } - /** - * @inheritDoc - */ + #[Override] public function getMTime(): int { return $this->preview->getMtime(); } - /** - * @inheritDoc - */ + #[Override] public function getContent(): string { $stream = $this->storage->readPreview($this->preview); return stream_get_contents($stream); } - /** - * @inheritDoc - */ + #[Override] public function putContent($data): void { } - /** - * @inheritDoc - */ + #[Override] public function delete(): void { $this->storage->deletePreview($this->preview); $this->previewMapper->delete($this->preview); } - /** - * @inheritDoc - */ + #[Override] public function getMimeType(): string { return $this->preview->getMimetypeValue(); } - /** - * @inheritDoc - */ + #[Override] public function getExtension(): string { return $this->preview->getExtension(); } - /** - * @inheritDoc - */ + #[Override] public function read() { return $this->storage->readPreview($this->preview); } - /** - * @inheritDoc - */ + #[Override] public function write() { return false; } diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 3438534d07e..5d1d130b300 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -16,6 +16,7 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\IConfig; use OCP\Server; +use Override; class StorageFactory implements IPreviewStorage { private ?IPreviewStorage $backend = null; @@ -27,14 +28,17 @@ class StorageFactory implements IPreviewStorage { ) { } + #[Override] public function writePreview(Preview $preview, mixed $stream): false|int { return $this->getBackend()->writePreview($preview, $stream); } + #[Override] public function readPreview(Preview $preview): mixed { return $this->getBackend()->readPreview($preview); } + #[Override] public function deletePreview(Preview $preview): void { $this->getBackend()->deletePreview($preview); } @@ -53,10 +57,12 @@ class StorageFactory implements IPreviewStorage { return $this->backend; } + #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { $this->getBackend()->migratePreview($preview, $file); } + #[Override] public function scan(): int { return $this->getBackend()->scan(); } From 66f50bd585f428862849db0db2847287e59453c9 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Tue, 30 Sep 2025 13:44:34 +0200 Subject: [PATCH 12/14] refactor(preview): Use same mimetype ids as filecache Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 13 ++- core/Command/Preview/ResetRenderedTexts.php | 2 +- lib/private/Files/Cache/LocalRootScanner.php | 14 +++ lib/private/Files/Cache/Scanner.php | 22 ++-- lib/private/Preview/Db/Preview.php | 73 ++++++------ lib/private/Preview/Db/PreviewMapper.php | 22 +--- lib/private/Preview/Generator.php | 106 ++++++------------ .../Preview/Storage/LocalPreviewStorage.php | 28 ++++- lib/private/Preview/Storage/PreviewFile.php | 8 +- .../Preview/Storage/StorageFactory.php | 6 +- lib/public/IPreview.php | 5 - tests/lib/Preview/GeneratorTest.php | 39 ++++--- tests/lib/Preview/MovePreviewJobTest.php | 23 +++- tests/lib/Preview/PreviewMapperTest.php | 5 +- tests/lib/Preview/PreviewServiceTest.php | 2 +- 15 files changed, 186 insertions(+), 182 deletions(-) diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index 0eaf409e433..ff13a6f37e2 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -19,12 +19,15 @@ use OCP\BackgroundJob\TimedJob; use OCP\DB\Exception; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\IAppData; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IAppConfig; use OCP\IDBConnection; use Override; +use Psr\Log\LoggerInterface; class MovePreviewJob extends TimedJob { private IAppData $appData; @@ -36,6 +39,9 @@ class MovePreviewJob extends TimedJob { private readonly StorageFactory $storageFactory, private readonly IDBConnection $connection, private readonly IRootFolder $rootFolder, + private readonly IMimeTypeDetector $mimeTypeDetector, + private readonly IMimeTypeLoader $mimeTypeLoader, + private readonly LoggerInterface $logger, IAppDataFactory $appDataFactory, ) { parent::__construct($time); @@ -125,8 +131,13 @@ class MovePreviewJob extends TimedJob { $previewFiles = []; foreach ($folder->getDirectoryListing() as $previewFile) { + $path = $fileId . '/' . $previewFile->getName(); /** @var SimpleFile $previewFile */ - $preview = Preview::fromPath($fileId . '/' . $previewFile->getName()); + $preview = Preview::fromPath($path, $this->mimeTypeDetector, $this->mimeTypeLoader); + if (!$preview) { + $this->logger->error('Unable to import old preview at path.'); + continue; + } $preview->setSize($previewFile->getSize()); $preview->setMtime($previewFile->getMtime()); $preview->setOldFileId($previewFile->getId()); diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index 6ec26ad6ac3..c4fcabbe500 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -93,7 +93,7 @@ class ResetRenderedTexts extends Command { $previewsToDeleteCount = 0; foreach ($this->getPreviewsToDelete() as $preview) { - $output->writeln('Deleting preview ' . $preview->getName() . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); + $output->writeln('Deleting preview ' . $preview->getName($this->mimeTypeLoader) . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); $previewsToDeleteCount++; diff --git a/lib/private/Files/Cache/LocalRootScanner.php b/lib/private/Files/Cache/LocalRootScanner.php index 3f4f70b865b..79908d63fe2 100644 --- a/lib/private/Files/Cache/LocalRootScanner.php +++ b/lib/private/Files/Cache/LocalRootScanner.php @@ -8,7 +8,18 @@ declare(strict_types=1); */ namespace OC\Files\Cache; +use OCP\IConfig; +use OCP\Server; + class LocalRootScanner extends Scanner { + private string $previewFolder; + + public function __construct(\OC\Files\Storage\Storage $storage) { + parent::__construct($storage); + $config = Server::get(IConfig::class); + $this->previewFolder = 'appdata_' . $config->getSystemValueString('instanceid', '') . '/preview'; + } + public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { if ($this->shouldScanPath($file)) { return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock, $data); @@ -27,6 +38,9 @@ class LocalRootScanner extends Scanner { private function shouldScanPath(string $path): bool { $path = trim($path, '/'); + if (str_starts_with($path, $this->previewFolder)) { + return false; + } return $path === '' || str_starts_with($path, 'appdata_') || str_starts_with($path, '__groupfolders'); } } diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 75ca9da0abe..2fa0dd09e4f 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -11,14 +11,15 @@ use Doctrine\DBAL\Exception; use OC\Files\Storage\Wrapper\Encryption; use OC\Files\Storage\Wrapper\Jail; use OC\Hooks\BasicEmitter; -use OC\SystemConfig; use OCP\Files\Cache\IScanner; use OCP\Files\ForbiddenException; use OCP\Files\NotFoundException; use OCP\Files\Storage\ILockingStorage; use OCP\Files\Storage\IReliableEtagStorage; +use OCP\IConfig; use OCP\IDBConnection; use OCP\Lock\ILockingProvider; +use OCP\Server; use Psr\Log\LoggerInterface; /** @@ -65,19 +66,15 @@ class Scanner extends BasicEmitter implements IScanner { protected IDBConnection $connection; - private string $previewFolder; - public function __construct(\OC\Files\Storage\Storage $storage) { $this->storage = $storage; $this->storageId = $this->storage->getId(); $this->cache = $storage->getCache(); - /** @var SystemConfig $config */ - $config = \OC::$server->get(SystemConfig::class); - $this->cacheActive = !$config->getValue('filesystem_cache_readonly', false); - $this->useTransactions = !$config->getValue('filescanner_no_transactions', false); - $this->lockingProvider = \OC::$server->get(ILockingProvider::class); - $this->connection = \OC::$server->get(IDBConnection::class); - $this->previewFolder = 'appdata_' . $config->getValue('instanceid', '') . '/preview'; + $config = Server::get(IConfig::class); + $this->cacheActive = !$config->getSystemValueBool('filesystem_cache_readonly', false); + $this->useTransactions = !$config->getSystemValueBool('filescanner_no_transactions', false); + $this->lockingProvider = Server::get(ILockingProvider::class); + $this->connection = Server::get(IDBConnection::class); } /** @@ -415,11 +412,6 @@ class Scanner extends BasicEmitter implements IScanner { $size = 0; $childQueue = $this->handleChildren($path, $recursive, $reuse, $folderId, $lock, $size, $etagChanged); - if (str_starts_with($path, $this->previewFolder)) { - // Preview scanning is handled in LocalPreviewStorage - return 0; - } - foreach ($childQueue as $child => [$childId, $childSize]) { // "etag changed" propagates up, but not down, so we pass `false` to the children even if we already know that the etag of the current folder changed $childEtagChanged = false; diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index fe30da6589c..7eb03e1e289 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -12,7 +12,9 @@ namespace OC\Preview\Db; use OCP\AppFramework\Db\Entity; use OCP\DB\Types; -use OCP\IPreview; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\IMimeTypeLoader; +use OCP\Server; /** * Preview entity mapped to the oc_previews and oc_preview_locations table. @@ -91,42 +93,39 @@ class Preview extends Entity { $this->addType('version', Types::BIGINT); } - public static function fromPath(string $path): Preview { + public static function fromPath(string $path, IMimeTypeDetector $mimeTypeDetector, IMimeTypeLoader $mimeTypeLoader): Preview|false { $preview = new self(); $preview->setFileId((int)basename(dirname($path))); $fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION); + $ok = preg_match('/(([0-9]+)-)?([0-9]+)-([0-9]+)(-(max))?(-(crop))?\.([a-z]{3,4})/', $fileName, $matches); - [0 => $baseName, 1 => $extension] = explode('.', $fileName); - $preview->setMimetype(match ($extension) { - 'jpg' | 'jpeg' => IPreview::MIMETYPE_JPEG, - 'png' => IPreview::MIMETYPE_PNG, - 'gif' => IPreview::MIMETYPE_GIF, - 'webp' => IPreview::MIMETYPE_WEBP, - default => IPreview::MIMETYPE_JPEG, - }); - $nameSplit = explode('-', $baseName); - - $offset = 0; - $preview->setVersion(null); - if (count($nameSplit) === 4 || (count($nameSplit) === 3 && is_numeric($nameSplit[2]))) { - $offset = 1; - $preview->setVersion((int)$nameSplit[0]); + if ($ok !== 1) { + return false; } - $preview->setWidth((int)$nameSplit[$offset + 0]); - $preview->setHeight((int)$nameSplit[$offset + 1]); + [ + 2 => $version, + 3 => $width, + 4 => $height, + 6 => $crop, + 8 => $max, + ] = $matches; - $preview->setCropped(false); - $preview->setMax(false); - if (isset($nameSplit[$offset + 2])) { - $preview->setCropped($nameSplit[$offset + 2] === 'crop'); - $preview->setMax($nameSplit[$offset + 2] === 'max'); + $preview->setMimetype($mimeTypeLoader->getId($mimeTypeDetector->detectPath($fileName))); + + $preview->setWidth((int)$width); + $preview->setHeight((int)$height); + $preview->setCropped($crop === 'crop'); + $preview->setMax($max === 'max'); + + if (!empty($version)) { + $preview->setVersion((int)$version); } return $preview; } - public function getName(): string { + public function getName(IMimeTypeLoader $mimeTypeLoader): string { $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); if ($this->isCropped()) { $path .= '-crop'; @@ -135,27 +134,23 @@ class Preview extends Entity { $path .= '-max'; } - $ext = $this->getExtension(); + $ext = $this->getExtension($mimeTypeLoader); $path .= '.' . $ext; return $path; } - public function getMimetypeValue(): string { - return match ($this->mimetype) { - IPreview::MIMETYPE_JPEG => 'image/jpeg', - IPreview::MIMETYPE_PNG => 'image/png', - IPreview::MIMETYPE_WEBP => 'image/webp', - IPreview::MIMETYPE_GIF => 'image/gif', + public function getExtension(IMimeTypeLoader $mimeTypeLoader): string { + return match ($this->getMimetypeValue($mimeTypeLoader)) { + 'image/png' => 'png', + 'image/gif' => 'gif', + 'image/jpeg' => 'jpg', + 'image/webp' => 'webp', + default => 'png', }; } - public function getExtension(): string { - return match ($this->mimetype) { - IPreview::MIMETYPE_JPEG => 'jpg', - IPreview::MIMETYPE_PNG => 'png', - IPreview::MIMETYPE_WEBP => 'webp', - IPreview::MIMETYPE_GIF => 'gif', - }; + public function getMimetypeValue(IMimeTypeLoader $mimeTypeLoader): string { + return $mimeTypeLoader->getMimetypeById($this->mimetype) ?? 'image/jpeg'; } public function setBucketName(string $bucketName): void { diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index d4ef242eb21..7d399dc5f3d 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -13,6 +13,7 @@ use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\QBMapper; use OCP\DB\Exception; use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; use OCP\IPreview; @@ -24,7 +25,9 @@ class PreviewMapper extends QBMapper { private const TABLE_NAME = 'previews'; private const LOCATION_TABLE_NAME = 'preview_locations'; - public function __construct(IDBConnection $db) { + public function __construct( + IDBConnection $db, + ) { parent::__construct($db, self::TABLE_NAME, Preview::class); } @@ -57,23 +60,6 @@ class PreviewMapper extends QBMapper { return $previews; } - public function getPreview(int $fileId, int $width, int $height, string $mode, int $mimetype = IPreview::MIMETYPE_JPEG): ?Preview { - $selectQb = $this->db->getQueryBuilder(); - $this->joinLocation($selectQb) - ->where( - $selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId)), - $selectQb->expr()->eq('width', $selectQb->createNamedParameter($width)), - $selectQb->expr()->eq('height', $selectQb->createNamedParameter($height)), - $selectQb->expr()->eq('mode', $selectQb->createNamedParameter($mode)), - $selectQb->expr()->eq('mimetype', $selectQb->createNamedParameter($mimetype)), - ); - try { - return $this->findEntity($selectQb); - } catch (DoesNotExistException) { - return null; - } - } - /** * @return \Generator */ diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 1743ca97e0d..664448b7d01 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -152,7 +152,7 @@ class Generator { // No need to generate a preview that is just the max preview if ($width === $maxWidth && $height === $maxHeight) { // ensure correct return value if this was the last one - $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper); + $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); continue; } @@ -163,14 +163,14 @@ class Generator { && $preview->getVersion() === $previewVersion && $preview->isCropped() === $crop); if ($preview) { - $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper); + $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); } else { if (!$this->previewManager->isMimeSupported($mimeType)) { throw new NotFoundException(); } if ($maxPreviewImage === null) { - $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); + $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader)); } $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); @@ -350,7 +350,21 @@ class Generator { } try { - return $this->savePreview($file, $preview->width(), $preview->height(), $crop, $max, $preview, $version); + $previewEntry = new Preview(); + $previewEntry->setFileId($file->getId()); + $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); + $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); + $previewEntry->setWidth($preview->width()); + $previewEntry->setHeight($preview->height()); + $previewEntry->setVersion($version); + $previewEntry->setMax($max); + $previewEntry->setCropped($crop); + $previewEntry->setEncrypted(false); + $previewEntry->setMimetype($this->mimeTypeLoader->getId($preview->dataMimeType())); + $previewEntry->setEtag($file->getEtag()); + $previewEntry->setMtime((new \DateTime())->getTimestamp()); + $previewEntry->setSize(0); + return $this->savePreview($previewEntry, $preview); } catch (NotPermittedException) { throw new NotFoundException(); } @@ -360,21 +374,6 @@ class Generator { throw new NotFoundException('No provider successfully handled the preview generation'); } - private function generatePath(int $width, int $height, bool $crop, bool $max, string $mimeType, int $version): string { - $path = ($version !== -1 ? $version . '-' : '') . $width . '-' . $height; - if ($crop) { - $path .= '-crop'; - } - if ($max) { - $path .= '-max'; - } - - $ext = $this->getExtension($mimeType); - $path .= '.' . $ext; - return $path; - } - - /** * @psalm-param IPreview::MODE_* $mode * @return int[] @@ -505,30 +504,25 @@ class Generator { self::unguardWithSemaphore($sem); } - $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $version); + $previewEntry = new Preview(); + $previewEntry->setFileId($file->getId()); + $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); + $previewEntry->setWidth($width); + $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); + $previewEntry->setHeight($height); + $previewEntry->setVersion($version); + $previewEntry->setMax(false); + $previewEntry->setCropped($crop); + $previewEntry->setEncrypted(false); + $previewEntry->setMimetype($this->mimeTypeLoader->getId($preview->dataMimeType())); + $previewEntry->setEtag($file->getEtag()); + $previewEntry->setMtime((new \DateTime())->getTimestamp()); + $previewEntry->setSize(0); if ($cacheResult) { - $previewEntry = $this->savePreview($file, $width, $height, $crop, false, $preview, $version); - return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper); + $previewEntry = $this->savePreview($previewEntry, $preview); + return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); } else { - return new InMemoryFile($path, $preview->data()); - } - } - - /** - * @throws \InvalidArgumentException - */ - private function getExtension(string $mimeType): string { - switch ($mimeType) { - case 'image/png': - return 'png'; - case 'image/jpeg': - return 'jpg'; - case 'image/webp': - return 'webp'; - case 'image/gif': - return 'gif'; - default: - throw new \InvalidArgumentException('Not a valid mimetype: "' . $mimeType . '"'); + return new InMemoryFile($previewEntry->getName($this->mimeTypeLoader), $preview->data()); } } @@ -538,35 +532,7 @@ class Generator { * @throws NotPermittedException * @throws \OCP\DB\Exception */ - public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, int $version): Preview { - $previewEntry = new Preview(); - $previewEntry->setFileId($file->getId()); - $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); - $previewEntry->setWidth($width); - $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); - $previewEntry->setHeight($height); - $previewEntry->setVersion($version); - $previewEntry->setMax($max); - $previewEntry->setCropped($crop); - $previewEntry->setEncrypted(false); - switch ($preview->dataMimeType()) { - case 'image/jpeg': - $previewEntry->setMimetype(IPreview::MIMETYPE_JPEG); - break; - case 'image/gif': - $previewEntry->setMimetype(IPreview::MIMETYPE_GIF); - break; - case 'image/webp': - $previewEntry->setMimetype(IPreview::MIMETYPE_WEBP); - break; - default: - $previewEntry->setMimetype(IPreview::MIMETYPE_PNG); - break; - } - $previewEntry->setEtag($file->getEtag()); - $previewEntry->setMtime((new \DateTime())->getTimestamp()); - $previewEntry->setSize(0); - + public function savePreview(Preview $previewEntry, IImage $preview): Preview { $previewEntry = $this->previewMapper->insert($previewEntry); // we need to save to DB first diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index 8d85897115b..e522df7da8e 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -16,10 +16,13 @@ use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\DB\Exception; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\IMimeTypeLoader; use OCP\IAppConfig; use OCP\IConfig; use OCP\IDBConnection; use Override; +use Psr\Log\LoggerInterface; use RecursiveDirectoryIterator; use RecursiveIteratorIterator; @@ -33,6 +36,9 @@ class LocalPreviewStorage implements IPreviewStorage { private readonly StorageFactory $previewStorage, private readonly IAppConfig $appConfig, private readonly IDBConnection $connection, + private readonly IMimeTypeLoader $mimeTypeLoader, + private readonly IMimeTypeDetector $mimeTypeDetector, + private readonly LoggerInterface $logger, ) { $this->instanceId = $this->config->getSystemValueString('instanceid'); $this->rootFolder = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); @@ -62,7 +68,7 @@ class LocalPreviewStorage implements IPreviewStorage { } private function constructPath(Preview $preview): string { - return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); + return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName($this->mimeTypeLoader); } private function createParentFiles(string $path): bool { @@ -74,7 +80,7 @@ class LocalPreviewStorage implements IPreviewStorage { #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { // legacy flat directory - $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName(); + $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName($this->mimeTypeLoader); if (!file_exists($sourcePath)) { return; } @@ -88,7 +94,7 @@ class LocalPreviewStorage implements IPreviewStorage { $this->createParentFiles($destinationPath); $ok = rename($sourcePath, $destinationPath); if (!$ok) { - throw new LogicException('Failed to copy ' . $sourcePath . ' to ' . $destinationPath); + throw new LogicException('Failed to move ' . $sourcePath . ' to ' . $destinationPath); } } @@ -100,7 +106,11 @@ class LocalPreviewStorage implements IPreviewStorage { $previewsFound = 0; foreach (new RecursiveIteratorIterator($scanner) as $file) { if ($file->isFile()) { - $preview = Preview::fromPath((string)$file); + $preview = Preview::fromPath((string)$file, $this->mimeTypeDetector, $this->mimeTypeLoader); + if ($preview === false) { + $this->logger->error('Unable to parse preview information for ' . $file->getRealPath()); + continue; + } try { $preview->setSize($file->getSize()); $preview->setMtime($file->getMtime()); @@ -139,7 +149,15 @@ class LocalPreviewStorage implements IPreviewStorage { $this->previewMapper->insert($preview); // Move old flat preview to new format - $this->previewStorage->migratePreview($preview, $file); + $dirName = str_replace($this->getPreviewRootFolder(), '', $file->getPath()); + if (preg_match('/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9a-e]\/[0-9]+/', $dirName) !== 1) { + $previewPath = $this->constructPath($preview); + $this->createParentFiles($previewPath); + $ok = rename($file->getRealPath(), $previewPath); + if (!$ok) { + throw new LogicException('Failed to move ' . $file->getRealPath() . ' to ' . $previewPath); + } + } } catch (Exception $e) { if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { throw $e; diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php index 8c9852673cc..c7c7b59c97c 100644 --- a/lib/private/Preview/Storage/PreviewFile.php +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -12,6 +12,7 @@ namespace OC\Preview\Storage; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; +use OCP\Files\IMimeTypeLoader; use OCP\Files\SimpleFS\ISimpleFile; use Override; @@ -20,12 +21,13 @@ class PreviewFile implements ISimpleFile { private readonly Preview $preview, private readonly IPreviewStorage $storage, private readonly PreviewMapper $previewMapper, + private readonly IMimeTypeLoader $mimeTypeLoader, ) { } #[Override] public function getName(): string { - return $this->preview->getName(); + return $this->preview->getName($this->mimeTypeLoader); } #[Override] @@ -61,12 +63,12 @@ class PreviewFile implements ISimpleFile { #[Override] public function getMimeType(): string { - return $this->preview->getMimetypeValue(); + return $this->preview->getMimetypeValue($this->mimeTypeLoader); } #[Override] public function getExtension(): string { - return $this->preview->getExtension(); + return $this->preview->getExtension($this->mimeTypeLoader); } #[Override] diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index 5d1d130b300..e33135be3ce 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -13,8 +13,6 @@ namespace OC\Preview\Storage; use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; -use OC\Preview\Db\PreviewMapper; -use OCP\IConfig; use OCP\Server; use Override; @@ -23,8 +21,6 @@ class StorageFactory implements IPreviewStorage { public function __construct( private readonly PrimaryObjectStoreConfig $objectStoreConfig, - private readonly IConfig $config, - private readonly PreviewMapper $previewMapper, ) { } @@ -49,7 +45,7 @@ class StorageFactory implements IPreviewStorage { } if ($this->objectStoreConfig->hasObjectStore()) { - $this->backend = new ObjectStorePreviewStorage($this->objectStoreConfig, $this->config, $this->previewMapper); + $this->backend = Server::get(ObjectStorePreviewStorage::class); } else { $this->backend = Server::get(LocalPreviewStorage::class); } diff --git a/lib/public/IPreview.php b/lib/public/IPreview.php index cbd0e0ae525..3c9eadd4577 100644 --- a/lib/public/IPreview.php +++ b/lib/public/IPreview.php @@ -29,11 +29,6 @@ interface IPreview { */ public const MODE_COVER = 'cover'; - public const MIMETYPE_JPEG = 0; - public const MIMETYPE_WEBP = 1; - public const MIMETYPE_PNG = 2; - public const MIMETYPE_GIF = 3; - /** * In order to improve lazy loading a closure can be registered which will be * called in case preview providers are actually requested diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index 640c2ce8b36..fc48ebc181a 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -47,7 +47,12 @@ class GeneratorTest extends TestCase { $this->logger = $this->createMock(LoggerInterface::class); $this->previewMapper = $this->createMock(PreviewMapper::class); $this->storageFactory = $this->createMock(StorageFactory::class); - $this->mimetypeLoader = $this->createMock(IMimeTypeLoader::class); + $this->mimeTypeLoader = $this->createMock(IMimeTypeLoader::class); + $this->mimeTypeLoader->method('getId') + ->willReturnCallback(fn ($mimeType) => $mimeType === 'image/png' ? 42 : 43); + $this->mimeTypeLoader->method('getMimetypeById') + ->with(42) + ->willReturn('image/png'); $this->generator = new Generator( $this->config, @@ -57,7 +62,7 @@ class GeneratorTest extends TestCase { $this->logger, $this->previewMapper, $this->storageFactory, - $this->mimetypeLoader, + $this->mimeTypeLoader, ); } @@ -91,7 +96,7 @@ class GeneratorTest extends TestCase { $maxPreview->setVersion(-1); $maxPreview->setCropped(false); $maxPreview->setStorageId(1); - $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); $previewFile = new Preview(); $previewFile->setWidth(256); @@ -101,7 +106,7 @@ class GeneratorTest extends TestCase { $previewFile->setVersion(-1); $previewFile->setCropped(false); $previewFile->setStorageId(1); - $previewFile->setMimetype(IPreview::MIMETYPE_PNG); + $previewFile->setMimetype($this->mimeTypeLoader->getId('image/png')); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -130,7 +135,7 @@ class GeneratorTest extends TestCase { ->with($this->equalTo([42])) ->willReturn([42 => []]); - $this->config->method('getSystemValue') + $this->config->method('getSystemValueString') ->willReturnCallback(function ($key, $default) { return $default; }); @@ -196,7 +201,7 @@ class GeneratorTest extends TestCase { $maxPreview->setHeight(2048); $maxPreview->setMax(true); $maxPreview->setSize(1000); - $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); $this->previewMapper->method('insert') ->willReturnCallback(fn (Preview $preview): Preview => $preview); @@ -206,7 +211,7 @@ class GeneratorTest extends TestCase { $this->storageFactory->method('writePreview') ->willReturnCallback(function (Preview $preview, string $data): int { - switch ($preview->getName()) { + switch ($preview->getName($this->mimeTypeLoader)) { case '2048-2048-max.png': $this->assertSame('my data', $data); return 1000; @@ -214,7 +219,7 @@ class GeneratorTest extends TestCase { $this->assertSame('my resized data', $data); return 1000; } - $this->fail('file name is wrong:' . $preview->getName()); + $this->fail('file name is wrong:' . $preview->getName($this->mimeTypeLoader)); }); $image = $this->getMockImage(2048, 2048, 'my resized data'); @@ -245,7 +250,7 @@ class GeneratorTest extends TestCase { $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); - $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + $maxPreview->setMimetype(42); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -269,7 +274,7 @@ class GeneratorTest extends TestCase { $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); - $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); $previewFile = new Preview(); $previewFile->setWidth(1024); @@ -278,7 +283,7 @@ class GeneratorTest extends TestCase { $previewFile->setSize(1000); $previewFile->setCropped(true); $previewFile->setVersion(-1); - $previewFile->setMimetype(IPreview::MIMETYPE_PNG); + $previewFile->setMimetype($this->mimeTypeLoader->getId('image/png')); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -316,7 +321,7 @@ class GeneratorTest extends TestCase { $this->generator->getPreview($file, 100, 100); } - private function getMockImage($width, $height, $data = null) { + private function getMockImage(int $width, int $height, $data = null) { $image = $this->createMock(IImage::class); $image->method('height')->willReturn($width); $image->method('width')->willReturn($height); @@ -387,10 +392,10 @@ class GeneratorTest extends TestCase { $maxPreview->setMax(true); $maxPreview->setSize(1000); $maxPreview->setVersion(-1); - $maxPreview->setMimetype(IPreview::MIMETYPE_PNG); + $maxPreview->setMimetype(42); - $this->assertSame($maxPreview->getName(), $maxX . '-' . $maxY . '-max.png'); - $this->assertSame($maxPreview->getMimetypeValue(), 'image/png'); + $this->assertSame($maxPreview->getName($this->mimeTypeLoader), $maxX . '-' . $maxY . '-max.png'); + $this->assertSame($maxPreview->getMimetypeValue($this->mimeTypeLoader), 'image/png'); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -410,7 +415,7 @@ class GeneratorTest extends TestCase { $this->previewMapper->method('insert') ->willReturnCallback(function (Preview $preview) use ($filename): Preview { - $this->assertSame($preview->getName(), $filename); + $this->assertSame($preview->getName($this->mimeTypeLoader), $filename); return $preview; }); @@ -426,7 +431,7 @@ class GeneratorTest extends TestCase { $result = $this->generator->getPreview($file, $reqX, $reqY, $crop, $mode); if ($expectedX === $maxX && $expectedY === $maxY) { - $this->assertSame($maxPreview->getName(), $result->getName()); + $this->assertSame($maxPreview->getName($this->mimeTypeLoader), $result->getName()); } else { $this->assertSame($filename, $result->getName()); } diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php index cd9f3253654..a5ac5ad51e3 100644 --- a/tests/lib/Preview/MovePreviewJobTest.php +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -17,12 +17,15 @@ use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\Utility\ITimeFactory; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\IAppData; +use OCP\Files\IMimeTypeDetector; +use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; use OCP\IAppConfig; use OCP\IDBConnection; use OCP\Server; use PHPUnit\Framework\Attributes\TestDox; use PHPUnit\Framework\MockObject\MockObject; +use Psr\Log\LoggerInterface; use Test\TestCase; /** @@ -35,6 +38,9 @@ class MovePreviewJobTest extends TestCase { private StorageFactory $storageFactory; private PreviewService $previewService; private IDBConnection $db; + private IMimeTypeLoader&MockObject $mimeTypeLoader; + private IMimeTypeDetector&MockObject $mimeTypeDetector; + private LoggerInterface&MockObject $logger; public function setUp(): void { parent::setUp(); @@ -75,6 +81,12 @@ class MovePreviewJobTest extends TestCase { 'permissions' => $qb->createNamedParameter(0), 'checksum' => $qb->createNamedParameter('abcdefg'), ])->executeStatement(); + + $this->mimeTypeDetector = $this->createMock(IMimeTypeDetector::class); + $this->mimeTypeDetector->method('detectPath')->willReturn('image/png'); + $this->mimeTypeLoader = $this->createMock(IMimeTypeLoader::class); + $this->mimeTypeLoader->method('getId')->with('image/png')->willReturn(42); + $this->logger = $this->createMock(LoggerInterface::class); } public function tearDown(): void { @@ -105,7 +117,10 @@ class MovePreviewJobTest extends TestCase { $this->storageFactory, Server::get(IDBConnection::class), Server::get(IRootFolder::class), - Server::get(IAppDataFactory::class) + $this->mimeTypeDetector, + $this->mimeTypeLoader, + $this->logger, + Server::get(IAppDataFactory::class), ); $this->invokePrivate($job, 'run', [[]]); $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); @@ -133,6 +148,9 @@ class MovePreviewJobTest extends TestCase { $this->storageFactory, Server::get(IDBConnection::class), Server::get(IRootFolder::class), + $this->mimeTypeDetector, + $this->mimeTypeLoader, + $this->logger, Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); @@ -169,6 +187,9 @@ class MovePreviewJobTest extends TestCase { $this->storageFactory, Server::get(IDBConnection::class), Server::get(IRootFolder::class), + $this->mimeTypeDetector, + $this->mimeTypeLoader, + $this->logger, Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index e4b2b1ab21a..19018c8b318 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -12,6 +12,7 @@ namespace Test\Preview; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; +use OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; use OCP\IPreview; use OCP\Server; @@ -23,10 +24,12 @@ use Test\TestCase; class PreviewMapperTest extends TestCase { private PreviewMapper $previewMapper; private IDBConnection $connection; + private IMimeTypeLoader $mimeTypeLoader; public function setUp(): void { $this->previewMapper = Server::get(PreviewMapper::class); $this->connection = Server::get(IDBConnection::class); + $this->mimeTypeLoader = Server::get(IMimeTypeLoader::class); } public function testGetAvailablePreviews(): void { @@ -72,7 +75,7 @@ class PreviewMapperTest extends TestCase { $preview->setHeight(100); $preview->setSize(100); $preview->setMtime(time()); - $preview->setMimetype(IPreview::MIMETYPE_PNG); + $preview->setMimetype($this->mimeTypeLoader->getId('image/jpeg')); $preview->setEtag('abcdefg'); if ($locationId !== null) { diff --git a/tests/lib/Preview/PreviewServiceTest.php b/tests/lib/Preview/PreviewServiceTest.php index f9a8d88f664..fe8dd1c3d33 100644 --- a/tests/lib/Preview/PreviewServiceTest.php +++ b/tests/lib/Preview/PreviewServiceTest.php @@ -46,7 +46,7 @@ class PreviewServiceTest extends TestCase { $preview->setSourceMimetype(1); $preview->setCropped(true); $preview->setEncrypted(false); - $preview->setMimetype(IPreview::MIMETYPE_JPEG); + $preview->setMimetype(42); $preview->setEtag('abc'); $preview->setMtime((new \DateTime())->getTimestamp()); $preview->setSize(0); From bef3996c3e21d401e11b3a5b2ea05be10189df2a Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Tue, 30 Sep 2025 15:56:31 +0200 Subject: [PATCH 13/14] fix(preview): Make version column a string And move it to a different table so that we don't have to pay the storage cost when not using it (most of the times). Signed-off-by: Carl Schwan --- core/BackgroundJobs/MovePreviewJob.php | 137 +++++++++++------- core/Command/Preview/ResetRenderedTexts.php | 10 +- .../Version33000Date20250819110529.php | 16 +- lib/private/Files/Cache/LocalRootScanner.php | 14 +- lib/private/Preview/BackgroundCleanupJob.php | 2 +- lib/private/Preview/Db/Preview.php | 74 ++++++---- lib/private/Preview/Db/PreviewMapper.php | 75 +++++++++- lib/private/Preview/Generator.php | 35 ++--- lib/private/Preview/PreviewService.php | 6 +- .../Preview/Storage/IPreviewStorage.php | 20 ++- .../Preview/Storage/LocalPreviewStorage.php | 35 +++-- .../Storage/ObjectStorePreviewStorage.php | 29 ++-- lib/private/Preview/Storage/PreviewFile.php | 8 +- .../Preview/Storage/StorageFactory.php | 2 +- lib/private/Preview/Watcher.php | 16 +- lib/private/PreviewManager.php | 2 - lib/private/Server.php | 1 + lib/public/Preview/IVersionedPreviewFile.php | 3 +- .../lib/Preview/BackgroundCleanupJobTest.php | 2 +- tests/lib/Preview/GeneratorTest.php | 114 ++++++++------- tests/lib/Preview/MovePreviewJobTest.php | 35 +++-- tests/lib/Preview/PreviewMapperTest.php | 8 +- tests/lib/Preview/PreviewServiceTest.php | 6 +- 23 files changed, 404 insertions(+), 246 deletions(-) diff --git a/core/BackgroundJobs/MovePreviewJob.php b/core/BackgroundJobs/MovePreviewJob.php index ff13a6f37e2..930f998a28b 100644 --- a/core/BackgroundJobs/MovePreviewJob.php +++ b/core/BackgroundJobs/MovePreviewJob.php @@ -17,24 +17,26 @@ use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; use OCP\DB\Exception; +use OCP\DB\IResult; use OCP\Files\AppData\IAppDataFactory; use OCP\Files\IAppData; use OCP\Files\IMimeTypeDetector; use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; -use OCP\Files\NotFoundException; -use OCP\Files\SimpleFS\ISimpleFolder; use OCP\IAppConfig; +use OCP\IConfig; use OCP\IDBConnection; use Override; use Psr\Log\LoggerInterface; class MovePreviewJob extends TimedJob { private IAppData $appData; + private string $previewRootPath; public function __construct( ITimeFactory $time, private readonly IAppConfig $appConfig, + private readonly IConfig $config, private readonly PreviewMapper $previewMapper, private readonly StorageFactory $storageFactory, private readonly IDBConnection $connection, @@ -49,6 +51,7 @@ class MovePreviewJob extends TimedJob { $this->appData = $appDataFactory->get('preview'); $this->setTimeSensitivity(self::TIME_INSENSITIVE); $this->setInterval(24 * 60 * 60); + $this->previewRootPath = 'appdata_' . $this->config->getSystemValueString('instanceid') . '/preview/'; } #[Override] @@ -57,49 +60,22 @@ class MovePreviewJob extends TimedJob { return; } - $emptyHierarchicalPreviewFolders = false; - $startTime = time(); while (true) { - // Check new hierarchical preview folders first - if (!$emptyHierarchicalPreviewFolders) { - $qb = $this->connection->getQueryBuilder(); - $qb->select('*') - ->from('filecache') - ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%/%/%/%/%/%/%/%'))) - ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) - ->setMaxResults(100); - - $result = $qb->executeQuery(); - while ($row = $result->fetch()) { - $pathSplit = explode('/', $row['path']); - assert(count($pathSplit) >= 2); - $fileId = $pathSplit[count($pathSplit) - 2]; - $this->processPreviews($fileId, false); - } - } - - // And then the flat preview folder (legacy) - $emptyHierarchicalPreviewFolders = true; $qb = $this->connection->getQueryBuilder(); - $qb->select('*') + $qb->select('path') ->from('filecache') - ->where($qb->expr()->like('path', $qb->createNamedParameter('appdata_%/preview/%/%.%'))) + // Hierarchical preview folder structure + ->where($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%'))) + // Legacy flat preview folder structure + ->orWhere($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%'))) ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) ->setMaxResults(100); $result = $qb->executeQuery(); - $foundOldPreview = false; - while ($row = $result->fetch()) { - $pathSplit = explode('/', $row['path']); - assert(count($pathSplit) >= 2); - $fileId = $pathSplit[count($pathSplit) - 2]; - array_pop($pathSplit); - $this->processPreviews($fileId, true); - $foundOldPreview = true; - } + $foundPreviews = $this->processQueryResult($result); - if (!$foundOldPreview) { + if (!$foundPreviews) { break; } @@ -109,20 +85,46 @@ class MovePreviewJob extends TimedJob { } } - try { - // Delete any leftover preview directory - $this->appData->getFolder('.')->delete(); - } catch (NotFoundException) { - // ignore - } $this->appConfig->setValueBool('core', 'previewMovedDone', true); } + private function processQueryResult(IResult $result): bool { + $foundPreview = false; + $fileIds = []; + $flatFileIds = []; + while ($row = $result->fetch()) { + $pathSplit = explode('/', $row['path']); + assert(count($pathSplit) >= 2); + $fileId = (int)$pathSplit[count($pathSplit) - 2]; + if (count($pathSplit) === 11) { + // Hierarchical structure + if (!in_array($fileId, $fileIds)) { + $fileIds[] = $fileId; + } + } else { + // Flat structure + if (!in_array($fileId, $flatFileIds)) { + $flatFileIds[] = $fileId; + } + } + $foundPreview = true; + } + + foreach ($fileIds as $fileId) { + $this->processPreviews($fileId, flatPath: false); + } + + foreach ($flatFileIds as $fileId) { + $this->processPreviews($fileId, flatPath: true); + } + return $foundPreview; + } + /** * @param array $previewFolders */ - private function processPreviews(int|string $fileId, bool $simplePaths): void { - $internalPath = $this->getInternalFolder((string)$fileId, $simplePaths); + private function processPreviews(int $fileId, bool $flatPath): void { + $internalPath = $this->getInternalFolder((string)$fileId, $flatPath); $folder = $this->appData->getFolder($internalPath); /** @@ -133,7 +135,7 @@ class MovePreviewJob extends TimedJob { foreach ($folder->getDirectoryListing() as $previewFile) { $path = $fileId . '/' . $previewFile->getName(); /** @var SimpleFile $previewFile */ - $preview = Preview::fromPath($path, $this->mimeTypeDetector, $this->mimeTypeLoader); + $preview = Preview::fromPath($path, $this->mimeTypeDetector); if (!$preview) { $this->logger->error('Unable to import old preview at path.'); continue; @@ -160,23 +162,30 @@ class MovePreviewJob extends TimedJob { if (count($result) > 0) { foreach ($previewFiles as $previewFile) { + /** @var Preview $preview */ $preview = $previewFile['preview']; /** @var SimpleFile $file */ $file = $previewFile['file']; $preview->setStorageId($result[0]['storage']); $preview->setEtag($result[0]['etag']); - $preview->setSourceMimetype($result[0]['mimetype']); + $preview->setSourceMimeType($this->mimeTypeLoader->getMimetypeById((int)$result[0]['mimetype'])); try { $preview = $this->previewMapper->insert($preview); - } catch (Exception $e) { + } catch (Exception) { // We already have this preview in the preview table, skip + $qb->delete('filecache') + ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId()))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) + ->executeStatement(); continue; } try { $this->storageFactory->migratePreview($preview, $file); + $qb = $this->connection->getQueryBuilder(); $qb->delete('filecache') ->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId()))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) ->executeStatement(); // Do not call $file->delete() as this will also delete the file from the file system } catch (\Exception $e) { @@ -184,35 +193,51 @@ class MovePreviewJob extends TimedJob { throw $e; } } + } else { + // No matching fileId, delete preview + try { + $this->connection->beginTransaction(); + foreach ($previewFiles as $previewFile) { + /** @var SimpleFile $file */ + $file = $previewFile['file']; + $file->delete(); + } + $this->connection->commit(); + } catch (Exception) { + $this->connection->rollback(); + } } - $this->deleteFolder($internalPath, $folder); + $this->deleteFolder($internalPath); } - public static function getInternalFolder(string $name, bool $simplePaths): string { - if ($simplePaths) { - return '/' . $name; + public static function getInternalFolder(string $name, bool $flatPath): string { + if ($flatPath) { + return $name; } return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name; } - private function deleteFolder(string $path, ISimpleFolder $folder): void { - $folder->delete(); - + private function deleteFolder(string $path): void { $current = $path; while (true) { + $appDataPath = $this->previewRootPath . $current; + $qb = $this->connection->getQueryBuilder(); + $qb->delete('filecache') + ->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($appDataPath)))) + ->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId()) + ->executeStatement(); + $current = dirname($current); if ($current === '/' || $current === '.' || $current === '') { break; } - $folder = $this->appData->getFolder($current); if (count($folder->getDirectoryListing()) !== 0) { break; } - $folder->delete(); } } } diff --git a/core/Command/Preview/ResetRenderedTexts.php b/core/Command/Preview/ResetRenderedTexts.php index c4fcabbe500..978128a609f 100644 --- a/core/Command/Preview/ResetRenderedTexts.php +++ b/core/Command/Preview/ResetRenderedTexts.php @@ -10,7 +10,6 @@ namespace OC\Core\Command\Preview; use OC\Preview\Db\Preview; use OC\Preview\PreviewService; -use OCP\Files\IMimeTypeLoader; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\IAvatarManager; @@ -28,7 +27,6 @@ class ResetRenderedTexts extends Command { protected readonly IUserManager $userManager, protected readonly IAvatarManager $avatarManager, private readonly PreviewService $previewService, - private readonly IMimeTypeLoader $mimeTypeLoader, ) { parent::__construct(); } @@ -93,7 +91,7 @@ class ResetRenderedTexts extends Command { $previewsToDeleteCount = 0; foreach ($this->getPreviewsToDelete() as $preview) { - $output->writeln('Deleting preview ' . $preview->getName($this->mimeTypeLoader) . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); + $output->writeln('Deleting preview ' . $preview->getName() . ' for fileId ' . $preview->getFileId(), OutputInterface::VERBOSITY_VERBOSE); $previewsToDeleteCount++; @@ -112,9 +110,9 @@ class ResetRenderedTexts extends Command { */ private function getPreviewsToDelete(): \Generator { return $this->previewService->getPreviewsForMimeTypes([ - $this->mimeTypeLoader->getId('text/plain'), - $this->mimeTypeLoader->getId('text/markdown'), - $this->mimeTypeLoader->getId('text/x-markdown'), + 'text/plain', + 'text/markdown', + 'text/x-markdown' ]); } } diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php index 27bf9ab89b7..32a25b0e33c 100644 --- a/core/Migrations/Version33000Date20250819110529.php +++ b/core/Migrations/Version33000Date20250819110529.php @@ -37,6 +37,14 @@ class Version33000Date20250819110529 extends SimpleMigrationStep { $table->setPrimaryKey(['id']); } + if (!$schema->hasTable('preview_versions')) { + $table = $schema->createTable('preview_versions'); + $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('version', Types::STRING, ['notnull' => true, 'default' => '', 'length' => 1024]); + $table->setPrimaryKey(['id']); + } + if (!$schema->hasTable('previews')) { $table = $schema->createTable('previews'); $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); @@ -46,18 +54,18 @@ class Version33000Date20250819110529 extends SimpleMigrationStep { $table->addColumn('location_id', Types::BIGINT, ['notnull' => false, 'length' => 20, 'unsigned' => true]); $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); - $table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]); - $table->addColumn('source_mimetype', Types::INTEGER, ['notnull' => true]); + $table->addColumn('mimetype_id', Types::INTEGER, ['notnull' => true]); + $table->addColumn('source_mimetype_id', Types::INTEGER, ['notnull' => true]); $table->addColumn('max', Types::BOOLEAN, ['notnull' => true, 'default' => false]); $table->addColumn('cropped', Types::BOOLEAN, ['notnull' => true, 'default' => false]); $table->addColumn('encrypted', Types::BOOLEAN, ['notnull' => true, 'default' => false]); $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40, 'fixed' => true]); $table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); $table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); - $table->addColumn('version', Types::BIGINT, ['notnull' => true, 'default' => -1]); // can not be null otherwise unique index doesn't work + $table->addColumn('version_id', Types::BIGINT, ['notnull' => true, 'default' => -1]); $table->setPrimaryKey(['id']); $table->addIndex(['file_id']); - $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'cropped', 'version'], 'previews_file_uniq_idx'); + $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype_id', 'cropped', 'version_id'], 'previews_file_uniq_idx'); } return $schema; diff --git a/lib/private/Files/Cache/LocalRootScanner.php b/lib/private/Files/Cache/LocalRootScanner.php index 79908d63fe2..d5f7d40e1b6 100644 --- a/lib/private/Files/Cache/LocalRootScanner.php +++ b/lib/private/Files/Cache/LocalRootScanner.php @@ -10,6 +10,7 @@ namespace OC\Files\Cache; use OCP\IConfig; use OCP\Server; +use Override; class LocalRootScanner extends Scanner { private string $previewFolder; @@ -20,6 +21,7 @@ class LocalRootScanner extends Scanner { $this->previewFolder = 'appdata_' . $config->getSystemValueString('instanceid', '') . '/preview'; } + #[Override] public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = null, $lock = true, $data = null) { if ($this->shouldScanPath($file)) { return parent::scanFile($file, $reuseExisting, $parentId, $cacheData, $lock, $data); @@ -28,6 +30,7 @@ class LocalRootScanner extends Scanner { } } + #[Override] public function scan($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $lock = true) { if ($this->shouldScanPath($path)) { return parent::scan($path, $recursive, $reuse, $lock); @@ -36,11 +39,16 @@ class LocalRootScanner extends Scanner { } } + #[Override] + protected function scanChildren(string $path, $recursive, int $reuse, int $folderId, bool $lock, int|float $oldSize, &$etagChanged = false) { + if (str_starts_with($path, $this->previewFolder)) { + return 0; + } + return parent::scanChildren($path, $recursive, $reuse, $folderId, $lock, $oldSize, $etagChanged); + } + private function shouldScanPath(string $path): bool { $path = trim($path, '/'); - if (str_starts_with($path, $this->previewFolder)) { - return false; - } return $path === '' || str_starts_with($path, 'appdata_') || str_starts_with($path, '__groupfolders'); } } diff --git a/lib/private/Preview/BackgroundCleanupJob.php b/lib/private/Preview/BackgroundCleanupJob.php index 257bebb1d5d..f6122fd0e12 100644 --- a/lib/private/Preview/BackgroundCleanupJob.php +++ b/lib/private/Preview/BackgroundCleanupJob.php @@ -35,7 +35,7 @@ class BackgroundCleanupJob extends TimedJob { public function run($argument): void { foreach ($this->getDeletedFiles() as $fileId) { $previewIds = []; - foreach ($this->previewService->getAvailablePreviewForFile($fileId) as $preview) { + foreach ($this->previewService->getAvailablePreviewsForFile($fileId) as $preview) { $this->previewService->deletePreview($preview); } } diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php index 7eb03e1e289..d3b3ab5ad07 100644 --- a/lib/private/Preview/Db/Preview.php +++ b/lib/private/Preview/Db/Preview.php @@ -13,8 +13,6 @@ namespace OC\Preview\Db; use OCP\AppFramework\Db\Entity; use OCP\DB\Types; use OCP\Files\IMimeTypeDetector; -use OCP\Files\IMimeTypeLoader; -use OCP\Server; /** * Preview entity mapped to the oc_previews and oc_preview_locations table. @@ -35,10 +33,10 @@ use OCP\Server; * @method void setHeight(int $height) * @method bool isCropped() Get whether the preview is cropped or not. * @method void setCropped(bool $cropped) - * @method void setMimetype(int $mimetype) Set the mimetype of the preview. - * @method int getMimetype() Get the mimetype of the preview. - * @method void setSourceMimetype(int $sourceMimetype) Set the mimetype of the source file. - * @method int getSourceMimetype() Get the mimetype of the source file. + * @method void setMimetypeId(int $mimetype) Set the mimetype of the preview. + * @method int getMimetypeId() Get the mimetype of the preview. + * @method void setSourceMimetypeId(int $sourceMimetype) Set the mimetype of the source file. + * @method int getSourceMimetypeId() Get the mimetype of the source file. * @method int getMtime() Get the modification time of the preview. * @method void setMtime(int $mtime) * @method int getSize() Get the size of the preview. @@ -47,8 +45,8 @@ use OCP\Server; * @method void setMax(bool $max) * @method string getEtag() Get the etag of the preview. * @method void setEtag(string $etag) - * @method int|null getVersion() Get the version for files_versions_s3 - * @method void setVersion(?int $version) + * @method string|null getVersion() Get the version for files_versions_s3 + * @method void setVersionId(int $versionId) * @method bool|null getIs() Get the version for files_versions_s3 * @method bool isEncrypted() Get whether the preview is encrypted. At the moment every preview is unencrypted. * @method void setEncrypted(bool $encrypted) @@ -64,15 +62,17 @@ class Preview extends Entity { protected ?string $objectStoreName = null; protected ?int $width = null; protected ?int $height = null; - protected ?int $mimetype = null; - - protected ?int $sourceMimetype = null; + protected ?int $mimetypeId = null; + protected ?int $sourceMimetypeId = null; + protected string $mimetype = 'application/octet-stream'; + protected string $sourceMimetype = 'application/octet-stream'; protected ?int $mtime = null; protected ?int $size = null; protected ?bool $max = null; protected ?bool $cropped = null; protected ?string $etag = null; - protected ?int $version = null; + protected ?string $version = null; + protected ?int $versionId = null; protected ?bool $encrypted = null; public function __construct() { @@ -82,23 +82,23 @@ class Preview extends Entity { $this->addType('locationId', Types::BIGINT); $this->addType('width', Types::INTEGER); $this->addType('height', Types::INTEGER); - $this->addType('mimetype', Types::INTEGER); - $this->addType('sourceMimetype', Types::INTEGER); + $this->addType('mimetypeId', Types::INTEGER); + $this->addType('sourceMimetypeId', Types::INTEGER); $this->addType('mtime', Types::INTEGER); $this->addType('size', Types::INTEGER); $this->addType('max', Types::BOOLEAN); $this->addType('cropped', Types::BOOLEAN); $this->addType('encrypted', Types::BOOLEAN); $this->addType('etag', Types::STRING); - $this->addType('version', Types::BIGINT); + $this->addType('versionId', Types::STRING); } - public static function fromPath(string $path, IMimeTypeDetector $mimeTypeDetector, IMimeTypeLoader $mimeTypeLoader): Preview|false { + public static function fromPath(string $path, IMimeTypeDetector $mimeTypeDetector): Preview|false { $preview = new self(); $preview->setFileId((int)basename(dirname($path))); $fileName = pathinfo($path, PATHINFO_FILENAME) . '.' . pathinfo($path, PATHINFO_EXTENSION); - $ok = preg_match('/(([0-9]+)-)?([0-9]+)-([0-9]+)(-(max))?(-(crop))?\.([a-z]{3,4})/', $fileName, $matches); + $ok = preg_match('/(([A-Za-z0-9\+\/]+)-)?([0-9]+)-([0-9]+)(-(max))?(-(crop))?\.([a-z]{3,4})/', $fileName, $matches); if ($ok !== 1) { return false; @@ -108,11 +108,11 @@ class Preview extends Entity { 2 => $version, 3 => $width, 4 => $height, - 6 => $crop, - 8 => $max, + 6 => $max, + 8 => $crop, ] = $matches; - $preview->setMimetype($mimeTypeLoader->getId($mimeTypeDetector->detectPath($fileName))); + $preview->setMimeType($mimeTypeDetector->detectPath($fileName)); $preview->setWidth((int)$width); $preview->setHeight((int)$height); @@ -120,12 +120,12 @@ class Preview extends Entity { $preview->setMax($max === 'max'); if (!empty($version)) { - $preview->setVersion((int)$version); + $preview->setVersion($version); } return $preview; } - public function getName(IMimeTypeLoader $mimeTypeLoader): string { + public function getName(): string { $path = ($this->getVersion() > -1 ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); if ($this->isCropped()) { $path .= '-crop'; @@ -134,13 +134,13 @@ class Preview extends Entity { $path .= '-max'; } - $ext = $this->getExtension($mimeTypeLoader); + $ext = $this->getExtension(); $path .= '.' . $ext; return $path; } - public function getExtension(IMimeTypeLoader $mimeTypeLoader): string { - return match ($this->getMimetypeValue($mimeTypeLoader)) { + public function getExtension(): string { + return match ($this->getMimeType()) { 'image/png' => 'png', 'image/gif' => 'gif', 'image/jpeg' => 'jpg', @@ -149,10 +149,6 @@ class Preview extends Entity { }; } - public function getMimetypeValue(IMimeTypeLoader $mimeTypeLoader): string { - return $mimeTypeLoader->getMimetypeById($this->mimetype) ?? 'image/jpeg'; - } - public function setBucketName(string $bucketName): void { $this->bucketName = $bucketName; } @@ -160,4 +156,24 @@ class Preview extends Entity { public function setObjectStoreName(string $objectStoreName): void { $this->objectStoreName = $objectStoreName; } + + public function setVersion(?string $version): void { + $this->version = $version; + } + + public function getMimeType(): string { + return $this->mimetype; + } + + public function setMimeType(string $mimeType): void { + $this->mimetype = $mimeType; + } + + public function getSourceMimeType(): string { + return $this->sourceMimetype; + } + + public function setSourceMimeType(string $mimeType): void { + $this->sourceMimetype = $mimeType; + } } diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php index 7d399dc5f3d..e6ca2e720f3 100644 --- a/lib/private/Preview/Db/PreviewMapper.php +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -9,13 +9,13 @@ declare(strict_types=1); namespace OC\Preview\Db; -use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\Entity; use OCP\AppFramework\Db\QBMapper; use OCP\DB\Exception; use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; -use OCP\IPreview; +use Override; /** * @template-extends QBMapper @@ -24,18 +24,74 @@ class PreviewMapper extends QBMapper { private const TABLE_NAME = 'previews'; private const LOCATION_TABLE_NAME = 'preview_locations'; + private const VERSION_TABLE_NAME = 'preview_versions'; public function __construct( IDBConnection $db, + private readonly IMimeTypeLoader $mimeTypeLoader, ) { parent::__construct($db, self::TABLE_NAME, Preview::class); } + protected function mapRowToEntity(array $row): Entity { + $row['mimetype'] = $this->mimeTypeLoader->getMimetypeById((int)$row['mimetype_id']); + $row['source_mimetype'] = $this->mimeTypeLoader->getMimetypeById((int)$row['source_mimetype_id']); + + return parent::mapRowToEntity($row); + } + + #[Override] + public function insert(Entity $entity): Entity { + /** @var Preview $preview */ + $preview = $entity; + + $preview->setMimetypeId($this->mimeTypeLoader->getId($preview->getMimeType())); + $preview->setSourceMimetypeId($this->mimeTypeLoader->getId($preview->getSourceMimeType())); + + if ($preview->getVersion() !== null && $preview->getVersion() !== '') { + $qb = $this->db->getQueryBuilder(); + $qb->insert(self::VERSION_TABLE_NAME) + ->values([ + 'version' => $preview->getVersion(), + 'file_id' => $preview->getFileId(), + ]) + ->executeStatement(); + $entity->setVersionId($qb->getLastInsertId()); + } + return parent::insert($preview); + } + + #[Override] + public function update(Entity $entity): Entity { + /** @var Preview $preview */ + $preview = $entity; + + $preview->setMimetypeId($this->mimeTypeLoader->getId($preview->getMimeType())); + $preview->setSourceMimetypeId($this->mimeTypeLoader->getId($preview->getSourceMimeType())); + + return parent::update($preview); + } + + #[Override] + public function delete(Entity $entity): Entity { + /** @var Preview $preview */ + $preview = $entity; + if ($preview->getVersion() !== null && $preview->getVersion() !== '') { + $qb = $this->db->getQueryBuilder(); + $qb->delete(self::VERSION_TABLE_NAME) + ->where($qb->expr()->eq('file_id', $qb->createNamedParameter($preview->getFileId()))) + ->andWhere($qb->expr()->eq('version', $qb->createNamedParameter($preview->getVersion()))) + ->executeStatement(); + } + + return parent::delete($entity); + } + /** * @return \Generator * @throws Exception */ - public function getAvailablePreviewForFile(int $fileId): \Generator { + public function getAvailablePreviewsForFile(int $fileId): \Generator { $selectQb = $this->db->getQueryBuilder(); $this->joinLocation($selectQb) ->where($selectQb->expr()->eq('p.file_id', $selectQb->createNamedParameter($fileId, IQueryBuilder::PARAM_INT))); @@ -82,10 +138,13 @@ class PreviewMapper extends QBMapper { } protected function joinLocation(IQueryBuilder $qb): IQueryBuilder { - return $qb->select('p.*', 'l.bucket_name', 'l.object_store_name') + return $qb->select('p.*', 'l.bucket_name', 'l.object_store_name', 'v.version') ->from(self::TABLE_NAME, 'p') - ->leftJoin('p', 'preview_locations', 'l', $qb->expr()->eq( + ->leftJoin('p', self::LOCATION_TABLE_NAME, 'l', $qb->expr()->eq( 'p.location_id', 'l.id' + )) + ->leftJoin('p', self::VERSION_TABLE_NAME, 'v', $qb->expr()->eq( + 'p.version_id', 'v.id' )); } @@ -127,15 +186,15 @@ class PreviewMapper extends QBMapper { } /** - * @param int[] $mimeTypes + * @param string[] $mimeTypes * @return \Generator */ public function getPreviewsForMimeTypes(array $mimeTypes): \Generator { $qb = $this->db->getQueryBuilder(); $this->joinLocation($qb) ->where($qb->expr()->orX( - ...array_map(function (int $mimeType) use ($qb) { - return $qb->expr()->eq('source_mimetype', $qb->createNamedParameter($mimeType, IQueryBuilder::PARAM_INT)); + ...array_map(function (string $mimeType) use ($qb): string { + return $qb->expr()->eq('source_mimetype_id', $qb->createNamedParameter($this->mimeTypeLoader->getId($mimeType), IQueryBuilder::PARAM_INT)); }, $mimeTypes) )); return $this->yieldEntities($qb); diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 664448b7d01..82c4ec88363 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -12,7 +12,6 @@ use OC\Preview\Storage\PreviewFile; use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IMimeTypeLoader; use OCP\Files\InvalidPathException; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; @@ -39,7 +38,6 @@ class Generator { private LoggerInterface $logger, private PreviewMapper $previewMapper, private StorageFactory $storageFactory, - private IMimeTypeLoader $mimeTypeLoader, ) { } @@ -111,9 +109,9 @@ class Generator { [$file->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$file->getId()]); - $previewVersion = -1; + $previewVersion = null; if ($file instanceof IVersionedPreviewFile) { - $previewVersion = (int)$file->getPreviewVersion(); + $previewVersion = $file->getPreviewVersion(); } // Get the max preview and infer the max preview sizes from that @@ -152,7 +150,7 @@ class Generator { // No need to generate a preview that is just the max preview if ($width === $maxWidth && $height === $maxHeight) { // ensure correct return value if this was the last one - $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); + $previewFile = new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper); continue; } @@ -163,14 +161,14 @@ class Generator { && $preview->getVersion() === $previewVersion && $preview->isCropped() === $crop); if ($preview) { - $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); + $previewFile = new PreviewFile($preview, $this->storageFactory, $this->previewMapper); } else { if (!$this->previewManager->isMimeSupported($mimeType)) { throw new NotFoundException(); } if ($maxPreviewImage === null) { - $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader)); + $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); } $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); @@ -298,7 +296,7 @@ class Generator { * @param Preview[] $previews * @throws NotFoundException */ - private function getMaxPreview(array $previews, File $file, string $mimeType, int $version): Preview { + private function getMaxPreview(array $previews, File $file, string $mimeType, ?string $version): Preview { // We don't know the max preview size, so we can't use getCachedPreview. // It might have been generated with a higher resolution than the current value. foreach ($previews as $preview) { @@ -313,7 +311,7 @@ class Generator { return $this->generateProviderPreview($file, $maxWidth, $maxHeight, false, true, $mimeType, $version); } - private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, int $version): Preview { + private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, ?string $version): Preview { $previewProviders = $this->previewManager->getProviders(); foreach ($previewProviders as $supportedMimeType => $providers) { // Filter out providers that does not support this mime @@ -353,14 +351,14 @@ class Generator { $previewEntry = new Preview(); $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); - $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); + $previewEntry->setSourceMimeType($file->getMimeType()); $previewEntry->setWidth($preview->width()); $previewEntry->setHeight($preview->height()); $previewEntry->setVersion($version); $previewEntry->setMax($max); $previewEntry->setCropped($crop); $previewEntry->setEncrypted(false); - $previewEntry->setMimetype($this->mimeTypeLoader->getId($preview->dataMimeType())); + $previewEntry->setMimetype($preview->dataMimeType()); $previewEntry->setEtag($file->getEtag()); $previewEntry->setMtime((new \DateTime())->getTimestamp()); $previewEntry->setSize(0); @@ -468,7 +466,7 @@ class Generator { bool $crop, int $maxWidth, int $maxHeight, - ?int $version, + ?string $version, bool $cacheResult, ): ISimpleFile { $preview = $maxPreview; @@ -508,21 +506,21 @@ class Generator { $previewEntry->setFileId($file->getId()); $previewEntry->setStorageId($file->getMountPoint()->getNumericStorageId()); $previewEntry->setWidth($width); - $previewEntry->setSourceMimetype($this->mimeTypeLoader->getId($file->getMimeType())); + $previewEntry->setSourceMimeType($file->getMimeType()); $previewEntry->setHeight($height); $previewEntry->setVersion($version); $previewEntry->setMax(false); $previewEntry->setCropped($crop); $previewEntry->setEncrypted(false); - $previewEntry->setMimetype($this->mimeTypeLoader->getId($preview->dataMimeType())); + $previewEntry->setMimeType($preview->dataMimeType()); $previewEntry->setEtag($file->getEtag()); $previewEntry->setMtime((new \DateTime())->getTimestamp()); $previewEntry->setSize(0); if ($cacheResult) { $previewEntry = $this->savePreview($previewEntry, $preview); - return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper, $this->mimeTypeLoader); + return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper); } else { - return new InMemoryFile($previewEntry->getName($this->mimeTypeLoader), $preview->data()); + return new InMemoryFile($previewEntry->getName(), $preview->data()); } } @@ -540,7 +538,10 @@ class Generator { if ($preview instanceof IStreamImage) { $size = $this->storageFactory->writePreview($previewEntry, $preview->resource()); } else { - $size = $this->storageFactory->writePreview($previewEntry, $preview->data()); + $stream = fopen('php://temp', 'w+'); + fwrite($stream, $preview->data()); + rewind($stream); + $size = $this->storageFactory->writePreview($previewEntry, $stream); } if (!$size) { throw new \RuntimeException('Unable to write preview file'); diff --git a/lib/private/Preview/PreviewService.php b/lib/private/Preview/PreviewService.php index 0e7c66dc12d..8d30ae8a402 100644 --- a/lib/private/Preview/PreviewService.php +++ b/lib/private/Preview/PreviewService.php @@ -70,12 +70,12 @@ class PreviewService { /** * @return \Generator */ - public function getAvailablePreviewForFile(int $fileId): \Generator { - return $this->previewMapper->getAvailablePreviewForFile($fileId); + public function getAvailablePreviewsForFile(int $fileId): \Generator { + return $this->previewMapper->getAvailablePreviewsForFile($fileId); } /** - * @param int[] $mimeTypes + * @param string[] $mimeTypes * @return \Generator */ public function getPreviewsForMimeTypes(array $mimeTypes): \Generator { diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php index 56464326838..1d6b128f8f0 100644 --- a/lib/private/Preview/Storage/IPreviewStorage.php +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -10,32 +10,44 @@ declare(strict_types=1); namespace OC\Preview\Storage; +use Exception; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; +use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; interface IPreviewStorage { /** - * @param resource|string $stream + * @param resource $stream * @throws NotPermittedException + * @throws NotFoundException */ - public function writePreview(Preview $preview, mixed $stream): false|int; + public function writePreview(Preview $preview, mixed $stream): int; /** * @param Preview $preview - * @return resource|false + * @return resource + * @throws NotPermittedException + * @throws NotFoundException */ public function readPreview(Preview $preview): mixed; + /** + * @throws NotPermittedException + */ public function deletePreview(Preview $preview): void; /** * Migration helper * * To remove at some point - * @throw \Exception + * @throws Exception */ public function migratePreview(Preview $preview, SimpleFile $file): void; + /** + * @throws NotPermittedException + * @throws NotFoundException + */ public function scan(): int; } diff --git a/lib/private/Preview/Storage/LocalPreviewStorage.php b/lib/private/Preview/Storage/LocalPreviewStorage.php index e522df7da8e..bd5e1a97818 100644 --- a/lib/private/Preview/Storage/LocalPreviewStorage.php +++ b/lib/private/Preview/Storage/LocalPreviewStorage.php @@ -17,7 +17,8 @@ use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OCP\DB\Exception; use OCP\Files\IMimeTypeDetector; -use OCP\Files\IMimeTypeLoader; +use OCP\Files\NotFoundException; +use OCP\Files\NotPermittedException; use OCP\IAppConfig; use OCP\IConfig; use OCP\IDBConnection; @@ -33,10 +34,8 @@ class LocalPreviewStorage implements IPreviewStorage { public function __construct( private readonly IConfig $config, private readonly PreviewMapper $previewMapper, - private readonly StorageFactory $previewStorage, private readonly IAppConfig $appConfig, private readonly IDBConnection $connection, - private readonly IMimeTypeLoader $mimeTypeLoader, private readonly IMimeTypeDetector $mimeTypeDetector, private readonly LoggerInterface $logger, ) { @@ -45,22 +44,28 @@ class LocalPreviewStorage implements IPreviewStorage { } #[Override] - public function writePreview(Preview $preview, mixed $stream): false|int { + public function writePreview(Preview $preview, mixed $stream): int { $previewPath = $this->constructPath($preview); - if (!$this->createParentFiles($previewPath)) { - return false; - } + $this->createParentFiles($previewPath); return file_put_contents($previewPath, $stream); } #[Override] public function readPreview(Preview $preview): mixed { - return @fopen($this->constructPath($preview), 'r'); + $previewPath = $this->constructPath($preview); + $resource = @fopen($previewPath, 'r'); + if ($resource === false) { + throw new NotFoundException('Unable to open preview stream at ' . $previewPath); + } + return $resource; } #[Override] public function deletePreview(Preview $preview): void { - @unlink($this->constructPath($preview)); + $previewPath = $this->constructPath($preview); + if (!@unlink($previewPath) && is_file($previewPath)) { + throw new NotPermittedException('Unable to delete preview at ' . $previewPath); + } } public function getPreviewRootFolder(): string { @@ -68,19 +73,21 @@ class LocalPreviewStorage implements IPreviewStorage { } private function constructPath(Preview $preview): string { - return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName($this->mimeTypeLoader); + return $this->getPreviewRootFolder() . implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); } - private function createParentFiles(string $path): bool { + private function createParentFiles(string $path): void { $dirname = dirname($path); @mkdir($dirname, recursive: true); - return is_dir($dirname); + if (!is_dir($dirname)) { + throw new NotPermittedException("Unable to create directory '$dirname'"); + } } #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { // legacy flat directory - $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName($this->mimeTypeLoader); + $sourcePath = $this->getPreviewRootFolder() . $preview->getFileId() . '/' . $preview->getName(); if (!file_exists($sourcePath)) { return; } @@ -106,7 +113,7 @@ class LocalPreviewStorage implements IPreviewStorage { $previewsFound = 0; foreach (new RecursiveIteratorIterator($scanner) as $file) { if ($file->isFile()) { - $preview = Preview::fromPath((string)$file, $this->mimeTypeDetector, $this->mimeTypeLoader); + $preview = Preview::fromPath((string)$file, $this->mimeTypeDetector); if ($preview === false) { $this->logger->error('Unable to parse preview information for ' . $file->getRealPath()); continue; diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 88470802667..3e4337fbf28 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -15,6 +15,7 @@ use OC\Files\ObjectStore\PrimaryObjectStoreConfig; use OC\Files\SimpleFS\SimpleFile; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; +use OCP\Files\NotPermittedException; use OCP\Files\ObjectStore\IObjectStore; use OCP\IConfig; use Override; @@ -41,15 +42,7 @@ class ObjectStorePreviewStorage implements IPreviewStorage { } #[Override] - public function writePreview(Preview $preview, mixed $stream): false|int { - if (!is_resource($stream)) { - $fh = fopen('php://temp', 'w+'); - fwrite($fh, $stream); - rewind($fh); - - $stream = $fh; - } - + public function writePreview(Preview $preview, mixed $stream): int { $size = 0; $countStream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size): void { $size = $writtenSize; @@ -61,7 +54,11 @@ class ObjectStorePreviewStorage implements IPreviewStorage { 'config' => $config, ] = $this->getObjectStoreForPreview($preview); - $store->writeObject($this->constructUrn($objectPrefix, $preview->getId()), $countStream); + try { + $store->writeObject($this->constructUrn($objectPrefix, $preview->getId()), $countStream); + } catch (\Exception $exception) { + throw new NotPermittedException('Unable to save preview to object store', previous: $exception); + } return $size; } @@ -71,7 +68,11 @@ class ObjectStorePreviewStorage implements IPreviewStorage { 'objectPrefix' => $objectPrefix, 'store' => $store, ] = $this->getObjectStoreForPreview($preview); - return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); + try { + return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); + } catch (\Exception $exception) { + throw new NotPermittedException('Unable to read preview from object store', previous: $exception); + } } #[Override] @@ -80,7 +81,11 @@ class ObjectStorePreviewStorage implements IPreviewStorage { 'objectPrefix' => $objectPrefix, 'store' => $store, ] = $this->getObjectStoreForPreview($preview); - $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); + try { + $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); + } catch (\Exception $exception) { + throw new NotPermittedException('Unable to delete preview from object store', previous: $exception); + } } #[Override] diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php index c7c7b59c97c..64cafb45470 100644 --- a/lib/private/Preview/Storage/PreviewFile.php +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -12,7 +12,6 @@ namespace OC\Preview\Storage; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; -use OCP\Files\IMimeTypeLoader; use OCP\Files\SimpleFS\ISimpleFile; use Override; @@ -21,13 +20,12 @@ class PreviewFile implements ISimpleFile { private readonly Preview $preview, private readonly IPreviewStorage $storage, private readonly PreviewMapper $previewMapper, - private readonly IMimeTypeLoader $mimeTypeLoader, ) { } #[Override] public function getName(): string { - return $this->preview->getName($this->mimeTypeLoader); + return $this->preview->getName(); } #[Override] @@ -63,12 +61,12 @@ class PreviewFile implements ISimpleFile { #[Override] public function getMimeType(): string { - return $this->preview->getMimetypeValue($this->mimeTypeLoader); + return $this->preview->getMimetype(); } #[Override] public function getExtension(): string { - return $this->preview->getExtension($this->mimeTypeLoader); + return $this->preview->getExtension(); } #[Override] diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php index e33135be3ce..b15031c6a12 100644 --- a/lib/private/Preview/Storage/StorageFactory.php +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -25,7 +25,7 @@ class StorageFactory implements IPreviewStorage { } #[Override] - public function writePreview(Preview $preview, mixed $stream): false|int { + public function writePreview(Preview $preview, mixed $stream): int { return $this->getBackend()->writePreview($preview, $stream); } diff --git a/lib/private/Preview/Watcher.php b/lib/private/Preview/Watcher.php index ea0f72796ae..9b95e87d2ed 100644 --- a/lib/private/Preview/Watcher.php +++ b/lib/private/Preview/Watcher.php @@ -13,6 +13,7 @@ use OC\Preview\Storage\StorageFactory; use OCP\Files\FileInfo; use OCP\Files\Folder; use OCP\Files\Node; +use OCP\IDBConnection; /** * Class Watcher @@ -26,8 +27,9 @@ class Watcher { * Watcher constructor. */ public function __construct( - readonly private StorageFactory $storageFactory, - readonly private PreviewMapper $previewMapper, + private readonly StorageFactory $storageFactory, + private readonly PreviewMapper $previewMapper, + private readonly IDBConnection $connection, ) { } @@ -47,8 +49,14 @@ class Watcher { } [$node->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$nodeId]); - foreach ($previews as $preview) { - $this->storageFactory->deletePreview($preview); + $this->connection->beginTransaction(); + try { + foreach ($previews as $preview) { + $this->storageFactory->deletePreview($preview); + $this->previewMapper->delete($preview); + } + } finally { + $this->connection->commit(); } } diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 480408d7724..fb884092872 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -16,7 +16,6 @@ use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\QueryException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\SimpleFS\ISimpleFile; @@ -140,7 +139,6 @@ class PreviewManager implements IPreview { $this->container->get(LoggerInterface::class), $this->container->get(PreviewMapper::class), $this->container->get(StorageFactory::class), - $this->container->get(IMimeTypeLoader::class), ); } return $this->generator; diff --git a/lib/private/Server.php b/lib/private/Server.php index 97483844333..927d2ce3224 100644 --- a/lib/private/Server.php +++ b/lib/private/Server.php @@ -308,6 +308,7 @@ class Server extends ServerContainer implements IServerContainer { return new Watcher( $c->get(\OC\Preview\Storage\StorageFactory::class), $c->get(PreviewMapper::class), + $c->get(IDBConnection::class), ); }); diff --git a/lib/public/Preview/IVersionedPreviewFile.php b/lib/public/Preview/IVersionedPreviewFile.php index 7d68fe8d15e..842ae877339 100644 --- a/lib/public/Preview/IVersionedPreviewFile.php +++ b/lib/public/Preview/IVersionedPreviewFile.php @@ -18,8 +18,7 @@ namespace OCP\Preview; */ interface IVersionedPreviewFile { /** - * @return numeric * @since 17.0.0 */ - public function getPreviewVersion(); + public function getPreviewVersion(): string; } diff --git a/tests/lib/Preview/BackgroundCleanupJobTest.php b/tests/lib/Preview/BackgroundCleanupJobTest.php index a2c72cbad57..80df690ad76 100644 --- a/tests/lib/Preview/BackgroundCleanupJobTest.php +++ b/tests/lib/Preview/BackgroundCleanupJobTest.php @@ -74,7 +74,7 @@ class BackgroundCleanupJobTest extends \Test\TestCase { $this->logout(); - foreach ($this->previewService->getAvailablePreviewForFile(5) as $preview) { + foreach ($this->previewService->getAvailablePreviewsForFile(5) as $preview) { $this->previewService->deletePreview($preview); } diff --git a/tests/lib/Preview/GeneratorTest.php b/tests/lib/Preview/GeneratorTest.php index fc48ebc181a..ceaf483a658 100644 --- a/tests/lib/Preview/GeneratorTest.php +++ b/tests/lib/Preview/GeneratorTest.php @@ -14,7 +14,6 @@ use OC\Preview\GeneratorHelper; use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; -use OCP\Files\IMimeTypeLoader; use OCP\Files\Mount\IMountPoint; use OCP\Files\NotFoundException; use OCP\IConfig; @@ -22,10 +21,17 @@ use OCP\IImage; use OCP\IPreview; use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IProviderV2; +use OCP\Preview\IVersionedPreviewFile; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\MockObject\MockObject; use Psr\Log\LoggerInterface; use Test\TestCase; +abstract class VersionedPreviewFile implements IVersionedPreviewFile, File { + +} + class GeneratorTest extends TestCase { private IConfig&MockObject $config; private IPreview&MockObject $previewManager; @@ -35,7 +41,6 @@ class GeneratorTest extends TestCase { private LoggerInterface&MockObject $logger; private StorageFactory&MockObject $storageFactory; private PreviewMapper&MockObject $previewMapper; - private IMimeTypeLoader&MockObject $mimeTypeLoader; protected function setUp(): void { parent::setUp(); @@ -47,12 +52,6 @@ class GeneratorTest extends TestCase { $this->logger = $this->createMock(LoggerInterface::class); $this->previewMapper = $this->createMock(PreviewMapper::class); $this->storageFactory = $this->createMock(StorageFactory::class); - $this->mimeTypeLoader = $this->createMock(IMimeTypeLoader::class); - $this->mimeTypeLoader->method('getId') - ->willReturnCallback(fn ($mimeType) => $mimeType === 'image/png' ? 42 : 43); - $this->mimeTypeLoader->method('getMimetypeById') - ->with(42) - ->willReturn('image/png'); $this->generator = new Generator( $this->config, @@ -62,14 +61,18 @@ class GeneratorTest extends TestCase { $this->logger, $this->previewMapper, $this->storageFactory, - $this->mimeTypeLoader, ); } - private function getFile(int $fileId, string $mimeType): File { + private function getFile(int $fileId, string $mimeType, bool $hasVersion = false): File { $mountPoint = $this->createMock(IMountPoint::class); $mountPoint->method('getNumericStorageId')->willReturn(42); - $file = $this->createMock(File::class); + if ($hasVersion) { + $file = $this->createMock(VersionedPreviewFile::class); + $file->method('getPreviewVersion')->willReturn('abc'); + } else { + $file = $this->createMock(File::class); + } $file->method('isReadable') ->willReturn(true); $file->method('getMimeType') @@ -81,8 +84,10 @@ class GeneratorTest extends TestCase { return $file; } - public function testGetCachedPreview(): void { - $file = $this->getFile(42, 'myMimeType'); + #[TestWith([true])] + #[TestWith([false])] + public function testGetCachedPreview(bool $hasPreview): void { + $file = $this->getFile(42, 'myMimeType', $hasPreview); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) @@ -93,20 +98,20 @@ class GeneratorTest extends TestCase { $maxPreview->setHeight(1000); $maxPreview->setMax(true); $maxPreview->setSize(1000); - $maxPreview->setVersion(-1); $maxPreview->setCropped(false); $maxPreview->setStorageId(1); - $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); + $maxPreview->setVersion($hasPreview ? 'abc' : null); + $maxPreview->setMimeType('image/png'); $previewFile = new Preview(); $previewFile->setWidth(256); $previewFile->setHeight(256); $previewFile->setMax(false); $previewFile->setSize(1000); - $previewFile->setVersion(-1); + $previewFile->setVersion($hasPreview ? 'abc' : null); $previewFile->setCropped(false); $previewFile->setStorageId(1); - $previewFile->setMimetype($this->mimeTypeLoader->getId('image/png')); + $previewFile->setMimeType('image/png'); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -120,12 +125,14 @@ class GeneratorTest extends TestCase { ->with(new BeforePreviewFetchedEvent($file, 100, 100, false, IPreview::MODE_FILL, null)); $result = $this->generator->getPreview($file, 100, 100); - $this->assertSame('256-256.png', $result->getName()); + $this->assertSame($hasPreview ? 'abc-256-256.png' : '256-256.png', $result->getName()); $this->assertSame(1000, $result->getSize()); } - public function testGetNewPreview(): void { - $file = $this->getFile(42, 'myMimeType'); + #[TestWith([true])] + #[TestWith([false])] + public function testGetNewPreview(bool $hasVersion): void { + $file = $this->getFile(42, 'myMimeType', $hasVersion); $this->previewManager->method('isMimeSupported') ->with($this->equalTo('myMimeType')) @@ -196,13 +203,6 @@ class GeneratorTest extends TestCase { $image->method('data') ->willReturn('my data'); - $maxPreview = new Preview(); - $maxPreview->setWidth(2048); - $maxPreview->setHeight(2048); - $maxPreview->setMax(true); - $maxPreview->setSize(1000); - $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); - $this->previewMapper->method('insert') ->willReturnCallback(fn (Preview $preview): Preview => $preview); @@ -210,16 +210,28 @@ class GeneratorTest extends TestCase { ->willReturnCallback(fn (Preview $preview): Preview => $preview); $this->storageFactory->method('writePreview') - ->willReturnCallback(function (Preview $preview, string $data): int { - switch ($preview->getName($this->mimeTypeLoader)) { - case '2048-2048-max.png': - $this->assertSame('my data', $data); - return 1000; - case '256-256.png': - $this->assertSame('my resized data', $data); - return 1000; + ->willReturnCallback(function (Preview $preview, mixed $data) use ($hasVersion): int { + $data = stream_get_contents($data); + if ($hasVersion) { + switch ($preview->getName()) { + case 'abc-2048-2048-max.png': + $this->assertSame('my data', $data); + return 1000; + case 'abc-256-256.png': + $this->assertSame('my resized data', $data); + return 1000; + } + } else { + switch ($preview->getName()) { + case '2048-2048-max.png': + $this->assertSame('my data', $data); + return 1000; + case '256-256.png': + $this->assertSame('my resized data', $data); + return 1000; + } } - $this->fail('file name is wrong:' . $preview->getName($this->mimeTypeLoader)); + $this->fail('file name is wrong:' . $preview->getName()); }); $image = $this->getMockImage(2048, 2048, 'my resized data'); @@ -231,7 +243,7 @@ class GeneratorTest extends TestCase { ->with(new BeforePreviewFetchedEvent($file, 100, 100, false, IPreview::MODE_FILL, null)); $result = $this->generator->getPreview($file, 100, 100); - $this->assertSame('256-256.png', $result->getName()); + $this->assertSame($hasVersion ? 'abc-256-256.png' : '256-256.png', $result->getName()); $this->assertSame(1000, $result->getSize()); } @@ -249,8 +261,8 @@ class GeneratorTest extends TestCase { $maxPreview->setHeight(2048); $maxPreview->setMax(true); $maxPreview->setSize(1000); - $maxPreview->setVersion(-1); - $maxPreview->setMimetype(42); + $maxPreview->setVersion(null); + $maxPreview->setMimetype('image/png'); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -273,8 +285,8 @@ class GeneratorTest extends TestCase { $maxPreview->setHeight(2048); $maxPreview->setMax(true); $maxPreview->setSize(1000); - $maxPreview->setVersion(-1); - $maxPreview->setMimetype($this->mimeTypeLoader->getId('image/png')); + $maxPreview->setVersion(null); + $maxPreview->setMimeType('image/png'); $previewFile = new Preview(); $previewFile->setWidth(1024); @@ -282,8 +294,8 @@ class GeneratorTest extends TestCase { $previewFile->setMax(false); $previewFile->setSize(1000); $previewFile->setCropped(true); - $previewFile->setVersion(-1); - $previewFile->setMimetype($this->mimeTypeLoader->getId('image/png')); + $previewFile->setVersion(null); + $previewFile->setMimeType('image/png'); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -321,7 +333,7 @@ class GeneratorTest extends TestCase { $this->generator->getPreview($file, 100, 100); } - private function getMockImage(int $width, int $height, $data = null) { + private function getMockImage(int $width, int $height, string $data = '') { $image = $this->createMock(IImage::class); $image->method('height')->willReturn($width); $image->method('width')->willReturn($height); @@ -378,7 +390,7 @@ class GeneratorTest extends TestCase { ]; } - #[\PHPUnit\Framework\Attributes\DataProvider('dataSize')] + #[DataProvider('dataSize')] public function testCorrectSize(int $maxX, int $maxY, int $reqX, int $reqY, bool $crop, string $mode, int $expectedX, int $expectedY): void { $file = $this->getFile(42, 'myMimeType'); @@ -391,11 +403,11 @@ class GeneratorTest extends TestCase { $maxPreview->setHeight($maxY); $maxPreview->setMax(true); $maxPreview->setSize(1000); - $maxPreview->setVersion(-1); - $maxPreview->setMimetype(42); + $maxPreview->setVersion(null); + $maxPreview->setMimeType('image/png'); - $this->assertSame($maxPreview->getName($this->mimeTypeLoader), $maxX . '-' . $maxY . '-max.png'); - $this->assertSame($maxPreview->getMimetypeValue($this->mimeTypeLoader), 'image/png'); + $this->assertSame($maxPreview->getName(), $maxX . '-' . $maxY . '-max.png'); + $this->assertSame($maxPreview->getMimeType(), 'image/png'); $this->previewMapper->method('getAvailablePreviews') ->with($this->equalTo([42])) @@ -415,7 +427,7 @@ class GeneratorTest extends TestCase { $this->previewMapper->method('insert') ->willReturnCallback(function (Preview $preview) use ($filename): Preview { - $this->assertSame($preview->getName($this->mimeTypeLoader), $filename); + $this->assertSame($preview->getName(), $filename); return $preview; }); @@ -431,7 +443,7 @@ class GeneratorTest extends TestCase { $result = $this->generator->getPreview($file, $reqX, $reqY, $crop, $mode); if ($expectedX === $maxX && $expectedY === $maxY) { - $this->assertSame($maxPreview->getName($this->mimeTypeLoader), $result->getName()); + $this->assertSame($maxPreview->getName(), $result->getName()); } else { $this->assertSame($filename, $result->getName()); } diff --git a/tests/lib/Preview/MovePreviewJobTest.php b/tests/lib/Preview/MovePreviewJobTest.php index a5ac5ad51e3..8c9df4274f0 100644 --- a/tests/lib/Preview/MovePreviewJobTest.php +++ b/tests/lib/Preview/MovePreviewJobTest.php @@ -21,6 +21,7 @@ use OCP\Files\IMimeTypeDetector; use OCP\Files\IMimeTypeLoader; use OCP\Files\IRootFolder; use OCP\IAppConfig; +use OCP\IConfig; use OCP\IDBConnection; use OCP\Server; use PHPUnit\Framework\Attributes\TestDox; @@ -35,6 +36,7 @@ class MovePreviewJobTest extends TestCase { private IAppData $previewAppData; private PreviewMapper $previewMapper; private IAppConfig&MockObject $appConfig; + private IConfig $config; private StorageFactory $storageFactory; private PreviewService $previewService; private IDBConnection $db; @@ -46,6 +48,7 @@ class MovePreviewJobTest extends TestCase { parent::setUp(); $this->previewAppData = Server::get(IAppDataFactory::class)->get('preview'); $this->previewMapper = Server::get(PreviewMapper::class); + $this->config = Server::get(IConfig::class); $this->appConfig = $this->createMock(IAppConfig::class); $this->appConfig->expects($this->any()) ->method('getValueBool') @@ -71,7 +74,7 @@ class MovePreviewJobTest extends TestCase { 'path_hash' => $qb->createNamedParameter(md5('test')), 'parent' => $qb->createNamedParameter(0), 'name' => $qb->createNamedParameter('abc'), - 'mimetype' => $qb->createNamedParameter(0), + 'mimetype' => $qb->createNamedParameter(42), 'size' => $qb->createNamedParameter(1000), 'mtime' => $qb->createNamedParameter(1000), 'storage_mtime' => $qb->createNamedParameter(1000), @@ -86,6 +89,7 @@ class MovePreviewJobTest extends TestCase { $this->mimeTypeDetector->method('detectPath')->willReturn('image/png'); $this->mimeTypeLoader = $this->createMock(IMimeTypeLoader::class); $this->mimeTypeLoader->method('getId')->with('image/png')->willReturn(42); + $this->mimeTypeLoader->method('getMimetypeById')->with(42)->willReturn('image/png'); $this->logger = $this->createMock(LoggerInterface::class); } @@ -108,11 +112,12 @@ class MovePreviewJobTest extends TestCase { $folder->newFile('128-128-crop.png', 'abcdefg'); $this->assertEquals(1, count($this->previewAppData->getDirectoryListing())); $this->assertEquals(2, count($folder->getDirectoryListing())); - $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); $job = new MovePreviewJob( Server::get(ITimeFactory::class), $this->appConfig, + $this->config, $this->previewMapper, $this->storageFactory, Server::get(IDBConnection::class), @@ -124,7 +129,7 @@ class MovePreviewJobTest extends TestCase { ); $this->invokePrivate($job, 'run', [[]]); $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); - $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); } private static function getInternalFolder(string $name): string { @@ -139,11 +144,12 @@ class MovePreviewJobTest extends TestCase { $folder = $this->previewAppData->getFolder(self::getInternalFolder((string)5)); $this->assertEquals(2, count($folder->getDirectoryListing())); - $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); $job = new MovePreviewJob( Server::get(ITimeFactory::class), $this->appConfig, + $this->config, $this->previewMapper, $this->storageFactory, Server::get(IDBConnection::class), @@ -155,7 +161,7 @@ class MovePreviewJobTest extends TestCase { ); $this->invokePrivate($job, 'run', [[]]); $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); - $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + $this->assertEquals(2, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); } #[TestDox("Test the migration from the 'new' nested hierarchy to the database format")] @@ -178,11 +184,12 @@ class MovePreviewJobTest extends TestCase { $folder = $this->previewAppData->getFolder(self::getInternalFolder((string)5)); $this->assertEquals(9, count($folder->getDirectoryListing())); - $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)))); + $this->assertEquals(0, count(iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)))); $job = new MovePreviewJob( Server::get(ITimeFactory::class), $this->appConfig, + $this->config, $this->previewMapper, $this->storageFactory, Server::get(IDBConnection::class), @@ -193,25 +200,25 @@ class MovePreviewJobTest extends TestCase { Server::get(IAppDataFactory::class) ); $this->invokePrivate($job, 'run', [[]]); - $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); - $previews = iterator_to_array($this->previewMapper->getAvailablePreviewForFile(5)); + $previews = iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5)); $this->assertEquals(9, count($previews)); + $this->assertEquals(0, count($this->previewAppData->getDirectoryListing())); $nameVersionMapping = []; foreach ($previews as $preview) { - $nameVersionMapping[$preview->getName()] = $preview->getVersion(); + $nameVersionMapping[$preview->getName($this->mimeTypeLoader)] = $preview->getVersion(); } $this->assertEquals([ - '1000-128-128.png' => 1000, '1000-128-128-crop.png' => 1000, + '1000-128-128.png' => 1000, '1000-256-256-max.png' => 1000, - '1001-128-128.png' => 1001, '1001-128-128-crop.png' => 1001, + '1001-128-128.png' => 1001, '1001-256-256-max.png' => 1001, - '128-128.png' => -1, - '128-128-crop.png' => -1, - '256-256-max.png' => -1, + '128-128-crop.png' => null, + '128-128.png' => null, + '256-256-max.png' => null, ], $nameVersionMapping); } } diff --git a/tests/lib/Preview/PreviewMapperTest.php b/tests/lib/Preview/PreviewMapperTest.php index 19018c8b318..8e27a642473 100644 --- a/tests/lib/Preview/PreviewMapperTest.php +++ b/tests/lib/Preview/PreviewMapperTest.php @@ -12,9 +12,7 @@ namespace Test\Preview; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; -use OCP\Files\IMimeTypeLoader; use OCP\IDBConnection; -use OCP\IPreview; use OCP\Server; use Test\TestCase; @@ -24,12 +22,10 @@ use Test\TestCase; class PreviewMapperTest extends TestCase { private PreviewMapper $previewMapper; private IDBConnection $connection; - private IMimeTypeLoader $mimeTypeLoader; public function setUp(): void { $this->previewMapper = Server::get(PreviewMapper::class); $this->connection = Server::get(IDBConnection::class); - $this->mimeTypeLoader = Server::get(IMimeTypeLoader::class); } public function testGetAvailablePreviews(): void { @@ -71,11 +67,11 @@ class PreviewMapperTest extends TestCase { $preview->setCropped(true); $preview->setMax(true); $preview->setWidth(100); - $preview->setSourceMimetype(1); + $preview->setSourceMimeType('image/jpeg'); $preview->setHeight(100); $preview->setSize(100); $preview->setMtime(time()); - $preview->setMimetype($this->mimeTypeLoader->getId('image/jpeg')); + $preview->setMimetype('image/jpeg'); $preview->setEtag('abcdefg'); if ($locationId !== null) { diff --git a/tests/lib/Preview/PreviewServiceTest.php b/tests/lib/Preview/PreviewServiceTest.php index fe8dd1c3d33..f3f9c8ae895 100644 --- a/tests/lib/Preview/PreviewServiceTest.php +++ b/tests/lib/Preview/PreviewServiceTest.php @@ -13,7 +13,6 @@ namespace Test\Preview; use OC\Preview\Db\Preview; use OC\Preview\Db\PreviewMapper; use OC\Preview\PreviewService; -use OCP\IPreview; use OCP\Server; use PHPUnit\Framework\Attributes\CoversClass; use PHPUnit\Framework\TestCase; @@ -24,6 +23,7 @@ use PHPUnit\Framework\TestCase; #[CoversClass(PreviewService::class)] class PreviewServiceTest extends TestCase { private PreviewService $previewService; + private PreviewMapper $previewMapper; protected function setUp(): void { $this->previewService = Server::get(PreviewService::class); @@ -43,10 +43,10 @@ class PreviewServiceTest extends TestCase { $preview->setWidth($i); $preview->setHeight($i); $preview->setMax(true); - $preview->setSourceMimetype(1); + $preview->setSourceMimeType('image/jpeg'); $preview->setCropped(true); $preview->setEncrypted(false); - $preview->setMimetype(42); + $preview->setMimetype('image/jpeg'); $preview->setEtag('abc'); $preview->setMtime((new \DateTime())->getTimestamp()); $preview->setSize(0); From fed7a33d1f4567ed9fa4dc4ceb55af28d6e4ecf5 Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Mon, 6 Oct 2025 17:49:05 +0200 Subject: [PATCH 14/14] refactor(preview-object-store): Refactor object store backend Simplify logic Signed-off-by: Carl Schwan --- .../Storage/ObjectStorePreviewStorage.php | 124 ++++++++++-------- 1 file changed, 68 insertions(+), 56 deletions(-) diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php index 3e4337fbf28..0c94071d28e 100644 --- a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -22,12 +22,12 @@ use Override; /** * @psalm-import-type ObjectStoreConfig from PrimaryObjectStoreConfig - * @psalm-type ObjectStoreDefinition = array{store: IObjectStore, objectPrefix: string, config?: ObjectStoreConfig} + * @psalm-type ObjectStoreDefinition = array{store: IObjectStore, urn: string} */ class ObjectStorePreviewStorage implements IPreviewStorage { /** - * @var array> + * @var array> */ private array $objectStoreCache = []; @@ -49,13 +49,12 @@ class ObjectStorePreviewStorage implements IPreviewStorage { }); [ - 'objectPrefix' => $objectPrefix, + 'urn' => $urn, 'store' => $store, - 'config' => $config, - ] = $this->getObjectStoreForPreview($preview); + ] = $this->getObjectStoreInfoForNewPreview($preview); try { - $store->writeObject($this->constructUrn($objectPrefix, $preview->getId()), $countStream); + $store->writeObject($urn, $countStream); } catch (\Exception $exception) { throw new NotPermittedException('Unable to save preview to object store', previous: $exception); } @@ -65,24 +64,26 @@ class ObjectStorePreviewStorage implements IPreviewStorage { #[Override] public function readPreview(Preview $preview): mixed { [ - 'objectPrefix' => $objectPrefix, + 'urn' => $urn, 'store' => $store, - ] = $this->getObjectStoreForPreview($preview); + ] = $this->getObjectStoreInfoForExistingPreview($preview); + try { - return $store->readObject($this->constructUrn($objectPrefix, $preview->getId())); + return $store->readObject($urn); } catch (\Exception $exception) { - throw new NotPermittedException('Unable to read preview from object store', previous: $exception); + throw new NotPermittedException('Unable to read preview from object store with urn:' . $urn, previous: $exception); } } #[Override] public function deletePreview(Preview $preview): void { [ - 'objectPrefix' => $objectPrefix, + 'urn' => $urn, 'store' => $store, - ] = $this->getObjectStoreForPreview($preview); + ] = $this->getObjectStoreInfoForExistingPreview($preview); + try { - $store->deleteObject($this->constructUrn($objectPrefix, $preview->getId())); + $store->deleteObject($urn); } catch (\Exception $exception) { throw new NotPermittedException('Unable to delete preview from object store', previous: $exception); } @@ -91,72 +92,83 @@ class ObjectStorePreviewStorage implements IPreviewStorage { #[Override] public function migratePreview(Preview $preview, SimpleFile $file): void { // Just set the Preview::bucket and Preview::objectStore - $this->getObjectStoreForPreview($preview, true); + $this->getObjectStoreInfoForNewPreview($preview, migration: true); $this->previewMapper->update($preview); } /** * @return ObjectStoreDefinition */ - private function getObjectStoreForPreview(Preview $preview, bool $oldFallback = false): array { - if ($preview->getObjectStoreName() === null) { - $config = $this->objectStoreConfig->getObjectStoreConfiguration($oldFallback ? 'root' : 'preview'); - $objectStoreName = $this->objectStoreConfig->resolveAlias($oldFallback ? 'root' : 'preview'); + private function getObjectStoreInfoForExistingPreview(Preview $preview): array { + assert(!empty($preview->getObjectStoreName())); + assert(!empty($preview->getBucketName())); - $bucketName = $config['arguments']['bucket']; - if ($config['arguments']['multibucket']) { - if ($this->isMultibucketPreviewDistributionEnabled) { - $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); - $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); - $bucketName .= '-preview-' . $bucketNumber; - } else { - $bucketName .= '0'; - } + $config = $this->objectStoreConfig->getObjectStoreConfiguration($preview->getObjectStoreName()); + $config['arguments']['bucket'] = $preview->getBucketName(); + $objectStoreName = $preview->getObjectStoreName(); + + return [ + 'urn' => $this->getUrn($preview, $config), + 'store' => $this->getObjectStore($objectStoreName, $config), + ]; + } + + /** + * @return ObjectStoreDefinition + */ + private function getObjectStoreInfoForNewPreview(Preview $preview, bool $migration = false): array { + // When migrating old previews, use the 'root' object store configuration + $config = $this->objectStoreConfig->getObjectStoreConfiguration($migration ? 'root' : 'preview'); + $objectStoreName = $this->objectStoreConfig->resolveAlias($migration ? 'root' : 'preview'); + + $bucketName = $config['arguments']['bucket']; + if ($config['arguments']['multibucket']) { + if ($this->isMultibucketPreviewDistributionEnabled) { + // Spread the previews on different buckets depending on their corresponding fileId + $oldLocationArray = str_split(substr(md5((string)$preview->getFileId()), 0, 2)); + $bucketNumber = hexdec('0x' . $oldLocationArray[0]) * 16 + hexdec('0x' . $oldLocationArray[0]); + $bucketName .= '-preview-' . $bucketNumber; + } else { + // Put all previews in the root (0) bucket + $bucketName .= '0'; } - $config['arguments']['bucket'] = $bucketName; - - $locationId = $this->previewMapper->getLocationId($bucketName, $objectStoreName); - $preview->setLocationId($locationId); - $preview->setObjectStoreName($objectStoreName); - $preview->setBucketName($bucketName); - } else { - $config = $this->objectStoreConfig->getObjectStoreConfiguration($preview->getObjectStoreName()); - $config['arguments']['bucket'] = $bucketName = $preview->getBucketName(); - $objectStoreName = $preview->getObjectStoreName(); } + $config['arguments']['bucket'] = $bucketName; - $objectPrefix = $this->getObjectPrefix($preview, $config); + // Get the locationId corresponding to the bucketName and objectStoreName, this will create + // a new one, if no matching location is found in the DB. + $locationId = $this->previewMapper->getLocationId($bucketName, $objectStoreName); + $preview->setLocationId($locationId); + $preview->setObjectStoreName($objectStoreName); + $preview->setBucketName($bucketName); + + return [ + 'urn' => $this->getUrn($preview, $config), + 'store' => $this->getObjectStore($objectStoreName, $config), + ]; + } + + private function getObjectStore(string $objectStoreName, array $config): IObjectStore { + $bucketName = $config['arguments']['bucket']; if (!isset($this->objectStoreCache[$objectStoreName])) { $this->objectStoreCache[$objectStoreName] = []; - $this->objectStoreCache[$objectStoreName][$bucketName] = [ - 'store' => $this->objectStoreConfig->buildObjectStore($config), - 'objectPrefix' => $objectPrefix, - 'config' => $config, - ]; + $this->objectStoreCache[$objectStoreName][$bucketName] = $this->objectStoreConfig->buildObjectStore($config); } elseif (!isset($this->objectStoreCache[$objectStoreName][$bucketName])) { - $this->objectStoreCache[$objectStoreName][$bucketName] = [ - 'store' => $this->objectStoreConfig->buildObjectStore($config), - 'objectPrefix' => $objectPrefix, - 'config' => $config, - ]; + $this->objectStoreCache[$objectStoreName][$bucketName] = $this->objectStoreConfig->buildObjectStore($config); } return $this->objectStoreCache[$objectStoreName][$bucketName]; } - private function constructUrn(string $objectPrefix, int $id): string { - return $objectPrefix . $id; - } - - public function getObjectPrefix(Preview $preview, array $config): string { + public function getUrn(Preview $preview, array $config): string { if ($preview->getOldFileId()) { - return $config['arguments']['objectPrefix'] ?? 'uri:oid:'; + return ($config['arguments']['objectPrefix'] ?? 'urn:oid:') . $preview->getOldFileId(); } if (isset($config['arguments']['objectPrefix'])) { - return $config['arguments']['objectPrefix'] . 'preview:'; + return ($config['arguments']['objectPrefix'] . 'preview:') . $preview->getId(); } else { - return 'uri:oid:preview:'; + return 'uri:oid:preview:' . $preview->getId(); } }