From 18fbacdd8d519e88e8cc53438a6209428f90085d Mon Sep 17 00:00:00 2001 From: Carl Schwan Date: Wed, 20 Aug 2025 17:34:07 +0200 Subject: [PATCH] perf(preview): Split preview data to new table The new oc_previews table is optimized for storing previews and should decrease significantly the space taken by previews in the filecache table. This attend to reuse the IObjectStore abstraction over S3/Swift/Azure but currently only support one single bucket configuration. Signed-off-by: Carl Schwan --- .../Version33000Date20250819110529.php | 48 ++++ lib/composer/composer/LICENSE | 2 - lib/composer/composer/autoload_classmap.php | 8 + lib/composer/composer/autoload_static.php | 8 + .../ObjectStore/PrimaryObjectStoreConfig.php | 2 +- lib/private/Preview/Db/Preview.php | 105 ++++++++ lib/private/Preview/Db/PreviewMapper.php | 66 +++++ lib/private/Preview/Generator.php | 226 ++++++++---------- .../Preview/Storage/IPreviewStorage.php | 22 ++ .../Preview/Storage/LocalPreviewStorage.php | 43 ++++ .../Storage/ObjectStorePreviewStorage.php | 50 ++++ lib/private/Preview/Storage/PreviewFile.php | 90 +++++++ .../Preview/Storage/StorageFactory.php | 44 ++++ lib/private/PreviewManager.php | 4 + lib/public/IPreview.php | 5 + 15 files changed, 596 insertions(+), 127 deletions(-) create mode 100644 core/Migrations/Version33000Date20250819110529.php create mode 100644 lib/private/Preview/Db/Preview.php create mode 100644 lib/private/Preview/Db/PreviewMapper.php create mode 100644 lib/private/Preview/Storage/IPreviewStorage.php create mode 100644 lib/private/Preview/Storage/LocalPreviewStorage.php create mode 100644 lib/private/Preview/Storage/ObjectStorePreviewStorage.php create mode 100644 lib/private/Preview/Storage/PreviewFile.php create mode 100644 lib/private/Preview/Storage/StorageFactory.php diff --git a/core/Migrations/Version33000Date20250819110529.php b/core/Migrations/Version33000Date20250819110529.php new file mode 100644 index 00000000000..fcc008b4283 --- /dev/null +++ b/core/Migrations/Version33000Date20250819110529.php @@ -0,0 +1,48 @@ +hasTable('previews')) { + $table = $schema->createTable('previews'); + $table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]); + $table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]); + $table->addColumn('is_max', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('crop', Types::BOOLEAN, ['notnull' => true, 'default' => false]); + $table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40]); + $table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]); + $table->addColumn('version', Types::BIGINT, ['notnull' => false, 'unsigned' => true]); + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx'); + } + + return $schema; + } +} diff --git a/lib/composer/composer/LICENSE b/lib/composer/composer/LICENSE index f27399a042d..62ecfd8d004 100644 --- a/lib/composer/composer/LICENSE +++ b/lib/composer/composer/LICENSE @@ -1,4 +1,3 @@ - Copyright (c) Nils Adermann, Jordi Boggiano Permission is hereby granted, free of charge, to any person obtaining a copy @@ -18,4 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index ee77fbd4cda..80191ae4b7f 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1528,6 +1528,7 @@ return array( 'OC\\Core\\Migrations\\Version32000Date20250620081925' => $baseDir . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110523' => $baseDir . '/core/Migrations/Version33000Date20250819110523.php', 'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php', @@ -1880,6 +1881,8 @@ return array( 'OC\\Preview\\BackgroundCleanupJob' => $baseDir . '/lib/private/Preview/BackgroundCleanupJob.php', 'OC\\Preview\\Bitmap' => $baseDir . '/lib/private/Preview/Bitmap.php', 'OC\\Preview\\Bundled' => $baseDir . '/lib/private/Preview/Bundled.php', + 'OC\\Preview\\Db\\Preview' => $baseDir . '/lib/private/Preview/Db/Preview.php', + 'OC\\Preview\\Db\\PreviewMapper' => $baseDir . '/lib/private/Preview/Db/PreviewMapper.php', 'OC\\Preview\\EMF' => $baseDir . '/lib/private/Preview/EMF.php', 'OC\\Preview\\Font' => $baseDir . '/lib/private/Preview/Font.php', 'OC\\Preview\\GIF' => $baseDir . '/lib/private/Preview/GIF.php', @@ -1912,7 +1915,12 @@ return array( 'OC\\Preview\\SGI' => $baseDir . '/lib/private/Preview/SGI.php', 'OC\\Preview\\SVG' => $baseDir . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => $baseDir . '/lib/private/Preview/StarOffice.php', + 'OC\\Preview\\Storage\\IPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/IPreviewStorage.php', + 'OC\\Preview\\Storage\\LocalPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/LocalPreviewStorage.php', + 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => $baseDir . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', + 'OC\\Preview\\Storage\\PreviewFile' => $baseDir . '/lib/private/Preview/Storage/PreviewFile.php', 'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php', + 'OC\\Preview\\Storage\\StorageFactory' => $baseDir . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => $baseDir . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => $baseDir . '/lib/private/Preview/TXT.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index 3b18f00da96..923cf1d5f0b 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1569,6 +1569,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Core\\Migrations\\Version32000Date20250620081925' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250620081925.php', 'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php', 'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php', + 'OC\\Core\\Migrations\\Version33000Date20250819110523' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110523.php', 'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php', 'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php', 'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php', @@ -1921,6 +1922,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\BackgroundCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Preview/BackgroundCleanupJob.php', 'OC\\Preview\\Bitmap' => __DIR__ . '/../../..' . '/lib/private/Preview/Bitmap.php', 'OC\\Preview\\Bundled' => __DIR__ . '/../../..' . '/lib/private/Preview/Bundled.php', + 'OC\\Preview\\Db\\Preview' => __DIR__ . '/../../..' . '/lib/private/Preview/Db/Preview.php', + 'OC\\Preview\\Db\\PreviewMapper' => __DIR__ . '/../../..' . '/lib/private/Preview/Db/PreviewMapper.php', 'OC\\Preview\\EMF' => __DIR__ . '/../../..' . '/lib/private/Preview/EMF.php', 'OC\\Preview\\Font' => __DIR__ . '/../../..' . '/lib/private/Preview/Font.php', 'OC\\Preview\\GIF' => __DIR__ . '/../../..' . '/lib/private/Preview/GIF.php', @@ -1953,7 +1956,12 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OC\\Preview\\SGI' => __DIR__ . '/../../..' . '/lib/private/Preview/SGI.php', 'OC\\Preview\\SVG' => __DIR__ . '/../../..' . '/lib/private/Preview/SVG.php', 'OC\\Preview\\StarOffice' => __DIR__ . '/../../..' . '/lib/private/Preview/StarOffice.php', + 'OC\\Preview\\Storage\\IPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/IPreviewStorage.php', + 'OC\\Preview\\Storage\\LocalPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/LocalPreviewStorage.php', + 'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php', + 'OC\\Preview\\Storage\\PreviewFile' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/PreviewFile.php', 'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php', + 'OC\\Preview\\Storage\\StorageFactory' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/StorageFactory.php', 'OC\\Preview\\TGA' => __DIR__ . '/../../..' . '/lib/private/Preview/TGA.php', 'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php', 'OC\\Preview\\TXT' => __DIR__ . '/../../..' . '/lib/private/Preview/TXT.php', diff --git a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php index 008431b3fbf..4a31d2e51f5 100644 --- a/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php +++ b/lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php @@ -14,7 +14,7 @@ use OCP\IConfig; use OCP\IUser; /** - * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, ...}} + * @psalm-type ObjectStoreConfig array{class: class-string, arguments: array{multibucket: bool, objectPrefix: ?string, ...}} */ class PrimaryObjectStoreConfig { public function __construct( diff --git a/lib/private/Preview/Db/Preview.php b/lib/private/Preview/Db/Preview.php new file mode 100644 index 00000000000..f2f05871082 --- /dev/null +++ b/lib/private/Preview/Db/Preview.php @@ -0,0 +1,105 @@ +addType('fileId', Types::INTEGER); + $this->addType('width', Types::INTEGER); + $this->addType('height', Types::INTEGER); + $this->addType('mimetype', Types::INTEGER); + $this->addType('mtime', Types::INTEGER); + $this->addType('size', Types::INTEGER); + $this->addType('isMax', Types::BOOLEAN); + $this->addType('crop', Types::BOOLEAN); + $this->addType('etag', Types::STRING); + $this->addType('version', Types::INTEGER); + } + + public function getName(): string { + $path = ($this->getVersion() ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight(); + if ($this->getCrop()) { + $path .= '-crop'; + } + if ($this->getIsMax()) { + $path .= '-max'; + } + + $ext = $this->getExtension(); + $path .= '.' . $ext; + return $path; + } + + public function getMimetypeValue(): string { + return match ($this->mimetype) { + IPreview::MIMETYPE_JPEG => 'image/jpeg', + IPreview::MIMETYPE_PNG => 'image/png', + IPreview::MIMETYPE_WEBP => 'image/webp', + IPreview::MIMETYPE_GIF => 'image/gif', + }; + } + + public function getExtension(): string { + return match ($this->mimetype) { + IPreview::MIMETYPE_JPEG => 'jpeg', + IPreview::MIMETYPE_PNG => 'png', + IPreview::MIMETYPE_WEBP => 'webp', + IPreview::MIMETYPE_GIF => 'gif', + }; + } +} diff --git a/lib/private/Preview/Db/PreviewMapper.php b/lib/private/Preview/Db/PreviewMapper.php new file mode 100644 index 00000000000..5faa508c8e0 --- /dev/null +++ b/lib/private/Preview/Db/PreviewMapper.php @@ -0,0 +1,66 @@ + + */ +class PreviewMapper extends QBMapper { + + private const TABLE_NAME = 'previews'; + + public function __construct(IDBConnection $db) { + parent::__construct($db, self::TABLE_NAME, Preview::class); + } + + /** + * @param int[] $fileIds + * @return array + * @throws Exception + */ + public function getAvailablePreviews(array $fileIds): array { + $selectQb = $this->db->getQueryBuilder(); + $selectQb->select('*') + ->from(self::TABLE_NAME) + ->where( + $selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)), + ); + $previews = array_fill_keys($fileIds, []); + foreach ($this->yieldEntities($selectQb) as $preview) { + $previews[$preview->getFileId()][] = $preview; + } + return $previews; + } + + public function getPreview(int $fileId, int $width, int $height, string $mode, int $mimetype = IPreview::MIMETYPE_JPEG): ?Preview { + $selectQb = $this->db->getQueryBuilder(); + $selectQb->select('*') + ->from(self::TABLE_NAME) + ->where( + $selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId)), + $selectQb->expr()->eq('width', $selectQb->createNamedParameter($width)), + $selectQb->expr()->eq('height', $selectQb->createNamedParameter($height)), + $selectQb->expr()->eq('mode', $selectQb->createNamedParameter($mode)), + $selectQb->expr()->eq('mimetype', $selectQb->createNamedParameter($mimetype)), + ); + try { + return $this->findEntity($selectQb); + } catch (DoesNotExistException) { + return null; + } + } +} diff --git a/lib/private/Preview/Generator.php b/lib/private/Preview/Generator.php index 4a7341896ef..39e1de54e69 100644 --- a/lib/private/Preview/Generator.php +++ b/lib/private/Preview/Generator.php @@ -6,6 +6,10 @@ */ namespace OC\Preview; +use OC\Preview\Db\Preview; +use OC\Preview\Db\PreviewMapper; +use OC\Preview\Storage\PreviewFile; +use OC\Preview\Storage\StorageFactory; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; use OCP\Files\IAppData; @@ -23,6 +27,7 @@ use OCP\Preview\BeforePreviewFetchedEvent; use OCP\Preview\IProviderV2; use OCP\Preview\IVersionedPreviewFile; use Psr\Log\LoggerInterface; +use function Symfony\Component\Translation\t; class Generator { public const SEMAPHORE_ID_ALL = 0x0a11; @@ -35,6 +40,8 @@ class Generator { private GeneratorHelper $helper, private IEventDispatcher $eventDispatcher, private LoggerInterface $logger, + private PreviewMapper $previewMapper, + private StorageFactory $storageFactory, ) { } @@ -104,25 +111,25 @@ class Generator { $mimeType = $file->getMimeType(); } - $previewFolder = $this->getPreviewFolder($file); - // List every existing preview first instead of trying to find them one by one - $previewFiles = $previewFolder->getDirectoryListing(); + [$file->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$file->getId()]); - $previewVersion = ''; + $previewVersion = null; if ($file instanceof IVersionedPreviewFile) { - $previewVersion = $file->getPreviewVersion() . '-'; + $previewVersion = (int)$file->getPreviewVersion(); } // Get the max preview and infer the max preview sizes from that - $maxPreview = $this->getMaxPreview($previewFolder, $previewFiles, $file, $mimeType, $previewVersion); + $maxPreview = $this->getMaxPreview($previews, $file, $mimeType, $previewVersion); $maxPreviewImage = null; // only load the image when we need it if ($maxPreview->getSize() === 0) { - $maxPreview->delete(); + $this->storageFactory->deletePreview($maxPreview); + $this->previewMapper->delete($maxPreview); $this->logger->error('Max preview generated for file {path} has size 0, deleting and throwing exception.', ['path' => $file->getPath()]); throw new NotFoundException('Max preview size 0, invalid!'); } - [$maxWidth, $maxHeight] = $this->getPreviewSize($maxPreview, $previewVersion); + $maxWidth = $maxPreview->getWidth(); + $maxHeight = $maxPreview->getHeight(); if ($maxWidth <= 0 || $maxHeight <= 0) { throw new NotFoundException('The maximum preview sizes are zero or less pixels'); @@ -154,32 +161,40 @@ class Generator { // Try to get a cached preview. Else generate (and store) one try { - try { - $preview = $this->getCachedPreview($previewFiles, $width, $height, $crop, $maxPreview->getMimeType(), $previewVersion); - } catch (NotFoundException $e) { + /** @var ISimpleFile $previewFile */ + $previewFile = null; + // TODO(php8.4) replace by array_find + foreach ($previews as $p) { + if ($p->getWidth() === $width && $p->getHeight() === $height && $p->getMimetype() === $maxPreview->getMimetype() && $p->getVersion() === $previewVersion && $p->getCrop() === $crop) { + $previewFile = new PreviewFile($p, $this->storageFactory, $this->previewMapper); + break; + } + } + + if ($previewFile === null) { if (!$this->previewManager->isMimeSupported($mimeType)) { throw new NotFoundException(); } if ($maxPreviewImage === null) { - $maxPreviewImage = $this->helper->getImage($maxPreview); + $maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper)); } $this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]); - $preview = $this->generatePreview($previewFolder, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); + $preview = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult); // New file, augment our array - $previewFiles[] = $preview; + //$previews[] = $preview; } } catch (\InvalidArgumentException $e) { throw new NotFoundException('', 0, $e); } - if ($preview->getSize() === 0) { - $preview->delete(); + if ($previewFile->getSize() === 0) { + $previewFile->delete(); throw new NotFoundException('Cached preview size 0, invalid!'); } } - assert($preview !== null); + assert($previewFile !== null); // Free memory being used by the embedded image resource. Without this the image is kept in memory indefinitely. // Garbage Collection does NOT free this memory. We have to do it ourselves. @@ -187,7 +202,7 @@ class Generator { $maxPreviewImage->destroy(); } - return $preview; + return $previewFile; } /** @@ -289,31 +304,25 @@ class Generator { } /** - * @param ISimpleFolder $previewFolder - * @param ISimpleFile[] $previewFiles - * @param File $file - * @param string $mimeType - * @param string $prefix - * @return ISimpleFile + * @param Preview[] $previews * @throws NotFoundException */ - private function getMaxPreview(ISimpleFolder $previewFolder, array $previewFiles, File $file, $mimeType, $prefix) { + private function getMaxPreview(array $previews, File $file, string $mimeType, ?int $version): Preview { // We don't know the max preview size, so we can't use getCachedPreview. // It might have been generated with a higher resolution than the current value. - foreach ($previewFiles as $node) { - $name = $node->getName(); - if (($prefix === '' || str_starts_with($name, $prefix)) && strpos($name, 'max')) { - return $node; + foreach ($previews as $preview) { + if ($preview->getIsMax() && ($version == $preview->getVersion())) { + return $preview; } } $maxWidth = $this->config->getSystemValueInt('preview_max_x', 4096); $maxHeight = $this->config->getSystemValueInt('preview_max_y', 4096); - return $this->generateProviderPreview($previewFolder, $file, $maxWidth, $maxHeight, false, true, $mimeType, $prefix); + return $this->generateProviderPreview($file, $maxWidth, $maxHeight, false, true, $mimeType, $version); } - private function generateProviderPreview(ISimpleFolder $previewFolder, File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, string $prefix) { + private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): Preview { $previewProviders = $this->previewManager->getProviders(); foreach ($previewProviders as $supportedMimeType => $providers) { // Filter out providers that does not support this mime @@ -348,45 +357,19 @@ class Generator { continue; } - $path = $this->generatePath($preview->width(), $preview->height(), $crop, $max, $preview->dataMimeType(), $prefix); try { - if ($preview instanceof IStreamImage) { - return $previewFolder->newFile($path, $preview->resource()); - } else { - return $previewFolder->newFile($path, $preview->data()); - } + return $this->savePreview($file, $width, $height, $crop, $max, $preview, $version); } catch (NotPermittedException $e) { throw new NotFoundException(); } - - return $file; } } throw new NotFoundException('No provider successfully handled the preview generation'); } - /** - * @param ISimpleFile $file - * @param string $prefix - * @return int[] - */ - private function getPreviewSize(ISimpleFile $file, string $prefix = '') { - $size = explode('-', substr($file->getName(), strlen($prefix))); - return [(int)$size[0], (int)$size[1]]; - } - - /** - * @param int $width - * @param int $height - * @param bool $crop - * @param bool $max - * @param string $mimeType - * @param string $prefix - * @return string - */ - private function generatePath($width, $height, $crop, $max, $mimeType, $prefix) { - $path = $prefix . (string)$width . '-' . (string)$height; + private function generatePath(int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): string { + $path = ($version ? $version . '-' : '') . $width . '-' . $height; if ($crop) { $path .= '-crop'; } @@ -401,15 +384,10 @@ class Generator { /** - * @param int $width - * @param int $height - * @param bool $crop - * @param string $mode - * @param int $maxWidth - * @param int $maxHeight + * @psalm-param IPreview::MODE_* $mode * @return int[] */ - private function calculateSize($width, $height, $crop, $mode, $maxWidth, $maxHeight) { + private function calculateSize(int $width, int $height, bool $crop, string $mode, int $maxWidth, int $maxHeight): array { /* * If we are not cropping we have to make sure the requested image * respects the aspect ratio of the original. @@ -492,14 +470,14 @@ class Generator { * @throws \InvalidArgumentException if the preview would be invalid (in case the original image is invalid) */ private function generatePreview( - ISimpleFolder $previewFolder, + File $file, IImage $maxPreview, int $width, int $height, bool $crop, int $maxWidth, int $maxHeight, - string $prefix, + ?int $version, bool $cacheResult, ): ISimpleFile { $preview = $maxPreview; @@ -536,62 +514,13 @@ class Generator { } - $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $prefix); - try { - if ($cacheResult) { - return $previewFolder->newFile($path, $preview->data()); - } else { - return new InMemoryFile($path, $preview->data()); - } - } catch (NotPermittedException $e) { - throw new NotFoundException(); + $path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $version); + if ($cacheResult) { + $previewEntry = $this->savePreview($file, $width, $height, $crop, false, $preview, $version); + return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper); + } else { + return new InMemoryFile($path, $preview->data()); } - return $file; - } - - /** - * @param ISimpleFile[] $files Array of FileInfo, as the result of getDirectoryListing() - * @param int $width - * @param int $height - * @param bool $crop - * @param string $mimeType - * @param string $prefix - * @return ISimpleFile - * - * @throws NotFoundException - */ - private function getCachedPreview($files, $width, $height, $crop, $mimeType, $prefix) { - $path = $this->generatePath($width, $height, $crop, false, $mimeType, $prefix); - foreach ($files as $file) { - if ($file->getName() === $path) { - $this->logger->debug('Found cached preview: {path}', ['path' => $path]); - return $file; - } - } - throw new NotFoundException(); - } - - /** - * Get the specific preview folder for this file - * - * @param File $file - * @return ISimpleFolder - * - * @throws InvalidPathException - * @throws NotFoundException - * @throws NotPermittedException - */ - private function getPreviewFolder(File $file) { - // Obtain file id outside of try catch block to prevent the creation of an existing folder - $fileId = (string)$file->getId(); - - try { - $folder = $this->appData->getFolder($fileId); - } catch (NotFoundException $e) { - $folder = $this->appData->newFolder($fileId); - } - - return $folder; } /** @@ -613,4 +542,53 @@ class Generator { throw new \InvalidArgumentException('Not a valid mimetype: "' . $mimeType . '"'); } } + + /** + * @throws InvalidPathException + * @throws NotFoundException + * @throws NotPermittedException + * @throws \OCP\DB\Exception + */ + public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, ?int $version): Preview { + $previewEntry = new Preview(); + $previewEntry->setFileId($file->getId()); + $previewEntry->setWidth($width); + $previewEntry->setHeight($height); + $previewEntry->setVersion($version); + $previewEntry->setIsMax($max); + $previewEntry->setCrop($crop); + switch ($preview->dataMimeType()) { + case 'image/jpeg': + $previewEntry->setMimetype(IPreview::MIMETYPE_JPEG); + break; + case 'image/gif': + $previewEntry->setMimetype(IPreview::MIMETYPE_GIF); + break; + case 'image/webp': + $previewEntry->setMimetype(IPreview::MIMETYPE_WEBP); + break; + default: + $previewEntry->setMimetype(IPreview::MIMETYPE_PNG); + break; + } + $previewEntry->setEtag($file->getEtag()); + $previewEntry->setMtime((new \DateTime())->getTimestamp()); + $previewEntry->setSize(0); + + $previewEntry = $this->previewMapper->insert($previewEntry); + + // we need to save to DB first + try { + if ($preview instanceof IStreamImage) { + $size = $this->storageFactory->writePreview($previewEntry, $preview->resource()); + } else { + $size = $this->storageFactory->writePreview($previewEntry, $preview->data()); + } + } catch (\Exception $e) { + $this->previewMapper->delete($previewEntry); + throw $e; + } + $previewEntry->setSize($size); + return $this->previewMapper->update($previewEntry); + } } diff --git a/lib/private/Preview/Storage/IPreviewStorage.php b/lib/private/Preview/Storage/IPreviewStorage.php new file mode 100644 index 00000000000..a989c492c76 --- /dev/null +++ b/lib/private/Preview/Storage/IPreviewStorage.php @@ -0,0 +1,22 @@ +constructPath($preview); + ['basename' => $basename, 'dirname' => $dirname] = pathinfo($previewPath); + $currentDir = $this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY; + mkdir($currentDir); + foreach (explode('/', $dirname) as $suffix) { + $currentDir .= "/$suffix"; + mkdir($currentDir); + } + $file = @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "w"); + return fwrite($file, $stream); + } + + public function readPreview(Preview $preview) { + $previewPath = $this->constructPath($preview); + return @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "r"); + } + + public function deletePreview(Preview $preview) { + $previewPath = $this->constructPath($preview); + @unlink($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath); + } + + private function constructPath(Preview $preview): string { + return implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName(); + } +} diff --git a/lib/private/Preview/Storage/ObjectStorePreviewStorage.php b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php new file mode 100644 index 00000000000..fc9a64ed761 --- /dev/null +++ b/lib/private/Preview/Storage/ObjectStorePreviewStorage.php @@ -0,0 +1,50 @@ +objectPrefix = $parameters['objectPrefix'] . 'preview:'; + } + } + + public function writePreview(Preview $preview, $stream): false|int { + if (!is_resource($stream)) { + $fh = fopen('php://temp', 'w+'); + fwrite($fh, $stream); + rewind($fh); + + $stream = $fh; + } + + $size = 0; + $countStream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size): void { + $size = $writtenSize; + }); + + $this->objectStore->writeObject($this->constructUrn($preview), $countStream); + return $size; + } + + public function readPreview(Preview $preview) { + return $this->objectStore->readObject($this->constructUrn($preview)); + } + + public function deletePreview(Preview $preview) { + return $this->objectStore->deleteObject($this->constructUrn($preview)); + } + + private function constructUrn(Preview $preview): string { + return $this->objectPrefix . $preview->getId(); + } +} diff --git a/lib/private/Preview/Storage/PreviewFile.php b/lib/private/Preview/Storage/PreviewFile.php new file mode 100644 index 00000000000..f9e4cc0f59a --- /dev/null +++ b/lib/private/Preview/Storage/PreviewFile.php @@ -0,0 +1,90 @@ +preview->getName(); + } + + /** + * @inheritDoc + */ + public function getSize(): int|float { + return $this->preview->getSize(); + } + + /** + * @inheritDoc + */ + public function getETag(): string { + return $this->preview->getEtag(); + } + + /** + * @inheritDoc + */ + public function getMTime(): int { + return $this->preview->getMtime(); + } + + /** + * @inheritDoc + */ + public function getContent(): string { + $stream = $this->storage->readPreview($this->preview); + return stream_get_contents($stream); + } + + /** + * @inheritDoc + */ + public function putContent($data): void { + } + + /** + * @inheritDoc + */ + public function delete(): void { + $this->storage->deletePreview($this->preview); + $this->previewMapper->delete($this->preview); + } + + /** + * @inheritDoc + */ + public function getMimeType(): string { + return $this->preview->getMimetypeValue(); + } + + /** + * @inheritDoc + */ + public function getExtension(): string { + return $this->preview->getExtension(); + } + + /** + * @inheritDoc + */ + public function read() { + return $this->storage->readPreview($this->preview); + } + + /** + * @inheritDoc + */ + public function write() { + return false; + } +} diff --git a/lib/private/Preview/Storage/StorageFactory.php b/lib/private/Preview/Storage/StorageFactory.php new file mode 100644 index 00000000000..15866276774 --- /dev/null +++ b/lib/private/Preview/Storage/StorageFactory.php @@ -0,0 +1,44 @@ +getBackend()->writePreview($preview, $stream); + } + + public function readPreview(Preview $preview) { + return $this->getBackend()->readPreview($preview); + } + + public function deletePreview(Preview $preview) { + $this->getBackend()->deletePreview($preview); + } + + private function getBackend(): IPreviewStorage { + if ($this->backend) { + return $this->backend; + } + + $objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot(); + + if ($objectStoreConfig) { + $objectStore = $this->objectStoreConfig->buildObjectStore($objectStoreConfig); + $this->backend = new ObjectStorePreviewStorage($objectStore, $objectStoreConfig['arguments']); + } else { + $configDataDirectory = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data'); + $this->backend = new LocalPreviewStorage($configDataDirectory); + } + + return $this->backend; + } +} diff --git a/lib/private/PreviewManager.php b/lib/private/PreviewManager.php index 97e9b5e313c..7c8e1b13eb9 100644 --- a/lib/private/PreviewManager.php +++ b/lib/private/PreviewManager.php @@ -8,9 +8,11 @@ namespace OC; use OC\AppFramework\Bootstrap\Coordinator; +use OC\Preview\Db\PreviewMapper; use OC\Preview\Generator; use OC\Preview\GeneratorHelper; use OC\Preview\IMagickSupport; +use OC\Preview\Storage\StorageFactory; use OCP\AppFramework\QueryException; use OCP\EventDispatcher\IEventDispatcher; use OCP\Files\File; @@ -140,6 +142,8 @@ class PreviewManager implements IPreview { ), $this->eventDispatcher, $this->container->get(LoggerInterface::class), + $this->container->get(PreviewMapper::class), + $this->container->get(StorageFactory::class), ); } return $this->generator; diff --git a/lib/public/IPreview.php b/lib/public/IPreview.php index 3c9eadd4577..cbd0e0ae525 100644 --- a/lib/public/IPreview.php +++ b/lib/public/IPreview.php @@ -29,6 +29,11 @@ interface IPreview { */ public const MODE_COVER = 'cover'; + public const MIMETYPE_JPEG = 0; + public const MIMETYPE_WEBP = 1; + public const MIMETYPE_PNG = 2; + public const MIMETYPE_GIF = 3; + /** * In order to improve lazy loading a closure can be registered which will be * called in case preview providers are actually requested