upstream/nextcloud
1
0
Fork 0
mirror of https://github.com/nextcloud/server.git synced 2026-02-18 18:28:50 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 14

perf(preview): Split preview data to new table

Browse source 
The new oc_previews table is optimized for storing previews and should
decrease significantly the space taken by previews in the filecache
table.

This attend to reuse the IObjectStore abstraction over S3/Swift/Azure
but currently only support one single bucket configuration.

Signed-off-by: Carl Schwan <carl.schwan@nextclound.com>
This commit is contained in:
Carl Schwan 2025-08-20 17:34:07 +02:00 committed by Carl Schwan
parent 057c0dcc98
commit 18fbacdd8d
15 changed files with 596 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
core/Migrations/Version33000Date20250819110529.php Normal file
View file

@ -0,0 +1,48 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OC\Core\Migrations;
use Closure;
use OCP\DB\ISchemaWrapper;
use OCP\DB\Types;
use OCP\Migration\IOutput;
use OCP\Migration\SimpleMigrationStep;
/**
*
*/
class Version33000Date20250819110529 extends SimpleMigrationStep {
/**
* @param Closure(): ISchemaWrapper $schemaClosure The `\Closure` returns a `ISchemaWrapper`
*/
public function changeSchema(IOutput $output, Closure $schemaClosure, array $options): ?ISchemaWrapper {
/** @var ISchemaWrapper $schema */
$schema = $schemaClosure();
if (!$schema->hasTable('previews')) {
$table = $schema->createTable('previews');
$table->addColumn('id', Types::BIGINT, ['autoincrement' => true, 'notnull' => true, 'length' => 20, 'unsigned' => true]);
$table->addColumn('file_id', Types::BIGINT, ['notnull' => true, 'length' => 20, 'unsigned' => true]);
$table->addColumn('width', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
$table->addColumn('height', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
$table->addColumn('mimetype', Types::INTEGER, ['notnull' => true]);
$table->addColumn('is_max', Types::BOOLEAN, ['notnull' => true, 'default' => false]);
$table->addColumn('crop', Types::BOOLEAN, ['notnull' => true, 'default' => false]);
$table->addColumn('etag', Types::STRING, ['notnull' => true, 'length' => 40]);
$table->addColumn('mtime', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
$table->addColumn('size', Types::INTEGER, ['notnull' => true, 'unsigned' => true]);
$table->addColumn('version', Types::BIGINT, ['notnull' => false, 'unsigned' => true]);
$table->setPrimaryKey(['id']);
$table->addUniqueIndex(['file_id', 'width', 'height', 'mimetype', 'crop', 'version'], 'previews_file_uniq_idx');
}
return $schema;
}
}

2
lib/composer/composer/LICENSE
View file

@ -1,4 +1,3 @@
Copyright (c) Nils Adermann, Jordi Boggiano
Permission is hereby granted, free of charge, to any person obtaining a copy
@ -18,4 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

8
lib/composer/composer/autoload_classmap.php
View file

@ -1528,6 +1528,7 @@ return array(
'OC\\Core\\Migrations\\Version32000Date20250620081925' => $baseDir . '/core/Migrations/Version32000Date20250620081925.php',
'OC\\Core\\Migrations\\Version32000Date20250731062008' => $baseDir . '/core/Migrations/Version32000Date20250731062008.php',
'OC\\Core\\Migrations\\Version32000Date20250806110519' => $baseDir . '/core/Migrations/Version32000Date20250806110519.php',
'OC\\Core\\Migrations\\Version33000Date20250819110523' => $baseDir . '/core/Migrations/Version33000Date20250819110523.php',
'OC\\Core\\Notification\\CoreNotifier' => $baseDir . '/core/Notification/CoreNotifier.php',
'OC\\Core\\ResponseDefinitions' => $baseDir . '/core/ResponseDefinitions.php',
'OC\\Core\\Service\\LoginFlowV2Service' => $baseDir . '/core/Service/LoginFlowV2Service.php',
@ -1880,6 +1881,8 @@ return array(
'OC\\Preview\\BackgroundCleanupJob' => $baseDir . '/lib/private/Preview/BackgroundCleanupJob.php',
'OC\\Preview\\Bitmap' => $baseDir . '/lib/private/Preview/Bitmap.php',
'OC\\Preview\\Bundled' => $baseDir . '/lib/private/Preview/Bundled.php',
'OC\\Preview\\Db\\Preview' => $baseDir . '/lib/private/Preview/Db/Preview.php',
'OC\\Preview\\Db\\PreviewMapper' => $baseDir . '/lib/private/Preview/Db/PreviewMapper.php',
'OC\\Preview\\EMF' => $baseDir . '/lib/private/Preview/EMF.php',
'OC\\Preview\\Font' => $baseDir . '/lib/private/Preview/Font.php',
'OC\\Preview\\GIF' => $baseDir . '/lib/private/Preview/GIF.php',
@ -1912,7 +1915,12 @@ return array(
'OC\\Preview\\SGI' => $baseDir . '/lib/private/Preview/SGI.php',
'OC\\Preview\\SVG' => $baseDir . '/lib/private/Preview/SVG.php',
'OC\\Preview\\StarOffice' => $baseDir . '/lib/private/Preview/StarOffice.php',
'OC\\Preview\\Storage\\IPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/IPreviewStorage.php',
'OC\\Preview\\Storage\\LocalPreviewStorage' => $baseDir . '/lib/private/Preview/Storage/LocalPreviewStorage.php',
'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => $baseDir . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php',
'OC\\Preview\\Storage\\PreviewFile' => $baseDir . '/lib/private/Preview/Storage/PreviewFile.php',
'OC\\Preview\\Storage\\Root' => $baseDir . '/lib/private/Preview/Storage/Root.php',
'OC\\Preview\\Storage\\StorageFactory' => $baseDir . '/lib/private/Preview/Storage/StorageFactory.php',
'OC\\Preview\\TGA' => $baseDir . '/lib/private/Preview/TGA.php',
'OC\\Preview\\TIFF' => $baseDir . '/lib/private/Preview/TIFF.php',
'OC\\Preview\\TXT' => $baseDir . '/lib/private/Preview/TXT.php',

8
lib/composer/composer/autoload_static.php
View file

@ -1569,6 +1569,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OC\\Core\\Migrations\\Version32000Date20250620081925' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250620081925.php',
'OC\\Core\\Migrations\\Version32000Date20250731062008' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250731062008.php',
'OC\\Core\\Migrations\\Version32000Date20250806110519' => __DIR__ . '/../../..' . '/core/Migrations/Version32000Date20250806110519.php',
'OC\\Core\\Migrations\\Version33000Date20250819110523' => __DIR__ . '/../../..' . '/core/Migrations/Version33000Date20250819110523.php',
'OC\\Core\\Notification\\CoreNotifier' => __DIR__ . '/../../..' . '/core/Notification/CoreNotifier.php',
'OC\\Core\\ResponseDefinitions' => __DIR__ . '/../../..' . '/core/ResponseDefinitions.php',
'OC\\Core\\Service\\LoginFlowV2Service' => __DIR__ . '/../../..' . '/core/Service/LoginFlowV2Service.php',
@ -1921,6 +1922,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OC\\Preview\\BackgroundCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Preview/BackgroundCleanupJob.php',
'OC\\Preview\\Bitmap' => __DIR__ . '/../../..' . '/lib/private/Preview/Bitmap.php',
'OC\\Preview\\Bundled' => __DIR__ . '/../../..' . '/lib/private/Preview/Bundled.php',
'OC\\Preview\\Db\\Preview' => __DIR__ . '/../../..' . '/lib/private/Preview/Db/Preview.php',
'OC\\Preview\\Db\\PreviewMapper' => __DIR__ . '/../../..' . '/lib/private/Preview/Db/PreviewMapper.php',
'OC\\Preview\\EMF' => __DIR__ . '/../../..' . '/lib/private/Preview/EMF.php',
'OC\\Preview\\Font' => __DIR__ . '/../../..' . '/lib/private/Preview/Font.php',
'OC\\Preview\\GIF' => __DIR__ . '/../../..' . '/lib/private/Preview/GIF.php',
@ -1953,7 +1956,12 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OC\\Preview\\SGI' => __DIR__ . '/../../..' . '/lib/private/Preview/SGI.php',
'OC\\Preview\\SVG' => __DIR__ . '/../../..' . '/lib/private/Preview/SVG.php',
'OC\\Preview\\StarOffice' => __DIR__ . '/../../..' . '/lib/private/Preview/StarOffice.php',
'OC\\Preview\\Storage\\IPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/IPreviewStorage.php',
'OC\\Preview\\Storage\\LocalPreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/LocalPreviewStorage.php',
'OC\\Preview\\Storage\\ObjectStorePreviewStorage' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/ObjectStorePreviewStorage.php',
'OC\\Preview\\Storage\\PreviewFile' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/PreviewFile.php',
'OC\\Preview\\Storage\\Root' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/Root.php',
'OC\\Preview\\Storage\\StorageFactory' => __DIR__ . '/../../..' . '/lib/private/Preview/Storage/StorageFactory.php',
'OC\\Preview\\TGA' => __DIR__ . '/../../..' . '/lib/private/Preview/TGA.php',
'OC\\Preview\\TIFF' => __DIR__ . '/../../..' . '/lib/private/Preview/TIFF.php',
'OC\\Preview\\TXT' => __DIR__ . '/../../..' . '/lib/private/Preview/TXT.php',

2
lib/private/Files/ObjectStore/PrimaryObjectStoreConfig.php
View file

@ -14,7 +14,7 @@ use OCP\IConfig;
use OCP\IUser;
/**
* @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, ...}}
* @psalm-type ObjectStoreConfig array{class: class-string<IObjectStore>, arguments: array{multibucket: bool, objectPrefix: ?string, ...}}
*/
class PrimaryObjectStoreConfig {
public function __construct(

105
lib/private/Preview/Db/Preview.php Normal file
View file

@ -0,0 +1,105 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
* SPDX-FileContributor: Carl Schwan
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OC\Preview\Db;
use OCP\AppFramework\Db\Entity;
use OCP\DB\Types;
use OCP\IPreview;
/**
* @method \int getFileId()
* @method void setFileId(int $fileId)
* @method \int getWidth()
* @method void setWidth(int $width)
* @method \int getHeight()
* @method void setHeight(int $height)
* @method \int getMode()
* @method void setMode(int $mode)
* @method \bool getCrop()
* @method void setCrop(bool $crop)
* @method void setMimetype(int $mimetype)
* @method IPreview::MIMETYPE_* getMimetype()
* @method \int getMtime()
* @method void setMtime(int $mtime)
* @method \int getSize()
* @method void setSize(int $size)
* @method \bool getIsMax()
* @method void setIsMax(bool $max)
* @method \string getEtag()
* @method void setEtag(string $etag)
* @method ?\int getVersion()
* @method void setVersion(?int $version)
*/
class Preview extends Entity {
protected ?int $fileId = null;
protected ?int $width = null;
protected ?int $height = null;
protected ?int $mimetype = null;
protected ?int $mtime = null;
protected ?int $size = null;
protected ?bool $isMax = null;
protected ?bool $crop = null;
protected ?string $etag = null;
protected ?int $version = null;
public function __construct() {
$this->addType('fileId', Types::INTEGER);
$this->addType('width', Types::INTEGER);
$this->addType('height', Types::INTEGER);
$this->addType('mimetype', Types::INTEGER);
$this->addType('mtime', Types::INTEGER);
$this->addType('size', Types::INTEGER);
$this->addType('isMax', Types::BOOLEAN);
$this->addType('crop', Types::BOOLEAN);
$this->addType('etag', Types::STRING);
$this->addType('version', Types::INTEGER);
}
public function getName(): string {
$path = ($this->getVersion() ? $this->getVersion() . '-' : '') . $this->getWidth() . '-' . $this->getHeight();
if ($this->getCrop()) {
$path .= '-crop';
}
if ($this->getIsMax()) {
$path .= '-max';
}
$ext = $this->getExtension();
$path .= '.' . $ext;
return $path;
}
public function getMimetypeValue(): string {
return match ($this->mimetype) {
IPreview::MIMETYPE_JPEG => 'image/jpeg',
IPreview::MIMETYPE_PNG => 'image/png',
IPreview::MIMETYPE_WEBP => 'image/webp',
IPreview::MIMETYPE_GIF => 'image/gif',
};
}
public function getExtension(): string {
return match ($this->mimetype) {
IPreview::MIMETYPE_JPEG => 'jpeg',
IPreview::MIMETYPE_PNG => 'png',
IPreview::MIMETYPE_WEBP => 'webp',
IPreview::MIMETYPE_GIF => 'gif',
};
}
}

66
lib/private/Preview/Db/PreviewMapper.php Normal file
View file

@ -0,0 +1,66 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OC\Preview\Db;
use OCP\AppFramework\Db\DoesNotExistException;
use OCP\AppFramework\Db\QBMapper;
use OCP\DB\Exception;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
use OCP\IPreview;
/**
* @template-extends QBMapper<Preview>
*/
class PreviewMapper extends QBMapper {
private const TABLE_NAME = 'previews';
public function __construct(IDBConnection $db) {
parent::__construct($db, self::TABLE_NAME, Preview::class);
}
/**
* @param int[] $fileIds
* @return array<int, Preview[]>
* @throws Exception
*/
public function getAvailablePreviews(array $fileIds): array {
$selectQb = $this->db->getQueryBuilder();
$selectQb->select('*')
->from(self::TABLE_NAME)
->where(
$selectQb->expr()->in('file_id', $selectQb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)),
);
$previews = array_fill_keys($fileIds, []);
foreach ($this->yieldEntities($selectQb) as $preview) {
$previews[$preview->getFileId()][] = $preview;
}
return $previews;
}
public function getPreview(int $fileId, int $width, int $height, string $mode, int $mimetype = IPreview::MIMETYPE_JPEG): ?Preview {
$selectQb = $this->db->getQueryBuilder();
$selectQb->select('*')
->from(self::TABLE_NAME)
->where(
$selectQb->expr()->eq('file_id', $selectQb->createNamedParameter($fileId)),
$selectQb->expr()->eq('width', $selectQb->createNamedParameter($width)),
$selectQb->expr()->eq('height', $selectQb->createNamedParameter($height)),
$selectQb->expr()->eq('mode', $selectQb->createNamedParameter($mode)),
$selectQb->expr()->eq('mimetype', $selectQb->createNamedParameter($mimetype)),
);
try {
return $this->findEntity($selectQb);
} catch (DoesNotExistException) {
return null;
}
}
}

226
lib/private/Preview/Generator.php
View file

@ -6,6 +6,10 @@
*/
namespace OC\Preview;
use OC\Preview\Db\Preview;
use OC\Preview\Db\PreviewMapper;
use OC\Preview\Storage\PreviewFile;
use OC\Preview\Storage\StorageFactory;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\File;
use OCP\Files\IAppData;
@ -23,6 +27,7 @@ use OCP\Preview\BeforePreviewFetchedEvent;
use OCP\Preview\IProviderV2;
use OCP\Preview\IVersionedPreviewFile;
use Psr\Log\LoggerInterface;
use function Symfony\Component\Translation\t;
class Generator {
public const SEMAPHORE_ID_ALL = 0x0a11;
@ -35,6 +40,8 @@ class Generator {
private GeneratorHelper $helper,
private IEventDispatcher $eventDispatcher,
private LoggerInterface $logger,
private PreviewMapper $previewMapper,
private StorageFactory $storageFactory,
) {
}
@ -104,25 +111,25 @@ class Generator {
$mimeType = $file->getMimeType();
}
$previewFolder = $this->getPreviewFolder($file);
// List every existing preview first instead of trying to find them one by one
$previewFiles = $previewFolder->getDirectoryListing();
[$file->getId() => $previews] = $this->previewMapper->getAvailablePreviews([$file->getId()]);
$previewVersion = '';
$previewVersion = null;
if ($file instanceof IVersionedPreviewFile) {
$previewVersion = $file->getPreviewVersion() . '-';
$previewVersion = (int)$file->getPreviewVersion();
}
// Get the max preview and infer the max preview sizes from that
$maxPreview = $this->getMaxPreview($previewFolder, $previewFiles, $file, $mimeType, $previewVersion);
$maxPreview = $this->getMaxPreview($previews, $file, $mimeType, $previewVersion);
$maxPreviewImage = null; // only load the image when we need it
if ($maxPreview->getSize() === 0) {
$maxPreview->delete();
$this->storageFactory->deletePreview($maxPreview);
$this->previewMapper->delete($maxPreview);
$this->logger->error('Max preview generated for file {path} has size 0, deleting and throwing exception.', ['path' => $file->getPath()]);
throw new NotFoundException('Max preview size 0, invalid!');
}
[$maxWidth, $maxHeight] = $this->getPreviewSize($maxPreview, $previewVersion);
$maxWidth = $maxPreview->getWidth();
$maxHeight = $maxPreview->getHeight();
if ($maxWidth <= 0 || $maxHeight <= 0) {
throw new NotFoundException('The maximum preview sizes are zero or less pixels');
@ -154,32 +161,40 @@ class Generator {
// Try to get a cached preview. Else generate (and store) one
try {
try {
$preview = $this->getCachedPreview($previewFiles, $width, $height, $crop, $maxPreview->getMimeType(), $previewVersion);
} catch (NotFoundException $e) {
/** @var ISimpleFile $previewFile */
$previewFile = null;
// TODO(php8.4) replace by array_find
foreach ($previews as $p) {
if ($p->getWidth() === $width && $p->getHeight() === $height && $p->getMimetype() === $maxPreview->getMimetype() && $p->getVersion() === $previewVersion && $p->getCrop() === $crop) {
$previewFile = new PreviewFile($p, $this->storageFactory, $this->previewMapper);
break;
}
}
if ($previewFile === null) {
if (!$this->previewManager->isMimeSupported($mimeType)) {
throw new NotFoundException();
}
if ($maxPreviewImage === null) {
$maxPreviewImage = $this->helper->getImage($maxPreview);
$maxPreviewImage = $this->helper->getImage(new PreviewFile($maxPreview, $this->storageFactory, $this->previewMapper));
}
$this->logger->debug('Cached preview not found for file {path}, generating a new preview.', ['path' => $file->getPath()]);
$preview = $this->generatePreview($previewFolder, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult);
$preview = $this->generatePreview($file, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion, $cacheResult);
// New file, augment our array
$previewFiles[] = $preview;
//$previews[] = $preview;
}
} catch (\InvalidArgumentException $e) {
throw new NotFoundException('', 0, $e);
}
if ($preview->getSize() === 0) {
$preview->delete();
if ($previewFile->getSize() === 0) {
$previewFile->delete();
throw new NotFoundException('Cached preview size 0, invalid!');
}
}
assert($preview !== null);
assert($previewFile !== null);
// Free memory being used by the embedded image resource. Without this the image is kept in memory indefinitely.
// Garbage Collection does NOT free this memory. We have to do it ourselves.
@ -187,7 +202,7 @@ class Generator {
$maxPreviewImage->destroy();
}
return $preview;
return $previewFile;
}
/**
@ -289,31 +304,25 @@ class Generator {
}
/**
* @param ISimpleFolder $previewFolder
* @param ISimpleFile[] $previewFiles
* @param File $file
* @param string $mimeType
* @param string $prefix
* @return ISimpleFile
* @param Preview[] $previews
* @throws NotFoundException
*/
private function getMaxPreview(ISimpleFolder $previewFolder, array $previewFiles, File $file, $mimeType, $prefix) {
private function getMaxPreview(array $previews, File $file, string $mimeType, ?int $version): Preview {
// We don't know the max preview size, so we can't use getCachedPreview.
// It might have been generated with a higher resolution than the current value.
foreach ($previewFiles as $node) {
$name = $node->getName();
if (($prefix === '' || str_starts_with($name, $prefix)) && strpos($name, 'max')) {
return $node;
foreach ($previews as $preview) {
if ($preview->getIsMax() && ($version == $preview->getVersion())) {
return $preview;
}
}
$maxWidth = $this->config->getSystemValueInt('preview_max_x', 4096);
$maxHeight = $this->config->getSystemValueInt('preview_max_y', 4096);
return $this->generateProviderPreview($previewFolder, $file, $maxWidth, $maxHeight, false, true, $mimeType, $prefix);
return $this->generateProviderPreview($file, $maxWidth, $maxHeight, false, true, $mimeType, $version);
}
private function generateProviderPreview(ISimpleFolder $previewFolder, File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, string $prefix) {
private function generateProviderPreview(File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): Preview {
$previewProviders = $this->previewManager->getProviders();
foreach ($previewProviders as $supportedMimeType => $providers) {
// Filter out providers that does not support this mime
@ -348,45 +357,19 @@ class Generator {
continue;
}
$path = $this->generatePath($preview->width(), $preview->height(), $crop, $max, $preview->dataMimeType(), $prefix);
try {
if ($preview instanceof IStreamImage) {
return $previewFolder->newFile($path, $preview->resource());
} else {
return $previewFolder->newFile($path, $preview->data());
}
return $this->savePreview($file, $width, $height, $crop, $max, $preview, $version);
} catch (NotPermittedException $e) {
throw new NotFoundException();
}
return $file;
}
}
throw new NotFoundException('No provider successfully handled the preview generation');
}
/**
* @param ISimpleFile $file
* @param string $prefix
* @return int[]
*/
private function getPreviewSize(ISimpleFile $file, string $prefix = '') {
$size = explode('-', substr($file->getName(), strlen($prefix)));
return [(int)$size[0], (int)$size[1]];
}
/**
* @param int $width
* @param int $height
* @param bool $crop
* @param bool $max
* @param string $mimeType
* @param string $prefix
* @return string
*/
private function generatePath($width, $height, $crop, $max, $mimeType, $prefix) {
$path = $prefix . (string)$width . '-' . (string)$height;
private function generatePath(int $width, int $height, bool $crop, bool $max, string $mimeType, ?int $version): string {
$path = ($version ? $version . '-' : '') . $width . '-' . $height;
if ($crop) {
$path .= '-crop';
}
@ -401,15 +384,10 @@ class Generator {
/**
* @param int $width
* @param int $height
* @param bool $crop
* @param string $mode
* @param int $maxWidth
* @param int $maxHeight
* @psalm-param IPreview::MODE_* $mode
* @return int[]
*/
private function calculateSize($width, $height, $crop, $mode, $maxWidth, $maxHeight) {
private function calculateSize(int $width, int $height, bool $crop, string $mode, int $maxWidth, int $maxHeight): array {
/*
* If we are not cropping we have to make sure the requested image
* respects the aspect ratio of the original.
@ -492,14 +470,14 @@ class Generator {
* @throws \InvalidArgumentException if the preview would be invalid (in case the original image is invalid)
*/
private function generatePreview(
ISimpleFolder $previewFolder,
File $file,
IImage $maxPreview,
int $width,
int $height,
bool $crop,
int $maxWidth,
int $maxHeight,
string $prefix,
?int $version,
bool $cacheResult,
): ISimpleFile {
$preview = $maxPreview;
@ -536,62 +514,13 @@ class Generator {
}
$path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $prefix);
try {
if ($cacheResult) {
return $previewFolder->newFile($path, $preview->data());
} else {
return new InMemoryFile($path, $preview->data());
}
} catch (NotPermittedException $e) {
throw new NotFoundException();
$path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $version);
if ($cacheResult) {
$previewEntry = $this->savePreview($file, $width, $height, $crop, false, $preview, $version);
return new PreviewFile($previewEntry, $this->storageFactory, $this->previewMapper);
} else {
return new InMemoryFile($path, $preview->data());
}
return $file;
}
/**
* @param ISimpleFile[] $files Array of FileInfo, as the result of getDirectoryListing()
* @param int $width
* @param int $height
* @param bool $crop
* @param string $mimeType
* @param string $prefix
* @return ISimpleFile
*
* @throws NotFoundException
*/
private function getCachedPreview($files, $width, $height, $crop, $mimeType, $prefix) {
$path = $this->generatePath($width, $height, $crop, false, $mimeType, $prefix);
foreach ($files as $file) {
if ($file->getName() === $path) {
$this->logger->debug('Found cached preview: {path}', ['path' => $path]);
return $file;
}
}
throw new NotFoundException();
}
/**
* Get the specific preview folder for this file
*
* @param File $file
* @return ISimpleFolder
*
* @throws InvalidPathException
* @throws NotFoundException
* @throws NotPermittedException
*/
private function getPreviewFolder(File $file) {
// Obtain file id outside of try catch block to prevent the creation of an existing folder
$fileId = (string)$file->getId();
try {
$folder = $this->appData->getFolder($fileId);
} catch (NotFoundException $e) {
$folder = $this->appData->newFolder($fileId);
}
return $folder;
}
/**
@ -613,4 +542,53 @@ class Generator {
throw new \InvalidArgumentException('Not a valid mimetype: "' . $mimeType . '"');
}
}
/**
* @throws InvalidPathException
* @throws NotFoundException
* @throws NotPermittedException
* @throws \OCP\DB\Exception
*/
public function savePreview(File $file, int $width, int $height, bool $crop, bool $max, IImage $preview, ?int $version): Preview {
$previewEntry = new Preview();
$previewEntry->setFileId($file->getId());
$previewEntry->setWidth($width);
$previewEntry->setHeight($height);
$previewEntry->setVersion($version);
$previewEntry->setIsMax($max);
$previewEntry->setCrop($crop);
switch ($preview->dataMimeType()) {
case 'image/jpeg':
$previewEntry->setMimetype(IPreview::MIMETYPE_JPEG);
break;
case 'image/gif':
$previewEntry->setMimetype(IPreview::MIMETYPE_GIF);
break;
case 'image/webp':
$previewEntry->setMimetype(IPreview::MIMETYPE_WEBP);
break;
default:
$previewEntry->setMimetype(IPreview::MIMETYPE_PNG);
break;
}
$previewEntry->setEtag($file->getEtag());
$previewEntry->setMtime((new \DateTime())->getTimestamp());
$previewEntry->setSize(0);
$previewEntry = $this->previewMapper->insert($previewEntry);
// we need to save to DB first
try {
if ($preview instanceof IStreamImage) {
$size = $this->storageFactory->writePreview($previewEntry, $preview->resource());
} else {
$size = $this->storageFactory->writePreview($previewEntry, $preview->data());
}
} catch (\Exception $e) {
$this->previewMapper->delete($previewEntry);
throw $e;
}
$previewEntry->setSize($size);
return $this->previewMapper->update($previewEntry);
}
}

22
lib/private/Preview/Storage/IPreviewStorage.php Normal file
View file

@ -0,0 +1,22 @@
<?php
namespace OC\Preview\Storage;
use OC\Preview\Db\Preview;
use OCP\Files\NotPermittedException;
interface IPreviewStorage {
/**
* @param resource|string $stream
* @throws NotPermittedException
*/
public function writePreview(Preview $preview, $stream): false|int;
/**
* @param Preview $preview
* @return resource|false
*/
public function readPreview(Preview $preview);
public function deletePreview(Preview $preview);
}

43
lib/private/Preview/Storage/LocalPreviewStorage.php Normal file
View file

@ -0,0 +1,43 @@
<?php
namespace OC\Preview\Storage;
use Icewind\Streams\CountWrapper;
use OC\Files\Storage\Local;
use OC\Preview\Db\Preview;
use OCP\Files\Storage\IStorage;
use OCP\IPreview;
class LocalPreviewStorage implements IPreviewStorage {
private const PREVIEW_DIRECTORY = "__preview";
public function __construct(private readonly string $rootFolder) {
}
public function writePreview(Preview $preview, $stream): false|int {
$previewPath = $this->constructPath($preview);
['basename' => $basename, 'dirname' => $dirname] = pathinfo($previewPath);
$currentDir = $this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY;
mkdir($currentDir);
foreach (explode('/', $dirname) as $suffix) {
$currentDir .= "/$suffix";
mkdir($currentDir);
}
$file = @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "w");
return fwrite($file, $stream);
}
public function readPreview(Preview $preview) {
$previewPath = $this->constructPath($preview);
return @fopen($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath, "r");
}
public function deletePreview(Preview $preview) {
$previewPath = $this->constructPath($preview);
@unlink($this->rootFolder . DIRECTORY_SEPARATOR . self::PREVIEW_DIRECTORY . DIRECTORY_SEPARATOR . $previewPath);
}
private function constructPath(Preview $preview): string {
return implode('/', str_split(substr(md5((string)$preview->getFileId()), 0, 7))) . '/' . $preview->getFileId() . '/' . $preview->getName();
}
}

50
lib/private/Preview/Storage/ObjectStorePreviewStorage.php Normal file
View file

@ -0,0 +1,50 @@
<?php
namespace OC\Preview\Storage;
use Icewind\Streams\CountWrapper;
use OC\Preview\Db\Preview;
use OCP\Files\ObjectStore\IObjectStore;
class ObjectStorePreviewStorage implements IPreviewStorage {
private string $objectPrefix = 'urn:oid:preview:';
/**
* @param array{objectPrefix: ?string} $parameters
*/
public function __construct(private IObjectStore $objectStore, array $parameters) {
if (isset($parameters['objectPrefix'])) {
$this->objectPrefix = $parameters['objectPrefix'] . 'preview:';
}
}
public function writePreview(Preview $preview, $stream): false|int {
if (!is_resource($stream)) {
$fh = fopen('php://temp', 'w+');
fwrite($fh, $stream);
rewind($fh);
$stream = $fh;
}
$size = 0;
$countStream = CountWrapper::wrap($stream, function (int $writtenSize) use (&$size): void {
$size = $writtenSize;
});
$this->objectStore->writeObject($this->constructUrn($preview), $countStream);
return $size;
}
public function readPreview(Preview $preview) {
return $this->objectStore->readObject($this->constructUrn($preview));
}
public function deletePreview(Preview $preview) {
return $this->objectStore->deleteObject($this->constructUrn($preview));
}
private function constructUrn(Preview $preview): string {
return $this->objectPrefix . $preview->getId();
}
}

90
lib/private/Preview/Storage/PreviewFile.php Normal file
View file

@ -0,0 +1,90 @@
<?php
namespace OC\Preview\Storage;
use OC\Preview\Db\Preview;
use OC\Preview\Db\PreviewMapper;
use OCP\Files\SimpleFS\ISimpleFile;
class PreviewFile implements ISimpleFile {
public function __construct(private Preview $preview, private IPreviewStorage $storage, private PreviewMapper $previewMapper) {
}
/**
* @inheritDoc
*/
public function getName(): string {
return $this->preview->getName();
}
/**
* @inheritDoc
*/
public function getSize(): int|float {
return $this->preview->getSize();
}
/**
* @inheritDoc
*/
public function getETag(): string {
return $this->preview->getEtag();
}
/**
* @inheritDoc
*/
public function getMTime(): int {
return $this->preview->getMtime();
}
/**
* @inheritDoc
*/
public function getContent(): string {
$stream = $this->storage->readPreview($this->preview);
return stream_get_contents($stream);
}
/**
* @inheritDoc
*/
public function putContent($data): void {
}
/**
* @inheritDoc
*/
public function delete(): void {
$this->storage->deletePreview($this->preview);
$this->previewMapper->delete($this->preview);
}
/**
* @inheritDoc
*/
public function getMimeType(): string {
return $this->preview->getMimetypeValue();
}
/**
* @inheritDoc
*/
public function getExtension(): string {
return $this->preview->getExtension();
}
/**
* @inheritDoc
*/
public function read() {
return $this->storage->readPreview($this->preview);
}
/**
* @inheritDoc
*/
public function write() {
return false;
}
}

44
lib/private/Preview/Storage/StorageFactory.php Normal file
View file

@ -0,0 +1,44 @@
<?php
namespace OC\Preview\Storage;
use OC;
use OC\Files\ObjectStore\PrimaryObjectStoreConfig;
use OC\Preview\Db\Preview;
use OCP\IConfig;
class StorageFactory implements IPreviewStorage {
private ?IPreviewStorage $backend = null;
public function __construct(private PrimaryObjectStoreConfig $objectStoreConfig, private IConfig $config) {}
public function writePreview(Preview $preview, $stream): false|int {
return $this->getBackend()->writePreview($preview, $stream);
}
public function readPreview(Preview $preview) {
return $this->getBackend()->readPreview($preview);
}
public function deletePreview(Preview $preview) {
$this->getBackend()->deletePreview($preview);
}
private function getBackend(): IPreviewStorage {
if ($this->backend) {
return $this->backend;
}
$objectStoreConfig = $this->objectStoreConfig->getObjectStoreConfigForRoot();
if ($objectStoreConfig) {
$objectStore = $this->objectStoreConfig->buildObjectStore($objectStoreConfig);
$this->backend = new ObjectStorePreviewStorage($objectStore, $objectStoreConfig['arguments']);
} else {
$configDataDirectory = $this->config->getSystemValue('datadirectory', OC::$SERVERROOT . '/data');
$this->backend = new LocalPreviewStorage($configDataDirectory);
}
return $this->backend;
}
}

4
lib/private/PreviewManager.php
View file

@ -8,9 +8,11 @@
namespace OC;
use OC\AppFramework\Bootstrap\Coordinator;
use OC\Preview\Db\PreviewMapper;
use OC\Preview\Generator;
use OC\Preview\GeneratorHelper;
use OC\Preview\IMagickSupport;
use OC\Preview\Storage\StorageFactory;
use OCP\AppFramework\QueryException;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\File;
@ -140,6 +142,8 @@ class PreviewManager implements IPreview {
),
$this->eventDispatcher,
$this->container->get(LoggerInterface::class),
$this->container->get(PreviewMapper::class),
$this->container->get(StorageFactory::class),
);
}
return $this->generator;

5
lib/public/IPreview.php
View file

@ -29,6 +29,11 @@ interface IPreview {
*/
public const MODE_COVER = 'cover';
public const MIMETYPE_JPEG = 0;
public const MIMETYPE_WEBP = 1;
public const MIMETYPE_PNG = 2;
public const MIMETYPE_GIF = 3;
/**
* In order to improve lazy loading a closure can be registered which will be
* called in case preview providers are actually requested