upstream/nextcloud
1
0
Fork 0
mirror of https://github.com/nextcloud/server.git synced 2026-05-28 04:32:30 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 25

Merge pull request #35129 from Glandos/previewperf2

Browse source 
[Performance] Reuse preview directory listing
This commit is contained in:
Simon L 2023-04-17 20:22:39 +02:00 committed by GitHub
commit 5cda8f0f45
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 54 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

140
lib/private/Preview/Generator.php
View file

@ -137,6 +137,8 @@ class Generator {
}
$previewFolder = $this->getPreviewFolder($file);
// List every existing preview first instead of trying to find them one by one
$previewFiles = $previewFolder->getDirectoryListing();
$previewVersion = '';
if ($file instanceof IVersionedPreviewFile) {
@ -150,7 +152,7 @@ class Generator {
&& preg_match(Imaginary::supportedMimeTypes(), $mimeType)
&& $this->config->getSystemValueString('preview_imaginary_url', 'invalid') !== 'invalid') {
$crop = $specifications[0]['crop'] ?? false;
$preview = $this->getSmallImagePreview($previewFolder, $file, $mimeType, $previewVersion, $crop);
$preview = $this->getSmallImagePreview($previewFolder, $previewFiles, $file, $mimeType, $previewVersion, $crop);
if ($preview->getSize() === 0) {
$preview->delete();
@ -161,7 +163,7 @@ class Generator {
}
// Get the max preview and infer the max preview sizes from that
$maxPreview = $this->getMaxPreview($previewFolder, $file, $mimeType, $previewVersion);
$maxPreview = $this->getMaxPreview($previewFolder, $previewFiles, $file, $mimeType, $previewVersion);
$maxPreviewImage = null; // only load the image when we need it
if ($maxPreview->getSize() === 0) {
$maxPreview->delete();
@ -197,7 +199,7 @@ class Generator {
// Try to get a cached preview. Else generate (and store) one
try {
try {
$preview = $this->getCachedPreview($previewFolder, $width, $height, $crop, $maxPreview->getMimeType(), $previewVersion);
$preview = $this->getCachedPreview($previewFiles, $width, $height, $crop, $maxPreview->getMimeType(), $previewVersion);
} catch (NotFoundException $e) {
if (!$this->previewManager->isMimeSupported($mimeType)) {
throw new NotFoundException();
@ -208,6 +210,8 @@ class Generator {
}
$preview = $this->generatePreview($previewFolder, $maxPreviewImage, $width, $height, $crop, $maxWidth, $maxHeight, $previewVersion);
// New file, augment our array
$previewFiles[] = $preview;
}
} catch (\InvalidArgumentException $e) {
throw new NotFoundException("", 0, $e);
@ -233,75 +237,19 @@ class Generator {
* Generate a small image straight away without generating a max preview first
* Preview generated is 256x256
*
* @param ISimpleFile[] $previewFiles
*
* @throws NotFoundException
*/
private function getSmallImagePreview(ISimpleFolder $previewFolder, File $file, string $mimeType, string $prefix, bool $crop): ISimpleFile {
$nodes = $previewFolder->getDirectoryListing();
private function getSmallImagePreview(ISimpleFolder $previewFolder, array $previewFiles, File $file, string $mimeType, string $prefix, bool $crop): ISimpleFile {
$width = 256;
$height = 256;
foreach ($nodes as $node) {
$name = $node->getName();
if (($prefix === '' || str_starts_with($name, $prefix))) {
// Prefix match
if (str_starts_with($name, $prefix . '256-256-crop') && $crop) {
// Cropped image
return $node;
}
if (str_starts_with($name, $prefix . '256-256.') && !$crop) {
// Uncropped image
return $node;
}
}
try {
return $this->getCachedPreview($previewFiles, $width, $height, $crop, $mimeType, $prefix);
} catch (NotFoundException $e) {
return $this->generateProviderPreview($previewFolder, $file, $width, $height, $crop, false, $mimeType, $prefix);
}
$previewProviders = $this->previewManager->getProviders();
foreach ($previewProviders as $supportedMimeType => $providers) {
// Filter out providers that does not support this mime
if (!preg_match($supportedMimeType, $mimeType)) {
continue;
}
foreach ($providers as $providerClosure) {
$provider = $this->helper->getProvider($providerClosure);
if (!($provider instanceof IProviderV2)) {
continue;
}
if (!$provider->isAvailable($file)) {
continue;
}
$preview = $this->helper->getThumbnail($provider, $file, 256, 256, $crop);
if (!($preview instanceof IImage)) {
continue;
}
// Try to get the extension.
try {
$ext = $this->getExtention($preview->dataMimeType());
} catch (\InvalidArgumentException $e) {
// Just continue to the next iteration if this preview doesn't have a valid mimetype
continue;
}
$path = $this->generatePath(256, 256, $crop, $preview->dataMimeType(), $prefix);
try {
$file = $previewFolder->newFile($path);
if ($preview instanceof IStreamImage) {
$file->putContent($preview->resource());
} else {
$file->putContent($preview->data());
}
} catch (NotPermittedException $e) {
throw new NotFoundException();
}
return $file;
}
}
throw new NotFoundException('No provider successfully handled the preview generation');
}
/**
@ -398,22 +346,30 @@ class Generator {
/**
* @param ISimpleFolder $previewFolder
* @param ISimpleFile[] $previewFiles
* @param File $file
* @param string $mimeType
* @param string $prefix
* @return ISimpleFile
* @throws NotFoundException
*/
private function getMaxPreview(ISimpleFolder $previewFolder, File $file, $mimeType, $prefix) {
$nodes = $previewFolder->getDirectoryListing();
foreach ($nodes as $node) {
private function getMaxPreview(ISimpleFolder $previewFolder, array $previewFiles, File $file, $mimeType, $prefix) {
// We don't know the max preview size, so we can't use getCachedPreview.
// It might have been generated with a higher resolution than the current value.
foreach ($previewFiles as $node) {
$name = $node->getName();
if (($prefix === '' || strpos($name, $prefix) === 0) && strpos($name, 'max')) {
return $node;
}
}
$maxWidth = $this->config->getSystemValueInt('preview_max_x', 4096);
$maxHeight = $this->config->getSystemValueInt('preview_max_y', 4096);
return $this->generateProviderPreview($previewFolder, $file, $maxWidth, $maxHeight, false, true, $mimeType, $prefix);
}
private function generateProviderPreview(ISimpleFolder $previewFolder, File $file, int $width, int $height, bool $crop, bool $max, string $mimeType, string $prefix) {
$previewProviders = $this->previewManager->getProviders();
foreach ($previewProviders as $supportedMimeType => $providers) {
// Filter out providers that does not support this mime
@ -431,13 +387,10 @@ class Generator {
continue;
}
$maxWidth = $this->config->getSystemValueInt('preview_max_x', 4096);
$maxHeight = $this->config->getSystemValueInt('preview_max_y', 4096);
$previewConcurrency = $this->getNumConcurrentPreviews('preview_concurrency_new');
$sem = self::guardWithSemaphore(self::SEMAPHORE_ID_NEW, $previewConcurrency);
try {
$preview = $this->helper->getThumbnail($provider, $file, $maxWidth, $maxHeight);
$preview = $this->helper->getThumbnail($provider, $file, $width, $height);
} finally {
self::unguardWithSemaphore($sem);
}
@ -446,15 +399,7 @@ class Generator {
continue;
}
// Try to get the extention.
try {
$ext = $this->getExtention($preview->dataMimeType());
} catch (\InvalidArgumentException $e) {
// Just continue to the next iteration if this preview doesn't have a valid mimetype
continue;
}
$path = $prefix . (string)$preview->width() . '-' . (string)$preview->height() . '-max.' . $ext;
$path = $this->generatePath($preview->width(), $preview->height(), $crop, $max, $preview->dataMimeType(), $prefix);
try {
$file = $previewFolder->newFile($path);
if ($preview instanceof IStreamImage) {
@ -470,7 +415,7 @@ class Generator {
}
}
throw new NotFoundException();
throw new NotFoundException('No provider successfully handled the preview generation');
}
/**
@ -487,15 +432,19 @@ class Generator {
* @param int $width
* @param int $height
* @param bool $crop
* @param bool $max
* @param string $mimeType
* @param string $prefix
* @return string
*/
private function generatePath($width, $height, $crop, $mimeType, $prefix) {
private function generatePath($width, $height, $crop, $max, $mimeType, $prefix) {
$path = $prefix . (string)$width . '-' . (string)$height;
if ($crop) {
$path .= '-crop';
}
if ($max) {
$path .= '-max';
}
$ext = $this->getExtention($mimeType);
$path .= '.' . $ext;
@ -637,7 +586,8 @@ class Generator {
self::unguardWithSemaphore($sem);
}
$path = $this->generatePath($width, $height, $crop, $preview->dataMimeType(), $prefix);
$path = $this->generatePath($width, $height, $crop, false, $preview->dataMimeType(), $prefix);
try {
$file = $previewFolder->newFile($path);
$file->putContent($preview->data());
@ -649,7 +599,7 @@ class Generator {
}
/**
* @param ISimpleFolder $previewFolder
* @param ISimpleFile[] $files Array of FileInfo, as the result of getDirectoryListing()
* @param int $width
* @param int $height
* @param bool $crop
@ -659,10 +609,14 @@ class Generator {
*
* @throws NotFoundException
*/
private function getCachedPreview(ISimpleFolder $previewFolder, $width, $height, $crop, $mimeType, $prefix) {
$path = $this->generatePath($width, $height, $crop, $mimeType, $prefix);
return $previewFolder->getFile($path);
private function getCachedPreview($files, $width, $height, $crop, $mimeType, $prefix) {
$path = $this->generatePath($width, $height, $crop, false, $mimeType, $prefix);
foreach ($files as $file) {
if ($file->getName() === $path) {
return $file;
}
}
throw new NotFoundException();
}
/**

19
tests/lib/Preview/GeneratorTest.php
View file

@ -105,15 +105,12 @@ class GeneratorTest extends \Test\TestCase {
$maxPreview->method('getMimeType')
->willReturn('image/png');
$previewFolder->method('getDirectoryListing')
->willReturn([$maxPreview]);
$previewFile = $this->createMock(ISimpleFile::class);
$previewFile->method('getSize')->willReturn(1000);
$previewFile->method('getName')->willReturn('256-256.png');
$previewFolder->method('getFile')
->with($this->equalTo('256-256.png'))
->willReturn($previewFile);
$previewFolder->method('getDirectoryListing')
->willReturn([$maxPreview, $previewFile]);
$this->legacyEventDispatcher->expects($this->once())
->method('dispatch')
@ -344,14 +341,12 @@ class GeneratorTest extends \Test\TestCase {
$maxPreview->method('getMimeType')
->willReturn('image/png');
$previewFolder->method('getDirectoryListing')
->willReturn([$maxPreview]);
$preview = $this->createMock(ISimpleFile::class);
$preview->method('getSize')->willReturn(1000);
$previewFolder->method('getFile')
->with($this->equalTo('1024-512-crop.png'))
->willReturn($preview);
$preview->method('getName')->willReturn('1024-512-crop.png');
$previewFolder->method('getDirectoryListing')
->willReturn([$maxPreview, $preview]);
$this->previewManager->expects($this->never())
->method('isMimeSupported');