upstream/nextcloud
1
0
Fork 0
mirror of https://github.com/nextcloud/server.git synced 2026-04-28 09:37:29 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 44

Merge pull request #33551 from nextcloud/scanner-dont-update-same-size

Browse source 
don't update the folder size if we know it hasn't changed
This commit is contained in:
Vincent Petry 2022-08-17 09:21:30 +02:00 committed by GitHub
commit 082432e01c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 47 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
lib/private/Files/Cache/Scanner.php
View file

@ -339,7 +339,7 @@ class Scanner extends BasicEmitter implements IScanner {
try {
$data = $this->scanFile($path, $reuse, -1, null, $lock);
if ($data and $data['mimetype'] === 'httpd/unix-directory') {
$size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock);
$size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data);
$data['size'] = $size;
}
} finally {
@ -376,9 +376,10 @@ class Scanner extends BasicEmitter implements IScanner {
* @param int $reuse
* @param int $folderId id for the folder to be scanned
* @param bool $lock set to false to disable getting an additional read lock during scanning
* @param array $data the data of the folder before (re)scanning the children
* @return int the size of the scanned folder or -1 if the size is unknown at this stage
*/
protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) {
protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true, array $data = []) {
if ($reuse === -1) {
$reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG;
}
@ -397,7 +398,8 @@ class Scanner extends BasicEmitter implements IScanner {
$size += $childSize;
}
}
if ($this->cacheActive) {
$oldSize = $data['size'] ?? null;
if ($this->cacheActive && $oldSize !== $size) {
$this->cache->update($folderId, ['size' => $size]);
}
$this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]);
@ -409,6 +411,11 @@ class Scanner extends BasicEmitter implements IScanner {
$existingChildren = $this->getExistingChildren($folderId);
$newChildren = iterator_to_array($this->storage->getDirectoryContent($path));
if (count($existingChildren) === 0 && count($newChildren) === 0) {
// no need to do a transaction
return [];
}
if ($this->useTransactions) {
\OC::$server->getDatabaseConnection()->beginTransaction();
}

2
lib/private/Files/ObjectStore/NoopScanner.php
View file

@ -68,7 +68,7 @@ class NoopScanner extends Scanner {
* @param array $folderData existing cache data for the folder to be scanned
* @return int the size of the scanned folder or -1 if the size is unknown at this stage
*/
protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) {
protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true, array $data = []) {
return 0;
}

78
tests/lib/Files/Cache/ScannerTest.php
View file

@ -8,7 +8,14 @@
namespace Test\Files\Cache;
use OC;
use OC\Files\Cache\Cache;
use OC\Files\Cache\CacheEntry;
use OC\Files\Cache\Scanner;
use OC\Files\Storage\Storage;
use OC\Files\Storage\Temporary;
use OCP\Files\Cache\IScanner;
use Test\TestCase;
/**
* Class ScannerTest
@ -17,34 +24,21 @@ use OC\Files\Cache\CacheEntry;
*
* @package Test\Files\Cache
*/
class ScannerTest extends \Test\TestCase {
/**
* @var \OC\Files\Storage\Storage $storage
*/
private $storage;
/**
* @var \OC\Files\Cache\Scanner $scanner
*/
private $scanner;
/**
* @var \OC\Files\Cache\Cache $cache
*/
private $cache;
class ScannerTest extends TestCase {
private Storage $storage;
private Scanner $scanner;
private Cache $cache;
protected function setUp(): void {
parent::setUp();
$this->storage = new \OC\Files\Storage\Temporary([]);
$this->scanner = new \OC\Files\Cache\Scanner($this->storage);
$this->cache = new \OC\Files\Cache\Cache($this->storage);
$this->storage = new Temporary([]);
$this->scanner = new Scanner($this->storage);
$this->cache = new Cache($this->storage);
}
protected function tearDown(): void {
if ($this->cache) {
$this->cache->clear();
}
$this->cache->clear();
parent::tearDown();
}
@ -60,7 +54,7 @@ class ScannerTest extends \Test\TestCase {
$this->assertEquals($cachedData['mimetype'], 'text/plain');
$this->assertNotEquals($cachedData['parent'], -1); //parent folders should be scanned automatically
$data = file_get_contents(\OC::$SERVERROOT . '/core/img/logo/logo.png');
$data = file_get_contents(OC::$SERVERROOT . '/core/img/logo/logo.png');
$this->storage->file_put_contents('foo.png', $data);
$this->scanner->scanFile('foo.png');
@ -74,7 +68,7 @@ class ScannerTest extends \Test\TestCase {
$data = "dummy file data\n";
$this->storage->file_put_contents('foo🙈.txt', $data);
if (\OC::$server->getDatabaseConnection()->supports4ByteText()) {
if (OC::$server->getDatabaseConnection()->supports4ByteText()) {
$this->assertNotNull($this->scanner->scanFile('foo🙈.txt'));
$this->assertTrue($this->cache->inCache('foo🙈.txt'), true);
@ -98,7 +92,7 @@ class ScannerTest extends \Test\TestCase {
private function fillTestFolders() {
$textData = "dummy file data\n";
$imgData = file_get_contents(\OC::$SERVERROOT . '/core/img/logo/logo.png');
$imgData = file_get_contents(OC::$SERVERROOT . '/core/img/logo/logo.png');
$this->storage->mkdir('folder');
$this->storage->file_put_contents('foo.txt', $textData);
$this->storage->file_put_contents('foo.png', $imgData);
@ -130,7 +124,7 @@ class ScannerTest extends \Test\TestCase {
public function testShallow() {
$this->fillTestFolders();
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW);
$this->scanner->scan('', IScanner::SCAN_SHALLOW);
$this->assertEquals($this->cache->inCache(''), true);
$this->assertEquals($this->cache->inCache('foo.txt'), true);
$this->assertEquals($this->cache->inCache('foo.png'), true);
@ -143,7 +137,7 @@ class ScannerTest extends \Test\TestCase {
$this->assertEquals(-1, $cachedDataFolder['size']);
$this->assertEquals(-1, $cachedDataFolder2['size']);
$this->scanner->scan('folder', \OC\Files\Cache\Scanner::SCAN_SHALLOW);
$this->scanner->scan('folder', IScanner::SCAN_SHALLOW);
$cachedDataFolder2 = $this->cache->get('folder');
@ -160,7 +154,7 @@ class ScannerTest extends \Test\TestCase {
$this->storage->mkdir('folder2');
$this->storage->file_put_contents('folder2/bar.txt', 'foobar');
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW);
$this->scanner->scan('', IScanner::SCAN_SHALLOW);
$this->assertFalse($this->cache->inCache('folder/bar.txt'));
$this->assertFalse($this->cache->inCache('folder/2bar.txt'));
$cachedData = $this->cache->get('');
@ -182,7 +176,7 @@ class ScannerTest extends \Test\TestCase {
$this->storage->mkdir('folder2');
$this->storage->file_put_contents('folder2/bar.txt', 'foobar');
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW);
$this->scanner->scan('', IScanner::SCAN_SHALLOW);
$this->assertFalse($this->cache->inCache('folder/bar.txt'));
$this->assertFalse($this->cache->inCache('folder/2bar.txt'));
$this->assertFalse($this->cache->inCache('folder2/bar.txt'));
@ -191,7 +185,7 @@ class ScannerTest extends \Test\TestCase {
$cachedData = $this->cache->get('');
$this->assertEquals(-1, $cachedData['size']);
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE_INCOMPLETE, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE);
$this->scanner->scan('', IScanner::SCAN_RECURSIVE_INCOMPLETE, IScanner::REUSE_ETAG | IScanner::REUSE_SIZE);
$this->assertTrue($this->cache->inCache('folder/bar.txt'));
$this->assertTrue($this->cache->inCache('folder/bar.txt'));
@ -248,7 +242,7 @@ class ScannerTest extends \Test\TestCase {
$oldData = $this->cache->get('');
$this->storage->unlink('folder/bar.txt');
$this->cache->put('folder', ['mtime' => $this->storage->filemtime('folder'), 'storage_mtime' => $this->storage->filemtime('folder')]);
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_SIZE);
$this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_SIZE);
$newData = $this->cache->get('');
$this->assertIsString($oldData['etag']);
$this->assertIsString($newData['etag']);
@ -256,25 +250,25 @@ class ScannerTest extends \Test\TestCase {
$this->assertEquals($oldData['size'], $newData['size']);
$oldData = $newData;
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_ETAG);
$this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG);
$newData = $this->cache->get('');
$this->assertSame($oldData['etag'], $newData['etag']);
$this->assertEquals(-1, $newData['size']);
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE);
$this->scanner->scan('', IScanner::SCAN_RECURSIVE);
$oldData = $this->cache->get('');
$this->assertNotEquals(-1, $oldData['size']);
$this->scanner->scanFile('', \OC\Files\Cache\Scanner::REUSE_ETAG + \OC\Files\Cache\Scanner::REUSE_SIZE);
$this->scanner->scanFile('', IScanner::REUSE_ETAG + IScanner::REUSE_SIZE);
$newData = $this->cache->get('');
$this->assertSame($oldData['etag'], $newData['etag']);
$this->assertEquals($oldData['size'], $newData['size']);
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG + \OC\Files\Cache\Scanner::REUSE_SIZE);
$this->scanner->scan('', IScanner::SCAN_RECURSIVE, IScanner::REUSE_ETAG + IScanner::REUSE_SIZE);
$newData = $this->cache->get('');
$this->assertSame($oldData['etag'], $newData['etag']);
$this->assertEquals($oldData['size'], $newData['size']);
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_ETAG + \OC\Files\Cache\Scanner::REUSE_SIZE);
$this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG + IScanner::REUSE_SIZE);
$newData = $this->cache->get('');
$this->assertSame($oldData['etag'], $newData['etag']);
$this->assertEquals($oldData['size'], $newData['size']);
@ -286,7 +280,7 @@ class ScannerTest extends \Test\TestCase {
$this->scanner->scan('');
$this->assertTrue($this->cache->inCache('foo.txt'));
$this->storage->unlink('foo.txt');
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW);
$this->scanner->scan('', IScanner::SCAN_SHALLOW);
$this->assertFalse($this->cache->inCache('foo.txt'));
}
@ -296,7 +290,7 @@ class ScannerTest extends \Test\TestCase {
$this->scanner->scan('');
$this->assertTrue($this->cache->inCache('folder/bar.txt'));
$this->storage->rmdir('/folder');
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW);
$this->scanner->scan('', IScanner::SCAN_SHALLOW);
$this->assertFalse($this->cache->inCache('folder'));
$this->assertFalse($this->cache->inCache('folder/bar.txt'));
}
@ -317,7 +311,7 @@ class ScannerTest extends \Test\TestCase {
$this->scanner->scan('folder/bar.txt');
// manipulate etag to simulate an empty etag
$this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_ETAG);
$this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG);
/** @var CacheEntry $data0 */
$data0 = $this->cache->get('folder/bar.txt');
$this->assertIsString($data0['etag']);
@ -329,7 +323,7 @@ class ScannerTest extends \Test\TestCase {
$this->cache->put('folder/bar.txt', $data0->getData());
// rescan
$this->scanner->scan('folder/bar.txt', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_ETAG);
$this->scanner->scan('folder/bar.txt', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG);
// verify cache content
$newData0 = $this->cache->get('folder/bar.txt');
@ -344,7 +338,7 @@ class ScannerTest extends \Test\TestCase {
$oldFolderId = $this->cache->getId('folder');
// delete the folder without removing the children
$query = \OC::$server->getDatabaseConnection()->getQueryBuilder();
$query = OC::$server->getDatabaseConnection()->getQueryBuilder();
$query->delete('filecache')
->where($query->expr()->eq('fileid', $query->createNamedParameter($oldFolderId)));
$query->execute();
@ -370,7 +364,7 @@ class ScannerTest extends \Test\TestCase {
$oldFolderId = $this->cache->getId('folder');
// delete the folder without removing the children
$query = \OC::$server->getDatabaseConnection()->getQueryBuilder();
$query = OC::$server->getDatabaseConnection()->getQueryBuilder();
$query->delete('filecache')
->where($query->expr()->eq('fileid', $query->createNamedParameter($oldFolderId)));
$query->execute();
@ -379,7 +373,7 @@ class ScannerTest extends \Test\TestCase {
$this->assertEquals($oldFolderId, $cachedData['parent']);
$this->assertFalse($this->cache->inCache('folder'));
$this->scanner->scan('folder', \OC\Files\Cache\Scanner::SCAN_SHALLOW);
$this->scanner->scan('folder', IScanner::SCAN_SHALLOW);
$this->assertTrue($this->cache->inCache('folder'));
$newFolderId = $this->cache->getId('folder');