upstream/nextcloud
1
0
Fork 0
mirror of https://github.com/nextcloud/server.git synced 2026-02-20 00:12:30 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 38

Merge pull request #46583 from nextcloud/filecache-sharding-compat

Browse source 
Make filecache queries compatible with sharding
This commit is contained in:
Robin Appelman 2024-07-19 09:49:45 +02:00 committed by GitHub
commit 64ca4b832d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 50 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
lib/private/Files/Cache/Cache.php
View file

@ -120,11 +120,11 @@ class Cache implements ICache {
// normalize file
$file = $this->normalize($file);
$query->whereStorageId($this->getNumericStorageId())
->wherePath($file);
$query->wherePath($file);
} else { //file id
$query->whereFileId($file);
}
$query->whereStorageId($this->getNumericStorageId());
$result = $query->execute();
$data = $result->fetch();
@ -199,6 +199,7 @@ class Cache implements ICache {
$query = $this->getQueryBuilder();
$query->selectFileCache()
->whereParent($fileId)
->whereStorageId($this->getNumericStorageId())
->orderBy('name', 'ASC');
$metadataQuery = $query->selectMetadata();
@ -337,6 +338,7 @@ class Cache implements ICache {
$query->update('filecache')
->whereFileId($id)
->whereStorageId($this->getNumericStorageId())
->andWhere($query->expr()->orX(...array_map(function ($key, $value) use ($query) {
return $query->expr()->orX(
$query->expr()->neq($key, $query->createNamedParameter($value)),
@ -512,6 +514,7 @@ class Cache implements ICache {
if ($entry instanceof ICacheEntry) {
$query = $this->getQueryBuilder();
$query->delete('filecache')
->whereStorageId($this->getNumericStorageId())
->whereFileId($entry->getId());
$query->execute();
@ -583,6 +586,7 @@ class Cache implements ICache {
$query = $this->getQueryBuilder();
$query->delete('filecache')
->whereStorageId($this->getNumericStorageId())
->whereParentInParameter('parentIds');
// Sorting before chunking allows the db to find the entries close to each
@ -929,6 +933,7 @@ class Cache implements ICache {
$query = $this->getQueryBuilder();
$query->select('size', 'unencrypted_size')
->from('filecache')
->whereStorageId($this->getNumericStorageId())
->whereParent($id);
if ($ignoreUnknown) {
$query->andWhere($query->expr()->gte('size', $query->createNamedParameter(0)));

8
lib/private/Files/Cache/Propagator.php
View file

@ -186,15 +186,15 @@ class Propagator implements IPropagator {
$query->update('filecache')
->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
->set('etag', $query->expr()->literal(uniqid()))
->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT)))
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')));
$sizeQuery = $this->connection->getQueryBuilder();
$sizeQuery->update('filecache')
->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size')))
->where($query->expr()->eq('storage', $query->expr()->literal($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')))
->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->expr()->literal(-1, IQueryBuilder::PARAM_INT)));
->where($query->expr()->eq('storage', $sizeQuery->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('path_hash', $sizeQuery->createParameter('hash')))
->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->createNamedParameter(-1, IQueryBuilder::PARAM_INT)));
foreach ($this->batch as $item) {
$query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT);

1
lib/private/Files/Cache/QuerySearchHelper.php
View file

@ -110,7 +110,6 @@ class QuerySearchHelper {
$query
->leftJoin('file', 'vcategory_to_object', 'tagmap', $query->expr()->eq('file.fileid', 'tagmap.objid'))
->leftJoin('tagmap', 'vcategory', 'tag', $query->expr()->andX(
$query->expr()->eq('tagmap.type', 'tag.type'),
$query->expr()->eq('tagmap.categoryid', 'tag.id'),
$query->expr()->eq('tag.type', $query->createNamedParameter('files')),
$query->expr()->eq('tag.uid', $query->createNamedParameter($user->getUID()))

10
lib/private/Files/Config/UserMountCache.php
View file

@ -237,7 +237,7 @@ class UserMountCache implements IUserMountCache {
$builder = $this->connection->getQueryBuilder();
$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'mount_provider_class')
->from('mounts', 'm')
->where($builder->expr()->eq('user_id', $builder->createPositionalParameter($userUID)));
->where($builder->expr()->eq('user_id', $builder->createNamedParameter($userUID)));
$result = $query->execute();
$rows = $result->fetchAll();
@ -264,7 +264,7 @@ class UserMountCache implements IUserMountCache {
$builder = $this->connection->getQueryBuilder();
$query = $builder->select('path')
->from('filecache')
->where($builder->expr()->eq('fileid', $builder->createPositionalParameter($info->getRootId())));
->where($builder->expr()->eq('fileid', $builder->createNamedParameter($info->getRootId())));
return $query->executeQuery()->fetchOne() ?: '';
}
@ -278,10 +278,10 @@ class UserMountCache implements IUserMountCache {
$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class')
->from('mounts', 'm')
->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid'))
->where($builder->expr()->eq('storage_id', $builder->createPositionalParameter($numericStorageId, IQueryBuilder::PARAM_INT)));
->where($builder->expr()->eq('storage_id', $builder->createNamedParameter($numericStorageId, IQueryBuilder::PARAM_INT)));
if ($user) {
$query->andWhere($builder->expr()->eq('user_id', $builder->createPositionalParameter($user)));
$query->andWhere($builder->expr()->eq('user_id', $builder->createNamedParameter($user)));
}
$result = $query->execute();
@ -300,7 +300,7 @@ class UserMountCache implements IUserMountCache {
$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class')
->from('mounts', 'm')
->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid'))
->where($builder->expr()->eq('root_id', $builder->createPositionalParameter($rootFileId, IQueryBuilder::PARAM_INT)));
->where($builder->expr()->eq('root_id', $builder->createNamedParameter($rootFileId, IQueryBuilder::PARAM_INT)));
$result = $query->execute();
$rows = $result->fetchAll();

2
tests/lib/Files/Cache/PropagatorTest.php
View file

@ -122,7 +122,7 @@ class PropagatorTest extends TestCase {
foreach ($oldInfos as $i => $oldInfo) {
if ($oldInfo->getPath() !== 'foo/baz') {
$this->assertNotEquals($oldInfo->getEtag(), $newInfos[$i]->getEtag());
$this->assertNotEquals($oldInfo->getEtag(), $newInfos[$i]->getEtag(), "etag for {$oldInfo->getPath()} not updated");
}
}

55
tests/lib/Files/Config/UserMountCacheTest.php
View file

@ -7,11 +7,13 @@
namespace Test\Files\Config;
use OC\DB\Exceptions\DbalException;
use OC\DB\QueryBuilder\Literal;
use OC\Files\Mount\MountPoint;
use OC\Files\Storage\Storage;
use OC\User\Manager;
use OCP\Cache\CappedMemoryCache;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Diagnostics\IEventLogger;
use OCP\EventDispatcher\IEventDispatcher;
use OCP\Files\Config\ICachedMountInfo;
@ -335,29 +337,38 @@ class UserMountCacheTest extends TestCase {
private function createCacheEntry($internalPath, $storageId, $size = 0) {
$internalPath = trim($internalPath, '/');
$inserted = $this->connection->insertIfNotExist('*PREFIX*filecache', [
'storage' => $storageId,
'path' => $internalPath,
'path_hash' => md5($internalPath),
'parent' => -1,
'name' => basename($internalPath),
'mimetype' => 0,
'mimepart' => 0,
'size' => $size,
'storage_mtime' => 0,
'encrypted' => 0,
'unencrypted_size' => 0,
'etag' => '',
'permissions' => 31
], ['storage', 'path_hash']);
if ($inserted) {
$id = (int)$this->connection->lastInsertId('*PREFIX*filecache');
try {
$query = $this->connection->getQueryBuilder();
$query->insert('filecache')
->values([
'storage' => $query->createNamedParameter($storageId),
'path' => $query->createNamedParameter($internalPath),
'path_hash' => $query->createNamedParameter(md5($internalPath)),
'parent' => $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT),
'name' => $query->createNamedParameter(basename($internalPath)),
'mimetype' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
'mimepart' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
'size' => $query->createNamedParameter($size),
'storage_mtime' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
'encrypted' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
'unencrypted_size' => $query->createNamedParameter(0, IQueryBuilder::PARAM_INT),
'etag' => $query->createNamedParameter(''),
'permissions' => $query->createNamedParameter(31, IQueryBuilder::PARAM_INT),
]);
$query->executeStatement();
$id = $query->getLastInsertId();
$this->fileIds[] = $id;
} else {
$sql = 'SELECT `fileid` FROM `*PREFIX*filecache` WHERE `storage` = ? AND `path_hash` =?';
$query = $this->connection->prepare($sql);
$query->execute([$storageId, md5($internalPath)]);
return (int)$query->fetchOne();
} catch (DbalException $e) {
if ($e->getReason() === DbalException::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
$query = $this->connection->getQueryBuilder();
$query->select('fileid')
->from('filecache')
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId)))
->andWhere($query->expr()->eq('path_hash', $query->createNamedParameter(md5($internalPath))));
$id = (int)$query->execute()->fetchColumn();
} else {
throw $e;
}
}
return $id;
}