From 17b8f15ea807f476fb64784b9a21ce30193448f8 Mon Sep 17 00:00:00 2001 From: Joas Schilling Date: Fri, 9 Feb 2024 13:21:56 +0100 Subject: [PATCH] perf(sharing): Split the getSharedWith() method First of all the method was huge and was handling multiple usecases: - User and group shares - "get all" (mount logic) and "get some" To improve the performance of the "get all" it was extracted into yet another method, to allow to further optimize it. E.g. the query was split into reading the shares and reading the filecache, which will be required for future work with sharding anyway. Without limits we also don't need to sort the entries. Signed-off-by: Joas Schilling --- lib/private/Share20/DefaultShareProvider.php | 377 ++++++++++++++----- 1 file changed, 284 insertions(+), 93 deletions(-) diff --git a/lib/private/Share20/DefaultShareProvider.php b/lib/private/Share20/DefaultShareProvider.php index fb511f9dcbd..a82aba056ea 100644 --- a/lib/private/Share20/DefaultShareProvider.php +++ b/lib/private/Share20/DefaultShareProvider.php @@ -886,11 +886,113 @@ class DefaultShareProvider implements IShareProvider { * @inheritdoc */ public function getSharedWith($userId, $shareType, $node, $limit, $offset) { + if ($shareType === IShare::TYPE_USER) { + if ($limit === -1 && $node === null && $offset === 0) { + return $this->getAllSharedWithUser((string) $userId); + } + return $this->getSharedWithUser($userId, $node, $limit, $offset); + } + if ($shareType === IShare::TYPE_GROUP) { + if ($limit === -1 && $node === null && $offset === 0) { + return $this->getAllSharedWithGroups((string) $userId); + } + return $this->getSharedWithGroups($userId, $node, $limit, $offset); + } + throw new BackendError('Invalid backend'); + } + + /** + * Get limited user-shares shared with the given user + * + * @param string $userId get shares where this user is the recipient + * @param Node|null $node + * @param int $limit The max number of entries returned, -1 for all + * @param int $offset + * @return IShare[] + */ + public function getSharedWithUser($userId, $node, $limit, $offset): array { /** @var Share[] $shares */ $shares = []; - if ($shareType === IShare::TYPE_USER) { - //Get shares directly with this user + //Get shares directly with this user + $qb = $this->dbConn->getQueryBuilder(); + $qb->select('s.*', + 'f.fileid', 'f.path', 'f.permissions AS f_permissions', 'f.storage', 'f.path_hash', + 'f.parent AS f_parent', 'f.name', 'f.mimetype', 'f.mimepart', 'f.size', 'f.mtime', 'f.storage_mtime', + 'f.encrypted', 'f.unencrypted_size', 'f.etag', 'f.checksum' + ) + ->selectAlias('st.id', 'storage_string_id') + ->from('share', 's') + ->leftJoin('s', 'filecache', 'f', $qb->expr()->eq('s.file_source', 'f.fileid')) + ->leftJoin('f', 'storages', 'st', $qb->expr()->eq('f.storage', 'st.numeric_id')); + + // Order by id + $qb->orderBy('s.id'); + + // Set limit and offset + if ($limit !== -1) { + $qb->setMaxResults($limit); + } + $qb->setFirstResult($offset); + + $qb->where($qb->expr()->eq('share_type', $qb->createNamedParameter(IShare::TYPE_USER))) + ->andWhere($qb->expr()->eq('share_with', $qb->createNamedParameter($userId))) + ->andWhere($qb->expr()->orX( + $qb->expr()->eq('item_type', $qb->createNamedParameter('file')), + $qb->expr()->eq('item_type', $qb->createNamedParameter('folder')) + )); + + // Filter by node if provided + if ($node !== null) { + $qb->andWhere($qb->expr()->eq('file_source', $qb->createNamedParameter($node->getId()))); + } + + $cursor = $qb->execute(); + + while ($data = $cursor->fetch()) { + if ($data['fileid'] && $data['path'] === null) { + $data['path'] = (string) $data['path']; + $data['name'] = (string) $data['name']; + $data['checksum'] = (string) $data['checksum']; + } + if ($this->isAccessibleResult($data)) { + $shares[] = $this->createShare($data); + } + } + $cursor->closeCursor(); + + return $shares; + } + + + /** + * Get limited group-shares shared with the groups of a given user + * + * @param string $userId get shares where this user is the recipient + * @param Node|null $node + * @param int $limit The max number of entries returned, -1 for all + * @param int $offset + * @return IShare[] + */ + public function getSharedWithGroups($userId, $node, $limit, $offset): array { + /** @var Share[] $shares */ + $shares = []; + + $user = $this->userManager->get($userId); + $allGroups = ($user instanceof IUser) ? $this->groupManager->getUserGroupIds($user) : []; + + /** @var Share[] $shares2 */ + $shares2 = []; + + $start = 0; + while (true) { + $groups = array_slice($allGroups, $start, 1000); + $start += 1000; + + if ($groups === []) { + break; + } + $qb = $this->dbConn->getQueryBuilder(); $qb->select('s.*', 'f.fileid', 'f.path', 'f.permissions AS f_permissions', 'f.storage', 'f.path_hash', @@ -900,119 +1002,208 @@ class DefaultShareProvider implements IShareProvider { ->selectAlias('st.id', 'storage_string_id') ->from('share', 's') ->leftJoin('s', 'filecache', 'f', $qb->expr()->eq('s.file_source', 'f.fileid')) - ->leftJoin('f', 'storages', 'st', $qb->expr()->eq('f.storage', 'st.numeric_id')); + ->leftJoin('f', 'storages', 'st', $qb->expr()->eq('f.storage', 'st.numeric_id')) + ->orderBy('s.id') + ->setFirstResult(0); - // Order by id - $qb->orderBy('s.id'); - - // Set limit and offset if ($limit !== -1) { - $qb->setMaxResults($limit); + $qb->setMaxResults($limit - count($shares)); } - $qb->setFirstResult($offset); - - $qb->where($qb->expr()->eq('share_type', $qb->createNamedParameter(IShare::TYPE_USER))) - ->andWhere($qb->expr()->eq('share_with', $qb->createNamedParameter($userId))) - ->andWhere($qb->expr()->orX( - $qb->expr()->eq('item_type', $qb->createNamedParameter('file')), - $qb->expr()->eq('item_type', $qb->createNamedParameter('folder')) - )); // Filter by node if provided if ($node !== null) { $qb->andWhere($qb->expr()->eq('file_source', $qb->createNamedParameter($node->getId()))); } - $cursor = $qb->execute(); + $groups = array_filter($groups); + + $qb->andWhere($qb->expr()->eq('share_type', $qb->createNamedParameter(IShare::TYPE_GROUP))) + ->andWhere($qb->expr()->in('share_with', $qb->createNamedParameter( + $groups, + IQueryBuilder::PARAM_STR_ARRAY + ))) + ->andWhere($qb->expr()->orX( + $qb->expr()->eq('item_type', $qb->createNamedParameter('file')), + $qb->expr()->eq('item_type', $qb->createNamedParameter('folder')) + )); + + $cursor = $qb->execute(); while ($data = $cursor->fetch()) { - if ($data['fileid'] && $data['path'] === null) { - $data['path'] = (string) $data['path']; - $data['name'] = (string) $data['name']; - $data['checksum'] = (string) $data['checksum']; + if ($offset > 0) { + $offset--; + continue; } + if ($this->isAccessibleResult($data)) { - $shares[] = $this->createShare($data); + $shares2[] = $this->createShare($data); } } $cursor->closeCursor(); - } elseif ($shareType === IShare::TYPE_GROUP) { - $user = $this->userManager->get($userId); - $allGroups = ($user instanceof IUser) ? $this->groupManager->getUserGroupIds($user) : []; - - /** @var Share[] $shares2 */ - $shares2 = []; - - $start = 0; - while (true) { - $groups = array_slice($allGroups, $start, 1000); - $start += 1000; - - if ($groups === []) { - break; - } - - $qb = $this->dbConn->getQueryBuilder(); - $qb->select('s.*', - 'f.fileid', 'f.path', 'f.permissions AS f_permissions', 'f.storage', 'f.path_hash', - 'f.parent AS f_parent', 'f.name', 'f.mimetype', 'f.mimepart', 'f.size', 'f.mtime', 'f.storage_mtime', - 'f.encrypted', 'f.unencrypted_size', 'f.etag', 'f.checksum' - ) - ->selectAlias('st.id', 'storage_string_id') - ->from('share', 's') - ->leftJoin('s', 'filecache', 'f', $qb->expr()->eq('s.file_source', 'f.fileid')) - ->leftJoin('f', 'storages', 'st', $qb->expr()->eq('f.storage', 'st.numeric_id')) - ->orderBy('s.id') - ->setFirstResult(0); - - if ($limit !== -1) { - $qb->setMaxResults($limit - count($shares)); - } - - // Filter by node if provided - if ($node !== null) { - $qb->andWhere($qb->expr()->eq('file_source', $qb->createNamedParameter($node->getId()))); - } - - - $groups = array_filter($groups); - - $qb->andWhere($qb->expr()->eq('share_type', $qb->createNamedParameter(IShare::TYPE_GROUP))) - ->andWhere($qb->expr()->in('share_with', $qb->createNamedParameter( - $groups, - IQueryBuilder::PARAM_STR_ARRAY - ))) - ->andWhere($qb->expr()->orX( - $qb->expr()->eq('item_type', $qb->createNamedParameter('file')), - $qb->expr()->eq('item_type', $qb->createNamedParameter('folder')) - )); - - $cursor = $qb->execute(); - while ($data = $cursor->fetch()) { - if ($offset > 0) { - $offset--; - continue; - } - - if ($this->isAccessibleResult($data)) { - $shares2[] = $this->createShare($data); - } - } - $cursor->closeCursor(); - } - - /* - * Resolve all group shares to user specific shares - */ - $shares = $this->resolveGroupShares($shares2, $userId); - } else { - throw new BackendError('Invalid backend'); } + /* + * Resolve all group shares to user specific shares + */ + $shares = $this->resolveGroupShares($shares2, $userId); return $shares; } + /** + * Get all user-shares shared with the given user + * + * @param string $userId get shares where this user is the recipient + * @return IShare[] + */ + public function getAllSharedWithUser(string $userId): array { + //Get shares directly with this user + $query = $this->dbConn->getQueryBuilder(); + $query->select('*') + ->from('share') + ->where($query->expr()->eq('share_type', $query->createNamedParameter(IShare::TYPE_USER))) + ->andWhere($query->expr()->eq('share_with', $query->createNamedParameter($userId))); + + /** @var array $shareRows */ + $shareRows = []; + + /** @var array $fileData */ + $fileData = []; + + $result = $query->executeQuery(); + while ($row = $result->fetch()) { + if ($row['item_type'] !== 'file' && $row['item_type'] !== 'folder') { + continue; + } + + $shareRows[(int)$row['id']] = $row; + $fileData[(int)$row['file_source']] = null; + } + $result->closeCursor(); + + if (empty($fileData)) { + return []; + } + + $queryFileCache = $this->dbConn->getQueryBuilder(); + $queryFileCache->select('f.fileid', 'f.path', 'f.permissions AS f_permissions', 'f.storage', 'f.path_hash', + 'f.parent AS f_parent', 'f.name', 'f.mimetype', 'f.mimepart', 'f.size', 'f.mtime', 'f.storage_mtime', + 'f.encrypted', 'f.unencrypted_size', 'f.etag', 'f.checksum') + ->selectAlias('st.id', 'storage_string_id') + ->from('filecache', 'f') + ->leftJoin('f', 'storages', 'st', $queryFileCache->expr()->eq('f.storage', 'st.numeric_id')) + ->where($queryFileCache->expr()->in('f.fileid', $queryFileCache->createParameter('fileIds'))); + + $allFileIds = array_keys($fileData); + foreach (array_chunk($allFileIds, 1000) as $fileIds) { + // Filecache and storage info + $queryFileCache->setParameter('fileIds', $fileIds, IQueryBuilder::PARAM_INT_ARRAY); + + $result = $queryFileCache->executeQuery(); + while ($row = $result->fetch()) { + if (!$this->isAccessibleResult($row)) { + continue; + } + + $fileData[(int) $row['fileid']] = $row; + } + $result->closeCursor(); + } + + /** @var IShare[] $shares */ + $shares = []; + foreach ($shareRows as $row) { + if (empty($fileData[(int)$row['file_source']])) { + continue; + } + $shares[] = $this->createShare(array_merge($row, $fileData[(int)$row['file_source']])); + } + + return $shares; + } + + + /** + * Get all group-shares shared with the groups of a given user + * + * @param string $userId get shares where this user is the recipient + * @return IShare[] + */ + public function getAllSharedWithGroups($userId): array { + $user = $this->userManager->get($userId); + $allGroups = ($user instanceof IUser) ? $this->groupManager->getUserGroupIds($user) : []; + + $query = $this->dbConn->getQueryBuilder(); + $query->select('s.*') + ->from('share', 's') + ->where($query->expr()->eq('s.share_type', $query->createNamedParameter(IShare::TYPE_GROUP))) + ->andWhere($query->expr()->in('s.share_with', $query->createParameter('groups'))); + + /** @var array $shareRows */ + $shareRows = []; + + /** @var array $fileData */ + $fileData = []; + + foreach (array_chunk($allGroups, 1000) as $groups) { + $query->setParameter('groups', $groups, IQueryBuilder::PARAM_STR_ARRAY); + + $result = $query->executeQuery(); + while ($row = $result->fetch()) { + if ($row['item_type'] !== 'file' && $row['item_type'] !== 'folder') { + continue; + } + + $shareRows[(int)$row['id']] = $row; + $fileData[(int)$row['file_source']] = null; + } + $result->closeCursor(); + } + + if (empty($fileData)) { + return []; + } + + $queryFileCache = $this->dbConn->getQueryBuilder(); + $queryFileCache->select('f.fileid', 'f.path', 'f.permissions AS f_permissions', 'f.storage', 'f.path_hash', + 'f.parent AS f_parent', 'f.name', 'f.mimetype', 'f.mimepart', 'f.size', 'f.mtime', 'f.storage_mtime', + 'f.encrypted', 'f.unencrypted_size', 'f.etag', 'f.checksum') + ->selectAlias('st.id', 'storage_string_id') + ->from('filecache', 'f') + ->leftJoin('f', 'storages', 'st', $queryFileCache->expr()->eq('f.storage', 'st.numeric_id')) + ->where($queryFileCache->expr()->in('f.fileid', $queryFileCache->createParameter('fileIds'))); + + $allFileIds = array_keys($fileData); + foreach (array_chunk($allFileIds, 1000) as $fileIds) { + // Filecache and storage info + $queryFileCache->setParameter('fileIds', $fileIds, IQueryBuilder::PARAM_INT_ARRAY); + + $result = $queryFileCache->executeQuery(); + while ($row = $result->fetch()) { + if (!$this->isAccessibleResult($row)) { + continue; + } + + $fileData[(int) $row['fileid']] = $row; + } + $result->closeCursor(); + } + + /** @var IShare[] $shares */ + $shares = []; + foreach ($shareRows as $row) { + if (empty($fileData[(int)$row['file_source']])) { + continue; + } + $shares[] = $this->createShare(array_merge($row, $fileData[(int)$row['file_source']])); + } + + /** + * Resolve all group shares to user specific shares + */ + return $this->resolveGroupShares($shares, $userId); + } + /** * Get a share by token *