squash: memory optimizations

fix: optimize FileUtils::getFilesByUser
fix: fix getNodeFromCacheEntryAndMount using relative path
fix: reduce memory usage for fetching cached mount into
fix: improve getMountsForFileId memory usage and performance
fix: fix "new mount" false positives
fix: skip registering mounts if there are no new mount providers
fix: fix oci string length with empty strings

Signed-off-by: Robin Appelman <robin@icewind.nl>

core/Command/Info/FileUtils.php

@@ -46,13 +46,12 @@ class FileUtils {
 
 		$mounts = $this->userMountCache->getMountsForFileId($id);
 		$result = [];
-		foreach ($mounts as $mount) {
-			if (isset($result[$mount->getUser()->getUID()])) {
-				continue;
-			}
-
-			$userFolder = $this->rootFolder->getUserFolder($mount->getUser()->getUID());
-			$result[$mount->getUser()->getUID()] = $userFolder->getById($id);
+		foreach ($mounts as $cachedMount) {
+			$mount = $this->rootFolder->getMount($cachedMount->getMountPoint());
+			$cache = $mount->getStorage()->getCache();
+			$cacheEntry = $cache->get($id);
+			$node = $this->rootFolder->getNodeFromCacheEntryAndMount($cacheEntry, $mount);
+			$result[$cachedMount->getUser()->getUID()][] = $node;
 		}
 
 		return $result;

lib/private/DB/QueryBuilder/FunctionBuilder/OCIFunctionBuilder.php

@@ -80,12 +80,12 @@ class OCIFunctionBuilder extends FunctionBuilder {
 	public function octetLength($field, $alias = ''): IQueryFunction {
 		$alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : '';
 		$quotedName = $this->helper->quoteColumnName($field);
-		return new QueryFunction('LENGTHB(' . $quotedName . ')' . $alias);
+		return new QueryFunction('COALESCE(LENGTHB(' . $quotedName . '), 0)' . $alias);
 	}
 
 	public function charLength($field, $alias = ''): IQueryFunction {
 		$alias = $alias ? (' AS ' . $this->helper->quoteColumnName($alias)) : '';
 		$quotedName = $this->helper->quoteColumnName($field);
-		return new QueryFunction('LENGTH(' . $quotedName . ')' . $alias);
+		return new QueryFunction('COALESCE(LENGTH(' . $quotedName . '), 0)' . $alias);
 	}
 }

lib/private/Files/Config/MountProviderCollection.php

@@ -238,4 +238,8 @@ class MountProviderCollection implements IMountProviderCollection, Emitter {
 	public function getProviders(): array {
 		return $this->providers;
 	}
+
+	public function getHomeProviders(): array {
+		return $this->homeProviders;
+	}
 }

lib/private/Files/Config/UserMountCache.php

@@ -5,10 +5,12 @@
  * SPDX-FileCopyrightText: 2016 ownCloud, Inc.
  * SPDX-License-Identifier: AGPL-3.0-only
  */
+
 namespace OC\Files\Config;
 
 use OC\User\LazyUser;
 use OCP\Cache\CappedMemoryCache;
+use OCP\DB\IResult;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Diagnostics\IEventLogger;
 use OCP\Files\Config\ICachedMountFileInfo;
@@ -29,11 +31,13 @@ class UserMountCache implements IUserMountCache {
 
 	/**
 	 * Cached mount info.
+	 *
 	 * @var CappedMemoryCache<ICachedMountInfo[]>
 	 **/
 	private CappedMemoryCache $mountsForUsers;
 	/**
 	 * fileid => internal path mapping for cached mount info.
+	 *
 	 * @var CappedMemoryCache<string>
 	 **/
 	private CappedMemoryCache $internalPathCache;
@@ -190,6 +194,19 @@ class UserMountCache implements IUserMountCache {
 		$query->execute();
 	}
 
+	/**
+	 * @param IResult $result
+	 * @return CachedMountInfo[]
+	 */
+	private function fetchMountInfo(IResult $result, ?callable $pathCallback = null): array {
+		$mounts = [];
+		while ($row = $result->fetch()) {
+			$mount = $this->dbRowToMountInfo($row, $pathCallback);
+			$mounts[] = $mount;
+		}
+		return $mounts;
+	}
+
 	/**
 	 * @param array $row
 	 * @param (callable(CachedMountInfo): string)|null $pathCallback
@@ -239,19 +256,11 @@ class UserMountCache implements IUserMountCache {
 				->from('mounts', 'm')
 				->where($builder->expr()->eq('user_id', $builder->createNamedParameter($userUID)));
 
-			$result = $query->execute();
-			$rows = $result->fetchAll();
-			$result->closeCursor();
-
-			/** @var array<string, ICachedMountInfo> $mounts */
-			$mounts = [];
-			foreach ($rows as $row) {
-				$mount = $this->dbRowToMountInfo($row, [$this, 'getInternalPathForMountInfo']);
-				if ($mount !== null) {
-					$mounts[$mount->getKey()] = $mount;
-				}
-			}
-			$this->mountsForUsers[$userUID] = $mounts;
+			$mounts = $this->fetchMountInfo($query->execute(), [$this, 'getInternalPathForMountInfo']);
+			$keys = array_map(function (ICachedMountInfo $mount) {
+				return $mount->getKey();
+			}, $mounts);
+			$this->mountsForUsers[$userUID] = array_combine($keys, $mounts);
 		}
 		return $this->mountsForUsers[$userUID];
 	}
@@ -274,8 +283,9 @@ class UserMountCache implements IUserMountCache {
 	 * @return CachedMountInfo[]
 	 */
 	public function getMountsForStorageId($numericStorageId, $user = null) {
+		$mounts = [];
 		$builder = $this->connection->getQueryBuilder();
-		$query = $builder->select('storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class')
+		$query = $builder->select('id', 'storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class')
 			->from('mounts', 'm')
 			->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid'))
 			->where($builder->expr()->eq('storage_id', $builder->createNamedParameter($numericStorageId, IQueryBuilder::PARAM_INT)));
@@ -284,11 +294,7 @@ class UserMountCache implements IUserMountCache {
 			$query->andWhere($builder->expr()->eq('user_id', $builder->createNamedParameter($user)));
 		}
 
-		$result = $query->execute();
-		$rows = $result->fetchAll();
-		$result->closeCursor();
-
-		return array_filter(array_map([$this, 'dbRowToMountInfo'], $rows));
+		return $this->fetchMountInfo($query->executeQuery());
 	}
 
 	/**
@@ -302,11 +308,7 @@ class UserMountCache implements IUserMountCache {
 			->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid'))
 			->where($builder->expr()->eq('root_id', $builder->createNamedParameter($rootFileId, IQueryBuilder::PARAM_INT)));
 
-		$result = $query->execute();
-		$rows = $result->fetchAll();
-		$result->closeCursor();
-
-		return array_filter(array_map([$this, 'dbRowToMountInfo'], $rows));
+		return $this->fetchMountInfo($query->executeQuery());
 	}
 
 	/**
@@ -329,7 +331,7 @@ class UserMountCache implements IUserMountCache {
 				$this->cacheInfoCache[$fileId] = [
 					(int)$row['storage'],
 					(string)$row['path'],
-					(int)$row['mimetype']
+					(int)$row['mimetype'],
 				];
 			} else {
 				throw new NotFoundException('File with id "' . $fileId . '" not found');
@@ -350,34 +352,54 @@ class UserMountCache implements IUserMountCache {
 		} catch (NotFoundException $e) {
 			return [];
 		}
-		$mountsForStorage = $this->getMountsForStorageId($storageId, $user);
 
-		// filter mounts that are from the same storage but not a parent of the file we care about
-		$filteredMounts = array_filter($mountsForStorage, function (ICachedMountInfo $mount) use ($internalPath, $fileId) {
-			if ($fileId === $mount->getRootId()) {
-				return true;
+		$builder = $this->connection->getQueryBuilder();
+		$query = $builder->select('id', 'storage_id', 'root_id', 'user_id', 'mount_point', 'mount_id', 'f.path', 'mount_provider_class')
+			->from('mounts', 'm')
+			->innerJoin('m', 'filecache', 'f', $builder->expr()->eq('m.root_id', 'f.fileid'))
+			->where($builder->expr()->eq('storage_id', $builder->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
+			->andWhere($builder->expr()->orX(
+				// filter mounts that are from the same storage but not a parent of the file we care about
+				$builder->expr()->eq('f.fileid', $builder->createNamedParameter($fileId, IQueryBuilder::PARAM_INT)),
+				$builder->expr()->eq('f.path', $builder->createNamedParameter('')),
+				$builder->expr()->isNull('f.path'),
+				$builder->expr()->eq(
+					$builder->func()->concat('f.path', $builder->createNamedParameter('/')),
+					$builder->func()->substring(
+						$builder->createNamedParameter($internalPath),
+						$builder->createNamedParameter(1, IQueryBuilder::PARAM_INT),
+						$builder->func()->add(
+							$builder->func()->charLength('f.path'),
+							$builder->createNamedParameter(1, IQueryBuilder::PARAM_INT),
+						),
+					),
+				),
+			));
+
+		if ($user) {
+			$query->andWhere($builder->expr()->eq('user_id', $builder->createNamedParameter($user)));
+		}
+
+		$result = $query->executeQuery();
+		$mounts = [];
+		while ($row = $result->fetch()) {
+			$user = $this->userManager->get($row['user_id']);
+
+			if ($user) {
+				$mounts[] = new CachedMountFileInfo(
+					$user,
+					(int)$row['storage_id'],
+					(int)$row['root_id'],
+					$row['mount_point'],
+					$row['mount_id'] ? (int)$row['mount_id'] : null,
+					$row['mount_provider_class'] ?? '',
+					$row['path'] ?? '',
+					$internalPath,
+				);
 			}
-			$internalMountPath = $mount->getRootInternalPath();
+		}
 
-			return $internalMountPath === '' || str_starts_with($internalPath, $internalMountPath . '/');
-		});
-
-		$filteredMounts = array_values(array_filter($filteredMounts, function (ICachedMountInfo $mount) {
-			return $this->userManager->userExists($mount->getUser()->getUID());
-		}));
-
-		return array_map(function (ICachedMountInfo $mount) use ($internalPath) {
-			return new CachedMountFileInfo(
-				$mount->getUser(),
-				$mount->getStorageId(),
-				$mount->getRootId(),
-				$mount->getMountPoint(),
-				$mount->getMountId(),
-				$mount->getMountProvider(),
-				$mount->getRootInternalPath(),
-				$internalPath
-			);
-		}, $filteredMounts);
+		return $mounts;
 	}
 
 	/**
@@ -421,7 +443,7 @@ class UserMountCache implements IUserMountCache {
 
 		$mountPoint = $builder->func()->concat(
 			$builder->func()->concat($slash, 'user_id'),
-			$slash
+			$slash,
 		);
 
 		$userIds = array_map(function (IUser $user) {
@@ -433,7 +455,7 @@ class UserMountCache implements IUserMountCache {
 			->innerJoin('m', 'filecache', 'f',
 				$builder->expr()->andX(
 					$builder->expr()->eq('m.storage_id', 'f.storage'),
-					$builder->expr()->eq('f.path_hash', $builder->createNamedParameter(md5('files')))
+					$builder->expr()->eq('f.path_hash', $builder->createNamedParameter(md5('files'))),
 				))
 			->where($builder->expr()->eq('m.mount_point', $mountPoint))
 			->andWhere($builder->expr()->in('m.user_id', $builder->createNamedParameter($userIds, IQueryBuilder::PARAM_STR_ARRAY)));

lib/private/Files/Node/Root.php

@@ -513,9 +513,9 @@ class Root extends Folder implements IRootFolder {
 		$isDir = $info->getType() === FileInfo::TYPE_FOLDER;
 		$view = new View('');
 		if ($isDir) {
-			return new Folder($this, $view, $path, $info, $parent);
+			return new Folder($this, $view, $fullPath, $info, $parent);
 		} else {
-			return new File($this, $view, $path, $info, $parent);
+			return new File($this, $view, $fullPath, $info, $parent);
 		}
 	}
 }

lib/private/Files/SetupManager.php

@@ -271,18 +271,20 @@ class SetupManager {
 	private function afterUserFullySetup(IUser $user, array $previouslySetupProviders): void {
 		$this->eventLogger->start('fs:setup:user:full:post', 'Housekeeping after user is setup');
 		$userRoot = '/' . $user->getUID() . '/';
-		$mounts = $this->mountManager->getAll();
-		$mounts = array_filter($mounts, function (IMountPoint $mount) use ($userRoot) {
-			return str_starts_with($mount->getMountPoint(), $userRoot);
-		});
-		$allProviders = array_map(function (IMountProvider $provider) {
+		$allProviders = array_map(function ($provider) {
 			return get_class($provider);
-		}, $this->mountProviderCollection->getProviders());
+		}, array_merge($this->mountProviderCollection->getProviders(), $this->mountProviderCollection->getHomeProviders()));
 		$newProviders = array_diff($allProviders, $previouslySetupProviders);
-		$mounts = array_filter($mounts, function (IMountPoint $mount) use ($previouslySetupProviders) {
-			return !in_array($mount->getMountProvider(), $previouslySetupProviders);
-		});
-		$this->userMountCache->registerMounts($user, $mounts, $newProviders);
+		if (count($newProviders) > 0) {
+			$mounts = $this->mountManager->getAll();
+			$mounts = array_filter($mounts, function (IMountPoint $mount) use ($userRoot, $previouslySetupProviders) {
+				if (!str_starts_with($mount->getMountPoint(), $userRoot)) {
+					return false;
+				}
+				return !in_array($mount->getMountProvider(), $previouslySetupProviders);
+			});
+			$this->userMountCache->registerMounts($user, $mounts, $newProviders);
+		}
 
 		$cacheDuration = $this->config->getSystemValueInt('fs_mount_cache_duration', 5 * 60);
 		if ($cacheDuration > 0) {