perf(metadata): Add optimized sharding for metadata deletion

Signed-off-by: Carl Schwan <carlschwan@kde.org>

lib/private/Files/Cache/Cache.php

@@ -635,18 +635,32 @@ class Cache implements ICache {
 		}
 
 		$cacheEntryRemovedEvents = [];
-		foreach (array_combine($deletedIds, $deletedPaths) as $fileId => $filePath) {
+		foreach (array_chunk(array_combine($deletedIds, $deletedPaths), 1000) as $chunk) {
-			$cacheEntryRemovedEvent = new CacheEntryRemovedEvent(
+			/** @var array<int, string> $chunk */
-				$this->storage,
+			foreach ($chunk as $fileId => $filePath) {
-				$filePath,
+				$cacheEntryRemovedEvents[] = new CacheEntryRemovedEvent(
-				$fileId,
+					$this->storage,
-				$this->getNumericStorageId()
+					$filePath,
-			);
+					$fileId,
-			$cacheEntryRemovedEvents[] = $cacheEntryRemovedEvent;
+					$this->getNumericStorageId()
-			$this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent);
+				);
-		}
+			}
-		$this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent($cacheEntryRemovedEvents));
 
+			$exception = null;
+			try {
+				$this->eventDispatcher->dispatchTyped(new CacheEntriesRemovedEvent($cacheEntryRemovedEvents));
+			} catch (\Exception $e) {
+				// still send the other event
+				$exception = $e;
+			}
+			foreach ($cacheEntryRemovedEvents as $cacheEntryRemovedEvent) {
+				$this->eventDispatcher->dispatchTyped($cacheEntryRemovedEvent);
+			}
+
+			if ($exception !== null) {
+				throw $exception;
+			}
+		}
 	}
 
 	/**

lib/private/FilesMetadata/FilesMetadataManager.php

@@ -214,9 +214,9 @@ class FilesMetadataManager implements IFilesMetadataManager {
 		}
 	}
 
-	public function deleteMetadataForFiles(array $fileIds): void {
+	public function deleteMetadataForFiles(int $storage, array $fileIds): void {
 		try {
-			$this->metadataRequestService->dropMetadataForFiles($fileIds);
+			$this->metadataRequestService->dropMetadataForFiles($storage, $fileIds);
 		} catch (Exception $e) {
 			$this->logger->warning('issue while deleteMetadata', ['exception' => $e, 'fileIds' => $fileIds]);
 		}

lib/private/FilesMetadata/Listener/MetadataDelete.php

@@ -33,18 +33,21 @@ class MetadataDelete implements IEventListener {
 		}
 
 		$entries = $event->getCacheEntryRemovedEvents();
-		$fileIds = [];
+		$storageToFileIds = [];
 
 		foreach ($entries as $entry) {
 			try {
-				$fileIds[] = $entry->getFileId();
+				$storageToFileIds[$entry->getStorageId()] ??= [];
+				$storageToFileIds[$entry->getStorageId()][] = $entry->getFileId();
 			} catch (Exception $e) {
 				$this->logger->warning('issue while running MetadataDelete', ['exception' => $e]);
 			}
 		}
 
 		try {
-			$this->filesMetadataManager->deleteMetadataForFiles($fileIds);
+			foreach ($storageToFileIds as $storageId => $fileIds) {
+				$this->filesMetadataManager->deleteMetadataForFiles($storageId, $fileIds);
+			}
 		} catch (Exception $e) {
 			$this->logger->warning('issue while running MetadataDelete', ['exception' => $e]);
 		}

lib/private/FilesMetadata/Service/MetadataRequestService.php

@@ -147,16 +147,16 @@ class MetadataRequestService {
 
 	/**
 	 * @param int[] $fileIds
-	 * @return void
 	 * @throws Exception
 	 */
-	public function dropMetadataForFiles(array $fileIds): void {
+	public function dropMetadataForFiles(int $storage, array $fileIds): void {
 		$chunks = array_chunk($fileIds, 1000);
 
 		foreach ($chunks as $chunk) {
 			$qb = $this->dbConnection->getQueryBuilder();
 			$qb->delete(self::TABLE_METADATA)
-				->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)));
+				->where($qb->expr()->in('file_id', $qb->createNamedParameter($fileIds, IQueryBuilder::PARAM_INT_ARRAY)))
+				->hintShardKey('storage', $storage);
 			$qb->executeStatement();
 		}
 	}

lib/public/Files/Cache/CacheEntriesRemovedEvent.php

@@ -8,26 +8,28 @@ declare(strict_types=1);
  */
 namespace OCP\Files\Cache;
 
+use OCP\AppFramework\Attribute\Listenable;
 use OCP\EventDispatcher\Event;
 
 /**
  * Meta-event wrapping multiple CacheEntryRemovedEvent for when an existing
  * entry in the cache gets removed.
  *
- * @since 32.0.0
+ * @since 34.0.0
  */
-#[\OCP\AppFramework\Attribute\Listenable(since: '32.0.0')]
+#[Listenable(since: '34.0.0')]
 class CacheEntriesRemovedEvent extends Event {
 	/**
-	 * @param CacheEntryRemovedEvent[] $cacheEntryRemovedEvents
+	 * @param ICacheEvent[] $cacheEntryRemovedEvents
 	 */
 	public function __construct(
 		private readonly array $cacheEntryRemovedEvents,
 	) {
+		Event::__construct();
 	}
 
 	/**
-	 * @return CacheEntryRemovedEvent[]
+	 * @return ICacheEvent[]
 	 */
 	public function getCacheEntryRemovedEvents(): array {
 		return $this->cacheEntryRemovedEvents;

lib/public/Files/Cache/CacheEntryRemovedEvent.php

@@ -11,7 +11,7 @@ namespace OCP\Files\Cache;
 /**
  * Event for when an existing entry in the cache gets removed
  *
- * Prefer using \c CacheEntriesRemovedEvent as it is more efficient when deleting
+ * Prefer using CacheEntriesRemovedEvent as it is more efficient when deleting
  * multiple files at the same time.
  *
  * @since 21.0.0

lib/public/FilesMetadata/IFilesMetadataManager.php

@@ -103,11 +103,12 @@ interface IFilesMetadataManager {
 	/**
 	 * Delete metadata and its indexes of multiple file ids
 	 *
+	 * @param int $storage The storage id coresponding to the $fileIds
 	 * @param array<int> $fileIds file ids
 	 * @return void
 	 * @since 32.0.0
 	 */
-	public function deleteMetadataForFiles(array $fileIds): void;
+	public function deleteMetadataForFiles(int $storage, array $fileIds): void;
 
 	/**
 	 * generate and return a MetadataQuery to help building sql queries