diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php index 10cf1753ad9..aba4b935660 100644 --- a/apps/dav/composer/composer/autoload_classmap.php +++ b/apps/dav/composer/composer/autoload_classmap.php @@ -16,6 +16,7 @@ return array( 'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => $baseDir . '/../lib/BackgroundJob/CalendarRetentionJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => $baseDir . '/../lib/BackgroundJob/CleanupDirectLinksJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => $baseDir . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php', + 'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => $baseDir . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php', 'OCA\\DAV\\BackgroundJob\\EventReminderJob' => $baseDir . '/../lib/BackgroundJob/EventReminderJob.php', 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\PruneOutdatedSyncTokensJob' => $baseDir . '/../lib/BackgroundJob/PruneOutdatedSyncTokensJob.php', @@ -270,6 +271,7 @@ return array( 'OCA\\DAV\\Migration\\BuildSocialSearchIndexBackgroundJob' => $baseDir . '/../lib/Migration/BuildSocialSearchIndexBackgroundJob.php', 'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => $baseDir . '/../lib/Migration/CalDAVRemoveEmptyValue.php', 'OCA\\DAV\\Migration\\ChunkCleanup' => $baseDir . '/../lib/Migration/ChunkCleanup.php', + 'OCA\\DAV\\Migration\\DeleteSchedulingObjects' => $baseDir . '/../lib/Migration/DeleteSchedulingObjects.php', 'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => $baseDir . '/../lib/Migration/FixBirthdayCalendarComponent.php', 'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => $baseDir . '/../lib/Migration/RefreshWebcalJobRegistrar.php', 'OCA\\DAV\\Migration\\RegenerateBirthdayCalendars' => $baseDir . '/../lib/Migration/RegenerateBirthdayCalendars.php', diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php index dc9daa1ac72..a7c42971448 100644 --- a/apps/dav/composer/composer/autoload_static.php +++ b/apps/dav/composer/composer/autoload_static.php @@ -31,6 +31,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\BackgroundJob\\CalendarRetentionJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CalendarRetentionJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupDirectLinksJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupDirectLinksJob.php', 'OCA\\DAV\\BackgroundJob\\CleanupInvitationTokenJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/CleanupInvitationTokenJob.php', + 'OCA\\DAV\\BackgroundJob\\DeleteOutdatedSchedulingObjects' => __DIR__ . '/..' . '/../lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php', 'OCA\\DAV\\BackgroundJob\\EventReminderJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/EventReminderJob.php', 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\PruneOutdatedSyncTokensJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/PruneOutdatedSyncTokensJob.php', @@ -285,6 +286,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\Migration\\BuildSocialSearchIndexBackgroundJob' => __DIR__ . '/..' . '/../lib/Migration/BuildSocialSearchIndexBackgroundJob.php', 'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => __DIR__ . '/..' . '/../lib/Migration/CalDAVRemoveEmptyValue.php', 'OCA\\DAV\\Migration\\ChunkCleanup' => __DIR__ . '/..' . '/../lib/Migration/ChunkCleanup.php', + 'OCA\\DAV\\Migration\\DeleteSchedulingObjects' => __DIR__ . '/..' . '/../lib/Migration/DeleteSchedulingObjects.php', 'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => __DIR__ . '/..' . '/../lib/Migration/FixBirthdayCalendarComponent.php', 'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => __DIR__ . '/..' . '/../lib/Migration/RefreshWebcalJobRegistrar.php', 'OCA\\DAV\\Migration\\RegenerateBirthdayCalendars' => __DIR__ . '/..' . '/../lib/Migration/RegenerateBirthdayCalendars.php', diff --git a/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php b/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php new file mode 100644 index 00000000000..fa53a8be4f0 --- /dev/null +++ b/apps/dav/lib/BackgroundJob/DeleteOutdatedSchedulingObjects.php @@ -0,0 +1,35 @@ +setInterval(23 * 60 * 60); + $this->setTimeSensitivity(self::TIME_INSENSITIVE); + } + + /** + * @param array $argument + */ + protected function run($argument): void { + $time = $this->time->getTime() - (60 * 60); + $this->calDavBackend->deleteOutdatedSchedulingObjects($time, 50000); + $this->logger->info("Removed outdated scheduling objects"); + } +} diff --git a/apps/dav/lib/CalDAV/CalDavBackend.php b/apps/dav/lib/CalDAV/CalDavBackend.php index 07dfa9fa84f..889dc842629 100644 --- a/apps/dav/lib/CalDAV/CalDavBackend.php +++ b/apps/dav/lib/CalDAV/CalDavBackend.php @@ -2792,6 +2792,44 @@ class CalDavBackend extends AbstractBackend implements SyncSupport, Subscription ->executeStatement(); } + /** + * Deletes all scheduling objects last modified before $modifiedBefore from the inbox collection. + * + * @param int $modifiedBefore + * @param int $limit + * @return void + */ + public function deleteOutdatedSchedulingObjects(int $modifiedBefore, int $limit): void { + $query = $this->db->getQueryBuilder(); + $query->select('id') + ->from('schedulingobjects') + ->where($query->expr()->lt('lastmodified', $query->createNamedParameter($modifiedBefore))) + ->setMaxResults($limit); + $result = $query->executeQuery(); + $count = $result->rowCount(); + if($count === 0) { + return; + } + $ids = array_map(static function (array $id) { + return (int)$id[0]; + }, $result->fetchAll(\PDO::FETCH_NUM)); + $result->closeCursor(); + + $numDeleted = 0; + $deleteQuery = $this->db->getQueryBuilder(); + $deleteQuery->delete('schedulingobjects') + ->where($deleteQuery->expr()->in('id', $deleteQuery->createParameter('ids'), IQueryBuilder::PARAM_INT_ARRAY)); + foreach(array_chunk($ids, 1000) as $chunk) { + $deleteQuery->setParameter('ids', $chunk, IQueryBuilder::PARAM_INT_ARRAY); + $numDeleted += $deleteQuery->executeStatement(); + } + + if($numDeleted === $limit) { + $this->logger->info("Deleted $limit scheduling objects, continuing with next batch"); + $this->deleteOutdatedSchedulingObjects($modifiedBefore, $limit); + } + } + /** * Creates a new scheduling object. This should land in a users' inbox. * diff --git a/apps/dav/lib/Migration/DeleteSchedulingObjects.php b/apps/dav/lib/Migration/DeleteSchedulingObjects.php new file mode 100644 index 00000000000..3919236788b --- /dev/null +++ b/apps/dav/lib/Migration/DeleteSchedulingObjects.php @@ -0,0 +1,38 @@ +info('Cleaning up old scheduling events'); + $time = $this->time->getTime() - (60 * 60); + $this->calDavBackend->deleteOutdatedSchedulingObjects($time, 50000); + if (!$this->jobList->has(DeleteOutdatedSchedulingObjects::class, null)) { + $output->info('Adding background job to delete old scheduling objects'); + $this->jobList->add(DeleteOutdatedSchedulingObjects::class, null); + } + } +} diff --git a/apps/dav/lib/Migration/Version1004Date20170825134824.php b/apps/dav/lib/Migration/Version1004Date20170825134824.php index a7cbaa78ef2..7321bba62ff 100644 --- a/apps/dav/lib/Migration/Version1004Date20170825134824.php +++ b/apps/dav/lib/Migration/Version1004Date20170825134824.php @@ -383,6 +383,7 @@ class Version1004Date20170825134824 extends SimpleMigrationStep { ]); $table->setPrimaryKey(['id']); $table->addIndex(['principaluri'], 'schedulobj_principuri_index'); + $table->addIndex(['lastmodified'], 'schedulobj_lastmodified_idx'); } if (!$schema->hasTable('cards_properties')) { diff --git a/apps/settings/composer/composer/autoload_classmap.php b/apps/settings/composer/composer/autoload_classmap.php index a1e0b7632ca..8f89ada8994 100644 --- a/apps/settings/composer/composer/autoload_classmap.php +++ b/apps/settings/composer/composer/autoload_classmap.php @@ -79,6 +79,7 @@ return array( 'OCA\\Settings\\SetupChecks\\NeedsSystemAddressBookSync' => $baseDir . '/../lib/SetupChecks/NeedsSystemAddressBookSync.php', 'OCA\\Settings\\SetupChecks\\PhpDefaultCharset' => $baseDir . '/../lib/SetupChecks/PhpDefaultCharset.php', 'OCA\\Settings\\SetupChecks\\PhpOutputBuffering' => $baseDir . '/../lib/SetupChecks/PhpOutputBuffering.php', + 'OCA\\Settings\\SetupChecks\\SchedulingTableSize' => $baseDir . '/../lib/SetupChecks/SchedulingTableSize.php', 'OCA\\Settings\\SetupChecks\\SupportedDatabase' => $baseDir . '/../lib/SetupChecks/SupportedDatabase.php', 'OCA\\Settings\\UserMigration\\AccountMigrator' => $baseDir . '/../lib/UserMigration/AccountMigrator.php', 'OCA\\Settings\\UserMigration\\AccountMigratorException' => $baseDir . '/../lib/UserMigration/AccountMigratorException.php', diff --git a/apps/settings/composer/composer/autoload_static.php b/apps/settings/composer/composer/autoload_static.php index 1eef91c5d91..77ec7c0fb61 100644 --- a/apps/settings/composer/composer/autoload_static.php +++ b/apps/settings/composer/composer/autoload_static.php @@ -94,6 +94,7 @@ class ComposerStaticInitSettings 'OCA\\Settings\\SetupChecks\\NeedsSystemAddressBookSync' => __DIR__ . '/..' . '/../lib/SetupChecks/NeedsSystemAddressBookSync.php', 'OCA\\Settings\\SetupChecks\\PhpDefaultCharset' => __DIR__ . '/..' . '/../lib/SetupChecks/PhpDefaultCharset.php', 'OCA\\Settings\\SetupChecks\\PhpOutputBuffering' => __DIR__ . '/..' . '/../lib/SetupChecks/PhpOutputBuffering.php', + 'OCA\\Settings\\SetupChecks\\SchedulingTableSize' => __DIR__ . '/..' . '/../lib/SetupChecks/SchedulingTableSize.php', 'OCA\\Settings\\SetupChecks\\SupportedDatabase' => __DIR__ . '/..' . '/../lib/SetupChecks/SupportedDatabase.php', 'OCA\\Settings\\UserMigration\\AccountMigrator' => __DIR__ . '/..' . '/../lib/UserMigration/AccountMigrator.php', 'OCA\\Settings\\UserMigration\\AccountMigratorException' => __DIR__ . '/..' . '/../lib/UserMigration/AccountMigratorException.php', diff --git a/apps/settings/lib/Controller/CheckSetupController.php b/apps/settings/lib/Controller/CheckSetupController.php index bf8de16a2b9..4de8e3c2389 100644 --- a/apps/settings/lib/Controller/CheckSetupController.php +++ b/apps/settings/lib/Controller/CheckSetupController.php @@ -68,6 +68,7 @@ use OCA\Settings\SetupChecks\LegacySSEKeyFormat; use OCA\Settings\SetupChecks\PhpDefaultCharset; use OCA\Settings\SetupChecks\PhpOutputBuffering; use OCA\Settings\SetupChecks\SupportedDatabase; +use OCA\Settings\SetupChecks\SchedulingTableSize; use OCP\App\IAppManager; use OCP\AppFramework\Controller; use OCP\AppFramework\Http\DataDisplayResponse; @@ -902,6 +903,7 @@ Raw output $supportedDatabases = new SupportedDatabase($this->l10n, $this->connection); $ldapInvalidUuids = new LdapInvalidUuids($this->appManager, $this->l10n, $this->serverContainer); $needsSystemAddressBookSync = new NeedsSystemAddressBookSync($this->config, $this->l10n); + $schedulingTableSize = new SchedulingTableSize($this->l10n, $this->connection); return new DataResponse( [ @@ -958,6 +960,7 @@ Raw output 'temporaryDirectoryWritable' => $this->isTemporaryDirectoryWritable(), LdapInvalidUuids::class => ['pass' => $ldapInvalidUuids->run(), 'description' => $ldapInvalidUuids->description(), 'severity' => $ldapInvalidUuids->severity()], NeedsSystemAddressBookSync::class => ['pass' => $needsSystemAddressBookSync->run(), 'description' => $needsSystemAddressBookSync->description(), 'severity' => $needsSystemAddressBookSync->severity()], + SchedulingTableSize::class => ['pass' => $schedulingTableSize->run(), 'description' => $schedulingTableSize->description(), 'severity' => $schedulingTableSize->severity()], ] ); } diff --git a/apps/settings/lib/SetupChecks/SchedulingTableSize.php b/apps/settings/lib/SetupChecks/SchedulingTableSize.php new file mode 100644 index 00000000000..ea1908215bf --- /dev/null +++ b/apps/settings/lib/SetupChecks/SchedulingTableSize.php @@ -0,0 +1,44 @@ +l10n = $l10n; + $this->connection = $connection; + } + + public function description(): string { + return $this->l10n->t('You have more than 500 000 rows in the scheduling objects table. Please run the expensive repair jobs via occ maintenance:repair --include-expensive'); + } + + public function severity(): string { + return 'warning'; + } + + public function run(): bool { + $qb = $this->connection->getQueryBuilder(); + $qb->select($qb->func()->count('id')) + ->from('schedulingobjects'); + $query = $qb->executeQuery(); + $count = $query->fetchOne(); + $query->closeCursor(); + + return $count <= 500000; + } +} diff --git a/apps/settings/tests/Controller/CheckSetupControllerTest.php b/apps/settings/tests/Controller/CheckSetupControllerTest.php index 50723703c56..67fd97aaeaa 100644 --- a/apps/settings/tests/Controller/CheckSetupControllerTest.php +++ b/apps/settings/tests/Controller/CheckSetupControllerTest.php @@ -47,6 +47,7 @@ use OCP\AppFramework\Http; use OCP\AppFramework\Http\DataDisplayResponse; use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\Http\RedirectResponse; +use OCP\DB\QueryBuilder\IQueryBuilder; use OCP\EventDispatcher\IEventDispatcher; use OCP\Http\Client\IClientService; use OCP\IConfig; @@ -610,6 +611,9 @@ class CheckSetupControllerTest extends TestCase { $sqlitePlatform = $this->getMockBuilder(SqlitePlatform::class)->getMock(); $this->connection->method('getDatabasePlatform') ->willReturn($sqlitePlatform); + $queryBuilder = $this->getMockBuilder(IQueryBuilder::class)->getMock(); + $this->connection->method('getQueryBuilder') + ->willReturn($queryBuilder); $expected = new DataResponse( [ @@ -671,6 +675,7 @@ class CheckSetupControllerTest extends TestCase { 'temporaryDirectoryWritable' => false, \OCA\Settings\SetupChecks\LdapInvalidUuids::class => ['pass' => true, 'description' => 'Invalid UUIDs of LDAP users or groups have been found. Please review your "Override UUID detection" settings in the Expert part of the LDAP configuration and use "occ ldap:update-uuid" to update them.', 'severity' => 'warning'], \OCA\Settings\SetupChecks\NeedsSystemAddressBookSync::class => ['pass' => true, 'description' => 'The DAV system address book sync has not run yet as your instance has more than 1000 users or because an error occured. Please run it manually by calling occ dav:sync-system-addressbook.', 'severity' => 'warning'], + \OCA\Settings\SetupChecks\SchedulingTableSize::class => ['pass' => true, 'description' => 'You have more than 500 000 rows in the scheduling objects table. Please run the expensive repair jobs via occ maintenance:repair --include-expensive', 'severity' => 'warning'], 'isBruteforceThrottled' => false, 'bruteforceRemoteAddress' => '', ] diff --git a/core/Application.php b/core/Application.php index 592e0929666..ee77e447b03 100644 --- a/core/Application.php +++ b/core/Application.php @@ -201,6 +201,10 @@ class Application extends App { if (!$table->hasIndex('schedulobj_principuri_index')) { $subject->addHintForMissingSubject($table->getName(), 'schedulobj_principuri_index'); } + $table = $schema->getTable('schedulingobjects'); + if (!$table->hasIndex('schedulobj_lastmodified_idx')) { + $subject->addHintForMissingSubject($table->getName(), 'schedulobj_lastmodified_idx'); + } } if ($schema->hasTable('properties')) { diff --git a/core/Command/Db/AddMissingIndices.php b/core/Command/Db/AddMissingIndices.php index 1044cea5547..78cde82ef8a 100644 --- a/core/Command/Db/AddMissingIndices.php +++ b/core/Command/Db/AddMissingIndices.php @@ -403,8 +403,18 @@ class AddMissingIndices extends Command { $output->writeln($sqlQueries); } $updated = true; - $output->writeln('schedulingobjects table updated successfully.'); } + if (!$table->hasIndex('schedulobj_lastmodified_idx')) { + $output->writeln('Adding schedulobj_lastmodified_idx index to the schedulingobjects table, this can take some time...'); + + $table->addIndex(['lastmodified'], 'schedulobj_lastmodified_idx'); + $sqlQueries = $this->connection->migrateToSchema($schema->getWrappedSchema(), $dryRun); + if ($dryRun && $sqlQueries !== null) { + $output->writeln($sqlQueries); + } + $updated = true; + } + $output->writeln('schedulingobjects table updated successfully.'); } $output->writeln('Check indices of the oc_properties table.'); diff --git a/core/js/setupchecks.js b/core/js/setupchecks.js index 3a836cd78c6..613aae1f084 100644 --- a/core/js/setupchecks.js +++ b/core/js/setupchecks.js @@ -543,6 +543,7 @@ OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\SupportedDatabase', messages) OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\LdapInvalidUuids', messages) OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\NeedsSystemAddressBookSync', messages) + OC.SetupChecks.addGenericSetupCheck(data, 'OCA\\Settings\\SetupChecks\\SchedulingTableSize', messages) } else { messages.push({ msg: t('core', 'Error occurred while checking server setup'), diff --git a/lib/private/Repair.php b/lib/private/Repair.php index 99af19ddb79..04f7981db96 100644 --- a/lib/private/Repair.php +++ b/lib/private/Repair.php @@ -81,6 +81,7 @@ use OC\Repair\RepairInvalidShares; use OC\Repair\RepairMimeTypes; use OC\Repair\SqliteAutoincrement; use OC\Template\JSCombiner; +use OCA\DAV\Migration\DeleteSchedulingObjects; use OCP\AppFramework\QueryException; use OCP\AppFramework\Utility\ITimeFactory; use OCP\Collaboration\Resources\IManager; @@ -227,6 +228,7 @@ class Repair implements IOutput { return [ new OldGroupMembershipShares(\OC::$server->getDatabaseConnection(), \OC::$server->getGroupManager()), \OC::$server->get(ValidatePhoneNumber::class), + \OC::$server->get(DeleteSchedulingObjects::class), ]; }