From abd6b354acc7e930302ffacb2a7ec92dd88fcad7 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 11 Mar 2021 16:37:23 +0100 Subject: [PATCH 1/2] add a prefix index to filecache.path The reason that `filecache.path` hasn't had an index added is the mysql limitation of ~1kb for indexeded fields, which is to small for the `path`, however mysql supports indexing only the first N bytes of a column instead of the entire column, allowing us to add an index even if the column is to long. Because the index doesn't cover the entire column it can't be used in all situations where a normal index would be used, but it does cover the `path like 'folder/path/%'` queries that are used in various places. Sqlite and Postgresql don't support prefix indexes, but they also don't have the 1kb limit and DBAL handles the differences in index creation. Signed-off-by: Robin Appelman --- core/Application.php | 4 ++++ core/Command/Db/AddMissingIndices.php | 7 +++++++ core/Migrations/Version13000Date20170718121200.php | 1 + 3 files changed, 12 insertions(+) diff --git a/core/Application.php b/core/Application.php index 43f32fcbdcc..06c3cc67b93 100644 --- a/core/Application.php +++ b/core/Application.php @@ -118,6 +118,10 @@ class Application extends App { if (!$table->hasIndex('fs_id_storage_size')) { $subject->addHintForMissingSubject($table->getName(), 'fs_id_storage_size'); } + + if (!$table->hasIndex('fs_storage_path_prefix')) { + $subject->addHintForMissingSubject($table->getName(), 'fs_storage_path_prefix'); + } } if ($schema->hasTable('twofactor_providers')) { diff --git a/core/Command/Db/AddMissingIndices.php b/core/Command/Db/AddMissingIndices.php index c4360157f72..449ee03d325 100644 --- a/core/Command/Db/AddMissingIndices.php +++ b/core/Command/Db/AddMissingIndices.php @@ -151,6 +151,13 @@ class AddMissingIndices extends Command { $updated = true; $output->writeln('Filecache table updated successfully.'); } + if (!$table->hasIndex('fs_storage_path_prefix')) { + $output->writeln('Adding additional path index to the filecache table, this can take some time...'); + $table->addIndex(['storage', 'path'], 'fs_storage_path_prefix', [], ['lengths' => [null, 64]]); + $this->connection->migrateToSchema($schema->getWrappedSchema()); + $updated = true; + $output->writeln('Filecache table updated successfully.'); + } } $output->writeln('Check indices of the twofactor_providers table.'); diff --git a/core/Migrations/Version13000Date20170718121200.php b/core/Migrations/Version13000Date20170718121200.php index e8dbf1fc0fb..bb4f7e895a3 100644 --- a/core/Migrations/Version13000Date20170718121200.php +++ b/core/Migrations/Version13000Date20170718121200.php @@ -264,6 +264,7 @@ class Version13000Date20170718121200 extends SimpleMigrationStep { $table->addIndex(['fileid', 'storage', 'size'], 'fs_id_storage_size'); $table->addIndex(['mtime'], 'fs_mtime'); $table->addIndex(['size'], 'fs_size'); + $table->addIndex(['storage', 'path'], 'fs_storage_path_prefix', [], ['lengths' => [null, 64]]); } if (!$schema->hasTable('group_user')) { From 695326534c18749f36e9172e7b7345824213ee60 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Thu, 26 Aug 2021 14:25:17 +0200 Subject: [PATCH 2/2] disable path prefix index on postgresql for now having the index work properly for the queries we need it for requires some additional options which dbal does not support at the momement. to prevent making it harder to add the correct index later on we don't create the index for now on postgresql Signed-off-by: Robin Appelman --- core/Application.php | 3 ++- core/Command/Db/AddMissingIndices.php | 3 ++- .../Migrations/Version13000Date20170718121200.php | 5 ++++- lib/private/DB/SchemaWrapper.php | 13 +++++++++++++ lib/public/DB/ISchemaWrapper.php | 15 ++++++++++++++- 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/core/Application.php b/core/Application.php index 06c3cc67b93..d987f7cfb17 100644 --- a/core/Application.php +++ b/core/Application.php @@ -31,6 +31,7 @@ */ namespace OC\Core; +use Doctrine\DBAL\Platforms\PostgreSQL94Platform; use OC\Authentication\Events\RemoteWipeFinished; use OC\Authentication\Events\RemoteWipeStarted; use OC\Authentication\Listeners\RemoteWipeActivityListener; @@ -119,7 +120,7 @@ class Application extends App { $subject->addHintForMissingSubject($table->getName(), 'fs_id_storage_size'); } - if (!$table->hasIndex('fs_storage_path_prefix')) { + if (!$table->hasIndex('fs_storage_path_prefix') && !$schema->getDatabasePlatform() instanceof PostgreSQL94Platform) { $subject->addHintForMissingSubject($table->getName(), 'fs_storage_path_prefix'); } } diff --git a/core/Command/Db/AddMissingIndices.php b/core/Command/Db/AddMissingIndices.php index 449ee03d325..348b4f94f0d 100644 --- a/core/Command/Db/AddMissingIndices.php +++ b/core/Command/Db/AddMissingIndices.php @@ -33,6 +33,7 @@ declare(strict_types=1); */ namespace OC\Core\Command\Db; +use Doctrine\DBAL\Platforms\PostgreSQL94Platform; use OC\DB\Connection; use OC\DB\SchemaWrapper; use OCP\IDBConnection; @@ -151,7 +152,7 @@ class AddMissingIndices extends Command { $updated = true; $output->writeln('Filecache table updated successfully.'); } - if (!$table->hasIndex('fs_storage_path_prefix')) { + if (!$table->hasIndex('fs_storage_path_prefix') && !$schema->getDatabasePlatform() instanceof PostgreSQL94Platform) { $output->writeln('Adding additional path index to the filecache table, this can take some time...'); $table->addIndex(['storage', 'path'], 'fs_storage_path_prefix', [], ['lengths' => [null, 64]]); $this->connection->migrateToSchema($schema->getWrappedSchema()); diff --git a/core/Migrations/Version13000Date20170718121200.php b/core/Migrations/Version13000Date20170718121200.php index bb4f7e895a3..4fa8fb3d1e9 100644 --- a/core/Migrations/Version13000Date20170718121200.php +++ b/core/Migrations/Version13000Date20170718121200.php @@ -31,6 +31,7 @@ */ namespace OC\Core\Migrations; +use Doctrine\DBAL\Platforms\PostgreSQL94Platform; use OCP\DB\Types; use OCP\DB\ISchemaWrapper; use OCP\IDBConnection; @@ -264,7 +265,9 @@ class Version13000Date20170718121200 extends SimpleMigrationStep { $table->addIndex(['fileid', 'storage', 'size'], 'fs_id_storage_size'); $table->addIndex(['mtime'], 'fs_mtime'); $table->addIndex(['size'], 'fs_size'); - $table->addIndex(['storage', 'path'], 'fs_storage_path_prefix', [], ['lengths' => [null, 64]]); + if (!$schema->getDatabasePlatform() instanceof PostgreSQL94Platform) { + $table->addIndex(['storage', 'path'], 'fs_storage_path_prefix', [], ['lengths' => [null, 64]]); + } } if (!$schema->hasTable('group_user')) { diff --git a/lib/private/DB/SchemaWrapper.php b/lib/private/DB/SchemaWrapper.php index e5f4656c9be..40d41f0dafc 100644 --- a/lib/private/DB/SchemaWrapper.php +++ b/lib/private/DB/SchemaWrapper.php @@ -23,6 +23,8 @@ */ namespace OC\DB; +use Doctrine\DBAL\Exception; +use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Schema\Schema; use OCP\DB\ISchemaWrapper; @@ -129,4 +131,15 @@ class SchemaWrapper implements ISchemaWrapper { public function getTables() { return $this->schema->getTables(); } + + /** + * Gets the DatabasePlatform for the database. + * + * @return AbstractPlatform + * + * @throws Exception + */ + public function getDatabasePlatform() { + return $this->connection->getDatabasePlatform(); + } } diff --git a/lib/public/DB/ISchemaWrapper.php b/lib/public/DB/ISchemaWrapper.php index 2031f807e76..82e4205f1b3 100644 --- a/lib/public/DB/ISchemaWrapper.php +++ b/lib/public/DB/ISchemaWrapper.php @@ -22,6 +22,9 @@ */ namespace OCP\DB; +use Doctrine\DBAL\Exception; +use Doctrine\DBAL\Platforms\AbstractPlatform; + /** * Interface ISchemaWrapper * @@ -81,7 +84,7 @@ interface ISchemaWrapper { * @since 13.0.0 */ public function getTableNames(); - + /** * Gets all table names * @@ -89,4 +92,14 @@ interface ISchemaWrapper { * @since 13.0.0 */ public function getTableNamesWithoutPrefix(); + + /** + * Gets the DatabasePlatform for the database. + * + * @return AbstractPlatform + * + * @throws Exception + * @since 23.0.0 + */ + public function getDatabasePlatform(); }