From e4c31b362ec724f83cd2fb0205ea4321496d8d0c Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Sat, 7 Dec 2013 14:44:23 +0100 Subject: [PATCH 01/48] Add command for converting sqlite database to server based one --- core/command/db/convertfromsqlite.php | 164 ++++++++++++++++++++++++++ core/register_command.php | 1 + 2 files changed, 165 insertions(+) create mode 100644 core/command/db/convertfromsqlite.php diff --git a/core/command/db/convertfromsqlite.php b/core/command/db/convertfromsqlite.php new file mode 100644 index 00000000000..85cf1e38c18 --- /dev/null +++ b/core/command/db/convertfromsqlite.php @@ -0,0 +1,164 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + * + */ + +namespace OC\Core\Command\Db; + +use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Output\OutputInterface; + +class ConvertFromSqlite extends Command { + protected function configure() { + $this + ->setName('db:convert-from-sqlite') + ->setDescription('Convert the owncloud sqlite database to the newly configured one') + ->addArgument( + 'type', + InputArgument::REQUIRED, + 'the type of the database to convert to' + ) + ->addArgument( + 'username', + InputArgument::REQUIRED, + 'the username of the database to convert to' + ) + ->addArgument( + 'hostname', + InputArgument::REQUIRED, + 'the hostname of the database to convert to' + ) + ->addArgument( + 'database', + InputArgument::REQUIRED, + 'the name of the database to convert to' + ) + ->addOption( + 'port', + null, + InputOption::VALUE_REQUIRED, + 'the port of the database to convert to' + ) + ->addOption( + 'password', + null, + InputOption::VALUE_REQUIRED, + 'the password of the database to convert to. Will be asked when not specified' + ) + ; + } + + private static $type2driver = array( + 'mysql' => 'pdo_mysql', + 'pgsql' => 'pdo_pgsql', + 'oci' => 'oci8', + 'mssql' => 'pdo_sqlsrv', + ); + protected function execute(InputInterface $input, OutputInterface $output) { + // connect 'from' database + $datadir = \OC_Config::getValue( "datadirectory", \OC::$SERVERROOT.'/data' ); + $name = \OC_Config::getValue( "dbname", "owncloud" ); + $dbfile = $datadir.'/'.$name.'.db'; + $connectionParams = array( + 'path' => $dbfile, + 'driver' => 'pdo_sqlite', + ); + $fromDB = \Doctrine\DBAL\DriverManager::getConnection($connectionParams); + + // connect 'to' database + $type = $input->getArgument('type'); + $username = $input->getArgument('username'); + $hostname = $input->getArgument('hostname'); + $dbname = $input->getArgument('database'); + + if ($input->getOption('password')) { + $password = $input->getOption('password'); + } else { + // TODO: should be moved to the interact function + $dialog = $this->getHelperSet()->get('dialog'); + $password = $dialog->askHiddenResponse( + $output, + 'What is the database password?', + false + ); + } + $connectionParams = array( + 'driver' => self::$type2driver[$type], + 'user' => $username, + 'password' => $password, + 'host' => $hostname, + 'dbname' => $dbname, + ); + if ($input->getOption('port')) { + $connectionParams['port'] = $input->getOption('port'); + } + $toDB = \Doctrine\DBAL\DriverManager::getConnection($connectionParams); + + // create tables in new database + $schemaManager = new \OC\DB\MDB2SchemaManager($toDB); + $schemaManager->createDbFromStructure(\OC::$SERVERROOT.'/db_structure.xml'); + $apps = \OC_App::getEnabledApps(); + foreach($apps as $app) { + if(file_exists(\OC_App::getAppPath($app).'/appinfo/database.xml')) { + $schemaManager->createDbFromStructure(\OC_App::getAppPath($app).'/appinfo/database.xml'); + } + } + + // get tables from 'to' database + $toTables = $this->getTables($toDB); + // get tables from 'from' database + $fromTables = $this->getTables($fromDB); + // warn/fail if there are more tables in 'from' database + $tables = array_diff($fromTables, $toTables); + if (!empty($tables)) { + $output->writeln('The following tables do NOT exist any more: '.join(', ', $tables).''); + $dialog = $this->getHelperSet()->get('dialog'); + if (!$dialog->askConfirmation( + $output, + 'Continue with the convertion?', + false + )) { + return; + } + } + // copy table rows + $tables = array_intersect($toTables, $fromTables); + foreach($tables as $table) { + $output->writeln($table); + $this->copyTable($fromDB, $toDB, $table, $output); + } + } + + private function getTables($db) { + $schemaManager = $db->getSchemaManager(); + return $schemaManager->listTableNames(); + } + + private function copyTable($fromDB, $toDB, $table, $output) { + $progress = $this->getHelperSet()->get('progress'); + $query = 'SELECT COUNT(*) from '.$table; + $count = $fromDB->fetchColumn($query); + $query = 'SELECT * from '.$table; + $statement = $fromDB->executeQuery($query); + $query = 'DELETE FROM '.$table; + $toDB->executeUpdate($query); + $progress->start($output, $count); + if ($count > 100) { + $progress->setRedrawFrequency(5); + } else { + $progress->setRedrawFrequency(1); + } + while($row = $statement->fetch()) { + $progress->advance(); + $toDB->insert($table, $row); + } + $progress->finish(); + } +} \ No newline at end of file diff --git a/core/register_command.php b/core/register_command.php index e4f3b124365..419747ebb44 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -9,6 +9,7 @@ /** @var $application Symfony\Component\Console\Application */ $application->add(new OC\Core\Command\Status); $application->add(new OC\Core\Command\Db\GenerateChangeScript()); +$application->add(new OC\Core\Command\Db\ConvertFromSqlite()); $application->add(new OC\Core\Command\Upgrade()); $application->add(new OC\Core\Command\Maintenance\SingleUser()); $application->add(new OC\Core\Command\App\Disable()); From 22edc42f548a148b323a375ecaf82141f18bc293 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Sat, 7 Dec 2013 15:09:36 +0100 Subject: [PATCH 02/48] Add UTF8 charset to connection params --- core/command/db/convertfromsqlite.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/command/db/convertfromsqlite.php b/core/command/db/convertfromsqlite.php index 85cf1e38c18..d5003edd6e9 100644 --- a/core/command/db/convertfromsqlite.php +++ b/core/command/db/convertfromsqlite.php @@ -99,6 +99,16 @@ class ConvertFromSqlite extends Command { if ($input->getOption('port')) { $connectionParams['port'] = $input->getOption('port'); } + switch ($type) { + case 'mysql': + case 'mssql': + $connectionParams['charset'] = 'UTF8'; + break; + case 'oci': + $connectionParams['charset'] = 'AL32UTF8'; + break; + } + $toDB = \Doctrine\DBAL\DriverManager::getConnection($connectionParams); // create tables in new database From 731e83c35a11657248f1d73c0a68e74573a1c3e8 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Sat, 7 Dec 2013 15:14:54 +0100 Subject: [PATCH 03/48] Save the new database config --- core/command/db/convertfromsqlite.php | 28 ++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/core/command/db/convertfromsqlite.php b/core/command/db/convertfromsqlite.php index d5003edd6e9..fb11409d2c3 100644 --- a/core/command/db/convertfromsqlite.php +++ b/core/command/db/convertfromsqlite.php @@ -138,12 +138,30 @@ class ConvertFromSqlite extends Command { return; } } - // copy table rows - $tables = array_intersect($toTables, $fromTables); - foreach($tables as $table) { - $output->writeln($table); - $this->copyTable($fromDB, $toDB, $table, $output); + // enable maintenance mode to prevent changes + \OC_Config::setValue('maintenance', true); + try { + // copy table rows + $tables = array_intersect($toTables, $fromTables); + foreach($tables as $table) { + $output->writeln($table); + $this->copyTable($fromDB, $toDB, $table, $output); + } + // save new database config + $dbhost = $hostname; + if ($input->getOption('port')) { + $dbhost = $hostname.':'.$input->getOption('port'); + } + \OC_Config::setValue('dbtype', $type); + \OC_Config::setValue('dbname', $dbname); + \OC_Config::setValue('dbhost', $dbhost); + \OC_Config::setValue('dbuser', $username); + \OC_Config::setValue('dbpassword', $password); + } catch(Exception $e) { + \OC_Config::setValue('maintenance', false); + throw $e; } + \OC_Config::setValue('maintenance', false); } private function getTables($db) { From 202e26647e534065271db153fde7207922fa2ecd Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Tue, 24 Dec 2013 13:36:32 +0100 Subject: [PATCH 04/48] Inject config object --- core/command/db/convertfromsqlite.php | 33 +++++++++++++++++++-------- core/register_command.php | 2 +- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/core/command/db/convertfromsqlite.php b/core/command/db/convertfromsqlite.php index fb11409d2c3..de65fe9a883 100644 --- a/core/command/db/convertfromsqlite.php +++ b/core/command/db/convertfromsqlite.php @@ -16,6 +16,19 @@ use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; class ConvertFromSqlite extends Command { + /** + * @var \OC\Config $config + */ + protected $config; + + /** + * @param \OC\Config $config + */ + public function __construct($config) { + $this->config = $config; + parent::__construct(); + } + protected function configure() { $this ->setName('db:convert-from-sqlite') @@ -63,8 +76,8 @@ class ConvertFromSqlite extends Command { ); protected function execute(InputInterface $input, OutputInterface $output) { // connect 'from' database - $datadir = \OC_Config::getValue( "datadirectory", \OC::$SERVERROOT.'/data' ); - $name = \OC_Config::getValue( "dbname", "owncloud" ); + $datadir = $this->config->getValue( "datadirectory", \OC::$SERVERROOT.'/data' ); + $name = $this->config->getValue( "dbname", "owncloud" ); $dbfile = $datadir.'/'.$name.'.db'; $connectionParams = array( 'path' => $dbfile, @@ -139,7 +152,7 @@ class ConvertFromSqlite extends Command { } } // enable maintenance mode to prevent changes - \OC_Config::setValue('maintenance', true); + $this->config->setValue('maintenance', true); try { // copy table rows $tables = array_intersect($toTables, $fromTables); @@ -152,16 +165,16 @@ class ConvertFromSqlite extends Command { if ($input->getOption('port')) { $dbhost = $hostname.':'.$input->getOption('port'); } - \OC_Config::setValue('dbtype', $type); - \OC_Config::setValue('dbname', $dbname); - \OC_Config::setValue('dbhost', $dbhost); - \OC_Config::setValue('dbuser', $username); - \OC_Config::setValue('dbpassword', $password); + $this->config->setValue('dbtype', $type); + $this->config->setValue('dbname', $dbname); + $this->config->setValue('dbhost', $dbhost); + $this->config->setValue('dbuser', $username); + $this->config->setValue('dbpassword', $password); } catch(Exception $e) { - \OC_Config::setValue('maintenance', false); + $this->config->setValue('maintenance', false); throw $e; } - \OC_Config::setValue('maintenance', false); + $this->config->setValue('maintenance', false); } private function getTables($db) { diff --git a/core/register_command.php b/core/register_command.php index 419747ebb44..350a817c02a 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -9,7 +9,7 @@ /** @var $application Symfony\Component\Console\Application */ $application->add(new OC\Core\Command\Status); $application->add(new OC\Core\Command\Db\GenerateChangeScript()); -$application->add(new OC\Core\Command\Db\ConvertFromSqlite()); +$application->add(new OC\Core\Command\Db\ConvertFromSqlite(OC_Config::getObject())); $application->add(new OC\Core\Command\Upgrade()); $application->add(new OC\Core\Command\Maintenance\SingleUser()); $application->add(new OC\Core\Command\App\Disable()); From 4e1a3f212f0c3f3f9e412b67b8deeec0963f15c9 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Tue, 24 Dec 2013 13:44:35 +0100 Subject: [PATCH 05/48] Review points --- core/command/db/convertfromsqlite.php | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/core/command/db/convertfromsqlite.php b/core/command/db/convertfromsqlite.php index de65fe9a883..ed0443e9806 100644 --- a/core/command/db/convertfromsqlite.php +++ b/core/command/db/convertfromsqlite.php @@ -184,22 +184,18 @@ class ConvertFromSqlite extends Command { private function copyTable($fromDB, $toDB, $table, $output) { $progress = $this->getHelperSet()->get('progress'); - $query = 'SELECT COUNT(*) from '.$table; + $query = 'SELECT COUNT(*) FROM '.$table; $count = $fromDB->fetchColumn($query); - $query = 'SELECT * from '.$table; + $query = 'SELECT * FROM '.$table; $statement = $fromDB->executeQuery($query); $query = 'DELETE FROM '.$table; $toDB->executeUpdate($query); $progress->start($output, $count); - if ($count > 100) { - $progress->setRedrawFrequency(5); - } else { - $progress->setRedrawFrequency(1); - } + $progress->setRedrawFrequency($count > 100 ? 5 : 1); while($row = $statement->fetch()) { $progress->advance(); $toDB->insert($table, $row); } $progress->finish(); } -} \ No newline at end of file +} From 1b7eb4dc6cfce382c7a3192bdc8bf6fd52791de1 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Sat, 11 Jan 2014 12:22:23 +0100 Subject: [PATCH 06/48] Quote column names on insert --- core/command/db/convertfromsqlite.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/command/db/convertfromsqlite.php b/core/command/db/convertfromsqlite.php index ed0443e9806..5ae54b68936 100644 --- a/core/command/db/convertfromsqlite.php +++ b/core/command/db/convertfromsqlite.php @@ -194,7 +194,11 @@ class ConvertFromSqlite extends Command { $progress->setRedrawFrequency($count > 100 ? 5 : 1); while($row = $statement->fetch()) { $progress->advance(); - $toDB->insert($table, $row); + $data = array(); + foreach ($row as $columnName => $value) { + $data[$toDB->quoteIdentifier($columnName)] = $value; + } + $toDB->insert($table, $data); } $progress->finish(); } From af3bedf9858b117b676c915829229fb8745a5e36 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Sat, 11 Jan 2014 12:23:28 +0100 Subject: [PATCH 07/48] Add option to remove all the tables from the destination database --- core/command/db/convertfromsqlite.php | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/core/command/db/convertfromsqlite.php b/core/command/db/convertfromsqlite.php index 5ae54b68936..4d5122ca2d7 100644 --- a/core/command/db/convertfromsqlite.php +++ b/core/command/db/convertfromsqlite.php @@ -65,6 +65,12 @@ class ConvertFromSqlite extends Command { InputOption::VALUE_REQUIRED, 'the password of the database to convert to. Will be asked when not specified' ) + ->addOption( + 'clear-schema', + null, + InputOption::VALUE_NONE, + 'remove all tables from the destination database' + ) ; } @@ -124,7 +130,20 @@ class ConvertFromSqlite extends Command { $toDB = \Doctrine\DBAL\DriverManager::getConnection($connectionParams); + // Clearing schema in new database + if ($input->getOption('clear-schema')) { + $schemaManager = $toDB->getSchemaManager(); + $toTables = $schemaManager->listTableNames(); + if (!empty($toTables)) { + $output->writeln('Clearing schema in new database'); + } + foreach($toTables as $table) { + $schemaManager->dropTable($table); + } + } + // create tables in new database + $output->writeln('Creating schema in new database'); $schemaManager = new \OC\DB\MDB2SchemaManager($toDB); $schemaManager->createDbFromStructure(\OC::$SERVERROOT.'/db_structure.xml'); $apps = \OC_App::getEnabledApps(); @@ -188,8 +207,6 @@ class ConvertFromSqlite extends Command { $count = $fromDB->fetchColumn($query); $query = 'SELECT * FROM '.$table; $statement = $fromDB->executeQuery($query); - $query = 'DELETE FROM '.$table; - $toDB->executeUpdate($query); $progress->start($output, $count); $progress->setRedrawFrequency($count > 100 ? 5 : 1); while($row = $statement->fetch()) { From 2638fd47674deaeb630b87f01a46ad94446a2097 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Sat, 11 Jan 2014 12:25:35 +0100 Subject: [PATCH 08/48] Postgresql needs the sequences adjusted after the inserts --- core/command/db/convertfromsqlite.php | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/command/db/convertfromsqlite.php b/core/command/db/convertfromsqlite.php index 4d5122ca2d7..cd3e494d7f2 100644 --- a/core/command/db/convertfromsqlite.php +++ b/core/command/db/convertfromsqlite.php @@ -179,6 +179,18 @@ class ConvertFromSqlite extends Command { $output->writeln($table); $this->copyTable($fromDB, $toDB, $table, $output); } + if ($type == 'pgsql') { + $sequences = $toDB->getSchemaManager()->listSequences(); + foreach($sequences as $sequence) { + $info = $toDB->fetchAssoc('SELECT table_schema, table_name, column_name ' + .'FROM information_schema.columns ' + .'WHERE column_default = ? AND table_catalog = ?', + array("nextval('".$sequence->getName()."'::regclass)", $dbname)); + $table_name = $info['table_name']; + $column_name = $info['column_name']; + $toDB->executeQuery("SELECT setval('" . $sequence->getName() . "', (SELECT MAX(" . $column_name . ") FROM " . $table_name . ")+1)"); + } + } // save new database config $dbhost = $hostname; if ($input->getOption('port')) { From 21426b7ff6c8b84ea3898aa4a95a7fd3c4376ddb Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Fri, 17 Jan 2014 14:54:13 +0100 Subject: [PATCH 09/48] Fixed doctrine code --- 3rdparty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty b/3rdparty index 42efd966284..b4db0b302aa 160000 --- a/3rdparty +++ b/3rdparty @@ -1 +1 @@ -Subproject commit 42efd966284debadf83b761367e529bc45f806d6 +Subproject commit b4db0b302aa8266b067012d0f862fafe70a665e0 From eede20c5acb4135f96f318a4ad0a146dd562861c Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Tue, 11 Feb 2014 17:59:50 +0100 Subject: [PATCH 10/48] Check target DB type --- core/command/db/convertfromsqlite.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/command/db/convertfromsqlite.php b/core/command/db/convertfromsqlite.php index cd3e494d7f2..71706580382 100644 --- a/core/command/db/convertfromsqlite.php +++ b/core/command/db/convertfromsqlite.php @@ -97,6 +97,9 @@ class ConvertFromSqlite extends Command { $hostname = $input->getArgument('hostname'); $dbname = $input->getArgument('database'); + if (!isset(self::$type2driver[$type])) { + throw new InvalidArgumentException('Unknown type: '.$type); + } if ($input->getOption('password')) { $password = $input->getOption('password'); } else { From 3abcd13979660309f9a6d672d3dc64a7c6d784ab Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Tue, 11 Feb 2014 18:01:41 +0100 Subject: [PATCH 11/48] Allow converting from any db type --- .../db/{convertfromsqlite.php => converttype.php} | 15 ++++----------- core/register_command.php | 2 +- 2 files changed, 5 insertions(+), 12 deletions(-) rename core/command/db/{convertfromsqlite.php => converttype.php} (92%) diff --git a/core/command/db/convertfromsqlite.php b/core/command/db/converttype.php similarity index 92% rename from core/command/db/convertfromsqlite.php rename to core/command/db/converttype.php index 71706580382..38527d3d55a 100644 --- a/core/command/db/convertfromsqlite.php +++ b/core/command/db/converttype.php @@ -15,7 +15,7 @@ use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; -class ConvertFromSqlite extends Command { +class ConvertType extends Command { /** * @var \OC\Config $config */ @@ -31,8 +31,8 @@ class ConvertFromSqlite extends Command { protected function configure() { $this - ->setName('db:convert-from-sqlite') - ->setDescription('Convert the owncloud sqlite database to the newly configured one') + ->setName('db:convert-type') + ->setDescription('Convert the owncloud database to the newly configured one') ->addArgument( 'type', InputArgument::REQUIRED, @@ -82,14 +82,7 @@ class ConvertFromSqlite extends Command { ); protected function execute(InputInterface $input, OutputInterface $output) { // connect 'from' database - $datadir = $this->config->getValue( "datadirectory", \OC::$SERVERROOT.'/data' ); - $name = $this->config->getValue( "dbname", "owncloud" ); - $dbfile = $datadir.'/'.$name.'.db'; - $connectionParams = array( - 'path' => $dbfile, - 'driver' => 'pdo_sqlite', - ); - $fromDB = \Doctrine\DBAL\DriverManager::getConnection($connectionParams); + $fromDB = \OC_DB::getConnection(); // connect 'to' database $type = $input->getArgument('type'); diff --git a/core/register_command.php b/core/register_command.php index 736953094b1..a3833214c21 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -9,7 +9,7 @@ /** @var $application Symfony\Component\Console\Application */ $application->add(new OC\Core\Command\Status); $application->add(new OC\Core\Command\Db\GenerateChangeScript()); -$application->add(new OC\Core\Command\Db\ConvertFromSqlite(OC_Config::getObject())); +$application->add(new OC\Core\Command\Db\ConvertType(OC_Config::getObject())); $application->add(new OC\Core\Command\Upgrade()); $application->add(new OC\Core\Command\Maintenance\SingleUser()); $application->add(new OC\Core\Command\App\Disable()); From ae525d1f125483dc14858938c5056f18d63f831e Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Wed, 12 Feb 2014 17:42:55 +0100 Subject: [PATCH 12/48] Fix namespace for Exception --- core/command/db/converttype.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 38527d3d55a..5f59a6be828 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -197,7 +197,7 @@ class ConvertType extends Command { $this->config->setValue('dbhost', $dbhost); $this->config->setValue('dbuser', $username); $this->config->setValue('dbpassword', $password); - } catch(Exception $e) { + } catch(\Exception $e) { $this->config->setValue('maintenance', false); throw $e; } From bcb78e48b283def05d21defaf554f5ce7c578c56 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Mon, 17 Feb 2014 18:02:58 +0100 Subject: [PATCH 13/48] Split execute function into multiple functions --- core/command/db/converttype.php | 168 ++++++++++++++++++-------------- 1 file changed, 94 insertions(+), 74 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 5f59a6be828..370ace7fd0c 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -85,46 +85,7 @@ class ConvertType extends Command { $fromDB = \OC_DB::getConnection(); // connect 'to' database - $type = $input->getArgument('type'); - $username = $input->getArgument('username'); - $hostname = $input->getArgument('hostname'); - $dbname = $input->getArgument('database'); - - if (!isset(self::$type2driver[$type])) { - throw new InvalidArgumentException('Unknown type: '.$type); - } - if ($input->getOption('password')) { - $password = $input->getOption('password'); - } else { - // TODO: should be moved to the interact function - $dialog = $this->getHelperSet()->get('dialog'); - $password = $dialog->askHiddenResponse( - $output, - 'What is the database password?', - false - ); - } - $connectionParams = array( - 'driver' => self::$type2driver[$type], - 'user' => $username, - 'password' => $password, - 'host' => $hostname, - 'dbname' => $dbname, - ); - if ($input->getOption('port')) { - $connectionParams['port'] = $input->getOption('port'); - } - switch ($type) { - case 'mysql': - case 'mssql': - $connectionParams['charset'] = 'UTF8'; - break; - case 'oci': - $connectionParams['charset'] = 'AL32UTF8'; - break; - } - - $toDB = \Doctrine\DBAL\DriverManager::getConnection($connectionParams); + $toDB = $this->getToDBConnection($input, $output); // Clearing schema in new database if ($input->getOption('clear-schema')) { @@ -167,41 +128,52 @@ class ConvertType extends Command { } } // enable maintenance mode to prevent changes - $this->config->setValue('maintenance', true); - try { - // copy table rows - $tables = array_intersect($toTables, $fromTables); - foreach($tables as $table) { - $output->writeln($table); - $this->copyTable($fromDB, $toDB, $table, $output); - } - if ($type == 'pgsql') { - $sequences = $toDB->getSchemaManager()->listSequences(); - foreach($sequences as $sequence) { - $info = $toDB->fetchAssoc('SELECT table_schema, table_name, column_name ' - .'FROM information_schema.columns ' - .'WHERE column_default = ? AND table_catalog = ?', - array("nextval('".$sequence->getName()."'::regclass)", $dbname)); - $table_name = $info['table_name']; - $column_name = $info['column_name']; - $toDB->executeQuery("SELECT setval('" . $sequence->getName() . "', (SELECT MAX(" . $column_name . ") FROM " . $table_name . ")+1)"); - } - } - // save new database config - $dbhost = $hostname; - if ($input->getOption('port')) { - $dbhost = $hostname.':'.$input->getOption('port'); - } - $this->config->setValue('dbtype', $type); - $this->config->setValue('dbname', $dbname); - $this->config->setValue('dbhost', $dbhost); - $this->config->setValue('dbuser', $username); - $this->config->setValue('dbpassword', $password); - } catch(\Exception $e) { - $this->config->setValue('maintenance', false); - throw $e; + $tables = array_intersect($toTables, $fromTables); + $this->convertDB($fromDB, $toDB, $tables, $input, $output); + } + + private function getToDBConnection($input, $output) { + $type = $input->getArgument('type'); + $username = $input->getArgument('username'); + $hostname = $input->getArgument('hostname'); + $dbname = $input->getArgument('database'); + + if (!isset(self::$type2driver[$type])) { + throw new InvalidArgumentException('Unknown type: '.$type); } - $this->config->setValue('maintenance', false); + if ($input->getOption('password')) { + $password = $input->getOption('password'); + } else { + // TODO: should be moved to the interact function + $dialog = $this->getHelperSet()->get('dialog'); + $password = $dialog->askHiddenResponse( + $output, + 'What is the database password?', + false + ); + $input->setOption('password', $password); + } + $connectionParams = array( + 'driver' => self::$type2driver[$type], + 'user' => $username, + 'password' => $password, + 'host' => $hostname, + 'dbname' => $dbname, + ); + if ($input->getOption('port')) { + $connectionParams['port'] = $input->getOption('port'); + } + switch ($type) { + case 'mysql': + case 'mssql': + $connectionParams['charset'] = 'UTF8'; + break; + case 'oci': + $connectionParams['charset'] = 'AL32UTF8'; + break; + } + + return \Doctrine\DBAL\DriverManager::getConnection($connectionParams); } private function getTables($db) { @@ -227,4 +199,52 @@ class ConvertType extends Command { } $progress->finish(); } + + private function convertDB($fromDB, $toDB, $tables, $input, $output) { + $this->config->setValue('maintenance', true); + $type = $input->getArgument('type'); + try { + // copy table rows + foreach($tables as $table) { + $output->writeln($table); + $this->copyTable($fromDB, $toDB, $table, $output); + } + if ($type == 'pgsql') { + $sequences = $toDB->getSchemaManager()->listSequences(); + $dbname = $input->getArgument('database'); + foreach($sequences as $sequence) { + $info = $toDB->fetchAssoc('SELECT table_schema, table_name, column_name ' + .'FROM information_schema.columns ' + .'WHERE column_default = ? AND table_catalog = ?', + array("nextval('".$sequence->getName()."'::regclass)", $dbname)); + $table_name = $info['table_name']; + $column_name = $info['column_name']; + $toDB->executeQuery("SELECT setval('" . $sequence->getName() . "', (SELECT MAX(" . $column_name . ") FROM " . $table_name . ")+1)"); + } + } + // save new database config + $this->saveDBInfo($input); + } catch(\Exception $e) { + $this->config->setValue('maintenance', false); + throw $e; + } + $this->config->setValue('maintenance', false); + } + + private function saveDBInfo($input) { + $type = $input->getArgument('type'); + $username = $input->getArgument('username'); + $dbhost = $input->getArgument('hostname'); + $dbname = $input->getArgument('database'); + $password = $input->getOption('password'); + if ($input->getOption('port')) { + $dbhost .= ':'.$input->getOption('port'); + } + + $this->config->setValue('dbtype', $type); + $this->config->setValue('dbname', $dbname); + $this->config->setValue('dbhost', $dbhost); + $this->config->setValue('dbuser', $username); + $this->config->setValue('dbpassword', $password); + } } From 8a6dcdf4a47fb45d417ca1fd2aec2901c247d16f Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Mon, 17 Feb 2014 18:09:42 +0100 Subject: [PATCH 14/48] Move password interaction to interact function --- core/command/db/converttype.php | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 370ace7fd0c..fb45ab33113 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -29,6 +29,19 @@ class ConvertType extends Command { parent::__construct(); } + protected function interact(InputInterface $input, OutputInterface $output) { + parent::interact($input, $output); + if (!$input->getOption('password')) { + $dialog = $this->getHelperSet()->get('dialog'); + $password = $dialog->askHiddenResponse( + $output, + 'What is the database password?', + false + ); + $input->setOption('password', $password); + } + } + protected function configure() { $this ->setName('db:convert-type') @@ -137,22 +150,11 @@ class ConvertType extends Command { $username = $input->getArgument('username'); $hostname = $input->getArgument('hostname'); $dbname = $input->getArgument('database'); + $password = $input->getOption('password'); if (!isset(self::$type2driver[$type])) { throw new InvalidArgumentException('Unknown type: '.$type); } - if ($input->getOption('password')) { - $password = $input->getOption('password'); - } else { - // TODO: should be moved to the interact function - $dialog = $this->getHelperSet()->get('dialog'); - $password = $dialog->askHiddenResponse( - $output, - 'What is the database password?', - false - ); - $input->setOption('password', $password); - } $connectionParams = array( 'driver' => self::$type2driver[$type], 'user' => $username, From a55c56c9e7c0c78c1592f2501b527b5c977cafb7 Mon Sep 17 00:00:00 2001 From: Bart Visscher Date: Tue, 18 Feb 2014 20:20:58 +0100 Subject: [PATCH 15/48] Use the patches as proposed in doctrine/dbal --- 3rdparty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty b/3rdparty index 3d42b540641..62402242330 160000 --- a/3rdparty +++ b/3rdparty @@ -1 +1 @@ -Subproject commit 3d42b540641cb3f9ce563d5a4ec6b385dbaba742 +Subproject commit 624022423304280dab2cf7ed2367aa7c99ff565c From e768ead82096b4ad885dd15388691e39bb2c55fc Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 31 Mar 2014 17:06:02 +0200 Subject: [PATCH 16/48] Remove whitespace at end of lines. --- core/command/db/converttype.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index fb45ab33113..1f3e296acc2 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -4,7 +4,7 @@ * This file is licensed under the Affero General Public License version 3 or * later. * See the COPYING-README file. - * + * */ namespace OC\Core\Command\Db; From b002f11771317d2098a9d93758ce7c442a76c203 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 31 Mar 2014 17:06:06 +0200 Subject: [PATCH 17/48] Use question color. --- core/command/db/converttype.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 1f3e296acc2..8b9c72d9039 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -35,7 +35,7 @@ class ConvertType extends Command { $dialog = $this->getHelperSet()->get('dialog'); $password = $dialog->askHiddenResponse( $output, - 'What is the database password?', + 'What is the database password?', false ); $input->setOption('password', $password); From 5dbbe6d08b6046777e20761057725f24f841e722 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 31 Mar 2014 18:20:04 +0200 Subject: [PATCH 18/48] Doc blocks for properties do not repeat the property name. --- core/command/db/converttype.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 8b9c72d9039..99f2807feee 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -17,7 +17,7 @@ use Symfony\Component\Console\Output\OutputInterface; class ConvertType extends Command { /** - * @var \OC\Config $config + * @var \OC\Config */ protected $config; From e67cf99cef596da7db5171730523e110a1a055d0 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 31 Mar 2014 18:20:24 +0200 Subject: [PATCH 19/48] \InvalidArgumentException is in root namespace. --- core/command/db/converttype.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 99f2807feee..b445378783a 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -153,7 +153,7 @@ class ConvertType extends Command { $password = $input->getOption('password'); if (!isset(self::$type2driver[$type])) { - throw new InvalidArgumentException('Unknown type: '.$type); + throw new \InvalidArgumentException('Unknown type: '.$type); } $connectionParams = array( 'driver' => self::$type2driver[$type], From a585cec5303a7b3d361559f73bba154fcbe0c978 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 31 Mar 2014 19:27:18 +0200 Subject: [PATCH 20/48] Remove unnecessary and incorrect comments. --- core/command/db/converttype.php | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index b445378783a..387f873adb6 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -94,13 +94,9 @@ class ConvertType extends Command { 'mssql' => 'pdo_sqlsrv', ); protected function execute(InputInterface $input, OutputInterface $output) { - // connect 'from' database $fromDB = \OC_DB::getConnection(); - - // connect 'to' database $toDB = $this->getToDBConnection($input, $output); - // Clearing schema in new database if ($input->getOption('clear-schema')) { $schemaManager = $toDB->getSchemaManager(); $toTables = $schemaManager->listTableNames(); @@ -112,7 +108,6 @@ class ConvertType extends Command { } } - // create tables in new database $output->writeln('Creating schema in new database'); $schemaManager = new \OC\DB\MDB2SchemaManager($toDB); $schemaManager->createDbFromStructure(\OC::$SERVERROOT.'/db_structure.xml'); @@ -123,10 +118,9 @@ class ConvertType extends Command { } } - // get tables from 'to' database $toTables = $this->getTables($toDB); - // get tables from 'from' database $fromTables = $this->getTables($fromDB); + // warn/fail if there are more tables in 'from' database $tables = array_diff($fromTables, $toTables); if (!empty($tables)) { @@ -140,7 +134,6 @@ class ConvertType extends Command { return; } } - // enable maintenance mode to prevent changes $tables = array_intersect($toTables, $fromTables); $this->convertDB($fromDB, $toDB, $tables, $input, $output); } From f9853b253c6ecb9a77a9d1c9006c5f548cdfd04e Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 31 Mar 2014 20:00:44 +0200 Subject: [PATCH 21/48] Deduplicate connection handling code into \OC\DB\ConnectionFactory --- core/command/db/converttype.php | 47 +++------ core/register_command.php | 2 +- lib/private/db.php | 137 ++++++++------------------- lib/private/db/connectionfactory.php | 117 +++++++++++++++++++++++ 4 files changed, 173 insertions(+), 130 deletions(-) create mode 100644 lib/private/db/connectionfactory.php diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 387f873adb6..2a4e6747e65 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -22,10 +22,17 @@ class ConvertType extends Command { protected $config; /** - * @param \OC\Config $config + * @var \OC\DB\ConnectionFactory */ - public function __construct($config) { + protected $connectionFactory; + + /** + * @param \OC\Config $config + * @param \OC\DB\ConnectionFactory $connectionFactory + */ + public function __construct($config, $connectionFactory) { $this->config = $config; + $this->connectionFactory = $connectionFactory; parent::__construct(); } @@ -87,12 +94,6 @@ class ConvertType extends Command { ; } - private static $type2driver = array( - 'mysql' => 'pdo_mysql', - 'pgsql' => 'pdo_pgsql', - 'oci' => 'oci8', - 'mssql' => 'pdo_sqlsrv', - ); protected function execute(InputInterface $input, OutputInterface $output) { $fromDB = \OC_DB::getConnection(); $toDB = $this->getToDBConnection($input, $output); @@ -140,35 +141,17 @@ class ConvertType extends Command { private function getToDBConnection($input, $output) { $type = $input->getArgument('type'); - $username = $input->getArgument('username'); - $hostname = $input->getArgument('hostname'); - $dbname = $input->getArgument('database'); - $password = $input->getOption('password'); - - if (!isset(self::$type2driver[$type])) { - throw new \InvalidArgumentException('Unknown type: '.$type); - } $connectionParams = array( - 'driver' => self::$type2driver[$type], - 'user' => $username, - 'password' => $password, - 'host' => $hostname, - 'dbname' => $dbname, + 'host' => $input->getArgument('hostname'), + 'user' => $input->getArgument('username'), + 'password' => $input->getOption('password'), + 'dbname' => $input->getArgument('database'), + 'tablePrefix' => $this->config->getValue('dbtableprefix', 'oc_'), ); if ($input->getOption('port')) { $connectionParams['port'] = $input->getOption('port'); } - switch ($type) { - case 'mysql': - case 'mssql': - $connectionParams['charset'] = 'UTF8'; - break; - case 'oci': - $connectionParams['charset'] = 'AL32UTF8'; - break; - } - - return \Doctrine\DBAL\DriverManager::getConnection($connectionParams); + return $this->connectionFactory->getConnection($type, $connectionParams); } private function getTables($db) { diff --git a/core/register_command.php b/core/register_command.php index a3833214c21..f1361c859fc 100644 --- a/core/register_command.php +++ b/core/register_command.php @@ -9,7 +9,7 @@ /** @var $application Symfony\Component\Console\Application */ $application->add(new OC\Core\Command\Status); $application->add(new OC\Core\Command\Db\GenerateChangeScript()); -$application->add(new OC\Core\Command\Db\ConvertType(OC_Config::getObject())); +$application->add(new OC\Core\Command\Db\ConvertType(OC_Config::getObject(), new \OC\DB\ConnectionFactory())); $application->add(new OC\Core\Command\Upgrade()); $application->add(new OC\Core\Command\Maintenance\SingleUser()); $application->add(new OC\Core\Command\App\Disable()); diff --git a/lib/private/db.php b/lib/private/db.php index cfdac766bff..11532d9fa54 100644 --- a/lib/private/db.php +++ b/lib/private/db.php @@ -72,102 +72,45 @@ class OC_DB { $port=false; } - // do nothing if the connection already has been established - if (!self::$connection) { - $config = new \Doctrine\DBAL\Configuration(); - $eventManager = new \Doctrine\Common\EventManager(); - switch($type) { - case 'sqlite': - case 'sqlite3': - $datadir=OC_Config::getValue( "datadirectory", OC::$SERVERROOT.'/data' ); - $connectionParams = array( - 'user' => $user, - 'password' => $pass, - 'path' => $datadir.'/'.$name.'.db', - 'driver' => 'pdo_sqlite', - ); - $connectionParams['adapter'] = '\OC\DB\AdapterSqlite'; - $connectionParams['wrapperClass'] = 'OC\DB\Connection'; - break; - case 'mysql': - $connectionParams = array( - 'user' => $user, - 'password' => $pass, - 'host' => $host, - 'port' => $port, - 'dbname' => $name, - 'charset' => 'UTF8', - 'driver' => 'pdo_mysql', - ); - $connectionParams['adapter'] = '\OC\DB\Adapter'; - $connectionParams['wrapperClass'] = 'OC\DB\Connection'; - // Send "SET NAMES utf8". Only required on PHP 5.3 below 5.3.6. - // See http://stackoverflow.com/questions/4361459/php-pdo-charset-set-names#4361485 - $eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\MysqlSessionInit); - break; - case 'pgsql': - $connectionParams = array( - 'user' => $user, - 'password' => $pass, - 'host' => $host, - 'port' => $port, - 'dbname' => $name, - 'driver' => 'pdo_pgsql', - ); - $connectionParams['adapter'] = '\OC\DB\AdapterPgSql'; - $connectionParams['wrapperClass'] = 'OC\DB\Connection'; - break; - case 'oci': - $connectionParams = array( - 'user' => $user, - 'password' => $pass, - 'host' => $host, - 'dbname' => $name, - 'charset' => 'AL32UTF8', - 'driver' => 'oci8', - ); - if (!empty($port)) { - $connectionParams['port'] = $port; - } - $connectionParams['adapter'] = '\OC\DB\AdapterOCI8'; - $connectionParams['wrapperClass'] = 'OC\DB\OracleConnection'; - $eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\OracleSessionInit); - break; - case 'mssql': - $connectionParams = array( - 'user' => $user, - 'password' => $pass, - 'host' => $host, - 'port' => $port, - 'dbname' => $name, - 'charset' => 'UTF8', - 'driver' => 'pdo_sqlsrv', - ); - $connectionParams['adapter'] = '\OC\DB\AdapterSQLSrv'; - $connectionParams['wrapperClass'] = 'OC\DB\Connection'; - break; - default: - return false; - } - $connectionParams['tablePrefix'] = OC_Config::getValue('dbtableprefix', 'oc_' ); - try { - self::$connection = \Doctrine\DBAL\DriverManager::getConnection($connectionParams, $config, $eventManager); - if ($type === 'sqlite' || $type === 'sqlite3') { - // Sqlite doesn't handle query caching and schema changes - // TODO: find a better way to handle this - self::$connection->disableQueryStatementCaching(); - } - } catch(\Doctrine\DBAL\DBALException $e) { - OC_Log::write('core', $e->getMessage(), OC_Log::FATAL); - OC_User::setUserId(null); + $factory = new \OC\DB\ConnectionFactory(); + if (!$factory->isValidType($type)) { + return false; + } - // send http status 503 - header('HTTP/1.1 503 Service Temporarily Unavailable'); - header('Status: 503 Service Temporarily Unavailable'); - OC_Template::printErrorPage('Failed to connect to database'); - die(); + if ($factory->normalizeType($type) === 'sqlite3') { + $datadir = OC_Config::getValue("datadirectory", OC::$SERVERROOT.'/data'); + $connectionParams = array( + 'user' => $user, + 'password' => $pass, + 'path' => $datadir.'/'.$name.'.db', + ); + } else { + $connectionParams = array( + 'user' => $user, + 'password' => $pass, + 'host' => $host, + 'dbname' => $name, + ); + if (!empty($port)) { + $connectionParams['port'] = $port; } } + + $connectionParams['tablePrefix'] = OC_Config::getValue('dbtableprefix', 'oc_'); + + try { + self::$connection = $factory->getConnection($type, $connectionParams); + } catch(\Doctrine\DBAL\DBALException $e) { + OC_Log::write('core', $e->getMessage(), OC_Log::FATAL); + OC_User::setUserId(null); + + // send http status 503 + header('HTTP/1.1 503 Service Temporarily Unavailable'); + header('Status: 503 Service Temporarily Unavailable'); + OC_Template::printErrorPage('Failed to connect to database'); + die(); + } + return true; } @@ -202,12 +145,12 @@ class OC_DB { */ static public function prepare( $query , $limit = null, $offset = null, $isManipulation = null) { self::connect(); - + if ($isManipulation === null) { //try to guess, so we return the number of rows on manipulations $isManipulation = self::isManipulation($query); } - + // return the result try { $result = self::$connection->prepare($query, $limit, $offset); @@ -222,7 +165,7 @@ class OC_DB { /** * tries to guess the type of statement based on the first 10 characters * the current check allows some whitespace but does not work with IF EXISTS or other more complex statements - * + * * @param string $sql * @return bool */ @@ -245,7 +188,7 @@ class OC_DB { } return false; } - + /** * @brief execute a prepared statement, on error write log and throw exception * @param mixed $stmt OC_DB_StatementWrapper, diff --git a/lib/private/db/connectionfactory.php b/lib/private/db/connectionfactory.php new file mode 100644 index 00000000000..14ffe1a4a56 --- /dev/null +++ b/lib/private/db/connectionfactory.php @@ -0,0 +1,117 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC\DB; + +/** +* Takes care of creating and configurating Doctrine connections. +*/ +class ConnectionFactory { + /** + * @var array + * + * Array mapping DBMS type to default connection parameters passed to + * \Doctrine\DBAL\DriverManager::getConnection(). + */ + protected $defaultConnectionParams = array( + 'mssql' => array( + 'adapter' => '\OC\DB\AdapterSQLSrv', + 'charset' => 'UTF8', + 'driver' => 'pdo_sqlsrv', + 'wrapperClass' => 'OC\DB\Connection', + ), + 'mysql' => array( + 'adapter' => '\OC\DB\Adapter', + 'charset' => 'UTF8', + 'driver' => 'pdo_mysql', + 'wrapperClass' => 'OC\DB\Connection', + ), + 'oci' => array( + 'adapter' => '\OC\DB\AdapterOCI8', + 'charset' => 'AL32UTF8', + 'driver' => 'oci8', + 'wrapperClass' => 'OC\DB\OracleConnection', + ), + 'pgsql' => array( + 'adapter' => '\OC\DB\AdapterPgSql', + 'driver' => 'pdo_pgsql', + 'wrapperClass' => 'OC\DB\Connection', + ), + 'sqlite3' => array( + 'adapter' => '\OC\DB\AdapterSqlite', + 'driver' => 'pdo_sqlite', + 'wrapperClass' => 'OC\DB\Connection', + ), + ); + + /** + * @brief Get default connection parameters for a given DBMS. + * @param string $type DBMS type + * @throws \InvalidArgumentException If $type is invalid + * @return array Default connection parameters. + */ + public function getDefaultConnectionParams($type) { + $normalizedType = $this->normalizeType($type); + if (!isset($this->defaultConnectionParams[$normalizedType])) { + throw new \InvalidArgumentException("Unsupported type: $type"); + } + return $this->defaultConnectionParams[$normalizedType]; + } + + /** + * @brief Get default connection parameters for a given DBMS. + * @param string $type DBMS type + * @param array $additionalConnectionParams Additional connection parameters + * @return \OC\DB\Connection + */ + public function getConnection($type, $additionalConnectionParams) { + $normalizedType = $this->normalizeType($type); + $eventManager = new \Doctrine\Common\EventManager(); + switch ($normalizedType) { + case 'mysql': + // Send "SET NAMES utf8". Only required on PHP 5.3 below 5.3.6. + // See http://stackoverflow.com/questions/4361459/php-pdo-charset-set-names#4361485 + $eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\MysqlSessionInit); + break; + case 'oci': + $eventManager->addEventSubscriber(new \Doctrine\DBAL\Event\Listeners\OracleSessionInit); + break; + } + $connection = \Doctrine\DBAL\DriverManager::getConnection( + array_merge($this->getDefaultConnectionParams($type), $additionalConnectionParams), + new \Doctrine\DBAL\Configuration(), + $eventManager + ); + switch ($normalizedType) { + case 'sqlite3': + // Sqlite doesn't handle query caching and schema changes + // TODO: find a better way to handle this + $connection->disableQueryStatementCaching(); + break; + } + return $connection; + } + + /** + * @brief Normalize DBMS type + * @param string $type DBMS type + * @return string Normalized DBMS type + */ + public function normalizeType($type) { + return $type === 'sqlite' ? 'sqlite3' : $type; + } + + /** + * @brief Checks whether the specififed DBMS type is valid. + * @return bool + */ + public function isValidType($type) { + $normalizedType = $this->normalizeType($type); + return isset($this->defaultConnectionParams[$normalizedType]); + } +} From b39a74ff6cc951e8334462ee08728b0d270d6f3a Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 31 Mar 2014 20:51:30 +0200 Subject: [PATCH 22/48] Some more colors. --- core/command/db/converttype.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 2a4e6747e65..464a5db3cdf 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -102,14 +102,14 @@ class ConvertType extends Command { $schemaManager = $toDB->getSchemaManager(); $toTables = $schemaManager->listTableNames(); if (!empty($toTables)) { - $output->writeln('Clearing schema in new database'); + $output->writeln('Clearing schema in new database'); } foreach($toTables as $table) { $schemaManager->dropTable($table); } } - $output->writeln('Creating schema in new database'); + $output->writeln('Creating schema in new database'); $schemaManager = new \OC\DB\MDB2SchemaManager($toDB); $schemaManager->createDbFromStructure(\OC::$SERVERROOT.'/db_structure.xml'); $apps = \OC_App::getEnabledApps(); From 01141e1520bb10c0fe915f1e86d5814004b57277 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 31 Mar 2014 21:07:48 +0200 Subject: [PATCH 23/48] Type hinting. --- core/command/db/converttype.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 464a5db3cdf..c38270d1536 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -9,6 +9,10 @@ namespace OC\Core\Command\Db; +use OC\Config; +use OC\DB\Connection; +use OC\DB\ConnectionFactory; + use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; @@ -30,7 +34,7 @@ class ConvertType extends Command { * @param \OC\Config $config * @param \OC\DB\ConnectionFactory $connectionFactory */ - public function __construct($config, $connectionFactory) { + public function __construct(Config $config, ConnectionFactory $connectionFactory) { $this->config = $config; $this->connectionFactory = $connectionFactory; parent::__construct(); @@ -139,7 +143,7 @@ class ConvertType extends Command { $this->convertDB($fromDB, $toDB, $tables, $input, $output); } - private function getToDBConnection($input, $output) { + private function getToDBConnection(InputInterface $input, OutputInterface $output) { $type = $input->getArgument('type'); $connectionParams = array( 'host' => $input->getArgument('hostname'), @@ -154,12 +158,12 @@ class ConvertType extends Command { return $this->connectionFactory->getConnection($type, $connectionParams); } - private function getTables($db) { + private function getTables(Connection $db) { $schemaManager = $db->getSchemaManager(); return $schemaManager->listTableNames(); } - private function copyTable($fromDB, $toDB, $table, $output) { + private function copyTable(Connection $fromDB, Connection $toDB, $table, OutputInterface $output) { $progress = $this->getHelperSet()->get('progress'); $query = 'SELECT COUNT(*) FROM '.$table; $count = $fromDB->fetchColumn($query); @@ -178,7 +182,7 @@ class ConvertType extends Command { $progress->finish(); } - private function convertDB($fromDB, $toDB, $tables, $input, $output) { + private function convertDB(Connection $fromDB, Connection $toDB, array $tables, InputInterface $input, OutputInterface $output) { $this->config->setValue('maintenance', true); $type = $input->getArgument('type'); try { @@ -209,7 +213,7 @@ class ConvertType extends Command { $this->config->setValue('maintenance', false); } - private function saveDBInfo($input) { + private function saveDBInfo(InputInterface $input) { $type = $input->getArgument('type'); $username = $input->getArgument('username'); $dbhost = $input->getArgument('hostname'); From 5b64a27df9c3f6c64db307753a70515b7123eb5b Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 31 Mar 2014 21:09:54 +0200 Subject: [PATCH 24/48] Undo 3rdparty update. --- 3rdparty | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty b/3rdparty index 62402242330..da3c9f651a2 160000 --- a/3rdparty +++ b/3rdparty @@ -1 +1 @@ -Subproject commit 624022423304280dab2cf7ed2367aa7c99ff565c +Subproject commit da3c9f651a26cf076249ebf25c477e3791e69ca3 From 86ab1cf476ab03d3c3e64ab9603bc1b6a1f27be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Tue, 1 Apr 2014 22:17:31 +0200 Subject: [PATCH 25/48] typos fixed PHPDoc comments added --- core/command/db/converttype.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index c38270d1536..d419ca61a8b 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -43,6 +43,7 @@ class ConvertType extends Command { protected function interact(InputInterface $input, OutputInterface $output) { parent::interact($input, $output); if (!$input->getOption('password')) { + /** @var $dialog \Symfony\Component\Console\Helper\DialogHelper */ $dialog = $this->getHelperSet()->get('dialog'); $password = $dialog->askHiddenResponse( $output, @@ -56,7 +57,7 @@ class ConvertType extends Command { protected function configure() { $this ->setName('db:convert-type') - ->setDescription('Convert the owncloud database to the newly configured one') + ->setDescription('Convert the ownCloud database to the newly configured one') ->addArgument( 'type', InputArgument::REQUIRED, @@ -130,10 +131,11 @@ class ConvertType extends Command { $tables = array_diff($fromTables, $toTables); if (!empty($tables)) { $output->writeln('The following tables do NOT exist any more: '.join(', ', $tables).''); + /** @var $dialog \Symfony\Component\Console\Helper\DialogHelper */ $dialog = $this->getHelperSet()->get('dialog'); if (!$dialog->askConfirmation( $output, - 'Continue with the convertion?', + 'Continue with the conversion?', false )) { return; @@ -164,6 +166,7 @@ class ConvertType extends Command { } private function copyTable(Connection $fromDB, Connection $toDB, $table, OutputInterface $output) { + /** @var $progress \Symfony\Component\Console\Helper\ProgressHelper */ $progress = $this->getHelperSet()->get('progress'); $query = 'SELECT COUNT(*) FROM '.$table; $count = $fromDB->fetchColumn($query); From 3e0858e51f885badb58b4ea3a7666937b3158bff Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Wed, 9 Apr 2014 15:21:57 +0200 Subject: [PATCH 26/48] private -> protected --- core/command/db/converttype.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index d419ca61a8b..81a89de97f8 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -145,7 +145,7 @@ class ConvertType extends Command { $this->convertDB($fromDB, $toDB, $tables, $input, $output); } - private function getToDBConnection(InputInterface $input, OutputInterface $output) { + protected function getToDBConnection(InputInterface $input, OutputInterface $output) { $type = $input->getArgument('type'); $connectionParams = array( 'host' => $input->getArgument('hostname'), @@ -160,12 +160,12 @@ class ConvertType extends Command { return $this->connectionFactory->getConnection($type, $connectionParams); } - private function getTables(Connection $db) { + protected function getTables(Connection $db) { $schemaManager = $db->getSchemaManager(); return $schemaManager->listTableNames(); } - private function copyTable(Connection $fromDB, Connection $toDB, $table, OutputInterface $output) { + protected function copyTable(Connection $fromDB, Connection $toDB, $table, OutputInterface $output) { /** @var $progress \Symfony\Component\Console\Helper\ProgressHelper */ $progress = $this->getHelperSet()->get('progress'); $query = 'SELECT COUNT(*) FROM '.$table; @@ -185,7 +185,7 @@ class ConvertType extends Command { $progress->finish(); } - private function convertDB(Connection $fromDB, Connection $toDB, array $tables, InputInterface $input, OutputInterface $output) { + protected function convertDB(Connection $fromDB, Connection $toDB, array $tables, InputInterface $input, OutputInterface $output) { $this->config->setValue('maintenance', true); $type = $input->getArgument('type'); try { @@ -216,7 +216,7 @@ class ConvertType extends Command { $this->config->setValue('maintenance', false); } - private function saveDBInfo(InputInterface $input) { + protected function saveDBInfo(InputInterface $input) { $type = $input->getArgument('type'); $username = $input->getArgument('username'); $dbhost = $input->getArgument('hostname'); From 03a3f668676486644a12261eba7e33f227ea960d Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Wed, 9 Apr 2014 15:45:30 +0200 Subject: [PATCH 27/48] Move schema clearing to extra method. --- core/command/db/converttype.php | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 81a89de97f8..7e65d1fb663 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -9,6 +9,8 @@ namespace OC\Core\Command\Db; +use Doctrine\DBAL\Schema\AbstractSchemaManager; + use OC\Config; use OC\DB\Connection; use OC\DB\ConnectionFactory; @@ -104,14 +106,7 @@ class ConvertType extends Command { $toDB = $this->getToDBConnection($input, $output); if ($input->getOption('clear-schema')) { - $schemaManager = $toDB->getSchemaManager(); - $toTables = $schemaManager->listTableNames(); - if (!empty($toTables)) { - $output->writeln('Clearing schema in new database'); - } - foreach($toTables as $table) { - $schemaManager->dropTable($table); - } + $this->clearSchema($toDB->getSchemaManager(), $input, $output); } $output->writeln('Creating schema in new database'); @@ -160,6 +155,16 @@ class ConvertType extends Command { return $this->connectionFactory->getConnection($type, $connectionParams); } + protected function clearSchema(AbstractSchemaManager $schemaManager, InputInterface $input, OutputInterface $output) { + $toTables = $schemaManager->listTableNames(); + if (!empty($toTables)) { + $output->writeln('Clearing schema in new database'); + } + foreach($toTables as $table) { + $schemaManager->dropTable($table); + } + } + protected function getTables(Connection $db) { $schemaManager = $db->getSchemaManager(); return $schemaManager->listTableNames(); From 5ef7d69d418ef4323cc3dc93f5ec6d24315eef96 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Wed, 9 Apr 2014 15:57:33 +0200 Subject: [PATCH 28/48] Do not attempt to covert to the same DBMS. --- core/command/db/converttype.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 7e65d1fb663..7e0a41b1b15 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -102,6 +102,14 @@ class ConvertType extends Command { } protected function execute(InputInterface $input, OutputInterface $output) { + if ($input->getArgument('type') === $this->config->getValue('dbtype', '')) { + $output->writeln(sprintf( + 'Can not convert from %1$s to %1$s.', + $input->getArgument('type') + )); + return 1; + } + $fromDB = \OC_DB::getConnection(); $toDB = $this->getToDBConnection($input, $output); From 370593361b89063d7bd65c018ccd537d0ea1c0ab Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Wed, 9 Apr 2014 16:42:22 +0200 Subject: [PATCH 29/48] Add option to create all app schemas instead of just installed app. --- core/command/db/converttype.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 7e0a41b1b15..f406148ebd5 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -98,6 +98,12 @@ class ConvertType extends Command { InputOption::VALUE_NONE, 'remove all tables from the destination database' ) + ->addOption( + 'all-apps', + null, + InputOption::VALUE_NONE, + 'whether to create schema for all apps instead of only installed apps' + ) ; } @@ -120,7 +126,7 @@ class ConvertType extends Command { $output->writeln('Creating schema in new database'); $schemaManager = new \OC\DB\MDB2SchemaManager($toDB); $schemaManager->createDbFromStructure(\OC::$SERVERROOT.'/db_structure.xml'); - $apps = \OC_App::getEnabledApps(); + $apps = $input->getOption('all-apps') ? \OC_App::getAllApps() : \OC_App::getEnabledApps(); foreach($apps as $app) { if(file_exists(\OC_App::getAppPath($app).'/appinfo/database.xml')) { $schemaManager->createDbFromStructure(\OC_App::getAppPath($app).'/appinfo/database.xml'); From 56b6504d59be62da618ad654e37faedb88f32242 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Wed, 9 Apr 2014 16:46:21 +0200 Subject: [PATCH 30/48] Extract schema creation code into its own method. --- core/command/db/converttype.php | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index f406148ebd5..170145940d3 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -123,15 +123,7 @@ class ConvertType extends Command { $this->clearSchema($toDB->getSchemaManager(), $input, $output); } - $output->writeln('Creating schema in new database'); - $schemaManager = new \OC\DB\MDB2SchemaManager($toDB); - $schemaManager->createDbFromStructure(\OC::$SERVERROOT.'/db_structure.xml'); - $apps = $input->getOption('all-apps') ? \OC_App::getAllApps() : \OC_App::getEnabledApps(); - foreach($apps as $app) { - if(file_exists(\OC_App::getAppPath($app).'/appinfo/database.xml')) { - $schemaManager->createDbFromStructure(\OC_App::getAppPath($app).'/appinfo/database.xml'); - } - } + $this->createSchema($toDB, $input, $output); $toTables = $this->getTables($toDB); $fromTables = $this->getTables($fromDB); @@ -154,6 +146,18 @@ class ConvertType extends Command { $this->convertDB($fromDB, $toDB, $tables, $input, $output); } + protected function createSchema(Connection $toDB, InputInterface $input, OutputInterface $output) { + $output->writeln('Creating schema in new database'); + $schemaManager = new \OC\DB\MDB2SchemaManager($toDB); + $schemaManager->createDbFromStructure(\OC::$SERVERROOT.'/db_structure.xml'); + $apps = $input->getOption('all-apps') ? \OC_App::getAllApps() : \OC_App::getEnabledApps(); + foreach($apps as $app) { + if (file_exists(\OC_App::getAppPath($app).'/appinfo/database.xml')) { + $schemaManager->createDbFromStructure(\OC_App::getAppPath($app).'/appinfo/database.xml'); + } + } + } + protected function getToDBConnection(InputInterface $input, OutputInterface $output) { $type = $input->getArgument('type'); $connectionParams = array( From 731571491f04c9a836b5f413d13f67571f1fe0b9 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 11 Apr 2014 21:40:45 +0200 Subject: [PATCH 31/48] Improve message about tables not going to be converted. Mention --all-apps option. --- core/command/db/converttype.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 170145940d3..47e824a3b1b 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -131,7 +131,12 @@ class ConvertType extends Command { // warn/fail if there are more tables in 'from' database $tables = array_diff($fromTables, $toTables); if (!empty($tables)) { - $output->writeln('The following tables do NOT exist any more: '.join(', ', $tables).''); + $output->writeln('The following tables will not be converted:'); + $output->writeln($tables); + if (!$input->getOption('all-apps')) { + $output->writeln('Please note that tables belonging to available but currently not installed apps'); + $output->writeln('can be included by specifying the --all-apps option.'); + } /** @var $dialog \Symfony\Component\Console\Helper\DialogHelper */ $dialog = $this->getHelperSet()->get('dialog'); if (!$dialog->askConfirmation( From 94522586c1717a55a2cc0f052f32ed9389c659fc Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 11 Apr 2014 21:46:02 +0200 Subject: [PATCH 32/48] Better variable names for table arrays. --- core/command/db/converttype.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 47e824a3b1b..863c739918d 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -129,10 +129,10 @@ class ConvertType extends Command { $fromTables = $this->getTables($fromDB); // warn/fail if there are more tables in 'from' database - $tables = array_diff($fromTables, $toTables); - if (!empty($tables)) { + $extraFromTables = array_diff($fromTables, $toTables); + if (!empty($extraFromTables)) { $output->writeln('The following tables will not be converted:'); - $output->writeln($tables); + $output->writeln($extraFromTables); if (!$input->getOption('all-apps')) { $output->writeln('Please note that tables belonging to available but currently not installed apps'); $output->writeln('can be included by specifying the --all-apps option.'); @@ -147,8 +147,8 @@ class ConvertType extends Command { return; } } - $tables = array_intersect($toTables, $fromTables); - $this->convertDB($fromDB, $toDB, $tables, $input, $output); + $intersectingTables = array_intersect($toTables, $fromTables); + $this->convertDB($fromDB, $toDB, $intersectingTables, $input, $output); } protected function createSchema(Connection $toDB, InputInterface $input, OutputInterface $output) { From f35eae64423b6d0346a13ff04fe2106d27276d66 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 11 Apr 2014 21:56:14 +0200 Subject: [PATCH 33/48] Remove unnecessary +1 from PostgreSQL sequence correction query. --- core/command/db/converttype.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 863c739918d..cf0f4437f73 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -232,7 +232,7 @@ class ConvertType extends Command { array("nextval('".$sequence->getName()."'::regclass)", $dbname)); $table_name = $info['table_name']; $column_name = $info['column_name']; - $toDB->executeQuery("SELECT setval('" . $sequence->getName() . "', (SELECT MAX(" . $column_name . ") FROM " . $table_name . ")+1)"); + $toDB->executeQuery("SELECT setval('" . $sequence->getName() . "', (SELECT MAX(" . $column_name . ") FROM " . $table_name . "))"); } } // save new database config From e3b0b40779369457e849396f2dfee3a3d652114f Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 14 Apr 2014 17:52:34 +0200 Subject: [PATCH 34/48] Do not quote table names on Oracle. They appear to be already quoted. Without this commit, Oracle complains as follows: [Doctrine\DBAL\DBALException] An exception occurred while executing 'INSERT INTO "oc_appconfig" ("""appid""", """configkey""", """configvalue""") VALUES (?, ?, ?)' with params ["core", "installedat", "1396972927.537"]: ORA-01741: illegal zero-length identifier --- core/command/db/converttype.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index cf0f4437f73..6ab2d893c60 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -193,7 +193,7 @@ class ConvertType extends Command { return $schemaManager->listTableNames(); } - protected function copyTable(Connection $fromDB, Connection $toDB, $table, OutputInterface $output) { + protected function copyTable(Connection $fromDB, Connection $toDB, $table, InputInterface $input, OutputInterface $output) { /** @var $progress \Symfony\Component\Console\Helper\ProgressHelper */ $progress = $this->getHelperSet()->get('progress'); $query = 'SELECT COUNT(*) FROM '.$table; @@ -204,9 +204,13 @@ class ConvertType extends Command { $progress->setRedrawFrequency($count > 100 ? 5 : 1); while($row = $statement->fetch()) { $progress->advance(); - $data = array(); - foreach ($row as $columnName => $value) { - $data[$toDB->quoteIdentifier($columnName)] = $value; + if ($input->getArgument('type') === 'oci') { + $data = $row; + } else { + $data = array(); + foreach ($row as $columnName => $value) { + $data[$toDB->quoteIdentifier($columnName)] = $value; + } } $toDB->insert($table, $data); } @@ -220,7 +224,7 @@ class ConvertType extends Command { // copy table rows foreach($tables as $table) { $output->writeln($table); - $this->copyTable($fromDB, $toDB, $table, $output); + $this->copyTable($fromDB, $toDB, $table, $input, $output); } if ($type == 'pgsql') { $sequences = $toDB->getSchemaManager()->listSequences(); From b0e6542dc2a72289a396d2e29604e4e83a6136a2 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 14 Apr 2014 18:29:47 +0200 Subject: [PATCH 35/48] Add error handling for --clear-schema on Oracle. --- core/command/db/converttype.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 6ab2d893c60..64178d1ef7f 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -120,6 +120,17 @@ class ConvertType extends Command { $toDB = $this->getToDBConnection($input, $output); if ($input->getOption('clear-schema')) { + if ($input->getArgument('type') === 'oci') { + // Doctrine unconditionally tries (at least in version 2.3) + // to drop sequence triggers when dropping a table, even though + // such triggers may not exist. This results in errors like + // "ORA-04080: trigger 'OC_STORAGES_AI_PK' does not exist". + $output->writeln(sprintf( + 'The --clear-schema option is not supported when converting to Oracle (oci).', + $input->getArgument('type') + )); + return 1; + } $this->clearSchema($toDB->getSchemaManager(), $input, $output); } From 9cc41a24603b250b9c657edfc7007bf10e795899 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Mon, 14 Apr 2014 18:33:21 +0200 Subject: [PATCH 36/48] Move PostgreSQL sequence resynchronisation out into PgSqlTools class. --- core/command/db/converttype.php | 17 ++++---------- lib/private/db/pgsqltools.php | 39 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 lib/private/db/pgsqltools.php diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 64178d1ef7f..3b405764e7f 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -1,6 +1,7 @@ + * Copyright (c) 2014 Andreas Fischer * This file is licensed under the Affero General Public License version 3 or * later. * See the COPYING-README file. @@ -230,25 +231,15 @@ class ConvertType extends Command { protected function convertDB(Connection $fromDB, Connection $toDB, array $tables, InputInterface $input, OutputInterface $output) { $this->config->setValue('maintenance', true); - $type = $input->getArgument('type'); try { // copy table rows foreach($tables as $table) { $output->writeln($table); $this->copyTable($fromDB, $toDB, $table, $input, $output); } - if ($type == 'pgsql') { - $sequences = $toDB->getSchemaManager()->listSequences(); - $dbname = $input->getArgument('database'); - foreach($sequences as $sequence) { - $info = $toDB->fetchAssoc('SELECT table_schema, table_name, column_name ' - .'FROM information_schema.columns ' - .'WHERE column_default = ? AND table_catalog = ?', - array("nextval('".$sequence->getName()."'::regclass)", $dbname)); - $table_name = $info['table_name']; - $column_name = $info['column_name']; - $toDB->executeQuery("SELECT setval('" . $sequence->getName() . "', (SELECT MAX(" . $column_name . ") FROM " . $table_name . "))"); - } + if ($input->getArgument('type') === 'pgsql') { + $tools = new \OC\DB\PgSqlTools; + $tools->resynchronizeDatabaseSequences($toDB); } // save new database config $this->saveDBInfo($input); diff --git a/lib/private/db/pgsqltools.php b/lib/private/db/pgsqltools.php new file mode 100644 index 00000000000..01e76148a2c --- /dev/null +++ b/lib/private/db/pgsqltools.php @@ -0,0 +1,39 @@ + + * This file is licensed under the Affero General Public License version 3 or + * later. + * See the COPYING-README file. + */ + +namespace OC\DB; + +/** +* Various PostgreSQL specific helper functions. +*/ +class PgSqlTools { + /** + * @brief Resynchronizes all sequences of a database after using INSERTs + * without leaving out the auto-incremented column. + * @param \OC\DB\Connection $conn + * @return null + */ + public function resynchronizeDatabaseSequences(Connection $conn) { + $databaseName = $conn->getDatabase(); + foreach ($conn->getSchemaManager()->listSequences() as $sequence) { + $sequenceName = $sequence->getName(); + $sqlInfo = 'SELECT table_schema, table_name, column_name + FROM information_schema.columns + WHERE column_default = ? AND table_catalog = ?'; + $sequenceInfo = $conn->fetchAssoc($sqlInfo, array( + "nextval('$sequenceName'::regclass)", + $databaseName + )); + $tableName = $sequenceInfo['table_name']; + $columnName = $sequenceInfo['column_name']; + $sqlMaxId = "SELECT MAX($columnName) FROM $tableName"; + $sqlSetval = "SELECT setval('$sequenceName', ($sqlMaxId))"; + $conn->executeQuery($sqlSetval); + } + } +} From 714343cd74af55c37c7eccb2f076013dc94db898 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 15 Apr 2014 16:29:43 +0200 Subject: [PATCH 37/48] Add Bart to PgSqlTools copyright. --- lib/private/db/pgsqltools.php | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/private/db/pgsqltools.php b/lib/private/db/pgsqltools.php index 01e76148a2c..c3ac140594d 100644 --- a/lib/private/db/pgsqltools.php +++ b/lib/private/db/pgsqltools.php @@ -1,5 +1,6 @@ * Copyright (c) 2014 Andreas Fischer * This file is licensed under the Affero General Public License version 3 or * later. From 9823d54bb8b88adeae9d976c57dc3f366a3d4f3b Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 15 Apr 2014 17:05:31 +0200 Subject: [PATCH 38/48] Use a variable for the DBMS tyoe we are converting to. --- core/command/db/converttype.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 3b405764e7f..cb6c7007380 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -109,10 +109,11 @@ class ConvertType extends Command { } protected function execute(InputInterface $input, OutputInterface $output) { - if ($input->getArgument('type') === $this->config->getValue('dbtype', '')) { + $type = $input->getArgument('type'); + if ($type === $this->config->getValue('dbtype', '')) { $output->writeln(sprintf( 'Can not convert from %1$s to %1$s.', - $input->getArgument('type') + $type )); return 1; } @@ -121,14 +122,14 @@ class ConvertType extends Command { $toDB = $this->getToDBConnection($input, $output); if ($input->getOption('clear-schema')) { - if ($input->getArgument('type') === 'oci') { + if ($type === 'oci') { // Doctrine unconditionally tries (at least in version 2.3) // to drop sequence triggers when dropping a table, even though // such triggers may not exist. This results in errors like // "ORA-04080: trigger 'OC_STORAGES_AI_PK' does not exist". $output->writeln(sprintf( 'The --clear-schema option is not supported when converting to Oracle (oci).', - $input->getArgument('type') + $type )); return 1; } From 5fe25868bcfd48cafe0fd569de13a593f6b46738 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 15 Apr 2014 17:14:26 +0200 Subject: [PATCH 39/48] Add message for converting to SQLite being unsupported. --- core/command/db/converttype.php | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index cb6c7007380..cf867d93924 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -110,6 +110,13 @@ class ConvertType extends Command { protected function execute(InputInterface $input, OutputInterface $output) { $type = $input->getArgument('type'); + if ($this->connectionFactory->normalizeType($type) === 'sqlite3') { + $output->writeln(sprintf( + 'Converting to SQLite (sqlite3) is currently not supported.', + $type + )); + return 1; + } if ($type === $this->config->getValue('dbtype', '')) { $output->writeln(sprintf( 'Can not convert from %1$s to %1$s.', From 6b2876c64dece60eaf43558a476dea5008023388 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 15 Apr 2014 17:19:47 +0200 Subject: [PATCH 40/48] Move (extended) input validation out into validateInput() method. --- core/command/db/converttype.php | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index cf867d93924..1fd546a7a87 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -108,7 +108,7 @@ class ConvertType extends Command { ; } - protected function execute(InputInterface $input, OutputInterface $output) { + protected function validateInput(InputInterface $input, OutputInterface $output) { $type = $input->getArgument('type'); if ($this->connectionFactory->normalizeType($type) === 'sqlite3') { $output->writeln(sprintf( @@ -124,22 +124,29 @@ class ConvertType extends Command { )); return 1; } + if ($type === 'oci' && $input->getOption('clear-schema')) { + // Doctrine unconditionally tries (at least in version 2.3) + // to drop sequence triggers when dropping a table, even though + // such triggers may not exist. This results in errors like + // "ORA-04080: trigger 'OC_STORAGES_AI_PK' does not exist". + $output->writeln(sprintf( + 'The --clear-schema option is not supported when converting to Oracle (oci).', + $type + )); + return 1; + } + } + + protected function execute(InputInterface $input, OutputInterface $output) { + $inputError = $this->validateInput($input, $output); + if ($inputError) { + return $inputError; + } $fromDB = \OC_DB::getConnection(); $toDB = $this->getToDBConnection($input, $output); if ($input->getOption('clear-schema')) { - if ($type === 'oci') { - // Doctrine unconditionally tries (at least in version 2.3) - // to drop sequence triggers when dropping a table, even though - // such triggers may not exist. This results in errors like - // "ORA-04080: trigger 'OC_STORAGES_AI_PK' does not exist". - $output->writeln(sprintf( - 'The --clear-schema option is not supported when converting to Oracle (oci).', - $type - )); - return 1; - } $this->clearSchema($toDB->getSchemaManager(), $input, $output); } From 8e758513c8f7a28300d006fd8b46a1bd01264606 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 15 Apr 2014 17:21:55 +0200 Subject: [PATCH 41/48] Remove unnecessary sprintf. --- core/command/db/converttype.php | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 1fd546a7a87..3382ebb0179 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -111,10 +111,7 @@ class ConvertType extends Command { protected function validateInput(InputInterface $input, OutputInterface $output) { $type = $input->getArgument('type'); if ($this->connectionFactory->normalizeType($type) === 'sqlite3') { - $output->writeln(sprintf( - 'Converting to SQLite (sqlite3) is currently not supported.', - $type - )); + $output->writeln('Converting to SQLite (sqlite3) is currently not supported.'); return 1; } if ($type === $this->config->getValue('dbtype', '')) { @@ -129,10 +126,7 @@ class ConvertType extends Command { // to drop sequence triggers when dropping a table, even though // such triggers may not exist. This results in errors like // "ORA-04080: trigger 'OC_STORAGES_AI_PK' does not exist". - $output->writeln(sprintf( - 'The --clear-schema option is not supported when converting to Oracle (oci).', - $type - )); + $output->writeln('The --clear-schema option is not supported when converting to Oracle (oci).'); return 1; } } From d2c7a8ee59d57be1441f771ff00350d5f677999f Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 15 Apr 2014 17:30:43 +0200 Subject: [PATCH 42/48] Do not ask for password before input parameter validation. --- core/command/db/converttype.php | 45 ++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 3382ebb0179..8f3047b8a0b 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -43,20 +43,6 @@ class ConvertType extends Command { parent::__construct(); } - protected function interact(InputInterface $input, OutputInterface $output) { - parent::interact($input, $output); - if (!$input->getOption('password')) { - /** @var $dialog \Symfony\Component\Console\Helper\DialogHelper */ - $dialog = $this->getHelperSet()->get('dialog'); - $password = $dialog->askHiddenResponse( - $output, - 'What is the database password?', - false - ); - $input->setOption('password', $password); - } - } - protected function configure() { $this ->setName('db:convert-type') @@ -91,7 +77,7 @@ class ConvertType extends Command { 'password', null, InputOption::VALUE_REQUIRED, - 'the password of the database to convert to. Will be asked when not specified' + 'the password of the database to convert to. Will be asked when not specified. Can also be passed via stdin.' ) ->addOption( 'clear-schema', @@ -131,12 +117,41 @@ class ConvertType extends Command { } } + protected function readPassword(InputInterface $input, OutputInterface $output) { + // Explicitly specified password + if ($input->getOption('password')) { + return; + } + + // Read from stdin + $password = file_get_contents('php://stdin'); + if (trim($password) !== '') { + $input->setOption('password', $password); + return; + } + + // Read password by interacting + if ($input->isInteractive()) { + /** @var $dialog \Symfony\Component\Console\Helper\DialogHelper */ + $dialog = $this->getHelperSet()->get('dialog'); + $password = $dialog->askHiddenResponse( + $output, + 'What is the database password?', + false + ); + $input->setOption('password', $password); + return; + } + } + protected function execute(InputInterface $input, OutputInterface $output) { $inputError = $this->validateInput($input, $output); if ($inputError) { return $inputError; } + $this->readPassword($input, $output); + $fromDB = \OC_DB::getConnection(); $toDB = $this->getToDBConnection($input, $output); From 854dd8fa7c793fdca6d9077c3519f4d55ad20eb9 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 15 Apr 2014 18:04:54 +0200 Subject: [PATCH 43/48] Do not block when nothing is passed via stdin. --- core/command/db/converttype.php | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 8f3047b8a0b..02dc45383e8 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -123,8 +123,11 @@ class ConvertType extends Command { return; } - // Read from stdin + // Read from stdin. stream_set_blocking is used to prevent blocking + // when nothing is passed via stdin. + stream_set_blocking(STDIN, 0); $password = file_get_contents('php://stdin'); + stream_set_blocking(STDIN, 1); if (trim($password) !== '') { $input->setOption('password', $password); return; From f1d05d204e5ce788ffcfaed04d3c4a06233eb231 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 15 Apr 2014 18:14:26 +0200 Subject: [PATCH 44/48] Use exceptions for in input validation. --- core/command/db/converttype.php | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 02dc45383e8..6fcaeacec49 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -97,23 +97,24 @@ class ConvertType extends Command { protected function validateInput(InputInterface $input, OutputInterface $output) { $type = $input->getArgument('type'); if ($this->connectionFactory->normalizeType($type) === 'sqlite3') { - $output->writeln('Converting to SQLite (sqlite3) is currently not supported.'); - return 1; + throw new \InvalidArgumentException( + 'Converting to SQLite (sqlite3) is currently not supported.' + ); } if ($type === $this->config->getValue('dbtype', '')) { - $output->writeln(sprintf( - 'Can not convert from %1$s to %1$s.', + throw new \InvalidArgumentException(sprintf( + 'Can not convert from %1$s to %1$s.', $type )); - return 1; } if ($type === 'oci' && $input->getOption('clear-schema')) { // Doctrine unconditionally tries (at least in version 2.3) // to drop sequence triggers when dropping a table, even though // such triggers may not exist. This results in errors like // "ORA-04080: trigger 'OC_STORAGES_AI_PK' does not exist". - $output->writeln('The --clear-schema option is not supported when converting to Oracle (oci).'); - return 1; + throw new \InvalidArgumentException( + 'The --clear-schema option is not supported when converting to Oracle (oci).' + ); } } @@ -148,11 +149,7 @@ class ConvertType extends Command { } protected function execute(InputInterface $input, OutputInterface $output) { - $inputError = $this->validateInput($input, $output); - if ($inputError) { - return $inputError; - } - + $this->validateInput($input, $output); $this->readPassword($input, $output); $fromDB = \OC_DB::getConnection(); From 0a78fb49f560248bac130405dea2b6af9a813d89 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Tue, 15 Apr 2014 18:20:36 +0200 Subject: [PATCH 45/48] Pass Connection instead of AbstractSchemaManager to clearSchema(). --- core/command/db/converttype.php | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 6fcaeacec49..4f3afb949e7 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -10,8 +10,6 @@ namespace OC\Core\Command\Db; -use Doctrine\DBAL\Schema\AbstractSchemaManager; - use OC\Config; use OC\DB\Connection; use OC\DB\ConnectionFactory; @@ -156,7 +154,7 @@ class ConvertType extends Command { $toDB = $this->getToDBConnection($input, $output); if ($input->getOption('clear-schema')) { - $this->clearSchema($toDB->getSchemaManager(), $input, $output); + $this->clearSchema($toDB, $input, $output); } $this->createSchema($toDB, $input, $output); @@ -214,19 +212,18 @@ class ConvertType extends Command { return $this->connectionFactory->getConnection($type, $connectionParams); } - protected function clearSchema(AbstractSchemaManager $schemaManager, InputInterface $input, OutputInterface $output) { - $toTables = $schemaManager->listTableNames(); + protected function clearSchema(Connection $db, InputInterface $input, OutputInterface $output) { + $toTables = $this->getTables($db); if (!empty($toTables)) { $output->writeln('Clearing schema in new database'); } foreach($toTables as $table) { - $schemaManager->dropTable($table); + $db->getSchemaManager()->dropTable($table); } } protected function getTables(Connection $db) { - $schemaManager = $db->getSchemaManager(); - return $schemaManager->listTableNames(); + return $db->getSchemaManager()->listTableNames(); } protected function copyTable(Connection $fromDB, Connection $toDB, $table, InputInterface $input, OutputInterface $output) { From de78be3891c7a44699dab7d276276061dd62586b Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 May 2014 12:31:08 +0200 Subject: [PATCH 46/48] Store normalised DBMS type in $type. --- core/command/db/converttype.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 4f3afb949e7..809374950d3 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -93,8 +93,8 @@ class ConvertType extends Command { } protected function validateInput(InputInterface $input, OutputInterface $output) { - $type = $input->getArgument('type'); - if ($this->connectionFactory->normalizeType($type) === 'sqlite3') { + $type = $this->connectionFactory->normalizeType($input->getArgument('type')); + if ($type === 'sqlite3') { throw new \InvalidArgumentException( 'Converting to SQLite (sqlite3) is currently not supported.' ); From 603ce95211a9c4a4e66f2fe19e0ef66288a5d122 Mon Sep 17 00:00:00 2001 From: Andreas Fischer Date: Fri, 9 May 2014 12:31:29 +0200 Subject: [PATCH 47/48] Add message about unsupported Microsoft SQL Server. --- core/command/db/converttype.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/command/db/converttype.php b/core/command/db/converttype.php index 809374950d3..39e87853d60 100644 --- a/core/command/db/converttype.php +++ b/core/command/db/converttype.php @@ -99,6 +99,11 @@ class ConvertType extends Command { 'Converting to SQLite (sqlite3) is currently not supported.' ); } + if ($type === 'mssql') { + throw new \InvalidArgumentException( + 'Converting to Microsoft SQL Server (mssql) is currently not supported.' + ); + } if ($type === $this->config->getValue('dbtype', '')) { throw new \InvalidArgumentException(sprintf( 'Can not convert from %1$s to %1$s.', From 8314e5f4d11d03a780334944d8578838c50600ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20M=C3=BCller?= Date: Mon, 12 May 2014 21:58:09 +0200 Subject: [PATCH 48/48] fixing typos and adding PHPDoc --- lib/private/db/connectionfactory.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/private/db/connectionfactory.php b/lib/private/db/connectionfactory.php index 14ffe1a4a56..8f852cf7127 100644 --- a/lib/private/db/connectionfactory.php +++ b/lib/private/db/connectionfactory.php @@ -9,7 +9,7 @@ namespace OC\DB; /** -* Takes care of creating and configurating Doctrine connections. +* Takes care of creating and configuring Doctrine connections. */ class ConnectionFactory { /** @@ -91,6 +91,7 @@ class ConnectionFactory { case 'sqlite3': // Sqlite doesn't handle query caching and schema changes // TODO: find a better way to handle this + /** @var $connection \OC\DB\Connection */ $connection->disableQueryStatementCaching(); break; } @@ -107,7 +108,7 @@ class ConnectionFactory { } /** - * @brief Checks whether the specififed DBMS type is valid. + * @brief Checks whether the specified DBMS type is valid. * @return bool */ public function isValidType($type) {