mirror of
https://github.com/nextcloud/server.git
synced 2026-05-28 04:32:30 -04:00
Add metadata generation job
Signed-off-by: Louis Chemineau <louis@chmn.me>
This commit is contained in:
Louis Chemineau
parent
9c3350b313
commit
a0ff998c00
6 changed files with 123 additions and 119 deletions
111
core/BackgroundJobs/GenerateMetadataJob.php
Normal file
111
core/BackgroundJobs/GenerateMetadataJob.php
Normal file
|
|
@ -0,0 +1,111 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* @copyright Copyright (c) 2023 Louis Chemineau <louis@chmn.me>
|
||||
*
|
||||
* @author Louis Chemineau <louis@chmn.me>
|
||||
*
|
||||
* @license AGPL-3.0-or-later
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
namespace OC\Core\BackgroundJobs;
|
||||
|
||||
use OCP\AppFramework\Utility\ITimeFactory;
|
||||
use OCP\BackgroundJob\IJobList;
|
||||
use OCP\BackgroundJob\TimedJob;
|
||||
use OCP\Files\Folder;
|
||||
use OCP\Files\IRootFolder;
|
||||
use OCP\FilesMetadata\IFilesMetadataManager;
|
||||
use OCP\IConfig;
|
||||
use OCP\IUserManager;
|
||||
|
||||
class GenerateMetadataJob extends TimedJob {
|
||||
public function __construct(
|
||||
ITimeFactory $time,
|
||||
private IConfig $config,
|
||||
private IRootFolder $rootFolder,
|
||||
private IUserManager $userManager,
|
||||
private IFilesMetadataManager $filesMetadataManager,
|
||||
private IJobList $jobList,
|
||||
) {
|
||||
parent::__construct($time);
|
||||
|
||||
$this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE);
|
||||
$this->setInterval(24 * 3600);
|
||||
}
|
||||
|
||||
protected function run(mixed $argument): void {
|
||||
$users = $this->userManager->search('');
|
||||
$lastMappedUser = $this->config->getAppValue('core', 'metadataGenerationLastHandledUser', '');
|
||||
|
||||
if ($lastMappedUser === '') {
|
||||
$user = array_key_first($users);
|
||||
if ($user === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
$lastMappedUser = $users[$user]->getUID();
|
||||
}
|
||||
|
||||
$startTime = null;
|
||||
foreach ($users as $user) {
|
||||
if ($startTime === null) {
|
||||
// Skip all user before lastMappedUser.
|
||||
if ($lastMappedUser !== $user->getUID()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
$startTime = time();
|
||||
}
|
||||
|
||||
// Stop if execution time is more than one hour.
|
||||
if (time() - $startTime > 60 * 60) {
|
||||
return;
|
||||
}
|
||||
|
||||
$this->scanFilesForUser($user->getUID());
|
||||
$this->config->setAppValue('core', 'metadataGenerationLastHandledUser', $user->getUID());
|
||||
}
|
||||
|
||||
$this->jobList->remove(GenerateMetadataJob::class);
|
||||
}
|
||||
|
||||
private function scanFilesForUser(string $userId): void {
|
||||
$userFolder = $this->rootFolder->getUserFolder($userId);
|
||||
$this->scanFolder($userFolder);
|
||||
}
|
||||
|
||||
private function scanFolder(Folder $folder): void {
|
||||
// Do not scan share and other moveable mounts.
|
||||
if ($folder->getMountPoint() instanceof \OC\Files\Mount\MoveableMount) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($folder->getDirectoryListing() as $node) {
|
||||
if ($node instanceof Folder) {
|
||||
$this->scanFolder($node);
|
||||
continue;
|
||||
}
|
||||
|
||||
$this->filesMetadataManager->refreshMetadata(
|
||||
$node,
|
||||
IFilesMetadataManager::PROCESS_LIVE | IFilesMetadataManager::PROCESS_BACKGROUND
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,107 +0,0 @@
|
|||
<?php
|
||||
|
||||
declare(strict_types=1);
|
||||
/**
|
||||
* @copyright Copyright (c) 2023 Louis Chemineau <louis@chmn.me>
|
||||
*
|
||||
* @author Louis Chemineau <louis@chmn.me>
|
||||
*
|
||||
* @license AGPL-3.0-or-later
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of the
|
||||
* License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
*/
|
||||
|
||||
namespace OC\Core\BackgroundJobs;
|
||||
|
||||
use OCP\AppFramework\Utility\ITimeFactory;
|
||||
use OCP\BackgroundJob\IJobList;
|
||||
use OCP\BackgroundJob\TimedJob;
|
||||
use OCP\FilesMetadata\IFilesMetadataManager;
|
||||
use OCP\IConfig;
|
||||
use OCP\IDBConnection;
|
||||
|
||||
class MigrateMetadataJob extends TimedJob {
|
||||
public function __construct(
|
||||
ITimeFactory $time,
|
||||
private IConfig $config,
|
||||
private IFilesMetadataManager $filesMetadataManager,
|
||||
private IDBConnection $connection,
|
||||
private IJobList $jobList,
|
||||
) {
|
||||
parent::__construct($time);
|
||||
|
||||
$this->setTimeSensitivity(\OCP\BackgroundJob\IJob::TIME_INSENSITIVE);
|
||||
$this->setInterval(24 * 3600);
|
||||
}
|
||||
|
||||
protected function run(mixed $argument): void {
|
||||
if (!$this->connection->tableExists('file_metadata')) {
|
||||
return;
|
||||
}
|
||||
|
||||
$startTime = time();
|
||||
|
||||
$selectQuery = $this->connection->getQueryBuilder()
|
||||
->select('*')
|
||||
->from('file_metadata')
|
||||
->setMaxResults(200);
|
||||
|
||||
$deleteQuery = $this->connection->getQueryBuilder();
|
||||
$deleteQuery->delete('file_metadata')
|
||||
->where($deleteQuery->expr()->eq('id', $deleteQuery->createParameter('id')))
|
||||
->where($deleteQuery->expr()->eq('group_name', $deleteQuery->createParameter('group_name')))
|
||||
->where($deleteQuery->expr()->eq('value', $deleteQuery->createParameter('value')));
|
||||
|
||||
do {
|
||||
$this->connection->beginTransaction();
|
||||
|
||||
$results = $selectQuery->executeQuery();
|
||||
|
||||
while ($row = $results->fetch()) {
|
||||
$metadata = $this->filesMetadataManager->getMetadata($row['id'], true);
|
||||
|
||||
switch ($row['group_name']) {
|
||||
case 'size':
|
||||
$metadata->setArray('photos-size', json_decode($row['value'], true));
|
||||
break;
|
||||
case 'gps':
|
||||
$metadata->setArray('photos-gps', json_decode($row['value'], true));
|
||||
break;
|
||||
case 'photos_place':
|
||||
$metadata->setString('photos-place', $row['value'], true);
|
||||
break;
|
||||
}
|
||||
|
||||
$this->filesMetadataManager->saveMetadata($metadata);
|
||||
$deleteQuery->setParameter('id', $row['id']);
|
||||
$deleteQuery->setParameter('group_name', $row['group_name']);
|
||||
$deleteQuery->setParameter('value', $row['value']);
|
||||
$deleteQuery->executeStatement();
|
||||
}
|
||||
|
||||
$results->closeCursor();
|
||||
|
||||
$this->connection->commit();
|
||||
|
||||
// Stop if execution time is more than one hour.
|
||||
if (time() - $startTime > 60 * 60) {
|
||||
return;
|
||||
}
|
||||
} while ($results->rowCount() !== 0);
|
||||
|
||||
$this->connection->dropTable('file_metadata');
|
||||
$this->jobList->remove(MigrateMetadataJob::class);
|
||||
}
|
||||
}
|
||||
|
|
@ -1002,8 +1002,8 @@ return array(
|
|||
'OC\\Core\\BackgroundJobs\\BackgroundCleanupUpdaterBackupsJob' => $baseDir . '/core/BackgroundJobs/BackgroundCleanupUpdaterBackupsJob.php',
|
||||
'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => $baseDir . '/core/BackgroundJobs/CheckForUserCertificates.php',
|
||||
'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => $baseDir . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
|
||||
'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => $baseDir . '/core/BackgroundJobs/GenerateMetadataJob.php',
|
||||
'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => $baseDir . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php',
|
||||
'OC\\Core\\BackgroundJobs\\MigrateMetadataJob' => $baseDir . '/core/BackgroundJobs/MigrateMetadataJob.php',
|
||||
'OC\\Core\\Command\\App\\Disable' => $baseDir . '/core/Command/App/Disable.php',
|
||||
'OC\\Core\\Command\\App\\Enable' => $baseDir . '/core/Command/App/Enable.php',
|
||||
'OC\\Core\\Command\\App\\GetPath' => $baseDir . '/core/Command/App/GetPath.php',
|
||||
|
|
@ -1603,7 +1603,7 @@ return array(
|
|||
'OC\\RepairException' => $baseDir . '/lib/private/RepairException.php',
|
||||
'OC\\Repair\\AddBruteForceCleanupJob' => $baseDir . '/lib/private/Repair/AddBruteForceCleanupJob.php',
|
||||
'OC\\Repair\\AddCleanupUpdaterBackupsJob' => $baseDir . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
|
||||
'OC\\Repair\\AddMetadataMigrationJob' => $baseDir . '/lib/private/Repair/AddMetadataMigrationJob.php',
|
||||
'OC\\Repair\\AddMetadataGenerationJob' => $baseDir . '/lib/private/Repair/AddMetadataGenerationJob.php',
|
||||
'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => $baseDir . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php',
|
||||
'OC\\Repair\\CleanTags' => $baseDir . '/lib/private/Repair/CleanTags.php',
|
||||
'OC\\Repair\\CleanUpAbandonedApps' => $baseDir . '/lib/private/Repair/CleanUpAbandonedApps.php',
|
||||
|
|
|
|||
|
|
@ -1035,8 +1035,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
|
|||
'OC\\Core\\BackgroundJobs\\BackgroundCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/BackgroundCleanupUpdaterBackupsJob.php',
|
||||
'OC\\Core\\BackgroundJobs\\CheckForUserCertificates' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CheckForUserCertificates.php',
|
||||
'OC\\Core\\BackgroundJobs\\CleanupLoginFlowV2' => __DIR__ . '/../../..' . '/core/BackgroundJobs/CleanupLoginFlowV2.php',
|
||||
'OC\\Core\\BackgroundJobs\\GenerateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/GenerateMetadataJob.php',
|
||||
'OC\\Core\\BackgroundJobs\\LookupServerSendCheckBackgroundJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/LookupServerSendCheckBackgroundJob.php',
|
||||
'OC\\Core\\BackgroundJobs\\MigrateMetadataJob' => __DIR__ . '/../../..' . '/core/BackgroundJobs/MigrateMetadataJob.php',
|
||||
'OC\\Core\\Command\\App\\Disable' => __DIR__ . '/../../..' . '/core/Command/App/Disable.php',
|
||||
'OC\\Core\\Command\\App\\Enable' => __DIR__ . '/../../..' . '/core/Command/App/Enable.php',
|
||||
'OC\\Core\\Command\\App\\GetPath' => __DIR__ . '/../../..' . '/core/Command/App/GetPath.php',
|
||||
|
|
@ -1636,7 +1636,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
|
|||
'OC\\RepairException' => __DIR__ . '/../../..' . '/lib/private/RepairException.php',
|
||||
'OC\\Repair\\AddBruteForceCleanupJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddBruteForceCleanupJob.php',
|
||||
'OC\\Repair\\AddCleanupUpdaterBackupsJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddCleanupUpdaterBackupsJob.php',
|
||||
'OC\\Repair\\AddMetadataMigrationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataMigrationJob.php',
|
||||
'OC\\Repair\\AddMetadataGenerationJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddMetadataGenerationJob.php',
|
||||
'OC\\Repair\\AddRemoveOldTasksBackgroundJob' => __DIR__ . '/../../..' . '/lib/private/Repair/AddRemoveOldTasksBackgroundJob.php',
|
||||
'OC\\Repair\\CleanTags' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanTags.php',
|
||||
'OC\\Repair\\CleanUpAbandonedApps' => __DIR__ . '/../../..' . '/lib/private/Repair/CleanUpAbandonedApps.php',
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ use OC\DB\Connection;
|
|||
use OC\DB\ConnectionAdapter;
|
||||
use OC\Repair\AddBruteForceCleanupJob;
|
||||
use OC\Repair\AddCleanupUpdaterBackupsJob;
|
||||
use OC\Repair\AddMetadataMigrationJob;
|
||||
use OC\Repair\AddMetadataGenerationJob;
|
||||
use OC\Repair\CleanTags;
|
||||
use OC\Repair\ClearFrontendCaches;
|
||||
use OC\Repair\ClearGeneratedAvatarCache;
|
||||
|
|
@ -212,7 +212,7 @@ class Repair implements IOutput {
|
|||
\OCP\Server::get(CleanUpAbandonedApps::class),
|
||||
\OCP\Server::get(AddMissingSecretJob::class),
|
||||
\OCP\Server::get(AddRemoveOldTasksBackgroundJob::class),
|
||||
\OCP\Server::get(AddMetadataMigrationJob::class),
|
||||
\OCP\Server::get(AddMetadataGenerationJob::class),
|
||||
];
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
<?php
|
||||
/**
|
||||
* @copyright Copyright (c) 2018 Morris Jobke <hey@morrisjobke.de>
|
||||
* @copyright Copyright (c) 2023 Louis Chmn <louis@chmn.me>
|
||||
*
|
||||
* @author Morris Jobke <hey@morrisjobke.de>
|
||||
* @author Louis Chmn <louis@chmn.me>
|
||||
*
|
||||
* @license GNU AGPL version 3 or any later version
|
||||
*
|
||||
|
|
@ -22,22 +22,22 @@
|
|||
*/
|
||||
namespace OC\Repair;
|
||||
|
||||
use OC\Core\BackgroundJobs\MigrateMetadataJob;
|
||||
use OC\Core\BackgroundJobs\GenerateMetadataJob;
|
||||
use OCP\BackgroundJob\IJobList;
|
||||
use OCP\Migration\IOutput;
|
||||
use OCP\Migration\IRepairStep;
|
||||
|
||||
class AddMetadataMigrationJob implements IRepairStep {
|
||||
class AddMetadataGenerationJob implements IRepairStep {
|
||||
public function __construct(
|
||||
private IJobList $jobList,
|
||||
) {
|
||||
}
|
||||
|
||||
public function getName() {
|
||||
return 'Queue a job to migrate legacy metadata tables';
|
||||
return 'Queue a job to generate metadata';
|
||||
}
|
||||
|
||||
public function run(IOutput $output) {
|
||||
$this->jobList->add(MigrateMetadataJob::class);
|
||||
$this->jobList->add(GenerateMetadataJob::class);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue