upstream/nextcloud
1
0
Fork 0
mirror of https://github.com/nextcloud/server.git synced 2026-06-11 01:30:50 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 79

optimize background image migration job

Browse source 
- separate in two stages: to prepare, and to actually migrate
- in step one, prepare a list of users to be migrated, and store it
  compressed as app config
  - gzcompress can be used, because we already require zlib
- upon the next calls (step two), slice off the first 5000 users
  and migrate them. Re-add job if necessary to repeat.
- downside is that an app config value will in the beginning use the
  RAM with any request, until it thins out. Examples: 2m UUIDs (75 MiB)
  result in ~40 MiB compressed data, while 0.2Mib for 10 000 UUIDs,
  0.4MiB for 20 000 and 4.1 MiB for 200 000. Acceptable.

Signed-off-by: Arthur Schiwon <blizzz@arthur-schiwon.de>
This commit is contained in:
Arthur Schiwon 2022-10-19 23:22:22 +02:00 committed by Vincent Petry
parent 30b2b86115
commit 20859a3f3f
No known key found for this signature in database
GPG key ID: E055D6A4D513575C
2 changed files with 70 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

94
apps/theming/lib/Jobs/MigrateBackgroundImages.php
View file

@ -36,45 +36,88 @@ use OCP\Files\NotFoundException;
use OCP\Files\NotPermittedException;
use OCP\Files\SimpleFS\ISimpleFolder;
use OCP\IConfig;
use OCP\IDBConnection;
class MigrateBackgroundImages extends QueuedJob {
public const TIME_SENSITIVE = 0;
protected const STAGE_PREPARE = 'prepare';
protected const STAGE_EXECUTE = 'execute';
private IConfig $config;
private IAppManager $appManager;
private IAppDataFactory $appDataFactory;
private IJobList $jobList;
private IDBConnection $dbc;
public function __construct(ITimeFactory $time, IAppDataFactory $appDataFactory, IConfig $config, IAppManager $appManager, IJobList $jobList) {
public function __construct(
ITimeFactory $time,
IAppDataFactory $appDataFactory,
IConfig $config,
IAppManager $appManager,
IJobList $jobList,
IDBConnection $dbc
) {
parent::__construct($time);
$this->config = $config;
$this->appManager = $appManager;
$this->appDataFactory = $appDataFactory;
$this->jobList = $jobList;
$this->dbc = $dbc;
}
protected function run($argument): void {
if (!$this->appManager->isEnabledForUser('dashboard')) {
return;
if (!isset($argument['stage'])) {
// not executed in 25.0.0?!
$argument['stage'] = 'prepare';
}
switch ($argument['stage']) {
case self::STAGE_PREPARE:
$this->runPreparation();
break;
case self::STAGE_EXECUTE:
$this->runMigration();
break;
default:
break;
}
}
protected function runPreparation(): void {
try {
$selector = $this->dbc->getQueryBuilder();
$result = $selector->select('userid')
->from('preferences')
->where($selector->expr()->eq('appid', $selector->createNamedParameter('theming')))
->andWhere($selector->expr()->eq('configkey', $selector->createNamedParameter('background')))
->andWhere($selector->expr()->eq('configvalue', $selector->createNamedParameter('custom')))
->executeQuery();
$userIds = $result->fetchAll(\PDO::FETCH_COLUMN);
$this->storeUserIdsToProcess($userIds);
} catch (\Throwable $t) {
$this->jobList->add(self::class, self::STAGE_PREPARE);
throw $t;
}
$this->jobList->add(self::class, self::STAGE_EXECUTE);
}
protected function runMigration(): void {
$storedUserIds = $this->config->getAppValue('theming', '25_background_image_migration');
if ($storedUserIds === '') {
return;
}
$allUserIds = \json_decode(\gzuncompress($storedUserIds), true);
$notSoFastMode = isset($allUserIds[5000]);
$dashboardData = $this->appDataFactory->get('dashboard');
$userIds = $this->config->getUsersForUserValue('theming', 'background', 'custom');
$notSoFastMode = \count($userIds) > 5000;
$reTrigger = false;
$processed = 0;
$userIds = $notSoFastMode ? array_slice($allUserIds, 0, 5000) : $allUserIds;
foreach ($userIds as $userId) {
try {
// precondition
if ($notSoFastMode) {
if ($this->config->getUserValue($userId, 'theming', 'background-migrated', '0') === '1') {
// already migrated
continue;
}
$reTrigger = true;
if (!$this->appManager->isEnabledForUser('dashboard', $userId)) {
continue;
}
// migration
@ -87,21 +130,22 @@ class MigrateBackgroundImages extends QueuedJob {
$file->delete();
} catch (NotFoundException|NotPermittedException $e) {
}
// capture state
if ($notSoFastMode) {
$this->config->setUserValue($userId, 'theming', 'background-migrated', '1');
$processed++;
}
if ($processed > 4999) {
break;
}
}
if ($reTrigger) {
$this->jobList->add(self::class);
if ($notSoFastMode) {
$remainingUserIds = array_slice($allUserIds, 5000);
$this->storeUserIdsToProcess($remainingUserIds);
$this->jobList->add(self::class, ['stage' => self::STAGE_EXECUTE]);
} else {
$this->config->deleteAppValue('theming', '25_background_image_migration');
}
}
protected function storeUserIdsToProcess(array $userIds): void {
$storableUserIds = \gzcompress(\json_encode($userIds), 9);
$this->config->setAppValue('theming', '25_background_image_migration', $storableUserIds);
}
/**
* Get the root location for users theming data
*/

2
apps/theming/lib/Migration/InitBackgroundImagesMigration.php
View file

@ -43,6 +43,6 @@ class InitBackgroundImagesMigration implements \OCP\Migration\IRepairStep {
}
public function run(IOutput $output) {
$this->jobList->add(MigrateBackgroundImages::class);
$this->jobList->add(MigrateBackgroundImages::class, ['stage' => 'prepare']);
}
}