upstream/nextcloud
1
0
Fork 0
mirror of https://github.com/nextcloud/server.git synced 2026-06-09 08:44:07 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 11

feat(files_trashbin): Refactor expire background job to support parallel run

Browse source 
- Follow-up of #51600

The original PR introduced the possibility to continue an `ExpireTrash` job by saving the offset. This was to prevent having to start over the whole user list when the job crashed or was killed.

But on big instances, one process is not enough to go through all the users in a timely manner. Supporting parallel run allows covering more ground faster.

This PR introduced this possibility. We are now storing the offset right away to allow another parallel job to pick up the task at that point. We are arbitrarily cutting the user list in chunk of 10 to not drastically overflow the 30 minutes time limit.

Signed-off-by: Louis Chemineau <louis@chmn.me>
This commit is contained in:
Louis Chemineau 2025-09-05 12:46:52 +02:00 committed by Louis
parent 7948e23441
commit 60455d724b
2 changed files with 112 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

107
apps/files_trashbin/lib/BackgroundJob/ExpireTrash.php
View file

@ -6,32 +6,42 @@
*/
namespace OCA\Files_Trashbin\BackgroundJob;
use OC\Files\SetupManager;
use OC\Files\View;
use OCA\Files_Trashbin\AppInfo\Application;
use OCA\Files_Trashbin\Expiration;
use OCA\Files_Trashbin\Helper;
use OCA\Files_Trashbin\Trashbin;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\TimedJob;
use OCP\IAppConfig;
use OCP\IUser;
use OCP\IUserManager;
use OCP\Lock\ILockingProvider;
use Psr\Log\LoggerInterface;
class ExpireTrash extends TimedJob {
public const TOGGLE_CONFIG_KEY_NAME = 'background_job_expire_trash';
public const OFFSET_CONFIG_KEY_NAME = 'background_job_expire_trash_offset';
private const THIRTY_MINUTES = 30 * 60;
private const USER_BATCH_SIZE = 10;
public function __construct(
private IAppConfig $appConfig,
private IUserManager $userManager,
private Expiration $expiration,
private LoggerInterface $logger,
private SetupManager $setupManager,
private ILockingProvider $lockingProvider,
ITimeFactory $time,
) {
parent::__construct($time);
// Run once per 30 minutes
$this->setInterval(60 * 30);
$this->setInterval(self::THIRTY_MINUTES);
}
protected function run($argument) {
$backgroundJob = $this->appConfig->getValueString('files_trashbin', 'background_job_expire_trash', 'yes');
if ($backgroundJob === 'no') {
$backgroundJob = $this->appConfig->getValueBool(Application::APP_ID, self::TOGGLE_CONFIG_KEY_NAME, true);
if (!$backgroundJob) {
return;
}
@ -40,48 +50,89 @@ class ExpireTrash extends TimedJob {
return;
}
$stopTime = time() + 60 * 30; // Stops after 30 minutes.
$offset = $this->appConfig->getValueInt('files_trashbin', 'background_job_expire_trash_offset', 0);
$users = $this->userManager->getSeenUsers($offset);
$startTime = time();
foreach ($users as $user) {
try {
// Process users in batches of 10, but don't run for more than 30 minutes
while (time() < $startTime + self::THIRTY_MINUTES) {
$offset = $this->getNextOffset();
$users = $this->userManager->getSeenUsers($offset, self::USER_BATCH_SIZE);
$count = 0;
foreach ($users as $user) {
$uid = $user->getUID();
if (!$this->setupFS($uid)) {
continue;
$count++;
try {
if ($this->setupFS($user)) {
$dirContent = Helper::getTrashFiles('/', $uid, 'mtime');
Trashbin::deleteExpiredFiles($dirContent, $uid);
}
} catch (\Throwable $e) {
$this->logger->error('Error while expiring trashbin for user ' . $uid, ['exception' => $e]);
} finally {
$this->setupManager->tearDown();
}
$dirContent = Helper::getTrashFiles('/', $uid, 'mtime');
Trashbin::deleteExpiredFiles($dirContent, $uid);
} catch (\Throwable $e) {
$this->logger->error('Error while expiring trashbin for user ' . $user->getUID(), ['exception' => $e]);
}
$offset++;
if ($stopTime < time()) {
$this->appConfig->setValueInt('files_trashbin', 'background_job_expire_trash_offset', $offset);
\OC_Util::tearDownFS();
return;
// If the last batch was not full it means that we reached the end of the user list.
if ($count < self::USER_BATCH_SIZE) {
$this->resetOffset();
}
}
$this->appConfig->setValueInt('files_trashbin', 'background_job_expire_trash_offset', 0);
\OC_Util::tearDownFS();
}
/**
* Act on behalf on trash item owner
*/
protected function setupFS(string $user): bool {
\OC_Util::tearDownFS();
\OC_Util::setupFS($user);
protected function setupFS(IUser $user): bool {
$this->setupManager->setupForUser($user);
// Check if this user has a trashbin directory
$view = new View('/' . $user);
$view = new View('/' . $user->getUID());
if (!$view->is_dir('/files_trashbin/files')) {
return false;
}
return true;
}
private function getNextOffset(): int {
return $this->runMutexOperation(function () {
$this->appConfig->clearCache();
$offset = $this->appConfig->getValueInt(Application::APP_ID, self::OFFSET_CONFIG_KEY_NAME, 0);
$this->appConfig->setValueInt(Application::APP_ID, self::OFFSET_CONFIG_KEY_NAME, $offset + self::USER_BATCH_SIZE);
return $offset;
});
}
private function resetOffset() {
$this->runMutexOperation(function () {
$this->appConfig->setValueInt(Application::APP_ID, self::OFFSET_CONFIG_KEY_NAME, 0);
});
}
private function runMutexOperation($operation): mixed {
$acquired = false;
while ($acquired === false) {
try {
$this->lockingProvider->acquireLock(self::OFFSET_CONFIG_KEY_NAME, ILockingProvider::LOCK_EXCLUSIVE, 'Expire trashbin background job offset');
$acquired = true;
} catch (\OCP\Lock\LockedException $e) {
// wait a bit and try again
usleep(100000);
}
}
try {
$result = $operation();
} finally {
$this->lockingProvider->releaseLock(self::OFFSET_CONFIG_KEY_NAME, ILockingProvider::LOCK_EXCLUSIVE);
}
return $result;
}
}

42
apps/files_trashbin/tests/BackgroundJob/ExpireTrashTest.php
View file

@ -7,12 +7,15 @@
namespace OCA\Files_Trashbin\Tests\BackgroundJob;
use OC\Files\SetupManager;
use OCA\Files_Trashbin\AppInfo\Application;
use OCA\Files_Trashbin\BackgroundJob\ExpireTrash;
use OCA\Files_Trashbin\Expiration;
use OCP\AppFramework\Utility\ITimeFactory;
use OCP\BackgroundJob\IJobList;
use OCP\IAppConfig;
use OCP\IUserManager;
use OCP\Lock\ILockingProvider;
use PHPUnit\Framework\MockObject\MockObject;
use Psr\Log\LoggerInterface;
use Test\TestCase;
@ -36,6 +39,9 @@ class ExpireTrashTest extends TestCase {
/** @var ITimeFactory&MockObject */
private $time;
private SetupManager&MockObject $setupManager;
private ILockingProvider&MockObject $lockingProvider;
protected function setUp(): void {
parent::setUp();
@ -44,6 +50,8 @@ class ExpireTrashTest extends TestCase {
$this->expiration = $this->createMock(Expiration::class);
$this->jobList = $this->createMock(IJobList::class);
$this->logger = $this->createMock(LoggerInterface::class);
$this->setupManager = $this->createMock(SetupManager::class);
$this->lockingProvider = $this->createMock(ILockingProvider::class);
$this->time = $this->createMock(ITimeFactory::class);
$this->time->method('getTime')
@ -56,25 +64,41 @@ class ExpireTrashTest extends TestCase {
}
public function testConstructAndRun(): void {
$this->appConfig->method('getValueString')
->with('files_trashbin', 'background_job_expire_trash', 'yes')
->willReturn('yes');
$this->appConfig->method('getValueBool')
->with(Application::APP_ID, ExpireTrash::TOGGLE_CONFIG_KEY_NAME, true)
->willReturn(true);
$this->appConfig->method('getValueInt')
->with('files_trashbin', 'background_job_expire_trash_offset', 0)
->with(Application::APP_ID, ExpireTrash::OFFSET_CONFIG_KEY_NAME, 0)
->willReturn(0);
$job = new ExpireTrash($this->appConfig, $this->userManager, $this->expiration, $this->logger, $this->time);
$job = new ExpireTrash(
$this->appConfig,
$this->userManager,
$this->expiration,
$this->logger,
$this->setupManager,
$this->lockingProvider,
$this->time,
);
$job->start($this->jobList);
}
public function testBackgroundJobDeactivated(): void {
$this->appConfig->method('getValueString')
->with('files_trashbin', 'background_job_expire_trash', 'yes')
->willReturn('no');
$this->appConfig->method('getValueBool')
->with(Application::APP_ID, ExpireTrash::TOGGLE_CONFIG_KEY_NAME, true)
->willReturn(false);
$this->expiration->expects($this->never())
->method('getMaxAgeAsTimestamp');
$job = new ExpireTrash($this->appConfig, $this->userManager, $this->expiration, $this->logger, $this->time);
$job = new ExpireTrash(
$this->appConfig,
$this->userManager,
$this->expiration,
$this->logger,
$this->setupManager,
$this->lockingProvider,
$this->time,
);
$job->start($this->jobList);
}
}