From cb742e7045d1f5af7994ad1e918b0ddc9ba01914 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Fri, 2 Nov 2018 21:19:27 +0100 Subject: [PATCH 1/2] Background job to cleanup leftover chunked uploads Signed-off-by: Roeland Jago Douma --- .../composer/composer/autoload_classmap.php | 2 + .../dav/composer/composer/autoload_static.php | 2 + apps/dav/lib/BackgroundJob/UploadCleanup.php | 87 +++++++++++++++++++ apps/dav/lib/RootCollection.php | 6 +- apps/dav/lib/Upload/CleanupService.php | 49 +++++++++++ apps/dav/lib/Upload/RootCollection.php | 24 +++-- apps/dav/lib/Upload/UploadFolder.php | 11 ++- apps/dav/lib/Upload/UploadHome.php | 39 +++++---- 8 files changed, 195 insertions(+), 25 deletions(-) create mode 100644 apps/dav/lib/BackgroundJob/UploadCleanup.php create mode 100644 apps/dav/lib/Upload/CleanupService.php diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php index 537bd72d4ca..89c6b991566 100644 --- a/apps/dav/composer/composer/autoload_classmap.php +++ b/apps/dav/composer/composer/autoload_classmap.php @@ -16,6 +16,7 @@ return array( 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => $baseDir . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => $baseDir . '/../lib/BackgroundJob/RefreshWebcalJob.php', 'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => $baseDir . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php', + 'OCA\\DAV\\BackgroundJob\\UploadCleanup' => $baseDir . '/../lib/BackgroundJob/UploadCleanup.php', 'OCA\\DAV\\CalDAV\\Activity\\Backend' => $baseDir . '/../lib/CalDAV/Activity/Backend.php', 'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => $baseDir . '/../lib/CalDAV/Activity/Filter/Calendar.php', 'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => $baseDir . '/../lib/CalDAV/Activity/Filter/Todo.php', @@ -181,6 +182,7 @@ return array( 'OCA\\DAV\\SystemTag\\SystemTagsRelationsCollection' => $baseDir . '/../lib/SystemTag/SystemTagsRelationsCollection.php', 'OCA\\DAV\\Upload\\AssemblyStream' => $baseDir . '/../lib/Upload/AssemblyStream.php', 'OCA\\DAV\\Upload\\ChunkingPlugin' => $baseDir . '/../lib/Upload/ChunkingPlugin.php', + 'OCA\\DAV\\Upload\\CleanupService' => $baseDir . '/../lib/Upload/CleanupService.php', 'OCA\\DAV\\Upload\\FutureFile' => $baseDir . '/../lib/Upload/FutureFile.php', 'OCA\\DAV\\Upload\\RootCollection' => $baseDir . '/../lib/Upload/RootCollection.php', 'OCA\\DAV\\Upload\\UploadFolder' => $baseDir . '/../lib/Upload/UploadFolder.php', diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php index d6c5628b191..f891e8e780a 100644 --- a/apps/dav/composer/composer/autoload_static.php +++ b/apps/dav/composer/composer/autoload_static.php @@ -31,6 +31,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\BackgroundJob\\GenerateBirthdayCalendarBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/GenerateBirthdayCalendarBackgroundJob.php', 'OCA\\DAV\\BackgroundJob\\RefreshWebcalJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/RefreshWebcalJob.php', 'OCA\\DAV\\BackgroundJob\\UpdateCalendarResourcesRoomsBackgroundJob' => __DIR__ . '/..' . '/../lib/BackgroundJob/UpdateCalendarResourcesRoomsBackgroundJob.php', + 'OCA\\DAV\\BackgroundJob\\UploadCleanup' => __DIR__ . '/..' . '/../lib/BackgroundJob/UploadCleanup.php', 'OCA\\DAV\\CalDAV\\Activity\\Backend' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Backend.php', 'OCA\\DAV\\CalDAV\\Activity\\Filter\\Calendar' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Calendar.php', 'OCA\\DAV\\CalDAV\\Activity\\Filter\\Todo' => __DIR__ . '/..' . '/../lib/CalDAV/Activity/Filter/Todo.php', @@ -196,6 +197,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\SystemTag\\SystemTagsRelationsCollection' => __DIR__ . '/..' . '/../lib/SystemTag/SystemTagsRelationsCollection.php', 'OCA\\DAV\\Upload\\AssemblyStream' => __DIR__ . '/..' . '/../lib/Upload/AssemblyStream.php', 'OCA\\DAV\\Upload\\ChunkingPlugin' => __DIR__ . '/..' . '/../lib/Upload/ChunkingPlugin.php', + 'OCA\\DAV\\Upload\\CleanupService' => __DIR__ . '/..' . '/../lib/Upload/CleanupService.php', 'OCA\\DAV\\Upload\\FutureFile' => __DIR__ . '/..' . '/../lib/Upload/FutureFile.php', 'OCA\\DAV\\Upload\\RootCollection' => __DIR__ . '/..' . '/../lib/Upload/RootCollection.php', 'OCA\\DAV\\Upload\\UploadFolder' => __DIR__ . '/..' . '/../lib/Upload/UploadFolder.php', diff --git a/apps/dav/lib/BackgroundJob/UploadCleanup.php b/apps/dav/lib/BackgroundJob/UploadCleanup.php new file mode 100644 index 00000000000..42262b9dc2c --- /dev/null +++ b/apps/dav/lib/BackgroundJob/UploadCleanup.php @@ -0,0 +1,87 @@ + + * + * @author Roeland Jago Douma + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\DAV\BackgroundJob; + +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\IJobList; +use OCP\BackgroundJob\TimedJob; +use OCP\Files\File; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; + +class UploadCleanup extends TimedJob { + + /** @var IRootFolder */ + private $rootFolder; + + /** @var IJobList */ + private $jobList; + + public function __construct(ITimeFactory $time, IRootFolder $rootFolder, IJobList $jobList) { + parent::__construct($time); + $this->rootFolder = $rootFolder; + $this->jobList = $jobList; + + // Run once a day + $this->setInterval(60*60*24); + } + + protected function run($argument) { + $uid = $argument['uid']; + $folder = $argument['folder']; + + $userFolder = $this->rootFolder->getUserFolder($uid); + $userRoot = $userFolder->getParent(); + + try { + /** @var Folder $uploads */ + $uploads = $userRoot->get('uploads'); + /** @var Folder $uploadFolder */ + $uploadFolder = $uploads->get($folder); + } catch (NotFoundException $e) { + $this->jobList->remove(self::class, $argument); + return; + } + + $files = $uploadFolder->getDirectoryListing(); + + // Remove if all files have an mtime of more than a day + $time = $this->time->getTime() - 60 * 60 * 24; + + // The folder has to be more than a day old + $initial = $uploadFolder->getMTime() < $time; + + $expire = array_reduce($files, function(bool $carry, File $file) use ($time) { + return $carry && $file->getMTime() < $time; + }, $initial); + + if ($expire) { + $uploadFolder->delete(); + $this->jobList->remove(self::class, $argument); + } + } + +} diff --git a/apps/dav/lib/RootCollection.php b/apps/dav/lib/RootCollection.php index a05781a7621..adf9d7b99c7 100644 --- a/apps/dav/lib/RootCollection.php +++ b/apps/dav/lib/RootCollection.php @@ -35,6 +35,7 @@ use OCA\DAV\Connector\Sabre\Principal; use OCA\DAV\DAV\GroupPrincipalBackend; use OCA\DAV\DAV\SystemPrincipalBackend; use OCA\DAV\CalDAV\Principal\Collection; +use OCA\DAV\Upload\CleanupService; use Sabre\DAV\SimpleCollection; class RootCollection extends SimpleCollection { @@ -120,7 +121,10 @@ class RootCollection extends SimpleCollection { $systemAddressBookRoot = new AddressBookRoot(new SystemPrincipalBackend(), $systemCardDavBackend, 'principals/system'); $systemAddressBookRoot->disableListing = $disableListing; - $uploadCollection = new Upload\RootCollection($userPrincipalBackend, 'principals/users'); + $uploadCollection = new Upload\RootCollection( + $userPrincipalBackend, + 'principals/users', + \OC::$server->query(CleanupService::class)); $uploadCollection->disableListing = $disableListing; $avatarCollection = new Avatars\RootCollection($userPrincipalBackend, 'principals/users'); diff --git a/apps/dav/lib/Upload/CleanupService.php b/apps/dav/lib/Upload/CleanupService.php new file mode 100644 index 00000000000..9e70a1cff90 --- /dev/null +++ b/apps/dav/lib/Upload/CleanupService.php @@ -0,0 +1,49 @@ + + * + * @author Roeland Jago Douma + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\DAV\Upload; + +use OCA\DAV\BackgroundJob\UploadCleanup; +use OCP\BackgroundJob\IJobList; +use OCP\IUserSession; + +class CleanupService { + /** @var IUserSession */ + private $userSession; + /** @var IJobList */ + private $jobList; + + public function __construct(IUserSession $userSession, IJobList $jobList) { + $this->userSession = $userSession; + $this->jobList = $jobList; + } + + public function addJob(string $folder) { + $this->jobList->add(UploadCleanup::class, ['uid' => $this->userSession->getUser()->getUID(), 'folder' => $folder]); + } + + public function removeJob(string $folder) { + $this->jobList->remove(UploadCleanup::class, ['uid' => $this->userSession->getUser()->getUID(), 'folder' => $folder]); + } +} diff --git a/apps/dav/lib/Upload/RootCollection.php b/apps/dav/lib/Upload/RootCollection.php index 696de064dd8..436792e25ec 100644 --- a/apps/dav/lib/Upload/RootCollection.php +++ b/apps/dav/lib/Upload/RootCollection.php @@ -1,4 +1,5 @@ cleanupService = $cleanupService; } /** * @inheritdoc */ - function getName() { + public function getChildForPrincipal(array $principalInfo): UploadHome { + return new UploadHome($principalInfo, $this->cleanupService); + } + + /** + * @inheritdoc + */ + public function getName(): string { return 'uploads'; } diff --git a/apps/dav/lib/Upload/UploadFolder.php b/apps/dav/lib/Upload/UploadFolder.php index b3d43fef95d..233c3ac3e52 100644 --- a/apps/dav/lib/Upload/UploadFolder.php +++ b/apps/dav/lib/Upload/UploadFolder.php @@ -22,16 +22,22 @@ */ namespace OCA\DAV\Upload; +use OCA\DAV\BackgroundJob\UploadCleanup; use OCA\DAV\Connector\Sabre\Directory; +use OCP\BackgroundJob\IJobList; use Sabre\DAV\Exception\Forbidden; use Sabre\DAV\ICollection; class UploadFolder implements ICollection { + /** @var Directory */ private $node; + /** @var CleanupService */ + private $cleanupService; - function __construct(Directory $node) { + function __construct(Directory $node, CleanupService $cleanupService) { $this->node = $node; + $this->cleanupService = $cleanupService; } function createFile($name, $data = null) { @@ -65,6 +71,9 @@ class UploadFolder implements ICollection { function delete() { $this->node->delete(); + + // Background cleanup job is not needed anymore + $this->cleanupService->removeJob($this->getName()); } function getName() { diff --git a/apps/dav/lib/Upload/UploadHome.php b/apps/dav/lib/Upload/UploadHome.php index 6976ece9943..fcfd978b33d 100644 --- a/apps/dav/lib/Upload/UploadHome.php +++ b/apps/dav/lib/Upload/UploadHome.php @@ -30,51 +30,56 @@ use Sabre\DAV\Exception\Forbidden; use Sabre\DAV\ICollection; class UploadHome implements ICollection { - /** - * UploadHome constructor. - * - * @param array $principalInfo - */ - public function __construct($principalInfo) { + + /** @var array */ + private $principalInfo; + /** @var CleanupService */ + private $cleanupService; + + public function __construct(array $principalInfo, CleanupService $cleanupService) { $this->principalInfo = $principalInfo; + $this->cleanupService = $cleanupService; } - function createFile($name, $data = null) { + public function createFile($name, $data = null) { throw new Forbidden('Permission denied to create file (filename ' . $name . ')'); } - function createDirectory($name) { + public function createDirectory($name) { $this->impl()->createDirectory($name); + + // Add a cleanup job + $this->cleanupService->addJob($name); } - function getChild($name) { - return new UploadFolder($this->impl()->getChild($name)); + public function getChild($name): UploadFolder { + return new UploadFolder($this->impl()->getChild($name), $this->cleanupService); } - function getChildren() { + public function getChildren(): array { return array_map(function($node) { - return new UploadFolder($node); + return new UploadFolder($node, $this->cleanupService); }, $this->impl()->getChildren()); } - function childExists($name) { + public function childExists($name): bool { return !is_null($this->getChild($name)); } - function delete() { + public function delete() { $this->impl()->delete(); } - function getName() { + public function getName() { list(,$name) = \Sabre\Uri\split($this->principalInfo['uri']); return $name; } - function setName($name) { + public function setName($name) { throw new Forbidden('Permission denied to rename this folder'); } - function getLastModified() { + public function getLastModified() { return $this->impl()->getLastModified(); } From adb2b9a2f82d80cfe8c89480f0f36642f6ebdc16 Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Fri, 4 Jan 2019 09:16:10 +0100 Subject: [PATCH 2/2] Add live repair step to cleanup old chunks Signed-off-by: Roeland Jago Douma --- apps/dav/appinfo/info.xml | 5 +- .../composer/composer/autoload_classmap.php | 1 + .../dav/composer/composer/autoload_static.php | 1 + apps/dav/lib/Migration/ChunkCleanup.php | 95 +++++++++++++++++++ 4 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 apps/dav/lib/Migration/ChunkCleanup.php diff --git a/apps/dav/appinfo/info.xml b/apps/dav/appinfo/info.xml index 633658674c5..9d6522e50b3 100644 --- a/apps/dav/appinfo/info.xml +++ b/apps/dav/appinfo/info.xml @@ -5,7 +5,7 @@ WebDAV WebDAV endpoint WebDAV endpoint - 1.9.0 + 1.9.1 agpl owncloud.org DAV @@ -32,6 +32,9 @@ OCA\DAV\Migration\BuildCalendarSearchIndex OCA\DAV\Migration\RefreshWebcalJobRegistrar + + OCA\DAV\Migration\ChunkCleanup + diff --git a/apps/dav/composer/composer/autoload_classmap.php b/apps/dav/composer/composer/autoload_classmap.php index 89c6b991566..d5d26ea9503 100644 --- a/apps/dav/composer/composer/autoload_classmap.php +++ b/apps/dav/composer/composer/autoload_classmap.php @@ -154,6 +154,7 @@ return array( 'OCA\\DAV\\Migration\\BuildCalendarSearchIndex' => $baseDir . '/../lib/Migration/BuildCalendarSearchIndex.php', 'OCA\\DAV\\Migration\\BuildCalendarSearchIndexBackgroundJob' => $baseDir . '/../lib/Migration/BuildCalendarSearchIndexBackgroundJob.php', 'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => $baseDir . '/../lib/Migration/CalDAVRemoveEmptyValue.php', + 'OCA\\DAV\\Migration\\ChunkCleanup' => $baseDir . '/../lib/Migration/ChunkCleanup.php', 'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => $baseDir . '/../lib/Migration/FixBirthdayCalendarComponent.php', 'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => $baseDir . '/../lib/Migration/RefreshWebcalJobRegistrar.php', 'OCA\\DAV\\Migration\\Version1004Date20170825134824' => $baseDir . '/../lib/Migration/Version1004Date20170825134824.php', diff --git a/apps/dav/composer/composer/autoload_static.php b/apps/dav/composer/composer/autoload_static.php index f891e8e780a..70dec97a91d 100644 --- a/apps/dav/composer/composer/autoload_static.php +++ b/apps/dav/composer/composer/autoload_static.php @@ -169,6 +169,7 @@ class ComposerStaticInitDAV 'OCA\\DAV\\Migration\\BuildCalendarSearchIndex' => __DIR__ . '/..' . '/../lib/Migration/BuildCalendarSearchIndex.php', 'OCA\\DAV\\Migration\\BuildCalendarSearchIndexBackgroundJob' => __DIR__ . '/..' . '/../lib/Migration/BuildCalendarSearchIndexBackgroundJob.php', 'OCA\\DAV\\Migration\\CalDAVRemoveEmptyValue' => __DIR__ . '/..' . '/../lib/Migration/CalDAVRemoveEmptyValue.php', + 'OCA\\DAV\\Migration\\ChunkCleanup' => __DIR__ . '/..' . '/../lib/Migration/ChunkCleanup.php', 'OCA\\DAV\\Migration\\FixBirthdayCalendarComponent' => __DIR__ . '/..' . '/../lib/Migration/FixBirthdayCalendarComponent.php', 'OCA\\DAV\\Migration\\RefreshWebcalJobRegistrar' => __DIR__ . '/..' . '/../lib/Migration/RefreshWebcalJobRegistrar.php', 'OCA\\DAV\\Migration\\Version1004Date20170825134824' => __DIR__ . '/..' . '/../lib/Migration/Version1004Date20170825134824.php', diff --git a/apps/dav/lib/Migration/ChunkCleanup.php b/apps/dav/lib/Migration/ChunkCleanup.php new file mode 100644 index 00000000000..037174806d6 --- /dev/null +++ b/apps/dav/lib/Migration/ChunkCleanup.php @@ -0,0 +1,95 @@ + + * + * @author Roeland Jago Douma + * + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +namespace OCA\DAV\Migration; + +use OCA\DAV\BackgroundJob\UploadCleanup; +use OCP\BackgroundJob\IJobList; +use OCP\Files\Folder; +use OCP\Files\IRootFolder; +use OCP\Files\NotFoundException; +use OCP\IConfig; +use OCP\IUser; +use OCP\IUserManager; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; + +class ChunkCleanup implements IRepairStep { + + /** @var IConfig */ + private $config; + /** @var IUserManager */ + private $userManager; + /** @var IRootFolder */ + private $rootFolder; + /** @var IJobList */ + private $jobList; + + public function __construct(IConfig $config, + IUserManager $userManager, + IRootFolder $rootFolder, + IJobList $jobList) { + $this->config = $config; + $this->userManager = $userManager; + $this->rootFolder = $rootFolder; + $this->jobList = $jobList; + } + + public function getName(): string { + return 'Chunk cleanup scheduler'; + } + + public function run(IOutput $output) { + // If we already ran this onec there is no need to run it again + if ($this->config->getAppValue('dav', 'chunks_migrated', '0') === '1') { + $output->info('Cleanup not required'); + } + + $output->startProgress(); + // Loop over all seen users + $this->userManager->callForSeenUsers(function (IUser $user) use ($output) { + try { + $userFolder = $this->rootFolder->getUserFolder($user->getUID()); + $userRoot = $userFolder->getParent(); + /** @var Folder $uploadFolder */ + $uploadFolder = $userRoot->get('uploads'); + } catch (NotFoundException $e) { + // No folder so skipping + return; + } + + // Insert a cleanup job for each folder we find + $uploads = $uploadFolder->getDirectoryListing(); + foreach ($uploads as $upload) { + $this->jobList->add(UploadCleanup::class, ['uid' => $user->getUID(), 'folder' => $upload->getName()]); + } + $output->advance(); + }); + $output->finishProgress(); + + + $this->config->setAppValue('dav', 'chunks_migrated', '1'); + } + +}