Merge pull request #30359 from nextcloud/enh/background-job-class

Allow calling cron jobs background job class with occ

core/Command/Background/JobBase.php

@@ -0,0 +1,95 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @copyright Copyright (c) 2022 Julius Härtl <jus@bitgrid.net>
+ *
+ * @author Julius Härtl <jus@bitgrid.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+
+namespace OC\Core\Command\Background;
+
+use OCP\BackgroundJob\IJob;
+use OCP\BackgroundJob\IJobList;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+abstract class JobBase extends \OC\Core\Command\Base {
+
+	public function __construct(
+		protected IJobList $jobList,
+		protected LoggerInterface $logger
+	) {
+		parent::__construct();
+	}
+
+	protected function printJobInfo(int $jobId, IJob $job, OutputInterface $output): void {
+		$row = $this->jobList->getDetailsById($jobId);
+
+		if ($row === null) {
+			return;
+		}
+
+		$lastRun = new \DateTime();
+		$lastRun->setTimestamp((int) $row['last_run']);
+		$lastChecked = new \DateTime();
+		$lastChecked->setTimestamp((int) $row['last_checked']);
+		$reservedAt = new \DateTime();
+		$reservedAt->setTimestamp((int) $row['reserved_at']);
+
+		$output->writeln('Job class:            ' . get_class($job));
+		$output->writeln('Arguments:            ' . json_encode($job->getArgument()));
+
+		$isTimedJob = $job instanceof \OCP\BackgroundJob\TimedJob;
+		if ($isTimedJob) {
+			$output->writeln('Type:                 timed');
+		} elseif ($job instanceof \OCP\BackgroundJob\QueuedJob) {
+			$output->writeln('Type:                 queued');
+		} else {
+			$output->writeln('Type:                 job');
+		}
+
+		$output->writeln('');
+		$output->writeln('Last checked:         ' . $lastChecked->format(\DateTimeInterface::ATOM));
+		if ((int) $row['reserved_at'] === 0) {
+			$output->writeln('Reserved at:          -');
+		} else {
+			$output->writeln('Reserved at:          <comment>' . $reservedAt->format(\DateTimeInterface::ATOM) . '</comment>');
+		}
+		$output->writeln('Last executed:        ' . $lastRun->format(\DateTimeInterface::ATOM));
+		$output->writeln('Last duration:        ' . $row['execution_duration']);
+
+		if ($isTimedJob) {
+			$reflection = new \ReflectionClass($job);
+			$intervalProperty = $reflection->getProperty('interval');
+			$intervalProperty->setAccessible(true);
+			$interval = $intervalProperty->getValue($job);
+
+			$nextRun = new \DateTime();
+			$nextRun->setTimestamp((int)$row['last_run'] + $interval);
+
+			if ($nextRun > new \DateTime()) {
+				$output->writeln('Next execution:       <comment>' . $nextRun->format(\DateTimeInterface::ATOM) . '</comment>');
+			} else {
+				$output->writeln('Next execution:       <info>' . $nextRun->format(\DateTimeInterface::ATOM) . '</info>');
+			}
+		}
+	}
+}

core/Command/Background/JobWorker.php

@@ -0,0 +1,157 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2021, Joas Schilling <coding@schilljs.com>
+ *
+ * @author Joas Schilling <coding@schilljs.com>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+namespace OC\Core\Command\Background;
+
+use OC\Core\Command\InterruptedException;
+use OC\Files\SetupManager;
+use OCP\BackgroundJob\IJobList;
+use OCP\ITempManager;
+use Psr\Log\LoggerInterface;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class JobWorker extends JobBase {
+
+	public function __construct(
+		protected IJobList $jobList,
+		protected LoggerInterface $logger,
+		private ITempManager $tempManager,
+		private SetupManager $setupManager,
+	) {
+		parent::__construct($jobList, $logger);
+	}
+	protected function configure(): void {
+		parent::configure();
+
+		$this
+			->setName('background-job:worker')
+			->setDescription('Run a background job worker')
+			->addArgument(
+				'job-classes',
+				InputArgument::OPTIONAL | InputArgument::IS_ARRAY,
+				'The classes of the jobs to look for in the database'
+			)
+			->addOption(
+				'once',
+				null,
+				InputOption::VALUE_NONE,
+				'Only execute the worker once (as a regular cron execution would do it)'
+			)
+			->addOption(
+				'interval',
+				'i',
+				InputOption::VALUE_OPTIONAL,
+				'Interval in seconds in which the worker should repeat already processed jobs (set to 0 for no repeat)',
+				5
+			)
+		;
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		$jobClasses = $input->getArgument('job-classes');
+		$jobClasses = empty($jobClasses) ? null : $jobClasses;
+
+		if ($jobClasses !== null) {
+			// at least one class is invalid
+			foreach ($jobClasses as $jobClass) {
+				if (!class_exists($jobClass)) {
+					$output->writeln('<error>Invalid job class: ' . $jobClass . '</error>');
+					return 1;
+				}
+			}
+		}
+
+		while (true) {
+			// Handle canceling of the process
+			try {
+				$this->abortIfInterrupted();
+			} catch (InterruptedException $e) {
+				$output->writeln('<info>Background job worker stopped</info>');
+				break;
+			}
+
+			$this->printSummary($input, $output);
+
+			usleep(50000);
+			$job = $this->jobList->getNext(false, $jobClasses);
+			if (!$job) {
+				if ($input->getOption('once') === true) {
+					if ($jobClasses === null) {
+						$output->writeln('No job is currently queued', OutputInterface::VERBOSITY_VERBOSE);
+					} else {
+						$output->writeln('No job of classes [' . implode(', ', $jobClasses) . '] is currently queued', OutputInterface::VERBOSITY_VERBOSE);
+					}
+					$output->writeln('Exiting...', OutputInterface::VERBOSITY_VERBOSE);
+					break;
+				}
+
+				$output->writeln('Waiting for new jobs to be queued', OutputInterface::VERBOSITY_VERBOSE);
+				// Re-check interval for new jobs
+				sleep(1);
+				continue;
+			}
+
+			$output->writeln('Running job ' . get_class($job) . ' with ID ' . $job->getId());
+
+			if ($output->isVerbose()) {
+				$this->printJobInfo($job->getId(), $job, $output);
+			}
+
+			/** @psalm-suppress DeprecatedMethod Calling execute until it is removed, then will switch to start */
+			$job->execute($this->jobList);
+
+			$output->writeln('Job ' . $job->getId() . ' has finished', OutputInterface::VERBOSITY_VERBOSE);
+
+			// clean up after unclean jobs
+			$this->setupManager->tearDown();
+			$this->tempManager->clean();
+
+			$this->jobList->setLastJob($job);
+			$this->jobList->unlockJob($job);
+
+			if ($input->getOption('once') === true) {
+				break;
+			}
+		}
+
+		return 0;
+	}
+
+	private function printSummary(InputInterface $input, OutputInterface $output): void {
+		if (!$output->isVeryVerbose()) {
+			return;
+		}
+		$output->writeln('<comment>Summary</comment>');
+
+		$counts = [];
+		foreach ($this->jobList->countByClass() as $row) {
+			$counts[] = $row;
+		}
+		$this->writeTableInOutputFormat($input, $output, $counts);
+	}
+}

core/register_command.php

@@ -50,7 +50,6 @@ declare(strict_types=1);
  * along with this program. If not, see <http://www.gnu.org/licenses/>
  *
  */
-
 use OC\Core\Command;
 use OCP\IConfig;
 use OCP\Server;
@@ -88,6 +87,7 @@ if ($config->getSystemValueBool('installed', false)) {
 	$application->add(Server::get(Command\Background\Job::class));
 	$application->add(Server::get(Command\Background\ListCommand::class));
 	$application->add(Server::get(Command\Background\Delete::class));
+	$application->add(Server::get(Command\Background\JobWorker::class));
 
 	$application->add(Server::get(Command\Broadcast\Test::class));
 

cron.php

@@ -57,6 +57,21 @@ use Psr\Log\LoggerInterface;
 try {
 	require_once __DIR__ . '/lib/base.php';
 
+	if ($argv[1] === '-h' || $argv[1] === '--help') {
+		echo 'Description:
+  Run the background job routine
+
+Usage:
+  php -f cron.php -- [-h] [<job-classes>...]
+
+Arguments:
+  job-classes                  Optional job class list to only run those jobs
+
+Options:
+  -h, --help                 Display this help message' . PHP_EOL;
+		exit(0);
+	}
+
 	if (Util::needUpgrade()) {
 		Server::get(LoggerInterface::class)->debug('Update required, skipping cron', ['app' => 'cron']);
 		exit;
@@ -160,7 +175,11 @@ try {
 		$endTime = time() + 14 * 60;
 
 		$executedJobs = [];
-		while ($job = $jobList->getNext($onlyTimeSensitive)) {
+		// a specific job class list can optionally be given as argument
+		$jobClasses = array_slice($argv, 1);
+		$jobClasses = empty($jobClasses) ? null : $jobClasses;
+
+		while ($job = $jobList->getNext($onlyTimeSensitive, $jobClasses)) {
 			if (isset($executedJobs[$job->getId()])) {
 				$jobList->unlockJob($job);
 				break;

lib/composer/composer/autoload_classmap.php

@@ -1056,6 +1056,8 @@ return array(
     'OC\\Core\\Command\\Background\\Cron' => $baseDir . '/core/Command/Background/Cron.php',
     'OC\\Core\\Command\\Background\\Delete' => $baseDir . '/core/Command/Background/Delete.php',
     'OC\\Core\\Command\\Background\\Job' => $baseDir . '/core/Command/Background/Job.php',
+    'OC\\Core\\Command\\Background\\JobBase' => $baseDir . '/core/Command/Background/JobBase.php',
+    'OC\\Core\\Command\\Background\\JobWorker' => $baseDir . '/core/Command/Background/JobWorker.php',
     'OC\\Core\\Command\\Background\\ListCommand' => $baseDir . '/core/Command/Background/ListCommand.php',
     'OC\\Core\\Command\\Background\\WebCron' => $baseDir . '/core/Command/Background/WebCron.php',
     'OC\\Core\\Command\\Base' => $baseDir . '/core/Command/Base.php',

lib/composer/composer/autoload_static.php

@@ -1089,6 +1089,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OC\\Core\\Command\\Background\\Cron' => __DIR__ . '/../../..' . '/core/Command/Background/Cron.php',
         'OC\\Core\\Command\\Background\\Delete' => __DIR__ . '/../../..' . '/core/Command/Background/Delete.php',
         'OC\\Core\\Command\\Background\\Job' => __DIR__ . '/../../..' . '/core/Command/Background/Job.php',
+        'OC\\Core\\Command\\Background\\JobBase' => __DIR__ . '/../../..' . '/core/Command/Background/JobBase.php',
+        'OC\\Core\\Command\\Background\\JobWorker' => __DIR__ . '/../../..' . '/core/Command/Background/JobWorker.php',
         'OC\\Core\\Command\\Background\\ListCommand' => __DIR__ . '/../../..' . '/core/Command/Background/ListCommand.php',
         'OC\\Core\\Command\\Background\\WebCron' => __DIR__ . '/../../..' . '/core/Command/Background/WebCron.php',
         'OC\\Core\\Command\\Base' => __DIR__ . '/../../..' . '/core/Command/Base.php',

lib/private/BackgroundJob/JobList.php

@@ -211,10 +211,9 @@ class JobList implements IJobList {
 	}
 
 	/**
-	 * Get the next job in the list
-	 * @return ?IJob the next job to run. Beware that this object may be a singleton and may be modified by the next call to buildJob.
+	 * @inheritDoc
 	 */
-	public function getNext(bool $onlyTimeSensitive = false): ?IJob {
+	public function getNext(bool $onlyTimeSensitive = false, ?array $jobClasses = null): ?IJob {
 		$query = $this->connection->getQueryBuilder();
 		$query->select('*')
 			->from('jobs')
@@ -227,6 +226,14 @@ class JobList implements IJobList {
 			$query->andWhere($query->expr()->eq('time_sensitive', $query->createNamedParameter(IJob::TIME_SENSITIVE, IQueryBuilder::PARAM_INT)));
 		}
 
+		if ($jobClasses !== null && count($jobClasses) > 0) {
+			$orClasses = $query->expr()->orx();
+			foreach ($jobClasses as $jobClass) {
+				$orClasses->add($query->expr()->eq('class', $query->createNamedParameter($jobClass, IQueryBuilder::PARAM_STR)));
+			}
+			$query->andWhere($orClasses);
+		}
+
 		$result = $query->executeQuery();
 		$row = $result->fetch();
 		$result->closeCursor();
@@ -261,7 +268,7 @@ class JobList implements IJobList {
 
 			if ($count === 0) {
 				// Background job already executed elsewhere, try again.
-				return $this->getNext($onlyTimeSensitive);
+				return $this->getNext($onlyTimeSensitive, $jobClasses);
 			}
 
 			if ($job === null) {
@@ -274,7 +281,7 @@ class JobList implements IJobList {
 				$reset->executeStatement();
 
 				// Background job from disabled app, try again.
-				return $this->getNext($onlyTimeSensitive);
+				return $this->getNext($onlyTimeSensitive, $jobClasses);
 			}
 
 			return $job;
@@ -432,4 +439,26 @@ class JobList implements IJobList {
 			return false;
 		}
 	}
+
+	public function countByClass(): array {
+		$query = $this->connection->getQueryBuilder();
+		$query->select('class')
+			->selectAlias($query->func()->count('id'), 'count')
+			->from('jobs')
+			->orderBy('count')
+			->groupBy('class');
+
+		$result = $query->executeQuery();
+
+		$jobs = [];
+
+		while (($row = $result->fetch()) !== false) {
+			/**
+			 * @var array{count:int, class:class-string} $row
+			 */
+			$jobs[] = $row;
+		}
+
+		return $jobs;
+	}
 }

lib/public/BackgroundJob/IJobList.php

@@ -108,11 +108,14 @@ interface IJobList {
 	public function getJobsIterator($job, ?int $limit, int $offset): iterable;
 
 	/**
-	 * get the next job in the list
+	 * Get the next job in the list
 	 *
-	 * @since 7.0.0 - In 24.0.0 parameter $onlyTimeSensitive got added
+	 * @param bool $onlyTimeSensitive Whether we get only time sensitive jobs or not
+	 * @param class-string<IJob>[]|null $jobClasses List of job classes to restrict which next job we get
+	 * @return ?IJob the next job to run. Beware that this object may be a singleton and may be modified by the next call to buildJob.
+	 * @since 7.0.0 - In 24.0.0 parameter $onlyTimeSensitive got added; In 30.0.0 parameter $jobClasses got added
 	 */
-	public function getNext(bool $onlyTimeSensitive = false): ?IJob;
+	public function getNext(bool $onlyTimeSensitive = false, ?array $jobClasses = null): ?IJob;
 
 	/**
 	 * @since 7.0.0
@@ -168,4 +171,12 @@ interface IJobList {
 	 * @since 27.0.0
 	 */
 	public function hasReservedJob(?string $className): bool;
+
+	/**
+	 * Returns a count of jobs per Job class
+	 *
+	 * @return list<array{class:class-string, count:int}>
+	 * @since 30.0.0
+	 */
+	public function countByClass(): array;
 }

tests/lib/BackgroundJob/DummyJobList.php

@@ -100,7 +100,7 @@ class DummyJobList extends \OC\BackgroundJob\JobList {
 	/**
 	 * get the next job in the list
 	 */
-	public function getNext(bool $onlyTimeSensitive = false): ?IJob {
+	public function getNext(bool $onlyTimeSensitive = false, ?array $jobClasses = null): ?IJob {
 		if (count($this->jobs) > 0) {
 			if ($this->last < (count($this->jobs) - 1)) {
 				$i = $this->last + 1;