upstream/nextcloud
1
0
Fork 0
mirror of https://github.com/nextcloud/server.git synced 2026-06-09 00:32:29 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 35

feat: add command to list orphan objects

Browse source 
Signed-off-by: Robin Appelman <robin@icewind.nl>
This commit is contained in:
Robin Appelman 2025-03-20 16:33:32 +01:00
parent fcde776683
commit f17cf83e16
No known key found for this signature in database
GPG key ID: 42B69D8A64526EFB
6 changed files with 154 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
apps/files/appinfo/info.xml
View file

@ -51,6 +51,7 @@
<command>OCA\Files\Command\Object\Put</command>
<command>OCA\Files\Command\Object\Info</command>
<command>OCA\Files\Command\Object\ListObject</command>
<command>OCA\Files\Command\Object\Orphans</command>
</commands>
<settings>

1
apps/files/composer/composer/autoload_classmap.php
View file

@ -38,6 +38,7 @@ return array(
'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php',
'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php',
'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php',

1
apps/files/composer/composer/autoload_static.php
View file

@ -53,6 +53,7 @@ class ComposerStaticInitFiles
'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php',
'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php',
'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php',

72
apps/files/lib/Command/Object/ListObject.php
View file

@ -41,79 +41,9 @@ class ListObject extends Base {
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
return self::FAILURE;
}
$outputType = $input->getOption('output');
$humanOutput = $outputType === self::OUTPUT_FORMAT_PLAIN;
if (!$humanOutput) {
$output->writeln('[');
}
$objects = $objectStore->listObjects();
$first = true;
foreach ($this->chunkIterator($objects, self::CHUNK_SIZE) as $chunk) {
if ($outputType === self::OUTPUT_FORMAT_PLAIN) {
$this->outputChunk($input, $output, $chunk);
} else {
foreach ($chunk as $object) {
if (!$first) {
$output->writeln(',');
}
$row = $this->formatObject($object, $humanOutput);
if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) {
$output->write(json_encode($row, JSON_PRETTY_PRINT));
} else {
$output->write(json_encode($row));
}
$first = false;
}
}
}
if (!$humanOutput) {
$output->writeln("\n]");
}
$this->objectUtils->writeIteratorToOutput($input, $output, $objects, self::CHUNK_SIZE);
return self::SUCCESS;
}
private function formatObject(array $object, bool $humanOutput): array {
$row = array_merge([
'urn' => $object['urn'],
], ($object['metadata'] ?? []));
if ($humanOutput && isset($row['size'])) {
$row['size'] = \OC_Helper::humanFileSize($row['size']);
}
if (isset($row['mtime'])) {
$row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM);
}
return $row;
}
private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void {
$result = [];
$humanOutput = $input->getOption('output') === "plain";
foreach ($chunk as $object) {
$result[] = $this->formatObject($object, $humanOutput);
}
$this->writeTableInOutputFormat($input, $output, $result);
}
function chunkIterator(\Iterator $iterator, int $count): \Iterator {
$chunk = [];
for($i = 0; $iterator->valid(); $i++){
$chunk[] = $iterator->current();
$iterator->next();
if(count($chunk) == $count){
yield $chunk;
$chunk = [];
}
}
if(count($chunk)){
yield $chunk;
}
}
}

78
apps/files/lib/Command/Object/ObjectUtil.php
View file

@ -8,13 +8,15 @@ declare(strict_types=1);
namespace OCA\Files\Command\Object;
use OC\Core\Command\Base;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\IConfig;
use OCP\IDBConnection;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class ObjectUtil {
class ObjectUtil extends Base {
public function __construct(
private IConfig $config,
private IDBConnection $connection,
@ -91,4 +93,78 @@ class ObjectUtil {
return $fileId;
}
public function writeIteratorToOutput(InputInterface $input, OutputInterface $output, \Iterator $objects, int $chunkSize): void {
$outputType = $input->getOption('output');
$humanOutput = $outputType === Base::OUTPUT_FORMAT_PLAIN;
$first = true;
if (!$humanOutput) {
$output->writeln('[');
}
foreach ($this->chunkIterator($objects, $chunkSize) as $chunk) {
if ($outputType === Base::OUTPUT_FORMAT_PLAIN) {
$this->outputChunk($input, $output, $chunk);
} else {
foreach ($chunk as $object) {
if (!$first) {
$output->writeln(',');
}
$row = $this->formatObject($object, $humanOutput);
if ($outputType === Base::OUTPUT_FORMAT_JSON_PRETTY) {
$output->write(json_encode($row, JSON_PRETTY_PRINT));
} else {
$output->write(json_encode($row));
}
$first = false;
}
}
}
if (!$humanOutput) {
$output->writeln("\n]");
}
}
private function formatObject(array $object, bool $humanOutput): array {
$row = array_merge([
'urn' => $object['urn'],
], ($object['metadata'] ?? []));
if ($humanOutput && isset($row['size'])) {
$row['size'] = \OC_Helper::humanFileSize($row['size']);
}
if (isset($row['mtime'])) {
$row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM);
}
return $row;
}
private function outputChunk(InputInterface $input, OutputInterface $output, iterable $chunk): void {
$result = [];
$humanOutput = $input->getOption('output') === 'plain';
foreach ($chunk as $object) {
$result[] = $this->formatObject($object, $humanOutput);
}
$this->writeTableInOutputFormat($input, $output, $result);
}
public function chunkIterator(\Iterator $iterator, int $count): \Iterator {
$chunk = [];
for ($i = 0; $iterator->valid(); $i++) {
$chunk[] = $iterator->current();
$iterator->next();
if (count($chunk) == $count) {
yield $chunk;
$chunk = [];
}
}
if (count($chunk)) {
yield $chunk;
}
}
}

73
apps/files/lib/Command/Object/Orphans.php Normal file
View file

@ -0,0 +1,73 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OCA\Files\Command\Object;
use OC\Core\Command\Base;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\ObjectStore\IObjectStoreMetaData;
use OCP\IDBConnection;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Orphans extends Base {
private const CHUNK_SIZE = 100;
private IQueryBuilder $query;
public function __construct(
private readonly ObjectUtil $objectUtils,
IDBConnection $connection,
) {
parent::__construct();
$this->query = $connection->getQueryBuilder();
$this->query->select('fileid')
->from('filecache')
->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id')));
}
protected function configure(): void {
parent::configure();
$this
->setName('files:object:orphans')
->setDescription('List all objects in the object store that don\'t have a matching entry in the database')
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
if (!$objectStore) {
return self::FAILURE;
}
if (!$objectStore instanceof IObjectStoreMetaData) {
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
return self::FAILURE;
}
$prefixLength = strlen('urn:oid:');
$objects = $objectStore->listObjects('urn:oid:');
$objects->rewind();
$orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) {
$fileId = (int)substr($object['urn'], $prefixLength);
return !$this->fileIdInDb($fileId);
});
$orphans = new \ArrayIterator(iterator_to_array($orphans));
$this->objectUtils->writeIteratorToOutput($input, $output, $orphans, self::CHUNK_SIZE);
return self::SUCCESS;
}
private function fileIdInDb(int $fileId): bool {
$this->query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT);
$result = $this->query->executeQuery();
return $result->fetchOne() !== false;
}
}