upstream/nextcloud
1
0
Fork 0
mirror of https://github.com/nextcloud/server.git synced 2026-02-20 00:12:30 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions 38

Merge pull request #52866 from nextcloud/backport/51603/stable31

Browse source 
[stable31] Add command to list orphan objects
This commit is contained in:
Robin Appelman 2025-05-16 21:36:33 +02:00 committed by GitHub
commit d18ff624d0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 367 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
apps/files/appinfo/info.xml
View file

@ -49,6 +49,9 @@
<command>OCA\Files\Command\Object\Delete</command>
<command>OCA\Files\Command\Object\Get</command>
<command>OCA\Files\Command\Object\Put</command>
<command>OCA\Files\Command\Object\Info</command>
<command>OCA\Files\Command\Object\ListObject</command>
<command>OCA\Files\Command\Object\Orphans</command>
</commands>
<settings>

3
apps/files/composer/composer/autoload_classmap.php
View file

@ -34,7 +34,10 @@ return array(
'OCA\\Files\\Command\\Move' => $baseDir . '/../lib/Command/Move.php',
'OCA\\Files\\Command\\Object\\Delete' => $baseDir . '/../lib/Command/Object/Delete.php',
'OCA\\Files\\Command\\Object\\Get' => $baseDir . '/../lib/Command/Object/Get.php',
'OCA\\Files\\Command\\Object\\Info' => $baseDir . '/../lib/Command/Object/Info.php',
'OCA\\Files\\Command\\Object\\ListObject' => $baseDir . '/../lib/Command/Object/ListObject.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => $baseDir . '/../lib/Command/Object/ObjectUtil.php',
'OCA\\Files\\Command\\Object\\Orphans' => $baseDir . '/../lib/Command/Object/Orphans.php',
'OCA\\Files\\Command\\Object\\Put' => $baseDir . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => $baseDir . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => $baseDir . '/../lib/Command/RepairTree.php',

3
apps/files/composer/composer/autoload_static.php
View file

@ -49,7 +49,10 @@ class ComposerStaticInitFiles
'OCA\\Files\\Command\\Move' => __DIR__ . '/..' . '/../lib/Command/Move.php',
'OCA\\Files\\Command\\Object\\Delete' => __DIR__ . '/..' . '/../lib/Command/Object/Delete.php',
'OCA\\Files\\Command\\Object\\Get' => __DIR__ . '/..' . '/../lib/Command/Object/Get.php',
'OCA\\Files\\Command\\Object\\Info' => __DIR__ . '/..' . '/../lib/Command/Object/Info.php',
'OCA\\Files\\Command\\Object\\ListObject' => __DIR__ . '/..' . '/../lib/Command/Object/ListObject.php',
'OCA\\Files\\Command\\Object\\ObjectUtil' => __DIR__ . '/..' . '/../lib/Command/Object/ObjectUtil.php',
'OCA\\Files\\Command\\Object\\Orphans' => __DIR__ . '/..' . '/../lib/Command/Object/Orphans.php',
'OCA\\Files\\Command\\Object\\Put' => __DIR__ . '/..' . '/../lib/Command/Object/Put.php',
'OCA\\Files\\Command\\Put' => __DIR__ . '/..' . '/../lib/Command/Put.php',
'OCA\\Files\\Command\\RepairTree' => __DIR__ . '/..' . '/../lib/Command/RepairTree.php',

80
apps/files/lib/Command/Object/Info.php Normal file
View file

@ -0,0 +1,80 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OCA\Files\Command\Object;
use OC\Core\Command\Base;
use OC\Files\ObjectStore\IObjectStoreMetaData;
use OCP\Files\IMimeTypeDetector;
use OCP\Util;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Info extends Base {
public function __construct(
private ObjectUtil $objectUtils,
private IMimeTypeDetector $mimeTypeDetector,
) {
parent::__construct();
}
protected function configure(): void {
parent::configure();
$this
->setName('files:object:info')
->setDescription('Get the metadata of an object')
->addArgument('object', InputArgument::REQUIRED, 'Object to get')
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to get the object from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$object = $input->getArgument('object');
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
if (!$objectStore) {
return self::FAILURE;
}
if (!$objectStore instanceof IObjectStoreMetaData) {
$output->writeln('<error>Configured object store does currently not support retrieve metadata</error>');
return self::FAILURE;
}
if (!$objectStore->objectExists($object)) {
$output->writeln("<error>Object $object does not exist</error>");
return self::FAILURE;
}
try {
$meta = $objectStore->getObjectMetaData($object);
} catch (\Exception $e) {
$msg = $e->getMessage();
$output->writeln("<error>Failed to read $object from object store: $msg</error>");
return self::FAILURE;
}
if ($input->getOption('output') === 'plain' && isset($meta['size'])) {
$meta['size'] = Util::humanFileSize($meta['size']);
}
if (isset($meta['mtime'])) {
$meta['mtime'] = $meta['mtime']->format(\DateTimeImmutable::ATOM);
}
if (!isset($meta['mimetype'])) {
$handle = $objectStore->readObject($object);
$head = fread($handle, 8192);
fclose($handle);
$meta['mimetype'] = $this->mimeTypeDetector->detectString($head);
}
$this->writeArrayInOutputFormat($input, $output, $meta);
return self::SUCCESS;
}
}

50
apps/files/lib/Command/Object/ListObject.php Normal file
View file

@ -0,0 +1,50 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OCA\Files\Command\Object;
use OC\Core\Command\Base;
use OC\Files\ObjectStore\IObjectStoreMetaData;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class ListObject extends Base {
private const CHUNK_SIZE = 100;
public function __construct(
private readonly ObjectUtil $objectUtils,
) {
parent::__construct();
}
protected function configure(): void {
parent::configure();
$this
->setName('files:object:list')
->setDescription('List all objects in the object store')
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
if (!$objectStore) {
return self::FAILURE;
}
if (!$objectStore instanceof IObjectStoreMetaData) {
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
return self::FAILURE;
}
$objects = $objectStore->listObjects();
$objects = $this->objectUtils->formatObjects($objects, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN);
$this->writeStreamingTableInOutputFormat($input, $output, $objects, self::CHUNK_SIZE);
return self::SUCCESS;
}
}

21
apps/files/lib/Command/Object/ObjectUtil.php
View file

@ -12,6 +12,7 @@ use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\IConfig;
use OCP\IDBConnection;
use OCP\Util;
use Symfony\Component\Console\Output\OutputInterface;
class ObjectUtil {
@ -91,4 +92,24 @@ class ObjectUtil {
return $fileId;
}
public function formatObjects(\Iterator $objects, bool $humanOutput): \Iterator {
foreach ($objects as $object) {
yield $this->formatObject($object, $humanOutput);
}
}
public function formatObject(array $object, bool $humanOutput): array {
$row = array_merge([
'urn' => $object['urn'],
], ($object['metadata'] ?? []));
if ($humanOutput && isset($row['size'])) {
$row['size'] = Util::humanFileSize($row['size']);
}
if (isset($row['mtime'])) {
$row['mtime'] = $row['mtime']->format(\DateTimeImmutable::ATOM);
}
return $row;
}
}

79
apps/files/lib/Command/Object/Orphans.php Normal file
View file

@ -0,0 +1,79 @@
<?php
declare(strict_types=1);
/**
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OCA\Files\Command\Object;
use OC\Core\Command\Base;
use OC\Files\ObjectStore\IObjectStoreMetaData;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class Orphans extends Base {
private const CHUNK_SIZE = 100;
private ?IQueryBuilder $query = null;
public function __construct(
private readonly ObjectUtil $objectUtils,
private readonly IDBConnection $connection,
) {
parent::__construct();
}
private function getQuery(): IQueryBuilder {
if (!$this->query) {
$this->query = $this->connection->getQueryBuilder();
$this->query->select('fileid')
->from('filecache')
->where($this->query->expr()->eq('fileid', $this->query->createParameter('file_id')));
}
return $this->query;
}
protected function configure(): void {
parent::configure();
$this
->setName('files:object:orphans')
->setDescription('List all objects in the object store that don\'t have a matching entry in the database')
->addOption('bucket', 'b', InputOption::VALUE_REQUIRED, "Bucket to list the objects from, only required in cases where it can't be determined from the config");
}
public function execute(InputInterface $input, OutputInterface $output): int {
$objectStore = $this->objectUtils->getObjectStore($input->getOption('bucket'), $output);
if (!$objectStore) {
return self::FAILURE;
}
if (!$objectStore instanceof IObjectStoreMetaData) {
$output->writeln('<error>Configured object store does currently not support listing objects</error>');
return self::FAILURE;
}
$prefixLength = strlen('urn:oid:');
$objects = $objectStore->listObjects('urn:oid:');
$orphans = new \CallbackFilterIterator($objects, function (array $object) use ($prefixLength) {
$fileId = (int)substr($object['urn'], $prefixLength);
return !$this->fileIdInDb($fileId);
});
$orphans = $this->objectUtils->formatObjects($orphans, $input->getOption('output') === self::OUTPUT_FORMAT_PLAIN);
$this->writeStreamingTableInOutputFormat($input, $output, $orphans, self::CHUNK_SIZE);
return self::SUCCESS;
}
private function fileIdInDb(int $fileId): bool {
$query = $this->getQuery();
$query->setParameter('file_id', $fileId, IQueryBuilder::PARAM_INT);
$result = $query->executeQuery();
return $result->fetchOne() !== false;
}
}

52
core/Command/Base.php
View file

@ -88,6 +88,58 @@ class Base extends Command implements CompletionAwareInterface {
}
}
protected function writeStreamingTableInOutputFormat(InputInterface $input, OutputInterface $output, \Iterator $items, int $tableGroupSize): void {
switch ($input->getOption('output')) {
case self::OUTPUT_FORMAT_JSON:
case self::OUTPUT_FORMAT_JSON_PRETTY:
$this->writeStreamingJsonArray($input, $output, $items);
break;
default:
foreach ($this->chunkIterator($items, $tableGroupSize) as $chunk) {
$this->writeTableInOutputFormat($input, $output, $chunk);
}
break;
}
}
protected function writeStreamingJsonArray(InputInterface $input, OutputInterface $output, \Iterator $items): void {
$first = true;
$outputType = $input->getOption('output');
$output->writeln('[');
foreach ($items as $item) {
if (!$first) {
$output->writeln(',');
}
if ($outputType === self::OUTPUT_FORMAT_JSON_PRETTY) {
$output->write(json_encode($item, JSON_PRETTY_PRINT));
} else {
$output->write(json_encode($item));
}
$first = false;
}
$output->writeln("\n]");
}
public function chunkIterator(\Iterator $iterator, int $count): \Iterator {
$chunk = [];
for ($i = 0; $iterator->valid(); $i++) {
$chunk[] = $iterator->current();
$iterator->next();
if (count($chunk) == $count) {
// Got a full chunk, yield and start a new one
yield $chunk;
$chunk = [];
}
}
if (count($chunk)) {
// Yield the last chunk even if incomplete
yield $chunk;
}
}
/**
* @param mixed $item

1
lib/composer/composer/autoload_classmap.php
View file

@ -1627,6 +1627,7 @@ return array(
'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Azure' => $baseDir . '/lib/private/Files/ObjectStore/Azure.php',
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\IObjectStoreMetaData' => $baseDir . '/lib/private/Files/ObjectStore/IObjectStoreMetaData.php',
'OC\\Files\\ObjectStore\\Mapper' => $baseDir . '/lib/private/Files/ObjectStore/Mapper.php',
'OC\\Files\\ObjectStore\\ObjectStoreScanner' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
'OC\\Files\\ObjectStore\\ObjectStoreStorage' => $baseDir . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',

1
lib/composer/composer/autoload_static.php
View file

@ -1676,6 +1676,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OC\\Files\\ObjectStore\\AppdataPreviewObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/AppdataPreviewObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\Azure' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Azure.php',
'OC\\Files\\ObjectStore\\HomeObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/HomeObjectStoreStorage.php',
'OC\\Files\\ObjectStore\\IObjectStoreMetaData' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/IObjectStoreMetaData.php',
'OC\\Files\\ObjectStore\\Mapper' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/Mapper.php',
'OC\\Files\\ObjectStore\\ObjectStoreScanner' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreScanner.php',
'OC\\Files\\ObjectStore\\ObjectStoreStorage' => __DIR__ . '/../../..' . '/lib/private/Files/ObjectStore/ObjectStoreStorage.php',

36
lib/private/Files/ObjectStore/IObjectStoreMetaData.php Normal file
View file

@ -0,0 +1,36 @@
<?php
/**
* SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
namespace OC\Files\ObjectStore;
/**
* Interface IObjectStoreMetaData
*
* @psalm-type ObjectMetaData = array{mtime?: \DateTime, etag?: string, size?: int, mimetype?: string, filename?: string}
*/
interface IObjectStoreMetaData {
/**
* Get metadata for an object.
*
* @param string $urn
* @return ObjectMetaData
*
* @since 32.0.0
*/
public function getObjectMetaData(string $urn): array;
/**
* List all objects in the object store.
*
* If the object store implementation can do it efficiently, the metadata for each object is also included.
*
* @param string $prefix
* @return \Iterator<array{urn: string, metadata: ?ObjectMetaData}>
*
* @since 32.0.0
*/
public function listObjects(string $prefix = ''): \Iterator;
}

41
lib/private/Files/ObjectStore/S3.php
View file

@ -3,6 +3,7 @@
* SPDX-FileCopyrightText: 2016 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/
namespace OC\Files\ObjectStore;
use Aws\Result;
@ -10,7 +11,7 @@ use Exception;
use OCP\Files\ObjectStore\IObjectStore;
use OCP\Files\ObjectStore\IObjectStoreMultiPartUpload;
class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
class S3 implements IObjectStore, IObjectStoreMultiPartUpload, IObjectStoreMetaData {
use S3ConnectionTrait;
use S3ObjectTrait;
@ -61,7 +62,7 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
'Key' => $urn,
'UploadId' => $uploadId,
'MaxParts' => 1000,
'PartNumberMarker' => $partNumberMarker
'PartNumberMarker' => $partNumberMarker,
] + $this->getSSECParameters());
$parts = array_merge($parts, $result->get('Parts') ?? []);
$isTruncated = $result->get('IsTruncated');
@ -89,7 +90,41 @@ class S3 implements IObjectStore, IObjectStoreMultiPartUpload {
$this->getConnection()->abortMultipartUpload([
'Bucket' => $this->bucket,
'Key' => $urn,
'UploadId' => $uploadId
'UploadId' => $uploadId,
]);
}
public function getObjectMetaData(string $urn): array {
$object = $this->getConnection()->headObject([
'Bucket' => $this->bucket,
'Key' => $urn
] + $this->getSSECParameters())->toArray();
return [
'mtime' => $object['LastModified'],
'etag' => trim($object['ETag'], '"'),
'size' => (int)($object['Size'] ?? $object['ContentLength']),
];
}
public function listObjects(string $prefix = ''): \Iterator {
$results = $this->getConnection()->getPaginator('ListObjectsV2', [
'Bucket' => $this->bucket,
'Prefix' => $prefix,
] + $this->getSSECParameters());
foreach ($results as $result) {
if (is_array($result['Contents'])) {
foreach ($result['Contents'] as $object) {
yield [
'urn' => basename($object['Key']),
'metadata' => [
'mtime' => $object['LastModified'],
'etag' => trim($object['ETag'], '"'),
'size' => (int)($object['Size'] ?? $object['ContentLength']),
],
];
}
}
}
}
}