mirror of
https://github.com/nextcloud/server.git
synced 2026-02-20 00:12:30 -05:00
s3 external storage listing rework
Signed-off-by: Robin Appelman <robin@icewind.nl>
This commit is contained in:
Robin Appelman
parent
fadeae8c8a
commit
09ffac5e6d
3 changed files with 217 additions and 101 deletions
65
.github/workflows/s3-external.yml
vendored
Normal file
65
.github/workflows/s3-external.yml
vendored
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
name: S3 External storage
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- stable*
|
||||
paths:
|
||||
- 'apps/files_external/**'
|
||||
pull_request:
|
||||
paths:
|
||||
- 'apps/files_external/**'
|
||||
|
||||
env:
|
||||
APP_NAME: files_external
|
||||
|
||||
jobs:
|
||||
s3-external-tests:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
# do not stop on another job's failure
|
||||
fail-fast: false
|
||||
matrix:
|
||||
php-versions: ['7.4', '8.0']
|
||||
|
||||
name: php${{ matrix.php-versions }}-${{ matrix.ftpd }}
|
||||
|
||||
services:
|
||||
minio:
|
||||
image: minio/minio:RELEASE.2021-10-06T23-36-31Z
|
||||
ports:
|
||||
- "9000:9000"
|
||||
|
||||
steps:
|
||||
- name: Checkout server
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
submodules: true
|
||||
|
||||
- name: Set up php ${{ matrix.php-versions }}
|
||||
uses: shivammathur/setup-php@v2
|
||||
with:
|
||||
php-version: ${{ matrix.php-versions }}
|
||||
tools: phpunit
|
||||
extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_sqlite, zip, gd
|
||||
|
||||
- name: Set up Nextcloud
|
||||
run: |
|
||||
mkdir data
|
||||
./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
|
||||
./occ app:enable --force ${{ env.APP_NAME }}
|
||||
php -S localhost:8080 &
|
||||
- name: PHPUnit
|
||||
run: |
|
||||
echo "<?php return ['run' => true,'hostname' => 'localhost','key' => 'minioadmin','secret' => 'minioadmin', 'bucket' => 'bucket', 'port' => 9000, 'use_ssl' => false, 'autocreate' => true, 'use_path_style' => true];" > apps/${{ env.APP_NAME }}/tests/config.amazons3.php
|
||||
phpunit --configuration tests/phpunit-autotest-external.xml apps/files_external/tests/Storage/Amazons3Test.php
|
||||
s3-external-summary:
|
||||
runs-on: ubuntu-latest
|
||||
needs: s3-external-tests
|
||||
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- name: Summary status
|
||||
run: if ${{ needs.s3-external-tests.result != 'success' }}; then exit 1; fi
|
||||
|
|
@ -46,6 +46,7 @@ use Icewind\Streams\CallbackWrapper;
|
|||
use Icewind\Streams\IteratorDirectory;
|
||||
use OC\Cache\CappedMemoryCache;
|
||||
use OC\Files\Cache\CacheEntry;
|
||||
use OC\Files\Filesystem;
|
||||
use OC\Files\ObjectStore\S3ConnectionTrait;
|
||||
use OC\Files\ObjectStore\S3ObjectTrait;
|
||||
use OCP\Constants;
|
||||
|
|
@ -71,6 +72,9 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
/** @var IMimeTypeDetector */
|
||||
private $mimeDetector;
|
||||
|
||||
/** @var bool|null */
|
||||
private $versioningEnabled = null;
|
||||
|
||||
public function __construct($parameters) {
|
||||
parent::__construct($parameters);
|
||||
$this->parseParams($parameters);
|
||||
|
|
@ -120,12 +124,20 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
unset($this->objectCache[$existingKey]);
|
||||
}
|
||||
}
|
||||
unset($this->directoryCache[$key], $this->filesCache[$key]);
|
||||
unset($this->filesCache[$key]);
|
||||
$keys = array_keys($this->directoryCache->getData());
|
||||
$keyLength = strlen($key);
|
||||
foreach ($keys as $existingKey) {
|
||||
if (substr($existingKey, 0, $keyLength) === $key) {
|
||||
unset($this->directoryCache[$existingKey]);
|
||||
}
|
||||
}
|
||||
unset($this->directoryCache[$key]);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param $key
|
||||
* @return Result|boolean
|
||||
* @return array|false
|
||||
*/
|
||||
private function headObject($key) {
|
||||
if (!isset($this->objectCache[$key])) {
|
||||
|
|
@ -133,7 +145,7 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
$this->objectCache[$key] = $this->getConnection()->headObject([
|
||||
'Bucket' => $this->bucket,
|
||||
'Key' => $key
|
||||
]);
|
||||
])->toArray();
|
||||
} catch (S3Exception $e) {
|
||||
if ($e->getStatusCode() >= 500) {
|
||||
throw $e;
|
||||
|
|
@ -159,32 +171,44 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
* @throws \Exception
|
||||
*/
|
||||
private function doesDirectoryExist($path) {
|
||||
if (!isset($this->directoryCache[$path])) {
|
||||
// Maybe this isn't an actual key, but a prefix.
|
||||
// Do a prefix listing of objects to determine.
|
||||
try {
|
||||
$result = $this->getConnection()->listObjects([
|
||||
'Bucket' => $this->bucket,
|
||||
'Prefix' => rtrim($path, '/'),
|
||||
'MaxKeys' => 1,
|
||||
'Delimiter' => '/',
|
||||
]);
|
||||
|
||||
if ((isset($result['Contents'][0]['Key']) && $result['Contents'][0]['Key'] === rtrim($path, '/') . '/')
|
||||
|| isset($result['CommonPrefixes'])) {
|
||||
$this->directoryCache[$path] = true;
|
||||
} else {
|
||||
$this->directoryCache[$path] = false;
|
||||
}
|
||||
} catch (S3Exception $e) {
|
||||
if ($e->getStatusCode() === 403) {
|
||||
$this->directoryCache[$path] = false;
|
||||
}
|
||||
throw $e;
|
||||
}
|
||||
if ($path === '.' || $path === '') {
|
||||
return true;
|
||||
}
|
||||
|
||||
return $this->directoryCache[$path];
|
||||
if (isset($this->directoryCache[$path])) {
|
||||
return $this->directoryCache[$path];
|
||||
}
|
||||
try {
|
||||
// Maybe this isn't an actual key, but a prefix.
|
||||
// Do a prefix listing of objects to determine.
|
||||
$result = $this->getConnection()->listObjectsV2([
|
||||
'Bucket' => $this->bucket,
|
||||
'Prefix' => rtrim($path, '/'),
|
||||
'MaxKeys' => 1,
|
||||
]);
|
||||
|
||||
if (isset($result['Contents'])) {
|
||||
$this->directoryCache[$path] = true;
|
||||
return true;
|
||||
}
|
||||
|
||||
// empty directories have their own object
|
||||
$object = $this->headObject($path);
|
||||
|
||||
if ($object) {
|
||||
$this->directoryCache[$path] = true;
|
||||
return true;
|
||||
}
|
||||
} catch (S3Exception $e) {
|
||||
if ($e->getStatusCode() === 403) {
|
||||
$this->directoryCache[$path] = false;
|
||||
}
|
||||
throw $e;
|
||||
}
|
||||
|
||||
|
||||
$this->directoryCache[$path] = false;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -284,7 +308,9 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
protected function clearBucket() {
|
||||
$this->clearCache();
|
||||
try {
|
||||
$this->getConnection()->clearBucket($this->bucket);
|
||||
$this->getConnection()->clearBucket([
|
||||
"Bucket" => $this->bucket
|
||||
]);
|
||||
return true;
|
||||
// clearBucket() is not working with Ceph, so if it fails we try the slower approach
|
||||
} catch (\Exception $e) {
|
||||
|
|
@ -318,7 +344,9 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
}
|
||||
// we reached the end when the list is no longer truncated
|
||||
} while ($objects['IsTruncated']);
|
||||
$this->deleteObject($path);
|
||||
if ($path !== '' && $path !== null) {
|
||||
$this->deleteObject($path);
|
||||
}
|
||||
} catch (S3Exception $e) {
|
||||
\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
|
||||
return false;
|
||||
|
|
@ -327,54 +355,12 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
}
|
||||
|
||||
public function opendir($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
if ($this->isRoot($path)) {
|
||||
$path = '';
|
||||
} else {
|
||||
$path .= '/';
|
||||
}
|
||||
|
||||
try {
|
||||
$files = [];
|
||||
$results = $this->getConnection()->getPaginator('ListObjects', [
|
||||
'Bucket' => $this->bucket,
|
||||
'Delimiter' => '/',
|
||||
'Prefix' => $path,
|
||||
]);
|
||||
|
||||
foreach ($results as $result) {
|
||||
// sub folders
|
||||
if (is_array($result['CommonPrefixes'])) {
|
||||
foreach ($result['CommonPrefixes'] as $prefix) {
|
||||
$directoryName = trim($prefix['Prefix'], '/');
|
||||
$files[] = substr($directoryName, strlen($path));
|
||||
$this->directoryCache[$directoryName] = true;
|
||||
}
|
||||
}
|
||||
if (is_array($result['Contents'])) {
|
||||
foreach ($result['Contents'] as $object) {
|
||||
if (isset($object['Key']) && $object['Key'] === $path) {
|
||||
// it's the directory itself, skip
|
||||
continue;
|
||||
}
|
||||
$file = basename(
|
||||
isset($object['Key']) ? $object['Key'] : $object['Prefix']
|
||||
);
|
||||
$files[] = $file;
|
||||
|
||||
// store this information for later usage
|
||||
$this->filesCache[$path . $file] = [
|
||||
'ContentLength' => $object['Size'],
|
||||
'LastModified' => (string)$object['LastModified'],
|
||||
];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return IteratorDirectory::wrap($files);
|
||||
} catch (S3Exception $e) {
|
||||
\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
|
||||
$content = iterator_to_array($this->getDirectoryContent($path));
|
||||
return IteratorDirectory::wrap(array_map(function (array $item) {
|
||||
return $item['name'];
|
||||
}, $content));
|
||||
} catch (S3Exception $e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
@ -382,33 +368,19 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
public function stat($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
try {
|
||||
$stat = [];
|
||||
if ($this->is_dir($path)) {
|
||||
$cacheEntry = $this->getCache()->get($path);
|
||||
if ($cacheEntry instanceof CacheEntry) {
|
||||
$stat['size'] = $cacheEntry->getSize();
|
||||
$stat['mtime'] = $cacheEntry->getMTime();
|
||||
} else {
|
||||
// Use dummy values
|
||||
$stat['size'] = -1; // Pending
|
||||
$stat['mtime'] = time();
|
||||
}
|
||||
} else {
|
||||
$stat['size'] = $this->getContentLength($path);
|
||||
$stat['mtime'] = strtotime($this->getLastModified($path));
|
||||
if ($this->is_dir($path)) {
|
||||
$stat = $this->getDirectoryMetaData($path);
|
||||
} else {
|
||||
$object = $this->headObject($path);
|
||||
if ($object === false) {
|
||||
return false;
|
||||
}
|
||||
$stat['atime'] = time();
|
||||
|
||||
return $stat;
|
||||
} catch (S3Exception $e) {
|
||||
\OC::$server->getLogger()->logException($e, ['app' => 'files_external']);
|
||||
return false;
|
||||
$object["Key"] = $path;
|
||||
$stat = $this->objectToMetaData($object);
|
||||
}
|
||||
}
|
||||
$stat['atime'] = time();
|
||||
|
||||
public function hasUpdated($path, $time) {
|
||||
return $this->getMountOption('filesystem_check_changes', 1) === 1 || parent::hasUpdated($path, $time);
|
||||
return $stat;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -711,4 +683,83 @@ class AmazonS3 extends \OC\Files\Storage\Common {
|
|||
public static function checkDependencies() {
|
||||
return true;
|
||||
}
|
||||
|
||||
public function getDirectoryContent($directory): \Traversable {
|
||||
$path = $this->normalizePath($directory);
|
||||
|
||||
if ($this->isRoot($path)) {
|
||||
$path = '';
|
||||
} else {
|
||||
$path .= '/';
|
||||
}
|
||||
|
||||
$results = $this->getConnection()->getPaginator('ListObjectsV2', [
|
||||
'Bucket' => $this->bucket,
|
||||
'Delimiter' => '/',
|
||||
'Prefix' => $path,
|
||||
]);
|
||||
|
||||
foreach ($results as $result) {
|
||||
// sub folders
|
||||
if (is_array($result['CommonPrefixes'])) {
|
||||
foreach ($result['CommonPrefixes'] as $prefix) {
|
||||
$dir = $this->getDirectoryMetaData($prefix['Prefix']);
|
||||
if ($dir) {
|
||||
yield $dir;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (is_array($result['Contents'])) {
|
||||
foreach ($result['Contents'] as $object) {
|
||||
$this->objectCache[$object['Key']] = $object;
|
||||
if ($object['Key'] !== $path) {
|
||||
yield $this->objectToMetaData($object);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private function objectToMetaData(array $object): array {
|
||||
return [
|
||||
'name' => basename($object['Key']),
|
||||
'mimetype' => $this->mimeDetector->detectPath($object['Key']),
|
||||
'mtime' => strtotime($object['LastModified']),
|
||||
'storage_mtime' => strtotime($object['LastModified']),
|
||||
'etag' => $object['ETag'],
|
||||
'permissions' => Constants::PERMISSION_ALL - Constants::PERMISSION_CREATE,
|
||||
'size' => (int)($object['Size'] ?? $object['ContentLength']),
|
||||
];
|
||||
}
|
||||
|
||||
private function getDirectoryMetaData(string $path): ?array {
|
||||
$path = trim($path, '/');
|
||||
// when versioning is enabled, delete markers are returned as part of CommonPrefixes
|
||||
// resulting in "ghost" folders, verify that each folder actually exists
|
||||
if ($this->versioningEnabled() && !$this->doesDirectoryExist($path)) {
|
||||
return null;
|
||||
}
|
||||
$cacheEntry = $this->getCache()->get($path);
|
||||
if ($cacheEntry instanceof CacheEntry) {
|
||||
return $cacheEntry->getData();
|
||||
} else {
|
||||
return [
|
||||
'name' => basename($path),
|
||||
'mimetype' => 'httpd/unix-directory',
|
||||
'mtime' => time(),
|
||||
'storage_mtime' => time(),
|
||||
'etag' => uniqid(),
|
||||
'permissions' => Constants::PERMISSION_ALL,
|
||||
'size' => -1,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
public function versioningEnabled(): bool {
|
||||
if ($this->versioningEnabled === null) {
|
||||
$result = $this->getConnection()->getBucketVersioning(['Bucket' => $this->getBucket()]);
|
||||
$this->versioningEnabled = $result->get('Status') === 'Enabled';
|
||||
}
|
||||
return $this->versioningEnabled;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ trait S3ObjectTrait {
|
|||
}
|
||||
$opts = [
|
||||
'http' => [
|
||||
'protocol_version' => 1.1,
|
||||
'protocol_version' => $request->getProtocolVersion(),
|
||||
'header' => $headers,
|
||||
],
|
||||
];
|
||||
|
|
|
|||
Loading…
Reference in a new issue