mirror of
https://github.com/nextcloud/server.git
synced 2026-02-20 00:12:30 -05:00
move to stream based IObjectStore interface, rearrange & reformat code
This commit is contained in:
Jörn Friedrich Dreyer
parent
f2fe00e972
commit
c112a1d323
3 changed files with 157 additions and 161 deletions
|
|
@ -24,21 +24,19 @@ use OCP\Files\ObjectStore\IObjectStore;
|
|||
|
||||
class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
||||
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
private static $tmpFiles = array();
|
||||
/**
|
||||
* @var \OCP\Files\ObjectStore\IObjectStore $objectStore
|
||||
*/
|
||||
protected $objectStore;
|
||||
|
||||
/**
|
||||
* @var \OC\User\User $user
|
||||
*/
|
||||
protected $user;
|
||||
|
||||
/**
|
||||
* @var array
|
||||
*/
|
||||
private static $tmpFiles = array();
|
||||
|
||||
public function __construct($params) {
|
||||
if (isset($params['objectstore']) && $params['objectstore'] instanceof IObjectStore) {
|
||||
$this->objectStore = $params['objectstore'];
|
||||
|
|
@ -46,31 +44,49 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
throw new \Exception('missing IObjectStore instance');
|
||||
}
|
||||
if (isset($params['storageid'])) {
|
||||
$this->id = 'object::store:'.$params['storageid'];
|
||||
$this->id = 'object::store:' . $params['storageid'];
|
||||
} else {
|
||||
$this->id = 'object::store:'.$this->objectStore->getStorageId();
|
||||
$this->id = 'object::store:' . $this->objectStore->getStorageId();
|
||||
}
|
||||
//initialize cache with root directory in cache
|
||||
if ( ! $this->is_dir('/') ) {
|
||||
if (!$this->is_dir('/')) {
|
||||
$this->mkdir('/');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Object Stores use a NoopScanner because metadata is directly stored in
|
||||
* the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere.
|
||||
* @param string $path
|
||||
* @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner
|
||||
* @return \OC\Files\ObjectStore\NoopScanner
|
||||
*/
|
||||
public function getScanner($path = '', $storage = null) {
|
||||
if (!$storage) {
|
||||
$storage = $this;
|
||||
public function mkdir($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
if ($this->is_dir($path)) {
|
||||
return false;
|
||||
}
|
||||
if (!isset($this->scanner)) {
|
||||
$this->scanner = new NoopScanner($storage);
|
||||
|
||||
$dirName = dirname($path);
|
||||
$parentExists = $this->is_dir($dirName);
|
||||
|
||||
$mTime = time();
|
||||
|
||||
$data = array(
|
||||
'mimetype' => 'httpd/unix-directory',
|
||||
'size' => 0,
|
||||
'mtime' => $mTime,
|
||||
'storage_mtime' => $mTime,
|
||||
'permissions' => \OCP\PERMISSION_ALL,
|
||||
);
|
||||
|
||||
if ($dirName === '.' && !$parentExists) {
|
||||
//create root on the fly
|
||||
$data['etag'] = $this->getETag($dirName);
|
||||
$this->getCache()->put('', $data);
|
||||
$parentExists = true;
|
||||
}
|
||||
return $this->scanner;
|
||||
|
||||
if ($parentExists) {
|
||||
$data['etag'] = $this->getETag($path);
|
||||
$this->getCache()->put($path, $data);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -89,61 +105,28 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
return $path;
|
||||
}
|
||||
|
||||
public function getId () {
|
||||
/**
|
||||
* Object Stores use a NoopScanner because metadata is directly stored in
|
||||
* the file cache and cannot really scan the filesystem. The storage passed in is not used anywhere.
|
||||
*
|
||||
* @param string $path
|
||||
* @param \OC\Files\Storage\Storage (optional) the storage to pass to the scanner
|
||||
* @return \OC\Files\ObjectStore\NoopScanner
|
||||
*/
|
||||
public function getScanner($path = '', $storage = null) {
|
||||
if (!$storage) {
|
||||
$storage = $this;
|
||||
}
|
||||
if (!isset($this->scanner)) {
|
||||
$this->scanner = new NoopScanner($storage);
|
||||
}
|
||||
return $this->scanner;
|
||||
}
|
||||
|
||||
public function getId() {
|
||||
return $this->id;
|
||||
}
|
||||
|
||||
public function mkdir($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
if($this->is_dir($path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$dirName = dirname($path);
|
||||
$parentExists = $this->is_dir($dirName);
|
||||
|
||||
$mTime = time();
|
||||
|
||||
$data = array(
|
||||
'mimetype' => 'httpd/unix-directory',
|
||||
'size' => 0,
|
||||
'mtime' => $mTime,
|
||||
'storage_mtime' => $mTime,
|
||||
'permissions' => \OCP\PERMISSION_ALL,
|
||||
);
|
||||
|
||||
if ($dirName === '.' && ! $parentExists ) {
|
||||
//create root on the fly
|
||||
$data['etag'] = $this->getETag($dirName);
|
||||
$this->getCache()->put('', $data);
|
||||
$parentExists = true;
|
||||
}
|
||||
|
||||
if ($parentExists) {
|
||||
$data['etag'] = $this->getETag($path);
|
||||
$this->getCache()->put($path, $data);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function file_exists($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
return (bool)$this->stat($path);
|
||||
}
|
||||
|
||||
private function rmObjects($path) {
|
||||
$children = $this->getCache()->getFolderContents($path);
|
||||
foreach ($children as $child) {
|
||||
if ($child['mimetype'] === 'httpd/unix-directory') {
|
||||
$this->rmObjects($child['path']);
|
||||
} else {
|
||||
$this->unlink($child['path']);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function rmdir($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
|
|
@ -158,9 +141,63 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
return true;
|
||||
}
|
||||
|
||||
private function rmObjects($path) {
|
||||
$children = $this->getCache()->getFolderContents($path);
|
||||
foreach ($children as $child) {
|
||||
if ($child['mimetype'] === 'httpd/unix-directory') {
|
||||
$this->rmObjects($child['path']);
|
||||
} else {
|
||||
$this->unlink($child['path']);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function unlink($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
$stat = $this->stat($path);
|
||||
|
||||
if ($stat && isset($stat['fileid'])) {
|
||||
if ($stat['mimetype'] === 'httpd/unix-directory') {
|
||||
return $this->rmdir($path);
|
||||
}
|
||||
try {
|
||||
$this->objectStore->deleteObject($this->getURN($stat['fileid']));
|
||||
} catch (\Exception $ex) {
|
||||
if ($ex->getCode() !== 404) {
|
||||
\OCP\Util::writeLog('objectstore', 'Could not delete object: ' . $ex->getMessage(), \OCP\Util::ERROR);
|
||||
return false;
|
||||
} else {
|
||||
//removing from cache is ok as it does not exist in the objectstore anyway
|
||||
}
|
||||
}
|
||||
$this->getCache()->remove($path);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function stat($path) {
|
||||
return $this->getCache()->get($path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Override this method if you need a different unique resource identifier for your object storage implementation.
|
||||
* The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users.
|
||||
* You may need a mapping table to store your URN if it cannot be generated from the fileid.
|
||||
*
|
||||
* @param int $fileId the fileid
|
||||
* @return null|string the unified resource name used to identify the object
|
||||
*/
|
||||
protected function getURN($fileId) {
|
||||
if (is_numeric($fileId)) {
|
||||
return 'urn:oid:' . $fileId;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public function opendir($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
|
||||
if ($path === '.') {
|
||||
$path = '';
|
||||
} else if ($path) {
|
||||
|
|
@ -173,7 +210,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
foreach ($folderContents as $file) {
|
||||
$files[] = $file['name'];
|
||||
}
|
||||
|
||||
|
||||
\OC\Files\Stream\Dir::register('objstore' . $path, $files);
|
||||
|
||||
return opendir('fakedir://objstore' . $path);
|
||||
|
|
@ -183,10 +220,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
}
|
||||
}
|
||||
|
||||
public function stat($path) {
|
||||
return $this->getCache()->get($path);
|
||||
}
|
||||
|
||||
public function filetype($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
$stat = $this->stat($path);
|
||||
|
|
@ -200,31 +233,6 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
public function unlink($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
$stat = $this->stat($path);
|
||||
|
||||
if ($stat && isset($stat['fileid'])) {
|
||||
if ($stat['mimetype'] === 'httpd/unix-directory') {
|
||||
return $this->rmdir($path);
|
||||
}
|
||||
try {
|
||||
$this->objectStore->deleteObject($this->getURN($stat['fileid']));
|
||||
} catch (\Exception $ex) {
|
||||
if ($ex->getCode() !== 404) {
|
||||
\OCP\Util::writeLog('objectstore', 'Could not delete object: '.$ex->getMessage(), \OCP\Util::ERROR);
|
||||
return false;
|
||||
} else {
|
||||
//removing from cache is ok as it does not exist in the objectstore anyway
|
||||
}
|
||||
}
|
||||
$this->getCache()->remove($path);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public function fopen($path, $mode) {
|
||||
$path = $this->normalizePath($path);
|
||||
|
||||
|
|
@ -233,15 +241,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
case 'rb':
|
||||
$stat = $this->stat($path);
|
||||
if (is_array($stat)) {
|
||||
$tmpFile = \OC_Helper::tmpFile();
|
||||
self::$tmpFiles[$tmpFile] = $path;
|
||||
try {
|
||||
$this->objectStore->getObject($this->getURN($stat['fileid']), $tmpFile);
|
||||
return $this->objectStore->readObject($this->getURN($stat['fileid']));
|
||||
} catch (\Exception $ex) {
|
||||
\OCP\Util::writeLog('objectstore', 'Could not get object: '.$ex->getMessage(), \OCP\Util::ERROR);
|
||||
\OCP\Util::writeLog('objectstore', 'Could not get object: ' . $ex->getMessage(), \OCP\Util::ERROR);
|
||||
return false;
|
||||
}
|
||||
return fopen($tmpFile, 'r');
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -275,6 +280,11 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
return false;
|
||||
}
|
||||
|
||||
public function file_exists($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
return (bool)$this->stat($path);
|
||||
}
|
||||
|
||||
public function rename($path1, $path2) {
|
||||
$path1 = $this->normalizePath($path1);
|
||||
$path2 = $this->normalizePath($path2);
|
||||
|
|
@ -297,12 +307,12 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
} else {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public function getMimeType($path) {
|
||||
$path = $this->normalizePath($path);
|
||||
$stat = $this->stat($path);
|
||||
|
|
@ -312,7 +322,7 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public function touch($path, $mtime = null) {
|
||||
if (is_null($mtime)) {
|
||||
$mtime = time();
|
||||
|
|
@ -343,30 +353,17 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
);
|
||||
$fileId = $this->getCache()->put($path, $stat);
|
||||
try {
|
||||
$this->objectStore->writeObject($this->getURN($fileId));
|
||||
//read an empty file from memory
|
||||
$this->objectStore->writeObject($this->getURN($fileId), fopen('php://memory', 'r'));
|
||||
} catch (\Exception $ex) {
|
||||
$this->getCache()->remove($path);
|
||||
\OCP\Util::writeLog('objectstore', 'Could not create object: '.$ex->getMessage(), \OCP\Util::ERROR);
|
||||
\OCP\Util::writeLog('objectstore', 'Could not create object: ' . $ex->getMessage(), \OCP\Util::ERROR);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Override this method if you need a different unique resource identifier for your object storage implementation.
|
||||
* The default implementations just appends the fileId to 'urn:oid:'. Make sure the URN is unique over all users.
|
||||
* You may need a mapping table to store your URN if it cannot be generated from the fileid.
|
||||
* @param int $fileId the fileid
|
||||
* @return null|string the unified resource name used to identify the object
|
||||
*/
|
||||
protected function getURN($fileId) {
|
||||
if (is_numeric($fileId)) {
|
||||
return 'urn:oid:'.$fileId;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public function writeBack($tmpFile) {
|
||||
if (!isset(self::$tmpFiles[$tmpFile])) {
|
||||
return false;
|
||||
|
|
@ -391,10 +388,10 @@ class ObjectStoreStorage extends \OC\Files\Storage\Common {
|
|||
$fileId = $this->getCache()->put($path, $stat);
|
||||
try {
|
||||
//upload to object storage
|
||||
$this->objectStore->writeObject($this->getURN($fileId), $tmpFile);
|
||||
$this->objectStore->writeObject($this->getURN($fileId), fopen($tmpFile, 'r'));
|
||||
} catch (\Exception $ex) {
|
||||
$this->getCache()->remove($path);
|
||||
\OCP\Util::writeLog('objectstore', 'Could not create object: '.$ex->getMessage(), \OCP\Util::ERROR);
|
||||
\OCP\Util::writeLog('objectstore', 'Could not create object: ' . $ex->getMessage(), \OCP\Util::ERROR);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -21,23 +21,24 @@
|
|||
namespace OC\Files\ObjectStore;
|
||||
|
||||
use Guzzle\Http\Exception\ClientErrorResponseException;
|
||||
use OCP\Files\ObjectStore\IObjectStore;
|
||||
use OpenCloud\OpenStack;
|
||||
|
||||
class Swift implements \OCP\Files\ObjectStore\IObjectStore {
|
||||
class Swift implements IObjectStore {
|
||||
|
||||
/**
|
||||
* @var \OpenCloud\ObjectStore\Service
|
||||
*/
|
||||
private $objectStoreService;
|
||||
|
||||
|
||||
/**
|
||||
* @var \OpenCloud\ObjectStore\Resource\Container
|
||||
*/
|
||||
private $container;
|
||||
|
||||
public function __construct($params) {
|
||||
if (!isset($params['username']) || !isset($params['password']) ) {
|
||||
throw new \Exception("Access Key and Secret have to be configured.");
|
||||
if (!isset($params['username']) || !isset($params['password'])) {
|
||||
throw new \Exception('Access Key and Secret have to be configured.');
|
||||
}
|
||||
if (!isset($params['container'])) {
|
||||
$params['container'] = 'owncloud';
|
||||
|
|
@ -80,40 +81,42 @@ class Swift implements \OCP\Files\ObjectStore\IObjectStore {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return string the container name where objects are stored
|
||||
*/
|
||||
public function getStorageId() {
|
||||
return $this->container->name;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $urn Unified Resource Name
|
||||
* @param string $tmpFile
|
||||
* @return void
|
||||
* @param string $urn the unified resource name used to identify the object
|
||||
* @param resource $stream stream with the data to write
|
||||
* @throws Exception from openstack lib when something goes wrong
|
||||
*/
|
||||
public function writeObject($urn, $tmpFile = null) {
|
||||
$fileData = '';
|
||||
if ($tmpFile) {
|
||||
$fileData = fopen($tmpFile, 'r');
|
||||
}
|
||||
|
||||
$this->container->uploadObject($urn, $fileData);
|
||||
public function writeObject($urn, $stream) {
|
||||
$this->container->uploadObject($urn, $stream);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param string $urn Unified Resource Name
|
||||
* @param string $tmpFile
|
||||
* @return void
|
||||
* @param string $urn the unified resource name used to identify the object
|
||||
* @return resource stream with the read data
|
||||
* @throws Exception from openstack lib when something goes wrong
|
||||
*/
|
||||
public function getObject($urn, $tmpFile) {
|
||||
public function readObject($urn) {
|
||||
$object = $this->container->getObject($urn);
|
||||
|
||||
/** @var $objectContent \Guzzle\Http\EntityBody **/
|
||||
// we need to keep a reference to objectContent or
|
||||
// the stream will be closed before we can do anything with it
|
||||
/** @var $objectContent \Guzzle\Http\EntityBody * */
|
||||
$objectContent = $object->getContent();
|
||||
|
||||
$objectContent->rewind();
|
||||
$stream = $objectContent->getStream();
|
||||
file_put_contents($tmpFile, $stream);
|
||||
|
||||
// directly returning the object stream does not work because the GC seems to collect it, so we need a copy
|
||||
$tmpStream = fopen('php://temp', 'r+');
|
||||
stream_copy_to_stream($objectContent->getStream(), $tmpStream);
|
||||
rewind($tmpStream);
|
||||
|
||||
return $tmpStream;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -11,21 +11,17 @@ interface IObjectStore {
|
|||
|
||||
/**
|
||||
* @param string $urn the unified resource name used to identify the object
|
||||
* @param string $tmpFile path to the local temporary file that should be
|
||||
* used to store the object
|
||||
* @return void
|
||||
* @return resource stream with the read data
|
||||
* @throws Exception when something goes wrong, message will be logged
|
||||
*/
|
||||
function getObject($urn, $tmpFile);
|
||||
function readObject($urn);
|
||||
|
||||
/**
|
||||
* @param string $urn the unified resource name used to identify the object
|
||||
* @param string $tmpFile path to the local temporary file that the object
|
||||
* should be loaded from
|
||||
* @return void
|
||||
* @param resource $stream stream with the data to write
|
||||
* @throws Exception when something goes wrong, message will be logged
|
||||
*/
|
||||
function writeObject($urn, $tmpFile = null);
|
||||
function writeObject($urn, $stream);
|
||||
|
||||
/**
|
||||
* @param string $urn the unified resource name used to identify the object
|
||||
|
|
|
|||
Loading…
Reference in a new issue