From 4a73f645e5a60d8adece72e6c231c2a1be13473d Mon Sep 17 00:00:00 2001 From: Roeland Jago Douma Date: Sat, 20 Jan 2018 11:41:04 +0100 Subject: [PATCH 1/7] Use zip32 if possible * OSX doesn't handle 64zip that well * Some other implentations don't handle it perfectly either * If the file is belog 4GiB (some overhead) => zip32 * This covers the 99% case I bet Signed-off-by: Roeland Jago Douma --- lib/private/Streamer.php | 23 +++++++++++++++++------ lib/private/legacy/files.php | 17 ++++++++++++++--- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/lib/private/Streamer.php b/lib/private/Streamer.php index 7b178fda652..3b033e265e7 100644 --- a/lib/private/Streamer.php +++ b/lib/private/Streamer.php @@ -24,6 +24,7 @@ namespace OC; +use OCP\IRequest; use ownCloud\TarStreamer\TarStreamer; use ZipStreamer\ZipStreamer; @@ -33,12 +34,22 @@ class Streamer { // streamer instance private $streamerInstance; - - public function __construct(){ - /** @var \OCP\IRequest */ - $request = \OC::$server->getRequest(); - - if ($request->isUserAgent($this->preferTarFor)) { + + /** + * Streamer constructor. + * + * @param IRequest $request + * @param int $size The size of the files in bytes + */ + public function __construct(IRequest $request, int $size){ + + /** + * If the size if below 4GB always use zip32 + * Use 4*1000*1000*1000 so we have a buffer for all the extra zip data + */ + if ($size < 4 * 1000 * 1000 * 1000) { + $this->streamerInstance = new ZipStreamer(['zip64' => false]); + } else if ($request->isUserAgent($this->preferTarFor)) { $this->streamerInstance = new TarStreamer(); } else { $this->streamerInstance = new ZipStreamer(['zip64' => PHP_INT_SIZE !== 4]); diff --git a/lib/private/legacy/files.php b/lib/private/legacy/files.php index def9f82fab9..b7c99b7fef8 100644 --- a/lib/private/legacy/files.php +++ b/lib/private/legacy/files.php @@ -144,17 +144,28 @@ class OC_Files { } } - $streamer = new Streamer(); - OC_Util::obEnd(); - self::lockFiles($view, $dir, $files); + /* Calculate filesize */ + if ($getType === self::ZIP_FILES) { + $fileSize = 0; + foreach ($files as $file) { + $fileSize += \OC\Files\Filesystem::getFileInfo($dir . '/' . $file)->getSize(); + } + } elseif ($getType === self::ZIP_DIR) { + $fileSize = \OC\Files\Filesystem::getFileInfo($dir . '/' . $files)->getSize(); + } + + $streamer = new Streamer(\OC::$server->getRequest(), $fileSize); + OC_Util::obEnd(); + $streamer->sendHeaders($name); $executionTime = (int)OC::$server->getIniWrapper()->getNumeric('max_execution_time'); if (strpos(@ini_get('disable_functions'), 'set_time_limit') === false) { @set_time_limit(0); } ignore_user_abort(true); + if ($getType === self::ZIP_FILES) { foreach ($files as $file) { $file = $dir . '/' . $file; From 90fdf83ca7648418f899e28490f65de53fcd31d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 6 Feb 2018 14:03:50 +0100 Subject: [PATCH 2/7] Use zip32 only if there are less than 65536 files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A zip32 file can contain, at most, 65535 files (and folders), so take that constraint into account. Signed-off-by: Daniel Calviño Sánchez --- lib/private/Streamer.php | 28 +++++++++++++++++++++++---- lib/private/legacy/files.php | 37 ++++++++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/lib/private/Streamer.php b/lib/private/Streamer.php index 3b033e265e7..51c2c923c23 100644 --- a/lib/private/Streamer.php +++ b/lib/private/Streamer.php @@ -40,14 +40,34 @@ class Streamer { * * @param IRequest $request * @param int $size The size of the files in bytes + * @param int $numberOfFiles The number of files (and directories) that will + * be included in the streamed file */ - public function __construct(IRequest $request, int $size){ + public function __construct(IRequest $request, int $size, int $numberOfFiles){ /** - * If the size if below 4GB always use zip32 - * Use 4*1000*1000*1000 so we have a buffer for all the extra zip data + * zip32 constraints for a basic (without compression, volumes nor + * encryption) zip file according to the Zip specification: + * - No file size is larger than 4 bytes (file size < 4294967296); see + * 4.4.9 uncompressed size + * - The size of all files plus their local headers is not larger than + * 4 bytes; see 4.4.16 relative offset of local header and 4.4.24 + * offset of start of central directory with respect to the starting + * disk number + * - The total number of entries (files and directories) in the zip file + * is not larger than 2 bytes (number of entries < 65536); see 4.4.22 + * total number of entries in the central dir + * - The size of the central directory is not larger than 4 bytes; see + * 4.4.23 size of the central directory + * + * Due to all that, zip32 is used if the size is below 4GB and there are + * less than 65536 files; the margin between 4*1000^3 and 4*1024^3 + * should give enough room for the extra zip metadata. Technically, it + * would still be possible to create an invalid zip32 file (for example, + * a zip file from files smaller than 4GB with a central directory + * larger than 4GiB), but it should not happen in the real world. */ - if ($size < 4 * 1000 * 1000 * 1000) { + if ($size < 4 * 1000 * 1000 * 1000 && $numberOfFiles < 65536) { $this->streamerInstance = new ZipStreamer(['zip64' => false]); } else if ($request->isUserAgent($this->preferTarFor)) { $this->streamerInstance = new TarStreamer(); diff --git a/lib/private/legacy/files.php b/lib/private/legacy/files.php index b7c99b7fef8..9281c1f7da4 100644 --- a/lib/private/legacy/files.php +++ b/lib/private/legacy/files.php @@ -146,17 +146,23 @@ class OC_Files { self::lockFiles($view, $dir, $files); - /* Calculate filesize */ + /* Calculate filesize and number of files */ if ($getType === self::ZIP_FILES) { + $fileInfos = array(); $fileSize = 0; foreach ($files as $file) { - $fileSize += \OC\Files\Filesystem::getFileInfo($dir . '/' . $file)->getSize(); + $fileInfo = \OC\Files\Filesystem::getFileInfo($dir . '/' . $file); + $fileSize += $fileInfo->getSize(); + $fileInfos[] = $fileInfo; } + $numberOfFiles = self::getNumberOfFiles($fileInfos); } elseif ($getType === self::ZIP_DIR) { - $fileSize = \OC\Files\Filesystem::getFileInfo($dir . '/' . $files)->getSize(); + $fileInfo = \OC\Files\Filesystem::getFileInfo($dir . '/' . $files); + $fileSize = $fileInfo->getSize(); + $numberOfFiles = self::getNumberOfFiles(array($fileInfo)); } - $streamer = new Streamer(\OC::$server->getRequest(), $fileSize); + $streamer = new Streamer(\OC::$server->getRequest(), $fileSize, $numberOfFiles); OC_Util::obEnd(); $streamer->sendHeaders($name); @@ -324,6 +330,29 @@ class OC_Files { } } + /** + * Returns the total (recursive) number of files and folders in the given + * FileInfos. + * + * @param \OCP\Files\FileInfo[] $fileInfos the FileInfos to count + * @return int the total number of files and folders + */ + private static function getNumberOfFiles($fileInfos) { + $numberOfFiles = 0; + + $view = new View(); + + while ($fileInfo = array_pop($fileInfos)) { + $numberOfFiles++; + + if ($fileInfo->getType() === \OCP\Files\FileInfo::TYPE_FOLDER) { + $fileInfos = array_merge($fileInfos, $view->getDirectoryContent($fileInfo->getPath())); + } + } + + return $numberOfFiles; + } + /** * @param View $view * @param string $dir From 5a7986c25d45678f6e7aa1aed707721e77c46f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 6 Feb 2018 17:55:15 +0100 Subject: [PATCH 3/7] Fix use of data directory in integration tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The data directory is not necessarily located at "../..". The proper directory is now got by running "php console.php config:system:get datadirectory". Signed-off-by: Daniel Calviño Sánchez --- .../features/bootstrap/BasicStructure.php | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/build/integration/features/bootstrap/BasicStructure.php b/build/integration/features/bootstrap/BasicStructure.php index 0aead766f2a..b3a45f66cea 100644 --- a/build/integration/features/bootstrap/BasicStructure.php +++ b/build/integration/features/bootstrap/BasicStructure.php @@ -356,8 +356,29 @@ trait BasicStructure { * @param string $text */ public function modifyTextOfFile($user, $filename, $text) { - self::removeFile("../../data/$user/files", "$filename"); - file_put_contents("../../data/$user/files" . "$filename", "$text"); + self::removeFile($this->getDataDirectory() . "/$user/files", "$filename"); + file_put_contents($this->getDataDirectory() . "/$user/files" . "$filename", "$text"); + } + + private function getDataDirectory() { + // Based on "runOcc" from CommandLine trait + $args = ['config:system:get', 'datadirectory']; + $args = array_map(function($arg) { + return escapeshellarg($arg); + }, $args); + $args[] = '--no-ansi --no-warnings'; + $args = implode(' ', $args); + + $descriptor = [ + 0 => ['pipe', 'r'], + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ]; + $process = proc_open('php console.php ' . $args, $descriptor, $pipes, $ocPath = '../..'); + $lastStdOut = stream_get_contents($pipes[1]); + proc_close($process); + + return trim($lastStdOut); } public function createFileSpecificSize($name, $size) { From 6ee5469a038c46d116a16f280c9de6b19526c939 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 6 Feb 2018 14:50:15 +0100 Subject: [PATCH 4/7] Add integration tests for downloading basic zip files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .drone.yml | 10 ++ .../features/bootstrap/BasicStructure.php | 13 ++ .../features/bootstrap/Download.php | 131 ++++++++++++++++ build/integration/features/download.feature | 144 ++++++++++++++++++ 4 files changed, 298 insertions(+) create mode 100644 build/integration/features/bootstrap/Download.php create mode 100644 build/integration/features/download.feature diff --git a/.drone.yml b/.drone.yml index c6f35114a25..b0f2664a01c 100644 --- a/.drone.yml +++ b/.drone.yml @@ -542,6 +542,15 @@ pipeline: when: matrix: TESTS: integration-remote-api + integration-download: + image: nextcloudci/integration-php7.0:integration-php7.0-6 + commands: + - ./occ maintenance:install --admin-pass=admin --data-dir=/dev/shm/nc_int + - cd build/integration + - ./run.sh features/download.feature + when: + matrix: + TESTS: integration-download acceptance-access-levels: image: nextcloudci/integration-php7.0:integration-php7.0-6 commands: @@ -738,6 +747,7 @@ matrix: - TESTS: integration-ldap-features - TESTS: integration-trashbin - TESTS: integration-remote-api + - TESTS: integration-download - TESTS: acceptance TESTS-ACCEPTANCE: access-levels - TESTS: acceptance diff --git a/build/integration/features/bootstrap/BasicStructure.php b/build/integration/features/bootstrap/BasicStructure.php index b3a45f66cea..9769037f190 100644 --- a/build/integration/features/bootstrap/BasicStructure.php +++ b/build/integration/features/bootstrap/BasicStructure.php @@ -39,6 +39,7 @@ require __DIR__ . '/../../vendor/autoload.php'; trait BasicStructure { use Auth; + use Download; use Trashbin; /** @var string */ @@ -381,6 +382,18 @@ trait BasicStructure { return trim($lastStdOut); } + /** + * @Given file :filename is created :times times in :user user data + * @param string $filename + * @param string $times + * @param string $user + */ + public function fileIsCreatedTimesInUserData($filename, $times, $user) { + for ($i = 0; $i < $times; $i++) { + file_put_contents($this->getDataDirectory() . "/$user/files" . "$filename-$i", "content-$i"); + } + } + public function createFileSpecificSize($name, $size) { $file = fopen("work/" . "$name", 'w'); fseek($file, $size - 1, SEEK_CUR); diff --git a/build/integration/features/bootstrap/Download.php b/build/integration/features/bootstrap/Download.php new file mode 100644 index 00000000000..88ab03d4568 --- /dev/null +++ b/build/integration/features/bootstrap/Download.php @@ -0,0 +1,131 @@ +. + * + */ + +use GuzzleHttp\Client; +use GuzzleHttp\Message\ResponseInterface; + +require __DIR__ . '/../../vendor/autoload.php'; + +trait Download { + + /** @var string **/ + private $downloadedFile; + + /** @AfterScenario **/ + public function cleanupDownloadedFile() { + $this->downloadedFile = null; + } + + /** + * @When user :user downloads zip file for entries :entries in folder :folder + */ + public function userDownloadsZipFileForEntriesInFolder($user, $entries, $folder) { + $this->asAn($user); + $this->sendingToDirectUrl('GET', "/index.php/apps/files/ajax/download.php?dir=" . $folder . "&files=[" . $entries . "]"); + $this->theHTTPStatusCodeShouldBe('200'); + + $this->getDownloadedFile(); + } + + private function getDownloadedFile() { + $this->downloadedFile = ''; + + $body = $this->response->getBody(); + while (!$body->eof()) { + $this->downloadedFile .= $body->read(8192); + } + $body->close(); + } + + /** + * @Then the downloaded zip file is a zip32 file + */ + public function theDownloadedZipFileIsAZip32File() { + // assertNotContains is not used to prevent the whole file from being + // printed in case of error. + PHPUnit_Framework_Assert::assertTrue( + strpos($this->downloadedFile, "\x50\x4B\x06\x06") === false, + "File contains the zip64 end of central dir signature" + ); + } + + /** + * @Then the downloaded zip file contains a file named :fileName with the contents of :sourceFileName from :user data + */ + public function theDownloadedZipFileContainsAFileNamedWithTheContentsOfFromData($fileName, $sourceFileName, $user) { + $fileHeaderRegExp = '/'; + $fileHeaderRegExp .= "\x50\x4B\x03\x04"; // Local file header signature + $fileHeaderRegExp .= '.{22,22}'; // Ignore from "version needed to extract" to "uncompressed size" + $fileHeaderRegExp .= preg_quote(pack('v', strlen($fileName)), '/'); // File name length + $fileHeaderRegExp .= '(.{2,2})'; // Get "extra field length" + $fileHeaderRegExp .= preg_quote($fileName, '/'); // File name + $fileHeaderRegExp .= '/s'; // PCRE_DOTALL, so all characters (including bytes that happen to be new line characters) match + + // assertRegExp is not used to prevent the whole file from being printed + // in case of error and to be able to get the extra field length. + PHPUnit_Framework_Assert::assertEquals( + 1, preg_match($fileHeaderRegExp, $this->downloadedFile, $matches), + "Local header for file did not appear once in zip file" + ); + + $extraFieldLength = unpack('vextraFieldLength', $matches[1])['extraFieldLength']; + $expectedFileContents = file_get_contents($this->getDataDirectory() . "/$user/files" . $sourceFileName); + + $fileHeaderAndContentRegExp = '/'; + $fileHeaderAndContentRegExp .= "\x50\x4B\x03\x04"; // Local file header signature + $fileHeaderAndContentRegExp .= '.{22,22}'; // Ignore from "version needed to extract" to "uncompressed size" + $fileHeaderAndContentRegExp .= preg_quote(pack('v', strlen($fileName)), '/'); // File name length + $fileHeaderAndContentRegExp .= '.{2,2}'; // Ignore "extra field length" + $fileHeaderAndContentRegExp .= preg_quote($fileName, '/'); // File name + $fileHeaderAndContentRegExp .= '.{' . $extraFieldLength . ',' . $extraFieldLength . '}'; // Ignore "extra field" + $fileHeaderAndContentRegExp .= preg_quote($expectedFileContents, '/'); // File contents + $fileHeaderAndContentRegExp .= '/s'; // PCRE_DOTALL, so all characters (including bytes that happen to be new line characters) match + + // assertRegExp is not used to prevent the whole file from being printed + // in case of error. + PHPUnit_Framework_Assert::assertEquals( + 1, preg_match($fileHeaderAndContentRegExp, $this->downloadedFile), + "Local header and contents for file did not appear once in zip file" + ); + } + + /** + * @Then the downloaded zip file contains a folder named :folderName + */ + public function theDownloadedZipFileContainsAFolderNamed($folderName) { + $folderHeaderRegExp = '/'; + $folderHeaderRegExp .= "\x50\x4B\x03\x04"; // Local file header signature + $folderHeaderRegExp .= '.{22,22}'; // Ignore from "version needed to extract" to "uncompressed size" + $folderHeaderRegExp .= preg_quote(pack('v', strlen($folderName)), '/'); // File name length + $folderHeaderRegExp .= '.{2,2}'; // Ignore "extra field length" + $folderHeaderRegExp .= preg_quote($folderName, '/'); // File name + $folderHeaderRegExp .= '/s'; // PCRE_DOTALL, so all characters (including bytes that happen to be new line characters) match + + // assertRegExp is not used to prevent the whole file from being printed + // in case of error. + PHPUnit_Framework_Assert::assertEquals( + 1, preg_match($folderHeaderRegExp, $this->downloadedFile), + "Local header for folder did not appear once in zip file" + ); + } +} diff --git a/build/integration/features/download.feature b/build/integration/features/download.feature new file mode 100644 index 00000000000..635dda36f30 --- /dev/null +++ b/build/integration/features/download.feature @@ -0,0 +1,144 @@ +Feature: download + + Scenario: downloading 2 small files returns a zip32 + Given using new dav path + And user "user0" exists + And User "user0" copies file "/welcome.txt" to "/welcome2.txt" + When user "user0" downloads zip file for entries '"welcome.txt","welcome2.txt"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/welcome.txt" from "user0" data + And the downloaded zip file contains a file named "welcome2.txt" with the contents of "/welcome2.txt" from "user0" data + + Scenario: downloading a small file and a directory returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/emptySubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","emptySubFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "emptySubFolder/" + + Scenario: downloading a small file and 2 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/subFolder" + And user "user0" created a folder "/subFolder/emptySubSubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","subFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "subFolder/" + And the downloaded zip file contains a folder named "subFolder/emptySubSubFolder/" + + Scenario: downloading dir with 2 small files returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/sparseFolder/welcome.txt" + And User "user0" copies file "/welcome.txt" to "/sparseFolder/welcome2.txt" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a file named "sparseFolder/welcome2.txt" with the contents of "/sparseFolder/welcome2.txt" from "user0" data + + Scenario: downloading dir with a small file and a directory returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/sparseFolder/welcome.txt" + And user "user0" created a folder "/sparseFolder/emptySubFolder" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "sparseFolder/emptySubFolder/" + + Scenario: downloading dir with a small file and 2 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/sparseFolder/welcome.txt" + And user "user0" created a folder "/sparseFolder/subFolder" + And user "user0" created a folder "/sparseFolder/subFolder/emptySubSubFolder" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "sparseFolder/subFolder/" + And the downloaded zip file contains a folder named "sparseFolder/subFolder/emptySubSubFolder/" + + Scenario: downloading (from folder) 2 small files returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/welcome.txt" + And User "user0" copies file "/welcome.txt" to "/baseFolder/welcome2.txt" + When user "user0" downloads zip file for entries '"welcome.txt","welcome2.txt"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/baseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a file named "welcome2.txt" with the contents of "/baseFolder/welcome2.txt" from "user0" data + + Scenario: downloading (from folder) a small file and a directory returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/welcome.txt" + And user "user0" created a folder "/baseFolder/emptySubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","emptySubFolder"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/baseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "emptySubFolder/" + + Scenario: downloading (from folder) a small file and 2 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/welcome.txt" + And user "user0" created a folder "/baseFolder/subFolder" + And user "user0" created a folder "/baseFolder/subFolder/emptySubSubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","subFolder"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/baseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "subFolder/" + And the downloaded zip file contains a folder named "subFolder/emptySubSubFolder/" + + Scenario: downloading (from folder) dir with 2 small files returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And user "user0" created a folder "/baseFolder/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/sparseFolder/welcome.txt" + And User "user0" copies file "/welcome.txt" to "/baseFolder/sparseFolder/welcome2.txt" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/baseFolder/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a file named "sparseFolder/welcome2.txt" with the contents of "/baseFolder/sparseFolder/welcome2.txt" from "user0" data + + Scenario: downloading (from folder) dir with a small file and a directory returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And user "user0" created a folder "/baseFolder/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/sparseFolder/welcome.txt" + And user "user0" created a folder "/baseFolder/sparseFolder/emptySubFolder" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/baseFolder/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "sparseFolder/emptySubFolder/" + + Scenario: downloading (from folder) dir with a small file and 2 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And user "user0" created a folder "/baseFolder/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/sparseFolder/welcome.txt" + And user "user0" created a folder "/baseFolder/sparseFolder/subFolder" + And user "user0" created a folder "/baseFolder/sparseFolder/subFolder/emptySubSubFolder" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/baseFolder/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "sparseFolder/subFolder/" + And the downloaded zip file contains a folder named "sparseFolder/subFolder/emptySubSubFolder/" From 65bc12960f77c172df271646c9c4458c1bbc5aaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 6 Feb 2018 14:51:43 +0100 Subject: [PATCH 5/7] Add integration tests for zip32/zip64 boundaries of number of files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Calviño Sánchez --- .../features/bootstrap/Download.php | 12 ++ build/integration/features/download.feature | 150 ++++++++++++++++++ 2 files changed, 162 insertions(+) diff --git a/build/integration/features/bootstrap/Download.php b/build/integration/features/bootstrap/Download.php index 88ab03d4568..90e2bdf67ac 100644 --- a/build/integration/features/bootstrap/Download.php +++ b/build/integration/features/bootstrap/Download.php @@ -69,6 +69,18 @@ trait Download { ); } + /** + * @Then the downloaded zip file is a zip64 file + */ + public function theDownloadedZipFileIsAZip64File() { + // assertNotContains is not used to prevent the whole file from being + // printed in case of error. + PHPUnit_Framework_Assert::assertTrue( + strpos($this->downloadedFile, "\x50\x4B\x06\x06") !== false, + "File does not contain the zip64 end of central dir signature" + ); + } + /** * @Then the downloaded zip file contains a file named :fileName with the contents of :sourceFileName from :user data */ diff --git a/build/integration/features/download.feature b/build/integration/features/download.feature index 635dda36f30..16d346b0150 100644 --- a/build/integration/features/download.feature +++ b/build/integration/features/download.feature @@ -142,3 +142,153 @@ Feature: download And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/baseFolder/sparseFolder/welcome.txt" from "user0" data And the downloaded zip file contains a folder named "sparseFolder/subFolder/" And the downloaded zip file contains a folder named "sparseFolder/subFolder/emptySubSubFolder/" + + @large + Scenario: downloading small file and dir with 65524 small files and 9 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/crowdedFolder" + And user "user0" created a folder "/crowdedFolder/subFolder1" + And file "/crowdedFolder/subFolder1/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder1" + And user "user0" created a folder "/crowdedFolder/subFolder2" + And file "/crowdedFolder/subFolder2/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder2" + And user "user0" created a folder "/crowdedFolder/subFolder3" + And file "/crowdedFolder/subFolder3/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder3" + And user "user0" created a folder "/crowdedFolder/subFolder4" + And file "/crowdedFolder/subFolder4/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder4" + And user "user0" created a folder "/crowdedFolder/subFolder5" + And file "/crowdedFolder/subFolder5/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder5" + And user "user0" created a folder "/crowdedFolder/subFolder6" + And file "/crowdedFolder/subFolder6/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder6" + And user "user0" created a folder "/crowdedFolder/subFolder7" + And file "/crowdedFolder/subFolder7/test.txt" is created "5524" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder7" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","crowdedFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder1/" + And the downloaded zip file contains a file named "crowdedFolder/subFolder1/test.txt-0" with the contents of "/crowdedFolder/subFolder1/test.txt-0" from "user0" data + And the downloaded zip file contains a file named "crowdedFolder/subFolder7/test.txt-5523" with the contents of "/crowdedFolder/subFolder7/test.txt-5523" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder/" + + @large + Scenario: downloading dir with 65525 small files and 9 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/crowdedFolder" + And user "user0" created a folder "/crowdedFolder/subFolder1" + And file "/crowdedFolder/subFolder1/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder1" + And user "user0" created a folder "/crowdedFolder/subFolder2" + And file "/crowdedFolder/subFolder2/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder2" + And user "user0" created a folder "/crowdedFolder/subFolder3" + And file "/crowdedFolder/subFolder3/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder3" + And user "user0" created a folder "/crowdedFolder/subFolder4" + And file "/crowdedFolder/subFolder4/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder4" + And user "user0" created a folder "/crowdedFolder/subFolder5" + And file "/crowdedFolder/subFolder5/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder5" + And user "user0" created a folder "/crowdedFolder/subFolder6" + And file "/crowdedFolder/subFolder6/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder6" + And user "user0" created a folder "/crowdedFolder/subFolder7" + And file "/crowdedFolder/subFolder7/test.txt" is created "5525" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder7" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder" + When user "user0" downloads zip file for entries '"crowdedFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "crowdedFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder1/" + And the downloaded zip file contains a file named "crowdedFolder/subFolder1/test.txt-0" with the contents of "/crowdedFolder/subFolder1/test.txt-0" from "user0" data + And the downloaded zip file contains a file named "crowdedFolder/subFolder7/test.txt-5524" with the contents of "/crowdedFolder/subFolder7/test.txt-5524" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder/" + + @large + Scenario: downloading small file and dir with 65524 small files and 10 nested directories returns a zip64 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/crowdedFolder" + And user "user0" created a folder "/crowdedFolder/subFolder1" + And file "/crowdedFolder/subFolder1/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder1" + And user "user0" created a folder "/crowdedFolder/subFolder2" + And file "/crowdedFolder/subFolder2/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder2" + And user "user0" created a folder "/crowdedFolder/subFolder3" + And file "/crowdedFolder/subFolder3/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder3" + And user "user0" created a folder "/crowdedFolder/subFolder4" + And file "/crowdedFolder/subFolder4/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder4" + And user "user0" created a folder "/crowdedFolder/subFolder5" + And file "/crowdedFolder/subFolder5/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder5" + And user "user0" created a folder "/crowdedFolder/subFolder6" + And file "/crowdedFolder/subFolder6/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder6" + And user "user0" created a folder "/crowdedFolder/subFolder7" + And file "/crowdedFolder/subFolder7/test.txt" is created "5524" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder7" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/emptySubSubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","crowdedFolder"' in folder "/" + Then the downloaded zip file is a zip64 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder1/" + And the downloaded zip file contains a file named "crowdedFolder/subFolder1/test.txt-0" with the contents of "/crowdedFolder/subFolder1/test.txt-0" from "user0" data + And the downloaded zip file contains a file named "crowdedFolder/subFolder7/test.txt-5523" with the contents of "/crowdedFolder/subFolder7/test.txt-5523" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/emptySubSubFolder/" + + @large + Scenario: downloading dir with 65525 small files and 10 nested directories returns a zip64 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/crowdedFolder" + And user "user0" created a folder "/crowdedFolder/subFolder1" + And file "/crowdedFolder/subFolder1/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder1" + And user "user0" created a folder "/crowdedFolder/subFolder2" + And file "/crowdedFolder/subFolder2/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder2" + And user "user0" created a folder "/crowdedFolder/subFolder3" + And file "/crowdedFolder/subFolder3/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder3" + And user "user0" created a folder "/crowdedFolder/subFolder4" + And file "/crowdedFolder/subFolder4/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder4" + And user "user0" created a folder "/crowdedFolder/subFolder5" + And file "/crowdedFolder/subFolder5/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder5" + And user "user0" created a folder "/crowdedFolder/subFolder6" + And file "/crowdedFolder/subFolder6/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder6" + And user "user0" created a folder "/crowdedFolder/subFolder7" + And file "/crowdedFolder/subFolder7/test.txt" is created "5525" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder7" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/emptySubSubFolder" + When user "user0" downloads zip file for entries '"crowdedFolder"' in folder "/" + Then the downloaded zip file is a zip64 file + And the downloaded zip file contains a folder named "crowdedFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder1/" + And the downloaded zip file contains a file named "crowdedFolder/subFolder1/test.txt-0" with the contents of "/crowdedFolder/subFolder1/test.txt-0" from "user0" data + And the downloaded zip file contains a file named "crowdedFolder/subFolder7/test.txt-5524" with the contents of "/crowdedFolder/subFolder7/test.txt-5524" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/emptySubSubFolder/" From 1d7bf328f8e3bbcd082b808825d435f88b9e484e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 6 Feb 2018 22:08:46 +0100 Subject: [PATCH 6/7] Make possible to provide "--tags=XXX" parameter to Behat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "--tags=XXX" limits the features or scenarios to be run to those matching the tag filter expression. Signed-off-by: Daniel Calviño Sánchez --- build/integration/run.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/build/integration/run.sh b/build/integration/run.sh index 45c2bcdaf2b..b747bb52c6b 100755 --- a/build/integration/run.sh +++ b/build/integration/run.sh @@ -2,6 +2,12 @@ OC_PATH=../../ OCC=${OC_PATH}occ +TAGS="" +if [ "$1" = "--tags" ]; then + TAGS="--tags=$2" + + shift 2 +fi SCENARIO_TO_RUN=$1 HIDE_OC_LOGS=$2 @@ -52,7 +58,7 @@ if [ "$INSTALLED" == "true" ]; then fi -vendor/bin/behat --strict -f junit -f pretty $SCENARIO_TO_RUN +vendor/bin/behat --strict -f junit -f pretty $TAGS $SCENARIO_TO_RUN RESULT=$? kill $PHPPID From 50ee978aa84197685107062f48074823ef328355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Calvi=C3=B1o=20S=C3=A1nchez?= Date: Tue, 6 Feb 2018 22:12:51 +0100 Subject: [PATCH 7/7] Do not run download scenarios tagged as "large" in Drone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Large scenarios take too long to run, so they would be cancelled before they were finished. Therefore, now they are not even run. Signed-off-by: Daniel Calviño Sánchez --- .drone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.drone.yml b/.drone.yml index b0f2664a01c..17557834428 100644 --- a/.drone.yml +++ b/.drone.yml @@ -547,7 +547,7 @@ pipeline: commands: - ./occ maintenance:install --admin-pass=admin --data-dir=/dev/shm/nc_int - cd build/integration - - ./run.sh features/download.feature + - ./run.sh --tags ~@large features/download.feature when: matrix: TESTS: integration-download