From fd4da92245bed6a87d2a0d64dd1f3a3d37efcd4a Mon Sep 17 00:00:00 2001 From: Chris Roberts Date: Tue, 23 Mar 2021 15:11:32 -0700 Subject: [PATCH] Check file size prior to upload and automatically adjust options When uploading box file, check if the size is greater than 5GB. If the size is larger and the direct to storage option is enabled, disable the option due to current 5GB restriction on direct uploads. --- lib/vagrant/util.rb | 1 + lib/vagrant/util/numeric.rb | 20 ++++++++++ plugins/commands/cloud/locales/en.yml | 4 ++ plugins/commands/cloud/provider/upload.rb | 10 +++++ plugins/commands/cloud/publish.rb | 10 +++++ .../commands/cloud/provider/upload_test.rb | 25 ++++++++++++ .../plugins/commands/cloud/publish_test.rb | 39 ++++++++++++++++--- 7 files changed, 103 insertions(+), 6 deletions(-) diff --git a/lib/vagrant/util.rb b/lib/vagrant/util.rb index f8be2baf0..b7f297169 100644 --- a/lib/vagrant/util.rb +++ b/lib/vagrant/util.rb @@ -11,6 +11,7 @@ module Vagrant autoload :HashWithIndifferentAccess, 'vagrant/util/hash_with_indifferent_access' autoload :GuestInspection, 'vagrant/util/guest_inspection' autoload :LoggingFormatter, 'vagrant/util/logging_formatter' + autoload :Numeric, 'vagrant/util/numeric' autoload :Platform, 'vagrant/util/platform' autoload :Retryable, 'vagrant/util/retryable' autoload :SafeExec, 'vagrant/util/safe_exec' diff --git a/lib/vagrant/util/numeric.rb b/lib/vagrant/util/numeric.rb index 93346b81d..93c73e113 100644 --- a/lib/vagrant/util/numeric.rb +++ b/lib/vagrant/util/numeric.rb @@ -49,6 +49,26 @@ module Vagrant bytes end + # Convert bytes to a user friendly string representation + # + # @param [Numeric] bytes Number of bytes to represent + # @return [String] user friendly output + def bytes_to_string(bytes) + # We want to locate the size that will return the + # smallest whole value number + BYTES_CONVERSION_MAP.sort { |a, b| + b.last <=> a.last + }.each do |suffix, size| + val = bytes.to_f / size + next if val < 1 + val = sprintf("%.2f", val) + val.slice!(-1, 1) while val.end_with?("0") + val.slice!(-1, 1) if val.end_with?(".") + return "#{val}#{suffix}" + end + "#{bytes} byte#{"s" if bytes > 1}" + end + # Rounds actual value to two decimal places # # @param [Integer] bytes diff --git a/plugins/commands/cloud/locales/en.yml b/plugins/commands/cloud/locales/en.yml index 713282261..c819aaaa5 100644 --- a/plugins/commands/cloud/locales/en.yml +++ b/plugins/commands/cloud/locales/en.yml @@ -75,6 +75,10 @@ en: Updated provider %{provider} on %{org}/%{box_name} for version %{version} not_found: |- Failed to locate %{provider_name} provider for %{org}/%{box_name} on version %{version} + direct_disable: |- + Vagrant is automatically disabling direct upload to backend storage. + Uploads directly to backend storage are currently only supported for + files 5G in size or smaller. Box file to upload is: %{size} version: create_success: |- Created version %{version} on %{org}/%{box_name} for version %{version} diff --git a/plugins/commands/cloud/provider/upload.rb b/plugins/commands/cloud/provider/upload.rb index 41b8e0613..6cf053b64 100644 --- a/plugins/commands/cloud/provider/upload.rb +++ b/plugins/commands/cloud/provider/upload.rb @@ -58,6 +58,16 @@ module VagrantPlugins access_token: access_token ) + # Include size check on file and disable direct if over 5G + if options[:direct] + fsize = File.stat(file).size + if fsize > (5 * Vagrant::Util::Numeric::GIGABYTE) + box_size = Vagrant::Util::Numeric.bytes_to_string(fsize) + @env.ui.warn(I18n.t("cloud_command.provider.direct_disable", size: box_size)) + options[:direct] = false + end + end + with_provider(account: account, org: org, box: box, version: version, provider: provider) do |p| p.upload(direct: options[:direct]) do |upload_url| m = options[:direct] ? :put : :put diff --git a/plugins/commands/cloud/publish.rb b/plugins/commands/cloud/publish.rb index a136eafb8..93acbcef4 100644 --- a/plugins/commands/cloud/publish.rb +++ b/plugins/commands/cloud/publish.rb @@ -129,6 +129,16 @@ module VagrantPlugins def upload_box_file(provider, box_file, options={}) box_file = File.absolute_path(box_file) @env.ui.info(I18n.t("cloud_command.publish.upload_provider", file: box_file)) + # Include size check on file and disable direct if over 5G + if options[:direct_upload] + fsize = File.stat(box_file).size + if fsize > (5 * Vagrant::Util::Numeric::GIGABYTE) + box_size = Vagrant::Util::Numeric.bytes_to_string(fsize) + @env.ui.warn(I18n.t("cloud_command.provider.direct_disable", size: box_size)) + options[:direct_upload] = false + end + end + provider.upload(direct: options[:direct_upload]) do |upload_url| Vagrant::Util::Uploader.new(upload_url, box_file, ui: @env.ui, method: :put).upload! end diff --git a/test/unit/plugins/commands/cloud/provider/upload_test.rb b/test/unit/plugins/commands/cloud/provider/upload_test.rb index 66d18c89f..a25868c4c 100644 --- a/test/unit/plugins/commands/cloud/provider/upload_test.rb +++ b/test/unit/plugins/commands/cloud/provider/upload_test.rb @@ -15,6 +15,7 @@ describe VagrantPlugins::CloudCommand::ProviderCommand::Command::Upload do let(:version) { double("version", version: box_version, provdiers: [provider]) } let(:provider) { double("provider", name: box_version_provider) } let(:provider_file) { double("provider-file") } + let(:provider_file_size) { 1 } describe "#upload_provider" do let(:argv) { [] } @@ -42,6 +43,8 @@ describe VagrantPlugins::CloudCommand::ProviderCommand::Command::Upload do allow(uploader).to receive(:upload!) allow(Vagrant::UI::Prefixed).to receive(:new).with(ui, "cloud").and_return(ui) allow(Vagrant::Util::Uploader).to receive(:new).and_return(uploader) + allow(File).to receive(:stat).with(provider_file). + and_return(double("provider-stat", size: provider_file_size)) end subject { described_class.new(argv, env) } @@ -91,6 +94,28 @@ describe VagrantPlugins::CloudCommand::ProviderCommand::Command::Upload do end subject.upload_provider(org_name, box_name, box_version, box_version_provider, provider_file, access_token, options) end + + context "when file size is 5GB" do + let(:provider_file_size) { 5368709120 } + + it "should use direct upload" do + expect(provider).to receive(:upload) do |**args| + expect(args[:direct]).to be_truthy + end + subject.upload_provider(org_name, box_name, box_version, box_version_provider, provider_file, access_token, options) + end + end + + context "when file size is greater than 5GB" do + let(:provider_file_size) { 5368709121 } + + it "should disable direct upload" do + expect(provider).to receive(:upload) do |**args| + expect(args[:direct]).to be_falsey + end + subject.upload_provider(org_name, box_name, box_version, box_version_provider, provider_file, access_token, options) + end + end end end diff --git a/test/unit/plugins/commands/cloud/publish_test.rb b/test/unit/plugins/commands/cloud/publish_test.rb index 4c7b584ad..bc25feaf0 100644 --- a/test/unit/plugins/commands/cloud/publish_test.rb +++ b/test/unit/plugins/commands/cloud/publish_test.rb @@ -10,6 +10,7 @@ describe VagrantPlugins::CloudCommand::Command::Publish do let(:account) { double("account") } let(:organization) { double("organization") } let(:box) { double("box") } + let(:box_size) { 1 } let(:version) { double("version") } let(:provider) { double("provider") } let(:uploader) { double("uploader") } @@ -25,6 +26,8 @@ describe VagrantPlugins::CloudCommand::Command::Publish do allow(ui).to receive(:success) allow(ui).to receive(:error) allow(iso_env).to receive(:ui).and_return(ui) + allow(File).to receive(:stat).with(box). + and_return(double("box_stat", size: box_size)) allow(VagrantCloud::Account).to receive(:new). with(custom_server: anything, access_token: anything). and_return(account) @@ -60,10 +63,30 @@ describe VagrantPlugins::CloudCommand::Command::Publish do subject.upload_box_file(provider, box) end - it "should upload with PUT method when direct upload option set" do - expect(Vagrant::Util::Uploader).to receive(:new). - with(upload_url, anything, hash_including(method: :put)).and_return(uploader) - subject.upload_box_file(provider, box, direct_upload: true) + context "with direct upload option enabled" do + it "should upload with PUT method when direct upload option set" do + expect(Vagrant::Util::Uploader).to receive(:new). + with(upload_url, anything, hash_including(method: :put)).and_return(uploader) + subject.upload_box_file(provider, box, direct_upload: true) + end + + context "with box size of 5GB" do + let(:box_size) { 5368709120 } + + it "should upload using direct to storage option" do + expect(provider).to receive(:upload).with(direct: true) + subject.upload_box_file(provider, box, direct_upload: true) + end + end + + context "with box size greater than 5GB" do + let(:box_size) { 5368709121 } + + it "should disable direct to storage upload" do + expect(provider).to receive(:upload).with(direct: false) + subject.upload_box_file(provider, box, direct_upload: true) + end + end end end @@ -220,6 +243,8 @@ describe VagrantPlugins::CloudCommand::Command::Publish do let(:client) { double("client", token: "1234token1234") } let(:action_runner) { double("action_runner") } let(:box_path) { "path/to/the/virtualbox.box" } + let(:full_box_path) { "/full/#{box_path}" } + let(:box) { full_box_path } before do allow(iso_env).to receive(:action_runner). @@ -229,8 +254,10 @@ describe VagrantPlugins::CloudCommand::Command::Publish do allow(subject).to receive(:format_box_results) allow(iso_env.ui).to receive(:ask).and_return("y") - allow(File).to receive(:absolute_path).and_return("/full/#{box_path}") - allow(File).to receive(:file?).and_return(true) + allow(File).to receive(:absolute_path).with(box_path) + .and_return("/full/#{box_path}") + allow(File).to receive(:file?).with(box_path) + .and_return(true) end context "with no arguments" do