From 03d30de6d60d1b7aad9bf665ce7d38fc15c6f24c Mon Sep 17 00:00:00 2001 From: Brett Cave Date: Fri, 29 Jul 2016 09:05:06 +0200 Subject: [PATCH 1/4] Skip multipart if body size is less than chunk. Fixes https://github.com/fog/fog/issues/3899 --- lib/fog/aws/models/storage/file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fog/aws/models/storage/file.rb b/lib/fog/aws/models/storage/file.rb index 4dae684a9..c3adfd4d1 100644 --- a/lib/fog/aws/models/storage/file.rb +++ b/lib/fog/aws/models/storage/file.rb @@ -205,7 +205,7 @@ module Fog options['x-amz-storage-class'] = storage_class if storage_class options.merge!(encryption_headers) - if multipart_chunk_size && body.respond_to?(:read) + if multipart_chunk_size && get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read) data = multipart_save(options) merge_attributes(data.body) else From ca995af1007cba9f1c5c84af21b4da5e33a2f862 Mon Sep 17 00:00:00 2001 From: Brett Cave Date: Fri, 29 Jul 2016 09:15:34 +0200 Subject: [PATCH 2/4] Static method from storage class for https://github.com/fog/fog/issues/3899 --- lib/fog/aws/models/storage/file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fog/aws/models/storage/file.rb b/lib/fog/aws/models/storage/file.rb index c3adfd4d1..064f96bc6 100644 --- a/lib/fog/aws/models/storage/file.rb +++ b/lib/fog/aws/models/storage/file.rb @@ -205,7 +205,7 @@ module Fog options['x-amz-storage-class'] = storage_class if storage_class options.merge!(encryption_headers) - if multipart_chunk_size && get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read) + if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read) data = multipart_save(options) merge_attributes(data.body) else From 0f5ff953ba110b3265a440b66d7b008477797367 Mon Sep 17 00:00:00 2001 From: Brett Cave Date: Sat, 30 Jul 2016 07:33:47 +0200 Subject: [PATCH 3/4] Raise an error (on the setter) if chunk size is too small - fixes #283 --- lib/fog/aws/models/storage/file.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/fog/aws/models/storage/file.rb b/lib/fog/aws/models/storage/file.rb index 064f96bc6..04bedbf2c 100644 --- a/lib/fog/aws/models/storage/file.rb +++ b/lib/fog/aws/models/storage/file.rb @@ -27,7 +27,11 @@ module Fog # @note Chunk size to use for multipart uploads. # Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb - attr_accessor :multipart_chunk_size + attr_reader :multipart_chunk_size + def multipart_chunk_size=(mp_chunk_size) + raise ArgumentError.new("minimum multipart_chunk_size is 5242880") if mp_chunk_size < 5242880 + @multipart_chunk_size = mp_chunk_size + end def acl requires :directory, :key From 46a92c05b29a987b6cd199d9afeb663861def1b7 Mon Sep 17 00:00:00 2001 From: Brett Cave Date: Sat, 30 Jul 2016 07:53:46 +0200 Subject: [PATCH 4/4] (Automatically) set multipart chunk size if the file is too big for a single PUT operation and mp was not set. --- lib/fog/aws/models/storage/file.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/fog/aws/models/storage/file.rb b/lib/fog/aws/models/storage/file.rb index 04bedbf2c..6c5f9971b 100644 --- a/lib/fog/aws/models/storage/file.rb +++ b/lib/fog/aws/models/storage/file.rb @@ -209,6 +209,9 @@ module Fog options['x-amz-storage-class'] = storage_class if storage_class options.merge!(encryption_headers) + # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects. + multipart_chunk_size=5242880 if !multipart_chunk_size && Fog::Storage.get_body_size(body) > 5368709120 + if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read) data = multipart_save(options) merge_attributes(data.body)