kotovalexarian-likes-github/fog--fog
1
0
Fork 0
mirror of https://github.com/fog/fog.git synced 2022-11-09 13:51:43 -05:00
Code Releases Activity

[Rackspace|Storage] updating large file upload based on @burns suggestions

Browse source
This commit is contained in:
Kyle Rames 2013-06-03 16:02:30 -05:00
parent 934726f126
commit 05d6390905
2 changed files with 33 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
lib/fog/rackspace/docs/storage.md
View file

@ -375,20 +375,21 @@ Cloud Files requires files larger than 5 GB to be uploaded into segments along w
SEGMENT_LIMIT = 5368709119.0 # 5GB -1
BUFFER_SIZE = 1024 * 1024 # 1MB
File.open("large_file") do |f|
num_segments = (f.size / SEGMENT_LIMIT).round + 1
1.upto(num_segments) do |segment|
File.open(file_name) do |f|
segment = 0
until file.eof?
segment += 1
offset = 0
read = 0
# upload segment to cloud files
service.put_object("my_container", "large_file/#{segment}", nil, options = {}) do
if (offset < SEGMENT_LIMIT) && !f.eof?
buf = f.read(BUFFER_SIZE)
read = buf.size
offset += read
segment_suffix = segment.to_s.rjust(10, '0')
service.put_object("my_container", "large_file/#{segment_suffix}", nil) do
if offset <= SEGMENT_LIMIT - BUFFER_SIZE
buf = file.read(BUFFER_SIZE).to_s
offset += buf.size
buf
else
""
''
end
end
end

40
lib/fog/rackspace/examples/storage/upload_large_files.rb
View file

@ -64,26 +64,30 @@ File.open(file_name) do |f|
num_segments = (f.size / SEGMENT_LIMIT).round + 1
puts "\nThis upload of '#{file_name}' will require #{num_segments} segment(s) and 1 manifest file\n"
1.upto(num_segments) do |segment|
print "\n\tUploading segment #{segment} "
offset = 0
read = 0
service.put_object(directory.key, "#{segment_name}/#{segment}", nil, options = {}) do
if (offset < SEGMENT_LIMIT) && !f.eof?
print "."
buf = f.read(BUFFER_SIZE)
read = buf.size
offset += read
buf
else
""
end
end
end
end
segment = 0
until f.eof?
segment += 1
offset = 0
# upload segment to cloud files
segment_suffix = segment.to_s.rjust(10, '0')
print "\n\tUploading segment #{segment_suffix} "
service.put_object(directory.key, "#{segment_name}/#{segment_suffix}", nil) do
if offset <= SEGMENT_LIMIT - BUFFER_SIZE
print "."
buf = f.read(BUFFER_SIZE).to_s
offset += buf.size
buf
else
''
end
end
end
end
puts "\n\n\tWriting manifest #{segment_name}\n\n"
service.put_object_manifest(directory.key, segment_name)
service.put_object_manifest(directory.key, segment_name, 'segments_prefix' => "#{segment_name}/")
puts <<-NOTE
You should now be able to download #{segment_name} from the cloud control panel or using the following code: