kotovalexarian-likes-github/fog--fog
1
0
Fork 0
mirror of https://github.com/fog/fog.git synced 2022-11-09 13:51:43 -05:00
Code Releases Activity

Merge pull request #1852 from rackspace/example_fix

Browse source 
[rackspace|storage] fixing large upload example and documentation
This commit is contained in:
Kyle Rames 2013-06-05 07:00:38 -07:00
commit 31c921eb5b
2 changed files with 37 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
lib/fog/rackspace/docs/storage.md
View file

@ -375,27 +375,28 @@ Cloud Files requires files larger than 5 GB to be uploaded into segments along w
SEGMENT_LIMIT = 5368709119.0 # 5GB -1
BUFFER_SIZE = 1024 * 1024 # 1MB
File.open("large_file") do |f|
num_segments = (f.size / SEGMENT_LIMIT).round + 1
1.upto(num_segments) do |segment|
File.open(file_name) do |f|
segment = 0
until file.eof?
segment += 1
offset = 0
read = 0
# upload segment to cloud files
service.put_object("my_container", "large_file/#{segment}", nil, options = {}) do
if (offset < SEGMENT_LIMIT) && (read.zero? || read == BUFFER_SIZE)
buf = f.sysread(BUFFER_SIZE)
read = buf.size
offset += read
segment_suffix = segment.to_s.rjust(10, '0')
service.put_object("my_container", "large_file/#{segment_suffix}", nil) do
if offset <= SEGMENT_LIMIT - BUFFER_SIZE
buf = file.read(BUFFER_SIZE).to_s
offset += buf.size
buf
else
""
''
end
end
end
end
# write manifest file
service.put_object_manifest("my_container", "large_file")
service.put_object_manifest("my_container", "large_file", 'X-Object-Manifest' => "my_container/large_file/")
Segmented files are downloaded like ordinary files. See [Download Files](#download-files) section for more information.

46
lib/fog/rackspace/examples/storage/upload_large_files.rb
View file

@ -61,38 +61,42 @@ file_name = get_user_input "Enter full path of file to upload"
segment_name = File.basename(file_name)
File.open(file_name) do |f|
num_segments = (f.size / SEGMENT_LIMIT).round + 1
num_segments = (f.stat.size / SEGMENT_LIMIT).round + 1
puts "\nThis upload of '#{file_name}' will require #{num_segments} segment(s) and 1 manifest file\n"
1.upto(num_segments) do |segment|
print "\n\tUploading segment #{segment} "
offset = 0
read = 0
service.put_object(directory.key, "#{segment_name}/#{segment}", nil, options = {}) do
if (offset < SEGMENT_LIMIT) && (read.zero? || read == BUFFER_SIZE)
print "."
buf = f.sysread(BUFFER_SIZE)
read = buf.size
offset += read
buf
else
""
end
end
end
end
segment = 0
until f.eof?
segment += 1
offset = 0
# upload segment to cloud files
segment_suffix = segment.to_s.rjust(10, '0')
print "\n\tUploading segment #{segment_suffix} "
service.put_object(directory.key, "#{segment_name}/#{segment_suffix}", nil) do
if offset <= SEGMENT_LIMIT - BUFFER_SIZE
print "."
buf = f.read(BUFFER_SIZE).to_s
offset += buf.size
buf
else
''
end
end
end
end
puts "\n\n\tWriting manifest #{segment_name}\n\n"
service.put_object_manifest(directory.key, segment_name)
service.put_object_manifest(directory.key, segment_name, 'X-Object-Manifest' => "#{directory.key}/#{segment_name}/" )
puts <<-NOTE
You should now be able to download #{segment_name} from the cloud control panel or using the following code:
directory = service.directories.get('#{directory.key}')
File.open('downloaded_#{segment_name}', 'w') do | f |
directory.files.get(#{segment_name}) do | data, remaining, content_length |
directory.files.get('#{segment_name}') do | data, remaining, content_length |
print "."
f.syswrite data
f.write data
end
end