diff --git a/astrails-safe.gemspec b/astrails-safe.gemspec index d3b1cb0..5fcf60f 100644 --- a/astrails-safe.gemspec +++ b/astrails-safe.gemspec @@ -89,18 +89,18 @@ Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain S s.specification_version = 3 if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then - s.add_runtime_dependency(%q, [">= 0"]) + s.add_runtime_dependency(%q, [">= 0"]) s.add_runtime_dependency(%q, [">= 0"]) s.add_runtime_dependency(%q, [">= 0"]) s.add_development_dependency(%q, [">= 0"]) else - s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) end else - s.add_dependency(%q, [">= 0"]) + s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) s.add_dependency(%q, [">= 0"]) diff --git a/lib/astrails/safe.rb b/lib/astrails/safe.rb index dd16376..af7cfd2 100644 --- a/lib/astrails/safe.rb +++ b/lib/astrails/safe.rb @@ -1,4 +1,4 @@ -require "aws/s3" +require "aws-sdk" require "cloudfiles" require 'net/sftp' require 'fileutils' diff --git a/lib/astrails/safe/s3.rb b/lib/astrails/safe/s3.rb index 9092e25..6b07208 100644 --- a/lib/astrails/safe/s3.rb +++ b/lib/astrails/safe/s3.rb @@ -17,9 +17,6 @@ def save # FIXME: user friendly error here :) raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path - # needed in cleanup even on dry run - AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless $LOCAL - puts "Uploading #{bucket}:#{full_path}" if $_VERBOSE || $DRY_RUN unless $DRY_RUN || $LOCAL if File.stat(@backup.path).size > MAX_S3_FILE_SIZE @@ -27,9 +24,8 @@ def save return end benchmark = Benchmark.realtime do - AWS::S3::Bucket.create(bucket) File.open(@backup.path) do |file| - AWS::S3::S3Object.store(full_path, file, bucket) + remote_bucket.objects.create(full_path, :data => file) end end puts "...done" if $_VERBOSE @@ -43,17 +39,23 @@ def cleanup return unless keep = @config[:keep, :s3] puts "listing files: #{bucket}:#{base}*" if $_VERBOSE - files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2) + files = remote_bucket.objects.with_prefix(:prefix => base) puts files.collect {|x| x.key} if $_VERBOSE - files = files. - collect {|x| x.key}. - sort + files = files.sort { |x,y| x.key <=> y.key } cleanup_with_limit(files, keep) do |f| puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE - AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN || $LOCAL + f.delete unless $DRY_RUN || $LOCAL + end + end + + def remote_bucket + unless @remote_bucket + s3 = AWS::S3.new(:access_key_id => key, :secret_access_key => secret) + @remote_bucket = s3.buckets.create(bucket) end + @remote_bucket end def bucket diff --git a/spec/unit/s3_spec.rb b/spec/unit/s3_spec.rb index 2deb0ad..060a487 100644 --- a/spec/unit/s3_spec.rb +++ b/spec/unit/s3_spec.rb @@ -5,7 +5,7 @@ def def_config { :s3 => { - :bucket => "_bucket", + :bucket => "bucket_name", :key => "_key", :secret => "_secret", }, @@ -37,10 +37,12 @@ def s3(config = def_config, backup = def_backup) before(:each) do @s3 = s3 - @files = [4,1,3,2].map { |i| stub(o = {}).key {"aaaaa#{i}"}; o } + @files = [4,1,3,2].map do |i| + stub(object = Object.new).key { "aaaaa#{i}" } + object + end - stub(AWS::S3::Bucket).objects("_bucket", :prefix => "_kind/_id/_kind-_id.", :max_keys => 4) {@files} - stub(AWS::S3::Bucket).find("_bucket").stub![anything].stub!.delete + stub(@s3).remote_bucket.stub!.objects.stub!.with_prefix(:prefix => '_kind/_id/_kind-_id.') { @files } end it "should check [:keep, :s3]" do @@ -50,8 +52,10 @@ def s3(config = def_config, backup = def_backup) end it "should delete extra files" do - mock(AWS::S3::Bucket).find("_bucket").mock!["aaaaa1"].mock!.delete - mock(AWS::S3::Bucket).find("_bucket").mock!["aaaaa2"].mock!.delete + dont_allow(@files[0]).delete + mock(@files[1]).delete + dont_allow(@files[2]).delete + mock(@files[3]).delete @s3.send :cleanup end @@ -107,16 +111,14 @@ def s3(config = def_config, backup = def_backup) def add_stubs(*stubs) stubs.each do |s| case s - when :connection - stub(AWS::S3::Base).establish_connection!(:access_key_id => "_key", :secret_access_key => "_secret", :use_ssl => true) when :stat stub(File).stat("foo").stub!.size {123} when :create_bucket - stub(AWS::S3::Bucket).create + stub.instance_of(AWS::S3).buckets.stub!.create.stub! when :file_open stub(File).open("foo") {|f, block| block.call(:opened_file)} when :s3_store - stub(AWS::S3::S3Object).store(@full_path, :opened_file, "_bucket") + stub.instance_of(AWS::S3).buckets.stub!.create.stub!.objects.stub!.create(@full_path, :data => :opened_file) end end end @@ -132,25 +134,26 @@ def add_stubs(*stubs) end it "should establish s3 connection" do - mock(AWS::S3::Base).establish_connection!(:access_key_id => "_key", :secret_access_key => "_secret", :use_ssl => true) + connection = AWS::S3.new(:access_key_id => "_key", :secret_access_key => "_secret") + mock(AWS::S3).new(:access_key_id => "_key", :secret_access_key => "_secret") { connection } add_stubs(:stat, :create_bucket, :file_open, :s3_store) + @s3.send(:save) end it "should open local file" do - add_stubs(:connection, :stat, :create_bucket) + add_stubs(:stat, :create_bucket) mock(File).open("foo") @s3.send(:save) end it "should upload file" do - add_stubs(:connection, :stat, :create_bucket, :file_open) - mock(AWS::S3::S3Object).store(@full_path, :opened_file, "_bucket") + add_stubs(:stat, :create_bucket, :file_open) + mock(@s3).remote_bucket.mock!.objects.mock!.create(@full_path, :data => :opened_file) @s3.send(:save) end it "should fail on files bigger then 5G" do - add_stubs(:connection) mock(File).stat("foo").stub!.size {5*1024*1024*1024+1} mock(STDERR).puts(anything) dont_allow(Benchmark).realtime