From 8d6f0ca8d733139b40adb7e59d10af83ea5f3e18 Mon Sep 17 00:00:00 2001 From: Adam Cecile Date: Tue, 25 May 2021 10:31:11 +0200 Subject: [PATCH 01/28] Implement AWS TagSpecifications (closes #603) --- lib/fog/aws/models/compute/server.rb | 44 ++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/lib/fog/aws/models/compute/server.rb b/lib/fog/aws/models/compute/server.rb index 13d3b66bd..08e9fa685 100644 --- a/lib/fog/aws/models/compute/server.rb +++ b/lib/fog/aws/models/compute/server.rb @@ -50,6 +50,7 @@ module Fog attribute :subnet_id, :aliases => 'subnetId' attribute :tenancy attribute :tags, :aliases => 'tagSet' + attribute :tag_specifications, :aliases => 'tagSpecifications' attribute :user_data attribute :virtualization_type, :aliases => 'virtualizationType' attribute :vpc_id, :aliases => 'vpcId' @@ -166,6 +167,7 @@ module Fog 'SecurityGroupId' => security_group_ids, 'SubnetId' => subnet_id, 'UserData' => user_data, + 'TagSpecifications' => tag_specifications, } options.delete_if {|key, value| value.nil?} @@ -196,6 +198,48 @@ module Fog else options.delete('SubnetId') end + if tag_specifications + # From https://docs.aws.amazon.com/sdk-for-ruby/v2/api/Aws/EC2/Client.html#run_instances-instance_method + # And https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html + # Discussed at https://github.com/fog/fog-aws/issues/603 + # + # Example + # + # tag_specifications: [ + # { + # resource_type: "instance", + # tags: [ + # { + # key: "Project", + # value: "MyProject", + # }, + # ], + # }, + # { + # resource_type: "volume", + # tags: [ + # { + # key: "Project", + # value: "MyProject", + # }, + # ], + # }, + # ] + options.delete('TagSpecifications') + tag_specifications.each_with_index do |val, idx| + resource_type = val["resource_type"] + tags = val["tags"] + options["TagSpecification.#{idx}.ResourceType"] = resource_type + tags.each_with_index do |tag, tag_idx| + aws_tag_key = "TagSpecification.#{idx}.Tag.#{tag_idx}.Key" + aws_tag_value = "TagSpecification.#{idx}.Tag.#{tag_idx}.Value" + options[aws_tag_key] = tag["key"] + options[aws_tag_value] = tag["value"] + end + end + else + options.delete('TagSpecifications') + end options end From 9b8695efb5072999d2b1f0ef78b9519ac3a362f0 Mon Sep 17 00:00:00 2001 From: Adam Cecile Date: Tue, 1 Jun 2021 09:47:27 +0200 Subject: [PATCH 02/28] Implement @geemus recommendations --- lib/fog/aws/models/compute/server.rb | 42 ------------------ lib/fog/aws/requests/compute/run_instances.rb | 44 +++++++++++++++++++ 2 files changed, 44 insertions(+), 42 deletions(-) diff --git a/lib/fog/aws/models/compute/server.rb b/lib/fog/aws/models/compute/server.rb index 08e9fa685..1062b2190 100644 --- a/lib/fog/aws/models/compute/server.rb +++ b/lib/fog/aws/models/compute/server.rb @@ -198,48 +198,6 @@ module Fog else options.delete('SubnetId') end - if tag_specifications - # From https://docs.aws.amazon.com/sdk-for-ruby/v2/api/Aws/EC2/Client.html#run_instances-instance_method - # And https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html - # Discussed at https://github.com/fog/fog-aws/issues/603 - # - # Example - # - # tag_specifications: [ - # { - # resource_type: "instance", - # tags: [ - # { - # key: "Project", - # value: "MyProject", - # }, - # ], - # }, - # { - # resource_type: "volume", - # tags: [ - # { - # key: "Project", - # value: "MyProject", - # }, - # ], - # }, - # ] - options.delete('TagSpecifications') - tag_specifications.each_with_index do |val, idx| - resource_type = val["resource_type"] - tags = val["tags"] - options["TagSpecification.#{idx}.ResourceType"] = resource_type - tags.each_with_index do |tag, tag_idx| - aws_tag_key = "TagSpecification.#{idx}.Tag.#{tag_idx}.Key" - aws_tag_value = "TagSpecification.#{idx}.Tag.#{tag_idx}.Value" - options[aws_tag_key] = tag["key"] - options[aws_tag_value] = tag["value"] - end - end - else - options.delete('TagSpecifications') - end options end diff --git a/lib/fog/aws/requests/compute/run_instances.rb b/lib/fog/aws/requests/compute/run_instances.rb index eaf438590..0461f723c 100644 --- a/lib/fog/aws/requests/compute/run_instances.rb +++ b/lib/fog/aws/requests/compute/run_instances.rb @@ -44,6 +44,11 @@ module Fog # * 'PrivateIpAddresses.Primary'<~Bool> - Indicates whether the private IP address is the primary private IP address. # * 'SecondaryPrivateIpAddressCount'<~Bool> - The number of private IP addresses to assign to the network interface. # * 'AssociatePublicIpAddress'<~String> - Indicates whether to assign a public IP address to an instance in a VPC. The public IP address is assigned to a specific network interface + # * 'TagSpecifications'<~Array>: array of hashes + # * 'ResourceType'<~String> - Type of resource to apply tags on, e.g: instance or volume + # * 'Tags'<~Array> - List of hashs reprensenting tag to be set + # * 'Key'<~String> - Tag name + # * 'Value'<~String> - Tag value # * 'ClientToken'<~String> - unique case-sensitive token for ensuring idempotency # * 'DisableApiTermination'<~Boolean> - specifies whether or not to allow termination of the instance from the api # * 'SecurityGroup'<~Array> or <~String> - Name of security group(s) for instances (not supported for VPC) @@ -144,6 +149,45 @@ module Fog end end end + if tag_specifications = options.delete('TagSpecifications') + # From https://docs.aws.amazon.com/sdk-for-ruby/v2/api/Aws/EC2/Client.html#run_instances-instance_method + # And https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html + # Discussed at https://github.com/fog/fog-aws/issues/603 + # + # Example + # + # TagSpecifications: [ + # { + # ResourceType: "instance", + # Tags: [ + # { + # Key: "Project", + # Value: "MyProject", + # }, + # ], + # }, + # { + # ResourceType: "volume", + # Tags: [ + # { + # Key: "Project", + # Value: "MyProject", + # }, + # ], + # }, + # ] + tag_specifications.each_with_index do |val, idx| + resource_type = val["ResourceType"] + tags = val["Tags"] + options["TagSpecification.#{idx}.ResourceType"] = resource_type + tags.each_with_index do |tag, tag_idx| + aws_tag_key = "TagSpecification.#{idx}.Tag.#{tag_idx}.Key" + aws_tag_value = "TagSpecification.#{idx}.Tag.#{tag_idx}.Value" + options[aws_tag_key] = tag["Key"] + options[aws_tag_value] = tag["Value"] + end + end + end idempotent = !(options['ClientToken'].nil? || options['ClientToken'].empty?) From f03bd6c98a490d9cc45b4443e1a998438501b84d Mon Sep 17 00:00:00 2001 From: geemus Date: Sun, 6 Jun 2021 07:11:10 -0500 Subject: [PATCH 03/28] update build status readme badge to github actions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bc37c3c52..13865c266 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Fog::Aws ![Gem Version](https://badge.fury.io/rb/fog-aws.svg) -[![Build Status](https://github.com/fog/fog-aws/actions/workflows/ruby.yml/badge.svg)](https://github.com/fog/fog-aws/workflows/ruby.yml) +[![Build Status](https://github.com/fog/fog-aws/actions/workflows/ruby.yml/badge.svg)](https://github.com/fog/fog-aws/actions/workflows/ruby.yml) [![Test Coverage](https://codeclimate.com/github/fog/fog-aws/badges/coverage.svg)](https://codeclimate.com/github/fog/fog-aws) [![Code Climate](https://codeclimate.com/github/fog/fog-aws.svg)](https://codeclimate.com/github/fog/fog-aws) From 412677cd1028efbf81e7c53a205cdeaa24c1aed7 Mon Sep 17 00:00:00 2001 From: vincent Date: Thu, 17 Jun 2021 09:45:54 +0200 Subject: [PATCH 04/28] fix storage for ruby 3.0 adapt storage#get_object with a block given. ruby 3 raises an error when instantiating a proc without a block @see https://blog.saeloun.com/2019/09/02/ruby-2-7-proc-without-block-warning.html --- lib/fog/aws/requests/storage/get_object.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fog/aws/requests/storage/get_object.rb b/lib/fog/aws/requests/storage/get_object.rb index 39cff00f3..f28a072fb 100644 --- a/lib/fog/aws/requests/storage/get_object.rb +++ b/lib/fog/aws/requests/storage/get_object.rb @@ -50,7 +50,7 @@ module Fog idempotent = true if block_given? - params[:response_block] = Proc.new + params[:response_block] = Proc.new(&block) idempotent = false end From c18e70e99fe321d8444573dc1a7bb7c9aa599e66 Mon Sep 17 00:00:00 2001 From: iqre8 <57012516+iqre8@users.noreply.github.com> Date: Sat, 10 Jul 2021 10:40:19 -0300 Subject: [PATCH 05/28] Update file.rb Add website_redirect_location --- lib/fog/aws/models/storage/file.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/fog/aws/models/storage/file.rb b/lib/fog/aws/models/storage/file.rb index 9c0d109fd..a56949b21 100644 --- a/lib/fog/aws/models/storage/file.rb +++ b/lib/fog/aws/models/storage/file.rb @@ -30,6 +30,7 @@ module Fog attribute :version, :aliases => 'x-amz-version-id' attribute :kms_key_id, :aliases => 'x-amz-server-side-encryption-aws-kms-key-id' attribute :tags, :aliases => 'x-amz-tagging' + attribute :website_redirect_location, :aliases => 'x-amz-website-redirect-location' UploadPartData = Struct.new(:part_number, :upload_options, :etag) @@ -249,6 +250,7 @@ module Fog # @option options [String] storage_class sets x-amz-storage-class HTTP header. Defaults to 'STANDARD'. Or, 'REDUCED_REDUNDANCY' # @option options [String] encryption sets HTTP encryption header. Set to 'AES256' to encrypt files at rest on S3 # @option options [String] tags sets x-amz-tagging HTTP header. For example, 'Org-Id=1' or 'Org-Id=1&Service=MyService' + # @option options [String] website_redirect_location sets x-amz-website-redirect-location HTTP header. For example, 'website_redirect_location=http://www.rubydoc.info/github/fog/fog-aws' # @return [Boolean] true if no errors # def save(options = {}) @@ -266,6 +268,7 @@ module Fog options.merge!(metadata) options['x-amz-storage-class'] = storage_class if storage_class options['x-amz-tagging'] = tags if tags + options['x-amz-website-redirect-location' = website_redirect_location if website_redirect_location options.merge!(encryption_headers) # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects. From 25d720e55c07c8377dd495f5141bcb2ea72f455d Mon Sep 17 00:00:00 2001 From: iqre8 <57012516+iqre8@users.noreply.github.com> Date: Wed, 14 Jul 2021 15:58:33 -0300 Subject: [PATCH 06/28] Update file.rb Added missing ] --- lib/fog/aws/models/storage/file.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fog/aws/models/storage/file.rb b/lib/fog/aws/models/storage/file.rb index a56949b21..b674930b6 100644 --- a/lib/fog/aws/models/storage/file.rb +++ b/lib/fog/aws/models/storage/file.rb @@ -268,7 +268,7 @@ module Fog options.merge!(metadata) options['x-amz-storage-class'] = storage_class if storage_class options['x-amz-tagging'] = tags if tags - options['x-amz-website-redirect-location' = website_redirect_location if website_redirect_location + options['x-amz-website-redirect-location'] = website_redirect_location if website_redirect_location options.merge!(encryption_headers) # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects. From 9670f3141c26afb16d635fdf10ccf0e9ef021269 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 15 Jul 2021 00:22:05 +0000 Subject: [PATCH 07/28] Bump actions/stale from 3.0.19 to 4 Bumps [actions/stale](https://github.com/actions/stale) from 3.0.19 to 4. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v3.0.19...v4) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 5996f2b8c..00b048ff4 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/stale@v3.0.19 + - uses: actions/stale@v4 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 60 From 04b3cd7cbf7002ac95ea487f825ea67c3730d02c Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 26 Jul 2021 18:42:34 +0300 Subject: [PATCH 08/28] Add storage option to configure multipart put/copy Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB. Other S3-compatible storages (like, Ceph) do not have such limit. Ceph shows much better performance when file is copied as a whole, in a single request. This commit adds `max_put_chunk_size`/`max_copy_chunk_size`storage options that allow user to configure respective chunk sizes. Usage of non-positive value will tell fog-aws to use a single put/copy request regardless of file size. This commit preserves backward compatibility: if user explicitly sets `multipart_chunk_size` on a `Fog::AWS::Storage::File` instance, that value will take precedence. --- lib/fog/aws/models/storage/file.rb | 20 +++++++-------- lib/fog/aws/storage.rb | 39 +++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/lib/fog/aws/models/storage/file.rb b/lib/fog/aws/models/storage/file.rb index b674930b6..7e1bdb1ce 100644 --- a/lib/fog/aws/models/storage/file.rb +++ b/lib/fog/aws/models/storage/file.rb @@ -4,8 +4,11 @@ module Fog module AWS class Storage class File < Fog::Model - MIN_MULTIPART_CHUNK_SIZE = 5242880 - MAX_SINGLE_PUT_SIZE = 5368709120 + # @deprecated use {Fog::AWS::Storage::MIN_MULTIPART_CHUNK_SIZE} instead + MIN_MULTIPART_CHUNK_SIZE = Fog::AWS::Storage::MIN_MULTIPART_CHUNK_SIZE + # @deprecated use {Fog::AWS::Storage::MAX_SINGLE_PUT_SIZE} instead + MAX_SINGLE_PUT_SIZE = Fog::AWS::Storage::MAX_SINGLE_PUT_SIZE + # @deprecated not used for anything MULTIPART_COPY_THRESHOLD = 15728640 # @see AWS Object docs http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectOps.html @@ -65,7 +68,7 @@ module Fog # Use small chunk sizes to minimize memory. E.g. 5242880 = 5mb attr_reader :multipart_chunk_size def multipart_chunk_size=(mp_chunk_size) - raise ArgumentError.new("minimum multipart_chunk_size is #{MIN_MULTIPART_CHUNK_SIZE}") if mp_chunk_size < MIN_MULTIPART_CHUNK_SIZE + service.validate_chunk_size(mp_chunk_size, 'multipart_chunk_size') @multipart_chunk_size = mp_chunk_size end @@ -145,10 +148,9 @@ module Fog def copy(target_directory_key, target_file_key, options = {}) requires :directory, :key - # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects. - self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE * 2 if !multipart_chunk_size && self.content_length.to_i > MAX_SINGLE_PUT_SIZE + self.multipart_chunk_size = service.max_copy_chunk_size if multipart_chunk_size.nil? - if multipart_chunk_size && self.content_length.to_i >= multipart_chunk_size + if multipart_chunk_size > 0 && self.content_length.to_i >= multipart_chunk_size upload_part_options = options.select { |key, _| ALLOWED_UPLOAD_PART_OPTIONS.include?(key.to_sym) } upload_part_options = upload_part_options.merge({ 'x-amz-copy-source' => "#{directory.key}/#{key}" }) multipart_copy(options, upload_part_options, target_directory_key, target_file_key) @@ -271,10 +273,8 @@ module Fog options['x-amz-website-redirect-location'] = website_redirect_location if website_redirect_location options.merge!(encryption_headers) - # With a single PUT operation you can upload objects up to 5 GB in size. Automatically set MP for larger objects. - self.multipart_chunk_size = MIN_MULTIPART_CHUNK_SIZE if !multipart_chunk_size && Fog::Storage.get_body_size(body) > MAX_SINGLE_PUT_SIZE - - if multipart_chunk_size && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read) + self.multipart_chunk_size = service.max_put_chunk_size if multipart_chunk_size.nil? + if multipart_chunk_size > 0 && Fog::Storage.get_body_size(body) >= multipart_chunk_size && body.respond_to?(:read) data = multipart_save(options) merge_attributes(data.body) else diff --git a/lib/fog/aws/storage.rb b/lib/fog/aws/storage.rb index c2d5e5925..ab9652cdb 100644 --- a/lib/fog/aws/storage.rb +++ b/lib/fog/aws/storage.rb @@ -14,6 +14,9 @@ module Fog 'https' => 443 } + MIN_MULTIPART_CHUNK_SIZE = 5242880 + MAX_SINGLE_PUT_SIZE = 5368709120 + VALID_QUERY_KEYS = %w[ acl cors @@ -43,7 +46,7 @@ module Fog ] requires :aws_access_key_id, :aws_secret_access_key - recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname + recognizes :endpoint, :region, :host, :port, :scheme, :persistent, :use_iam_profile, :aws_session_token, :aws_credentials_expire_at, :path_style, :acceleration, :instrumentor, :instrumentor_name, :aws_signature_version, :enable_signature_v4_streaming, :virtual_host, :cname, :max_put_chunk_size, :max_copy_chunk_size secrets :aws_secret_access_key, :hmac @@ -117,6 +120,17 @@ module Fog module Utils attr_accessor :region + # Amazon S3 limits max chunk size that can be uploaded/copied in a single request to 5GB. + # Other S3-compatible storages (like, Ceph) do not have such limit. + # Ceph shows much better performance when file is copied as a whole, in a single request. + # fog-aws user can use these settings to configure chunk sizes. + # A non-positive value will tell fog-aws to use a single put/copy request regardless of file size. + # + # @return [Integer] + # @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/copy-object.html + attr_reader :max_put_chunk_size + attr_reader :max_copy_chunk_size + def cdn @cdn ||= Fog::AWS::CDN.new( :aws_access_key_id => @aws_access_key_id, @@ -171,6 +185,12 @@ module Fog params_to_url(params) end + # @param value [int] + # @param description [str] + def validate_chunk_size(value, description) + raise "#{description} (#{value}) is less than minimum #{MIN_MULTIPART_CHUNK_SIZE}" unless value <= 0 || value >= MIN_MULTIPART_CHUNK_SIZE + end + private def validate_signature_version! @@ -179,6 +199,16 @@ module Fog end end + def init_max_put_chunk_size!(options = {}) + @max_put_chunk_size = options.fetch(:max_put_chunk_size, MAX_SINGLE_PUT_SIZE) + validate_chunk_size(@max_put_chunk_size, 'max_put_chunk_size') + end + + def init_max_copy_chunk_size!(options = {}) + @max_copy_chunk_size = options.fetch(:max_copy_chunk_size, MAX_SINGLE_PUT_SIZE) + validate_chunk_size(@max_copy_chunk_size, 'max_copy_chunk_size') + end + def v4_signed_params_for_url(params, expires) now = Fog::Time.now @@ -452,6 +482,10 @@ module Fog @path_style = options[:path_style] || false + + init_max_put_chunk_size!(options) + init_max_copy_chunk_size!(options) + @signature_version = options.fetch(:aws_signature_version, 4) validate_signature_version! setup_credentials(options) @@ -515,6 +549,9 @@ module Fog validate_signature_version! @path_style = options[:path_style] || false + init_max_put_chunk_size!(options) + init_max_copy_chunk_size!(options) + @region = options[:region] || DEFAULT_REGION if @endpoint = options[:endpoint] From ed5190434dcf1ca6a1c9421e2ecc77bd9595e788 Mon Sep 17 00:00:00 2001 From: geemus Date: Thu, 5 Aug 2021 11:20:41 -0500 Subject: [PATCH 09/28] v3.11.0 --- CHANGELOG.md | 121 ++++++++++++++++++++++++++++++++++++++++- lib/fog/aws/version.rb | 2 +- 2 files changed, 120 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b4be7af59..7e10ab638 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,123 @@ # Changelog +## [Unreleased](https://github.com/fog/fog-aws/tree/HEAD) + +[Full Changelog](https://github.com/fog/fog-aws/compare/v3.10.0...HEAD) + +**Closed issues:** + +- Support for Regional STS Endpoints [\#604](https://github.com/fog/fog-aws/issues/604) +- Pass tags when creating EBS ? [\#603](https://github.com/fog/fog-aws/issues/603) +- S3 multiple delete api should handle delete for multiple versions for a single object [\#598](https://github.com/fog/fog-aws/issues/598) +- Fog does not return error from s3 [\#586](https://github.com/fog/fog-aws/issues/586) +- Add support for r6g and c6g instance types [\#580](https://github.com/fog/fog-aws/issues/580) +- Using internal S3 provider ... and something funky is going on! [\#575](https://github.com/fog/fog-aws/issues/575) +- How to upload File to S3 with with accelerate? [\#554](https://github.com/fog/fog-aws/issues/554) +- How to use assume\_role\_with\_web\_identity auth for S3 storage? [\#543](https://github.com/fog/fog-aws/issues/543) +- Fog::AWS::Glacier::TreeHash::add\_part works only sometimes [\#520](https://github.com/fog/fog-aws/issues/520) +- fog-aws: Fog::AWS::Glacier::Job doesn't support RetrievalByteRange [\#519](https://github.com/fog/fog-aws/issues/519) +- Path style is being deprecated [\#516](https://github.com/fog/fog-aws/issues/516) +- Fog::DNS::AWS can't read zones [\#513](https://github.com/fog/fog-aws/issues/513) +- Lambda Parser can't handle VPC config, containing array of hash values [\#509](https://github.com/fog/fog-aws/issues/509) +- Excon::Error::Forbidden: Expected\(200\) \<=\> Actual\(403 Forbidden\) [\#508](https://github.com/fog/fog-aws/issues/508) +- file.save does not work with STDIN [\#500](https://github.com/fog/fog-aws/issues/500) +- ELBv2 Support ? [\#489](https://github.com/fog/fog-aws/issues/489) +- S3 Select Support? [\#484](https://github.com/fog/fog-aws/issues/484) +- nil:NilClass error [\#483](https://github.com/fog/fog-aws/issues/483) +- Mocks for VM creation require access and secret key when using instance profiles [\#482](https://github.com/fog/fog-aws/issues/482) +- Always use bucket virtual hostname? [\#464](https://github.com/fog/fog-aws/issues/464) +- Spot instance creation : Support for BlockDurationMinutes [\#461](https://github.com/fog/fog-aws/issues/461) +- How can I remove the `Content-Encoding` metadata properties if I upload from fog [\#447](https://github.com/fog/fog-aws/issues/447) +- AWS::ECS with `use_iam_profile` errors out [\#441](https://github.com/fog/fog-aws/issues/441) +- Option to turn off Warnings [\#426](https://github.com/fog/fog-aws/issues/426) +- Accessing AWS S3 using EC2 Instance Profile [\#423](https://github.com/fog/fog-aws/issues/423) +- Support step and target tracking auto scaling policies [\#422](https://github.com/fog/fog-aws/issues/422) +- could not create ec2 volume with custom encryption key, volume only create with default 'aws/ebs' encryption key [\#420](https://github.com/fog/fog-aws/issues/420) +- Download File with content\_disposition [\#418](https://github.com/fog/fog-aws/issues/418) +- Fog::Compute::AWS::Error iamInstanceProfile.name is invalid [\#410](https://github.com/fog/fog-aws/issues/410) +- Mocks for EC2 instance creation do not behave as expected [\#404](https://github.com/fog/fog-aws/issues/404) +- Cannot copy an encrypted snapshot from one account to another account [\#398](https://github.com/fog/fog-aws/issues/398) +- Fog::Compute::AWS::Image\#deregister ignores non-root snapshots. [\#380](https://github.com/fog/fog-aws/issues/380) +- AWS S3 overwrites files with same name [\#378](https://github.com/fog/fog-aws/issues/378) +- Support S3 object tagging [\#377](https://github.com/fog/fog-aws/issues/377) +- Reqeust to support Aws::DynamoDBStreams [\#373](https://github.com/fog/fog-aws/issues/373) +- Not all Rds versions and Instance Types are rendered [\#371](https://github.com/fog/fog-aws/issues/371) +- Tag instances upon creation of new instance [\#359](https://github.com/fog/fog-aws/issues/359) +- Creating instances in AWS fails with Socket Error [\#352](https://github.com/fog/fog-aws/issues/352) +- `NameError: uninitialized constant Fog::ServicesMixin` when requiring `fog/storage` [\#345](https://github.com/fog/fog-aws/issues/345) +- Add full support for target groups [\#328](https://github.com/fog/fog-aws/issues/328) +- Fog transfer acceleration endpoints [\#303](https://github.com/fog/fog-aws/issues/303) +- "Fog::DNS\[:aws\] | change\_resource\_record\_sets \(aws, dns\)" test suite flaky [\#301](https://github.com/fog/fog-aws/issues/301) +- Cross account access using IAM role [\#294](https://github.com/fog/fog-aws/issues/294) +- Write timeout trying to upload a large file to S3 [\#291](https://github.com/fog/fog-aws/issues/291) +- Support Autoscaling lifecycle hooks [\#289](https://github.com/fog/fog-aws/issues/289) +- directories ignore region option [\#287](https://github.com/fog/fog-aws/issues/287) +- Feature: Access logs for ELB [\#271](https://github.com/fog/fog-aws/issues/271) +- S3: retry on 500 internal server error [\#264](https://github.com/fog/fog-aws/issues/264) +- Alias for server side encryption not working [\#260](https://github.com/fog/fog-aws/issues/260) +- InvalidParameterCombination =\> You cannot move a DB instance with Single-Az enabled to a VPC \(Fog::AWS::RDS::Error\) [\#255](https://github.com/fog/fog-aws/issues/255) +- Using STS [\#253](https://github.com/fog/fog-aws/issues/253) +- Auto Scaling Group does not enable metrics [\#251](https://github.com/fog/fog-aws/issues/251) +- aws has no storage service [\#248](https://github.com/fog/fog-aws/issues/248) +- Timeouts on Compute\#describe\_volumes due to extreme numbers of volumes [\#244](https://github.com/fog/fog-aws/issues/244) +- Support CreateReusableDelegationSet [\#243](https://github.com/fog/fog-aws/issues/243) +- Tags server creation in Mock vs Real [\#239](https://github.com/fog/fog-aws/issues/239) +- Excon::Errors::SocketError Broken pipe \(Errno::EPIPE\) when use Activeadmin upload image by nested form [\#237](https://github.com/fog/fog-aws/issues/237) +- Fog Mock doesn't update [\#236](https://github.com/fog/fog-aws/issues/236) +- ECS service\_update does not support "deploymentConfig" [\#234](https://github.com/fog/fog-aws/issues/234) +- Fog::Storage::AWS::Files\#each always iterates over entire collection [\#232](https://github.com/fog/fog-aws/issues/232) +- repeated bucket name in the URL on AWS and issue with :path\_style [\#228](https://github.com/fog/fog-aws/issues/228) +- Already initialized constant warnings [\#212](https://github.com/fog/fog-aws/issues/212) +- SQS API version is outdated [\#198](https://github.com/fog/fog-aws/issues/198) +- Problem when using irb [\#195](https://github.com/fog/fog-aws/issues/195) +- compute.servers \(via DescribeInstances\) does not include tags reliably [\#192](https://github.com/fog/fog-aws/issues/192) +- EBS create volume io1 or gp2 [\#186](https://github.com/fog/fog-aws/issues/186) +- Aws cloudformation stack-policy-body [\#179](https://github.com/fog/fog-aws/issues/179) +- EXCON\_DEBUG and DEBUG env variables do not help debug -S key issues [\#177](https://github.com/fog/fog-aws/issues/177) +- AWS4 SignatureDoesNotMatch if header contains two spaces [\#160](https://github.com/fog/fog-aws/issues/160) +- Add support for elasticache redis replication groups [\#136](https://github.com/fog/fog-aws/issues/136) +- Getting SignatureDoesNotMatch error with eu-central-1 [\#127](https://github.com/fog/fog-aws/issues/127) +- Cannot saving auto scaling group [\#125](https://github.com/fog/fog-aws/issues/125) +- fog-aws not working with dynamoDB Local [\#118](https://github.com/fog/fog-aws/issues/118) +- Fog::Compute::AWS::Error InvalidParameterValue =\> secondary-addresses [\#115](https://github.com/fog/fog-aws/issues/115) +- Is there an equivalent to describe-instance-status? [\#66](https://github.com/fog/fog-aws/issues/66) +- No usage instructions in Readme [\#64](https://github.com/fog/fog-aws/issues/64) +- AWS - distributionConfig.enabled' failed to satisfy constraint: Member must not be null [\#48](https://github.com/fog/fog-aws/issues/48) +- Clarify versioning on README [\#42](https://github.com/fog/fog-aws/issues/42) +- AWS SQS AddPermission API missing [\#26](https://github.com/fog/fog-aws/issues/26) +- AWS China region [\#25](https://github.com/fog/fog-aws/issues/25) +- AWS CloudFormation ListStacks options [\#24](https://github.com/fog/fog-aws/issues/24) +- Setting region of AWS::Compute after initialization [\#23](https://github.com/fog/fog-aws/issues/23) +- Support AWS Support API [\#22](https://github.com/fog/fog-aws/issues/22) +- InvalidClientTokenId =\> The security token included in the request is invalid [\#21](https://github.com/fog/fog-aws/issues/21) +- Change architecture attribute in AWS::Compute::Server model [\#20](https://github.com/fog/fog-aws/issues/20) +- Add support for Amazon Kinesis [\#19](https://github.com/fog/fog-aws/issues/19) +- Bring AWS CloudFront API Models/Requests up to date [\#17](https://github.com/fog/fog-aws/issues/17) +- AWS security group tests have become unstable [\#16](https://github.com/fog/fog-aws/issues/16) +- AWS auto scaling: availability zones are not a required parameter [\#15](https://github.com/fog/fog-aws/issues/15) +- Is anyone going to add support for AWS ElasticTranscoder [\#14](https://github.com/fog/fog-aws/issues/14) +- add missing attributes to aws describe\_reserved\_instances parser [\#13](https://github.com/fog/fog-aws/issues/13) +- AWS AutoScaling group min\_size & max\_size getting set to 0 [\#12](https://github.com/fog/fog-aws/issues/12) +- auto\_scaling\_group.instances does not return only instances for that group [\#11](https://github.com/fog/fog-aws/issues/11) +- Why are the credential keys not generalized? [\#10](https://github.com/fog/fog-aws/issues/10) +- Invalid XML Character in S3 Response [\#8](https://github.com/fog/fog-aws/issues/8) +- reading s3 upload progress [\#7](https://github.com/fog/fog-aws/issues/7) +- delete\_on\_termination=true attribute on new volume is not set on create [\#6](https://github.com/fog/fog-aws/issues/6) +- user\_data is still base64 encoded in Real launch\_configurations [\#5](https://github.com/fog/fog-aws/issues/5) + +**Merged pull requests:** + +- Add storage option to configure multipart put/copy [\#616](https://github.com/fog/fog-aws/pull/616) ([slonopotamus](https://github.com/slonopotamus)) +- Bump actions/stale from 3.0.19 to 4 [\#615](https://github.com/fog/fog-aws/pull/615) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Update file.rb [\#613](https://github.com/fog/fog-aws/pull/613) ([iqre8](https://github.com/iqre8)) +- fix storage for ruby 3.0 [\#611](https://github.com/fog/fog-aws/pull/611) ([vincentjoseph](https://github.com/vincentjoseph)) +- Implement AWS TagSpecifications \(closes \#603\) [\#610](https://github.com/fog/fog-aws/pull/610) ([eLvErDe](https://github.com/eLvErDe)) +- Bump actions/stale from 3.0.18 to 3.0.19 [\#609](https://github.com/fog/fog-aws/pull/609) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump actions/stale from 3 to 3.0.18 [\#608](https://github.com/fog/fog-aws/pull/608) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump actions/checkout from 2 to 2.3.4 [\#607](https://github.com/fog/fog-aws/pull/607) ([dependabot[bot]](https://github.com/apps/dependabot)) +- drop git in gemspec [\#602](https://github.com/fog/fog-aws/pull/602) ([abrahamparayil](https://github.com/abrahamparayil)) +- Update rubyzip requirement from ~\> 1.3.0 to ~\> 2.3.0 [\#601](https://github.com/fog/fog-aws/pull/601) ([dependabot[bot]](https://github.com/apps/dependabot)) + ## [v3.10.0](https://github.com/fog/fog-aws/tree/v3.10.0) (2021-03-22) [Full Changelog](https://github.com/fog/fog-aws/compare/v3.9.0...v3.10.0) @@ -12,7 +130,6 @@ - Enable hibernation on creation of ec2 instance [\#566](https://github.com/fog/fog-aws/issues/566) - Tests broken with fog-core 2.1.0 [\#504](https://github.com/fog/fog-aws/issues/504) - changelog? [\#471](https://github.com/fog/fog-aws/issues/471) -- Changelog: Listing breaking changes [\#419](https://github.com/fog/fog-aws/issues/419) - How to use iam\_instance\_profile? [\#342](https://github.com/fog/fog-aws/issues/342) - how to support additional aws regions, e.g. cn-north-1 [\#164](https://github.com/fog/fog-aws/issues/164) - Still empty content-encoding when it is not set [\#130](https://github.com/fog/fog-aws/issues/130) @@ -883,7 +1000,7 @@ - Fix mock VPC ELB creation in regions other than us-east-1 [\#94](https://github.com/fog/fog-aws/pull/94) ([mrpoundsign](https://github.com/mrpoundsign)) - Fix repository URL in README.md [\#91](https://github.com/fog/fog-aws/pull/91) ([tricknotes](https://github.com/tricknotes)) - adding support for d2 instance type [\#90](https://github.com/fog/fog-aws/pull/90) ([yumminhuang](https://github.com/yumminhuang)) -- Support weight round robin mock [\#89](https://github.com/fog/fog-aws/pull/89) ([freddy1666](https://github.com/freddy1666)) +- Support weight round robin mock [\#89](https://github.com/fog/fog-aws/pull/89) ([freddy61025](https://github.com/freddy61025)) - Update README.md [\#87](https://github.com/fog/fog-aws/pull/87) ([nomadium](https://github.com/nomadium)) - Add mock for EC2 request\_spot\_instances API request [\#86](https://github.com/fog/fog-aws/pull/86) ([nomadium](https://github.com/nomadium)) - Move more requires to autoload [\#85](https://github.com/fog/fog-aws/pull/85) ([plribeiro3000](https://github.com/plribeiro3000)) diff --git a/lib/fog/aws/version.rb b/lib/fog/aws/version.rb index 676602d0b..756a3deca 100644 --- a/lib/fog/aws/version.rb +++ b/lib/fog/aws/version.rb @@ -1,5 +1,5 @@ module Fog module AWS - VERSION = "3.10.0" + VERSION = "3.11.0" end end From 0ce9184b64f64af125f251e679574609fd188a2a Mon Sep 17 00:00:00 2001 From: Sunny Ripert Date: Mon, 16 Aug 2021 12:18:12 +0200 Subject: [PATCH 10/28] Separate CHANGELOG entry for 3.11.0 --- CHANGELOG.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e10ab638..66dcd1e1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,11 @@ ## [Unreleased](https://github.com/fog/fog-aws/tree/HEAD) -[Full Changelog](https://github.com/fog/fog-aws/compare/v3.10.0...HEAD) +[Full Changelog](https://github.com/fog/fog-aws/compare/v3.11.0...HEAD) + +## [v3.11.0](https://github.com/fog/fog-aws/tree/v3.11.0) (2021-08-05) + +[Full Changelog](https://github.com/fog/fog-aws/compare/v3.10.0...v3.11.0) **Closed issues:** From 171ad6ff5fe63956cbffc45e31d13cda343f5526 Mon Sep 17 00:00:00 2001 From: Petrik Date: Wed, 18 Aug 2021 14:06:42 +0200 Subject: [PATCH 11/28] Add IPv6 support for Ingress Security Groups --- lib/fog/aws/compute.rb | 9 +++++--- lib/fog/aws/models/compute/security_group.rb | 18 ++++++++++----- .../compute/describe_security_groups.rb | 22 +++++++++++++++---- .../authorize_security_group_ingress.rb | 15 +++++++++++++ .../compute/describe_security_groups.rb | 2 ++ .../requests/compute/security_group_tests.rb | 13 ++++++++++- 6 files changed, 66 insertions(+), 13 deletions(-) diff --git a/lib/fog/aws/compute.rb b/lib/fog/aws/compute.rb index 492ab57ce..0a8dd35f8 100644 --- a/lib/fog/aws/compute.rb +++ b/lib/fog/aws/compute.rb @@ -233,21 +233,24 @@ module Fog 'fromPort' => -1, 'toPort' => -1, 'ipProtocol' => 'icmp', - 'ipRanges' => [] + 'ipRanges' => [], + 'ipv6Ranges' => [] }, { 'groups' => [{'groupName' => 'default', 'userId' => owner_id, 'groupId' => security_group_id}], 'fromPort' => 0, 'toPort' => 65535, 'ipProtocol' => 'tcp', - 'ipRanges' => [] + 'ipRanges' => [], + 'ipv6Ranges' => [] }, { 'groups' => [{'groupName' => 'default', 'userId' => owner_id, 'groupId' => security_group_id}], 'fromPort' => 0, 'toPort' => 65535, 'ipProtocol' => 'udp', - 'ipRanges' => [] + 'ipRanges' => [], + 'ipv6Ranges' => [] } ], 'ownerId' => owner_id diff --git a/lib/fog/aws/models/compute/security_group.rb b/lib/fog/aws/models/compute/security_group.rb index ce3f98e7c..cf0eb61b0 100644 --- a/lib/fog/aws/models/compute/security_group.rb +++ b/lib/fog/aws/models/compute/security_group.rb @@ -62,7 +62,8 @@ module Fog # options:: # A hash that can contain any of the following keys: # :cidr_ip (defaults to "0.0.0.0/0") - # :group - ("account:group_name" or "account:group_id"), cannot be used with :cidr_ip + # :cidr_ipv6 cannot be used with :cidr_ip + # :group - ("account:group_name" or "account:group_id"), cannot be used with :cidr_ip or :cidr_ipv6 # :ip_protocol (defaults to "tcp") # # == Returns: @@ -178,7 +179,8 @@ module Fog # options:: # A hash that can contain any of the following keys: # :cidr_ip (defaults to "0.0.0.0/0") - # :group - ("account:group_name" or "account:group_id"), cannot be used with :cidr_ip + # :cidr_ipv6 cannot be used with :cidr_ip + # :group - ("account:group_name" or "account:group_id"), cannot be used with :cidr_ip or :cidr_ipv6 # :ip_protocol (defaults to "tcp") # # == Returns: @@ -327,9 +329,15 @@ module Fog } if options[:group].nil? - ip_permission['IpRanges'] = [ - { 'CidrIp' => options[:cidr_ip] || '0.0.0.0/0' } - ] + if options[:cidr_ipv6].nil? + ip_permission['IpRanges'] = [ + { 'CidrIp' => options[:cidr_ip] || '0.0.0.0/0' } + ] + else + ip_permission['Ipv6Ranges'] = [ + { 'CidrIpv6' => options[:cidr_ipv6] } + ] + end else ip_permission['Groups'] = [ group_info(options[:group]) diff --git a/lib/fog/aws/parsers/compute/describe_security_groups.rb b/lib/fog/aws/parsers/compute/describe_security_groups.rb index e1a46f091..0d38dbb8c 100644 --- a/lib/fog/aws/parsers/compute/describe_security_groups.rb +++ b/lib/fog/aws/parsers/compute/describe_security_groups.rb @@ -5,9 +5,10 @@ module Fog class DescribeSecurityGroups < Fog::Parsers::Base def reset @group = {} - @ip_permission = { 'groups' => [], 'ipRanges' => []} - @ip_permission_egress = { 'groups' => [], 'ipRanges' => []} + @ip_permission = { 'groups' => [], 'ipRanges' => [], 'ipv6Ranges' => []} + @ip_permission_egress = { 'groups' => [], 'ipRanges' => [], 'ipv6Ranges' => []} @ip_range = {} + @ipv6_range = {} @security_group = { 'ipPermissions' => [], 'ipPermissionsEgress' => [], 'tagSet' => {} } @response = { 'securityGroupInfo' => [] } @tag = {} @@ -24,6 +25,8 @@ module Fog @in_ip_permissions_egress = true when 'ipRanges' @in_ip_ranges = true + when 'ipv6Ranges' + @in_ipv6_ranges = true when 'tagSet' @in_tag_set = true end @@ -44,6 +47,8 @@ module Fog case name when 'cidrIp' @ip_range[name] = value + when 'cidrIpv6' + @ipv6_range[name] = value when 'fromPort', 'toPort' if @in_ip_permissions_egress @ip_permission_egress[name] = value.to_i @@ -72,6 +77,8 @@ module Fog end when 'ipRanges' @in_ip_ranges = false + when 'ipv6Ranges' + @in_ipv6_ranges = false when 'item' if @in_groups if @in_ip_permissions_egress @@ -87,12 +94,19 @@ module Fog @ip_permission['ipRanges'] << @ip_range end @ip_range = {} + elsif @in_ipv6_ranges + if @in_ip_permissions_egress + @ip_permission_egress['ipv6Ranges'] << @ipv6_range + else + @ip_permission['ipv6Ranges'] << @ipv6_range + end + @ipv6_range = {} elsif @in_ip_permissions @security_group['ipPermissions'] << @ip_permission - @ip_permission = { 'groups' => [], 'ipRanges' => []} + @ip_permission = { 'groups' => [], 'ipRanges' => [], 'ipv6Ranges' => []} elsif @in_ip_permissions_egress @security_group['ipPermissionsEgress'] << @ip_permission_egress - @ip_permission_egress = { 'groups' => [], 'ipRanges' => []} + @ip_permission_egress = { 'groups' => [], 'ipRanges' => [], 'ipv6Ranges' => []} else @response['securityGroupInfo'] << @security_group @security_group = { 'ipPermissions' => [], 'ipPermissionsEgress' => [], 'tagSet' => {} } diff --git a/lib/fog/aws/requests/compute/authorize_security_group_ingress.rb b/lib/fog/aws/requests/compute/authorize_security_group_ingress.rb index 159c9a6dd..deeb814c5 100644 --- a/lib/fog/aws/requests/compute/authorize_security_group_ingress.rb +++ b/lib/fog/aws/requests/compute/authorize_security_group_ingress.rb @@ -30,6 +30,9 @@ module Fog # * 'IpRanges'<~Array>: # * ip_range<~Hash>: # * 'CidrIp'<~String> - CIDR range + # * 'Ipv6Ranges'<~Array>: + # * ip_range<~Hash>: + # * 'CidrIpv6'<~String> - CIDR range # * 'ToPort'<~Integer> - End of port range (or -1 for ICMP wildcard) # # === Returns @@ -72,6 +75,10 @@ module Fog range_index += 1 params[format('IpPermissions.%d.IpRanges.%d.CidrIp', key_index, range_index)] = ip_range['CidrIp'] end + (permission['Ipv6Ranges'] || []).each_with_index do |ip_range, range_index| + range_index += 1 + params[format('IpPermissions.%d.Ipv6Ranges.%d.CidrIpv6', key_index, range_index)] = ip_range['CidrIpv6'] + end end params.reject {|k, v| v.nil? } end @@ -186,6 +193,14 @@ module Fog 'groups' => [], 'ipRanges' => [{'cidrIp' => options['CidrIp']}] } + elsif options['CidrIpv6'] + normalized_permissions << { + 'ipProtocol' => options['IpProtocol'], + 'fromPort' => Integer(options['FromPort']), + 'toPort' => Integer(options['ToPort']), + 'groups' => [], + 'ipv6Ranges' => [{'cidrIpv6' => options['CidrIpv6']}] + } elsif options['IpPermissions'] options['IpPermissions'].each do |permission| diff --git a/lib/fog/aws/requests/compute/describe_security_groups.rb b/lib/fog/aws/requests/compute/describe_security_groups.rb index a589e216c..a39d6c053 100644 --- a/lib/fog/aws/requests/compute/describe_security_groups.rb +++ b/lib/fog/aws/requests/compute/describe_security_groups.rb @@ -27,6 +27,8 @@ module Fog # * 'ipProtocol'<~String> - Ip protocol, must be in ['tcp', 'udp', 'icmp'] # * 'ipRanges'<~Array>: # * 'cidrIp'<~String> - CIDR range + # * 'ipv6Ranges'<~Array>: + # * 'cidrIpv6'<~String> - CIDR ipv6 range # * 'toPort'<~Integer> - End of port range (or -1 for ICMP wildcard) # * 'ownerId'<~String> - AWS Access Key Id of the owner of the security group # * 'NextToken'<~String> - The token to retrieve the next page of results diff --git a/tests/requests/compute/security_group_tests.rb b/tests/requests/compute/security_group_tests.rb index aad6e1f33..cb3404b52 100644 --- a/tests/requests/compute/security_group_tests.rb +++ b/tests/requests/compute/security_group_tests.rb @@ -19,6 +19,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do 'groups' => [{ 'groupName' => Fog::Nullable::String, 'userId' => String, 'groupId' => String }], 'ipProtocol' => String, 'ipRanges' => [Fog::Nullable::Hash], + 'ipv6Ranges' => [Fog::Nullable::Hash], 'toPort' => Fog::Nullable::Integer, }], 'ipPermissionsEgress' => [], @@ -54,16 +55,19 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}], "fromPort"=>1, "ipRanges"=>[], + "ipv6Ranges"=>[], "ipProtocol"=>"tcp", "toPort"=>65535}, {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}], "fromPort"=>1, "ipRanges"=>[], + "ipv6Ranges"=>[], "ipProtocol"=>"udp", "toPort"=>65535}, {"groups"=>[{"groupName"=>"default", "userId"=>@owner_id, "groupId"=>@group_id_default}], "fromPort"=>-1, "ipRanges"=>[], + "ipv6Ranges"=>[], "ipProtocol"=>"icmp", "toPort"=>-1} ] @@ -88,6 +92,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default}, {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}], "ipRanges"=>[], + "ipv6Ranges"=>[], "ipProtocol"=>"tcp", "fromPort"=>1, "toPort"=>65535}, @@ -95,6 +100,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default}, {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}], "ipRanges"=>[], + "ipv6Ranges"=>[], "ipProtocol"=>"udp", "fromPort"=>1, "toPort"=>65535}, @@ -102,6 +108,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do [{"userId"=>@owner_id, "groupName"=>"default", "groupId"=>@group_id_default}, {"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}], "ipRanges"=>[], + "ipv6Ranges"=>[], "ipProtocol"=>"icmp", "fromPort"=>-1, "toPort"=>-1} @@ -133,6 +140,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do expected_permissions += [ {"groups"=>[], "ipRanges"=>[{"cidrIp"=>"10.0.0.0/8"}], + "ipv6Ranges"=>[], "ipProtocol"=>"tcp", "fromPort"=>22, "toPort"=>22} @@ -164,7 +172,8 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do 'IpPermissions' => [ { 'IpProtocol' => 'tcp', 'FromPort' => '80', 'ToPort' => '80', - 'IpRanges' => [{ 'CidrIp' => '192.168.0.0/24' }] + 'IpRanges' => [{ 'CidrIp' => '192.168.0.0/24' }], + 'Ipv6Ranges' => [] } ] } @@ -177,6 +186,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do expected_permissions += [ {"groups"=>[], "ipRanges"=>[{"cidrIp"=>"192.168.0.0/24"}], + "ipv6Ranges"=>[], "ipProtocol"=>"tcp", "fromPort"=>80, "toPort"=>80} @@ -204,6 +214,7 @@ Shindo.tests('Fog::Compute[:aws] | security group requests', ['aws']) do expected_permissions += [ {"groups"=>[{"userId"=>@owner_id, "groupName"=>"fog_security_group_two", "groupId"=>@group_id_two}], "ipRanges"=>[], + "ipv6Ranges"=>[], "ipProtocol"=>"tcp", "fromPort"=>8000, "toPort"=>8000} From cbd3354777d14a42c4083534ba6862bb28d75291 Mon Sep 17 00:00:00 2001 From: Michael Gerhart Date: Mon, 3 May 2021 15:56:42 -0400 Subject: [PATCH 12/28] Add support for regional STS endpoints --- lib/fog/aws/credential_fetcher.rb | 11 ++++++++--- tests/credentials_tests.rb | 18 ++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/lib/fog/aws/credential_fetcher.rb b/lib/fog/aws/credential_fetcher.rb index 65d5447a3..e0ef2ec86 100644 --- a/lib/fog/aws/credential_fetcher.rb +++ b/lib/fog/aws/credential_fetcher.rb @@ -13,8 +13,6 @@ module Fog CONTAINER_CREDENTIALS_HOST = "http://169.254.170.2" - STS_GLOBAL_ENDPOINT = "https://sts.amazonaws.com" - module ServiceMethods def fetch_credentials(options) if options[:use_iam_profile] && Fog.mocking? @@ -44,7 +42,14 @@ module Fog :WebIdentityToken => File.read(options[:aws_web_identity_token_file] || ENV.fetch("AWS_WEB_IDENTITY_TOKEN_FILE")), :Version => "2011-06-15", } - connection = options[:connection] || Excon.new(STS_GLOBAL_ENDPOINT, :query => params) + + if ENV["AWS_STS_REGIONAL_ENDPOINTS"] == "regional" && ENV["AWS_DEFAULT_REGION"] + STS_ENDPOINT = "https://sts.#{ENV['AWS_DEFAULT_REGION']}.amazonaws.com" + else + STS_ENDPOINT = "https://sts.amazonaws.com" + end + + connection = options[:connection] || Excon.new(STS_ENDPOINT, :query => params) document = Nokogiri::XML(connection.get(:idempotent => true, :expects => 200).body) session = { diff --git a/tests/credentials_tests.rb b/tests/credentials_tests.rb index 3f6f69414..5976b46f1 100644 --- a/tests/credentials_tests.rb +++ b/tests/credentials_tests.rb @@ -83,6 +83,7 @@ Shindo.tests('AWS | credentials', ['aws']) do aws_secret_access_key: 'dummysecret', aws_session_token: 'dummytoken', region: 'us-west-1', + STS_ENDPOINT: "https://sts.amazonaws.com" aws_credentials_expire_at: expires_at ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) } end @@ -95,10 +96,27 @@ Shindo.tests('AWS | credentials', ['aws']) do aws_secret_access_key: 'dummysecret', aws_session_token: 'dummytoken', region: 'us-west-1', + STS_ENDPOINT: "https://sts.amazonaws.com" aws_credentials_expire_at: expires_at ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) } end + ENV["AWS_STS_REGIONAL_ENDPOINTS"] = "regional" + ENV["AWS_REGION"] = "us-west-1" + + tests('#fetch_credentials with regional sts endpoint') do + returns( + aws_access_key_id: 'dummykey', + aws_secret_access_key: 'dummysecret', + aws_session_token: 'dummytoken', + region: 'us-west-1', + STS_ENDPOINT: "https://sts.us-west-1.amazonaws.com" + aws_credentials_expire_at: expires_at + ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) } + end + + ENV["AWS_STS_REGIONAL_ENDPOINTS"] = nil + ENV["AWS_REGION"] = nil ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil compute = Fog::AWS::Compute.new(use_iam_profile: true) From dad90af22d19f2f1ee8bd0e9e3b12ff1d27406dd Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 20 Aug 2021 11:10:44 -0700 Subject: [PATCH 13/28] Clean up STS regional work We use the AWS_DEFAULT_REGION environment variable only if the region isn't specified explicitly in the configuration. --- lib/fog/aws/credential_fetcher.rb | 22 ++++++++++++---------- tests/credentials_tests.rb | 26 +++++++++++++++++++------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/lib/fog/aws/credential_fetcher.rb b/lib/fog/aws/credential_fetcher.rb index e0ef2ec86..0ce411419 100644 --- a/lib/fog/aws/credential_fetcher.rb +++ b/lib/fog/aws/credential_fetcher.rb @@ -21,7 +21,7 @@ module Fog if options[:use_iam_profile] begin role_data = nil - region = options[:region] + region = options[:region] || ENV["AWS_DEFAULT_REGION"] if ENV["AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"] connection = options[:connection] || Excon.new(CONTAINER_CREDENTIALS_HOST) @@ -43,13 +43,14 @@ module Fog :Version => "2011-06-15", } - if ENV["AWS_STS_REGIONAL_ENDPOINTS"] == "regional" && ENV["AWS_DEFAULT_REGION"] - STS_ENDPOINT = "https://sts.#{ENV['AWS_DEFAULT_REGION']}.amazonaws.com" - else - STS_ENDPOINT = "https://sts.amazonaws.com" - end + sts_endpoint = + if ENV["AWS_STS_REGIONAL_ENDPOINTS"] == "regional" + "https://sts.#{region}.amazonaws.com" + else + "https://sts.amazonaws.com" + end - connection = options[:connection] || Excon.new(STS_ENDPOINT, :query => params) + connection = options[:connection] || Excon.new(sts_endpoint, :query => params) document = Nokogiri::XML(connection.get(:idempotent => true, :expects => 200).body) session = { @@ -70,18 +71,19 @@ module Fog role_name = connection.get(:path => INSTANCE_METADATA_PATH, :idempotent => true, :expects => 200, :headers => token_header).body role_data = connection.get(:path => INSTANCE_METADATA_PATH+role_name, :idempotent => true, :expects => 200, :headers => token_header).body session = Fog::JSON.decode(role_data) - + region ||= connection.get(:path => INSTANCE_METADATA_AZ, :idempotent => true, :expects => 200, :headers => token_header).body[0..-2] end - + credentials = {} credentials[:aws_access_key_id] = session['AccessKeyId'] credentials[:aws_secret_access_key] = session['SecretAccessKey'] credentials[:aws_session_token] = session['Token'] credentials[:aws_credentials_expire_at] = Time.xmlschema session['Expiration'] - + # set region by default to the one the instance is in. credentials[:region] = region + credentials[:sts_endpoint] = sts_endpoint if sts_endpoint #these indicate the metadata service is unavailable or has no profile setup credentials rescue Excon::Error => e diff --git a/tests/credentials_tests.rb b/tests/credentials_tests.rb index 5976b46f1..e3981769e 100644 --- a/tests/credentials_tests.rb +++ b/tests/credentials_tests.rb @@ -83,7 +83,7 @@ Shindo.tests('AWS | credentials', ['aws']) do aws_secret_access_key: 'dummysecret', aws_session_token: 'dummytoken', region: 'us-west-1', - STS_ENDPOINT: "https://sts.amazonaws.com" + sts_endpoint: "https://sts.amazonaws.com", aws_credentials_expire_at: expires_at ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) } end @@ -96,27 +96,39 @@ Shindo.tests('AWS | credentials', ['aws']) do aws_secret_access_key: 'dummysecret', aws_session_token: 'dummytoken', region: 'us-west-1', - STS_ENDPOINT: "https://sts.amazonaws.com" + sts_endpoint: "https://sts.amazonaws.com", aws_credentials_expire_at: expires_at - ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) } + ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true, region: 'us-west-1') } end ENV["AWS_STS_REGIONAL_ENDPOINTS"] = "regional" - ENV["AWS_REGION"] = "us-west-1" - tests('#fetch_credentials with regional sts endpoint') do + tests('#fetch_credentials with regional STS endpoint') do returns( aws_access_key_id: 'dummykey', aws_secret_access_key: 'dummysecret', aws_session_token: 'dummytoken', region: 'us-west-1', - STS_ENDPOINT: "https://sts.us-west-1.amazonaws.com" + sts_endpoint: "https://sts.us-west-1.amazonaws.com", + aws_credentials_expire_at: expires_at + ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true, region: 'us-west-1') } + end + + ENV["AWS_DEFAULT_REGION"] = "us-west-1" + + tests('#fetch_credentials with regional STS endpoint with region in env') do + returns( + aws_access_key_id: 'dummykey', + aws_secret_access_key: 'dummysecret', + aws_session_token: 'dummytoken', + region: 'us-west-1', + sts_endpoint: "https://sts.us-west-1.amazonaws.com", aws_credentials_expire_at: expires_at ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) } end ENV["AWS_STS_REGIONAL_ENDPOINTS"] = nil - ENV["AWS_REGION"] = nil + ENV["AWS_DEFAULT_REGION"] = nil ENV['AWS_WEB_IDENTITY_TOKEN_FILE'] = nil compute = Fog::AWS::Compute.new(use_iam_profile: true) From 1eccc9627c803c95444d287a46e0cd27a66e430e Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Fri, 20 Aug 2021 11:25:21 -0700 Subject: [PATCH 14/28] Only use STS regional endpoint if a region is specified --- lib/fog/aws/credential_fetcher.rb | 2 +- tests/credentials_tests.rb | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/fog/aws/credential_fetcher.rb b/lib/fog/aws/credential_fetcher.rb index 0ce411419..7699b865d 100644 --- a/lib/fog/aws/credential_fetcher.rb +++ b/lib/fog/aws/credential_fetcher.rb @@ -44,7 +44,7 @@ module Fog } sts_endpoint = - if ENV["AWS_STS_REGIONAL_ENDPOINTS"] == "regional" + if ENV["AWS_STS_REGIONAL_ENDPOINTS"] == "regional" && region "https://sts.#{region}.amazonaws.com" else "https://sts.amazonaws.com" diff --git a/tests/credentials_tests.rb b/tests/credentials_tests.rb index e3981769e..36f031e52 100644 --- a/tests/credentials_tests.rb +++ b/tests/credentials_tests.rb @@ -103,6 +103,17 @@ Shindo.tests('AWS | credentials', ['aws']) do ENV["AWS_STS_REGIONAL_ENDPOINTS"] = "regional" + tests('#fetch_credentials with no region specified') do + returns( + aws_access_key_id: 'dummykey', + aws_secret_access_key: 'dummysecret', + aws_session_token: 'dummytoken', + region: 'us-west-1', + sts_endpoint: "https://sts.amazonaws.com", + aws_credentials_expire_at: expires_at + ) { Fog::AWS::Compute.fetch_credentials(use_iam_profile: true) } + end + tests('#fetch_credentials with regional STS endpoint') do returns( aws_access_key_id: 'dummykey', From 07044fc331cf83358615a89fb593a6b82e862144 Mon Sep 17 00:00:00 2001 From: geemus Date: Mon, 23 Aug 2021 11:35:26 -0500 Subject: [PATCH 15/28] v3.12.0 --- lib/fog/aws/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/fog/aws/version.rb b/lib/fog/aws/version.rb index 756a3deca..04cdd30c3 100644 --- a/lib/fog/aws/version.rb +++ b/lib/fog/aws/version.rb @@ -1,5 +1,5 @@ module Fog module AWS - VERSION = "3.11.0" + VERSION = "3.12.0" end end From 63858927271d5093aac28fbb6b14186c513b7725 Mon Sep 17 00:00:00 2001 From: geemus Date: Mon, 23 Aug 2021 11:42:07 -0500 Subject: [PATCH 16/28] update changelog for v3.12.0 --- CHANGELOG.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66dcd1e1e..6373ea098 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,14 @@ # Changelog -## [Unreleased](https://github.com/fog/fog-aws/tree/HEAD) +## [v3.12.0](https://github.com/fog/fog-aws/tree/v3.12.0) (2021-08-23) -[Full Changelog](https://github.com/fog/fog-aws/compare/v3.11.0...HEAD) +[Full Changelog](https://github.com/fog/fog-aws/compare/v3.11.0...v3.12.0) + +**Merged pull requests:** + +- Add support for regional STS endpoints [\#620](https://github.com/fog/fog-aws/pull/620) ([stanhu](https://github.com/stanhu)) +- Add IPv6 support for Ingress Security Groups [\#619](https://github.com/fog/fog-aws/pull/619) ([p8](https://github.com/p8)) +- Separate CHANGELOG entry for 3.11.0 [\#618](https://github.com/fog/fog-aws/pull/618) ([sunny](https://github.com/sunny)) ## [v3.11.0](https://github.com/fog/fog-aws/tree/v3.11.0) (2021-08-05) From 2ad3fb14a52e7373870aa58380db1b49e5005430 Mon Sep 17 00:00:00 2001 From: Andrew Sullivan Cant Date: Tue, 12 Oct 2021 15:12:03 -0400 Subject: [PATCH 17/28] Add json files to the gem file This will add following IAM related JSON files back into the gem file: * lib/fog/aws/iam/default_policies.json * lib/fog/aws/iam/default_policy_versions.json These files appears to have been accidentally dropped as of v3.10.0. --- fog-aws.gemspec | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fog-aws.gemspec b/fog-aws.gemspec index 74589491a..c03d58a9b 100644 --- a/fog-aws.gemspec +++ b/fog-aws.gemspec @@ -14,8 +14,9 @@ Gem::Specification.new do |spec| spec.homepage = "https://github.com/fog/fog-aws" spec.license = "MIT" - spec.files = Dir['lib/**/*.rb', 'tests/**/*', 'CHANGELOG.md', 'CONTRIBUTING.md', - 'CONTRIBUTORS.md', 'LICENSE.md', 'README.md', 'fog-aws.gemspec',] + spec.files = Dir['lib/**/*.rb', 'lib/**/*.json', 'tests/**/*', + 'CHANGELOG.md', 'CONTRIBUTING.md', 'CONTRIBUTORS.md', + 'LICENSE.md', 'README.md', 'fog-aws.gemspec',] spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) spec.require_paths = ["lib"] From 711c6238fadece545e61974941521645387ba055 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Oct 2021 00:44:51 +0000 Subject: [PATCH 18/28] Bump actions/checkout from 2.3.4 to 2.3.5 Bumps [actions/checkout](https://github.com/actions/checkout) from 2.3.4 to 2.3.5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2.3.4...v2.3.5) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index cdafbc2ed..b5a485845 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -24,7 +24,7 @@ jobs: ruby-version: ['2.5', '2.6', '2.7', '3.0', 'head'] steps: - - uses: actions/checkout@v2.3.4 + - uses: actions/checkout@v2.3.5 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: From 866f0ce3cddb36a9bedeb832c31482c06b4e9ac3 Mon Sep 17 00:00:00 2001 From: David Fernandez Date: Fri, 29 Oct 2021 15:41:27 +0200 Subject: [PATCH 19/28] Don't use last_modified when loading the body Port of https://github.com/fog/fog-google/pull/545 where: - last_modified condition was removed - method has been rewritten to be simpler --- lib/fog/aws/models/storage/file.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/fog/aws/models/storage/file.rb b/lib/fog/aws/models/storage/file.rb index 7e1bdb1ce..5f17bd842 100644 --- a/lib/fog/aws/models/storage/file.rb +++ b/lib/fog/aws/models/storage/file.rb @@ -108,15 +108,16 @@ module Fog # @return [File] # def body - return attributes[:body] if attributes[:body] - return '' unless last_modified + return attributes[:body] if attributes.key?(:body) file = collection.get(identity) - if file - attributes[:body] = file.body - else - attributes[:body] = '' - end + + attributes[:body] = + if file + file.body + else + '' + end end # Set body attribute. From 5079dbeb09784060f6eec544b1cb7389b7435066 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Nov 2021 00:23:52 +0000 Subject: [PATCH 20/28] Bump actions/checkout from 2.3.5 to 2.4.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 2.3.5 to 2.4.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2.3.5...v2.4.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index b5a485845..68ebd4ecc 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -24,7 +24,7 @@ jobs: ruby-version: ['2.5', '2.6', '2.7', '3.0', 'head'] steps: - - uses: actions/checkout@v2.3.5 + - uses: actions/checkout@v2.4.0 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: From ff982f438954d499eed03b9aa7e93bb67a3c97e8 Mon Sep 17 00:00:00 2001 From: Igor Victor Date: Sat, 13 Nov 2021 14:06:02 +0100 Subject: [PATCH 21/28] Update ruby.yml --- .github/workflows/ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 68ebd4ecc..278fed21e 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - ruby-version: ['2.5', '2.6', '2.7', '3.0', 'head'] + ruby-version: ['2.5', '2.6', '2.7', '3.0', 'head', truffleruby-head] steps: - uses: actions/checkout@v2.4.0 From 82e14de7c61a5182a8015e1488cd2432b04f94b5 Mon Sep 17 00:00:00 2001 From: geemus Date: Wed, 17 Nov 2021 20:43:06 -0600 Subject: [PATCH 22/28] add truffle-ruby to allowed errors --- .github/workflows/ruby.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 278fed21e..533be6900 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -16,12 +16,13 @@ on: jobs: test: + continue-on-error: ${{ matrix.ruby-version == 'truffleruby-head' }} env: BUNDLER_GEMFILE: gemfiles/Gemfile-edge runs-on: ubuntu-latest strategy: matrix: - ruby-version: ['2.5', '2.6', '2.7', '3.0', 'head', truffleruby-head] + ruby-version: ['2.5', '2.6', '2.7', '3.0', 'head', 'truffleruby-head'] steps: - uses: actions/checkout@v2.4.0 From 1eec8b5acd3fbeb1e4557ee7d8d49c45eb0cfc25 Mon Sep 17 00:00:00 2001 From: Keita Urashima Date: Thu, 18 Nov 2021 19:26:51 +0900 Subject: [PATCH 23/28] Exclude test files from gem --- fog-aws.gemspec | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fog-aws.gemspec b/fog-aws.gemspec index c03d58a9b..efd3aa7c5 100644 --- a/fog-aws.gemspec +++ b/fog-aws.gemspec @@ -14,11 +14,10 @@ Gem::Specification.new do |spec| spec.homepage = "https://github.com/fog/fog-aws" spec.license = "MIT" - spec.files = Dir['lib/**/*.rb', 'lib/**/*.json', 'tests/**/*', + spec.files = Dir['lib/**/*.{rb,json}', 'CHANGELOG.md', 'CONTRIBUTING.md', 'CONTRIBUTORS.md', 'LICENSE.md', 'README.md', 'fog-aws.gemspec',] spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } - spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) spec.require_paths = ["lib"] spec.required_ruby_version = '>= 2.0.0' From db8e8a706c27aed16d7af8d81c8f7e6fa403ef89 Mon Sep 17 00:00:00 2001 From: Ewoud Kohl van Wijngaarden Date: Fri, 4 Feb 2022 14:08:22 +0100 Subject: [PATCH 24/28] Drop ipaddress dependency in favor of built in ipaddr The ipaddr module is part of Ruby itself and can be used to check for range inclusion. --- fog-aws.gemspec | 1 - .../aws/requests/compute/create_network_interface.rb | 11 +++++++---- lib/fog/aws/requests/compute/create_subnet.rb | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/fog-aws.gemspec b/fog-aws.gemspec index efd3aa7c5..72510bc3b 100644 --- a/fog-aws.gemspec +++ b/fog-aws.gemspec @@ -31,5 +31,4 @@ Gem::Specification.new do |spec| spec.add_dependency 'fog-core', '~> 2.1' spec.add_dependency 'fog-json', '~> 1.1' spec.add_dependency 'fog-xml', '~> 0.1' - spec.add_dependency 'ipaddress', '~> 0.8' end diff --git a/lib/fog/aws/requests/compute/create_network_interface.rb b/lib/fog/aws/requests/compute/create_network_interface.rb index 33e3225eb..ab7bcd9ac 100644 --- a/lib/fog/aws/requests/compute/create_network_interface.rb +++ b/lib/fog/aws/requests/compute/create_network_interface.rb @@ -2,6 +2,7 @@ module Fog module AWS class Compute class Real + require 'ipaddr' require 'fog/aws/parsers/compute/create_network_interface' # Creates a network interface @@ -68,7 +69,7 @@ module Fog raise Fog::AWS::Compute::Error.new("Unknown subnet '#{subnetId}' specified") else id = Fog::AWS::Mock.network_interface_id - cidr_block = IPAddress.parse(subnet['cidrBlock']) + cidr_block = IPAddr.new(subnet['cidrBlock']) groups = {} if options['GroupSet'] @@ -82,12 +83,14 @@ module Fog end if options['PrivateIpAddress'].nil? + range = cidr_block.to_range # Here we try to act like a DHCP server and pick the first # available IP (not including the first in the cidr block, # which is typically reserved for the gateway). - cidr_block.each_host do |p_ip| - unless self.data[:network_interfaces].map{ |ni, ni_conf| ni_conf['privateIpAddress'] }.include?p_ip.to_s || - cidr_block.first == p_ip + range = range.drop(2)[0..-2] if cidr_block.ipv4? + + range.each do |p_ip| + unless self.data[:network_interfaces].map{ |ni, ni_conf| ni_conf['privateIpAddress'] }.include?p_ip.to_s options['PrivateIpAddress'] = p_ip.to_s break end diff --git a/lib/fog/aws/requests/compute/create_subnet.rb b/lib/fog/aws/requests/compute/create_subnet.rb index d9302ba5e..074e68e72 100644 --- a/lib/fog/aws/requests/compute/create_subnet.rb +++ b/lib/fog/aws/requests/compute/create_subnet.rb @@ -2,7 +2,7 @@ module Fog module AWS class Compute class Real - require 'ipaddress' + require 'ipaddr' require 'fog/aws/parsers/compute/create_subnet' # Creates a Subnet with the CIDR block you specify. @@ -50,11 +50,11 @@ module Fog if vpc.nil? raise Fog::AWS::Compute::NotFound.new("The vpc ID '#{vpcId}' does not exist") end - if ! ::IPAddress.parse(vpc['cidrBlock']).include?(::IPAddress.parse(cidrBlock)) + if ! ::IPAddr.new(vpc['cidrBlock']).include?(::IPAddr.new(cidrBlock)) raise Fog::AWS::Compute::Error.new("Range => The CIDR '#{cidrBlock}' is invalid.") end self.data[:subnets].select{ |s| s['vpcId'] == vpcId }.each do |subnet| - if ::IPAddress.parse(subnet['cidrBlock']).include?(::IPAddress.parse(cidrBlock)) + if ::IPAddr.new(subnet['cidrBlock']).include?(::IPAddr.new(cidrBlock)) raise Fog::AWS::Compute::Error.new("Conflict => The CIDR '#{cidrBlock}' conflicts with another subnet") end end From 6ed140a1059e40dd0da50d8f3fd097380f750b97 Mon Sep 17 00:00:00 2001 From: geemus Date: Sat, 12 Feb 2022 20:24:24 -0600 Subject: [PATCH 25/28] v3.13.0 --- CHANGELOG.md | 22 +++++++++++++++++++++- lib/fog/aws/version.rb | 2 +- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6373ea098..8d47640df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [v3.13.0](https://github.com/fog/fog-aws/tree/v3.12.0) (2022-02-12) + +[Full Changelog](https://github.com/fog/fog-aws/compare/v3.12.0...v3.13.0) + +**Closed issues:** + +- Please include all files required for running tests in the gem [\#625](https://github.com/fog/fog-aws/issues/625) +- Using Hitachi compatible S3 and still see the AWS S3 host [\#624](https://github.com/fog/fog-aws/issues/624) +- Spawn compute node with Elastic Inference [\#617](https://github.com/fog/fog-aws/issues/617) + +**Merged pull requests:** + +- Drop ipaddress dependency in favor of built in ipaddr [\#630](https://github.com/fog/fog-aws/pull/630) ([ekohl](https://github.com/ekohl)) +- Exclude test files from gem [\#629](https://github.com/fog/fog-aws/pull/629) ([ursm](https://github.com/ursm)) +- Add Truffleruby head to CI [\#628](https://github.com/fog/fog-aws/pull/628) ([gogainda](https://github.com/gogainda)) +- Bump actions/checkout from 2.3.5 to 2.4.0 [\#627](https://github.com/fog/fog-aws/pull/627) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Update Fog::AWS::Storage::File\#body [\#626](https://github.com/fog/fog-aws/pull/626) ([10io](https://github.com/10io)) +- Bump actions/checkout from 2.3.4 to 2.3.5 [\#623](https://github.com/fog/fog-aws/pull/623) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Add json files to the gem file [\#622](https://github.com/fog/fog-aws/pull/622) ([acant](https://github.com/acant)) + ## [v3.12.0](https://github.com/fog/fog-aws/tree/v3.12.0) (2021-08-23) [Full Changelog](https://github.com/fog/fog-aws/compare/v3.11.0...v3.12.0) @@ -146,7 +166,7 @@ **Merged pull requests:** -- Handle multiple versions of objects in multiple delete request [\#600](https://github.com/fog/fog-aws/pull/600) ([shanu-idrive](https://github.com/shanu-idrive)) +- Handle multiple versions of objects in multiple delete request [\#600](https://github.com/fog/fog-aws/pull/600) ([shanu-kr](https://github.com/shanu-kr)) - Add Truffleruby head to CI [\#596](https://github.com/fog/fog-aws/pull/596) ([gogainda](https://github.com/gogainda)) - Fixes domain name duplication in url [\#593](https://github.com/fog/fog-aws/pull/593) ([midhunkrishna](https://github.com/midhunkrishna)) diff --git a/lib/fog/aws/version.rb b/lib/fog/aws/version.rb index 04cdd30c3..ae426c9d8 100644 --- a/lib/fog/aws/version.rb +++ b/lib/fog/aws/version.rb @@ -1,5 +1,5 @@ module Fog module AWS - VERSION = "3.12.0" + VERSION = "3.13.0" end end From 2a231cb8eff78364607dc5c64728963af0e7061e Mon Sep 17 00:00:00 2001 From: Peter Goldstein Date: Sun, 13 Feb 2022 07:49:39 -0800 Subject: [PATCH 26/28] Add Ruby 3.1 to the CI matrix Also removes an unneeded bundle install step --- .github/workflows/ruby.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 533be6900..bf3de5768 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - ruby-version: ['2.5', '2.6', '2.7', '3.0', 'head', 'truffleruby-head'] + ruby-version: ['2.5', '2.6', '2.7', '3.0', '3.1', 'head', 'truffleruby-head'] steps: - uses: actions/checkout@v2.4.0 @@ -31,7 +31,5 @@ jobs: with: ruby-version: ${{ matrix.ruby-version }} bundler-cache: true # runs 'bundle install' and caches installed gems automatically - - name: Install dependencies - run: bundle install - name: Run tests run: bundle exec rake From 0491d78681ba537e7b02bc143be4ac0f6420160f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Mar 2022 00:23:56 +0000 Subject: [PATCH 27/28] Bump actions/checkout from 2.4.0 to 3 Bumps [actions/checkout](https://github.com/actions/checkout) from 2.4.0 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2.4.0...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ruby.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index bf3de5768..5f84887ae 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -25,7 +25,7 @@ jobs: ruby-version: ['2.5', '2.6', '2.7', '3.0', '3.1', 'head', 'truffleruby-head'] steps: - - uses: actions/checkout@v2.4.0 + - uses: actions/checkout@v3 - name: Set up Ruby uses: ruby/setup-ruby@v1 with: From 3124da9cb7462ab1c5bc2870b0684eb7f9ae5da5 Mon Sep 17 00:00:00 2001 From: Orrin Naylor Date: Thu, 3 Mar 2022 22:02:14 -0700 Subject: [PATCH 28/28] Add warning messages around directories.get --- README.md | 1 + lib/fog/aws/models/storage/directories.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 13865c266..c7b571f44 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,7 @@ file = directory.files.create(key: 'user/1/Gemfile', body: File.open('Gemfile'), directory = s3.directories.get('gaudi-portal-dev', prefix: 'user/1/') directory.files ``` +**Warning!** `s3.directories.get` retrieves and caches meta data for the first 10,000 objects in the bucket, which can be very expensive. When possible use `s3.directories.new`. #### Generating a URL for a file: diff --git a/lib/fog/aws/models/storage/directories.rb b/lib/fog/aws/models/storage/directories.rb index 64341509a..43c9741d5 100644 --- a/lib/fog/aws/models/storage/directories.rb +++ b/lib/fog/aws/models/storage/directories.rb @@ -11,6 +11,7 @@ module Fog load(data) end + # Warning! This retrieves and caches meta data for the first 10,000 objects in the bucket, which can be very expensive. When possible use directories.new def get(key, options = {}) remap_attributes(options, { :delimiter => 'delimiter',