Skip to content

Commit a7a4a00

Browse files
Ankk98Ankk98
authored
Fix #741 Added list objects v2 api support (#744)
* Added support for ListObjectsV2 #741 * Fix #741 Add tests for ListObjectsV2 API in AWS storage module - Introduced comprehensive tests for the ListObjectsV2 API, covering basic functionality, parameter handling (prefix, max-keys, start-after, delimiter, fetch-owner), and pagination with continuation tokens. - Created separate test files for both model and request layers to ensure thorough validation of the API's behavior and response structure. * Update ListObjectsV2 response structure to include unique common prefixes in KeyCount calculation - Modified the response body to ensure that the 'KeyCount' accurately reflects the total number of objects and unique common prefixes. - Adjusted the handling of 'CommonPrefixes' to use a unique set, improving the integrity of the response data.
1 parent d796c0e commit a7a4a00

File tree

5 files changed

+509
-0
lines changed

5 files changed

+509
-0
lines changed

‎lib/fog/aws/parsers/storage/list_objects_v2.rb

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
module Fog
2+
module Parsers
3+
module AWS
4+
module Storage
5+
class ListObjectsV2 < Fog::Parsers::Base
6+
# Initialize parser state
7+
def initialize
8+
super
9+
@common_prefix = {}
10+
@object = { 'Owner' => {} }
11+
reset
12+
end
13+
14+
def reset
15+
@object = { 'Owner' => {} }
16+
@response = { 'Contents' => [], 'CommonPrefixes' => [] }
17+
end
18+
19+
def start_element(name, attrs = [])
20+
super
21+
case name
22+
when 'CommonPrefixes'
23+
@in_common_prefixes = true
24+
end
25+
end
26+
27+
def end_element(name)
28+
case name
29+
when 'CommonPrefixes'
30+
@in_common_prefixes = false
31+
when 'Contents'
32+
@response['Contents'] << @object
33+
@object = { 'Owner' => {} }
34+
when 'DisplayName', 'ID'
35+
@object['Owner'][name] = value
36+
when 'ETag'
37+
@object[name] = value.gsub('"', '') if value != nil
38+
when 'IsTruncated'
39+
if value == 'true'
40+
@response['IsTruncated'] = true
41+
else
42+
@response['IsTruncated'] = false
43+
end
44+
when 'LastModified'
45+
@object['LastModified'] = Time.parse(value)
46+
when 'ContinuationToken', 'NextContinuationToken', 'Name', 'StartAfter'
47+
@response[name] = value
48+
when 'MaxKeys', 'KeyCount'
49+
@response[name] = value.to_i
50+
when 'Prefix'
51+
if @in_common_prefixes
52+
@response['CommonPrefixes'] << value
53+
else
54+
@response[name] = value
55+
end
56+
when 'Size'
57+
@object['Size'] = value.to_i
58+
when 'Delimiter', 'Key', 'StorageClass'
59+
@object[name] = value
60+
end
61+
end
62+
end
63+
end
64+
end
65+
end
66+
end

‎lib/fog/aws/requests/storage/list_objects_v2.rb

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
module Fog
2+
module AWS
3+
class Storage
4+
class Real
5+
require 'fog/aws/parsers/storage/list_objects_v2'
6+
7+
# List information about objects in an S3 bucket using ListObjectsV2
8+
#
9+
# @param bucket_name [String] name of bucket to list object keys from
10+
# @param options [Hash] config arguments for list. Defaults to {}.
11+
# @option options delimiter [String] causes keys with the same string between the prefix
12+
# value and the first occurrence of delimiter to be rolled up
13+
# @option options continuation-token [String] continuation token from a previous request
14+
# @option options fetch-owner [Boolean] specifies whether to return owner information
15+
# @option options max-keys [Integer] limits number of object keys returned
16+
# @option options prefix [String] limits object keys to those beginning with its value
17+
# @option options start-after [String] starts listing after this specified key
18+
#
19+
# @return [Excon::Response] response:
20+
# * body [Hash]:
21+
# * Delimiter [String] - Delimiter specified for query
22+
# * IsTruncated [Boolean] - Whether or not the listing is truncated
23+
# * ContinuationToken [String] - Token specified in the request
24+
# * NextContinuationToken [String] - Token to use in subsequent requests
25+
# * KeyCount [Integer] - Number of keys returned
26+
# * MaxKeys [Integer] - Maximum number of keys specified for query
27+
# * Name [String] - Name of the bucket
28+
# * Prefix [String] - Prefix specified for query
29+
# * StartAfter [String] - StartAfter specified in the request
30+
# * CommonPrefixes [Array] - Array of strings for common prefixes
31+
# * Contents [Array]:
32+
# * ETag [String] - Etag of object
33+
# * Key [String] - Name of object
34+
# * LastModified [String] - Timestamp of last modification of object
35+
# * Owner [Hash]:
36+
# * DisplayName [String] - Display name of object owner
37+
# * ID [String] - Id of object owner
38+
# * Size [Integer] - Size of object
39+
# * StorageClass [String] - Storage class of object
40+
#
41+
# @see https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
42+
43+
def list_objects_v2(bucket_name, options = {})
44+
unless bucket_name
45+
raise ArgumentError.new('bucket_name is required')
46+
end
47+
48+
# Add list-type=2 to indicate ListObjectsV2
49+
options = options.merge('list-type' => '2')
50+
51+
request({
52+
:expects => 200,
53+
:headers => {},
54+
:bucket_name => bucket_name,
55+
:idempotent => true,
56+
:method => 'GET',
57+
:parser => Fog::Parsers::AWS::Storage::ListObjectsV2.new,
58+
:query => options
59+
})
60+
end
61+
end
62+
63+
class Mock # :nodoc:all
64+
def list_objects_v2(bucket_name, options = {})
65+
prefix = options['prefix']
66+
continuation_token = options['continuation-token']
67+
delimiter = options['delimiter']
68+
max_keys = options['max-keys']
69+
start_after = options['start-after']
70+
fetch_owner = options['fetch-owner']
71+
common_prefixes = []
72+
73+
unless bucket_name
74+
raise ArgumentError.new('bucket_name is required')
75+
end
76+
77+
response = Excon::Response.new
78+
if bucket = self.data[:buckets][bucket_name]
79+
contents = bucket[:objects].values.map(&:first).sort {|x,y| x['Key'] <=> y['Key']}.reject do |object|
80+
(prefix && object['Key'][0...prefix.length] != prefix) ||
81+
(start_after && object['Key'] <= start_after) ||
82+
(continuation_token && object['Key'] <= continuation_token) ||
83+
(delimiter && object['Key'][(prefix ? prefix.length : 0)..-1].include?(delimiter) \
84+
&& common_prefixes << object['Key'].sub(/^(#{prefix}[^#{delimiter}]+.).*/, '\1')) ||
85+
object.key?(:delete_marker)
86+
end.map do |object|
87+
data = object.reject {|key, value| !['ETag', 'Key', 'StorageClass'].include?(key)}
88+
data.merge!({
89+
'LastModified' => Time.parse(object['Last-Modified']),
90+
'Owner' => fetch_owner ? bucket['Owner'] : nil,
91+
'Size' => object['Content-Length'].to_i
92+
})
93+
data
94+
end
95+
96+
max_keys = max_keys || 1000
97+
size = [max_keys, 1000].min
98+
truncated_contents = contents[0...size]
99+
next_token = truncated_contents.size != contents.size ? truncated_contents.last['Key'] : nil
100+
101+
response.status = 200
102+
common_prefixes_uniq = common_prefixes.uniq
103+
response.body = {
104+
'CommonPrefixes' => common_prefixes_uniq,
105+
'Contents' => truncated_contents,
106+
'IsTruncated' => truncated_contents.size != contents.size,
107+
'ContinuationToken' => continuation_token,
108+
'NextContinuationToken' => next_token,
109+
'KeyCount' => truncated_contents.size + common_prefixes_uniq.size,
110+
'MaxKeys' => max_keys,
111+
'Name' => bucket['Name'],
112+
'Prefix' => prefix,
113+
'StartAfter' => start_after
114+
}
115+
if max_keys && max_keys < response.body['Contents'].length
116+
response.body['IsTruncated'] = true
117+
response.body['Contents'] = response.body['Contents'][0...max_keys]
118+
response.body['KeyCount'] = response.body['Contents'].size + response.body['CommonPrefixes'].size
119+
end
120+
else
121+
response.status = 404
122+
raise(Excon::Errors.status_error({:expects => 200}, response))
123+
end
124+
response
125+
end
126+
end
127+
end
128+
end
129+
end

‎lib/fog/aws/storage.rb

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,12 @@ class Storage < Fog::Service
2626

2727
VALID_QUERY_KEYS = %w[
2828
acl
29+
continuation-token
2930
cors
3031
delete
32+
fetch-owner
3133
lifecycle
34+
list-type
3235
location
3336
logging
3437
notification
@@ -42,6 +45,7 @@ class Storage < Fog::Service
4245
response-content-type
4346
response-expires
4447
restore
48+
start-after
4549
tagging
4650
torrent
4751
uploadId
@@ -102,6 +106,7 @@ class Storage < Fog::Service
102106
request :head_object_url
103107
request :initiate_multipart_upload
104108
request :list_multipart_uploads
109+
request :list_objects_v2
105110
request :list_parts
106111
request :post_object_hidden_fields
107112
request :post_object_restore

‎tests/models/storage/files_v2_tests.rb

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
Shindo.tests("Storage[:aws] | ListObjectsV2 API", ["aws"]) do
2+
3+
directory_attributes = {
4+
:key => uniq_id('foglistobjectsv2tests')
5+
}
6+
7+
@directory = Fog::Storage[:aws].directories.create(directory_attributes)
8+
9+
tests('Direct ListObjectsV2 API usage') do
10+
11+
# Create some test files
12+
file1 = @directory.files.create(:body => 'test1', :key => 'prefix/file1.txt')
13+
file2 = @directory.files.create(:body => 'test2', :key => 'prefix/file2.txt')
14+
file3 = @directory.files.create(:body => 'test3', :key => 'other/file3.txt')
15+
file4 = @directory.files.create(:body => 'test4', :key => 'file4.txt')
16+
17+
tests('#list_objects_v2 basic functionality') do
18+
response = Fog::Storage[:aws].list_objects_v2(@directory.key)
19+
20+
tests('returns proper response structure').returns(true) do
21+
response.body.has_key?('Contents') &&
22+
response.body.has_key?('KeyCount') &&
23+
response.body.has_key?('IsTruncated')
24+
end
25+
26+
tests('returns all files').returns(4) do
27+
response.body['Contents'].size
28+
end
29+
30+
tests('has V2-specific KeyCount').returns(4) do
31+
response.body['KeyCount']
32+
end
33+
end
34+
35+
tests('#list_objects_v2 with parameters') do
36+
37+
tests('with prefix') do
38+
response = Fog::Storage[:aws].list_objects_v2(@directory.key, 'prefix' => 'prefix/')
39+
40+
tests('filters by prefix').returns(2) do
41+
response.body['Contents'].size
42+
end
43+
44+
tests('KeyCount reflects filtered results').returns(2) do
45+
response.body['KeyCount']
46+
end
47+
end
48+
49+
tests('with max-keys') do
50+
response = Fog::Storage[:aws].list_objects_v2(@directory.key, 'max-keys' => 2)
51+
52+
tests('limits results').returns(2) do
53+
response.body['Contents'].size
54+
end
55+
56+
tests('is truncated').returns(true) do
57+
response.body['IsTruncated']
58+
end
59+
60+
tests('has next continuation token').returns(true) do
61+
!response.body['NextContinuationToken'].nil?
62+
end
63+
end
64+
65+
tests('with start-after') do
66+
response = Fog::Storage[:aws].list_objects_v2(@directory.key, 'start-after' => 'other/file3.txt')
67+
68+
tests('starts after specified key').returns(true) do
69+
keys = response.body['Contents'].map { |obj| obj['Key'] }
70+
keys.none? { |key| key <= 'other/file3.txt' }
71+
end
72+
end
73+
74+
tests('with delimiter') do
75+
response = Fog::Storage[:aws].list_objects_v2(@directory.key, 'delimiter' => '/')
76+
77+
tests('respects delimiter').returns(true) do
78+
# Should have common prefixes and fewer direct contents
79+
response.body.has_key?('CommonPrefixes') && response.body['CommonPrefixes'].size > 0
80+
end
81+
end
82+
83+
tests('with fetch-owner') do
84+
response = Fog::Storage[:aws].list_objects_v2(@directory.key, 'fetch-owner' => true)
85+
86+
tests('request succeeds').returns(true) do
87+
response.body.has_key?('Contents')
88+
end
89+
end unless Fog.mocking?
90+
91+
end
92+
93+
tests('pagination with continuation token') do
94+
first_page = Fog::Storage[:aws].list_objects_v2(@directory.key, 'max-keys' => 2)
95+
96+
if first_page.body['IsTruncated'] && first_page.body['NextContinuationToken']
97+
second_page = Fog::Storage[:aws].list_objects_v2(@directory.key, 'continuation-token' => first_page.body['NextContinuationToken'])
98+
99+
tests('second page has different objects').returns(true) do
100+
first_keys = first_page.body['Contents'].map { |obj| obj['Key'] }
101+
second_keys = second_page.body['Contents'].map { |obj| obj['Key'] }
102+
(first_keys & second_keys).empty?
103+
end
104+
end
105+
end
106+
107+
# Clean up test files
108+
file1.destroy
109+
file2.destroy
110+
file3.destroy
111+
file4.destroy
112+
end
113+
114+
@directory.destroy
115+
116+
end

0 commit comments

Comments
 (0)