Class: Bricolage::S3DataSource

Inherits:
DataSource show all
Defined in:
lib/bricolage/psqldatasource.rb,
lib/bricolage/s3datasource.rb

Overview

reopen

Constant Summary

Constants inherited from DataSource

DataSource::CLASSES

Instance Attribute Summary collapse

Attributes inherited from DataSource

#context, #logger, #name

Instance Method Summary collapse

Methods inherited from DataSource

get_class, new_for_type, #open, #open_for_batch

Constructor Details

#initialize(endpoint: 's3-ap-northeast-1.amazonaws.com', region: 'ap-northeast-1', bucket: nil, prefix: nil, access_key_id: nil, secret_access_key: nil, iam_role: nil, master_symmetric_key: nil, encryption: nil, s3cfg: nil) ⇒ S3DataSource

Returns a new instance of S3DataSource.



12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
 
# File 'lib/bricolage/s3datasource.rb', line 12

def initialize(
    endpoint: 's3-ap-northeast-1.amazonaws.com',
    region: 'ap-northeast-1',
    bucket: nil,
    prefix: nil,
    access_key_id: nil,
    secret_access_key: nil,
    iam_role: nil,
    master_symmetric_key: nil,
    encryption: nil,
    s3cfg: nil
)
  @endpoint = (/\Ahttps?:/ =~ endpoint) ? endpoint : "https://#{endpoint}"
  @region = region
  @bucket_name = bucket
  @prefix = (prefix && prefix.empty?) ? nil : prefix
  @access_key_id = access_key_id
  @secret_access_key = secret_access_key
  @iam_role = iam_role
  @master_symmetric_key = master_symmetric_key
  @encryption = encryption
  @s3cfg_path = s3cfg
  @s3cfg = @s3cfg_path ? load_s3cfg(@s3cfg_path) : nil
end

Instance Attribute Details

#bucket_name ⇒ Object (readonly)

Returns the value of attribute bucket_name.



39
40
41
 
# File 'lib/bricolage/s3datasource.rb', line 39

def bucket_name
  @bucket_name
end

#encryption ⇒ Object (readonly)

Returns the value of attribute encryption.



95
96
97
 
# File 'lib/bricolage/s3datasource.rb', line 95

def encryption
  @encryption
end

#endpoint ⇒ Object (readonly)

Returns the value of attribute endpoint.



37
38
39
 
# File 'lib/bricolage/s3datasource.rb', line 37

def endpoint
  @endpoint
end

#iam_role ⇒ Object (readonly)

Redshift attached IAM role ARN



73
74
75
 
# File 'lib/bricolage/s3datasource.rb', line 73

def iam_role
  @iam_role
end

#prefix ⇒ Object (readonly)

Returns the value of attribute prefix.



40
41
42
 
# File 'lib/bricolage/s3datasource.rb', line 40

def prefix
  @prefix
end

#region ⇒ Object (readonly)

Returns the value of attribute region.



38
39
40
 
# File 'lib/bricolage/s3datasource.rb', line 38

def region
  @region
end

Instance Method Details

#access_key ⇒ Object

AWS access key ID. This property may be nil, we can use EC2 instance or ECS task attached IAM role in that case.



63
64
65
 
# File 'lib/bricolage/s3datasource.rb', line 63

def access_key
  @access_key_id || get_s3cfg('access_key')
end

#bucket ⇒ Object 



109
110
111
112
 
# File 'lib/bricolage/s3datasource.rb', line 109

def bucket
  @resource ||= Aws::S3::Resource.new(client: client)
  @bucket ||= @resource.bucket(@bucket_name)
end

#client ⇒ Object

Ruby Interface



105
106
107
 
# File 'lib/bricolage/s3datasource.rb', line 105

def client
  @client ||= Aws::S3::Client.new(region: @region, endpoint: @endpoint, access_key_id: access_key, secret_access_key: secret_key)
end

#credential_string ⇒ Object

For Redshift COPY/UNLOAD



47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/bricolage/s3datasource.rb', line 47

def credential_string
  if @iam_role
    "aws_iam_role=#{@iam_role}"
  elsif access_key
    [
      "aws_access_key_id=#{access_key}",
      "aws_secret_access_key=#{secret_key}",
      (@master_symmetric_key && "master_symmetric_key=#{@master_symmetric_key}")
    ].compact.join(';')
  else
    raise ParameterError, "[s3:#{@bucket_name}] credential string requested but no credentials exist"
  end
end

#encrypted? ⇒ Boolean

Returns:

  • (Boolean) 


97
98
99
 
# File 'lib/bricolage/s3datasource.rb', line 97

def encrypted?
  !!(@master_symmetric_key or @encryption)
end

#new_task ⇒ Object 



42
43
44
 
# File 'lib/bricolage/s3datasource.rb', line 42

def new_task
  S3Task.new(self)
end

#object(rel, no_prefix: false) ⇒ Object 



114
115
116
 
# File 'lib/bricolage/s3datasource.rb', line 114

def object(rel, no_prefix: false)
  bucket.object(path(rel, no_prefix: no_prefix))
end

#path(rel, no_prefix: false) ⇒ Object 



122
123
124
125
 
# File 'lib/bricolage/s3datasource.rb', line 122

def path(rel, no_prefix: false)
  path = (no_prefix || !@prefix) ? rel.to_s : "#{@prefix}/#{rel}"
  path.sub(%r<\A/>, '').gsub(%r<//>, '/')
end

#redshift_loader_source? ⇒ Boolean

Returns:

  • (Boolean) 


157
158
159
 
# File 'lib/bricolage/psqldatasource.rb', line 157

def redshift_loader_source?
  true
end

#secret_key ⇒ Object

AWS secret access key.



68
69
70
 
# File 'lib/bricolage/s3datasource.rb', line 68

def secret_key
  @secret_access_key || get_s3cfg('secret_key')
end

#traverse(rel, no_prefix: false) ⇒ Object 



127
128
129
130
131
132
133
134
135
136
137
138
139
140
 
# File 'lib/bricolage/s3datasource.rb', line 127

def traverse(rel, no_prefix: false)
  retries = client.config.retry_limit
  begin
    bucket.objects(prefix: path(rel, no_prefix: no_prefix)).to_a
  rescue Aws::Xml::Parser::ParsingError => e
    retries -= 1
    if retries >= 0
      logger.warn "Retry Bucket#objects() for XML parsing error: #{e.message}"
      sleep 1
      retry
    end
    raise
  end
end

#url(rel, no_prefix: false) ⇒ Object 



118
119
120
 
# File 'lib/bricolage/s3datasource.rb', line 118

def url(rel, no_prefix: false)
  "s3://#{@bucket_name}/#{path(rel, no_prefix: no_prefix)}"
end