Class: Longleaf::S3StorageLocation

Inherits:
StorageLocation show all
Includes:
Logging
Defined in:
lib/longleaf/models/s3_storage_location.rb

Overview

A storage location in a s3 bucket

Optionally, the location configuration may include an “options” sub-hash in order to provide any of the s3 client options specified in Client initializer: docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Client.html#constructor_details

Constant Summary collapse

IS_URI_REGEX = 
/\A#{URI::regexp}\z/
CLIENT_OPTIONS_FIELD = 
'options'

Instance Attribute Summary

Attributes inherited from StorageLocation

#metadata_location, #name, #path

Instance Method Summary collapse

Methods included from Logging

#initialize_logger, initialize_logger, logger, #logger

Methods inherited from StorageLocation

#contains?, #get_metadata_path_for

Constructor Details

#initialize(name, config, md_loc) ⇒ S3StorageLocation

Returns a new instance of S3StorageLocation.

Parameters:

  • name (String) —

    the name of this storage location

  • config (Hash) —

    hash containing the configuration options for this location

  • md_loc (MetadataLocation) —

    metadata location associated with this storage location



25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/longleaf/models/s3_storage_location.rb', line 25

def initialize(name, config, md_loc)
  super(name, config, md_loc)

  @bucket_name = S3UriHelper.extract_bucket(@path)
  if @bucket_name.nil?
    raise ArgumentError.new("Unable to identify bucket for location #{@name} from path #{@path}")
  end

  # Force path to always end with a slash
  @path += '/' unless @path.end_with?('/')

  custom_options = config[CLIENT_OPTIONS_FIELD]
  if custom_options.nil?
    @client_options = Hash.new
  else
    # Clone options and convert keys to symbols
    @client_options = Hash[custom_options.map { |(k,v)| [k.to_sym,v] } ]
  end
  @client_options[:logger] = logger
  @client_options[:log_level] = :debug if @client_options[:log_level].nil?
  
  # If no region directly configured, use region from path
  if !@client_options.key?(:region)
    region = S3UriHelper.extract_region(@path)
    @client_options[:region] = region unless region.nil?
  end
  
  @subpath_prefix = S3UriHelper.extract_path(@path)
end

Instance Method Details

#available? ⇒ Boolean

Checks that the path and metadata path defined in this location are available

Returns:

  • (Boolean)

Raises:



74
75
76
77
78
79
80
81
82
 
# File 'lib/longleaf/models/s3_storage_location.rb', line 74

def available?
  begin
    s3_client().head_bucket({ bucket: @bucket_name, use_accelerate_endpoint: false })
  rescue StandardError => e
    raise StorageLocationUnavailableError.new("Destination bucket #{@bucket_name} does not exist " \
        + "or is not accessible: #{e.message}")
  end
  @metadata_location.available?
end

#get_path_from_metadata_path(md_path) ⇒ String

Get that absolute path to the file associated with the provided metadata path

Parameters:

  • md_path (String) —

    metadata file path

Returns:

  • (String) —

    the path for the file associated with this metadata

Raises:

  • (ArgumentError) —

    if the md_path is not in this storage location



64
65
66
67
68
69
70
 
# File 'lib/longleaf/models/s3_storage_location.rb', line 64

def get_path_from_metadata_path(md_path)
  raise ArgumentError.new("A file_path parameter is required") if md_path.nil? || md_path.empty?

  rel_path = @metadata_location.relative_file_path_for(md_path)

  URI.join(@path, rel_path).to_s
end

#relative_to_bucket_path(rel_path) ⇒ Object

Prefixes the provided path with the query path portion of the location’s path after the bucket uri, used to place relative paths into the same sub-URL of a bucket. For example: Given a location with ‘path’ example.s3-amazonaws.com/env/test/ Where rel_path = ‘path/to/text.txt’ The result would be ‘env/test/path/to/text.txt’

Parameters:

  • rel_path —

    relative path to work with

Returns:

  • the given relative path prefixed with the path portion of the storage location path

Raises:

  • (ArgumentError) 


111
112
113
114
115
116
117
118
119
 
# File 'lib/longleaf/models/s3_storage_location.rb', line 111

def relative_to_bucket_path(rel_path)
  raise ArgumentError.new("Must provide a non-nil path") if rel_path.nil?
  
  if @subpath_prefix.nil?
    return rel_path
  end
  
  @subpath_prefix + rel_path
end

#relativize(file_path) ⇒ Object

Get the file path relative to this location

Parameters:

  • file_path (String) —

    file path

Returns:

  • the file path relative to this location

Raises:

  • (ArgumentError) —

    if the file path is not contained by this location



88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/longleaf/models/s3_storage_location.rb', line 88

def relativize(file_path)
  raise ArgumentError.new("Must provide a non-nil path to relativize") if file_path.nil?

  if file_path.start_with?(@path)
    file_path[@path.length..-1]
  else
    if file_path =~ IS_URI_REGEX
      raise ArgumentError.new("Path #{file_path} is not contained by #{@name}")
    else
      # path already relative
      file_path
    end
  end
end

#s3_bucket ⇒ Object

Returns the bucket used by this storage location.

Returns:

  • the bucket used by this storage location



122
123
124
125
126
127
128
 
# File 'lib/longleaf/models/s3_storage_location.rb', line 122

def s3_bucket
  if @bucket.nil?
    @s3 = Aws::S3::Resource.new(client: s3_client())
    @bucket = @s3.bucket(@bucket_name)
  end
  @bucket
end

#s3_client ⇒ Object

Returns the s3 client used by this storage locatio.

Returns:

  • the s3 client used by this storage locatio



131
132
133
134
135
136
 
# File 'lib/longleaf/models/s3_storage_location.rb', line 131

def s3_client
  if @client.nil?
    @client = Aws::S3::Client.new(**@client_options)
  end
  @client
end

#type ⇒ Object

Returns the storage type for this location.

Returns:

  • the storage type for this location



56
57
58
 
# File 'lib/longleaf/models/s3_storage_location.rb', line 56

def type
  StorageTypes::S3_STORAGE_TYPE
end