Class: Aspera::Api::Node

Inherits:
Rest
  • Object
show all
Defined in:
lib/aspera/api/node.rb

Overview

Provides additional functions using node API with gen4 extensions (access keys)

Direct Known Subclasses

CosNode

Constant Summary collapse

ACCESS_LEVELS =

node api permissions

%w[delete list mkdir preview read rename write].freeze
HEADER_X_ASPERA_ACCESS_KEY = 
'X-Aspera-AccessKey'
HEADER_X_TOTAL_COUNT = 
'X-Total-Count'
HEADER_X_CACHE_CONTROL = 
'X-Aspera-Cache-Control'
HEADER_X_NEXT_ITER_TOKEN = 
'X-Aspera-Next-Iteration-Token'
SCOPE_USER = 
'user:all'
SCOPE_ADMIN = 
'admin:all'
PATH_SEPARATOR = 
'/'

Constants inherited from Rest

Rest::ENTITY_NOT_FOUND, Rest::JSON_DECODE

Class Attribute Summary collapse

Instance Attribute Summary collapse

Attributes inherited from Rest

#auth_params, #base_url

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Rest

array_params, array_params?, basic_authorization, build_uri, #call, #cancel, #create, #delete, io_http_session, #lookup_by_name, #oauth, #params, parse_header, query_to_h, #read, remote_certificate_chain, start_http_session, #update

Constructor Details

#initialize(app_info: nil, add_tspec: nil, **rest_args) ⇒ Node

Returns a new instance of Node.

Parameters:

  • app_info (Hash, NilClass) (defaults to: nil) —

    Special processing for AoC

  • add_tspec (Hash, NilClass) (defaults to: nil) —

    Additional transfer spec

  • base_url (String) —

    Rest parameters

  • auth (String, NilClass) —

    Rest parameters

  • headers (String, NilClass) —

    Rest parameters


133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
 
# File 'lib/aspera/api/node.rb', line 133

def initialize(app_info: nil, add_tspec: nil, **rest_args)
  # init Rest
  super(**rest_args)
  @app_info = app_info
  # this is added to transfer spec, for instance to add tags (COS)
  @add_tspec = add_tspec
  @std_t_spec_cache = nil
  if !@app_info.nil?
    REQUIRED_APP_INFO_FIELDS.each do |field|
      Aspera.assert(@app_info.key?(field)){"app_info lacks field #{field}"}
    end
    REQUIRED_APP_API_METHODS.each do |method|
      Aspera.assert(@app_info[:api].respond_to?(method)){"#{@app_info[:api].class} lacks method #{method}"}
    end
  end
end

Class Attribute Details

.use_node_cache ⇒ Object

set to false to bypass cache in redis


51
52
53
 
# File 'lib/aspera/api/node.rb', line 51

def use_node_cache
  @use_node_cache
end

.use_standard_ports ⇒ Object

set to false to read transfer parameters from download_setup


49
50
51
 
# File 'lib/aspera/api/node.rb', line 49

def use_standard_ports
  @use_standard_ports
end

Instance Attribute Details

#app_info ⇒ Object (readonly)

Returns the value of attribute app_info.


126
127
128
 
# File 'lib/aspera/api/node.rb', line 126

def app_info
  @app_info
end

Class Method Details

.bearer_headers(bearer_auth, access_key: nil) ⇒ Object 


113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/aspera/api/node.rb', line 113

def bearer_headers(bearer_auth, access_key: nil)
  # if username is not provided, use the access key from the token
  if access_key.nil?
    access_key = Node.decode_scope(Node.decode_bearer_token(OAuth::Factory.bearer_extract(bearer_auth))['scope'])[:access_key]
    Aspera.assert(!access_key.nil?)
  end
  return {
    Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
    'Authorization'                  => bearer_auth
  }
end

.bearer_token(access_key:, payload:, private_key:) ⇒ Object

Create an Aspera Node bearer token

Parameters:

  • payload (String) —

    JSON payload to be included in the token

  • private_key (OpenSSL::PKey::RSA) —

    Private key to sign the token


86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/aspera/api/node.rb', line 86

def bearer_token(access_key:, payload:, private_key:)
  Aspera.assert_type(payload, Hash)
  Aspera.assert(payload.key?('user_id'))
  Aspera.assert_type(payload['user_id'], String)
  Aspera.assert(!payload['user_id'].empty?)
  Aspera.assert_type(private_key, OpenSSL::PKey::RSA)
  # manage convenience parameters
  expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
  payload.delete('_validity')
  scope = payload['_scope'] || SCOPE_USER
  payload.delete('_scope')
  payload['scope'] ||= token_scope(access_key, scope)
  payload['auth_type'] ||= 'access_key'
  payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
  payload_json = JSON.generate(payload)
  return Base64.strict_encode64(Zlib::Deflate.deflate([
    payload_json,
    SIGNATURE_DELIMITER,
    Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
    ''
  ].join("\n")))
end

.decode_bearer_token(token) ⇒ Object 


109
110
111
 
# File 'lib/aspera/api/node.rb', line 109

def decode_bearer_token(token)
  return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
end

.decode_scope(scope) ⇒ Object 


76
77
78
79
80
81
 
# File 'lib/aspera/api/node.rb', line 76

def decode_scope(scope)
  items = scope.split(SCOPE_SEPARATOR, 2)
  Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
  Aspera.assert(items[0].start_with?(SCOPE_NODE_PREFIX)){"invalid scope: #{scope}"}
  return {access_key: items[0][SCOPE_NODE_PREFIX.length..-1], scope: items[1]}
end

.file_matcher(match_expression) ⇒ Object

For access keys: provide expression to match entry in folder

Parameters:

  • match_expression —

    one of supported types

Returns:

  • lambda function


56
57
58
59
60
61
62
63
64
65
 
# File 'lib/aspera/api/node.rb', line 56

def file_matcher(match_expression)
  case match_expression
  when Proc then return match_expression
  when Regexp then return ->(f){f['name'].match?(match_expression)}
  when String
    return ->(f){File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
  when NilClass then return ->(_){true}
  else Aspera.error_unexpected_value(match_expression.class.name, exception_class: Cli::BadArgument)
  end
end

.file_matcher_from_argument(options) ⇒ Object 


67
68
69
 
# File 'lib/aspera/api/node.rb', line 67

def file_matcher_from_argument(options)
  return file_matcher(options.get_next_argument('filter', validation: MATCH_TYPES, mandatory: false))
end

.token_scope(access_key, scope) ⇒ Object

node API scopes


72
73
74
 
# File 'lib/aspera/api/node.rb', line 72

def token_scope(access_key, scope)
  return [SCOPE_NODE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
end

Instance Method Details

#add_tspec_info(tspec) ⇒ Object

update transfer spec with special additional tags


162
163
164
165
 
# File 'lib/aspera/api/node.rb', line 162

def add_tspec_info(tspec)
  tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
  return tspec
end

#entry_has_link_information(entry) ⇒ Boolean

Check if a link entry in folder has target information

Parameters:

  • entry (Hash) —

    entry in folder

Returns:

  • (Boolean) —

    true if target information is available


183
184
185
186
187
188
189
190
191
192
193
 
# File 'lib/aspera/api/node.rb', line 183

def entry_has_link_information(entry)
  # if target information is missing in folder, try to get it on entry
  if entry['target_node_id'].nil? || entry['target_id'].nil?
    link_entry = read("files/#{entry['id']}")
    entry['target_node_id'] = link_entry['target_node_id']
    entry['target_id'] = link_entry['target_id']
  end
  return true unless entry['target_node_id'].nil? || entry['target_id'].nil?
  Log.log.warn{"Missing target information for link: #{entry['name']}"}
  return false
end

#find_files(top_file_id, test_lambda) ⇒ Object 


269
270
271
272
273
274
 
# File 'lib/aspera/api/node.rb', line 269

def find_files(top_file_id, test_lambda)
  Log.log.debug{"find_files: file id=#{top_file_id}"}
  find_state = {found: [], test_lambda: test_lambda}
  process_folder_tree(method_sym: :process_find_files, state: find_state, top_file_id: top_file_id)
  return find_state[:found]
end

#list_files(top_file_id) ⇒ Object 


276
277
278
279
280
 
# File 'lib/aspera/api/node.rb', line 276

def list_files(top_file_id)
  find_state = {found: []}
  process_folder_tree(method_sym: :process_list_files, state: find_state, top_file_id: top_file_id)
  return find_state[:found]
end

#node_id_to_node(node_id) ⇒ Object 


168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/aspera/api/node.rb', line 168

def node_id_to_node(node_id)
  if !@app_info.nil?
    return self if node_id.eql?(@app_info[:node_info]['id'])
    return @app_info[:api].node_api_from(
      node_id: node_id,
      workspace_id: @app_info[:workspace_id],
      workspace_name: @app_info[:workspace_name])
  end
  Log.log.warn{"Cannot resolve link with node id #{node_id}, no resolver"}
  return nil
end

#process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil) ⇒ Object

Recursively browse in a folder (with non-recursive method) sub folders are processed if the processing method returns true links are processed on the respective node

Parameters:

  • method_sym (Symbol) —

    processing method, arguments: entry, path, state

  • state (Object) —

    state object sent to processing method

  • top_file_id (String) —

    file id to start at (default = access key root file id)

  • top_file_path (String) (defaults to: '/') —

    path of top folder (default = /)


202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
 
# File 'lib/aspera/api/node.rb', line 202

def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil)
  Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
  Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info[:node_info]['id'] : 'nil'}, file id=#{top_file_id},  path=#{top_file_path}"}
  # start at top folder
  folders_to_explore = [{id: top_file_id, path: top_file_path}]
  Log.log.debug{Log.dump(:folders_to_explore, folders_to_explore)}
  until folders_to_explore.empty?
    # consume first in job list
    current_item = folders_to_explore.shift
    Log.log.debug{"Exploring #{current_item[:path]}".bg_green}
    # get folder content
    folder_contents =
      begin
        # TODO: use header
        read_with_cache("files/#{current_item[:id]}/files")
      rescue StandardError => e
        Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
        []
      end
    Log.log.debug{Log.dump(:folder_contents, folder_contents)}
    folder_contents.each do |entry|
      if entry.key?('error')
        if entry['error'].is_a?(Hash) && entry['error'].key?('user_message')
          Log.log.error(entry['error']['user_message'])
        end
        next
      end
      current_path = File.join(current_item[:path], entry['name'])
      Log.log.debug{"process_folder_tree: checking #{current_path}"}
      # call block, continue only if method returns true
      next unless send(method_sym, entry, current_path, state)
      # entry type is file, folder or link
      case entry['type']
      when 'folder'
        folders_to_explore.push({id: entry['id'], path: current_path})
      when 'link'
        if entry_has_link_information(entry)
          node_id_to_node(entry['target_node_id'])&.process_folder_tree(
            method_sym:    method_sym,
            state:         state,
            top_file_id:   entry['target_id'],
            top_file_path: current_path)
        end
      end
    end
  end
end

#read_with_cache(subpath, query = nil) ⇒ Object

Call node API, possibly adding cache control header, as globally specified


151
152
153
154
155
156
157
158
159
 
# File 'lib/aspera/api/node.rb', line 151

def read_with_cache(subpath, query=nil)
  headers = {'Accept' => 'application/json'}
  headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless self.class.use_node_cache
  return call(
    operation: 'GET',
    subpath:   subpath,
    headers:   headers,
    query:     query)[:data]
end

#refreshed_transfer_token ⇒ Object 


282
283
284
 
# File 'lib/aspera/api/node.rb', line 282

def refreshed_transfer_token
  return oauth.authorization(refresh: true)
end

#resolve_api_fid(top_file_id, path, process_last_link = false) ⇒ Hash

Navigate the path from given file id on current node, and return the node and file id of target. If the path ends with a “/” or process_last_link is true then if the last item in path is a link, it is followed.

Parameters:

  • top_file_id (String) —

    id initial file id

  • path (String) —

    file or folder path (end with “/” is like setting process_last_link)

  • process_last_link (Boolean) (defaults to: false) —

    if true, follow the last link

Returns:

  • (Hash) —

    Aspera::Api::Node.api,.api,.file_id


256
257
258
259
260
261
262
263
264
265
266
267
 
# File 'lib/aspera/api/node.rb', line 256

def resolve_api_fid(top_file_id, path, process_last_link=false)
  Aspera.assert_type(top_file_id, String)
  Aspera.assert_type(path, String)
  process_last_link ||= path.end_with?(PATH_SEPARATOR)
  path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
  return {api: self, file_id: top_file_id} if path_elements.empty?
  resolve_state = {path: path_elements, result: nil, process_last_link: process_last_link}
  process_folder_tree(method_sym: :process_api_fid, state: resolve_state, top_file_id: top_file_id)
  raise "entry not found: #{resolve_state[:path]}" if resolve_state[:result].nil?
  Log.log.debug{"resolve_api_fid: #{path} -> #{resolve_state[:result][:api].base_url} #{resolve_state[:result][:file_id]}"}
  return resolve_state[:result]
end

#transfer_spec_gen4(file_id, direction, ts_merge = nil) ⇒ Object

Create transfer spec for gen4

Parameters:

  • file_id —

    destination or source folder (id)

  • direction —

    one of Transfer::Spec::DIRECTION_SEND, Transfer::Spec::DIRECTION_RECEIVE

  • ts_merge (defaults to: nil) —

    additional transfer spec to merge


306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
 
# File 'lib/aspera/api/node.rb', line 306

def transfer_spec_gen4(file_id, direction, ts_merge=nil)
  ak_name = nil
  ak_token = nil
  case auth_params[:type]
  when :basic
    ak_name = auth_params[:username]
    Aspera.assert(auth_params[:password]){'no secret in node object'}
    ak_token = Rest.basic_authorization(auth_params[:username], auth_params[:password])
  when :oauth2
    ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
    # TODO: token_generation_lambda = lambda{|do_refresh|oauth.authorization(refresh: do_refresh)}
    # get bearer token, possibly use cache
    ak_token = oauth.authorization
  when :none
    ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
    ak_token = params[:headers]['Authorization']
  else Aspera.error_unexpected_value(auth_params[:type])
  end
  transfer_spec = {
    'direction' => direction,
    'token'     => ak_token,
    'tags'      => {
      Transfer::Spec::TAG_RESERVED => {
        'node' => {
          'access_key' => ak_name,
          'file_id'    => file_id
        } # node
      } # aspera
    } # tags
  }
  # add specials tags (cos)
  add_tspec_info(transfer_spec)
  transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
  # add application specific tags (AoC)
  app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: app_info) unless app_info.nil?
  # add remote host info
  if self.class.use_standard_ports
    # get default TCP/UDP ports and transfer user
    transfer_spec.merge!(Transfer::Spec::AK_TSPEC_BASE)
    # by default: same address as node API
    transfer_spec['remote_host'] = URI.parse(base_url).host
    # AoC allows specification of other url
    if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
      transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url']
    end
    info = read('info')
    # get the transfer user from info on access key
    transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
    # get settings from name.value array to hash key.value
    settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
    # check WSS ports
    Transfer::Spec::WSS_FIELDS.each do |i|
      transfer_spec[i] = settings[i] if settings.key?(i)
    end if settings.is_a?(Hash)
  else
    transfer_spec.merge!(transport_params)
  end
  Log.log.warn{"Expected transfer user: #{Transfer::Spec::ACCESS_KEY_TRANSFER_USER}, but have #{transfer_spec['remote_user']}"} \
    unless transfer_spec['remote_user'].eql?(Transfer::Spec::ACCESS_KEY_TRANSFER_USER)
  return transfer_spec
end

#transport_params ⇒ Object

Returns part of transfer spec with transport parameters only.

Returns:

  • part of transfer spec with transport parameters only


287
288
289
290
291
292
293
294
295
296
297
298
299
300
 
# File 'lib/aspera/api/node.rb', line 287

def transport_params
  if @std_t_spec_cache.nil?
    # retrieve values from API (and keep a copy/cache)
    full_spec = create(
      'files/download_setup',
      {transfer_requests: [{transfer_request: {paths: [{source: '/'}]}}]}
    )['transfer_specs'].first['transfer_spec']
    # set available fields
    @std_t_spec_cache = Transfer::Spec::TRANSPORT_FIELDS.each_with_object({}) do |i, h|
      h[i] = full_spec[i] if full_spec.key?(i)
    end
  end
  return @std_t_spec_cache
end