Class: Aspera::Api::Node

Inherits:
Rest
  • Object
show all
Defined in:
lib/aspera/api/node.rb

Overview

Provides additional functions using node API with gen4 extensions (access keys)

Direct Known Subclasses

CosNode

Defined Under Namespace

Modules: Scope

Constant Summary collapse

ACCESS_LEVELS =

Node API permissions: delete list mkdir preview read rename write

%w[delete list mkdir preview read rename write].freeze
HEADER_X_ASPERA_ACCESS_KEY =

Special HTTP Headers

'X-Aspera-AccessKey'
HEADER_X_TOTAL_COUNT = 
'X-Total-Count'
HEADER_X_CACHE_CONTROL = 
'X-Aspera-Cache-Control'
HEADER_X_NEXT_ITER_TOKEN = 
'X-Aspera-Next-Iteration-Token'
PATH_SEPARATOR =

/ in cloud

'/'

Constants inherited from Rest

Rest::MAX_ITEMS, Rest::MAX_PAGES

Class Attribute Summary collapse

Instance Attribute Summary collapse

Attributes inherited from Rest

#auth_params, #base_url, #headers

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Rest

basic_authorization, build_uri, #call, #cancel, #create, #delete, h_to_query_array, io_http_session, #lookup_by_name, #oauth, #params, parse_header, php_style, query_to_h, #read, remote_certificate_chain, start_http_session, #update

Constructor Details

#initialize(app_info: nil, add_tspec: nil, **rest_args) ⇒ Node

Returns a new instance of Node.

Parameters:

  • app_info (Hash, NilClass) (defaults to: nil) —

    App information, typically AoC

  • add_tspec (Hash, NilClass) (defaults to: nil) —

    Additional transfer spec

  • base_url (String) —

    Rest parameters

  • auth (String, NilClass) —

    Rest parameters

  • headers (String, NilClass) —

    Rest parameters



196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
 
# File 'lib/aspera/api/node.rb', line 196

def initialize(app_info: nil, add_tspec: nil, **rest_args)
  # Init Rest
  super(**rest_args)
  @dynamic_key = nil
  @app_info = app_info
  # This is added to transfer spec, for instance to add tags (COS)
  @add_tspec = add_tspec
  @std_t_spec_cache = nil
  if !@app_info.nil?
    REQUIRED_APP_INFO_FIELDS.each do |field|
      Aspera.assert(@app_info.key?(field)){"app_info lacks field #{field}"}
    end
    REQUIRED_APP_API_METHODS.each do |method|
      Aspera.assert(@app_info[:api].respond_to?(method)){"#{@app_info[:api].class} lacks method #{method}"}
    end
  end
end

Class Attribute Details

.use_dynamic_key ⇒ Object

Returns the value of attribute use_dynamic_key.



63
64
65
 
# File 'lib/aspera/api/node.rb', line 63

def use_dynamic_key
  @use_dynamic_key
end

.use_node_cache ⇒ Object

Set to false to bypass cache in redis



62
63
64
 
# File 'lib/aspera/api/node.rb', line 62

def use_node_cache
  @use_node_cache
end

.use_standard_ports ⇒ Object

Set to false to read transfer parameters from download_setup



60
61
62
 
# File 'lib/aspera/api/node.rb', line 60

def use_standard_ports
  @use_standard_ports
end

Instance Attribute Details

#app_info ⇒ Object (readonly)

Returns the value of attribute app_info.



189
190
191
 
# File 'lib/aspera/api/node.rb', line 189

def app_info
  @app_info
end

Class Method Details

.add_private_key(h) ⇒ Object

Adds fields ssh_private_key in provided Hash, if dynamic key is set.

Parameters:

  • h (Hash) —

    Hash to add private key to



96
97
98
99
 
# File 'lib/aspera/api/node.rb', line 96

def add_private_key(h)
  h['ssh_private_key'] = @dynamic_key.to_pem if @dynamic_key
  return h
end

.add_public_key(h) ⇒ Object

Adds fields public_keys in provided Hash, if dynamic key is set.

Parameters:

  • h (Hash) —

    Hash to add public key to



82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/aspera/api/node.rb', line 82

def add_public_key(h)
  if @dynamic_key
    ssh_key = Net::SSH::Buffer.from(:key, @dynamic_key)
    # Get pub key in OpenSSH public key format (authorized_keys)
    h['public_keys'] = [
      ssh_key.read_string,
      Base64.strict_encode64(ssh_key.to_s)
    ].join(' ')
  end
  return h
end

.bearer_headers(bearer_auth, access_key: nil) ⇒ Hash

Returns Headers to call node API with access key and auth.

Returns:

  • (Hash) —

    Headers to call node API with access key and auth



176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/aspera/api/node.rb', line 176

def bearer_headers(bearer_auth, access_key: nil)
  # If username is not provided, use the access key from the token
  if access_key.nil?
    access_key = Node.decode_scope(Node.decode_bearer_token(OAuth::Factory.bearer_token(bearer_auth))['scope'])[:access_key]
    Aspera.assert(!access_key.nil?)
  end
  return {
    Node::HEADER_X_ASPERA_ACCESS_KEY => access_key,
    'Authorization'                  => bearer_auth
  }
end

.bearer_token(access_key:, payload:, private_key:) ⇒ Object

Create an Aspera Node bearer token

Parameters:

  • access_key (String) —

    Access key identifier

  • payload (String) —

    JSON payload to be included in the token

  • private_key (OpenSSL::PKey::RSA) —

    Private key to sign the token



147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
 
# File 'lib/aspera/api/node.rb', line 147

def bearer_token(access_key:, payload:, private_key:)
  Aspera.assert_type(payload, Hash)
  Aspera.assert(payload.key?('user_id'))
  Aspera.assert_type(payload['user_id'], String)
  Aspera.assert(!payload['user_id'].empty?)
  Aspera.assert_type(private_key, OpenSSL::PKey::RSA)
  # Manage convenience parameters
  expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
  payload.delete('_validity')
  scope = payload['_scope'] || Scope::USER
  payload.delete('_scope')
  payload['scope'] ||= token_scope(access_key, scope)
  payload['auth_type'] ||= 'access_key'
  payload['expires_at'] ||= (Time.now + expiration_sec).utc.strftime('%FT%TZ')
  payload_json = JSON.generate(payload)
  return Base64.strict_encode64(Zlib::Deflate.deflate([
    payload_json,
    SIGNATURE_DELIMITER,
    Base64.strict_encode64(private_key.sign(OpenSSL::Digest.new('sha512'), payload_json)).scan(/.{1,60}/).join("\n"),
    ''
  ].join("\n")))
end

.cache_control ⇒ Object

Adds cache control header, as globally specified to read request Use like this: read(…,**cache_control)



67
68
69
70
71
 
# File 'lib/aspera/api/node.rb', line 67

def cache_control
  headers = {'Accept' => Mime::JSON}
  headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless use_node_cache
  {headers: headers}
end

.decode_bearer_token(token) ⇒ Object

Decode an Aspera Node bearer token



171
172
173
 
# File 'lib/aspera/api/node.rb', line 171

def decode_bearer_token(token)
  return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
end

.decode_scope(scope) ⇒ Hash

Decode node scope into access key and scope

Returns:



136
137
138
139
140
141
 
# File 'lib/aspera/api/node.rb', line 136

def decode_scope(scope)
  items = scope.split(Scope::SEPARATOR, 2)
  Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
  Aspera.assert(items[0].start_with?(Scope::NODE_PREFIX)){"invalid scope: #{scope}"}
  return {access_key: items[0][Scope::NODE_PREFIX.length..-1], scope: items[1]}
end

.file_matcher(match_expression) ⇒ Object

For access keys: provide expression to match entry in folder

Parameters:

  • match_expression —

    one of supported types

Returns:

  • lambda function



104
105
106
107
108
109
110
111
112
113
 
# File 'lib/aspera/api/node.rb', line 104

def file_matcher(match_expression)
  case match_expression
  when Proc then return match_expression
  when Regexp then return ->(f){f['name'].match?(match_expression)}
  when String
    return ->(f){File.fnmatch(match_expression, f['name'], File::FNM_DOTMATCH)}
  when NilClass then return ->(_){true}
  else Aspera.error_unexpected_value(match_expression.class.name, type: ParameterError)
  end
end

.file_matcher_from_argument(options) ⇒ Proc

Returns lambda from provided CLI options.

Returns:

  • (Proc) —

    lambda from provided CLI options



116
117
118
 
# File 'lib/aspera/api/node.rb', line 116

def file_matcher_from_argument(options)
  return file_matcher(options.get_next_argument('filter', validation: MATCH_TYPES, mandatory: false))
end

.split_folder(path) ⇒ Array

Split path into folder + filename

Returns:

  • (Array) —

    containing folder + inside folder/file



122
123
124
125
126
 
# File 'lib/aspera/api/node.rb', line 122

def split_folder(path)
  folder = path.split(PATH_SEPARATOR)
  inside = folder.pop
  [folder.join(PATH_SEPARATOR), inside]
end

.token_scope(access_key, scope) ⇒ String

Node API scopes

Returns:



130
131
132
 
# File 'lib/aspera/api/node.rb', line 130

def token_scope(access_key, scope)
  return [Scope::NODE_PREFIX, access_key, Scope::SEPARATOR, scope].join('')
end

Instance Method Details

#add_tspec_info(tspec) ⇒ Hash

Update transfer spec with special additional tags

Parameters:

  • tspec (Hash) —

    Transfer spec to be modified

Returns:

  • (Hash) —

    initial modified tspec



217
218
219
220
 
# File 'lib/aspera/api/node.rb', line 217

def add_tspec_info(tspec)
  tspec.deep_merge!(@add_tspec) unless @add_tspec.nil?
  return tspec
end

#base_spec ⇒ Hash

Get a base download transfer spec (gen3)

Returns:

  • (Hash) —

    Base transfer spec



395
396
397
398
399
400
 
# File 'lib/aspera/api/node.rb', line 395

def base_spec
  create(
    'files/download_setup',
    {transfer_requests: [{transfer_request: {paths: [{source: '/'}]}}]}
  )['transfer_specs'].first['transfer_spec']
end

#entry_has_link_information(entry) ⇒ Boolean

Check if a link entry in folder has target information

Parameters:

  • entry (Hash) —

    entry in folder

Returns:

  • (Boolean) —

    true if target information is available



239
240
241
242
243
244
245
246
247
248
249
 
# File 'lib/aspera/api/node.rb', line 239

def entry_has_link_information(entry)
  # If target information is missing in folder, try to get it on entry
  if entry['target_node_id'].nil? || entry['target_id'].nil?
    link_entry = read("files/#{entry['id']}")
    entry['target_node_id'] = link_entry['target_node_id']
    entry['target_id'] = link_entry['target_id']
  end
  return true unless entry['target_node_id'].nil? || entry['target_id'].nil?
  Log.log.warn{"Missing target information for link: #{entry['name']}"}
  return false
end

#find_files(top_file_id, test_lambda) ⇒ Object

Recursively find files matching lambda

Parameters:

  • top_file_id (String) —

    Search root

  • test_lambda (Proc) —

    Test function



374
375
376
377
378
379
 
# File 'lib/aspera/api/node.rb', line 374

def find_files(top_file_id, test_lambda)
  Log.log.debug{"find_files: file id=#{top_file_id}"}
  find_state = {found: [], test_lambda: test_lambda}
  process_folder_tree(method_sym: :process_find_files, state: find_state, top_file_id: top_file_id)
  return find_state[:found]
end

#list_files(top_file_id, query: nil) ⇒ Object

Recursively list all files and folders



382
383
384
385
386
 
# File 'lib/aspera/api/node.rb', line 382

def list_files(top_file_id, query: nil)
  find_state = {found: []}
  process_folder_tree(method_sym: :process_list_files, state: find_state, top_file_id: top_file_id, query: query)
  return find_state[:found]
end

#node_id_to_node(node_id) ⇒ Object 



223
224
225
226
227
228
229
230
231
232
233
234
 
# File 'lib/aspera/api/node.rb', line 223

def node_id_to_node(node_id)
  if !@app_info.nil?
    return self if node_id.eql?(@app_info[:node_info]['id'])
    return @app_info[:api].node_api_from(
      node_id: node_id,
      workspace_id: @app_info[:workspace_id],
      workspace_name: @app_info[:workspace_name]
    )
  end
  Log.log.warn{"Cannot resolve link with node id #{node_id}, no resolver"}
  return
end

#process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil) ⇒ Object

Recursively browse in a folder (with non-recursive method) Entries of folders are processed if the processing method returns true Links are processed on the respective node

Parameters:

  • method_sym (Symbol) —

    processing method, arguments: entry, path, state

  • state (Object) —

    state object sent to processing method

  • top_file_id (String) —

    file id to start at (default = access key root file id)

  • top_file_path (String) (defaults to: '/') —

    path of top folder (default = /)



259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
 
# File 'lib/aspera/api/node.rb', line 259

def process_folder_tree(method_sym:, state:, top_file_id:, top_file_path: '/', query: nil)
  Aspera.assert(!top_file_path.nil?){'top_file_path not set'}
  Log.log.debug{"process_folder_tree: node=#{@app_info ? @app_info[:node_info]['id'] : 'nil'}, file id=#{top_file_id},  path=#{top_file_path}"}
  # Start at top folder
  folders_to_explore = [{id: top_file_id, path: top_file_path}]
  Log.dump(:folders_to_explore, folders_to_explore)
  until folders_to_explore.empty?
    # Consume first in job list
    current_item = folders_to_explore.shift
    Log.log.debug{"Exploring #{current_item[:path]}".bg_green}
    # Get folder content
    folder_contents =
      begin
        # TODO: use header
        read("files/#{current_item[:id]}/files", query, **self.class.cache_control)
      rescue StandardError => e
        Log.log.warn{"#{current_item[:path]}: #{e.class} #{e.message}"}
        []
      end
    Log.dump(:folder_contents, folder_contents)
    folder_contents.each do |entry|
      if entry.key?('error')
        Log.log.error(entry['error']['user_message']) if entry['error'].is_a?(Hash) && entry['error'].key?('user_message')
        next
      end
      current_path = File.join(current_item[:path], entry['name'])
      Log.log.debug{"process_folder_tree: checking #{current_path}"}
      # Call block, continue only if method returns true
      next unless send(method_sym, entry, current_path, state)
      # Entry type is file, folder or link
      case entry['type']
      when 'folder'
        folders_to_explore.push({id: entry['id'], path: current_path})
      when 'link'
        if entry_has_link_information(entry)
          node_id_to_node(entry['target_node_id'])&.process_folder_tree(
            method_sym:    method_sym,
            state:         state,
            top_file_id:   entry['target_id'],
            top_file_path: current_path
          )
        end
      end
    end
  end
end

#refreshed_transfer_token ⇒ Object

Generate a refreshed auth token



389
390
391
 
# File 'lib/aspera/api/node.rb', line 389

def refreshed_transfer_token
  return oauth.authorization(refresh: true)
end

#resolve_api_fid(top_file_id, path, process_last_link = false) ⇒ Hash

Navigate the path from given file id on current node, and return the node and file id of target. If the path ends with a “/” or process_last_link is true then if the last item in path is a link, it is followed.

Parameters:

  • top_file_id (String) —

    id initial file id

  • path (String) —

    file or folder path (end with “/” is like setting process_last_link)

  • process_last_link (Boolean) (defaults to: false) —

    if true, follow the last link

  • return (Hash) —

    a customizable set of options

Returns:

  • (Hash) —

    Result data

Raises:



314
315
316
317
318
319
320
321
322
323
324
325
 
# File 'lib/aspera/api/node.rb', line 314

def resolve_api_fid(top_file_id, path, process_last_link = false)
  Aspera.assert_type(top_file_id, String)
  Aspera.assert_type(path, String)
  process_last_link ||= path.end_with?(PATH_SEPARATOR)
  path_elements = path.split(PATH_SEPARATOR).reject(&:empty?)
  return {api: self, file_id: top_file_id} if path_elements.empty?
  resolve_state = {path: path_elements, consumed: [], result: nil, process_last_link: process_last_link}
  process_folder_tree(method_sym: :process_api_fid, state: resolve_state, top_file_id: top_file_id)
  raise ParameterError, "Entry not found: #{resolve_state[:path].first} in /#{resolve_state[:consumed].join(PATH_SEPARATOR)}" if resolve_state[:result].nil?
  Log.log.debug{"resolve_api_fid: #{path} -> #{resolve_state[:result][:api].base_url} #{resolve_state[:result][:file_id]}"}
  return resolve_state[:result]
end

#resolve_api_fid_paths(top_file_id, paths) ⇒ Array

Given a list of paths, finds a common root and list of sub-paths

Parameters:

  • top_file_id (String) —

    Root file id

  • paths (Array(Hash)) —

    List of paths

Returns:

  • (Array) —

    size=2: apfid, paths (Array(Hash))



331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
 
# File 'lib/aspera/api/node.rb', line 331

def resolve_api_fid_paths(top_file_id, paths)
  Aspera.assert_type(paths, Array)
  Aspera.assert(paths.size.positive?)
  split_sources = paths.map{ |p| Pathname(p['source']).each_filename.to_a}
  root = []
  split_sources.map(&:size).min.times do |i|
    parts = split_sources.map{ |s| s[i]}
    break unless parts.uniq.size == 1
    root << parts.first
  end
  source_folder = File.join(root)
  source_paths = paths.each_with_index.map do |p, i|
    m = {'source' => File.join(split_sources[i][root.size..])}
    m['destination'] = p['destination'] if p.key?('destination')
    m
  end
  apifid = resolve_api_fid(top_file_id, source_folder, true)
  # If a single item
  if source_paths.size.eql?(1)
    # Get precise info in this element
    file_info = apifid[:api].read("files/#{apifid[:file_id]}")
    source_paths =
      case file_info['type']
      when 'file'
        # If the single source is a file, we need to split into folder path and filename
        src_dir_elements = source_folder.split(Api::Node::PATH_SEPARATOR)
        filename = src_dir_elements.pop
        apifid = resolve_api_fid(top_file_id, src_dir_elements.join(Api::Node::PATH_SEPARATOR), true)
        # Filename is the last one, source folder is what remains
        [{'source' => filename}]
      when 'link', 'folder'
        # Single source is 'folder' or 'link'
        # TODO: add this ? , 'destination'=>file_info['name']
        [{'source' => '.'}]
      else Aspera.error_unexpected_value(file_info['type']){'source type'}
      end
  end
  [apifid, source_paths]
end

#transfer_spec_gen4(file_id, direction, ts_merge = nil) ⇒ Object

Create transfer spec for gen4

Parameters:

  • file_id (String) —

    Destination or source folder (id)

  • direction (Symbol) —

    One of Transfer::Spec::DIRECTION_SEND, Transfer::Spec::DIRECTION_RECEIVE

  • ts_merge (Hash, nil) (defaults to: nil) —

    Additional transfer spec to merge



412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
 
# File 'lib/aspera/api/node.rb', line 412

def transfer_spec_gen4(file_id, direction, ts_merge = nil)
  ak_name = nil
  ak_token = nil
  case auth_params[:type]
  when :basic
    ak_name = auth_params[:username]
    Aspera.assert(auth_params[:password]){'no secret in node object'}
    ak_token = Rest.basic_authorization(auth_params[:username], auth_params[:password])
  when :oauth2
    ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
    # TODO: token_generation_lambda = lambda{|do_refresh|oauth.authorization(refresh: do_refresh)}
    # Get bearer token, possibly use cache
    ak_token = oauth.authorization
  when :none
    ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
    ak_token = params[:headers]['Authorization']
  else Aspera.error_unexpected_value(auth_params[:type])
  end
  transfer_spec = {
    'direction' => direction,
    'token'     => ak_token,
    'tags'      => {
      Transfer::Spec::TAG_RESERVED => {
        'node' => {
          'access_key' => ak_name,
          'file_id'    => file_id
        }
      }
    }
  }
  # Add specials tags (cos)
  add_tspec_info(transfer_spec)
  transfer_spec.deep_merge!(ts_merge) unless ts_merge.nil?
  # Add application specific tags (AoC)
  @app_info[:api].add_ts_tags(transfer_spec: transfer_spec, app_info: @app_info) unless @app_info.nil?
  # Add remote host info
  if self.class.use_standard_ports
    # Get default TCP/UDP ports and transfer user
    transfer_spec.merge!(Transfer::Spec::AK_TSPEC_BASE)
    # By default: same address as node API
    transfer_spec['remote_host'] = URI.parse(base_url).host
    # AoC allows specification of other url
    transfer_spec['remote_host'] = @app_info[:node_info]['transfer_url'] if !@app_info.nil? && !@app_info[:node_info]['transfer_url'].nil? && !@app_info[:node_info]['transfer_url'].empty?
    info = read('info')
    # Get the transfer user from info on access key
    transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
    # Get settings from name.value array to hash key.value
    settings = info['settings']&.each_with_object({}){ |i, h| h[i['name']] = i['value']}
    # Check WSS ports
    Transfer::Spec::WSS_FIELDS.each do |i|
      transfer_spec[i] = settings[i] if settings.key?(i)
    end if settings.is_a?(Hash)
  else
    transfer_spec.merge!(transport_params)
  end
  Aspera.assert_values(transfer_spec['remote_user'], Transfer::Spec::ACCESS_KEY_TRANSFER_USER, type: :warn){'transfer user'}
  return transfer_spec
end

#transport_params ⇒ Hash

Get generic part of transfer spec with transport parameters only

Returns:

  • (Hash) —

    Base transfer spec



404
405
406
 
# File 'lib/aspera/api/node.rb', line 404

def transport_params
  @std_t_spec_cache ||= base_spec.slice(*Transfer::Spec::TRANSPORT_FIELDS).freeze
end