Class: Fog::Backblaze::Storage::Real

Inherits:
Object
  • Object
show all
Defined in:
lib/fog/backblaze/storage/real.rb

Constant Summary collapse

USER_AGENT =
"fog-backblaze/#{Fog::Backblaze::VERSION}+ruby/#{RUBY_VERSION}".freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(options = {}) ⇒ Real

Returns a new instance of Real.


7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# File 'lib/fog/backblaze/storage/real.rb', line 7

def initialize(options = {})
  @options = options
  @logger = @options[:logger] || begin
    require 'logger'
    Logger.new("/dev/null")
  end

  @token_cache = if options[:token_cache].nil? || options[:token_cache] == :memory
    Fog::Backblaze::TokenCache.new
  elsif options[:token_cache] === false
    Fog::Backblaze::TokenCache::NullTokenCache.new
  elsif options[:token_cache].is_a?(Fog::Backblaze::TokenCache)
    options[:token_cache]
  else
    Fog::Backblaze::TokenCache::FileTokenCache.new(options[:token_cache])
  end
end

Instance Attribute Details

#optionsObject (readonly)

Returns the value of attribute options


2
3
4
# File 'lib/fog/backblaze/storage/real.rb', line 2

def options
  @options
end

#token_cacheObject (readonly)

Returns the value of attribute token_cache


2
3
4
# File 'lib/fog/backblaze/storage/real.rb', line 2

def token_cache
  @token_cache
end

Instance Method Details

#_cached_buchets_hash(force_fetch: false) ⇒ Object


484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
# File 'lib/fog/backblaze/storage/real.rb', line 484

def _cached_buchets_hash(force_fetch: false)

  if !force_fetch && cached = @token_cache.buckets
    cached
  end

  buckets_hash = {}
  list_buckets.json['buckets'].each do |bucket|
    buckets_hash[bucket['bucketName']] = bucket
  end

  @token_cache.buckets = buckets_hash

  buckets_hash
end

#_get_bucket_id(bucket_name) ⇒ Object


460
461
462
463
464
465
466
467
468
469
470
471
472
473
# File 'lib/fog/backblaze/storage/real.rb', line 460

def _get_bucket_id(bucket_name)
  if @options[:b2_bucket_name] == bucket_name && @options[:b2_bucket_id]
    return @options[:b2_bucket_id]
  else
    cached = @token_cache && @token_cache.buckets

    if cached && cached[bucket_name]
      return cached[bucket_name]['bucketId']
    else
      fetched = _cached_buchets_hash(force_fetch: !!cached)
      return fetched[bucket_name] && fetched[bucket_name]['bucketId']
    end
  end
end

#_get_bucket_id!(bucket_name) ⇒ Object


475
476
477
478
479
480
481
482
# File 'lib/fog/backblaze/storage/real.rb', line 475

def _get_bucket_id!(bucket_name)
  bucket_id = _get_bucket_id(bucket_name)
  unless bucket_id
    raise Fog::Errors::NotFound, "Can not find bucket \"#{bucket_name}\""
  end

  return bucket_id
end

#_get_object_version_ids(bucket_name, file_name) ⇒ Object


435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
# File 'lib/fog/backblaze/storage/real.rb', line 435

def _get_object_version_ids(bucket_name, file_name)
  response = b2_command(:b2_list_file_versions,
    body: {
      startFileName: file_name,
      prefix: file_name,
      bucketId: _get_bucket_id!(bucket_name),
      maxFileCount: 1000
    }
  )

  if response.status >= 400
    raise Fog::Errors::Error, "Fetch error: #{response.json['message']} (status = #{response.status})"
  end

  if response.json['files']
    version_ids = []
    response.json['files'].map do |file_version|
      version_ids << file_version['fileId'] if file_version['fileName'] == file_name
    end
    version_ids
  else
    []
  end
end

#auth_responseObject


500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
# File 'lib/fog/backblaze/storage/real.rb', line 500

def auth_response
  #return @auth_response.json if @auth_response

  if cached = @token_cache.auth_response
    logger.info("get token from cache")
    return cached
  end

  auth_string = if @options[:b2_account_id] && @options[:b2_account_token]
    "#{@options[:b2_account_id]}:#{@options[:b2_account_token]}"
  elsif @options[:b2_key_id] && @options[:b2_key_token]
    "#{@options[:b2_key_id]}:#{@options[:b2_key_token]}"
  else
    raise Fog::Errors::Error, "B2 credentials are required, "\
                              "please use b2_account_id and b2_account_token or "\
                              "b2_key_id and b2_key_token"
  end

  @auth_response = json_req(:get, "https://api.backblazeb2.com/b2api/v1/b2_authorize_account",
    headers: {
      "Authorization" => "Basic " + Base64.strict_encode64(auth_string)
    },
    persistent: false
  )

  if @auth_response.status >= 400
    raise Fog::Errors::Error, "Authentication error: #{@auth_response.json['message']} (status = #{@auth_response.status})\n#{@auth_response.body}"
  end

  @token_cache.auth_response = @auth_response.json

  @auth_response.json
end

#b2_account_idObject

return @options or call b2_authorize_account when using application key


602
603
604
605
# File 'lib/fog/backblaze/storage/real.rb', line 602

def 
  return @options[:b2_account_id] if @options[:b2_account_id]
  auth_response['accountId']
end

#b2_command(command, options = {}) ⇒ Object


534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
# File 'lib/fog/backblaze/storage/real.rb', line 534

def b2_command(command, options = {})
  auth_response = self.auth_response
  options[:headers] ||= {}
  options[:headers]['Authorization'] ||= auth_response['authorizationToken']

  if options[:body] && !options[:body].is_a?(String)
    options[:body] = ::JSON.generate(options[:body])
  end

  request_url = options.delete(:url) || "#{auth_response['apiUrl']}/b2api/v1/#{command}"

  #pp [:b2_command, request_url, options]

  json_req(options.delete(:method) || :post, request_url, options)
end

#b2_copy_object(source_object:, target_object:, bucket: nil, source_bucket: nil, target_bucket: nil, options: {}) ⇒ Object


152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
# File 'lib/fog/backblaze/storage/real.rb', line 152

def b2_copy_object(source_object:, target_object:, bucket: nil, source_bucket: nil, target_bucket: nil, options: {})
  if bucket.nil? && source_bucket.nil?
    raise ArgumentError, "arguemnt bucket either source_bucket is required for copy_object()"
  end
  source_bucket ||= bucket

  file_id = _get_object_version_ids(source_bucket, source_object)[0]

  if file_id == nil
    raise Fog::Errors::NotFound, "Command copy_object failed: Can not find source object: #{source_object} in bucket #{source_bucket}"
  end

  target_bucket_id = target_bucket ? _get_bucket_id(target_bucket) : nil

  if target_bucket && !target_bucket_id
    raise Fog::Errors::NotFound, "Command copy_object failed: Can not find target bucket: #{target_bucket}"
  end

  response = b2_command(
    :b2_copy_file,
    body: {
      sourceFileId: file_id,
      destinationBucketId: target_bucket_id,
      fileName: target_object
    }
  )

  if response.status >= 400
    raise Fog::Errors::Error, "Backblaze respond with status = #{response.status} - #{response.body}"
  end

  response
end

#b2_url_encode(str) ⇒ Object


597
598
599
# File 'lib/fog/backblaze/storage/real.rb', line 597

def b2_url_encode(str)
  URI.encode_www_form_component(str.force_encoding(Encoding::UTF_8)).gsub("%2F", "/")
end

#copy_object(source_bucket, source_object, target_bucket, target_object, options = {}) ⇒ Object

call b2_copy_file


142
143
144
145
146
147
148
149
150
# File 'lib/fog/backblaze/storage/real.rb', line 142

def copy_object(source_bucket, source_object, target_bucket, target_object, options = {})
  b2_copy_object(
    source_object: source_object,
    target_object: target_object,
    source_bucket: source_bucket,
    target_bucket: target_bucket,
    options: options
  )
end

#create_key(name, capabilities: nil, bucket_id: nil, name_prefix: nil) ⇒ Object

call b2_create_key


378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
# File 'lib/fog/backblaze/storage/real.rb', line 378

def create_key(name, capabilities: nil, bucket_id: nil, name_prefix: nil)
  capabilities ||= [
    'listKeys',
    'writeKeys',
    'deleteKeys',
    'listBuckets',
    'writeBuckets',
    'deleteBuckets',
    'listFiles',
    'readFiles',
    'shareFiles',
    'writeFiles',
    'deleteFiles'
  ]

  response = b2_command(:b2_create_key,
    body: {
      accountId: ,
      keyName: name,
      capabilities: capabilities,
      bucketId: bucket_id,
      namePrefix: name_prefix
    }
  )
end

#delete_bucket(bucket_name, options = {}) ⇒ Object

call b2_delete_bucket


108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
# File 'lib/fog/backblaze/storage/real.rb', line 108

def delete_bucket(bucket_name, options = {})
  bucket_id = _get_bucket_id!(bucket_name)

  response = b2_command(:b2_delete_bucket,
    body: {
      bucketId: bucket_id,
      accountId: 
    }
  )

  if !options[:is_retrying]
    if response.status == 400 && response.json['message'] =~ /Bucket .+ does not exist/
      logger.info("Try drop cache and try again")
      @token_cache.buckets = nil
      return delete_bucket(bucket_name, is_retrying: true)
    end
  end

  if response.status >= 400
    raise Fog::Errors::Error, "Failed delete_bucket, status = #{response.status} #{response.body}"
  end

  if cached = @token_cache.buckets
    #cached.delete(bucket_name)
    #@token_cache.buckets = cached
    @token_cache.buckets = nil
  end

  response
end

#delete_key(key_id) ⇒ Object

call b2_delete_key


421
422
423
424
425
426
427
428
429
430
431
432
433
# File 'lib/fog/backblaze/storage/real.rb', line 421

def delete_key(key_id)
  response = b2_command(:b2_list_keys,
    body: {
      applicationKeyId: key_id
    }
  )

  if response.status > 400
    raise Fog::Errors::Error, "Failed delete_key, status = #{response.status} #{response.body}"
  end

  response
end

#delete_object(bucket_name, file_name) ⇒ Object

call b2_delete_file_version


357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
# File 'lib/fog/backblaze/storage/real.rb', line 357

def delete_object(bucket_name, file_name)
  version_ids = _get_object_version_ids(bucket_name, file_name)

  if version_ids.size == 0
    raise Fog::Errors::NotFound, "Can not find #{file_name} in in bucket #{bucket_name}"
  end

  logger.info("Deleting #{version_ids.size} versions of #{file_name}")

  last_response = nil
  version_ids.each do |version_id|
    last_response = b2_command(:b2_delete_file_version, body: {
      fileName: file_name,
      fileId: version_id
    })
  end

  last_response
end

#get_bucket(bucket_name) ⇒ Object

call b2_list_buckets


91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
# File 'lib/fog/backblaze/storage/real.rb', line 91

def get_bucket(bucket_name)
  response = list_buckets(bucketName: bucket_name)

  bucket = response.json['buckets'].detect do |bucket|
    bucket['bucketName'] == bucket_name
  end

  unless bucket
    raise Fog::Errors::NotFound, "No bucket with name: #{bucket_name}"
  end

  response.body = bucket
  response.json = bucket
  return response
end

#get_object(bucket_name, file_name) ⇒ Object


337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
# File 'lib/fog/backblaze/storage/real.rb', line 337

def get_object(bucket_name, file_name)
  file_url = get_object_url(bucket_name, file_name)

  response = b2_command(nil,
    method: :get,
    url: file_url
  )

  if response.status == 404
    raise Fog::Errors::NotFound, "Can not find #{file_name.inspect} in bucket #{bucket_name}"
  end

  if response.status > 400
    raise Fog::Errors::Error, "Failed get_object, status = #{response.status} #{response.body}"
  end

  return response
end

#get_object_url(bucket_name, file_path, expires_or_options = nil, options = {}) ⇒ Object Also known as: get_object_https_url

generates url regardless if bucket is private or not


286
287
288
289
290
291
292
293
294
295
296
297
298
# File 'lib/fog/backblaze/storage/real.rb', line 286

def get_object_url(bucket_name, file_path, expires_or_options = nil, options = {})
  if options == nil && expires_or_options.is_a?(Hash)
    options = expires_or_options
    expires_or_options = nil
  end

  if expires_or_options
    options[:validDurationInSeconds] = expires_or_options.to_i
    return get_public_object_url(bucket_name, file_path, options)
  end

  "#{auth_response['downloadUrl']}/file/#{b2_url_encode(bucket_name)}/#{b2_url_encode(file_path)}"
end

#get_public_object_url(bucket_name, file_path, options = {}) ⇒ Object

call b2_get_download_authorization


303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
# File 'lib/fog/backblaze/storage/real.rb', line 303

def get_public_object_url(bucket_name, file_path, options = {})
  bucket_id = _get_bucket_id!(bucket_name)

  optional_params = {}.tap do |params|
    params['b2ContentDisposition'] = options[:content_disposition] if options[:content_disposition]
  end

  # Request HTTP Message Body Parameters
  # bucketId: required
  # fileNamePrefix: required
  # validDurationInSeconds: required
  # b2ContentDisposition: optional
  result = b2_command(:b2_get_download_authorization, body: {
    bucketId: bucket_id,
    fileNamePrefix: file_path,
    validDurationInSeconds: 604800
  }.merge(optional_params))

  if result.status == 404
    raise Fog::Errors::NotFound, "Can not find #{file_path.inspect} in bucket #{bucket_name}"
  end

  if result.status >= 400
    raise Fog::Errors::NotFound, "Backblaze respond with status = #{result.status} - #{result.reason_phrase}"
  end

  query_params = {}.tap do |params|
    params['Authorization'] = result.json['authorizationToken']
    params['b2ContentDisposition'] = options[:content_disposition] if options[:content_disposition]
  end

  "#{get_object_url(bucket_name, file_path)}?#{URI.encode_www_form(query_params)}"
end

#head_object(bucket_name, file_path, options = {}) ⇒ Object


196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
# File 'lib/fog/backblaze/storage/real.rb', line 196

def head_object(bucket_name, file_path, options = {})
  file_url = get_object_url(bucket_name, file_path)

  result = b2_command(nil,
    method: :head,
    url: file_url
  )

  if result.status == 404
    raise Fog::Errors::NotFound, "Can not find #{file_path.inspect} in bucket #{bucket_name}"
  end

  if result.status >= 400
    raise Fog::Errors::Error, "Backblaze respond with status = #{result.status} - #{result.reason_phrase}"
  end

  result
end

#json_req(method, url, options = {}) ⇒ Object


550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
# File 'lib/fog/backblaze/storage/real.rb', line 550

def json_req(method, url, options = {})
  start_time = Time.now.to_f
  logger.info("Req #{method.to_s.upcase} #{url}")
  if options[:body] && options[:body].size > 300
    logger.debug(options.merge(body: "-- Body #{options[:body].size} bytes --").to_s)
  else
    logger.debug(options.to_s)
  end

  options[:headers] ||= {}
  options[:headers]['User-Agent'] ||= USER_AGENT
  if @options[:headers] && @options[:headers].is_a?(Hash)
    options[:headers].merge!(@options[:headers])
  end

  if !options.has_key?(:persistent) || options[:persistent] == true
    @connections ||= {}
    full_path = [URI.parse(url).request_uri, URI.parse(url).fragment].compact.join("#")
    host_url = url.sub(full_path, "")
    connection = @connections[host_url] ||= Excon.new(host_url, persistent: true)
    http_response = connection.send(method, options.merge(path: full_path, idempotent: true))
  else
    http_response = Excon.send(method, url, options)
  end

  http_response.extend(Fog::Backblaze::JSONResponse)
  if http_response.json_response? && http_response.body.size > 0
    http_response.assign_json_body!
  end

  http_response
ensure
  status = http_response && http_response.status
  logger.info("    Done #{method.to_s.upcase} #{url} = #{status} (#{(Time.now.to_f - start_time).round(3)} sec)")
  if http_response
    logger.debug("    Headers: #{http_response.headers}")
    if method != :head && http_response.headers['Content-Type'].to_s !~ %r{^image/}
      logger.debug("    Body: #{http_response.body}")
    end
  end
end

#list_buckets(options = {}) ⇒ Object

call b2_list_buckets


82
83
84
85
86
87
88
# File 'lib/fog/backblaze/storage/real.rb', line 82

def list_buckets(options = {})
  response = b2_command(:b2_list_buckets, body: {
    accountId: 
  }.merge(options))

  response
end

#list_keysObject

call b2_list_keys


405
406
407
408
409
410
411
412
413
414
415
416
417
418
# File 'lib/fog/backblaze/storage/real.rb', line 405

def list_keys
  response = b2_command(:b2_list_keys,
    body: {
      accountId: ,
      maxKeyCount: 1000
    }
  )

  if response.status > 400
    raise Fog::Errors::Error, "Failed list_keys, status = #{response.status} #{response.body}"
  end

  response
end

#list_objects(bucket_name, options = {}) ⇒ Object

call b2_list_file_names


187
188
189
190
191
192
193
194
# File 'lib/fog/backblaze/storage/real.rb', line 187

def list_objects(bucket_name, options = {})
  bucket_id = _get_bucket_id!(bucket_name)

  b2_command(:b2_list_file_names, body: {
    bucketId: bucket_id,
    maxFileCount: 10_000
  }.merge(options))
end

#loggerObject


25
26
27
# File 'lib/fog/backblaze/storage/real.rb', line 25

def logger
  @logger
end

#put_bucket(bucket_name, extra_options = {}) ⇒ Object

call b2_create_bucket


32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
# File 'lib/fog/backblaze/storage/real.rb', line 32

def put_bucket(bucket_name, extra_options = {})
  options = {
    accountId: ,
    bucketType: extra_options.delete(:public) ? 'allPublic' : 'allPrivate',
    bucketName: bucket_name,
  }.merge(extra_options)

  response = b2_command(:b2_create_bucket, body: options)

  if response.status >= 400
    raise Fog::Errors::Error, "Failed put_bucket, status = #{response.status} #{response.body}"
  end

  if cached = @token_cache.buckets
    @token_cache.buckets = cached.merge(bucket_name => response.json)
  else
    @token_cache.buckets = {bucket_name => response.json}
  end

  response
end

#put_object(bucket_name, file_path, content, options = {}) ⇒ Object

call b2_get_upload_url

connection.put_object("a-bucket", "/some_file.txt", string_or_io, options)

Possible options:

  • content_type

  • last_modified - time object or number of miliseconds

  • content_disposition

  • extra_headers - hash, list of custom headers


224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
# File 'lib/fog/backblaze/storage/real.rb', line 224

def put_object(bucket_name, file_path, content, options = {})
  upload_url = @token_cache.fetch("upload_url/#{bucket_name}") do
    bucket_id = _get_bucket_id!(bucket_name)
    result = b2_command(:b2_get_upload_url, body: {bucketId: bucket_id})
    result.json
  end

  if content.is_a?(IO)
    content = content.read
  end

  extra_headers = {}
  if options[:content_type]
    extra_headers['Content-Type'] = options[:content_type]
  end

  if options[:last_modified]
    value = if options[:last_modified].is_a?(::Time)
      (options[:last_modified].to_f * 1000).round
    else
      value
    end
    extra_headers['X-Bz-Info-src_last_modified_millis'] = value
  end

  if options[:content_disposition]
    extra_headers['X-Bz-Info-b2-content-disposition'] = options[:content_disposition]
  end

  if options[:extra_headers]
    options[:extra_headers].each do |key, value|
      extra_headers["X-Bz-Info-#{key}"] = value
    end
  end

  c_name = content.class.to_s
  if c_name == "Rack::Test::UploadedFile" || c_name == "ActionDispatch::Http::UploadedFile" || content.is_a?(File)
    content.rewind
    content = content.read
  else
    Digest::SHA1.hexdigest(content)
  end

  response = b2_command(nil,
    url: upload_url['uploadUrl'],
    body: content,
    headers: {
      'Authorization': upload_url['authorizationToken'],
      'Content-Type': 'b2/x-auto',
      'X-Bz-File-Name': "#{b2_url_encode(file_path)}",
      'X-Bz-Content-Sha1': Digest::SHA1.hexdigest(content)
    }.merge(extra_headers)
  )

  if response.json['fileId'] == nil
    raise Fog::Errors::Error, "Failed put_object, status = #{response.status} #{response.body}"
  end

  response
end

#reset_token_cacheObject


592
593
594
# File 'lib/fog/backblaze/storage/real.rb', line 592

def reset_token_cache
  @token_cache.reset
end

#update_bucket(bucket_name, extra_options) ⇒ Object

call b2_update_bucket if options presents, then bucket_name is option


56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
# File 'lib/fog/backblaze/storage/real.rb', line 56

def update_bucket(bucket_name, extra_options)
  options = {
    accountId: ,
    bucketId: extra_options[:bucketId] || _get_bucket_id!(bucket_name),
  }
  if extra_options.has_key?(:public)
    options[:bucketType] = extra_options.delete(:public) ? 'allPublic' : 'allPrivate'
  end
  options.merge!(extra_options)

  response = b2_command(:b2_update_bucket, body: options)

  if response.status >= 400
    raise Fog::Errors::Error, "Failed update_bucket, status = #{response.status} #{response.body}"
  end

  if cached = @token_cache.buckets
    @token_cache.buckets = cached.merge(bucket_name => response.json)
  else
    @token_cache.buckets = {bucket_name => response.json}
  end

  response
end