Class: Goodcheck::ImportLoader

Inherits:
Object
  • Object
show all
Defined in:
lib/goodcheck/import_loader.rb

Defined Under Namespace

Classes: FileNotFound, HTTPGetError, UnexpectedSchemaError

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(cache_path:, expires_in: 3 * 60, force_download:, config_path:) ⇒ ImportLoader

Returns a new instance of ImportLoader.



35
36
37
38
39
40
 
# File 'lib/goodcheck/import_loader.rb', line 35

def initialize(cache_path:, expires_in: 3 * 60, force_download:, config_path:)
  @cache_path = cache_path
  @expires_in = expires_in
  @force_download = force_download
  @config_path = config_path
end

Instance Attribute Details

#cache_path ⇒ Object (readonly)

Returns the value of attribute cache_path.



30
31
32
 
# File 'lib/goodcheck/import_loader.rb', line 30

def cache_path
  @cache_path
end

#config_path ⇒ Object (readonly)

Returns the value of attribute config_path.



33
34
35
 
# File 'lib/goodcheck/import_loader.rb', line 33

def config_path
  @config_path
end

#expires_in ⇒ Object (readonly)

Returns the value of attribute expires_in.



31
32
33
 
# File 'lib/goodcheck/import_loader.rb', line 31

def expires_in
  @expires_in
end

#force_download ⇒ Object (readonly)

Returns the value of attribute force_download.



32
33
34
 
# File 'lib/goodcheck/import_loader.rb', line 32

def force_download
  @force_download
end

Instance Method Details

#cache_name(uri) ⇒ Object 



79
80
81
 
# File 'lib/goodcheck/import_loader.rb', line 79

def cache_name(uri)
  Digest::SHA2.hexdigest(uri.to_s)
end

#http_get(uri, limit = 10) ⇒ Object

Raises:

  • (ArgumentError)

See Also:



131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/goodcheck/import_loader.rb', line 131

def http_get(uri, limit = 10)
  raise ArgumentError, "Too many HTTP redirects" if limit == 0

  max_retry_count = 2
  retry_count = 0
  begin
    res = Net::HTTP.get_response URI(uri)
    case res
    when Net::HTTPSuccess
      res.body
    when Net::HTTPRedirection
      location = res['Location']
      http_get location, limit - 1
    when Net::HTTPClientError, Net::HTTPServerError
      raise HTTPGetError.new(res)
    else
      raise Error, "HTTP GET failed due to #{res.inspect}"
    end
  rescue Net::OpenTimeout, HTTPGetError => exn
    if retry_count < max_retry_count
      retry_count += 1
      Goodcheck.logger.info "Retry ##{retry_count} - HTTP GET #{uri} due to #{exn.inspect}..."
      sleep 1
      retry
    else
      raise
    end
  end
end

#load(name, &block) ⇒ Object 



42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
 
# File 'lib/goodcheck/import_loader.rb', line 42

def load(name, &block)
  uri = begin
    URI.parse(name)
  rescue URI::InvalidURIError
    nil
  end

  case uri&.scheme
  when nil
    load_file name, &block
  when "file"
    load_file uri.path, &block
  when "http", "https"
    load_http uri, &block
  else
    raise UnexpectedSchemaError.new(uri)
  end
end

#load_file(path, &block) ⇒ Object 



61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/goodcheck/import_loader.rb', line 61

def load_file(path, &block)
  files = Pathname.glob(File.join(config_path.parent.to_path, path), File::FNM_DOTMATCH | File::FNM_EXTGLOB).sort
  if files.empty?
    raise FileNotFound.new(path)
  else
    files.each do |file|
      Goodcheck.logger.info "Reading file: #{file}"
      if unarchiver.tar_gz?(file)
        unarchiver.tar_gz(file.read) do |content, filename|
          block.call(content, filename)
        end
      else
        block.call(file.read, file.to_path)
      end
    end
  end
end

#load_http(uri, &block) ⇒ Object 



83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
 
# File 'lib/goodcheck/import_loader.rb', line 83

def load_http(uri, &block)
  hash = cache_name(uri)
  path = cache_path + hash

  Goodcheck.logger.info "Calculated cache name: #{hash}"

  download = false

  if force_download
    Goodcheck.logger.debug "Downloading: force flag"
    download = true
  end

  if !download && !path.file?
    Goodcheck.logger.debug "Downloading: no cache found"
    download = true
  end

  if !download && path.mtime + expires_in < Time.now
    Goodcheck.logger.debug "Downloading: cache expired"
    download = true
  end

  if download
    path.rmtree if path.exist?
    Goodcheck.logger.info "Downloading content..."
    if unarchiver.tar_gz?(uri.path)
      unarchiver.tar_gz(http_get(uri)) do |content, filename|
        block.call(content, filename)
        write_cache "#{uri}/#{filename}", content
      end
    else
      content = http_get(uri)
      block.call(content, uri.path)
      write_cache uri, content
    end
  else
    Goodcheck.logger.info "Reading content from cache..."
    block.call(path.read, path.to_path)
  end
end

#write_cache(uri, content) ⇒ Object 



125
126
127
128
 
# File 'lib/goodcheck/import_loader.rb', line 125

def write_cache(uri, content)
  path = cache_path + cache_name(uri)
  path.write(content)
end