Class: BoltServer::FileCache

Inherits:
Object
  • Object
show all
Defined in:
lib/bolt_server/file_cache.rb

Defined Under Namespace

Classes: Error

Constant Summary collapse

PURGE_TIMEOUT = 
60 * 60
PURGE_INTERVAL = 
24 * PURGE_TIMEOUT
PURGE_TTL = 
7 * PURGE_INTERVAL

Instance Method Summary collapse

Constructor Details

#initialize(config, executor: Concurrent::SingleThreadExecutor.new, purge_interval: PURGE_INTERVAL, purge_timeout: PURGE_TIMEOUT, purge_ttl: PURGE_TTL, cache_dir_mutex: Concurrent::ReadWriteLock.new, do_purge: true) ⇒ FileCache

Returns a new instance of FileCache.



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/bolt_server/file_cache.rb', line 27

def initialize(config,
               executor: Concurrent::SingleThreadExecutor.new,
               purge_interval: PURGE_INTERVAL,
               purge_timeout: PURGE_TIMEOUT,
               purge_ttl: PURGE_TTL,
               cache_dir_mutex: Concurrent::ReadWriteLock.new,
               do_purge: true)
  @executor = executor
  @cache_dir = config['cache-dir']
  @config = config
  @logger = Bolt::Logger.logger(self)
  @cache_dir_mutex = cache_dir_mutex

  if do_purge
    @purge = Concurrent::TimerTask.new(execution_interval: purge_interval,
                                       run_now: true) { expire(purge_ttl, purge_timeout) }
    @purge.execute
  end
end

Instance Method Details

#cache_project_file(versioned_project, file_name, data) ⇒ Object 



194
195
196
197
198
 
# File 'lib/bolt_server/file_cache.rb', line 194

def cache_project_file(versioned_project, file_name, data)
  file_dir = create_cache_dir(versioned_project)
  file_path = File.join(file_dir, file_name)
  serial_execute { File.open(file_path, 'w') { |f| f.write(data) } }
end

#check_file(file_path, sha) ⇒ Object 



112
113
114
 
# File 'lib/bolt_server/file_cache.rb', line 112

def check_file(file_path, sha)
  File.exist?(file_path) && Digest::SHA256.file(file_path) == sha
end

#client ⇒ Object 



65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
 
# File 'lib/bolt_server/file_cache.rb', line 65

def client
  # rubocop:disable Naming/VariableNumber
  @client ||= begin
    uri = URI(@config['file-server-uri'])
    https = Net::HTTP.new(uri.host, uri.port)
    https.use_ssl = true
    https.ssl_version = :TLSv1_2
    https.ca_file = @config['ssl-ca-cert']
    https.cert = OpenSSL::X509::Certificate.new(ssl_cert)
    https.key = OpenSSL::PKey::RSA.new(ssl_key)
    https.verify_mode = OpenSSL::SSL::VERIFY_PEER
    https.open_timeout = @config['file-server-conn-timeout']
    https
  end
  # rubocop:enable Naming/VariableNumber
end

#create_cache_dir(sha) ⇒ Object

Create a cache dir if necessary and update it’s last write time. Returns the dir. Acquires @cache_dir_mutex to ensure we don’t try to purge the directory at the same time. Uses the directory mtime because it’s simpler to ensure the directory exists and update mtime in a single place than with a file in a directory that may not exist.



126
127
128
129
130
131
132
133
134
 
# File 'lib/bolt_server/file_cache.rb', line 126

def create_cache_dir(sha)
  file_dir = File.join(@cache_dir, sha)
  @cache_dir_mutex.with_read_lock do
    # mkdir_p doesn't error if the file exists
    FileUtils.mkdir_p(file_dir, mode: 0o750)
    FileUtils.touch(file_dir)
  end
  file_dir
end

#download_file(file_path, sha, uri) ⇒ Object 



136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
 
# File 'lib/bolt_server/file_cache.rb', line 136

def download_file(file_path, sha, uri)
  if check_file(file_path, sha)
    @logger.debug("File was downloaded while queued: #{file_path}")
    return file_path
  end

  @logger.debug("Downloading file: #{file_path}")

  tmpfile = Tempfile.new(sha, tmppath)
  request_file(uri['path'], uri['params'], tmpfile)

  if Digest::SHA256.file(tmpfile.path) == sha
    # mv doesn't error if the file exists
    FileUtils.mv(tmpfile.path, file_path)
    @logger.debug("Downloaded file: #{file_path}")
    file_path
  else
    msg = "Downloaded file did not match checksum for: #{file_path}"
    @logger.warn msg
    raise Error, msg
  end
end

#expire(purge_ttl, purge_timeout) ⇒ Object 



174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/bolt_server/file_cache.rb', line 174

def expire(purge_ttl, purge_timeout)
  expired_time = Time.now - purge_ttl
  Timeout.timeout(purge_timeout) do
    @cache_dir_mutex.with_write_lock do
      Dir.glob(File.join(@cache_dir, '*')).select { |f| File.directory?(f) }.each do |dir|
        if (mtime = File.mtime(dir)) < expired_time && dir != tmppath
          @logger.debug("Removing #{dir}, last used at #{mtime}")
          FileUtils.remove_dir(dir)
        end
      end
    end
  end
end

#get_cached_project_file(versioned_project, file_name) ⇒ Object 



188
189
190
191
192
 
# File 'lib/bolt_server/file_cache.rb', line 188

def get_cached_project_file(versioned_project, file_name)
  file_dir = create_cache_dir(versioned_project)
  file_path = File.join(file_dir, file_name)
  serial_execute { File.read(file_path) if File.exist?(file_path) }
end

#request_file(path, params, file) ⇒ Object 



82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
 
# File 'lib/bolt_server/file_cache.rb', line 82

def request_file(path, params, file)
  uri = "#{@config['file-server-uri'].chomp('/')}#{path}"
  uri = URI(uri)
  uri.query = URI.encode_www_form(params)

  req = Net::HTTP::Get.new(uri)

  begin
    client.request(req) do |resp|
      if resp.code != "200"
        msg = "Failed to download file: #{resp.body}"
        @logger.warn resp.body
        raise Error, msg
      end
      resp.read_body do |chunk|
        file.write(chunk)
      end
    end
  rescue StandardError => e
    if e.is_a?(Bolt::Error)
      raise e
    else
      @logger.warn e
      raise Error, "Failed to download file: #{e.message}"
    end
  end
ensure
  file.close
end

#serial_execute(&block) ⇒ Object 



116
117
118
119
120
 
# File 'lib/bolt_server/file_cache.rb', line 116

def serial_execute(&block)
  promise = Concurrent::Promise.new(executor: @executor, &block).execute.wait
  raise promise.reason if promise.rejected?
  promise.value
end

#setup ⇒ Object 



51
52
53
54
55
 
# File 'lib/bolt_server/file_cache.rb', line 51

def setup
  FileUtils.mkdir_p(@cache_dir)
  FileUtils.mkdir_p(tmppath)
  self
end

#ssl_cert ⇒ Object 



57
58
59
 
# File 'lib/bolt_server/file_cache.rb', line 57

def ssl_cert
  @ssl_cert ||= File.read(@config['ssl-cert'])
end

#ssl_key ⇒ Object 



61
62
63
 
# File 'lib/bolt_server/file_cache.rb', line 61

def ssl_key
  @ssl_key ||= File.read(@config['ssl-key'])
end

#tmppath ⇒ Object 



47
48
49
 
# File 'lib/bolt_server/file_cache.rb', line 47

def tmppath
  File.join(@cache_dir, 'tmp')
end

#update_file(file_data) ⇒ Object

If the file doesn’t exist or is invalid redownload it This downloads, validates and moves into place



161
162
163
164
165
166
167
168
169
170
171
172
 
# File 'lib/bolt_server/file_cache.rb', line 161

def update_file(file_data)
  sha = file_data['sha256']
  file_dir = create_cache_dir(file_data['sha256'])
  file_path = File.join(file_dir, File.basename(file_data['filename']))
  if check_file(file_path, sha)
    @logger.debug("Using prexisting file: #{file_path}")
    return file_path
  end

  @logger.debug("Queueing download for: #{file_path}")
  serial_execute { download_file(file_path, sha, file_data['uri']) }
end