Class: Scout::Server

Inherits:
Object
  • Object
show all
Defined in:
lib/es-scout/server.rb

Defined Under Namespace

Classes: APITimeoutError, PluginTimeoutError

Constant Summary collapse

HTTP_HEADERS =

Headers passed up with all API requests.

{ "Client-Version"  => Scout::VERSION,
"Client-Hostname" => Socket.gethostname,
"Accept-Encoding" => "gzip" }
DEFAULT_PLUGIN_TIMEOUT =

A plugin cannot take more than DEFAULT_PLUGIN_TIMEOUT seconds to execute, otherwise, a timeout error is generated. This can be overriden by individual plugins.

60
RUN_DELTA =

A fuzzy range of seconds in which it is okay to rerun a plugin. We consider the interval close enough at this point.

30

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(server, client_key, history_file, logger = nil) ⇒ Server

Creates a new Scout Server connection.



44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/es-scout/server.rb', line 44

def initialize(server, client_key, history_file, logger = nil)
  @server       = server
  @client_key   = client_key
  @history_file = history_file
  @history      = Hash.new
  @logger       = logger
  @plugin_plan  = []
  @directives   = {} # take_snapshots, interval, sleep_interval
  @new_plan     = false
  @local_plugin_path = File.dirname(history_file) # just put overrides and ad-hoc plugins in same directory as history file.
  @plugin_config_path = File.join(@local_plugin_path, "plugins.properties")
  @plugin_config = load_plugin_configs(@plugin_config_path)

  # the block is only passed for install and test, since we split plan retrieval outside the lockfile for run
  if block_given?
    load_history
    yield self
    save_history
  end
end

Dynamic Method Handling

This class handles dynamic methods through the method_missing method

#method_missing(meth, *args, &block) ⇒ Object (private)

Forward Logger methods to an active instance, when there is one.



544
545
546
547
548
549
550
 
# File 'lib/es-scout/server.rb', line 544

def method_missing(meth, *args, &block)
  if (Logger::SEV_LABEL - %w[ANY]).include? meth.to_s.upcase
    @logger.send(meth, *args, &block) unless @logger.nil?
  else
    super
  end
end

Instance Attribute Details

#directives ⇒ Object (readonly)

Returns the value of attribute directives.



40
41
42
 
# File 'lib/es-scout/server.rb', line 40

def directives
  @directives
end

#new_plan ⇒ Object (readonly)

Returns the value of attribute new_plan.



39
40
41
 
# File 'lib/es-scout/server.rb', line 39

def new_plan
  @new_plan
end

#plugin_config ⇒ Object (readonly)

Returns the value of attribute plugin_config.



41
42
43
 
# File 'lib/es-scout/server.rb', line 41

def plugin_config
  @plugin_config
end

Instance Method Details

#create_blank_history ⇒ Object

creates a blank history file



432
433
434
435
436
437
438
 
# File 'lib/es-scout/server.rb', line 432

def create_blank_history
  debug "Creating empty history file..."
  File.open(@history_file, "w") do |file|
    YAML.dump({"last_runs" => Hash.new, "memory" => Hash.new}, file)
  end
  info "History file created."      
end

#fetch_plan ⇒ Object

Retrieves the Plugin Plan from the server. This is the list of plugins to execute, along with all options.

This method has a couple of side effects: 1) it sets the @plugin_plan with either A) whatever is in history, B) the results of the /plan retrieval 2) it sets @checkin_to = true IF so directed by the scout server



92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
 
# File 'lib/es-scout/server.rb', line 92

def fetch_plan
  if refresh?

    url = urlify(:plan)
    info "Fetching plan from server at #{url}..."
    headers = {"x-scout-tty" => ($stdin.tty? ? 'true' : 'false')}

    get(url, "Could not retrieve plan from server.", headers) do |res|
      begin
        body = res.body
        if res["Content-Encoding"] == "gzip" and body and not body.empty?
          body = Zlib::GzipReader.new(StringIO.new(body)).read
        end

        body_as_hash = JSON.parse(body)

        # Ensure all the plugins in the new plan are properly signed. Load the public key for this.
        public_key_text = File.read(File.join( File.dirname(__FILE__), *%w[.. .. data code_key.pub] ))
        debug "Loaded public key used for verifying code signatures (#{public_key_text.size} bytes)"
        code_public_key = OpenSSL::PKey::RSA.new(public_key_text)

        temp_plugins=Array(body_as_hash["plugins"])
        plugin_signature_error = false
        temp_plugins.each do |plugin|
          signature=plugin['signature']
          id_and_name = "#{plugin['id']}-#{plugin['name']}".sub(/\A-/, "")
          if signature
            code=plugin['code'].gsub(/ +$/,'') # we strip trailing whitespace before calculating signatures. Same here.
            decoded_signature=Base64.decode64(signature)
            if !code_public_key.verify(OpenSSL::Digest::SHA1.new, decoded_signature, code)
              info "#{id_and_name} signature doesn't match!"
              plugin_signature_error=true
            end
          else
            info "#{id_and_name} has no signature!"
            plugin_signature_error=true
          end
        end


        if(!plugin_signature_error)
          @plugin_plan = temp_plugins
          @directives = body_as_hash["directives"].is_a?(Hash) ? body_as_hash["directives"] : Hash.new
          @history["plan_last_modified"] = res["last-modified"]
          @history["old_plugins"]        = @plugin_plan.clone # important that the plan is cloned -- we're going to add local plugins, and they shouldn't go into history
          @history["directives"]         = @directives

          info "Plan loaded.  (#{@plugin_plan.size} plugins:  " +
               "#{@plugin_plan.map { |p| p['name'] }.join(', ')})" +
               ". Directives: #{@directives.to_a.map{|a|  "#{a.first}:#{a.last}"}.join(", ")}"

          @new_plan = true # used in determination if we should checkin this time or not
        else
          info "There was a problem with plugin signatures. Reusing old plan."
          @plugin_plan = Array(@history["old_plugins"])
          @directives = @history["directives"] || Hash.new
        end

        # Add local plugins to the plan. Note that local plugins are NOT saved to history file
        @plugin_plan += get_local_plugins
      rescue Exception =>e
        fatal "Plan from server was malformed: #{e.message} - #{e.backtrace}"
        exit
      end
    end
  else
    info "Plan not modified."
    @plugin_plan = Array(@history["old_plugins"])
    @plugin_plan += get_local_plugins
    @directives = @history["directives"] || Hash.new
  end
end

#get_local_plugins ⇒ Object

returns an array of hashes representing local plugins found on the filesystem The glob pattern requires that filenames begin with a letter, which excludes plugin overrides (like 12345.rb)



168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
 
# File 'lib/es-scout/server.rb', line 168

def get_local_plugins
  local_plugin_paths=Dir.glob(File.join(@local_plugin_path,"[a-zA-Z]*.rb"))
  local_plugin_paths.map do |plugin_path|
    begin
      {
        'name' => File.basename(plugin_path),
        'local_filename' => File.basename(plugin_path),
        'origin' => 'LOCAL',
        'code' => File.read(plugin_path),
        'interval' => 0
      }
    rescue => e
      info "Error trying to read local plugin: #{plugin_path} -- #{e.backtrace.join('\n')}"
      nil
    end
  end.compact
end

#load_history ⇒ Object

Loads the history file from disk. If the file does not exist, it creates one.



411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
 
# File 'lib/es-scout/server.rb', line 411

def load_history
  if !File.exist?(@history_file) || File.zero?(@history_file)
    create_blank_history
  end
  debug "Loading history file..."
  contents=File.read(@history_file)
  begin
    @history = YAML.load(contents)
  rescue => e
    backup_path=File.join(File.dirname(@history_file), "history.corrupt")
    info "Couldn't parse the history file. Deleting it and resetting to an empty history file. Keeping a backup at #{backup_path}"
    File.open(backup_path,"w"){|f|f.write contents}
    File.delete(@history_file)
    create_blank_history
    @history = File.open(@history_file) { |file| YAML.load(file) }
  end

  info "History file loaded."
end

#next_checkin ⇒ Object

returns a human-readable representation of the next checkin, i.e., 5min 30sec



220
221
222
223
224
225
226
227
 
# File 'lib/es-scout/server.rb', line 220

def next_checkin
  secs= @directives['interval'].to_i*60 - (Time.now.to_i - Time.at(@history['last_checkin']).to_i).abs
  minutes=(secs.to_f/60).floor
  secs=secs%60
  "#{minutes}min #{secs} sec"
rescue
  "[next scout invocation]"
end

#ping_key ⇒ Object 



192
193
194
 
# File 'lib/es-scout/server.rb', line 192

def ping_key
  (@history['directives'] || {})['ping_key']
end

#prepare_checkin ⇒ Object

Prepares a check-in data structure to hold Plugin generated data.



394
395
396
397
398
399
400
401
 
# File 'lib/es-scout/server.rb', line 394

def prepare_checkin
  @checkin = { :reports   => Array.new,
               :alerts    => Array.new,
               :errors    => Array.new,
               :summaries => Array.new,
               :snapshot  => '',
               :config_path => File.expand_path(File.dirname(@history_file))}
end

#process_plugin(plugin) ⇒ Object

This is the heart of Scout.

First, it determines if a plugin is past interval and needs to be run. If it is, it simply evals the code, compiling it. It then loads the plugin and runs it with a PLUGIN_TIMEOUT time limit. The plugin generates data, alerts, and errors. In addition, it will set memory and last_run information in the history file.

The plugin argument is a hash with keys: id, name, code, timeout, options, signature.



260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
 
# File 'lib/es-scout/server.rb', line 260

def process_plugin(plugin)
  info "Processing the '#{plugin['name']}' plugin:"
  id_and_name = "#{plugin['id']}-#{plugin['name']}".sub(/\A-/, "")
  plugin_id = plugin['id']
  last_run    = @history["last_runs"][id_and_name] ||
                @history["last_runs"][plugin['name']]
  memory      = @history["memory"][id_and_name] ||
                @history["memory"][plugin['name']]
  run_time    = Time.now
  delta       = last_run.nil? ? nil : run_time -
                                      (last_run + plugin['interval'] * 60)
  if last_run.nil? or delta.between?(-RUN_DELTA, 0) or delta >= 0
    debug "Plugin is past interval and needs to be run.  " +
          "(last run:  #{last_run || 'nil'})"
    code_to_run = plugin['code']
    if plugin_id && plugin_id != ""
      override_path=File.join(@local_plugin_path, "#{plugin_id}.rb")
      # debug "Checking for local plugin override file at #{override_path}"
      if File.exist?(override_path)
        code_to_run = File.read(override_path)
        debug "Override file found - Using #{code_to_run.size} chars of code in #{override_path} for plugin id=#{plugin_id}"
        plugin['origin'] = "OVERRIDE"
      else
        plugin['origin'] = nil
      end
    end
    debug "Compiling plugin..."
    begin
      eval( code_to_run,
            TOPLEVEL_BINDING,
            plugin['path'] || plugin['name'] )
      info "Plugin compiled."
    rescue Exception
      raise if $!.is_a? SystemExit
      error "Plugin would not compile: #{$!.message}"
      @checkin[:errors] << build_report(plugin,:subject => "Plugin would not compile", :body=>"#{$!.message}\n\n#{$!.backtrace}")
      return
    end

    # Lookup any local options in plugin_config.properies as needed
    options=(plugin['options'] || Hash.new)
    options.each_pair do |k,v|
      if v=~/^lookup:(.+)$/
        lookup_key = $1.strip
        if plugin_config[lookup_key]
          options[k]=plugin_config[lookup_key]
        else
          info "Plugin #{id_and_name}: option #{k} appears to be a lookup, but we can't find #{lookup_key} in #{@plugin_config_path}"
        end
      end
    end


    debug "Loading plugin..."
    if job = Plugin.last_defined.load( last_run, (memory || Hash.new), options)
      info "Plugin loaded."
      debug "Running plugin..."
      begin
        data    = {}
        timeout = plugin['timeout'].to_i
        timeout = DEFAULT_PLUGIN_TIMEOUT unless timeout > 0
        Timeout.timeout(timeout, PluginTimeoutError) do
          data = job.run
        end
      rescue Timeout::Error, PluginTimeoutError
        error "Plugin took too long to run."
        @checkin[:errors] << build_report(plugin,
                                          :subject => "Plugin took too long to run",
                                          :body=>"Execution timed out.")
        return
      rescue Exception
        raise if $!.is_a? SystemExit
        error "Plugin failed to run: #{$!.class}: #{$!.message}\n" +
              "#{$!.backtrace.join("\n")}"
        @checkin[:errors] << build_report(plugin,
                                          :subject => "Plugin failed to run",
                                          :body=>"#{$!.class}: #{$!.message}\n#{$!.backtrace.join("\n")}")
      end
      info "Plugin completed its run."
      
      %w[report alert error summary].each do |type|
        plural  = "#{type}s".sub(/ys\z/, "ies").to_sym
        reports = data[plural].is_a?(Array) ? data[plural] :
                                              [data[plural]].compact
        if report = data[type.to_sym]
          reports << report
        end
        reports.each do |fields|
          @checkin[plural] << build_report(plugin, fields)
        end
      end
      
      @history["last_runs"].delete(plugin['name'])
      @history["memory"].delete(plugin['name'])
      @history["last_runs"][id_and_name] = run_time
      @history["memory"][id_and_name]    = data[:memory]
    else
      @checkin[:errors] << build_report(
        plugin,
        :subject => "Plugin would not load."
      )
    end
  else
    debug "Plugin does not need to be run at this time.  " +
          "(last run:  #{last_run || 'nil'})"
  end
  data
ensure
  if Plugin.last_defined
    debug "Removing plugin code..."
    begin
      Object.send(:remove_const, Plugin.last_defined.to_s.split("::").first)
      Plugin.last_defined = nil
      info "Plugin Removed."
    rescue
      raise if $!.is_a? SystemExit
      error "Unable to remove plugin."
    end
  end
  info "Plugin '#{plugin['name']}' processing complete."
end

#refresh? ⇒ Boolean

Returns:

  • (Boolean) 


65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/es-scout/server.rb', line 65

def refresh?
  return true if !ping_key

  url=URI.join( @server.sub("https://","http://"), "/clients/#{ping_key}/ping.scout")

  headers = {"x-scout-tty" => ($stdin.tty? ? 'true' : 'false')}
  if @history["plan_last_modified"] and @history["old_plugins"]
    headers["If-Modified-Since"] = @history["plan_last_modified"]
  end
  get(url, "Could not ping #{url} for refresh info", headers) do |res|
    if res.is_a?(Net::HTTPNotModified)
      return false
    else
      info "Plan has been modified!"
      return true
    end
  end
end

#run_plugins_by_plan ⇒ Object

Runs all plugins from the given plan. Calls process_plugin on each plugin.



231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
 
# File 'lib/es-scout/server.rb', line 231

def run_plugins_by_plan
  prepare_checkin
  @plugin_plan.each do |plugin|
    begin
      process_plugin(plugin)
    rescue Exception
      @checkin[:errors] << build_report(
        plugin,
        :subject => "Exception:  #{$!.message}.",
        :body    => $!.backtrace
      )
      error("Encountered an error: #{$!.message}")
      puts $!.backtrace.join('\n')
    end
  end
  take_snapshot if @directives['take_snapshots']
  checkin
end

#save_history ⇒ Object

Saves the history file to disk.



441
442
443
444
445
 
# File 'lib/es-scout/server.rb', line 441

def save_history
  debug "Saving history file..."
  File.open(@history_file, "w") { |file| YAML.dump(@history, file) }
  info "History file saved."
end

#show_checkin(printer = :p) ⇒ Object 



403
404
405
 
# File 'lib/es-scout/server.rb', line 403

def show_checkin(printer = :p)
  send(printer, @checkin)
end

#sleep_interval ⇒ Object

To distribute pings across a longer timeframe, the agent will sleep for a given amount of time. When using the –force option the sleep_interval is ignored.



188
189
190
 
# File 'lib/es-scout/server.rb', line 188

def sleep_interval
  (@history['directives'] || {})['sleep_interval'].to_f
end

#take_snapshot ⇒ Object

captures a list of processes running at this moment



384
385
386
387
388
389
390
391
 
# File 'lib/es-scout/server.rb', line 384

def take_snapshot
  info "Taking a process snapshot"
  ps=%x(ps aux).split("\n")[1..-1].join("\n") # get rid of the header line
  @checkin[:snapshot]=ps
  rescue Exception
    error "unable to capture processes on this server. #{$!.message}"
    return nil
end

#time_to_checkin? ⇒ Boolean

uses values from history and current time to determine if we should checkin at this time

Returns:

  • (Boolean) 


197
198
199
200
201
202
203
204
205
 
# File 'lib/es-scout/server.rb', line 197

def time_to_checkin?
  @history['last_checkin'] == nil ||
          @directives['interval'] == nil ||
          (Time.now.to_i - Time.at(@history['last_checkin']).to_i).abs+15 > @directives['interval'].to_i*60
rescue
  debug "Failed to calculate time_to_checkin. @history['last_checkin']=#{@history['last_checkin']}. "+
          "@directives['interval']=#{@directives['interval']}. Time.now.to_i=#{Time.now.to_i}"
  return true
end

#time_to_ping? ⇒ Boolean

uses values from history and current time to determine if we should ping the server at this time

Returns:

  • (Boolean) 


208
209
210
211
212
213
214
215
216
217
 
# File 'lib/es-scout/server.rb', line 208

def time_to_ping?
  return true if
  @history['last_ping'] == nil ||
          @directives['ping_interval'] == nil ||
          (Time.now.to_i - Time.at(@history['last_ping']).to_i).abs+15 > @directives['ping_interval'].to_i*60
rescue
  debug "Failed to calculate time_to_ping. @history['last_ping']=#{@history['last_ping']}. "+
          "@directives['ping_interval']=#{@directives['ping_interval']}. Time.now.to_i=#{Time.now.to_i}"
  return true
end