Class: Fluent::AnalyzeConfigFilter

Inherits:
Filter
  • Object
show all
Includes:
selfself::Constants, Config, PluginHelper::Mixin
Defined in:
lib/fluent/plugin/filter_analyze_config.rb

Overview

Fluentd filter plugin to analyze configuration usage.

For documentation on inspecting parsed configuration elements, see www.rubydoc.info/github/fluent/fluentd/Fluent/Config/Element

Defined Under Namespace

Modules: Constants

Instance Method Summary collapse

Instance Method Details

#configure(conf) ⇒ Object 



217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 217

def configure(conf)
  super
  @log.info('analyze_config plugin: Starting to configure the plugin.')
  if File.file?(@google_fluentd_config_path) &&
     File.file?(@google_fluentd_baseline_config_path)
    @log.info(
      'analyze_config plugin: google-fluentd configuration file found at' \
      " #{@google_fluentd_config_path}. " \
      'google-fluentd baseline configuration file found at' \
      " #{@google_fluentd_baseline_config_path}. " \
      'google-fluentd Analyzing configuration.'
    )

    utils = Common::Utils.new(@log)
    platform = utils.detect_platform(true)
    project_id = utils.get_project_id(platform, nil)
    vm_id = utils.get_vm_id(platform, nil)
    zone = utils.get_location(platform, nil, true)

    # All metadata parameters must now be set.
    utils.check_required_metadata_variables(
      platform, project_id, zone, vm_id
    )

    # Retrieve monitored resource.
    # Fail over to retrieve monitored resource via the legacy path if we
    # fail to get it from Metadata Agent.
    resource = utils.determine_agent_level_monitored_resource_via_legacy(
      platform, nil, false, vm_id, zone
    )

    unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
      @monitoring_type
    )
      @log.warn(
        "analyze_config plugin: monitoring_type #{@monitoring_type} is " \
        'unknown; there will be no metrics.'
      )
    end

    @registry = Monitoring::MonitoringRegistryFactory.create(
      @monitoring_type, project_id, resource, @gcm_service_address
    )
    # Export metrics every 60 seconds.
    timer_execute(:export_config_analysis_metrics, 60) do
      @registry.update_timestamps(PREFIX) if @registry.respond_to? :update_timestamps
      @registry.export
    end

    @log.info('analyze_config plugin: Registering counters.')
    enabled_plugins_counter = @registry.counter(
      :enabled_plugins,
      %i[plugin_name is_default_plugin has_default_config has_ruby_snippet],
      'Enabled plugins',
      PREFIX,
      'GAUGE'
    )
    @log.info(
      'analyze_config plugin: registered enable_plugins counter. ' \
      "#{enabled_plugins_counter}"
    )
    plugin_config_counter = @registry.counter(
      :plugin_config,
      %i[plugin_name param is_present has_default_config],
      'Configuration parameter usage for plugins relevant to Google Cloud.',
      PREFIX,
      'GAUGE'
    )
    @log.info('analyze_config plugin: registered plugin_config counter. ' \
      "#{plugin_config_counter}")
    config_bool_values_counter = @registry.counter(
      :config_bool_values,
      %i[plugin_name param value],
      'Values for bool parameters in Google Cloud plugins',
      PREFIX,
      'GAUGE'
    )
    @log.info('analyze_config plugin: registered config_bool_values ' \
      "counter. #{config_bool_values_counter}")

    config = parse_config(@google_fluentd_config_path)
    @log.debug(
      'analyze_config plugin: successfully parsed google-fluentd' \
      " configuration file at #{@google_fluentd_config_path}. #{config}"
    )
    baseline_config = parse_config(@google_fluentd_baseline_config_path)
    @log.debug(
      'analyze_config plugin: successfully parsed google-fluentd' \
      ' baseline configuration file at' \
      " #{@google_fluentd_baseline_config_path}: #{baseline_config}"
    )

    # Create hash of all baseline elements by their plugin names.
    baseline_elements = Hash[baseline_config.elements.collect do |e|
                               [default_plugin_name(e), e]
                             end]
    baseline_google_element = baseline_config.elements.find do |e|
      e['@type'] == 'google_cloud'
    end

    # Look at each top-level config element and see whether it
    # matches the baseline value.
    #
    # Note on custom configurations: If the plugin has a custom
    # value (e.g. if a tail plugin has pos_file
    # /var/lib/google-fluentd/pos/my-custom-value.pos), then the
    # default_plugin_name (e.g. source/tail/my-custom-value) won't
    # be a key in baseline_elements below, so it won't be
    # used.  Instead it will use the custom_plugin_name
    # (e.g. source/tail).
    config.elements.each do |e|
      plugin_name = default_plugin_name(e)
      if baseline_elements.key?(plugin_name)
        is_default_plugin = true
        has_default_config = (baseline_elements[plugin_name] == e)
      else
        plugin_name = custom_plugin_name(e)
        is_default_plugin = false
        has_default_config = false
      end
      enabled_plugins_counter.increment(
        labels: {
          plugin_name: plugin_name,
          is_default_plugin: is_default_plugin,
          has_default_config: has_default_config,
          has_ruby_snippet: embedded_ruby?(e)
        },
        by: 1
      )

      # Additional metric for Google plugins (google_cloud and
      # detect_exceptions).
      next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])

      GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
        plugin_config_counter.increment(
          labels: {
            plugin_name: e['@type'],
            param: p,
            is_present: e.key?(p),
            has_default_config: (e.key?(p) &&
                                baseline_google_element.key?(p) &&
                                e[p] == baseline_google_element[p])
          },
          by: 1
        )
        next unless e.key?(p) && %w[true false].include?(e[p])

        config_bool_values_counter.increment(
          labels: {
            plugin_name: e['@type'],
            param: p,
            value: e[p] == 'true'
          },
          by: 1
        )
      end
    end
    @log.info(
      'analyze_config plugin: Successfully finished analyzing config.'
    )
  else
    @log.info(
      'analyze_config plugin: google-fluentd configuration file does not ' \
      "exist at #{@google_fluentd_config_path} or google-fluentd " \
      'baseline configuration file does not exist at' \
      " #{@google_fluentd_baseline_config_path}. Skipping configuration " \
      'analysis.'
    )
  end
rescue StandardError => e
  # Do not crash the agent due to configuration analysis failures.
  @log.warn(
    'analyze_config plugin: Failed to optionally analyze the ' \
    "google-fluentd configuration file. Proceeding anyway. Error: #{e}. " \
    "Trace: #{e.backtrace}"
  )
end

#custom_plugin_name(conf_element) ⇒ Object

Returns a name for identifying plugins not in our default config. This should not contain arbitrary user-supplied data.



202
203
204
205
206
207
208
209
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 202

def custom_plugin_name(conf_element)
  if KNOWN_PLUGINS.key?(conf_element.name) &&
     KNOWN_PLUGINS[conf_element.name].include?(conf_element['@type'])
    "#{conf_element.name}/#{conf_element['@type']}"
  else
    conf_element.name.to_s
  end
end

#default_plugin_name(conf_element) ⇒ Object

Returns a name for identifying plugins we ship by default.



189
190
191
192
193
194
195
196
197
198
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 189

def default_plugin_name(conf_element)
  case conf_element['@type']
  when 'syslog'
    "#{conf_element.name}/syslog/#{conf_element['protocol_type']}"
  when 'tail'
    "#{conf_element.name}/tail/#{File.basename(conf_element['pos_file'], '.pos')}"
  else
    "#{conf_element.name}/#{conf_element['@type']}"
  end
end

#embedded_ruby?(conf_element) ⇒ Boolean

Returns:

  • (Boolean) 


211
212
213
214
215
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 211

def embedded_ruby?(conf_element)
  (conf_element.arg.include?('#{') ||
   conf_element.any? { |_, v| v.include?('#{') } ||
   conf_element.elements.any? { |e| embedded_ruby?(e) })
end

#filter(tag, time, record) ⇒ Object

rubocop:disable Lint/UnusedMethodArgument



404
405
406
407
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 404

def filter(tag, time, record)
  # Skip the actual filtering process.
  record
end

#parse_config(path) ⇒ Object 



173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 173

def parse_config(path)
  data = File.open(path, 'r', &:read)
  fname = File.basename(path)
  basepath = File.dirname(path)
  eval_context = Kernel.binding
  # Override instance_eval so that LiteralParser does not actually
  # evaluate the embedded Ruby, but instead just returns the
  # source string.  See
  # https://github.com/fluent/fluentd/blob/master/lib/fluent/config/literal_parser.rb
  def eval_context.instance_eval(code)
    code
  end
  Fluent::Config::V1Parser.parse(data, fname, basepath, eval_context)
end

#shutdown ⇒ Object 



396
397
398
399
400
401
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 396

def shutdown
  super
  # Export metrics on shutdown. This is a best-effort attempt, and it might
  # fail, for instance if there was a recent write to the same time series.
  @registry&.export
end

#start ⇒ Object

rubocop:enable Style/HashSyntax



164
165
166
167
168
169
170
171
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 164

def start
  super
  @log = $log # rubocop:disable Style/GlobalVars

  @log.info(
    'analyze_config plugin: Started the plugin to analyze configuration.'
  )
end