Class: Fluent::AnalyzeConfigFilter

Inherits:
Filter
  • Object
show all
Includes:
selfself::Constants, Config, PluginHelper::Mixin
Defined in:
lib/fluent/plugin/filter_analyze_config.rb

Overview

Fluentd filter plugin to analyze configuration usage.

For documentation on inspecting parsed configuration elements, see www.rubydoc.info/github/fluent/fluentd/Fluent/Config/Element

Defined Under Namespace

Modules: Constants

Instance Method Summary collapse

Instance Method Details

#configure(conf) ⇒ Object 



214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 214

def configure(conf)
  super
  @log.info('analyze_config plugin: Starting to configure the plugin.')
  if File.file?(@google_fluentd_config_path) &&
     File.file?(@google_fluentd_baseline_config_path)
    @log.info(
      'analyze_config plugin: google-fluentd configuration file found at' \
      " #{@google_fluentd_config_path}. " \
      'google-fluentd baseline configuration file found at' \
      " #{@google_fluentd_baseline_config_path}. " \
      'google-fluentd Analyzing configuration.')

    utils = Common::Utils.new(@log)
    platform = utils.detect_platform(true)
    project_id = utils.get_project_id(platform, nil)
    vm_id = utils.get_vm_id(platform, nil)
    zone = utils.get_location(platform, nil, true)

    # All metadata parameters must now be set.
    utils.check_required_metadata_variables(
      platform, project_id, zone, vm_id)

    # Retrieve monitored resource.
    # Fail over to retrieve monitored resource via the legacy path if we
    # fail to get it from Metadata Agent.
    resource = utils.determine_agent_level_monitored_resource_via_legacy(
      platform, nil, false, vm_id, zone)

    unless Monitoring::MonitoringRegistryFactory.supports_monitoring_type(
      @monitoring_type)
      @log.warn(
        "analyze_config plugin: monitoring_type #{@monitoring_type} is " \
        'unknown; there will be no metrics.')
    end

    @registry = Monitoring::MonitoringRegistryFactory.create(
      @monitoring_type, project_id, resource, @gcm_service_address)
    # Export metrics every 60 seconds.
    timer_execute(:export_config_analysis_metrics, 60) { @registry.export }

    @log.info('analyze_config plugin: Registering counters.')
    enabled_plugins_counter = @registry.counter(
      :enabled_plugins,
      [:plugin_name, :is_default_plugin,
       :has_default_config, :has_ruby_snippet],
      'Enabled plugins',
      'agent.googleapis.com/agent/internal/logging/config',
      'GAUGE')
    @log.info(
      'analyze_config plugin: registered enable_plugins counter. ' \
      "#{enabled_plugins_counter}")
    plugin_config_counter = @registry.counter(
      :plugin_config,
      [:plugin_name, :param, :is_present, :has_default_config],
      'Configuration parameter usage for plugins relevant to Google Cloud.',
      'agent.googleapis.com/agent/internal/logging/config',
      'GAUGE')
    @log.info('analyze_config plugin: registered plugin_config counter. ' \
      "#{plugin_config_counter}")
    config_bool_values_counter = @registry.counter(
      :config_bool_values,
      [:plugin_name, :param, :value],
      'Values for bool parameters in Google Cloud plugins',
      'agent.googleapis.com/agent/internal/logging/config',
      'GAUGE')
    @log.info('analyze_config plugin: registered config_bool_values ' \
      "counter. #{config_bool_values_counter}")

    config = parse_config(@google_fluentd_config_path)
    @log.debug(
      'analyze_config plugin: successfully parsed google-fluentd' \
      " configuration file at #{@google_fluentd_config_path}. #{config}")
    baseline_config = parse_config(@google_fluentd_baseline_config_path)
    @log.debug(
      'analyze_config plugin: successfully parsed google-fluentd' \
      ' baseline configuration file at' \
      " #{@google_fluentd_baseline_config_path}: #{baseline_config}")

    # Create hash of all baseline elements by their plugin names.
    baseline_elements = Hash[baseline_config.elements.collect do |e|
                               [default_plugin_name(e), e]
                             end]
    baseline_google_element = baseline_config.elements.find do |e|
      e['@type'] == 'google_cloud'
    end

    # Look at each top-level config element and see whether it
    # matches the baseline value.
    #
    # Note on custom configurations: If the plugin has a custom
    # value (e.g. if a tail plugin has pos_file
    # /var/lib/google-fluentd/pos/my-custom-value.pos), then the
    # default_plugin_name (e.g. source/tail/my-custom-value) won't
    # be a key in baseline_elements below, so it won't be
    # used.  Instead it will use the custom_plugin_name
    # (e.g. source/tail).
    config.elements.each do |e|
      plugin_name = default_plugin_name(e)
      if baseline_elements.key?(plugin_name)
        is_default_plugin = true
        has_default_config = (baseline_elements[plugin_name] == e)
      else
        plugin_name = custom_plugin_name(e)
        is_default_plugin = false
        has_default_config = false
      end
      enabled_plugins_counter.increment(
        labels: {
          plugin_name: plugin_name,
          is_default_plugin: is_default_plugin,
          has_default_config: has_default_config,
          has_ruby_snippet: embedded_ruby?(e)
        },
        by: 1)

      # Additional metric for Google plugins (google_cloud and
      # detect_exceptions).
      next unless GOOGLE_PLUGIN_PARAMS.key?(e['@type'])
      GOOGLE_PLUGIN_PARAMS[e['@type']].each do |p|
        plugin_config_counter.increment(
          labels: {
            plugin_name: e['@type'],
            param: p,
            is_present: e.key?(p),
            has_default_config: (e.key?(p) &&
                                baseline_google_element.key?(p) &&
                                e[p] == baseline_google_element[p])
          },
          by: 1)
        next unless e.key?(p) && %w(true false).include?(e[p])
        config_bool_values_counter.increment(
          labels: {
            plugin_name: e['@type'],
            param: p,
            value: e[p] == 'true'
          },
          by: 1)
      end
    end
    @log.info(
      'analyze_config plugin: Successfully finished analyzing config.')
  else
    @log.info(
      'analyze_config plugin: google-fluentd configuration file does not ' \
      "exist at #{@google_fluentd_config_path} or google-fluentd " \
      'baseline configuration file does not exist at' \
      " #{@google_fluentd_baseline_config_path}. Skipping configuration " \
      'analysis.')
  end
rescue => e
  # Do not crash the agent due to configuration analysis failures.
  @log.warn(
    'analyze_config plugin: Failed to optionally analyze the ' \
    "google-fluentd configuration file. Proceeding anyway. Error: #{e}. " \
    "Trace: #{e.backtrace}")
end

#custom_plugin_name(e) ⇒ Object

Returns a name for identifying plugins not in our default config. This should not contain arbitrary user-supplied data.



199
200
201
202
203
204
205
206
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 199

def custom_plugin_name(e)
  if KNOWN_PLUGINS.key?(e.name) &&
     KNOWN_PLUGINS[e.name].include?(e['@type'])
    "#{e.name}/#{e['@type']}"
  else
    e.name.to_s
  end
end

#default_plugin_name(e) ⇒ Object

Returns a name for identifying plugins we ship by default.



186
187
188
189
190
191
192
193
194
195
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 186

def default_plugin_name(e)
  case e['@type']
  when 'syslog'
    "#{e.name}/syslog/#{e['protocol_type']}"
  when 'tail'
    "#{e.name}/tail/#{File.basename(e['pos_file'], '.pos')}"
  else
    "#{e.name}/#{e['@type']}"
  end
end

#embedded_ruby?(e) ⇒ Boolean

Returns:

  • (Boolean) 


208
209
210
211
212
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 208

def embedded_ruby?(e)
  (e.arg.include?('#{') ||
   e.any? { |_, v| v.include?('#{') } ||
   e.elements.any? { |ee| embedded_ruby?(ee) })
end

#filter(tag, time, record) ⇒ Object

rubocop:disable Lint/UnusedMethodArgument



379
380
381
382
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 379

def filter(tag, time, record)
  # Skip the actual filtering process.
  record
end

#parse_config(path) ⇒ Object 



170
171
172
173
174
175
176
177
178
179
180
181
182
183
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 170

def parse_config(path)
  data = File.open(path, 'r', &:read)
  fname = File.basename(path)
  basepath = File.dirname(path)
  eval_context = Kernel.binding
  # Override instance_eval so that LiteralParser does not actually
  # evaluate the embedded Ruby, but instead just returns the
  # source string.  See
  # https://github.com/fluent/fluentd/blob/master/lib/fluent/config/literal_parser.rb
  def eval_context.instance_eval(code)
    code
  end
  Fluent::Config::V1Parser.parse(data, fname, basepath, eval_context)
end

#shutdown ⇒ Object 



371
372
373
374
375
376
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 371

def shutdown
  super
  # Export metrics on shutdown. This is a best-effort attempt, and it might
  # fail, for instance if there was a recent write to the same time series.
  @registry.export unless @registry.nil?
end

#start ⇒ Object

rubocop:enable Style/HashSyntax



162
163
164
165
166
167
168
 
# File 'lib/fluent/plugin/filter_analyze_config.rb', line 162

def start
  super
  @log = $log # rubocop:disable Style/GlobalVars

  @log.info(
    'analyze_config plugin: Started the plugin to analyze configuration.')
end