Class: Fluent::Plugin::SecondaryFileOutput

Inherits:
Output
  • Object
show all
Defined in:
lib/fluent/plugin/out_secondary_file.rb

Defined Under Namespace

Classes: FileAlreadyExist

Constant Summary collapse

PLACEHOLDER_REGEX = 
/\${(tag(\[\d+\])?|[\w.@-]+)}/

Constants inherited from Output

Output::BUFFER_STATS_KEYS, Output::CHUNKING_FIELD_WARN_NUM, Output::CHUNK_ID_PLACEHOLDER_PATTERN, Output::CHUNK_KEY_PATTERN, Output::CHUNK_KEY_PLACEHOLDER_PATTERN, Output::CHUNK_TAG_PLACEHOLDER_PATTERN, Output::FORMAT_COMPRESSED_MSGPACK_STREAM, Output::FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT, Output::FORMAT_MSGPACK_STREAM, Output::FORMAT_MSGPACK_STREAM_TIME_INT, Output::TIMESTAMP_CHECK_BASE_TIME, Output::TIME_KEY_PLACEHOLDER_THRESHOLDS, Output::UNRECOVERABLE_ERRORS

Constants included from Configurable

Configurable::CONFIG_TYPE_REGISTRY

Instance Attribute Summary

Attributes inherited from Output

#as_secondary, #buffer, #chunk_key_accessors, #chunk_key_tag, #chunk_key_time, #chunk_keys, #delayed_commit, #delayed_commit_timeout, #dequeued_chunks, #dequeued_chunks_mutex, #output_enqueue_thread_waiting, #retry, #retry_for_error_chunk, #secondary, #timekey_zone

Attributes included from Fluent::PluginLoggerMixin

#log

Attributes inherited from Base

#under_plugin_development

Instance Method Summary collapse

Methods inherited from Output

#acts_as_secondary, #actual_flush_thread_count, #after_shutdown, #after_start, #backup_chunk, #before_shutdown, #calculate_timekey, #check_slow_flush, #chunk_for_test, #close, #commit_write, #emit_buffered, #emit_count, #emit_events, #emit_records, #emit_size, #emit_sync, #enqueue_thread_run, #enqueue_thread_wait, #execute_chunking, #extract_placeholders, #flush_thread_run, #flush_thread_wakeup, #force_flush, #format, #formatted_to_msgpack_binary, #formatted_to_msgpack_binary?, #generate_format_proc, #get_placeholders_keys, #get_placeholders_tag, #get_placeholders_time, #handle_limit_reached, #handle_stream_simple, #handle_stream_with_custom_format, #handle_stream_with_standard_format, #implement?, #initialize, #interrupt_flushes, #keep_buffer_config_compat, #log_retry_error, #metadata, #next_flush_time, #num_errors, #placeholder_validate!, #placeholder_validators, #prefer_buffered_processing, #prefer_delayed_commit, #process, #retry_state, #rollback_count, #rollback_write, #shutdown, #start, #statistics, #stop, #submit_flush_all, #submit_flush_once, #support_in_v12_style?, #synchronize_in_threads, #synchronize_path, #synchronize_path_in_workers, #terminate, #try_flush, #try_rollback_all, #try_rollback_write, #try_write, #update_retry_state, #write_count, #write_guard

Methods included from UniqueId::Mixin

#dump_unique_id_hex, #generate_unique_id

Methods included from Fluent::PluginHelper::Mixin

included

Methods included from Fluent::PluginLoggerMixin

included, #initialize, #terminate

Methods included from Fluent::PluginId

#initialize, #plugin_id, #plugin_id_configured?, #plugin_id_for_test?, #plugin_root_dir, #stop

Methods inherited from Base

#acquire_worker_lock, #after_shutdown, #after_shutdown?, #after_start, #after_started?, #before_shutdown, #before_shutdown?, #called_in_test?, #close, #closed?, #configured?, #context_router, #context_router=, #fluentd_worker_id, #get_lock_path, #has_router?, #initialize, #inspect, #plugin_root_dir, #reloadable_plugin?, #shutdown, #shutdown?, #start, #started?, #stop, #stopped?, #string_safe_encoding, #terminate, #terminated?

Methods included from SystemConfig::Mixin

#system_config, #system_config_override

Methods included from Configurable

#config, #configure_proxy_generate, #configured_section_create, included, #initialize, lookup_type, register_type

Constructor Details

This class inherits a constructor from Fluent::Plugin::Output

Instance Method Details

#configure(conf) ⇒ Object 



36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/fluent/plugin/out_secondary_file.rb', line 36

def configure(conf)
  super

  unless @as_secondary
    raise Fluent::ConfigError, "This plugin can only be used in the <secondary> section"
  end

  if @basename.include?("/")
    raise Fluent::ConfigError, "basename should not include `/`"
  end

  @path_without_suffix = File.join(@directory, @basename)
  validate_compatible_with_primary_buffer!(@path_without_suffix)

  @suffix = case @compress
            when :text
              ""
            when :gzip
              ".gz"
            end

  test_path = @path_without_suffix
  unless Fluent::FileUtil.writable_p?(test_path)
    raise Fluent::ConfigError, "out_secondary_file: `#{@directory}` should be writable"
  end

  @dir_perm = system_config.dir_permission || Fluent::DEFAULT_DIR_PERMISSION
  @file_perm = system_config.file_permission || Fluent::DEFAULT_FILE_PERMISSION
end

#multi_workers_ready? ⇒ Boolean

Returns:

  • (Boolean) 


66
67
68
 
# File 'lib/fluent/plugin/out_secondary_file.rb', line 66

def multi_workers_ready?
  true
end

#write(chunk) ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
 
# File 'lib/fluent/plugin/out_secondary_file.rb', line 70

def write(chunk)
  path_without_suffix = extract_placeholders(@path_without_suffix, chunk)
  generate_path(path_without_suffix) do |path|
    FileUtils.mkdir_p File.dirname(path), mode: @dir_perm

    case @compress
    when :text
      File.open(path, "ab", @file_perm) {|f|
        f.flock(File::LOCK_EX)
        chunk.write_to(f)
      }
    when :gzip
      File.open(path, "ab", @file_perm) {|f|
        f.flock(File::LOCK_EX)
        gz = Zlib::GzipWriter.new(f)
        chunk.write_to(gz)
        gz.close
      }
    end
  end
end