Class: LogStash::Filters::Foreach

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/filters/foreach.rb

Overview

This filter will split event by array field, and later join back

Constant Summary collapse

FAILURE_TAG = 
'_foreach_failure'.freeze
@@configuration_data = 
{}
@@event_data = 
{}
@@mutex = 
Mutex.new

Instance Method Summary collapse

Instance Method Details

#close ⇒ Object 



249
250
251
 
# File 'lib/logstash/filters/foreach.rb', line 249

def close
  @@configuration_data = {}
end

#filter(event) ⇒ Object 



74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
 
# File 'lib/logstash/filters/foreach.rb', line 74

def filter(event)

  @logger.debug("Foreach plugin:", :task_id => @task_id, :array_field => @array_field, :join_fields => @join_fields, :end => @end, :timeout => @timeout, :event => event.to_hash, :metadata => event.get('@metadata'))

  passthrough = false

  task_id = event.sprintf(@task_id)
  if task_id.nil? || task_id == @task_id

    @logger.trace("Foreach plugin: if task_id.nil? || task_id == @task_id");

    @logger.warn("Foreach plugin: #{@task_id} should be calculated into value (not '#{task_id}'). Passing through")
    event.tag(FAILURE_TAG)
    passthrough = true

  else

    @logger.trace("Foreach plugin: else task_id.nil? || task_id == @task_id");

    @@mutex.synchronize do

      if !@@configuration_data.has_key?(@task_id) or !@@configuration_data[@task_id].end_filter_configured

        @logger.trace("Foreach plugin: if !@@configuration_data.has_key?(@task_id) or !@@configuration_data[@task_id].end_filter_configured");

        raise LogStash::ConfigurationError, "Foreach plugin: For task_id pattern '#{@task_id}', there should be one `start` and one `end` filter."
      end

      configuration = @@configuration_data[@task_id]

      if !@end

        @logger.trace("Foreach plugin: if !@end");

        array_field = event.get(@array_field)

        if !array_field.is_a?(Array)

          @logger.trace("Foreach plugin: if !array_field.is_a?(Array)");

          @logger.warn("Foreach plugin: Field should be of Array type. field:#{@array_field} is of type = #{array_field.class}. Passing through")
          event.tag(FAILURE_TAG)
          passthrough = true

        elsif @@event_data.has_key?(task_id)

          @logger.trace("Foreach plugin: elsif @@event_data.has_key?(task_id)");

          @logger.warn("Foreach plugin: task_id should be unique. Duplicate value found: '#{task_id}'. Passing through")
          event.tag(FAILURE_TAG)
          passthrough = true

        else

          @logger.trace("Foreach plugin: else !array_field.is_a?(Array)");

          @@event_data[task_id] = LogStash::Filters::Foreach::Element.new(configuration, Time.now(), event.clone, event.get('@metadata').clone, configuration.join_fields)
          event_data = @@event_data[task_id]

          if array_field.length == 0

            passthrough = true

          else
            array_field.each do |value|

              @logger.trace("Foreach plugin: array_field.each do |value|", :value => value);

              next if value.nil? or value.empty?

              event_split = event.clone
              @logger.debug("Foreach plugin: Split event", :field => @array_field, :value => value)
              event_split.set(@array_field, value)
              event_data.counter += 1

              filter_matched(event_split)
              yield event_split
            end

            event.cancel
          end

        end

      else # if !@end

        @logger.trace("Foreach plugin: else !@end");

        if !@@event_data.has_key?(task_id)

          @logger.trace("Foreach plugin: if !@@event_data.has_key?(task_id)");

          @logger.warn("Foreach plugin: found `end` event for task_id = '#{task_id}' without `start` event. Passing through")
          event.tag(FAILURE_TAG)
          passthrough = true

        else

          @logger.trace("Foreach plugin: else !@@event_data.has_key?(task_id)");

          @logger.debug("Foreach plugin: Join event back", :field => configuration.array_field, :value => event.get(configuration.array_field))

          event_data = @@event_data[task_id]

          if event_data.sub_events_count == 0

            ret_event = event_data.event()
            filter_matched(ret_event)
            yield ret_event
            @@event_data.delete(task_id)

          else

            event_data.lastevent_timestamp = Time.now()

            event_data.add_join_fields_values(event)
            event_data.counter -= 1

            if event_data.counter == 0

              @logger.trace("Foreach plugin: if event_data.counter == 0");

              ret_event = event_data.event()
              filter_matched(ret_event)
              yield ret_event
              @@event_data.delete(task_id)
            end

          end # if event_data.sub_events_count == 0

          event.cancel

        end

      end

    end # @@mutex.synchronize

  end # task_id.nil? || task_id == @task_id

  if passthrough

    @logger.trace("Foreach plugin: if passthrough");

    filter_matched(event)
  end

end

#flush(options = {}) ⇒ Object

def filter



223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
 
# File 'lib/logstash/filters/foreach.rb', line 223

def flush(options = {})
  events_to_flush = []
  if @end
    @@mutex.synchronize do
      @@event_data.each do |task_id, obj|
        if obj.lastevent_timestamp < Time.now() - obj.configuration.timeout
          if obj.counter < obj.sub_events_count
            @logger.warn("Foreach plugin: Flushing partly processed event with task_id = '#{obj.initial_event.sprintf(@task_id)}' after timeout = '#{obj.configuration.timeout.to_s}'")
            events_to_flush << obj.event()
          else
            @logger.warn("Foreach plugin: Removing unprocessed event with task_id = '#{obj.initial_event.sprintf(@task_id)}' after timeout = '#{obj.configuration.timeout.to_s}'")
          end
          @@event_data.delete(task_id)
        end
      end
    end # @@mutex.synchronize
  end
  return events_to_flush

end

#periodic_flush ⇒ Object

def flush



244
245
246
 
# File 'lib/logstash/filters/foreach.rb', line 244

def periodic_flush
  true
end

#register ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/logstash/filters/foreach.rb', line 43

def register

  # validate task_id option
  if !@task_id.match(/%\{.+\}/)
    raise LogStash::ConfigurationError, "Foreach plugin: task_id pattern '#{@task_id}' must contain a dynamic expression like '%{field}'"
  end

  if !@end
    if @@configuration_data.has_key?(@task_id)
      raise LogStash::ConfigurationError, "Foreach plugin: For task_id pattern '#{@task_id}', there are more than one filters defined. There should be only one `start` and one `end` filter with the same task_id."
    end
    if !@array_field.is_a?(String)
      raise LogStash::ConfigurationError, "Foreach plugin: For task_id pattern '#{@task_id}': array_field should be a field name, but it is of type = #{@array_field.class}"
    end
    if !@join_fields.is_a?(Array)
      raise LogStash::ConfigurationError, "Foreach plugin: For task_id pattern '#{@task_id}': join_fields should be an Array of fields, but it is of type = #{@join_fields.class}"
    end
    @@configuration_data[@task_id] = LogStash::Filters::Foreach::Configuration.new(@array_field, @join_fields, @timeout)
  else
    if !@@configuration_data.has_key?(@task_id)
      raise LogStash::ConfigurationError, "Foreach plugin: For task_id pattern '#{@task_id}', there are no `start` filter. You should declare `start` filter before `end` filter."
    elsif @@configuration_data[@task_id].end_filter_configured
      raise LogStash::ConfigurationError, "Foreach plugin: For task_id pattern '#{@task_id}', there are more than one filters defined. There should be only one `start` and one `end` filter with the same task_id."
    end
    @@configuration_data[@task_id].end_filter_configured = true
  end


end