Class: LogStash::Filters::LookUp

Inherits:
Base
  • Object
show all
Defined in:
lib/logstash/filters/lookup.rb

Overview

A general search and replace tool which uses a Web service with a YAML, CSV or JSON response to determine replacement values.

The map entries can be specified with a Web service who your request produces a YML, CSV or JSON response.

Operationally, if the event field specified in the ‘field` configuration matches the EXACT contents of a map entry key, the field’s value will be substituted with the matched key’s value from the map.

The lookup filter will put the contents of the maching event fields into field result.

Instance Method Summary collapse

Instance Method Details

#csv_loader(data) ⇒ Object 



89
90
91
92
93
94
95
 
# File 'lib/logstash/filters/lookup.rb', line 89

def csv_loader(data)
  data = CSV.parse(data).inject(Hash.new) do |acc, v|
    acc[v[0]] = v[1]
    acc
  end
  fill_map(data)
end

#fill_map(data) ⇒ Object 



69
70
71
 
# File 'lib/logstash/filters/lookup.rb', line 69

def fill_map(data)
  get_map.merge!(data)
end

#filter(event) ⇒ Object 



165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
 
# File 'lib/logstash/filters/lookup.rb', line 165

def filter(event)
  if @next_refresh < Time.now
    load()
    @next_refresh = Time.now + @refresh_interval
    @logger.info('downloading and refreshing map file')
  end
  begin
    matched = true
    if @override
      result = event;
    else
      event[@result_key] = {}
      result = event[@result_key]
    end
    @fields.each { |key|
      key_string = key.to_s
      if event.include?(key_string)
        val_string = event[key_string]
        if get_map.include?(val_string)
          result[key_string] = get_map[val_string]
        else
          if @default_values and @default_values.include?(key_string)
            result[key_string] = event.sprintf(@default_values[key_string])
          end
          matched = false
        end
      end
    }
    filter_matched(event) if matched and result.length == @fields.length
  rescue Exception => e
    @logger.error('Something went wrong when attempting to map from my_map', :exception => e, :field => @fields, :event => event)
  end
end

#get_file_content(file) ⇒ Object

def load



138
139
140
141
142
143
144
 
# File 'lib/logstash/filters/lookup.rb', line 138

def get_file_content(file)
  data = ''
  File.open(file, 'r') do |read_file|
    data +=read_file.read
  end
  data
end

#get_map ⇒ Object 



65
66
67
 
# File 'lib/logstash/filters/lookup.rb', line 65

def get_map
  @my_map
end

#get_webservice_content(path) ⇒ Object

def get_file_content



148
149
150
151
152
153
154
155
156
 
# File 'lib/logstash/filters/lookup.rb', line 148

def get_webservice_content(path)
  case @method
    when "get"
      data = RestClient.get sprint(path), sprint(@headers)
    when "post"
      data = RestClient.post sprint(path), sprint(@params), sprint(@headers)
  end
  data
end

#json_loader(data) ⇒ Object

def register



85
86
87
 
# File 'lib/logstash/filters/lookup.rb', line 85

def json_loader(data)
  fill_map(JSON.parse(data))
end

#load(registering = false) ⇒ Object 



118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
 
# File 'lib/logstash/filters/lookup.rb', line 118

def load(registering=false)
  begin
    if @type=='webservice'
      route = @url
      data = get_webservice_content(route)
    elsif @type=='file'
      route = @path
      data = get_file_content(route)
    end
    load_data(registering, data)
  rescue Exception => _
    if registering
      raise "#{self.class.name}: Failed to initialize with type #{type} and route #{route}"
    end
    @logger.warn("#{self.class.name}: Something happened with load. Continuing with old map", :type => @type, :route => route)
  end
end

#load_data(registering, data) ⇒ Object 



101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
 
# File 'lib/logstash/filters/lookup.rb', line 101

def load_data(registering, data)
  begin
    if @format.eql?('json')
      return json_loader(data)
    elsif @format.eql?('csv')
      return csv_loader(data)
    end
    yml_loader(data)
  rescue Exception => _
    if registering
      raise "#{self.class.name}: Bad Syntax in data #{data}"
    else
      @logger.warn("#{self.class.name}: Bad Syntax in map file, continuing with old map", :map_path => data)
    end
  end
end

#register ⇒ Object 



74
75
76
77
78
79
80
81
 
# File 'lib/logstash/filters/lookup.rb', line 74

def register
  @my_map = {}
  @next_refresh = Time.now + @refresh_interval
  load(true)
  @logger.debug? and @logger.debug("#{self.class.name}: map - ", :map => get_map)
  type = 'Exact'
  @logger.debug? and @logger.debug("#{self.class.name}: map mapping method - "+type)
end

#sprint(hash) ⇒ Object

def get_webservice_content



160
161
162
 
# File 'lib/logstash/filters/lookup.rb', line 160

def sprint(hash)
  hash
end

#yml_loader(data) ⇒ Object 



97
98
99
 
# File 'lib/logstash/filters/lookup.rb', line 97

def yml_loader(data)
  fill_map(YAML.load(data))
end