Class: Threatinator::FeedRunner

Inherits:
Object
  • Object
show all
Includes:
Observable
Defined in:
lib/threatinator/feed_runner.rb

Overview

Runs those feeds!

Has the following observations:
  :start                 - start of feed parsing

  :start_fetch           - start of fetching
  :end_fetch             - end of fetching

  :start_decode          - start of decoding
  :end_decode            - end of decoding

  :start_parse_record    - start of record parse
     - record                - The record

  :record_filtered       - Indicates that the record was filtered.
     - record                - The record

  :record_missed         - Indicates that the record was not parsed
     - record                - The record

  :record_parsed         - Indicates that the record WAS parsed
     - record                - The record
     - events                - The events that were parsed out of the 
                               record

  :end_parse_record      - when a record has been parsed
     - record                - The record

  :end                   - completion of feed parsing

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(feed, output_formatter, opts = {}) ⇒ FeedRunner

Returns a new instance of FeedRunner.

Parameters:



40
41
42
43
44
45
46
47
48
 
# File 'lib/threatinator/feed_runner.rb', line 40

def initialize(feed, output_formatter, opts = {})
  @feed = feed
  @output_formatter = output_formatter
  @event_builder = Threatinator::EventBuilder.new(@feed)
  @feed_filters = @feed.filter_builders.map { |x| x.call } 
  @decoders = @feed.decoder_builders.map { |x| x.call } 
  @parser_block = @feed.parser_block
  @create_event_proc = @event_builder.create_event_proc()
end

Class Method Details

.run(feed, output, run_opts = {}) ⇒ Object

Runs a feed

Parameters:



113
114
115
 
# File 'lib/threatinator/feed_runner.rb', line 113

def self.run(feed, output, run_opts = {})
  self.new(feed, output).run(run_opts)
end

Instance Method Details

#parse_record(record) ⇒ Object 



84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
 
# File 'lib/threatinator/feed_runner.rb', line 84

def parse_record(record)
  @event_builder.clear
  events = []
  changed(true); notify_observers(:start_parse_record, record)

  if @feed_filters.any? { |filter| filter.filter?(record) }
    changed(true); notify_observers(:record_filtered, record)
    return
  end
  @parser_block.call(@create_event_proc, record)
  if @event_builder.count == 0
    changed(true); notify_observers(:record_missed, record)
    # Keep track of the fact that this line did not generate any events?
  else 
    @event_builder.each_built_event do |event|
      events << event
      @output_formatter.handle_event(event)
    end
    changed(true); notify_observers(:record_parsed, record, events)
  end
  return
ensure 
  changed(true); notify_observers(:end_parse_record, record)
end

#run(opts = {}) ⇒ Object

Parameters:

  • opts (Hash) (defaults to: {}) —

    The options hash

Options Hash (opts):

  • :io (IO-like) —

    Override the fetcher by providing an IO directly.

  • :skip_decoding (Boolean) — default: false —

    Skip all decoding if set to true. Useful for testing.



55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
 
# File 'lib/threatinator/feed_runner.rb', line 55

def run(opts = {})
  changed(true); notify_observers(:start)
  skip_decoding = !!opts.delete(:skip_decoding)

  unless io = opts.delete(:io)
    fetcher = @feed.fetcher_builder.call()
    changed(true); notify_observers(:start_fetch)
    io = fetcher.fetch()
    changed(true); notify_observers(:end_fetch)
  end

  unless skip_decoding == true
    changed(true); notify_observers(:start_decode)
    @decoders.each do |decoder|
      io = decoder.decode(io)
    end
    changed(true); notify_observers(:end_decode)
  end

  parser = @feed.parser_builder.call()

  parser.run(io) do |record|
    rr = parse_record(record)
  end

  changed(true); notify_observers(:end)
  nil
end