Class: Clockker::Watcher

Inherits:
Object
  • Object
show all
Defined in:
lib/clockker/watcher.rb

Instance Method Summary collapse

Constructor Details

#initialize(config, logger: Logger.new) ⇒ Watcher

Returns a new instance of Watcher.



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/clockker/watcher.rb', line 8

def initialize(config, logger: Logger.new)
  Thread.abort_on_exception = true

  trap "INT" do
    @fsevent.stop
    @timer.kill  # the timer can just go away; it doesn't need a graceful shutdown.
    @safari_watcher_queue << :exit
    @submitter_queue << :exit  # this shuts down the submitter, first letting it submit its changeset
  end

  white_black_list = WhiteBlackList.new(config)

  @submitter_queue = Queue.new
  @safari_watcher_queue = Queue.new
  @timer = Thread.new { create_timer(@safari_watcher_queue) }
  @file_watcher = Thread.new { create_file_watcher('/', white_black_list, logger) }
  @safari_watcher = Thread.new { create_safari_watcher(white_black_list, logger) }
  @submitter = Thread.new { create_submitter(config.submit_frequency, config.region, config.token, config.identifier, logger, config.development) }
  
  @file_watcher.join
  @safari_watcher.join
  @submitter.join
end

Instance Method Details

#create_file_watcher(dir, white_black_list, logger) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
 
# File 'lib/clockker/watcher.rb', line 39

def create_file_watcher(dir, white_black_list, logger)
  @fsevent = FSEvent.new
  @fsevent.watch dir, {file_events: true} do |directories, event_meta|
    t = Time.now.utc
    count = 0
    event_meta['events'].each do |event|
      next if white_black_list.ignore?(event['path'])
      count += 1
      file_path = ::Pathname.new(event['path'])
      dirs, name = file_path.split
      last_dir = dirs.split[-1]
      title = (last_dir + name).to_s
      @submitter_queue << {touched_at: t, contents: file_path.to_s, meta_type: "file", metadata: {title: title, file_path: file_path.to_s}}
    end
    logger.debug "#{Time.now} checked #{event_meta['numEvents']} events and added #{count} to the submitter_queue"
  end
  logger.info "#{Time.now} now watching /"
  logger.info "#{Time.now} whitelist paths: #{white_black_list.whitelist}"
  logger.info "#{Time.now} blacklist paths: #{white_black_list.blacklist}"
  logger.info "#{Time.now} url whitelist:   #{white_black_list.url_whitelist}"
  @fsevent.run
  logger.info "#{Time.now} fswatcher done"
end

#create_safari_watcher(white_black_list, logger) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
 
# File 'lib/clockker/watcher.rb', line 63

def create_safari_watcher(white_black_list, logger)
  safari_db = SQLite3::Database.new(File.expand_path("~/Library/Safari/History.db"))
  logger.info "#{Time.now} now watching Safari"
  last_time = Time.now.to_i - 978307200   # convert to macOS Core Data time
  loop do
    submit_now = false
    exit_now = false

    value = @safari_watcher_queue.pop
    case value
    when :exit
      submit_now = true
      exit_now = true
    when :run
      # we just move forward
    else
      # On the off chance the message isn't :run
    end

    safari_db.execute("
      SELECT hv.id, datetime(hv.visit_time+978307200, \"unixepoch\") as visited_at, hv.visit_time, hv.title, hi.url,
            hv.load_successful, hv.http_non_get, hv.redirect_source, hv.redirect_destination, hv.origin, hv.generation, hv.attributes, hv.score
      FROM history_visits hv, history_items hi
      WHERE hv.history_item=hi.id
      AND hv.visit_time > #{last_time}
      ORDER BY hv.visit_time ASC
    ") do |row|
      id, visited_at, visit_time, title, url, load_successful, http_non_get, redirect_source, redirect_destination, origin, generation, attributes, score = row

      next if load_successful == 0
      next if redirect_destination
      next if white_black_list.ignore_url?(url)

      visited_at = Time.parse(visited_at+"Z") # the time is already UTC; let's make doubly sure!
      logger.debug "- #{url} http_non_get: #{http_non_get} origin: #{origin} generation: #{generation} attributes: #{attributes} score: #{score}"
      @submitter_queue << {touched_at: visited_at, contents: url, meta_type: "url", metadata: {url: url, title: title}}
      last_time = visit_time
    end

    if exit_now
      logger.info "#{Time.now} Safari watcher preparing to exit"
      break
    end
  end
  logger.info "#{Time.now} Safari watcher done"
end

#create_submitter(submit_frequency, region, token, identifier, logger, dev_mode) ⇒ Object 



110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
 
# File 'lib/clockker/watcher.rb', line 110

def create_submitter(submit_frequency, region, token, identifier, logger, dev_mode)
  logger.info "#{Time.now} submitting to clockk.com #{"and localhost:4000 " if dev_mode}every #{submit_frequency} seconds with token #{token} from identifier #{identifier} with version #{Clockker.version}"
  t_start = Time.now
  clockk_agent_uri = URI("https://#{region}.clockk.io/api/v1/agent/artifacts")
  clockk_local_uri = URI("http://localhost:4000/api/v1/agent/artifacts")
  @changeset = []
  loop do
    submit_now = false
    exit_now = false

    value = @submitter_queue.pop
    case value
    when :submit
      submit_now = true
    when :exit
      submit_now = true
      exit_now = true
    else
      @changeset << value
    end

    if Time.now - t_start > submit_frequency
      submit_now = true
    end

    if exit_now
      logger.info "#{Time.now} Submitter preparing to exit"
    end

    if submit_now && @changeset.length > 0
      logger.info "#{Time.now} submitting #{@changeset.length} items"
      @changeset.each_with_index do |cs, idx|
        logger.info " - #{idx} - #{cs[:touched_at].strftime("%H:%M:%S")} #{cs[:contents]}"
      end

      begin
        dev_data = {artifacts: @changeset.map{|c| {contents: c[:contents], touched_at: c[:touched_at].strftime("%Y-%m-%d %H:%M:%S"), meta_type: c[:meta_type], metadata: c[:metadata], identifier: identifier, agent: Clockker.version}}}.to_json
        prod_data = dev_data   # at this point the same data goes to prod as to dev
        
        header = {"Authorization" => "Bearer #{token}", "Content-Type" => "application/json"}

        logger.debug "PROD submission: #{prod_data.inspect}"
        Net::HTTP.start(clockk_agent_uri.hostname, clockk_agent_uri.port, use_ssl: clockk_agent_uri.scheme == 'https') do |http|
          http.post(clockk_agent_uri.path, prod_data, header)
        end

        if dev_mode
          begin
            logger.debug "DEV submission: #{dev_data.inspect}"
            Net::HTTP.start(clockk_local_uri.hostname, clockk_local_uri.port, use_ssl: clockk_local_uri.scheme == 'https') do |http|
              http.post(clockk_local_uri.path, dev_data, header)
            end
          rescue Exception => q
            logger.info "#{Time.now} Failed to reach localhost:4000; ignoring"
          end
        end

        @changeset = []
      rescue Exception => e
        logger.info "#{Time.now} Reached exception #{e}"
      end

      t_start = Time.now
    end

    if exit_now
      logger.info "#{Time.now} Submitter done"
      break
    end
  end
end

#create_timer(*queues) ⇒ Object 



32
33
34
35
36
37
 
# File 'lib/clockker/watcher.rb', line 32

def create_timer(*queues)
  loop do
    sleep 5
    queues.each {|q| q << :run}
  end
end