Class: HQ::LogMonitorServer::Script

Inherits:
Object
  • Object
show all
Includes:
Tools::Escape
Defined in:
lib/hq/log-monitor-server/misc.rb,
lib/hq/log-monitor-server/logic.rb,
lib/hq/log-monitor-server/script.rb,
lib/hq/log-monitor-server/do-checks.rb,
lib/hq/log-monitor-server/event-page.rb,
lib/hq/log-monitor-server/service-page.rb,
lib/hq/log-monitor-server/overview-page.rb,
lib/hq/log-monitor-server/submit-log-event.rb,
lib/hq/log-monitor-server/service-host-page.rb

Constant Summary collapse

CHECK_FREQUENCY = 
60

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize ⇒ Script

Returns a new instance of Script.



20
21
22
 
# File 'lib/hq/log-monitor-server/script.rb', line 20

def initialize
	@status = 0
end

Instance Attribute Details

#args ⇒ Object

Returns the value of attribute args.



17
18
19
 
# File 'lib/hq/log-monitor-server/script.rb', line 17

def args
  @args
end

#status ⇒ Object

Returns the value of attribute status.



18
19
20
 
# File 'lib/hq/log-monitor-server/script.rb', line 18

def status
  @status
end

Instance Method Details

#background ⇒ Object 



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/hq/log-monitor-server/do-checks.rb', line 8

def background

	@mutex = Mutex.new
	@run_now_signal = ConditionVariable.new
	@next_check_done_signal = nil
	@current_check_done_signal = nil
	@stop_checks_signal = nil

	@next_check = Time.now

	loop do

		# wait till next invocation

		@mutex.synchronize do

			until @next_check_done_signal

				if @next_check < Time.now
					@next_check_done_signal = ConditionVariable.new
					break
				end

				@run_now_signal.wait @mutex, 1

				if @stop_checks_signal
					@stop_checks_signal.broadcast
					return
				end

			end

			@current_check_done_signal = @next_check_done_signal

		end

		# perform the checks

		begin
			do_checks_real
		rescue => e
			$stderr.puts e, *e.backtrace
			sleep 10
		ensure

			# notify waiting threads

			@mutex.synchronize do

				if @next_check_done_signal == @current_check_done_signal
					@next_check_done_signal = nil
				end

				@current_check_done_signal.broadcast

			end

		end

		@next_check = Time.now + CHECK_FREQUENCY

	end

end

#call(env) ⇒ Object 



135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
 
# File 'lib/hq/log-monitor-server/script.rb', line 135

def call env

	case env["PATH_INFO"]

	when "/submit-log-event"
		submit_log_event env

	when "/"
		overview_page env

	when /^\/service\/([^\/]+)$/
		service_page env, :service => $1

	when /^\/service-host\/([^\/]+)\/([^\/]+)\/([^\/]+)$/
		service_host_page env, :service => $1, :class => $2, :host => $3

	when /^\/event\/([^\/]+)$/
		event_page env, :event_id => $1

	when "/favicon.ico"
		[ 404, {}, [] ]

	else
		raise "Not found: #{env["PATH_INFO"]}"

	end

end

#class_for_event(event) ⇒ Object 



84
85
86
87
88
89
90
91
92
93
 
# File 'lib/hq/log-monitor-server/misc.rb', line 84

def class_for_event event

	return nil \
		unless event["status"] == "unseen"

	return class_for_type \
		event["source"]["service"],
		event["type"]

end

#class_for_level(level) ⇒ Object 



62
63
64
65
66
67
68
69
70
71
72
73
74
 
# File 'lib/hq/log-monitor-server/misc.rb', line 62

def class_for_level level

	case level

		when "critical"
			return "error"

		when "warning"
			return "warning"

	end

end

#class_for_summary(summary) ⇒ Object 



124
125
126
127
128
129
130
 
# File 'lib/hq/log-monitor-server/misc.rb', line 124

def class_for_summary summary

	level = level_for_summary summary

	return class_for_level level

end

#class_for_type(service, type) ⇒ Object 



76
77
78
79
80
81
82
 
# File 'lib/hq/log-monitor-server/misc.rb', line 76

def class_for_type service, type

	level = level_for_type service, type

	return class_for_level level

end

#connect_db ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/hq/log-monitor-server/script.rb', line 95

def connect_db

	@mongo =
		Mongo::MongoClient.new \
			@db_elem["host"],
			@db_elem["port"].to_i

	@db =
		@mongo[@db_elem["name"]]

	@db["events"].ensure_index \
		"source.service" => Mongo::ASCENDING,
		"source.class" => Mongo::ASCENDING,
		"source.host" => Mongo::ASCENDING,
		"timestamp" => Mongo::ASCENDING

end

#delete_all(source) ⇒ Object 



188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
 
# File 'lib/hq/log-monitor-server/logic.rb', line 188

def delete_all source

	# get types

	query = {
		"source.service" => source["service"],
		"source.class" => source["class"],
		"source.host" => source["host"],
	}

	types =
		@db["events"].distinct "type", query

	types.each do
		|type|

		# delete events

		query = {
			"source.service" => source["service"],
			"source.class" => source["class"],
			"source.host" => source["host"],
			"status" => "unseen",
			"type" => type,
		}

		@db["events"].remove query

		unseen_count =
			@db.get_last_error["n"]

		query = {
			"source.service" => source["service"],
			"source.class" => source["class"],
			"source.host" => source["host"],
			"status" => "seen",
			"type" => type,
		}

		@db["events"].remove query

		seen_count =
			@db.get_last_error["n"]

		# update summaries

		@db["summaries"].update(
			{
				"_id.service" => source["service"],
				"_id.class" => source["class"],
				"_id.host" => source["host"],
			}, {
				"$inc" => {
					"combined.new" => - unseen_count,
					"combined.total" => - unseen_count - seen_count,
					"types.#{type}.new" => - unseen_count,
					"types.#{type}.total" => - unseen_count - seen_count,
				}
			}
		)

	end

	# notify icinga checks

	do_checks

end

#delete_event(event_id) ⇒ Object 



127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
# File 'lib/hq/log-monitor-server/logic.rb', line 127

def delete_event event_id

	# fetch it

	event = get_event event_id

	return nil unless event

	# delete it

	@db["events"].remove({
		"_id" => event["_id"],
	})

	event_count =
		@db.get_last_error["n"]

	return event unless event_count > 0

	# update summary

	case event["status"]

	when "unseen"

		@db["summaries"].update(
			{ "_id" => event["source"] },
			{ "$inc" => {
				"combined.new" => -1,
				"combined.total" => -1,
				"types.#{event["type"]}.new" => -1,
				"types.#{event["type"]}.total" => -1,
			} }
		)

	when "seen"

		@db["summaries"].update(
			{ "_id" => event["source"] },
			{ "$inc" => {
				"combined.total" => -1,
				"types.#{event["type"]}.total" => -1,
			} }
		)

	else

		raise "Error 3084789190"

	end

	# notify icinga checks

	do_checks

	# and return

	return event

end

#do_checks ⇒ Object 



89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
 
# File 'lib/hq/log-monitor-server/do-checks.rb', line 89

def do_checks

	@mutex.synchronize do

		if @next_check_done_signal

			# next run already scheduled, just wait for it

			@next_check_done_signal.wait @mutex

		else

			# create new next run and wait for it

			@next_check_done_signal = ConditionVariable.new

			@run_now_signal.signal

			@next_check_done_signal.wait @mutex

		end

	end

end

#do_checks_real ⇒ Object 



115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
 
# File 'lib/hq/log-monitor-server/do-checks.rb', line 115

def do_checks_real

	File.open @icinga_elem["command-file"], "a" do
		|command_io|

		summaries_by_service =
			get_summaries_by_service

		@icinga_elem.find("service").each do
			|service_elem|

			service_name = service_elem["name"]

			critical_count = 0
			warning_count = 0
			other_count = 0
			unknown_count = 0

			summaries =
				summaries_by_service[service_name]

			if summaries
				summaries["types"].each do
					|type_name, type_info|

					case level_for_type service_name, type_name
						when "critical"
							critical_count += type_info["new"]
						when "warning"
							warning_count += type_info["new"]
						when "none"
							other_count += type_info["new"]
						else
							unknown_count += type_info["new"]
					end

				end
			end

			status_int =
				if critical_count > 0
					2
				elsif warning_count > 0
					1
				elsif unknown_count > 0
					3
				else
					0
				end

			status_str =
				if critical_count > 0
					"CRITICAL"
				elsif warning_count > 0
					"WARNING"
				elsif unknown_count > 0
					"UNKNOWN"
				else
					"OK"
				end

			parts = []

			if critical_count > 0
				parts << "%d critical" % critical_count
			end

			if warning_count > 0
				parts << "%d warning" % warning_count
			end

			if other_count > 0
				parts << "%d other" % other_count
			end

			if unknown_count > 0
				parts << "%d unknown" % unknown_count
			end

			if parts.empty?
				parts << "no new events"
			end


			command_io.print "[%s] %s\n" % [
				Time.now.to_i,
				[
					"PROCESS_SERVICE_CHECK_RESULT",
					service_elem["icinga-host"],
					service_elem["icinga-service"],
					status_int,
					"%s %s" % [
						status_str,
						parts.join(", "),
					]
				].join(";"),
			]

		end

	end

end

#event_page(env, context) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
 
# File 'lib/hq/log-monitor-server/event-page.rb', line 6

def event_page env, context

	req = Rack::Request.new env

	# process form stuff

	if req.request_method == "POST" \
		&& req.params["mark-as-seen"]

		mark_event_as_seen context[:event_id]

	end

	if req.request_method == "POST" \
		&& req.params["mark-as-unseen"]

		mark_event_as_unseen context[:event_id]

	end

	if req.request_method == "POST" \
		&& req.params["delete"]

		event =
			delete_event context[:event_id]

		resp =
			Rack::Response.new

		resp.redirect "/service-host/%s/%s/%s" % [
			event["source"]["service"],
			event["source"]["class"],
			event["source"]["host"],
		]

		return resp

	end

	# read from database

	event =
		@db["events"]
			.find_one({
				"_id" => BSON::ObjectId.from_string(context[:event_id]),
			})

	# set headers

	headers = {}

	headers["content-type"] = "text/html; charset=utf-8"

	# create page

	html = []

	title =
		"Event %s \u2014 Log monitor" % [
			context[:event_id],
		]

	html << "<!DOCTYPE html>\n"
	html << "<html>\n"
	html << "<head>\n"

	html << "<title>%s</title>\n" % [
		esc_ht(title),
	]

	html << "<link href=\"%s\" rel=\"stylesheet\">" % [
		"%s/css/bootstrap-combined.min.css" % [
			@assets_elem["bootstrap"],
		],
	]

	html << "<script src=\"%s\"></script>" % [
		"%s/js/bootstrap.min.js" % [
			@assets_elem["bootstrap"],
		],
	]

	html << "</head>\n"
	html << "<body>\n"

	html << "<div class=\"navbar navbar-static-top\">\n"
	html << "<div class=\"navbar-inner\">\n"
	html << "<div class=\"container\">\n"
	html << "<ul class=\"nav\">\n"
	html << "<li><a href=\"/\">Overview</a></li>\n"
	html << "<li><a href=\"%s\">Service</a></li>\n" % [
		esc_ht("/service/%s" % [
			event["source"]["service"],
		]),
	]
	html << "<li><a href=\"%s\">Host</a></li>\n" % [
		esc_ht("/service-host/%s/%s/%s" % [
			event["source"]["service"],
			event["source"]["class"],
			event["source"]["host"],
		])
	]
	html << "<li class=\"active\"><a href=\"%s\">Event</a></li>\n" % [
		esc_ht("/event/%s" % [
			context[:event_id],
		])
	]
	html << "</ul>\n"
	html << "</div>\n"
	html << "</div>\n"
	html << "</div>\n"

	html << "<div class=\"container\">\n"
	html << "<div class=\"row\">\n"
	html << "<div class=\"span12\">\n"

	html << "<h1>%s</h1>\n" % [
		esc_ht(title),
	]

	unless event

		html << "<p>Event id not recognised</p>\n"

	else

		html << "<table id=\"event\" class=\"table table-striped\">\n"
		html << "<tbody>\n"

		html << "<tr id=\"id\">\n"
		html << "<th>ID</th>\n"
		html << "<td>%s</td>\n" % [
			esc_ht(event["_id"].to_s),
		]
		html << "</tr>\n"

		html << "<tr>\n"
		html << "<th>Timestamp</th>\n"
		html << "<td>%s</td>\n" % [
			esc_ht(event["timestamp"].to_s),
		]
		html << "</tr>\n"

		html << "<tr>\n"
		html << "<th>Status</th>\n"
		html << "<td>%s</td>\n" % [
			esc_ht(event["status"].to_s),
		]
		html << "</tr>\n"

		html << "<tr>\n"
		html << "<th>Service</th>\n"
		html << "<td>%s</td>\n" % [
			esc_ht(event["source"]["service"]),
		]
		html << "</tr>\n"

		html << "<tr>\n"
		html << "<th>Host</th>\n"
		html << "<td>%s</td>\n" % [
			esc_ht(event["source"]["host"]),
		]
		html << "</tr>\n"

		html << "<tr>\n"
		html << "<th>Class</th>\n"
		html << "<td>%s</td>\n" % [
			esc_ht(event["source"]["class"]),
		]
		html << "</tr>\n"

		html << "<tr>\n"
		html << "<th>Type</th>\n"
		html << "<td>%s</td>\n" % [
			esc_ht(event["type"]),
		]
		html << "</tr>\n"

		html << "<tr>\n"
		html << "<th>Filename</th>\n"
		html << "<td>%s</td>\n" % [
			esc_ht(event["location"]["file"]),
		]
		html << "</tr>\n"

		html << "<tr>\n"
		html << "<th>Line number</th>\n"
		html << "<td>%s</td>\n" % [
			esc_ht((event["location"]["line"].to_i + 1).to_s),
		]
		html << "</tr>\n"

		unless event["lines"]["before"].empty?

			html << "<tr>\n"
			html << "<th>Before</th>\n"
			html << "<td>%s</td>\n" % [
				event["lines"]["before"]
					.map { |line| esc_ht(line) }
					.join("<br>")
			]

		end

		html << "<tr>\n"
		html << "<th>Matching</th>\n"
		html << "<td>%s</td>\n" % [
			esc_ht(event["lines"]["matching"]),
		]

		unless event["lines"]["after"].empty?

			html << "<tr>\n"
			html << "<th>Before</th>\n"
			html << "<td>%s</td>\n" % [
				event["lines"]["after"]
					.map { |line| esc_ht(line) }
					.join("<br>")
			]

		end

		html << "</tbody>\n"
		html << "</table>\n"

	end

	html << "<form method=\"post\">\n"

	html << "<p>\n"

	if event["status"] != "seen"

		html <<
			"<input " +
				"type=\"submit\" " +
				"name=\"mark-as-seen\" " +
				"value=\"mark as seen\">\n"

	end

	if event["status"] != "unseen"

		html <<
			"<input " +
				"type=\"submit\" " +
				"name=\"mark-as-unseen\" " +
				"value=\"mark as unseen\">\n"

	end

	html <<
		"<input " +
			"type=\"submit\" " +
			"name=\"delete\" " +
			"value=\"delete\">\n"

	html << "</p>\n"

	html << "</form>\n"

	html << "</div>\n"
	html << "</div>\n"
	html << "</div>\n"

	html << "</body>\n"
	html << "</html>\n"

	# return

	return 200, headers, html

end

#get_event(event_id) ⇒ Object 



6
7
8
9
10
11
12
 
# File 'lib/hq/log-monitor-server/logic.rb', line 6

def get_event event_id

	return @db["events"].find({
		"_id" => BSON::ObjectId.from_string(event_id),
	}).first

end

#get_summaries_by_service ⇒ Object 



257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
 
# File 'lib/hq/log-monitor-server/logic.rb', line 257

def get_summaries_by_service

	summaries_by_service = {}

	@db["summaries"].find.each do
		|summary|

		service =
			summary["_id"]["service"]

		summary_by_service =
			summaries_by_service[service] ||= {
				"_id" => {
					"service" => service,
				},
				"combined" => { "new" => 0, "total" => 0 },
				"types" => {},
			}

		summary_by_service["combined"]["new"] +=
			summary["combined"]["new"]

		summary_by_service["combined"]["total"] +=
			summary["combined"]["total"]

		summary["types"].each do
			|type, type_summary|

			type_summary_by_service =
				summary_by_service["types"][type] ||= {
					"new" => 0,
					"total" => 0,
				}

			type_summary_by_service["new"] +=
				type_summary["new"]

			type_summary_by_service["total"] +=
				type_summary["total"]

		end

	end

	return summaries_by_service

end

#init_server ⇒ Object 



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/hq/log-monitor-server/script.rb', line 113

def init_server

	@web_config = {
		:Port => @server_elem["port"].to_i,
		:AccessLog => [],
	}

	if @opts[:quiet]
		@web_config.merge!({
			:Logger => WEBrick::Log::new("/dev/null", 7),
			:DoNotReverseLookup => true,
		})
	end

	@web_server =
		WEBrick::HTTPServer.new \
			@web_config

	@web_server.mount "/", Rack::Handler::WEBrick, self

end

#level_for_summary(summary) ⇒ Object 



95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/hq/log-monitor-server/misc.rb', line 95

def level_for_summary summary

	critical = false
	warning = false

	summary["types"].each do
		|type_name, type_info|

		next unless type_info["new"] > 0

		case level_for_type summary["_id"]["service"], type_name

		when "critical"
			critical = true

		when "warning"
			warning = true

		end

	end

	return "critical" if critical
	return "warning" if warning

	return nil

end

#level_for_type(service_name, type_name) ⇒ Object 



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/hq/log-monitor-server/misc.rb', line 39

def level_for_type service_name, type_name

	service_elem =
		@icinga_elem.find_first("
			service [@name = #{esc_xp service_name}]
		")

	return nil unless service_elem

	type_elem =
		service_elem.find_first("
			type [@name = #{esc_xp type_name}]
		")

	return nil unless type_elem

	level = type_elem["level"]

	return level == "" ? nil : level


end

#main ⇒ Object 



24
25
26
27
28
29
30
31
 
# File 'lib/hq/log-monitor-server/script.rb', line 24

def main
	setup
	trap "INT" do
		@web_server.shutdown
	end
	Thread.new { background }
	run
end

#mark_all_as_seen(source) ⇒ Object 



40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
 
# File 'lib/hq/log-monitor-server/logic.rb', line 40

def mark_all_as_seen source

	# get types

	query = {
		"source.service" => source["service"],
		"source.class" => source["class"],
		"source.host" => source["host"],
		"status" => "unseen",
	}

	types =
		@db["events"].distinct "type", query

	types.each do
		|type|

		# update events

		query = {
			"source.service" => source["service"],
			"source.class" => source["class"],
			"source.host" => source["host"],
			"status" => "unseen",
			"type" => type,
		}

		update = {
			"$set" => {
				"status" => "seen",
			},
		}

		@db["events"].update query, update, :multi => true

		event_count =
			@db.get_last_error["n"]

		# update summaries

		@db["summaries"].update(
			{
				"_id.service" => source["service"],
				"_id.class" => source["class"],
				"_id.host" => source["host"],
			}, {
				"$inc" => {
					"combined.new" => -event_count,
					"types.#{type}.new" => -event_count,
				}
			}
		)

	end

	# notify icinga checks

	do_checks

end

#mark_event_as_seen(event_id) ⇒ Object 



14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'lib/hq/log-monitor-server/logic.rb', line 14

def mark_event_as_seen event_id

	event = get_event event_id

	return if event["status"] == "seen"

	event["status"] = "seen"

	@db["events"].save event

	# update summary

	@db["summaries"].update(
		{ "_id" => event["source"] },
		{ "$inc" => {
			"combined.new" => -1,
			"types.#{event["type"]}.new" => -1,
		} }
	)

	# notify icinga checks

	do_checks

end

#mark_event_as_unseen(event_id) ⇒ Object 



101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
 
# File 'lib/hq/log-monitor-server/logic.rb', line 101

def mark_event_as_unseen event_id

	event = get_event event_id

	return if event["status"] == "unseen"

	event["status"] = "unseen"

	@db["events"].save event

	# update summary

	@db["summaries"].update(
		{ "_id" => event["source"] },
		{ "$inc" => {
			"combined.new" => 1,
			"types.#{event["type"]}.new" => 1,
		} }
	)

	# notify icinga checks

	do_checks

end

#overview_page(env) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
 
# File 'lib/hq/log-monitor-server/overview-page.rb', line 6

def overview_page env

	summaries_by_service =
		get_summaries_by_service

	summaries =
		summaries_by_service
			.values
			.sort_by { |summary| summary["_id"]["service"] }

	headers = {}
	html = []

	headers["content-type"] = "text/html; charset=utf-8"

	html << "<!DOCTYPE html>\n"
	html << "<html>\n"
	html << "<head>\n"

	title =
		"Overview \u2014 Log monitor"

	html << "<title>%s</title>\n" % [
		esc_ht(title),
	]

	html << "<link href=\"%s\" rel=\"stylesheet\">" % [
		"%s/css/bootstrap-combined.min.css" % [
			@assets_elem["bootstrap"],
		],
	]

	html << "<script src=\"%s\"></script>" % [
		"%s/js/bootstrap.min.js" % [
			@assets_elem["bootstrap"],
		],
	]

	html << "</head>\n"
	html << "<body>\n"

	html << "<div class=\"navbar navbar-static-top\">\n"
	html << "<div class=\"navbar-inner\">\n"
	html << "<div class=\"container\">\n"
	html << "<ul class=\"nav\">\n"
	html << "<li class=\"active\"><a href=\"/\">Overview</a></li>\n"
	html << "</ul>\n"
	html << "</div>\n"
	html << "</div>\n"
	html << "</div>\n"

	html << "<div class=\"container\">\n"
	html << "<div class=\"row\">\n"
	html << "<div class=\"span12\">\n"

	html << "<h1>%s</h1>\n" % [
		esc_ht(title),
	]

	if summaries.empty?
		html << "<p>No events have been logged</p>\n"
	else

		html << "<table class=\"table table-striped\" id=\"summaries\">\n"
		html << "<thead>\n"

		html << "<tr>\n"
		html << "<th>Service</th>\n"
		html << "<th>New</th>\n"
		html << "<th>Total</th>\n"
		html << "<th>View</th>\n"
		html << "</tr>\n"

		html << "</thead>\n"
		html << "<tbody>\n"

		summaries.each do
			|summary|

			html << "<tr class=\"%s\">\n" % [
				esc_ht([
					"summary",
					class_for_summary(summary),
				].compact.join(" "))
			]

			html << "<td class=\"service\">%s</td>\n" % [
				esc_ht(summary["_id"]["service"]),
			]

			html << "<td class=\"new\">%s</td>\n" % [
				esc_ht(status_breakdown(summary, "new")),
			]

			html << "<td class=\"total\">%s</td>\n" % [
				esc_ht(status_breakdown(summary, "total")),
			]

			html << "<td class=\"view\">%s</td>\n" % [
				"<a href=\"%s\">view</a>" % [
					"/service/%s" % [
						esc_ue(summary["_id"]["service"]),
					],
				],
			]

			html << "</tr>\n"

		end

		html << "</tbody>\n"
		html << "</table>\n"

	end

	html << "</div>\n"
	html << "</div>\n"
	html << "</div>\n"

	html << "</body>\n"
	html << "</html>\n"

	return 200, headers, html

end

#pagination(html, page_count, page_num) ⇒ Object 



249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
 
# File 'lib/hq/log-monitor-server/service-host-page.rb', line 249

def pagination html, page_count, page_num

	html << "<div class=\"pagination\">\n"
	html << "<ul>\n"

	page_count.times do
		|num|

		if num == page_num
			html << "<li class=\"active\">"
		else
			html << "<li>"
		end

		html << "<a href=\"?page=#{num}\">#{num+1}</a>\n"

	end

	html << "</ul>\n"
	html << "</div>\n"

end

#process_args ⇒ Object 



55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/hq/log-monitor-server/script.rb', line 55

def process_args

	@opts, @args =
		Tools::Getopt.process @args, [

		{ :name => :config,
			:required => true },

		{ :name => :quiet,
			:boolean => true },

	]

	@args.empty? \
		or raise "Extra args on command line"

end

#read_config ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/hq/log-monitor-server/script.rb', line 73

def read_config

	config_doc =
		XML::Document.file @opts[:config]

	@config_elem =
		config_doc.root

	@server_elem =
		@config_elem.find_first("server")

	@db_elem =
		@config_elem.find_first("db")

	@icinga_elem =
		@config_elem.find_first("icinga")

	@assets_elem =
		@config_elem.find_first("assets")

end

#run ⇒ Object 



51
52
53
 
# File 'lib/hq/log-monitor-server/script.rb', line 51

def run
	@web_server.start
end

#service_host_page(env, context) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
 
# File 'lib/hq/log-monitor-server/service-host-page.rb', line 6

def service_host_page env, context

	req = Rack::Request.new env

	source = {
		"class" => context[:class],
		"host" => context[:host],
		"service" => context[:service],
	}

	# process form stuff

	if req.request_method == "POST" \
		&& req.params["mark-all-as-seen"]

		mark_all_as_seen source

	end

	if req.request_method == "POST" \
		&& req.params["delete-all"]

		delete_all source

	end

	# read from database

	all_events_count =
		@db["events"]
			.find({
				"source.class" => source["class"],
				"source.host" => source["host"],
				"source.service" => source["service"],
			})
			.count

	page_size = 100
	page_num = req.GET["page"].to_i

	page_start = page_num * page_size
	page_end = page_start + page_size
	page_count = (all_events_count + page_size - 1) / page_size

	page_events =
		@db["events"]
			.find({
				"source.class" => source["class"],
				"source.host" => source["host"],
				"source.service" => source["service"],
			})
			.sort({
				"timestamp" => -1,
			})
			.skip(page_start)
			.limit(page_size)
			.to_a

	title =
		"%s %s \u2014 Log monitor" % [
			context[:host],
			context[:service],
		]

	headers = {}
	html = []

	headers["content-type"] = "text/html; charset=utf-8"

	html << "<!DOCTYPE html>\n"
	html << "<html>\n"
	html << "<head>\n"

	html << "<title>%s</title>\n" % [
		esc_ht(title),
	]

	html << "<link href=\"%s\" rel=\"stylesheet\">" % [
		"%s/css/bootstrap-combined.min.css" % [
			@assets_elem["bootstrap"],
		],
	]

	html << "<script src=\"%s\"></script>" % [
		"%s/js/bootstrap.min.js" % [
			@assets_elem["bootstrap"],
		],
	]

	html << "</head>\n"
	html << "<body>\n"

	html << "<div class=\"navbar navbar-static-top\">\n"
	html << "<div class=\"navbar-inner\">\n"
	html << "<div class=\"container\">\n"
	html << "<ul class=\"nav\">\n"
	html << "<li><a href=\"/\">Overview</a></li>\n"
	html << "<li><a href=\"%s\">Service</a></li>\n" % [
		esc_ht("/service/%s" % [
			context[:service],
		]),
	]
	html << "<li class=\"active\"><a href=\"%s\">Host</a></li>\n" % [
		esc_ht("/service/%s/host/%s" % [
			context[:service],
			context[:host],
		])
	]
	html << "</ul>\n"
	html << "</div>\n"
	html << "</div>\n"
	html << "</div>\n"

	html << "<div class=\"container\">\n"
	html << "<div class=\"row\">\n"
	html << "<div class=\"span12\">\n"

	html << "<h1>%s</h1>\n" % [
		esc_ht(title),
	]

	if all_events_count > page_size || page_num > 0
		pagination html, page_count, page_num
	end

	if all_events_count == 0

		html << "<p>No events have been logged for this service on this " +
			"host</p>\n"

	elsif page_events.empty?

		html << "<p>No events on this page</p>\n"

	else


		html << "<table id=\"events\" class=\"table table-striped\">\n"
		html << "<thead>\n"

		html << "<tr>\n"
		html << "<th>Timestamp</th>\n"
		html << "<th>File</th>\n"
		html << "<th>Line</th>\n"
		html << "<th>Type</th>\n"
		html << "<th>Status</th>\n"
		html << "<th>View</th>\n"
		html << "</tr>\n"

		html << "</thead>\n"
		html << "<tbody>\n"

		unseen_count = 0

		page_events.each do
			|event|

			html << "<tr class=\"%s\">\n" % [
				[
					"event",
					class_for_event(event),
				].compact.join(" "),
			]

			html << "<td class=\"timestamp\">%s</td>\n" % [
				esc_ht(event["timestamp"].to_s),
			]

			html << "<td class=\"file\">%s</td>\n" % [
				esc_ht(event["location"]["file"]),
			]

			html << "<td class=\"line\">%s</td>\n" % [
				esc_ht((event["location"]["line"] + 1).to_s),
			]

			html << "<td class=\"type\">%s</td>\n" % [
				esc_ht(event["type"]),
			]

			html << "<td class=\"status\">%s</td>\n" % [
				esc_ht(event["status"]),
			]

			html << "<td class=\"view\">%s</td>\n" % [
				"<a href=\"%s\">view</a>" % [
					"/event/%s" % [
						esc_ue(event["_id"].to_s),
					],
				],
			]

			html << "</tr>\n"

			unseen_count += 1 \
				if event["status"] == "unseen"

		end

		html << "</tbody>\n"
		html << "</table>\n"

		if all_events_count > page_size || page_num > 0
			pagination html, page_count, page_num
		end

		html << "<form method=\"post\">\n"

		html << "<p>\n"

		if unseen_count > 0

			html <<
				"<input " +
					"type=\"submit\" " +
					"name=\"mark-all-as-seen\" " +
					"value=\"mark all as seen\">\n"

		end

		html <<
			"<input " +
				"type=\"submit\" " +
				"name=\"delete-all\" " +
				"value=\"delete all\">\n"

		html << "</p>\n"

		html << "</form>\n"

	end

	html << "</div>\n"
	html << "</div>\n"
	html << "</div>\n"

	html << "</body>\n"
	html << "</html>\n"

	return 200, headers, html

end

#service_page(env, context) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/hq/log-monitor-server/service-page.rb', line 6

def service_page env, context

	summaries =
		@db["summaries"]
			.find({
				"_id.service" => context[:service]
			})
			.to_a

	summaries.sort_by! {
		|summary|
		[
			summary["_id"]["host"],
			summary["_id"]["class"],
		]
	}

	title =
		"%s - Log monitor" % [
			context[:service],
		]

	headers = {}
	html = []

	headers["content-type"] = "text/html; charset=utf-8"

	html << "<!DOCTYPE html>\n"
	html << "<html>\n"
	html << "<head>\n"

	html << "<title>%s</title>\n" % [
		esc_ht(title),
	]

	html << "<link href=\"%s\" rel=\"stylesheet\">" % [
		"%s/css/bootstrap-combined.min.css" % [
			@assets_elem["bootstrap"],
		],
	]

	html << "<script src=\"%s\"></script>" % [
		"%s/js/bootstrap.min.js" % [
			@assets_elem["bootstrap"],
		],
	]

	html << "</head>\n"
	html << "<body>\n"

	html << "<div class=\"navbar navbar-static-top\">\n"
	html << "<div class=\"navbar-inner\">\n"
	html << "<div class=\"container\">\n"
	html << "<ul class=\"nav\">\n"
	html << "<li><a href=\"/\">Overview</a></li>\n"
	html << "<li class=\"active\"><a href=\"%s\">Service</a></li>\n" % [
		"/service/#{context[:service]}",
	]
	html << "</ul>\n"
	html << "</div>\n"
	html << "</div>\n"
	html << "</div>\n"

	html << "<div class=\"container\">\n"
	html << "<div class=\"row\">\n"
	html << "<div class=\"span12\">\n"

	html << "<h1>%s</h1>\n" % [
		esc_ht(title),
	]

	if summaries.empty?
		html << "<p>No events have been logged for this service</p>\n"
	else

		html << "<table id=\"summaries\" class=\"table table-striped\">\n"
		html << "<thead>\n"

		html << "<tr>\n"
		html << "<th>Host</th>\n"
		html << "<th>Class</th>\n"
		html << "<th>New</th>\n"
		html << "<th>Total</th>\n"
		html << "<th>View</th>\n"
		html << "</tr>\n"

		html << "</thead>\n"
		html << "<tbody>\n"

		summaries.each do
			|summary|

			html << "<tr class=\"%s\">\n" % [
				esc_ht([
					"summary",
					class_for_summary(summary),
				].compact.join(" "))
			]

			html << "<td class=\"host\">%s</td>\n" % [
				esc_ht(summary["_id"]["host"]),
			]

			html << "<td class=\"service\">%s</td>\n" % [
				esc_ht(summary["_id"]["class"]),
			]

			html << "<td class=\"new\">%s</td>\n" % [
				esc_ht(status_breakdown(summary, "new")),
			]

			html << "<td class=\"total\">%s</td>\n" % [
				esc_ht(status_breakdown(summary, "total")),
			]

			html << "<td class=\"view\">%s</td>\n" % [
				"<a href=\"%s\">view</a>" % [
					"/service-host/%s/%s/%s" % [
						esc_ue(summary["_id"]["service"]),
						esc_ue(summary["_id"]["class"]),
						esc_ue(summary["_id"]["host"]),
					],
				],
			]

			html << "</tr>\n"

		end

		html << "</tbody>\n"
		html << "</table>\n"

	end

	html << "</div>\n"
	html << "</div>\n"
	html << "</div>\n"

	html << "</body>\n"
	html << "</html>\n"

	return 200, headers, html

end

#setup ⇒ Object 



44
45
46
47
48
49
 
# File 'lib/hq/log-monitor-server/script.rb', line 44

def setup
	process_args
	read_config
	connect_db
	init_server
end

#sf(format, *args) ⇒ Object 



164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
 
# File 'lib/hq/log-monitor-server/script.rb', line 164

def sf format, *args

	ret = []

	format.scan(/%.|%%|[^%]+|%/).each do
		|match|

		case match

		when ?%
			raise "Error"

		when "%%"
			ret << ?%

		when /^%(.)$/
			ret << send("format_#{$1}", args.shift)

		else
			ret << match

		end

	end

	return ret.join

end

#start ⇒ Object 



33
34
35
36
37
 
# File 'lib/hq/log-monitor-server/script.rb', line 33

def start
	setup
	Thread.new { run }
	@background_thread = Thread.new { background }
end

#status_breakdown(summary, status) ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/hq/log-monitor-server/misc.rb', line 10

def status_breakdown summary, status

	if summary["combined"][status] == 0

		return "0"

	else

		return "%s (%s)" % [

			summary["combined"][status].to_s,

			summary["types"]
				.select {
					|type, counts|
					counts[status] > 0
				}
				.map {
					|type, counts|
					"%s %s" % [ counts[status], type ]
				}
				.join(", ")

		]

	end

end

#stop ⇒ Object 



39
40
41
42
 
# File 'lib/hq/log-monitor-server/script.rb', line 39

def stop
	@web_server.shutdown
	stop_checks
end

#stop_checks ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/hq/log-monitor-server/do-checks.rb', line 73

def stop_checks

	@mutex.synchronize do

		unless @stop_checks_signal
			@stop_checks_signal = ConditionVariable.new
		end

		@run_now_signal.signal

		@stop_checks_signal.wait @mutex

	end

end

#submit_log_event(env) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/hq/log-monitor-server/submit-log-event.rb', line 6

def submit_log_event env

	# decode it

	event = MultiJson.load env["rack.input"].read

	# add some fields

	event["timestamp"] = Time.now
	event["status"] = "unseen"

	# insert it

	@db["events"].insert event

	# update summary

	summary =
		@db["summaries"].find({
			"_id" => event["source"],
		}).first

	summary ||= {
		"_id" => event["source"],
		"combined" => { "new" => 0, "total" => 0 },
		"types" => {},
	}

	summary["types"][event["type"]] ||=
		{ "new" => 0, "total" => 0 }

	summary["types"][event["type"]]["new"] += 1
	summary["types"][event["type"]]["total"] += 1

	summary["combined"]["new"] += 1
	summary["combined"]["total"] += 1

	@db["summaries"].save summary

	# perform checks

	do_checks

	# respond successfully

	return 202, {}, []

end