Module: GreenHat::Query

Defined in:
lib/greenhat/shell/query.rb

Overview

Query Handlers rubocop:disable Metrics/ModuleLength

Class Method Summary collapse

Class Method Details

.build_time_index(results, interval = 5.minutes) ⇒ Object

Create list of Times



231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
 
# File 'lib/greenhat/shell/query.rb', line 231

def self.build_time_index(results, interval = 5.minutes)
  index = {}
  start = results.min_by(&:time).time.floor(interval)
  finish = results.max_by(&:time).time.floor(interval)

  loop do
    index[start] = 0
    start += interval
    break if start > finish
  end

  index
rescue StandardError => e
  LogBot.fatal('Index Error', e.message)
end

.calculate_duration(results) ⇒ Object 



135
136
137
138
139
140
141
142
143
144
 
# File 'lib/greenhat/shell/query.rb', line 135

def self.calculate_duration(results)
  # Skip for Pluck
  only_with_time = results.select { |x| x.instance_of?(Hash) && x.key?(:time) && x[:time].instance_of?(Time) }

  # If slice is used ignore
  return nil if only_with_time.empty?

  sorted = only_with_time.map(&:time).sort
  humanize_time(sorted.first, sorted.last)
end

.combine_src_add(files) ⇒ Object 



24
25
26
27
28
29
30
31
32
33
34
35
36
 
# File 'lib/greenhat/shell/query.rb', line 24

def self.combine_src_add(files)
  files.reject(&:blank?).map do |file|
    src = file.archive&.friendly_name || file&.name
    file.data.map do |entry|
      # Some entries are nil?
      next if entry.nil?

      bit = entry.clone
      bit[:src] = src
      bit
    end.compact
  end
end

.create_title(file, flags, results) ⇒ Object

Helper to simplify Title Creation



69
70
71
72
73
74
 
# File 'lib/greenhat/shell/query.rb', line 69

def self.create_title(file, flags, results)
  title = file.friendly_name
  # Ignore for Total Results
  title += " #{results.count}".pastel(:bright_black) unless flags.key?(:total)
  title
end

.file_filter(files, flags = {}, args = {}) ⇒ Object

Simplify Loop for Query.start



39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
 
# File 'lib/greenhat/shell/query.rb', line 39

def self.file_filter(files, flags = {}, args = {})
  # Iterate and Preserve Archive/Host Index
  files.each_with_object([]) do |file, obj|
    # Ignore Empty Results / No Thing
    next if file&.blank?

    # Include Total Count in Name
    results = Query.filter(file.data, flags, args)

    next if results.count.zero? # Skip if there are no results

    duration = calculate_duration(results)

    # Create Title
    title = create_title(file, flags, results)

    # Append Duration
    title += " #{duration.pastel(:cyan, :dim)}" unless duration.blank?

    # Add Title
    obj.push(title)

    # Add Results
    obj.concat(results)

    obj
  end
end

.filter(data, flags = {}, args = {}) ⇒ Object

Filter Logic TODO: Simplify



163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
 
# File 'lib/greenhat/shell/query.rb', line 163

def self.filter(data, flags = {}, args = {})
  results = data.clone

  # Handle Transform
  process_transform(flags[:transform], results) if flags.key? :transform

  results.select! do |row|
    # Skip Integers?
    # next if row.is_a? Integer

    args.send(flags.logic || :all?) do |arg|
      filter_row_key(row, arg, flags)
    end
  end

  # Ensure presecense of a specific field
  results = filter_exists(results, flags[:exists]) if flags.key?(:exists)

  # Time Zone
  results = filter_modify_timezone(results, flags[:time_zone]) if flags.key?(:time_zone)

  # Time Filtering
  results = filter_time(results, flags) if flags.key?(:start) || flags.key?(:end)

  # Strip Results if Slice is defined
  results = filter_slice(results, flags[:slice]) if flags.key?(:slice)

  # Strip Results if Except is defined
  results = filter_except(results, flags[:except]) if flags.key?(:except)

  # Remove Blank from either slice or except
  results.reject!(&:blank?)

  # Sort / Reverse by default
  if flags.key?(:sort)
    results.sort_by! { |x| x.slice(*flags[:sort]).values }
    results.reverse!
  end

  # JSON Formatting
  results = results.map { |x| Oj.dump(x) } if flags.key?(:json)

  # Show Unique Only
  results = filter_uniq(results, flags[:uniq]) if flags.key?(:uniq)

  # Reverse
  results.reverse! if flags[:reverse]

  # Count occurrences / Skip Results
  return filter_stats(results, flags) if flags.key?(:stats)

  # Percentile Breakdown
  return filter_percentile(results, flags) if flags.key?(:percentile)

  # Limit before Pluck / Flattening
  results = filter_limit(results, flags[:limit]) if flags.key?(:limit)

  # Pluck
  results = filter_pluck(results, flags[:pluck]) if flags.key?(:pluck)

  # Total Counter
  results = ShellHelper.total_count(results, flags) if flags.key?(:total)

  results
end

.filter_count_occurrences(results, field, flags = {}) ⇒ Object

Helper to Count occurrences



441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
 
# File 'lib/greenhat/shell/query.rb', line 441

def self.filter_count_occurrences(results, field, flags = {})
  results.each_with_object(Hash.new(0)) do |entry, counts|
    if entry.key? field
      # Rounding in pagination breaks stats
      key = if flags.key?(:round) && entry[field].numeric?
              entry[field].to_f.round(flags.round)
            else
              entry[field]
            end

      counts[key] += 1
    else
      counts['None'.pastel(:bright_black)] += 1
    end

    counts
  end
end

.filter_empty_arg(arg) ⇒ Object 



475
476
477
478
479
480
481
 
# File 'lib/greenhat/shell/query.rb', line 475

def self.filter_empty_arg(arg)
  puts [
    'Ignoring'.pastel(:bright_yellow),
    "--#{arg}".pastel(:cyan),
    'it requires an argument'.pastel(:red)
  ].join(' ')
end

.filter_entry_cast(entry, arg, flags) ⇒ Object

Handle casting to strings or integers



517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
 
# File 'lib/greenhat/shell/query.rb', line 517

def self.filter_entry_cast(entry, arg, flags)
  # Cast to String
  value = arg.value.to_s

  # No Logic on Empty Entries
  return [entry, value] if entry.empty?

  case arg.logic
  when :include?

    # Exact Case argument
    unless flags.key?(:case)
      entry = entry.downcase
      value = value.downcase
    end
  when :>=, :<=
    entry = entry.to_i if entry.numeric?
    value = value.to_i if value&.numeric?
  end

  [entry, value]
end

.filter_entry_logic(entry, arg) ⇒ Object 



540
541
542
 
# File 'lib/greenhat/shell/query.rb', line 540

def self.filter_entry_logic(entry, arg)
  entry.send(arg.logic, arg.value)
end

.filter_except(results, except) ⇒ Object 



303
304
305
306
307
308
309
310
311
 
# File 'lib/greenhat/shell/query.rb', line 303

def self.filter_except(results, except)
  # Avoid Empty Results
  if except.empty?
    filter_empty_arg('except')
    return results
  end

  results.map { |row| row.except(*except) }
end

.filter_exists(results, exists) ⇒ Object 



313
314
315
316
317
318
319
320
321
 
# File 'lib/greenhat/shell/query.rb', line 313

def self.filter_exists(results, exists)
  # Avoid Empty Results
  if exists.empty?
    filter_empty_arg('exists')
    return results
  end

  results.select { |row| (exists - row.keys).empty? }
end

.filter_limit(results, limit) ⇒ Object

Limit / Ensure Exists and Valid Number



248
249
250
251
252
 
# File 'lib/greenhat/shell/query.rb', line 248

def self.filter_limit(results, limit)
  return results unless limit.integer? && limit.positive?

  results.shift limit
end

.filter_modify_timezone(results, time_zone) ⇒ Object 



254
255
256
257
258
259
260
261
262
263
 
# File 'lib/greenhat/shell/query.rb', line 254

def self.filter_modify_timezone(results, time_zone)
  results.map do |x|
    next unless x.key? :time

    x = x.clone # Prevent Top Level Modification
    x[:time] = x[:time].in_time_zone time_zone.upcase

    x
  end
end

.filter_percentile(results, flags) ⇒ Object

TODO: Make better rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity



357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
 
# File 'lib/greenhat/shell/query.rb', line 357

def self.filter_percentile(results, flags)
  output = {}

  results.each do |entry|
    entry.each do |key, value|
      # Numbers only
      next unless value.instance_of?(Integer) || value.instance_of?(Float)

      output[key] ||= []
      output[key].push value
    end
  end

  data = output.map do |key, values|
    # Support Rounding
    l99 = values.percentile(0.99)
    l99 = l99.round(flags.round) if flags.round

    l95 = values.percentile(0.99)
    l95 = l95.round(flags.round) if flags.round

    {
      key: key,
      '99' => l99,
      '95' => l95,
      mean: flags.round ? values.mean.round(flags.round) : values.mean,
      min: flags.round ? values.min.round(flags.round) : values.min,
      max: flags.round ? values.max.round(flags.round) : values.max,
      count: values.count
    }
  end

  headers = data.flat_map(&:keys).uniq
  [[headers, data.map(&:values)]] # Multiple Arrays for Results Flatten
end

.filter_pluck(results, pluck) ⇒ Object 



333
334
335
336
337
338
339
340
341
 
# File 'lib/greenhat/shell/query.rb', line 333

def self.filter_pluck(results, pluck)
  # Avoid Empty Results
  if pluck.empty?
    filter_empty_arg('pluck')
    return results
  end

  results.map { |x| x.slice(*pluck).values }.flatten
end

.filter_row_entry(entry, arg, flags) ⇒ Object

Field Partial / Case / Exact search



506
507
508
509
510
511
512
513
514
 
# File 'lib/greenhat/shell/query.rb', line 506

def self.filter_row_entry(entry, arg, flags)
  # Exact Matching / Unless doing full text search
  return entry == arg.value.to_s if flags.key?(:exact) && arg.field != :text

  # Cast to String/Integer Helper
  entry, value = filter_entry_cast(entry, arg, flags)

  entry.send(arg.logic, value)
end

.filter_row_key(row, arg, flags) ⇒ Object

Break out filter row logic into separate method



484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
 
# File 'lib/greenhat/shell/query.rb', line 484

def self.filter_row_key(row, arg, flags)
  return false if row.nil? || row.zero? # Nothing to filter if row empty

  # Ignore Other Logic if Field isn't even included / Full Text Searching
  return false unless row.key?(arg[:field]) || arg[:field] == :text

  # Sensitivity Check / Check for Match / Full Text Searching
  search_data = arg[:field] == :text ? row : row[arg.field]
  match = filter_row_entry(search_data.to_s, arg, flags)

  # Pivot of off include vs exclude
  if arg.bang
    !match
  else
    match
  end
rescue StandardError => e
  LogBot.fatal('[Query] Filter Row Failure', message: e.message, backtrace: e.backtrace.first, row: row)
  false
end

.filter_slice(results, slice) ⇒ Object 



323
324
325
326
327
328
329
330
331
 
# File 'lib/greenhat/shell/query.rb', line 323

def self.filter_slice(results, slice)
  # Avoid Empty Results
  if slice.empty?
    filter_empty_arg('slice')
    return results
  end

  results.compact.map { |row| row.slice(*slice) }
end

.filter_stats(results, flags) ⇒ Object

rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity



394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
 
# File 'lib/greenhat/shell/query.rb', line 394

def self.filter_stats(results, flags)
  stats = flags[:stats]

  # Avoid Empty Results
  if stats.empty?
    filter_empty_arg('stats')
    return results
  end

  # Loop through Stats, Separate Hash/Tables
  stats.map do |field|
    occurrences = filter_count_occurrences(results, field, flags)

    # Use Truncate For Long Keys
    occurrences.transform_keys! { |key| key.to_s[0..flags[:truncate]] } if flags[:truncate]

    # Total Occurences
    total = occurrences.values.sum

    # Limit Output
    filter_stats_limiter(occurrences, total, flags[:stats_limit]) if flags.key?(:stats_limit)

    # Percs
    occurrences.transform_values! do |count|
      [
        count,
        " #{percent(count, total)}%".pastel(:bright_black)
      ]
    end

    # Sort by total occurances / New Variable for Total
    output = occurrences.sort_by(&:last).to_h.transform_values!(&:join).to_a

    # Append Header / Total with field name
    output.unshift([field.to_s.pastel(:bright_black), total])

    # Format
    output.to_h
  end
end

.filter_stats_limiter(results, total, limits) ⇒ Object

Additional Filtering through stats_limit



461
462
463
464
465
466
467
468
469
470
471
472
473
 
# File 'lib/greenhat/shell/query.rb', line 461

def self.filter_stats_limiter(results, total, limits)
  results.select! do |_k, v|
    limits.all? do |limit|
      # Shim for Integers
      if limit.logic == :include?
        percent(v, total).send(:>=, limit.value)
      else
        percent(v, total).send(limit.logic, limit.value)
      end
      # ------------
    end
  end
end

.filter_time(results, flags) ⇒ Object

Filter Start and End Times TODO: This is a bit icky, simplify/dry



267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
 
# File 'lib/greenhat/shell/query.rb', line 267

def self.filter_time(results, flags)
  if flags.key?(:start)
    begin
      time_start = Time.parse(flags[:start])

      results.select! do |x|
        if x.time
          time_start < x.time
        else
          true
        end
      end
    rescue StandardError
      puts 'Unable to Process Start Time Filter'.pastel(:red)
    end
  end

  if flags.key?(:end)
    begin
      time_start = Time.parse(flags[:end])

      results.select! do |x|
        if x.time
          time_start > x.time
        else
          true
        end
      end
    rescue StandardError
      puts 'Unable to Process End Time Filter'.pastel(:red)
    end
  end

  results
end

.filter_uniq(results, unique) ⇒ Object 



343
344
345
346
347
348
349
350
351
352
353
 
# File 'lib/greenhat/shell/query.rb', line 343

def self.filter_uniq(results, unique)
  # Avoid Empty Results
  if unique.empty?
    filter_empty_arg('uniq')
    return results
  end

  unique.map do |field|
    results.uniq { |x| x[field] }
  end.inject(:&)
end

.humanize_time(time_start, time_end, increments = 2) ⇒ Object

Replace TimeDifference with stackoverflow.com/a/4136485/1678507



147
148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/greenhat/shell/query.rb', line 147

def self.humanize_time(time_start, time_end, increments = 2)
  miliseconds = (time_end - time_start) * 1000

  list = [[1000, :ms], [60, :s], [60, :m], [24, :h]].map do |count, name|
    next unless miliseconds.positive?

    miliseconds, n = miliseconds.divmod(count)

    "#{n.to_i}#{name}" unless n.to_i.zero?
  end

  list.compact.reverse[0..increments - 1].join(' ')
end

.percent(value, total) ⇒ Object

Percent Helper



436
437
438
 
# File 'lib/greenhat/shell/query.rb', line 436

def self.percent(value, total)
  ((value / total.to_f) * 100).round
end

.process_interval(files, flags, args) ⇒ Object

Organize Results by Interval TODO Simplify rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity



93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/greenhat/shell/query.rb', line 93

def self.process_interval(files, flags, args)
  data = files.reject(&:blank?).map(&:data).flatten.compact

  # Handle Transform
  process_transform(flags[:transform], data) if flags.key? :transform

  interval = ChronicDuration.parse(flags[:interval]) || 5.minutes
  index = build_time_index(data, interval)
  index.transform_values! { |_y| [] }

  data.each do |r|
    index[r.time.floor(interval)] << r
  end

  output = index.each_with_object({}) do |(time, entries), obj|
    results = Query.filter(entries, flags, args)

    next if results.count.zero? # Skip if No Results

    duration = calculate_duration(results)

    # Timzone Manipulation
    time_title = flags.key?(:time_zone) ? time.in_time_zone(flags[:time_zone]) : time

    title = time_title.to_s.pastel(:bright_blue, :bold)
    title += " #{results.count}".pastel(:bright_black) unless flags.key?(:total)

    # Append Duration
    title += " #{duration.pastel(:cyan, :dim)}" unless duration.blank?

    # Add Results
    obj[title] = results

    obj
  end

  # Remove Empty Intervals / There may be empty here due to results
  output.reject! { |_k, v| v.flatten.empty? }

  output.to_a.flatten(2)
end

.process_transform(transform, results) ⇒ Object

Helper to transform fields from one to another (timestamp => time)



77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/greenhat/shell/query.rb', line 77

def self.process_transform(transform, results)
  # TODO: Eventually provide more splat options
  from, to, *splat = transform
  results.each do |entry|
    entry[to.to_sym] = case splat
                       when [:to_i]
                         entry[from.to_sym].to_i
                       else
                         entry[from.to_sym]
                       end
  end
end

.start(files, flags = {}, args = {}) ⇒ Object

Main Entry Point for Filtering



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/greenhat/shell/query.rb', line 7

def self.start(files, flags = {}, args = {})
  # Convert to Things / Thing.list == processed files
  files = ShellHelper.find_things(files, flags, Thing.list).select(&:query?)

  # Breakdown by Interval
  return process_interval(files, flags, args) if flags.key? :interval

  # Ignore Archive/Host Dividers
  if flags[:combine]
    # Add SRC Field to Combined Entries
    results = combine_src_add(files)
    Query.filter(results.flatten.compact, flags, args)
  else
    file_filter(files, flags, args)
  end
end