Class: Flex::Admin::Tasks

Inherits:
Object
  • Object
show all
Defined in:
lib/flex/admin.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(overrides = {}) ⇒ Tasks

Returns a new instance of Tasks.



8
9
10
11
12
13
14
15
16
17
 
# File 'lib/flex/admin.rb', line 8

def initialize(overrides={})
  options = Flex::Utils.env2options *default_options.keys

  options[:size]       = options[:size].to_i       if options[:size]
  options[:timeout]    = options[:timeout].to_i    if options[:timeout]
  options[:batch_size] = options[:batch_size].to_i if options[:batch_size]
  options[:index_map]  = Hash[options[:index_map].split(',').map{|i|i.split(':')}] if options[:index_map] && options[:index_map].is_a?(String)

  @options = default_options.merge(options).merge(overrides)
end

Instance Attribute Details

#options ⇒ Object (readonly)

Returns the value of attribute options.



6
7
8
 
# File 'lib/flex/admin.rb', line 6

def options
  @options
end

Instance Method Details

#default_options ⇒ Object 



19
20
21
22
23
24
25
26
27
28
29
 
# File 'lib/flex/admin.rb', line 19

def default_options
  @default_options ||= { :file       => './flex.dump',
                         :index      => Conf.variables[:index],
                         :type       => Conf.variables[:type],
                         :scroll     => '5m',
                         :size       => 50,
                         :timeout    => 20,
                         :batch_size => 1000,
                         :verbose    => true,
                         :index_map  => nil }
end

#dump_to_file(cli = false) ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
 
# File 'lib/flex/admin.rb', line 31

def dump_to_file(cli=false)
  vars = { :index => cli ? options[:index] : (options[:index] || Flex::Tasks.new.config_hash.keys),
           :type  => options[:type] }
  if options[:verbose]
    total_hits  = Flex.count(vars)['count'].to_i
    total_count = 0
    pbar        = ProgBar.new(total_hits)
    dump_stats  = Hash.new { |hash, key| hash[key] = Hash.new { |h, k| h[k] = 0 } }
    file_size   = 0
  end
  vars.merge! :params => { :scroll => options[:scroll],
                           :size   => options[:size],
                           :fields => '_source,*' }

  file = options[:file].is_a?(String) ? File.open(options[:file], 'wb') : options[:file]
  path = file.path

  Flex.dump_all(vars) do |batch|
    bulk_string = ''
    batch.each do |document|
      dump_stats[document['_index']][document['_type']] += 1 if options[:verbose]
      bulk_string << Flex.build_bulk_string(document)
    end
    file.puts bulk_string
    if options[:verbose]
      total_count += batch.size
      pbar.pbar.inc(batch.size)
    end
  end
  file_size = file.size if options[:verbose]
  file.close

  if options[:verbose]
    formatted_file_size = file_size.to_s.reverse.gsub(/...(?=.)/, '\&,').reverse
    pbar.pbar.finish
    puts "\n***** WARNING: Expected document to dump: #{total_hits}, dumped: #{total_count}. *****" \
         unless total_hits == total_count
    puts "\nDumped #{total_count} documents to #{path} (size: #{formatted_file_size} bytes)"
    puts dump_stats.to_yaml
  end
end

#load_from_file ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
 
# File 'lib/flex/admin.rb', line 73

def load_from_file
  Configuration.http_client.options[:timeout] = options[:timeout]
  chunk_size  = options[:batch_size] * 2 # 2 lines per doc
  bulk_string = ''
  file        = options[:file].is_a?(String) ? File.open(options[:file]) : options[:file]
  path        = file.path
  if options[:verbose]
    line_count = 0
    file.lines { line_count += 1 }
    file.rewind
    puts "\nLoading from #{path}...\n"
    pbar = ProgBar.new(line_count / 2, options[:batch_size])
  end
  file.lines do |line|
    bulk_string << (options[:index_map] ? map_index(line) : line)
    if (file.lineno % chunk_size) == 0
      result = Flex.post_bulk_string :bulk_string => bulk_string
      bulk_string  = ''
      pbar.process_result(result, options[:batch_size]) if options[:verbose]
    end
  end
  # last chunk
  unless bulk_string == ''
    result = Flex.post_bulk_string :bulk_string => bulk_string
    pbar.process_result(result, (file.lineno % chunk_size) / 2) if options[:verbose]
  end
  file.close
  pbar.finish if options[:verbose]
end