Class: EScleaner
- Inherits:
-
Object
- Object
- EScleaner
- Defined in:
- lib/elasticsearch-data-cleaner.rb
Instance Method Summary collapse
- #acknowledged_test(response, answer) ⇒ Object
- #delete(object) ⇒ Object
- #docs(index) ⇒ Object
- #exists(index, type) ⇒ Object
- #info ⇒ Object
-
#initialize(options) ⇒ EScleaner
constructor
A new instance of EScleaner.
- #list ⇒ Object
- #optimize(index) ⇒ Object
- #parse_response(response) ⇒ Object
- #run(config) ⇒ Object
Constructor Details
#initialize(options) ⇒ EScleaner
Returns a new instance of EScleaner.
14 15 16 17 18 |
# File 'lib/elasticsearch-data-cleaner.rb', line 14 def initialize() @connection = ::Faraday.new .url, { :request => { :timeout => .timeout } } @options = info() end |
Instance Method Details
#acknowledged_test(response, answer) ⇒ Object
38 39 40 41 42 43 44 45 |
# File 'lib/elasticsearch-data-cleaner.rb', line 38 def acknowledged_test(response, answer) # Testing ES response for errors r = parse_response response.body if r.has_key? answer and r[answer] == true else raise 'request failed' end end |
#delete(object) ⇒ Object
85 86 87 88 89 90 91 92 |
# File 'lib/elasticsearch-data-cleaner.rb', line 85 def delete(object) # Deleting indices or types $logger.info("deleting #{object}#{@options.dry_run ? ' (dry_run)': ''}") if !@options.dry_run resp = @connection.delete "#{object}" acknowledged_test resp, 'acknowledged' end end |
#docs(index) ⇒ Object
71 72 73 74 75 76 77 78 79 80 81 82 83 |
# File 'lib/elasticsearch-data-cleaner.rb', line 71 def docs(index) # Returns number of documents for index resp = @connection.get "#{index}/_stats/docs" r = (parse_response resp.body)['_all']['primaries']['docs']['count'] if r if r < 10 $logger.warn("index #{index} has #{r} docs") end return r else raise "could not get count of docs for index #{index}" end end |
#exists(index, type) ⇒ Object
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 |
# File 'lib/elasticsearch-data-cleaner.rb', line 54 def exists(index, type) #checks if type exists if @es_version['major'] >= 5 url = "#{index}/_mapping/#{type}" else url = "#{index}/#{type}" end resp = @connection.head url if resp.status == 200 return true elsif resp.status == 404 return false else raise "could not check if type #{index}/#{type} exists. got response code #{resp.status}" end end |
#info ⇒ Object
20 21 22 23 24 25 26 27 |
# File 'lib/elasticsearch-data-cleaner.rb', line 20 def info #get ES information resp = @connection.get @es_info = parse_response resp.body @es_version = {} @es_version['major'], @es_version['minor'], @es_version['patch'] = @es_info['version']['number'].split('.').map { |i| i.to_i } $logger.debug("detected Elasticsearch #{@es_version['major']}.#{@es_version['minor']}") end |
#list ⇒ Object
47 48 49 50 51 52 |
# File 'lib/elasticsearch-data-cleaner.rb', line 47 def list # Get list of indices resp = @connection.get "_aliases" r = parse_response resp.body r.keys end |
#optimize(index) ⇒ Object
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 |
# File 'lib/elasticsearch-data-cleaner.rb', line 94 def optimize(index) # Run _forcemerge or _optimize if @es_version['major'] > 2 or (@es_version['major'] == 2 and @es_version['minor'] >= 1) api = '_forcemerge' else api = '_optimize' end $logger.info("starting optimize for index #{index}#{@options.dry_run ? ' (dry_run)': ''}") if !@options.dry_run resp = @connection.post "#{index}/#{api}?only_expunge_deletes=true" failed = (parse_response resp.body)['_shards']['failed'] if failed > 0 $logger.warn("optimizing of index #{index} returns #{failed} failed shards") end end end |
#parse_response(response) ⇒ Object
29 30 31 32 33 34 35 36 |
# File 'lib/elasticsearch-data-cleaner.rb', line 29 def parse_response(response) j = ::JSON.parse response if j.has_key?('error') raise j['error'] else j end end |
#run(config) ⇒ Object
111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 |
# File 'lib/elasticsearch-data-cleaner.rb', line 111 def run(config) # Run rotate of ES data config.each do |pattern, settings| a = [] list().each do |index| # check if empty just delete it if !@options.empty and docs(index) == 0 delete(index) else begin a << DateTime.strptime(index, pattern) rescue end end end today = Date.today sorted = a.sort while sorted.length > 0 need_to_optimize = false date = sorted.pop d = date.to_date index = date.strftime(pattern) if d > today if settings['future'] == false $logger.debug("going to delete index #{index}. It is future") delete(index) end else if settings['number'] if settings['number'] <= 0 delete(index) else if settings['types'] # Checking types settings settings['types'].each do |type, type_settings| if exists(index, type) if type_settings['number'] if type_settings['number'] <= 0 delete("#{index}/#{type}") if type_settings['optimize'] and need_to_optimize == false need_to_optimize = true end else type_settings['number'] -= 1 end end end end end end settings['number'] -= 1 end end a.delete(date) # do optimize if needed if need_to_optimize optimize(index) end end end end |