Class: BarnyardHarvester::Sync

Inherits:
Object
  • Object
show all
Defined in:
lib/barnyard_harvester.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(args) ⇒ Sync

Returns a new instance of Sync.



22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/barnyard_harvester.rb', line 22

def initialize(args)

  @queueing = args.fetch(:queueing) { raise "You must provide :queueing" }
  @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
  @redis_settings = args.fetch(:redis_settings) { DEFAULT_REDIS_SETTINGS }
  @debug = args.fetch(:debug) { false }
  @log = args.fetch(:logger) { Logger.new(STDOUT) }

  @q = BarnyardHarvester::Queue.new(args)

  #@queueing = args[:queueing]
  #
  #case @queueing
  #  when :rabbitmq
  #    @rabbitmq_settings = args.fetch(:rabbitmq_settings) { raise "You must provide :rabbitmq_settings" }
  #  when :sqs
  #    @sqs_settings = args.fetch(:sqs_settings) { raise "You must provide :sqs_settings" }
  #  else
  #    @queueing = :resque
  #end

  @backend = args.fetch(:backend) { :redis }

  if @backend == :mongodb
    @mongodb_settings = args.fetch(:mongodb_settings) { raise "You must provide :mongodb_settings" }
  end

  require "barnyard_harvester/#{@backend.to_s}_helper" if File.exist? "barnyard_harvester/#{@backend.to_s}_helper"
#      require "barnyard_harvester/#{@queueing.to_s}_queue"
  require "barnyard_harvester/#{@backend.to_s}"

#      YAML::ENGINE.yamler = 'syck'

  @uuid = UUID.new
  @harvester_uuid = @uuid.generate

  @key_store = Hash.new

  # Setup barn and queues
  @my_barn = BarnyardHarvester::Barn.new args
  @my_add_queue = BarnyardHarvester::AddQueue.new args
  @my_change_queue = BarnyardHarvester::ChangeQueue.new args
  @my_delete_queue = BarnyardHarvester::DeleteQueue.new args

  @add_count = @change_count = @delete_count = @source_count = @cache_count = 0

end

Instance Attribute Details

#add_count ⇒ Object (readonly)

Returns the value of attribute add_count.



18
19
20
 
# File 'lib/barnyard_harvester.rb', line 18

def add_count
  @add_count
end

#backend ⇒ Object (readonly)

Returns the value of attribute backend.



20
21
22
 
# File 'lib/barnyard_harvester.rb', line 20

def backend
  @backend
end

#cache_count ⇒ Object (readonly)

Returns the value of attribute cache_count.



18
19
20
 
# File 'lib/barnyard_harvester.rb', line 18

def cache_count
  @cache_count
end

#change_count ⇒ Object (readonly)

Returns the value of attribute change_count.



18
19
20
 
# File 'lib/barnyard_harvester.rb', line 18

def change_count
  @change_count
end

#crop_number ⇒ Object (readonly)

Returns the value of attribute crop_number.



20
21
22
 
# File 'lib/barnyard_harvester.rb', line 20

def crop_number
  @crop_number
end

#delete_count ⇒ Object (readonly)

Returns the value of attribute delete_count.



18
19
20
 
# File 'lib/barnyard_harvester.rb', line 18

def delete_count
  @delete_count
end

#harvester_uuid ⇒ Object (readonly)

Returns the value of attribute harvester_uuid.



20
21
22
 
# File 'lib/barnyard_harvester.rb', line 20

def harvester_uuid
  @harvester_uuid
end

#my_add_queue ⇒ Object (readonly)

Returns the value of attribute my_add_queue.



17
18
19
 
# File 'lib/barnyard_harvester.rb', line 17

def my_add_queue
  @my_add_queue
end

#my_barn ⇒ Object (readonly)

Returns the value of attribute my_barn.



17
18
19
 
# File 'lib/barnyard_harvester.rb', line 17

def my_barn
  @my_barn
end

#my_change_queue ⇒ Object (readonly)

Returns the value of attribute my_change_queue.



17
18
19
 
# File 'lib/barnyard_harvester.rb', line 17

def my_change_queue
  @my_change_queue
end

#my_delete_queue ⇒ Object (readonly)

Returns the value of attribute my_delete_queue.



17
18
19
 
# File 'lib/barnyard_harvester.rb', line 17

def my_delete_queue
  @my_delete_queue
end

#redis_settings ⇒ Object (readonly)

Returns the value of attribute redis_settings.



19
20
21
 
# File 'lib/barnyard_harvester.rb', line 19

def redis_settings
  @redis_settings
end

#source_count ⇒ Object (readonly)

Returns the value of attribute source_count.



18
19
20
 
# File 'lib/barnyard_harvester.rb', line 18

def source_count
  @source_count
end

Instance Method Details

#delete_run ⇒ Object 



70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
 
# File 'lib/barnyard_harvester.rb', line 70

def delete_run

  deletes = Array.new

  # Iterate Cache Data, detect deletes.
  @my_barn.each do |primary_key, value|

    @cache_count += 1

    next unless @key_store[primary_key].nil?

    # We got delete
    begin
      change_uuid = @uuid.generate
      @my_delete_queue.push @harvester_uuid, change_uuid, @crop_number, primary_key, BarnyardHarvester::DELETE, value
    rescue Exception => e
      @log.fatal "FATAL error pushing delete #{primary_key} to queue. #{e}"
      exit 1
    end

    deletes << primary_key

    @delete_count += 1
  end

  # Remove Deletes from the Cache Data
  deletes.each do |primary_key|
    @my_barn.delete primary_key
  end

end

#process(primary_key, value) ⇒ Object 



102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
 
# File 'lib/barnyard_harvester.rb', line 102

def process primary_key, value

  @source_count += 1

  change_uuid = @uuid.generate

  @key_store[primary_key] = :present # TODO What did this do: if @call_back.nil?

  if @my_barn.has_key? primary_key

    @log.debug "original: #{@my_barn[primary_key]}"

    YAML::ENGINE.yamler = 'syck'

    @log.debug "current : #{Crack::JSON.parse(value.to_json)}"

    if @my_barn[primary_key] != Crack::JSON.parse(value.to_json)
      #We got change!
      begin
        @my_change_queue.push(@harvester_uuid, change_uuid, @crop_number, primary_key, BarnyardHarvester::CHANGE, value, @my_barn[primary_key])
      rescue Exception => e
        @log.fatal "FATAL error pushing change #{primary_key} to queue. #{e}"
        exit 1
      end

      @my_barn[primary_key] = value
      @change_count += 1
    end
  else
    # We got add!
    #begin
      @my_add_queue.push(@harvester_uuid, change_uuid, @crop_number, primary_key, BarnyardHarvester::ADD, value)
    #rescue Exception => e
    #  @log.fatal "FATAL error pushing add #{primary_key} to queue. #{e}"
    #  exit 1
    #end

    @my_barn[primary_key] = value
    @add_count += 1
  end
end

#run ⇒ Object

def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)

#begin
#rescue Exception => e
#  @log.fatal "#{self.class} Fail in Resque.enqueue of HarvesterLogs. #{e.backtrace}"
#end

end



157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
 
# File 'lib/barnyard_harvester.rb', line 157

def run

  @began_at = Time.now

  yield

  # Detect and queue Deletes
  delete_run

  @ended_at = Time.now


  @my_add_queue.log_run(@harvester_uuid, @crop_number, @began_at, @ended_at, @source_count, @change_count, @add_count, @delete_count)

  # Let Farmer know I'm done and to flush the updates
  @my_barn.flush
  @my_add_queue.flush
  @my_change_queue.flush
  @my_delete_queue.flush


end

#stats ⇒ Object 



144
145
146
 
# File 'lib/barnyard_harvester.rb', line 144

def stats
  "(#{@add_count}) adds, (#{@delete_count}) deletes, (#{@change_count}) changes, (#{@source_count}) source records, (#{@cache_count}) cache records"
end