Class: Bricolage::StreamingLoad::Dispatcher

Inherits:
Bricolage::SQSDataSource::MessageHandler show all
Defined in:
lib/bricolage/streamingload/dispatcher.rb

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Bricolage::SQSDataSource::MessageHandler

#call_handler_method, #handle, #handleable?, #handler_method

Constructor Details

#initialize(event_queue:, task_queue:, object_buffer:, url_patterns:, dispatch_interval:, logger:) ⇒ Dispatcher

Returns a new instance of Dispatcher.



85
86
87
88
89
90
91
92
93
94
95
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 85

def initialize(event_queue:, task_queue:, object_buffer:, url_patterns:, dispatch_interval:, logger:)
  @event_queue = event_queue
  @task_queue = task_queue
  @object_buffer = object_buffer
  @url_patterns = url_patterns
  @dispatch_interval = dispatch_interval
  @dispatch_message_id = nil
  @logger = logger
  @dispatch_requested = false
  @checkpoint_requested = false
end

Instance Attribute Details

#logger ⇒ Object (readonly)

Returns the value of attribute logger.



97
98
99
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 97

def logger
  @logger
end

Class Method Details

.create_pid_file(path) ⇒ Object 



77
78
79
80
81
82
83
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 77

def Dispatcher.create_pid_file(path)
  File.open(path, 'w') {|f|
    f.puts $$
  }
rescue
  # ignore
end

.main ⇒ Object 



21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 21

def Dispatcher.main
  opts = DispatcherOptions.new(ARGV)
  opts.parse
  unless opts.rest_arguments.size == 1
    $stderr.puts opts.usage
    exit 1
  end
  config_path, * = opts.rest_arguments
  config = YAML.load(File.read(config_path))
  log = opts.log_file_path ? new_logger(File.expand_path(opts.log_file_path), config) : nil
  ctx = Context.for_application('.', environment: opts.environment, logger: log)
  logger = raw_logger = ctx.logger
  event_queue = ctx.get_data_source('sqs', config.fetch('event-queue-ds', 'sqs_event'))
  task_queue = ctx.get_data_source('sqs', config.fetch('task-queue-ds', 'sqs_task'))
  if config['alert-level']
    logger = AlertingLogger.new(
      logger: raw_logger,
      sns_datasource: ctx.get_data_source('sns', config.fetch('sns-ds', 'sns')),
      alert_level: config.fetch('alert-level', 'warn')
    )
  end

  object_buffer = ObjectBuffer.new(
    control_data_source: ctx.get_data_source('sql', config.fetch('ctl-postgres-ds', 'db_ctl')),
    logger: logger
  )

  url_patterns = URLPatterns.for_config(config.fetch('url_patterns'))

  dispatcher = Dispatcher.new(
    event_queue: event_queue,
    task_queue: task_queue,
    object_buffer: object_buffer,
    url_patterns: url_patterns,
    dispatch_interval: 60,
    logger: logger
  )

  Process.daemon(true) if opts.daemon?
  create_pid_file opts.pid_file_path if opts.pid_file_path
  Dir.chdir '/'
  dispatcher.event_loop
rescue Exception => e
  logger.exception e
  logger.error "dispatcher abort: pid=#{$$}"
  raise
end

.new_logger(path, config) ⇒ Object 



69
70
71
72
73
74
75
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 69

def Dispatcher.new_logger(path, config)
  Logger.new(
    device: path,
    rotation_period: config.fetch('log-rotation-period', 'daily'),
    rotation_size: config.fetch('log-rotation-size', nil)
  )
end

Instance Method Details

#after_message_batch ⇒ Object

override



108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 108

def after_message_batch
  # must be processed first
  @event_queue.process_async_delete

  if @dispatch_requested
    logger.info "*** dispatch requested"
    dispatch_tasks
    @dispatch_requested = false
  end

  if @checkpoint_requested
    create_checkpoint
    @checkpoint_requested = false   # is needless, but reset it just in case
  end
end

#create_checkpoint ⇒ Object 



144
145
146
147
148
149
150
151
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 144

def create_checkpoint
  logger.info "*** checkpoint requested"
  logger.info "Force-flushing all objects..."
  tasks = @object_buffer.flush_tasks_force
  send_tasks tasks
  logger.info "All objects flushed; shutting down..."
  @event_queue.initiate_terminate
end

#dispatch_tasks ⇒ Object 



172
173
174
175
176
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 172

def dispatch_tasks
  tasks = @object_buffer.flush_tasks
  send_tasks tasks
  set_dispatch_timer
end

#event_loop ⇒ Object 



99
100
101
102
103
104
105
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 99

def event_loop
  logger.info "*** dispatcher started: pid=#{$$}"
  set_dispatch_timer
  @event_queue.handle_messages(handler: self, message_class: Event)
  @event_queue.process_async_delete_force
  logger.info "*** shutdown gracefully: pid=#{$$}"
end

#handle_checkpoint(e) ⇒ Object 



136
137
138
139
140
141
142
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 136

def handle_checkpoint(e)
  # Delay creating CHECKPOINT after the current message batch,
  # because any other extra events are already received.
  @checkpoint_requested = true
  # Delete this event immediately
  @event_queue.delete_message(e)
end

#handle_data(e) ⇒ Object 



153
154
155
156
157
158
159
160
161
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 153

def handle_data(e)
  unless e.created?
    @event_queue.delete_message_async(e)
    return
  end
  obj = e.loadable_object(@url_patterns)
  @object_buffer.put(obj)
  @event_queue.delete_message_async(e)
end

#handle_dispatch(e) ⇒ Object 



163
164
165
166
167
168
169
170
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 163

def handle_dispatch(e)
  # Dispatching tasks may takes 10 minutes or more, it can exceeds visibility timeout.
  # To avoid this, delay dispatching until all events of current message batch are processed.
  if @dispatch_message_id == e.message_id
    @dispatch_requested = true
  end
  @event_queue.delete_message_async(e)
end

#handle_flushtable(e) ⇒ Object 



183
184
185
186
187
188
189
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 183

def handle_flushtable(e)
  logger.info "*** flushtable requested: table=#{e.table_name}"
  tasks = @object_buffer.flush_table_force(e.table_name)
  send_tasks tasks
  # Delete this event immediately
  @event_queue.delete_message(e)
end

#handle_shutdown(e) ⇒ Object 



129
130
131
132
133
134
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 129

def handle_shutdown(e)
  logger.info "*** shutdown requested"
  @event_queue.initiate_terminate
  # Delete this event immediately
  @event_queue.delete_message(e)
end

#handle_unknown(e) ⇒ Object 



124
125
126
127
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 124

def handle_unknown(e)
  logger.warn "unknown event: #{e.message_body}"
  @event_queue.delete_message_async(e)
end

#send_tasks(tasks) ⇒ Object 



191
192
193
194
195
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 191

def send_tasks(tasks)
  tasks.each do |task|
    @task_queue.put task
  end
end

#set_dispatch_timer ⇒ Object 



178
179
180
181
 
# File 'lib/bricolage/streamingload/dispatcher.rb', line 178

def set_dispatch_timer
  res = @event_queue.send_message(DispatchEvent.create(delay_seconds: @dispatch_interval))
  @dispatch_message_id = res.message_id
end