Class: SimpleMapReduce::Worker::RunReduceTaskWorker

Inherits:
Object
  • Object
show all
Defined in:
lib/simple_map_reduce/worker/run_reduce_task_worker.rb

Instance Method Summary collapse

Instance Method Details

#perform(task, reduce_worker) ⇒ Object 



6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/simple_map_reduce/worker/run_reduce_task_worker.rb', line 6

def perform(task, reduce_worker)
  task_wrapper_class_name = "TaskWrapper#{task.id.delete('-')}"
  self.class.class_eval("class #{task_wrapper_class_name}; end", 'Task Wrapper Class')
  task_wrapper_class = self.class.const_get(task_wrapper_class_name)
  task_wrapper_class.class_eval(task.task_script, 'Reduce task script')
  reduce_task = task_wrapper_class.const_get(task.task_class_name, false).new
  unless reduce_task.respond_to?(:reduce)
    # TODO: notifying to job_tracker that this task have failed
    logger.error('no reduce method')
    return
  end

  logger.info('reduce task start')

  local_input_cache = Tempfile.new
  s3_client.get_object(
    response_target: local_input_cache.path,
    bucket: task.task_input_bucket_name,
    key: task.task_input_file_path
  )
  local_input_cache.rewind

  local_output_cache = Tempfile.new
  reduce_task.reduce(local_input_cache, local_output_cache)

  local_output_cache.rewind
  s3_client.put_object(
    body: local_output_cache.read,
    bucket: task.task_output_bucket_name,
    key: "#{task.task_output_directory_path}/#{task.job_id}/#{task.id}_reduce_task_output.txt"
  )

  s3_client.delete_object(
    bucket: task.task_input_bucket_name,
    key: task.task_input_file_path
  )

  # TODO: Notify the task succeeded
rescue => e
  logger.error(e.inspect)
  logger.error(e.backtrace.take(50))

  # TODO: Notify the task failed
ensure
  local_input_cache&.delete
  local_output_cache&.delete
  if self.class.const_defined?(task_wrapper_class_name.to_sym)
    self.class.send(:remove_const, task_wrapper_class_name.to_sym)
  end

  begin
    reduce_worker.ready!
  rescue => notify_error
    logger.fatal(notify_error.inspect)
    logger.fatal(notify_error.backtrace.take(50))
  end

  logger.info('reduce task end')
end