6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
|
# File 'lib/simple_map_reduce/worker/run_reduce_task_worker.rb', line 6
def perform(task, reduce_worker_id)
task_wrapper_class_name = "TaskWrapper#{task.id.delete('-')}"
self.class.class_eval("class #{task_wrapper_class_name}; end", 'Task Wrapper Class')
task_wrapper_class = self.class.const_get(task_wrapper_class_name)
task_wrapper_class.class_eval(task.task_script, 'Reduce task script')
reduce_task = task_wrapper_class.const_get(task.task_class_name, false).new
unless reduce_task.respond_to?(:reduce)
logger.error('no reduce method')
return
end
logger.info('reduce task start')
local_input_cache = Tempfile.new
s3_client.get_object(
response_target: local_input_cache.path,
bucket: task.task_input_bucket_name,
key: task.task_input_file_path
)
local_input_cache.rewind
local_output_cache = Tempfile.new
reduce_task.reduce(local_input_cache, local_output_cache)
local_output_cache.rewind
s3_client.put_object(
body: local_output_cache.read,
bucket: task.task_output_bucket_name,
key: "#{task.task_output_directory_path}/#{task.job_id}/#{task.id}_reduce_task_output.txt"
)
s3_client.delete_object(
bucket: task.task_input_bucket_name,
key: task.task_input_file_path
)
rescue => e
logger.error(e.inspect)
logger.error(e.backtrace.take(50))
ensure
local_input_cache&.delete
local_output_cache&.delete
if self.class.const_defined?(task_wrapper_class_name.to_sym)
self.class.send(:remove_const, task_wrapper_class_name.to_sym)
end
begin
response = http_client(SimpleMapReduce.job_tracker_url).put do |request|
request.url("/workers/#{reduce_worker_id}")
request.body = { event: 'ready' }.to_json
end
logger.debug(response.body)
rescue => notify_error
logger.fatal(notify_error.inspect)
logger.fatal(notify_error.backtrace.take(50))
end
logger.info('reduce task end')
end
|