Class: Pmux::TaskScheduler

Inherits:
Object
  • Object
show all
Defined in:
lib/pmux/task_scheduler.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(adapter = nil) ⇒ TaskScheduler

Returns a new instance of TaskScheduler.



6
7
8
9
10
11
12
13
 
# File 'lib/pmux/task_scheduler.rb', line 6

def initialize adapter=nil
  @adapter = adapter
  @node_table = adapter
  @job_table = {}
  @task_queue = TaskQueue.new
  @allocated_tasks = {}
  @shipped = {}
end

Instance Attribute Details

#shipped ⇒ Object (readonly)

Returns the value of attribute shipped.



4
5
6
 
# File 'lib/pmux/task_scheduler.rb', line 4

def shipped
  @shipped
end

Instance Method Details

#allocate_map_task_to_node(job_map, task) ⇒ Object 



143
144
145
146
147
148
149
150
151
152
153
154
155
 
# File 'lib/pmux/task_scheduler.rb', line 143

def allocate_map_task_to_node job_map, task
  node_addrs = task[:node_addrs].dup
  for node_addr, path in node_addrs
    next unless @node_table[node_addr]
    next unless path
    task[:path] = path
    if allocate_task_to_node job_map, task, node_addr
      task[:node_addrs].delete [node_addr, path]
      return true
    end
  end
  return false
end

#allocate_reduce_task_to_node(job_map, task) ⇒ Object 



158
159
160
161
162
163
164
165
 
# File 'lib/pmux/task_scheduler.rb', line 158

def allocate_reduce_task_to_node job_map, task
  return false unless (node_addr = task[:node_addr])
  if allocate_task_to_node job_map, task, node_addr
    return true
  else
    return false
  end
end

#allocate_task_to_node(job_map, task, node_addr) ⇒ Object 



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 
# File 'lib/pmux/task_scheduler.rb', line 90

def allocate_task_to_node job_map, task, node_addr
  if (ff = task[:ff]) and !task[:pindex]
    # task fusion
    job_id = task[:job_id]
    if (nt_map = job_map[job_id]) and (fslots = nt_map[node_addr])
      for fslot in fslots
        if fslot.size < ff
          fslot.push task
          return true
        end
      end
    end
  end

  #if @node_table.allocate_task node_addr, task
  if allocate_task_to_slot node_addr, task
    # success
    job_id = task[:job_id]
    nt_map = (job_map[job_id] ||= {})

    #task = task.dup #???
    task[:alloc_time] = Time.now
    (nt_map[node_addr] ||= []).push [task]

    return true
  end
  return false
end

#allocate_task_to_slot(node_addr, task) ⇒ Object 



120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'lib/pmux/task_scheduler.rb', line 120

def allocate_task_to_slot node_addr, task
  if (node = @node_table[node_addr]) and @shipped[node_addr]
    slot = (@allocated_tasks[node_addr] ||= [])
    num_workers = node[:num_workers] || 2
    if slot.size >= num_workers
      return false
    else
      return false if slot.include? task
      task[:node_addr] = node_addr
      slot.push task
      return true
    end
  end
end

#attach_flush_callback(&block) ⇒ Object 



68
69
70
 
# File 'lib/pmux/task_scheduler.rb', line 68

def attach_flush_callback &block
  @flush_callback = block
end

#delete_task_from_job(job, task, node_addr = nil) ⇒ Object 



24
25
26
27
28
 
# File 'lib/pmux/task_scheduler.rb', line 24

def delete_task_from_job job, task, node_addr=nil
  node_addr ||= task[:node_addr]
  remove_allocated_task node_addr, job.id, task[:task_id]
  job.delete_task_by_id task[:task_id]
end

#flush_job_map(job_map) ⇒ Object 



73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
 
# File 'lib/pmux/task_scheduler.rb', line 73

def flush_job_map job_map
  for job_id, nt_map in job_map
    for node_addr, fslots in nt_map
      for fslot in fslots
        if fslot.size > 1
          task_keys =
            Hash[*(fslot.map {|t| [t[:task_id], t[:path]]}).flatten]
          task = fslot.first.merge :task_keys=>task_keys
          @flush_callback.call node_addr, task if @flush_callback
        else
          @flush_callback.call node_addr, fslot.first if @flush_callback
        end
      end
    end
  end
end

#inject_tasks(tasks) ⇒ Object 



20
21
22
 
# File 'lib/pmux/task_scheduler.rb', line 20

def inject_tasks tasks
  @task_queue.inject_tasks tasks
end

#process_queue ⇒ Object 



30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
 
# File 'lib/pmux/task_scheduler.rb', line 30

def process_queue
  job_map = {}
  success_count = 0
  fail_count = 0
  fail_lim = @node_table.size * 2
  tmp_queue = []

  task_queue_size = @task_queue.size
  loop_count = 0
  while task = @task_queue.shift
    if task[:pindex] and task[:node_addr]
      allocated_p = allocate_reduce_task_to_node job_map, task
    else
      unless task[:node_addrs]
        task[:node_addrs] = @adapter.lookup_file task[:file]
      end
      allocated_p = allocate_map_task_to_node job_map, task
    end
    if allocated_p
      # success
      fail_count = 0
    else
      # fail
      tmp_queue.push task
      fail_count += 1
      break if fail_count >= fail_lim
    end

    loop_count += 1

    break if loop_count >= task_queue_size
  end
  @task_queue.replace tmp_queue + @task_queue

  flush_job_map job_map
end

#push_job(job) ⇒ Object 



15
16
17
18
 
# File 'lib/pmux/task_scheduler.rb', line 15

def push_job job
  @job_table[job.id] = job
  @task_queue.inject_tasks job.tasks
end

#remove_allocated_task(node_addr, job_id, task_id) ⇒ Object 



136
137
138
139
140
 
# File 'lib/pmux/task_scheduler.rb', line 136

def remove_allocated_task node_addr, job_id, task_id
  if (slot = @allocated_tasks[node_addr])
    slot.delete_if {|t| t[:job_id] == job_id and t[:task_id] == task_id}
  end
end