Class: Workflow::Orchestrator
- Inherits:
-
Object
- Object
- Workflow::Orchestrator
- Defined in:
- lib/rbbt/workflow/util/orchestrator.rb
Instance Attribute Summary collapse
-
#available_resources ⇒ Object
Returns the value of attribute available_resources.
-
#resources_requested ⇒ Object
Returns the value of attribute resources_requested.
-
#resources_used ⇒ Object
Returns the value of attribute resources_used.
-
#timer ⇒ Object
Returns the value of attribute timer.
Class Method Summary collapse
- .candidates(workload, rules) ⇒ Object
- .job_resources(rules, job) ⇒ Object
- .job_rules(rules, job) ⇒ Object
- .job_workload(job) ⇒ Object
- .purge_duplicates(candidates) ⇒ Object
- .sort_candidates(candidates, rules) ⇒ Object
- .workload(jobs) ⇒ Object
Instance Method Summary collapse
- #check_resources(rules, job) ⇒ Object
- #erase_job_dependencies(job, rules, all_jobs, top_level_jobs) ⇒ Object
-
#initialize(timer = 5, available_resources = {}) ⇒ Orchestrator
constructor
A new instance of Orchestrator.
- #process(rules, jobs) ⇒ Object
- #release_resources(job) ⇒ Object
- #run_with_rules(rules, job) ⇒ Object
Constructor Details
#initialize(timer = 5, available_resources = {}) ⇒ Orchestrator
Returns a new instance of Orchestrator.
103 104 105 106 107 108 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 103 def initialize(timer = 5, available_resources = {}) @timer = timer @available_resources = IndiferentHash.setup(available_resources) @resources_requested = IndiferentHash.setup({}) @resources_used = IndiferentHash.setup({}) end |
Instance Attribute Details
#available_resources ⇒ Object
Returns the value of attribute available_resources.
101 102 103 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 101 def available_resources @available_resources end |
#resources_requested ⇒ Object
Returns the value of attribute resources_requested.
101 102 103 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 101 def resources_requested @resources_requested end |
#resources_used ⇒ Object
Returns the value of attribute resources_used.
101 102 103 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 101 def resources_used @resources_used end |
#timer ⇒ Object
Returns the value of attribute timer.
101 102 103 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 101 def timer @timer end |
Class Method Details
.candidates(workload, rules) ⇒ Object
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 81 def self.candidates(workload, rules) if rules.empty? candidates = workload.select{|k,v| v.empty? }. collect{|k,v| k}. reject{|k| k.done? } else candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }. select{|k,v| v.empty? }. collect{|k,v| k }. reject{|k| k.done? } end top_level = workload.keys - workload.values.flatten candidates = purge_duplicates candidates candidates = sort_candidates candidates, rules candidates end |
.job_resources(rules, job) ⇒ Object
63 64 65 66 67 68 69 70 71 72 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 63 def self.job_resources(rules, job) resources = (job_rules(rules, job) || {})["resources"] || {} IndiferentHash.setup(resources) default_resources = rules["default_resources"] || rules["defaults"]["resources"] default_resources.each{|k,v| resources[k] ||= v } if default_resources resources end |
.job_rules(rules, job) ⇒ Object
38 39 40 41 42 43 44 45 46 47 48 49 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 38 def self.job_rules(rules, job) workflow = job.workflow.to_s task_name = job.task_name.to_s defaults = rules["defaults"] || {} return IndiferentHash.setup(defaults) unless rules[workflow] return IndiferentHash.setup(defaults) unless rules[workflow][task_name] job_rules = IndiferentHash.setup(rules[workflow][task_name]) defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults job_rules end |
.job_workload(job) ⇒ Object
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 6 def self.job_workload(job) workload = {job => []} return workload if job.done? job.dependencies.each do |dep| next if dep.done? workload.merge!(job_workload(dep)) workload[job] += workload[dep] workload[job] << dep workload[job].uniq! end job.input_dependencies.each do |dep| next if dep.done? workload.merge!(job_workload(dep)) workload[job] += workload[dep] workload[job] << dep workload[job].uniq! end workload end |
.purge_duplicates(candidates) ⇒ Object
51 52 53 54 55 56 57 58 59 60 61 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 51 def self.purge_duplicates(candidates) seen = Set.new candidates.select do |job| if seen.include? job.path false else seen << job.path true end end end |
.sort_candidates(candidates, rules) ⇒ Object
74 75 76 77 78 79 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 74 def self.sort_candidates(candidates, rules) seen = Set.new candidates.sort_by do |job| - job_resources(rules, job).values.inject(0){|acc,e| acc += e} end end |
.workload(jobs) ⇒ Object
29 30 31 32 33 34 35 36 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 29 def self.workload(jobs) jobs.inject({}) do |acc,job| Orchestrator.job_workload(job).each do |j,d| acc[j] = d unless acc.keys.collect{|k| k.path }.include? j.path end acc end end |
Instance Method Details
#check_resources(rules, job) ⇒ Object
121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 121 def check_resources(rules, job) resources = Orchestrator.job_resources(rules, job) limit_resources = resources.select{|resource,value| available_resources[resource] && ((resources_requested[resource] || 0) + value) > available_resources[resource] }.collect{|resource,v| resource } if limit_resources.any? Log.debug "Orchestrator waiting on #{job.path} due to #{limit_resources * ", "}" else resources_used[job] = resources resources.each do |resource,value| resources_requested[resource] ||= 0 resources_requested[resource] += value.to_i end Log.low "Orchestrator producing #{job.path} with resources #{resources}" return yield end end |
#erase_job_dependencies(job, rules, all_jobs, top_level_jobs) ⇒ Object
156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 156 def erase_job_dependencies(job, rules, all_jobs, top_level_jobs) job.dependencies.each do |dep| next if top_level_jobs.include? dep.path next unless Orchestrator.job_rules(rules, dep)["erase"].to_s == 'true' dep_path = dep.path parents = all_jobs.select do |parent| paths = parent.info[:dependencies].nil? ? parent.dependencies.collect{|d| d.path } : parent.info[:dependencies].collect{|d| d.last } paths.include? dep_path end next unless parents.reject{|parent| parent.done? }.empty? parents.each do |parent| Log.high "Erasing #{dep.path} from #{parent.path}" parent.archive_deps parent.copy_files_dir parent.dependencies = parent.dependencies - [dep] end dep.clean end end |
#process(rules, jobs) ⇒ Object
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 179 def process(rules, jobs) begin workload = Orchestrator.workload(jobs) all_jobs = workload.keys top_level_jobs = jobs.collect{|job| job.path } while workload.any? candidates = resources_used.keys + Orchestrator.candidates(workload, rules) raise "No candidates and no running jobs" if candidates.empty? candidates.each do |job| case when (job.error? || job.aborted?) begin if job.recoverable_error? job.clean raise TryAgain else next end ensure Log.warn "Releases resources from failed job: #{job.path}" release_resources(job) end when job.done? Log.debug "Orchestrator done #{job.path}" release_resources(job) erase_job_dependencies(job, rules, all_jobs, top_level_jobs) when job.running? next else check_resources(rules, job) do run_with_rules(rules, job) end end end new_workload = {} workload.each do |k,v| next if k.done? new_workload[k] = v.reject{|d| d.done? || (d.error? && ! d.recoverable_error?)} end workload = new_workload sleep timer end rescue TryAgain retry end end |
#release_resources(job) ⇒ Object
110 111 112 113 114 115 116 117 118 119 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 110 def release_resources(job) if resources_used[job] Log.debug "Orchestrator releasing resouces from #{job.path}" resources_used[job].each do |resource,value| next if resource == 'size' resources_requested[resource] -= value.to_i end resources_used.delete job end end |
#run_with_rules(rules, job) ⇒ Object
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 |
# File 'lib/rbbt/workflow/util/orchestrator.rb', line 140 def run_with_rules(rules, job) job_rules = Orchestrator.job_rules(rules, job) Rbbt::Config.with_config do job_rules[:config_keys].each do |config| Rbbt::Config.process_config config end if job_rules && job_rules[:config_keys] log = job_rules[:log] if job_rules log = Log.severity if log.nil? Log.with_severity log do job.produce(false, true) end end end |