Class: RQ::Feeder
- Inherits:
-
MainHelper
- Object
- MainHelper
- RQ::Feeder
- Defined in:
- lib/rq-0.1.7/feeder.rb
Constant Summary collapse
- DEFAULT_MIN_SLEEP =
{{{
42
- DEFAULT_MAX_SLEEP =
240
- DEFAULT_FEED =
2
Constants included from Logging
Logging::DIV0, Logging::DIV1, Logging::DIV2, Logging::DIV3, Logging::EOL, Logging::SEC0, Logging::SEC1, Logging::SEC2, Logging::SEC3
Class Attribute Summary collapse
-
.feed ⇒ Object
Returns the value of attribute feed.
-
.max_sleep ⇒ Object
Returns the value of attribute max_sleep.
-
.min_sleep ⇒ Object
{{{.
Attributes inherited from MainHelper
#argv, #cmd, #env, #logger, #main, #mode, #options, #q, #qpath
Instance Method Summary collapse
-
#busy? ⇒ Boolean
}}}.
-
#daemon ⇒ Object
}}}.
- #feed ⇒ Object
-
#fill_morgue ⇒ Object
}}}.
-
#finish_job(job, status) ⇒ Object
}}}.
-
#gen_feeder_name(path) ⇒ Object
}}}.
-
#gen_pidfile(name = nil) ⇒ Object
}}}.
-
#handle_signal ⇒ Object
}}}.
-
#install_signal_handlers ⇒ Object
}}}.
-
#nothing_running? ⇒ Boolean
}}}.
-
#reap_jobs(reap_only = false, blocking = true) ⇒ Object
}}}.
-
#relax ⇒ Object
}}}.
-
#start_job(job) ⇒ Object
}}}.
-
#start_jobs ⇒ Object
}}}.
-
#throttle(rate = @min_sleep) ⇒ Object
}}}.
-
#transaction ⇒ Object
}}}.
Methods inherited from MainHelper
Methods included from Logging
Methods included from Logging::LogMethods
#debug, #error, #fatal, #info, #logerr, #logger, #logger=, #warn
Methods included from Util
#alive?, append_features, #btrace, #columnize, #defval, #emsg, #erreq, #errmsg, #escape, #escape!, #exec, export, #fork, #getopt, #hashify, #host, #hostname, #klass, #maim, #mcp, #realpath, #stamptime, #system, #timestamp, #tmpnam, #uncache
Constructor Details
This class inherits a constructor from RQ::MainHelper
Class Attribute Details
.feed ⇒ Object
Returns the value of attribute feed.
23 24 25 |
# File 'lib/rq-0.1.7/feeder.rb', line 23 def feed @feed end |
.max_sleep ⇒ Object
Returns the value of attribute max_sleep.
22 23 24 |
# File 'lib/rq-0.1.7/feeder.rb', line 22 def max_sleep @max_sleep end |
.min_sleep ⇒ Object
{{{
21 22 23 |
# File 'lib/rq-0.1.7/feeder.rb', line 21 def min_sleep @min_sleep end |
Instance Method Details
#busy? ⇒ Boolean
}}}
350 351 352 353 354 |
# File 'lib/rq-0.1.7/feeder.rb', line 350 def busy? #{{{ @children.size >= @max_feed #}}} end |
#daemon ⇒ Object
}}}
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
# File 'lib/rq-0.1.7/feeder.rb', line 73 def daemon #{{{ if @options['daemon'] fork do Process::setsid fork do Dir::chdir(Util.realpath('~')) File::umask 0 open('/dev/null','r+') do |f| STDIN.reopen f STDOUT.reopen f STDERR.reopen f end @daemon = true yield exit EXIT_SUCCESS end exit! end exit! else @daemon = false yield exit EXIT_SUCCESS end #}}} end |
#feed ⇒ Object
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 |
# File 'lib/rq-0.1.7/feeder.rb', line 27 def feed #{{{ daemon do gen_pidfile @main.init_logging @logger = @main.logger set_q @pid = Process::pid @cmd = @main.cmd @started = Util:: @min_sleep = Integer(@options['min_sleep'] || defval('min_sleep')) @max_sleep = Integer(@options['max_sleep'] || defval('max_sleep')) @max_feed = Integer(@options['feed'] || defval('feed')) @children = Hash::new @jrd = JobRunnerDaemon::daemon install_signal_handlers info{ "** STARTED **" } info{ "version <#{ RQ::VERSION }>" } info{ "cmd <#{ @cmd }>" } info{ "pid <#{ @pid }>" } info{ "pidfile <#{ @pidfile.path }>" } info{ "jobrunnerdaemon uri <#{ @jrd.uri }> pid <#{ @jrd.pid }>" } info{ "qpath <#{ @qpath }>" } info{ "mode <#{ @mode }>" } debug{ "min_sleep <#{ @min_sleep }>" } debug{ "max_sleep <#{ @max_sleep }>" } fill_morgue loop do handle_signal if $rq_signaled throttle(@min_sleep) do start_jobs unless busy? if nothing_running? relax else reap_jobs end end end end #}}} end |
#fill_morgue ⇒ Object
}}}
158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 |
# File 'lib/rq-0.1.7/feeder.rb', line 158 def fill_morgue #{{{ debug{ "filling morgue" } sql = <<-sql select * from jobs where state = 'running' and runner = '#{ Util::hostname }' and started <= '#{ @started }' sql db = @q.qdb transaction do tuples = db.execute sql tuples.each do |tuple| jid = tuple['jid'] if jid sql = "update jobs set state='dead' where jid='#{ jid }'" db.execute sql info{ "burried job <#{ jid }>" } end end end debug{ "filled morgue" } #}}} end |
#finish_job(job, status) ⇒ Object
}}}
320 321 322 323 324 325 326 327 328 329 330 331 332 333 |
# File 'lib/rq-0.1.7/feeder.rb', line 320 def finish_job job, status #{{{ job['finished'] = Util::(Time::now) job['elapsed'] = Util::stamptime(job['finished']) - Util::stamptime(job['started']) t = status.exitstatus rescue nil job['exit_status'] = t job['state'] = 'finished' if t and t == 0 info{ "finished - jid <#{ job['jid'] }> pid <#{ job['pid'] }> exit_status <#{ job['exit_status'] }>" } else warn{ "finished - jid <#{ job['jid'] }> pid <#{ job['pid'] }> exit_status <#{ job['exit_status'] }>" } end #}}} end |
#gen_feeder_name(path) ⇒ Object
}}}
126 127 128 129 130 131 |
# File 'lib/rq-0.1.7/feeder.rb', line 126 def gen_feeder_name path #{{{ path = Util::realpath(path).gsub(%r|/|o, '_') File::join(Util::realpath('~'), ".#{ path }.feeder") #}}} end |
#gen_pidfile(name = nil) ⇒ Object
}}}
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 |
# File 'lib/rq-0.1.7/feeder.rb', line 100 def gen_pidfile name = nil #{{{ name ||= gen_feeder_name(@options['name'] || @qpath) @pidfile = begin open name, File::CREAT | File::EXCL | File::RDWR rescue open name, File::RDWR end unless @pidfile and @pidfile.posixlock(File::LOCK_EX | File::LOCK_NB) pid = IO::read(name) rescue nil pid ||= 'unknown' if @options['quiet'] exit EXIT_FAILURE else raise "process <#{ pid }> is already feeding from this queue" end else @pidfile.rewind @pidfile.sync = true @pidfile.print Process::pid @pidfile.truncate @pidfile.pos at_exit{ FileUtils::rm_f name rescue nil } end #}}} end |
#handle_signal ⇒ Object
}}}
179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 |
# File 'lib/rq-0.1.7/feeder.rb', line 179 def handle_signal #{{{ if $rq_sigterm or $rq_sigint reap_jobs(reap_only = true) until nothing_running? info{ "** STOPPING **" } @jrd.shutdown rescue nil @pidfile.posixlock File::LOCK_UN exit EXIT_SUCCESS end if $rq_sighup reap_jobs(reap_only = true) until nothing_running? info{ "** RESTARTING **" } @jrd.shutdown rescue nil Util::uncache __FILE__ @pidfile.posixlock File::LOCK_UN Util::exec @cmd end #}}} end |
#install_signal_handlers ⇒ Object
}}}
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 |
# File 'lib/rq-0.1.7/feeder.rb', line 132 def install_signal_handlers #{{{ if @daemon $rq_signaled = false $rq_sighup = false $rq_sigterm = false $rq_sigint = false trap('SIGHUP') do $rq_signaled = $rq_sighup = 'SIGHUP' warn{ "signal <SIGHUP>" } warn{ "finishing running jobs before handling signal" } end trap('SIGTERM') do $rq_signaled = $rq_sigterm = 'SIGTERM' warn{ "signal <SIGTERM>" } warn{ "finishing running jobs before handling signal" } end trap('SIGINT') do $rq_signaled = $rq_sigint = 'SIGINT' warn{ "signal <SIGINT>" } warn{ "finishing running jobs before handling signal" } end @jrd.install_signal_handlers end #}}} end |
#nothing_running? ⇒ Boolean
}}}
257 258 259 260 261 |
# File 'lib/rq-0.1.7/feeder.rb', line 257 def nothing_running? #{{{ @children.size == 0 #}}} end |
#reap_jobs(reap_only = false, blocking = true) ⇒ Object
}}}
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 |
# File 'lib/rq-0.1.7/feeder.rb', line 262 def reap_jobs reap_only = false, blocking = true #{{{ reaped = [] cid = status = nil if blocking if busy? or reap_only cid, status = @jrd.waitpid2 -1, Process::WUNTRACED else loop do debug{ "not busy wait loop" } cid, status = @jrd.waitpid2 -1, Process::WNOHANG | Process::WUNTRACED break if cid start_jobs unless $rq_signaled break if busy? cid, status = @jrd.waitpid2 -1, Process::WNOHANG | Process::WUNTRACED break if cid sleep 4.2 end cid, status = @jrd.waitpid2 -1, Process::WUNTRACED unless cid end else cid, status = @jrd.waitpid2 -1, Process::WNOHANG | Process::WUNTRACED end if cid and status job = @children[cid] finish_job job, status transaction do loopno = 0 loop do @q.jobisdone job @children.delete cid reaped << cid start_jobs unless reap_only or $rq_signaled if @children.size == 0 or loopno > 42 break else sleep 0.1 cid, status = @jrd.waitpid2 -1, Process::WNOHANG | Process::WUNTRACED break unless cid and status job = @children[cid] finish_job job, status end loopno += 1 end end end debug{ "<#{ reaped.size }> jobs reaped" } reaped #}}} end |
#relax ⇒ Object
}}}
355 356 357 358 359 360 361 |
# File 'lib/rq-0.1.7/feeder.rb', line 355 def relax #{{{ seconds = rand(@max_sleep - @min_sleep + 1) + @min_sleep debug{ "relaxing <#{ seconds }>" } sleep seconds #}}} end |
#start_job(job) ⇒ Object
}}}
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 |
# File 'lib/rq-0.1.7/feeder.rb', line 230 def start_job job #{{{ jid, command = job['jid'], job['command'] # # we setup state slightly prematurely so jobrunner will have it availible # job['state'] = 'running' job['started'] = Util:: Time::now job['runner'] = Util::hostname jr = @jrd.runner job cid = jr.cid if jr and cid jr.run job['pid'] = cid @children[cid] = job @q.jobisrunning job info{ "started - jid <#{ job['jid'] }> pid <#{ job['pid'] }> command <#{ job['command'] }>" } else error{ "not started - jid <#{ job['jid'] }> command <#{ job['command'] }>" } end cid #}}} end |
#start_jobs ⇒ Object
}}}
216 217 218 219 220 221 222 223 224 225 226 227 228 229 |
# File 'lib/rq-0.1.7/feeder.rb', line 216 def start_jobs #{{{ n_started = 0 transaction do until busy? break unless((job = @q.getjob)) start_job job n_started += 1 end end debug{ "<#{ n_started }> jobs started" } n_started #}}} end |
#throttle(rate = @min_sleep) ⇒ Object
}}}
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 |
# File 'lib/rq-0.1.7/feeder.rb', line 199 def throttle rate = @min_sleep #{{{ if Numeric === rate and rate > 0 if defined? @last_throttle_time and @last_throttle_time elapsed = Time.now - @last_throttle_time timeout = rate - elapsed if timeout > 0 timeout = timeout + rand(rate * 0.10) debug{ "throttle rate of <#{ rate }> exceeded - sleeping <#{ timeout }>" } sleep timeout end end @last_throttle_time = Time.now end yield #}}} end |
#transaction ⇒ Object
}}}
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 |
# File 'lib/rq-0.1.7/feeder.rb', line 334 def transaction #{{{ ret = nil if @in_transaction ret = yield else begin @in_transaction = true @q.transaction{ ret = yield } ensure @in_transaction = false end end ret #}}} end |