Class: SSHotgun

Inherits:
Object
  • Object
show all
Defined in:
lib/sshotgun.rb

Overview

This is the main class of SSHotgun. This class stores the configuration and launches the processing threads.

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(hostlist, processingclass) ⇒ SSHotgun

Returns a new instance of SSHotgun.



344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
 
# File 'lib/sshotgun.rb', line 344

def initialize(hostlist, processingclass)
  @versionStr = "1.0.5"
  @hostlist = hostlist
  @processingclass = processingclass
  @logmutex = Mutex.new
  @threadStartMutex = Mutex.new
  @threadEndMutex = Mutex.new
  @maxThreads = 30  # allow NN processing threads to run at any time
  @threadTimeoutInSeconds = 1800  # timeout after 30 minutes
  @monitorPollIntervalInSeconds = 30 # period for displaying thread monitor
  timestr = Time.now.strftime("%Y%m%d%H%M%S")
  logfilename = File.basename($0) + ".#{timestr}.log"
  @filelogger = Logger.new(logfilename)
  @isFileLogging = true
  @isDebug = false
  @isStopOnNonZeroExit = false
  @exitCode = 0
end

Instance Attribute Details

#exitCode ⇒ Object

Exit code for this script, default is zero. Settable by the script programmer.



342
343
344
 
# File 'lib/sshotgun.rb', line 342

def exitCode
  @exitCode
end

#filelogger ⇒ Object

File logger. Set your own if desired. (optional)



320
321
322
 
# File 'lib/sshotgun.rb', line 320

def filelogger
  @filelogger
end

#gatewayhost ⇒ Object

The gateway host for use as an ssh relay. (optional)



297
298
299
 
# File 'lib/sshotgun.rb', line 297

def gatewayhost
  @gatewayhost
end

#hostlist ⇒ Object

Array containing all of the HostDefinitions (hosts to process)



288
289
290
 
# File 'lib/sshotgun.rb', line 288

def hostlist
  @hostlist
end

#isDebug ⇒ Object

Turn on/off debugging. Default is off. (optional)



326
327
328
 
# File 'lib/sshotgun.rb', line 326

def isDebug
  @isDebug
end

#isFileLogging ⇒ Object

Turn on/off file logging. Default is on. (optional)



323
324
325
 
# File 'lib/sshotgun.rb', line 323

def isFileLogging
  @isFileLogging
end

#isStopOnNonZeroExit ⇒ Object

Turn on/off behavior where processing for a server is stopped if any command (run, runSudo, etc) returns with a non-zero exit code. Default is off (don’t stop). Note that if isStopOnNonZeroExit is true then the processing thread exits immediately, you never have the opportunity to check the exit status in your processing code.



339
340
341
 
# File 'lib/sshotgun.rb', line 339

def isStopOnNonZeroExit
  @isStopOnNonZeroExit
end

#logmutex ⇒ Object (readonly)

For internal use. Mutex for writing to the log



311
312
313
 
# File 'lib/sshotgun.rb', line 311

def logmutex
  @logmutex
end

#maxThreads ⇒ Object

The maximum number of processing threads to run simultaneously. Each processing thread roughly equates to one ssh client running on your local machine. Default is 30. (optional)



302
303
304
 
# File 'lib/sshotgun.rb', line 302

def maxThreads
  @maxThreads
end

#monitorPollIntervalInSeconds ⇒ Object

Interval time to periodically display thread status information. Default is 30 sec. (optional)



308
309
310
 
# File 'lib/sshotgun.rb', line 308

def monitorPollIntervalInSeconds
  @monitorPollIntervalInSeconds
end

#processingclass ⇒ Object

Your custom processing class



291
292
293
 
# File 'lib/sshotgun.rb', line 291

def processingclass
  @processingclass
end

#startTime ⇒ Object (readonly)

For internal use. Script start time



329
330
331
 
# File 'lib/sshotgun.rb', line 329

def startTime
  @startTime
end

#sudopassword ⇒ Object

The sudo password, if needed. (optional)



294
295
296
 
# File 'lib/sshotgun.rb', line 294

def sudopassword
  @sudopassword
end

#threadEndMutex ⇒ Object (readonly)

For internal use. Mutex for exiting



317
318
319
 
# File 'lib/sshotgun.rb', line 317

def threadEndMutex
  @threadEndMutex
end

#threadStartMutex ⇒ Object (readonly)

For internal use. Mutex for launching processing threads



314
315
316
 
# File 'lib/sshotgun.rb', line 314

def threadStartMutex
  @threadStartMutex
end

#threadTimeoutInSeconds ⇒ Object

Timeout for processing threads. Default is 30 minutes. (optional)



305
306
307
 
# File 'lib/sshotgun.rb', line 305

def threadTimeoutInSeconds
  @threadTimeoutInSeconds
end

#versionStr ⇒ Object (readonly)

For internal use. sshotgun version



332
333
334
 
# File 'lib/sshotgun.rb', line 332

def versionStr
  @versionStr
end

Instance Method Details

#log(s) ⇒ Object

Log a string to the console and to the file logger

s is the string to log



510
511
512
513
514
515
516
517
 
# File 'lib/sshotgun.rb', line 510

def log(s)
  @logmutex.synchronize {
    puts s.chomp
    if @isFileLogging
      @filelogger.info s.chomp
    end
  }
end

#showRunStatsAndExit ⇒ Object

Display status and exit. May be called by pressing ctrl-c, fatal error or normal termination



489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
 
# File 'lib/sshotgun.rb', line 489

def showRunStatsAndExit
  @threadEndMutex.synchronize {
    # Display script ending message
    stopTime = Time.now
    log "[_sshotgun_] info: Ended script " + $0 + " at " + stopTime.to_s
    log "[_sshotgun_] info: Elapsed script " + $0 + " time " + (stopTime - @startTime).to_s + "s"
    log "[_sshotgun_] info: Script exit code is = " + @exitCode.to_s

    # Display status of each host
    @hostlist.each do |hostdef| 
      log "[_sshotgun_] status: " + hostdef.hostname + " = " + hostdef.status
    end
    exit @exitCode
  }
end

#start ⇒ Object

Start the processing. In most cases, this call will be the last line in your SSHotgun processing script.



366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
 
# File 'lib/sshotgun.rb', line 366

def start
  # trap ctrl-c, show stats and exit
  trap("INT") {
    log "[_sshotgun_] info: Script terminated early by ctrl-c"
    showRunStatsAndExit
  }

  @startTime = Time.now
  log "[_sshotgun_] info: SSHotgun version = " + @versionStr
  log "[_sshotgun_] info: Started script " + $0 + " at " + @startTime.to_s

  # Display configuration info
  log "[_sshotgun_] info: User=" + ENV["USER"].to_s
  log "[_sshotgun_] info: Gateway host=" + @gatewayhost.to_s
  log "[_sshotgun_] info: Thread timeout=" + @threadTimeoutInSeconds.to_s + "s"
  log "[_sshotgun_] info: Max concurrent threads=" + @maxThreads.to_s
  log "[_sshotgun_] info: Monitor poll interval=" + @monitorPollIntervalInSeconds.to_s + "s"
  log "[_sshotgun_] info: File logging is=" + @isFileLogging.to_s
  log "[_sshotgun_] info: Will stop thread upon non-zero exit=" + @isStopOnNonZeroExit.to_s

  hostarr = []
  @hostlist.each do |hostdef| 
    hostarr << hostdef.hostname
  end
  log "[_sshotgun_] info: Host list=" + hostarr.join(", ")

  currentHostIndex = 0
  processingThreads = []
  checkRunningThreadsCounter = 0
  while true

    # how many active threads now?
    numThreadsAlive = 0
    processingThreads.each { |aThread|  
      if aThread.alive?
        numThreadsAlive += 1
      end
    }

    # calc number of threads to launch with this 
    # pass through the loop
    numThreadsToLaunch = @maxThreads - numThreadsAlive 
    numThreadsLeftToLaunch = @hostlist.length - currentHostIndex
    if numThreadsToLaunch > numThreadsLeftToLaunch
      numThreadsToLaunch = numThreadsLeftToLaunch
    end

    # from current index in hosts array loop and launch threads up to maxThreads running
    i = 0
    while i < numThreadsToLaunch do
      # I hate to admit it but I don't know why i needed to synchronize this block
      # Without it, some threads seemed to freeze at startup
      @threadStartMutex.synchronize {
        hostdef = @hostlist[currentHostIndex]
        processingThreads << Thread.new {
          aThread = Thread.current
          aThread["threadstarttime"] = Time.now  # store start time in threadlocal variable
          aThread["hostname"] = hostdef.hostname  # store hostname time in threadlocal variable
          log "[_sshotgun_] info: Started thread for host = " + hostdef.hostname + " at " + aThread["threadstarttime"].to_s
          begin 
            o = @processingclass.new(self, hostdef)
            o.doProcessing
          rescue => ex
            log "[_sshotgun_] stderr: Thread blew up for host = " + hostdef.hostname + ". ex = " + ex.inspect + "\n" + ex.backtrace.join("\n")
          end
        }

        # bump loop counter
        i += 1
        currentHostIndex += 1
      }
    end

    # kill expired threads
    processingThreads.each { |aThread|  
      if aThread.alive?
        # ensure that the threadlocal var is actually set before
        # doing a calc with it. This is race condition where you
        # get here so fast and threadstarttime is nil
        if aThread["threadstarttime"]
          if Time.now - aThread["threadstarttime"] > @threadTimeoutInSeconds
            log "[_sshotgun_] info: Killed thread for host = " + aThread["hostname"]
            aThread.kill
          end
        end
      end
    }

    # display list of running threads as visual indicator to user
    checkRunningThreadsCounter += 1
    if checkRunningThreadsCounter > @monitorPollIntervalInSeconds
      # display running threads
      processingThreads.each { |aThread|  
        if aThread.alive?
          log "[_sshotgun_] monitor: Thread still running for host = " + aThread["hostname"] + ". Elapsed time is " + (Time.now - aThread["threadstarttime"] ).to_s + "s"
        end
      }
      checkRunningThreadsCounter = 0
    end

    # any processing threads still alive?
    aThreadIsAlive = false
    processingThreads.each { |aThread|  
      if aThread.alive?
        aThreadIsAlive = true
        break
      end
    }

    # break out of this main loop if no processing threads are alive
    if aThreadIsAlive
      sleep 1 
    else
      break
    end
  end

  # ended normally, show stats
  log "[_sshotgun_] info: Script terminated normally"
  showRunStatsAndExit
end