Top Level Namespace

Includes:
DefaultLogger

Defined Under Namespace

Modules: DefaultLogger, DefaultPreproc, DefaultRecon, DefaultStats, JohnsonMerit220Visit1Preproc, JohnsonMerit220Visit1Stats, JohnsonTbiLongitudinalSnodPreproc, JohnsonTbiLongitudinalSnodStats, ReconWithHello, Rpipe Classes: DriverConfigError, FlashFormatter, Float, JobGenerator, JobStep, Logfile, MatlabQueue, PreprocJobGenerator, Preprocessing, RPipe, ReconJobGenerator, Reconstruction, Stats, StatsJobGenerator, String, WorkflowGenerator

Constant Summary collapse

STEPS = 
%w(recon preproc stats)
VERSION_NUMBER = 
"0.0.0"
VERSION_LINE = 
"rpipe %s WADRC Imaging Core" % VERSION_NUMBER
<<-EOS
A utility for running neuroimaging rpipe jobs.

Usage:\trpipe [options] <driver(s)>

EOS

Instance Method Summary collapse

Methods included from DefaultLogger

#setup_logger

Instance Method Details

#create! ⇒ Object 



10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
 
# File 'bin/create_driver.rb', line 10

def create!
  # Parse CLI Options and Spec File
  cli_options = parse_options
  spec_options = cli_options[:spec_file] ? load_spec(cli_options[:spec_file]) : {}
  
  workflow_options_defaults = {}
  workflow_options = workflow_options_defaults.merge(spec_options).merge(cli_options[:config])

  # Create a Workflow Generator and use it to create configure job.
  while ARGV.size >= 1
    begin
      rawdir = ARGV.shift
      workflow = WorkflowGenerator.new(rawdir, workflow_options)
      if cli_options[:dry_run]
        pp workflow.build
      else
        begin
          puts write_driver_hash workflow.build, :cli_options => cli_options
        rescue IOError => e
          puts e
        end
      end
    rescue StandardError => e
      puts "Problem creating driver for #{rawdir}"
      puts e 
      puts e.backtrace if cli_options[:verbose]
    end
  end
end

#flash(msg) ⇒ Object

Global Method to display a message and the date/time to standard output.



22
23
24
 
# File 'lib/global_additions.rb', line 22

def flash(msg)
	$Log.info msg
end

#load_spec(spec_file) ⇒ Object 



56
57
58
59
60
61
62
63
64
 
# File 'bin/create_driver.rb', line 56

def load_spec(spec_file)
  if File.exist?(spec_file)
    spec = YAML::load_file(spec_file)
  else
    raise IOError, "Cannot find yaml spec file #{spec_file}"
  end
  
  return spec
end

#parse_options ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
 
# File 'bin/create_driver.rb', line 66

def parse_options
  options = { :config => {} }
  parser = OptionParser.new do |opts|
    opts.banner = "Usage: #{File.basename(__FILE__)} [options] RAWDIR(S)"

    opts.on('-s', '--spec SPEC', "Spec File for common study parameters")     do |spec_file| 
      options[:spec_file] = spec_file
    end
    
    opts.on('-r', '--responses_dir RESPONSES_DIR', "Directory containing Button-press Response Logfiles")     do |responses_dir| 
      options[:config][:responses_dir.to_s] = File.expand_path(responses_dir)
    end

    opts.on('-d', '--dry-run', "Display Driver without executing it.") do
      options[:dry_run] = true
    end
    
    opts.on('-f', '--force', "Overwrite drivers if they exist.") do
      options[:force] = true
    end
    
    opts.on('-v', '--verbose', "Print extra info.") do
      options[:verbose] = true
    end

    opts.on_tail('-h', '--help', "Show this message")  { puts(parser); exit }
    opts.on_tail("Example: #{File.basename(__FILE__)} mrt00001")
    opts.on_tail("Example: #{File.basename(__FILE__)} raw/johnson.merit220.visit1/mri/mrt*")
  end
  parser.parse!(ARGV)

  if ARGV.size == 0
    puts "Problem with arguments - Missing RAWDIR"
    puts(parser); exit
  end
  
  return options
end

#run(command) ⇒ Object

Global Method to Log and Run system commands.



10
11
12
13
14
15
16
17
18
19
 
# File 'lib/global_additions.rb', line 10

def run(command)
  $CommandLog.info command
  
  status = POpen4::popen4(command) do |stdout, stderr|
    puts stdout.read.strip
    $Log.error stderr.read.strip
  end
      
  status && status.exitstatus == 0 ? true : false
end

#run! ⇒ Object

Main Function for the CLI runner.



27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
 
# File 'bin/rpipe', line 27

def run!
  options = setup_options
  drivers = ARGV
  while drivers.size >= 1    
    driver = drivers.shift
    run_with_logging File.basename(driver, File.extname(driver)), options do  
      begin
        $Log.info "Begin Pipelining #{driver} | #{Etc.getlogin} on #{`hostname`}"
        run_pipe(options, driver)
        $Log.info "Finished Pipelining #{driver}"
      rescue StandardError => e
        $Log.error e
      end
    end
  end
end

#run_pipe(options, driver) ⇒ Object

Perform Each of the Jobs in a Pipe



45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'bin/rpipe', line 45

def run_pipe(options, driver)

  pipe = RPipe.new(driver)
  pp pipe if options[:debug]

  if options[:only_given]
  	case options[:only]
  	when "recon"
  		pipe.recon_jobs.each { |job| job.perform }
  	when "preproc"
  		pipe.preproc_jobs.each { |job| job.perform }
  	when "stats"
  		pipe.stats_jobs.each { |job| job.perform }
  	end
  else
  	pipe.recon_jobs.each { |job| job.perform }
  	pipe.preproc_jobs.each { |job| job.perform }
  	pipe.stats_jobs.each { |job| job.perform }
  end
end

#run_with_logging(logfile_stem, options, &block) ⇒ Object

Setup Tee IO’s for Out and Error and start logs for them.



90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
 
# File 'bin/rpipe', line 90

def run_with_logging(logfile_stem, options, &block)
  if options[:log_directory]
    log_dir = options[:log_directory]
  else 
    # If no explicit directory has been passed as an argument, check to see if
    # a directory named "logs" exists.  If so, use that, otherwise use the current directory.
    log_dir = File.directory?('logs') ? 'logs' : '.'
  end
  out_logfile     = File.join(log_dir, logfile_stem + '.out')
  error_logfile   = File.join(log_dir, logfile_stem + '.err')
  command_logfile = File.join(log_dir, logfile_stem + '.sh')
  
  teeout = Tee.new(out_logfile, :out, 'a')
  teeerr = Tee.new(error_logfile, :err, 'a')
  
  setup_runner_logger(command_logfile, teeout, teeerr, options)
  
  begin
    yield
  rescue Exception => e
    $ErrorLog.error e
    raise e
  ensure
    # Discard the error log if there were no errors.
    # Size returns nil for an empty file.
    teeerr.close
    File.delete(error_logfile) unless File.size?(error_logfile)
  end
end

#setup_options ⇒ Object

Setup Trollop Options



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'bin/rpipe', line 67

def setup_options
  opts = Trollop::options do
  	version VERSION_LINE
  	banner BANNER
  	opt :only, "Perform only a certain step (recon, preproc, stats)", :type => String
  	opt :debug, "Be more wordy than usual for debugging"
  	opt :log_directory, "Output Directory for processing logs.", :type => String
  end

  if opts[:only_given]
  	unless STEPS.include?(opts[:only])
  		Trollop::die :only, "must be one of recon, preproc, or stats"
  	end
  end

  Trollop::die "Driver file not given" if (ARGV.size < 1)

  pp opts if opts[:debug]
  
  return opts
end

#setup_runner_logger(command_logfile, teeout, teeerr, options) ⇒ Object

Log Commands to a file and Output to stdout



121
122
123
124
125
126
127
128
129
130
131
132
133
 
# File 'bin/rpipe', line 121

def setup_runner_logger(command_logfile, teeout, teeerr, options)
  console_pattern = "#{'*' * 10} %m [ %d ]"
  $Log = Log4r::Logger.new('output')
  $Log.level = Log4r::DEBUG
  $Log.add Log4r::IOOutputter.new(:stdout, teeout, :formatter => FlashFormatter.new)

  $CommandLog = Log4r::Logger.new('command::output')
  $CommandLog.add Log4r::FileOutputter.new(:file, :filename => command_logfile, :trunc => false, :formatter => Log4r::PatternFormatter.new(:pattern => "%m"))
  $CommandLog.add Log4r::IOOutputter.new(:stdout, teeout, :formatter => FlashFormatter.new)
  
  $ErrorLog = Log4r::Logger.new('error::output')
  $ErrorLog.add Log4r::IOOutputter.new(:stderr, teeerr, :formatter => Log4r::PatternFormatter.new(:pattern => "%m"))
end

#summarize! ⇒ Object 



7
8
9
10
11
12
13
14
15
16
 
# File 'bin/summarize_responses.rb', line 7

def summarize!
  # Parse CLI Options and Spec File
  cli_options = parse_options  
  begin
    # Summarize a Directory of Logfiles
    Logfile.write_summary(cli_options[:filename], cli_options[:directory], cli_options[:grouping])
  rescue IOError => e
    puts e
  end
end

#write_driver_hash(workflow_spec, driver_options = {:filename => nil}) ⇒ Object 



40
41
42
43
44
45
46
47
48
 
# File 'bin/create_driver.rb', line 40

def write_driver_hash (workflow_spec, driver_options = {:filename => nil})
  filename = driver_options[:filename] ||= workflow_spec['subid'] + '.yaml'
  
  if File.exist? filename and ! driver_options[:cli_options][:force]
    raise IOError, "Driver #{filename} already exists; use -f option to force overwrite." 
  else
    write_file(workflow_spec, filename)
  end
end

#write_file(workflow_spec, filename) ⇒ Object

Raises:

  • (IOError) 


50
51
52
53
54
 
# File 'bin/create_driver.rb', line 50

def write_file(workflow_spec, filename)
  File.open(filename, 'w') { |f| f.puts workflow_spec.to_yaml }
  raise IOError, "Couldn't write #{filename}}" unless File.exist?(filename)
  "Wrote file #{filename}..."
end