Module: Resource

Extended by:
MetaExtension
Included in:
Scout
Defined in:
lib/scout/resource.rb,
lib/scout/resource/util.rb,
lib/scout/resource/produce.rb,
lib/scout/resource/software.rb

Class Method Summary collapse

Instance Method Summary collapse

Methods included from MetaExtension

extended, is_extended?, purge

Dynamic Method Handling

This class handles dynamic methods through the method_missing method

#method_missing(name, prev = nil, *args) ⇒ Object 



33
34
35
36
37
38
39
 
# File 'lib/scout/resource.rb', line 33

def method_missing(name, prev = nil, *args)
  if prev.nil?
    root.send(name, *args)
  else
    root.send(name, prev, *args)
  end
end

Class Method Details

.default_lock_dir ⇒ Object 



13
14
15
 
# File 'lib/scout/resource.rb', line 13

def self.default_lock_dir
  Path.setup('tmp/produce_locks').find
end

.identify(path) ⇒ Object 



47
48
49
50
51
 
# File 'lib/scout/resource/util.rb', line 47

def self.identify(path)
  resource = path.pkgdir if Path === path
  resource = Scout unless Resource === resource
  unlocated = resource.identify path
end

.install(content, name, software_dir = Path.setup('software'), &block) ⇒ Object 



7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
 
# File 'lib/scout/resource/software.rb', line 7

def self.install(content, name, software_dir = Path.setup('software'), &block)
  software_dir ||= Path.setup('software')
  software_dir = software_dir.find(:user) if Path === software_dir

  content = block if block_given?

  preamble = <<-EOF
#!/bin/bash

SOFTWARE_DIR="#{software_dir}"

INSTALL_HELPER_FILE="#{install_helpers}"
source "$INSTALL_HELPER_FILE"
  EOF

  content = content.call if Proc === content

  name = content[:name] if Hash === content && content.include?(:name)
  content = 
    if content =~ /git:|\.git$/
      {:git => content}
    else
      {:src => content}
    end if String === content and Open.remote?(content)

    script_text = 
      case content
      when nil
        raise "No way to install #{name}"
      when Path
        Open.read(content) 
      when String
        if Path.is_filename?(content) and Open.exists?(content)
          Open.read(content) 
        else
          content
        end
      when Hash
        name = content[:name] || name
        git = content[:git]
        src = content[:src]
        url = content[:url]
        jar = content[:jar]
        extra = content[:extra]
        commands = content[:commands]
        if git
          <<-EOF

name='#{name}'
url='#{git}'

install_git "$name" "$url" #{extra}

#{commands}
          EOF
        elsif src
          <<-EOF

name='#{name}'
url='#{src}'

install_src "$name" "$url" #{extra}

#{commands}
          EOF
        elsif jar
          <<-EOF

name='#{name}'
url='#{jar}'

install_jar "$name" "$url" #{extra}

#{commands}
          EOF
        else
          <<-EOF

name='#{name}'
url='#{url}'

#{commands}
          EOF
        end
      end

    script = preamble + "\n" + script_text
    Log.debug "Installing software #{name} into #{software_dir} with script:\n" << script
    CMD.cmd_log('bash', :in => script)
    Resource.set_software_env(software_dir)
end

.install_helpers ⇒ Object 



3
4
5
 
# File 'lib/scout/resource/software.rb', line 3

def self.install_helpers
  File.expand_path(Scout.share.software.install_helpers.find(:lib))
end

.relocate(path) ⇒ Object 



53
54
55
56
57
 
# File 'lib/scout/resource/util.rb', line 53

def self.relocate(path)
  return path if Open.exists?(path)
  unlocated = identify(path)
  unlocated.find
end

.set_software_env(software_dir = Path.setup('software')) ⇒ Object 



99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
 
# File 'lib/scout/resource/software.rb', line 99

def self.set_software_env(software_dir = Path.setup('software'))
  software_dir.opt.find_all.collect{|d| d.annotate(File.dirname(d)) }.reverse.each do |software_dir|
    next unless software_dir.exists?
    Log.medium "Preparing software env at #{software_dir}"

    software_dir = File.expand_path(software_dir)
    opt_dir = File.join(software_dir, 'opt')
    bin_dir = File.join(opt_dir, 'bin')

    Misc.env_add 'PATH', bin_dir

    FileUtils.mkdir_p opt_dir unless File.exist? opt_dir

    %w(.ld-paths .c-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
      filename = File.join(opt_dir, file)
      begin
        FileUtils.touch filename unless File.exist? filename
      rescue
        Log.warn("Could not touch #{ filename }")
      end
    end

    Open.read(File.join opt_dir, '.c-paths').split(/\n/).each do |line|
      dir = line.chomp
      dir = File.join(opt_dir, dir) unless dir[0] == "/"
      Misc.env_add('CPLUS_INCLUDE_PATH',dir)
      Misc.env_add('C_INCLUDE_PATH',dir)
    end if File.exist? File.join(opt_dir, '.c-paths')

    Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
      dir = line.chomp
      dir = File.join(opt_dir, dir) unless dir[0] == "/"
      Misc.env_add('LIBRARY_PATH',dir)
      Misc.env_add('LD_LIBRARY_PATH',dir)
      Misc.env_add('LD_RUN_PATH',dir)
    end if File.exist? File.join(opt_dir, '.ld-paths')

    Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
      dir = line.chomp
      dir = File.join(opt_dir, dir) unless dir[0] == "/"
      Misc.env_add('PKG_CONFIG_PATH',dir)
    end if File.exist? File.join(opt_dir, '.pkgconfig-paths')

    Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
      dir = line.chomp
      dir = File.join(opt_dir, dir) unless dir[0] == "/"
      Misc.env_add('ACLOCAL_FLAGS', "-I #{dir}", ' ')
    end if File.exist? File.join(opt_dir, '.aclocal-paths')

    Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
      dir = line.chomp
      dir = File.join(opt_dir, dir) unless dir[0] == "/"
      Misc.env_add('CLASSPATH', "#{dir}")
    end if File.exist? File.join(opt_dir, '.java-classpaths')

    Dir.glob(File.join opt_dir, 'jars', '*.jar').each do |file|
      Misc.env_add('CLASSPATH', "#{file}")
    end

    if File.exist?(File.join(opt_dir, '.post_install')) and File.directory?(File.join(opt_dir, '.post_install'))
      Dir.glob(File.join(opt_dir, '.post_install','*')).each do |file|

        # Load exports
        Open.read(file).split("\n").each do |line|
          next unless line =~ /^\s*export\s+([^=]+)=(.*)/
          var = $1.strip
          value = $2.strip
          value.sub!(/^['"]/,'')
          value.sub!(/['"]$/,'')
          value.gsub!(/\$[a-z_0-9]+/i){|var| ENV[var[1..-1]] }
          Log.debug "Set variable export from .post_install: #{Log.fingerprint [var,value]*"="}"
          ENV[var] = value
        end
      end
    end
  end
end

Instance Method Details

#claim(path, type, content = nil, &block) ⇒ Object 



6
7
8
9
10
11
12
13
14
 
# File 'lib/scout/resource/produce.rb', line 6

def claim(path, type, content = nil, &block)
  if type == :rake
    @rake_dirs ||= {}
    @rake_dirs[path] = content || block
  else
    @resources ||= {}
    @resources[path] = [type, content || block]
  end
end

#has_rake(path) ⇒ Object 



25
26
27
 
# File 'lib/scout/resource/produce.rb', line 25

def has_rake(path)
  !! rake_for(path)
end

#identify(path) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/scout/resource/util.rb', line 2

def identify(path)
  return path unless path.start_with?("/")
  path_maps = path.path_maps if Path === path
  path_maps ||= self.path_maps || Path.path_maps
  path = File.expand_path(path)
  path += "/" if File.directory?(path)

  map_order ||= (path_maps.keys & Path.basic_map_order) + (path_maps.keys - Path.basic_map_order)
  map_order -= [:current, "current"]

  choices = []
  map_order.uniq.each do |name|
    pattern = path_maps[name]
    pattern = path_maps[pattern] while Symbol === pattern
    next if pattern.nil?

    pattern = pattern.sub('{PWD}', Dir.pwd)
    pattern = pattern.sub('{HOME}', ENV["HOME"])
    if String ===  pattern and pattern.include?('{')
      regexp = "^" + pattern
        .gsub(/{(TOPLEVEL)}/,'(?<\1>[^/]+)')
        .gsub(/{([^}]+)}/,'(?<\1>[^/]+)?') +
      "(?:/(?<REST>.*))?/?$"
      if m = path.match(regexp) 
        if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == self.pkgdir
          unlocated = %w(TOPLEVEL SUBPATH PATH REST).collect{|c| 
            m.named_captures.include?(c) ? m[c] : nil
          }.compact * "/"
          unlocated.gsub!(/\/+/,'/')
          if self.subdir && ! self.subdir.empty?
            subdir = self.subdir
            subdir += "/" unless subdir.end_with?("/")
            unlocated[subdir] = "" 
          end
          choices << self.annotate(unlocated)
        end
      end
    end
  end

  identified = choices.sort_by{|s| s.length }.first

  Path.setup(identified || path, self, nil, path_maps)
end

#lock_dir ⇒ Object 



21
22
23
 
# File 'lib/scout/resource.rb', line 21

def lock_dir
  @lock_dir ||= Resource.default_lock_dir
end

#pkgdir ⇒ Object 



25
26
27
 
# File 'lib/scout/resource.rb', line 25

def pkgdir
  @pkgdir ||= Path.default_pkgdir
end

#produce(path, force = false) ⇒ Object 



52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
 
# File 'lib/scout/resource/produce.rb', line 52

def produce(path, force = false)
  case
  when (@resources && @resources.include?(path))
    type, content = @resources[path]
  when (Path === path && @resources && @resources.include?(path.original))
    type, content = @resources[path.original]
  when has_rake(path)
    type = :rake
    rake_dir, content = rake_for(path)
    rake_dir = Path.setup(rake_dir.dup, self.pkgdir, self)
  else
    if path !~ /\.(gz|bgz)$/
      begin
        produce(path.annotate(path + '.gz'), force)
      rescue ResourceNotFound
        begin
          produce(path.annotate(path + '.bgz'), force)
        rescue ResourceNotFound
          raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
        end
      end
    else
      raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
    end
  end

  if path.respond_to?(:find) 
    final_path = force ? path.find(:default) : path.find
  else
    final_path = path
  end

  if type and not File.exist?(final_path) or force
    Log.medium "Producing: (#{self.to_s}) #{ final_path }"
    lock_filename = TmpFile.tmp_for_file(final_path, :dir => lock_dir)

    Open.lock lock_filename do
      FileUtils.rm_rf final_path if force and File.exist? final_path

      if ! File.exist?(final_path) || force 

        begin
          case type
          when :string
            Open.sensible_write(final_path, content)
          when :csv
            raise "TSV/CSV Not implemented yet"
            #require 'rbbt/tsv/csv'
            #tsv = TSV.csv Open.open(content)
            #Open.sensible_write(final_path, tsv.to_s)
          when :url
            options = {}
            options[:noz] = true if Open.gzip?(final_path) || Open.bgzip?(final_path) || Open.zip?(final_path)
            Open.sensible_write(final_path, Open.open(content, options))
          when :proc
            data = case content.arity
                   when 0
                     content.call
                   when 1
                     content.call final_path
                   end
            case data
            when String, IO, StringIO
              Open.sensible_write(final_path, data) 
            when Array
              Open.sensible_write(final_path, data * "\n")
            when TSV
              Open.sensible_write(final_path, data.dumper_stream) 
            when TSV::Dumper
              Open.sensible_write(final_path, data.stream) 
            when nil
            else
              raise "Unkown object produced: #{Log.fingerprint data}"
            end
          when :rake
            run_rake(path, content, rake_dir)
          when :install
            software_dir = self.root.software
            name = File.basename(path)
            Resource.install(content, name, software_dir)
            set_software_env(software_dir)
          else
            raise "Could not produce #{ resource }. (#{ type }, #{ content })"
          end
        rescue
          FileUtils.rm_rf final_path if File.exist? final_path
          raise $!
        end
      end
    end
  end

  # After producing a file, make sure we recheck all locations, the file
  # might have appeared with '.gz' extension for instance
  path.instance_variable_set("@path", {})

  path
end

#rake_for(path) ⇒ Object 



16
17
18
19
20
21
22
23
 
# File 'lib/scout/resource/produce.rb', line 16

def rake_for(path)
  @rake_dirs ||= {}
  @rake_dirs.select{|dir, content|
    Misc.path_relative_to(dir, path)
  }.sort_by{|dir, content|
    dir.length
  }.last
end

#root ⇒ Object 



29
30
31
 
# File 'lib/scout/resource.rb', line 29

def root
  Path.setup(subdir, self, self.libdir, @path_maps)
end

#run_rake(path, rakefile, rake_dir) ⇒ Object 



29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
 
# File 'lib/scout/resource/produce.rb', line 29

def run_rake(path, rakefile, rake_dir)
  task = Misc.path_relative_to rake_dir, path
  rakefile = rakefile.produce if rakefile.respond_to? :produce
  rakefile = rakefile.find if rakefile.respond_to? :find

  rake_dir = rake_dir.find(:user) if rake_dir.respond_to? :find

  begin
    if Proc === rakefile
      ScoutRake.run(nil, rake_dir, task, &rakefile)
    else
      ScoutRake.run(rakefile, rake_dir, task)
    end
  rescue ScoutRake::TaskNotFound
    if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
      raise $! 
    end
    task = File.join(File.basename(rake_dir), task)
    rake_dir = File.dirname(rake_dir)
    retry
  end
end

#subdir ⇒ Object 



17
18
19
 
# File 'lib/scout/resource.rb', line 17

def subdir
  @subdir ||= ""
end