Module: Resource

Included in:
Rbbt
Defined in:
lib/rbbt/resource.rb,
lib/rbbt/resource/util.rb,
lib/rbbt/resource/with_key.rb

Defined Under Namespace

Modules: WithKey Classes: ResourceNotFound

Class Attribute Summary collapse

Instance Attribute Summary collapse

Class Method Summary collapse

Instance Method Summary collapse

Dynamic Method Handling

This class handles dynamic methods through the method_missing method

#method_missing(name, prev = nil, *args) ⇒ Object 



44
45
46
47
48
49
50
 
# File 'lib/rbbt/resource.rb', line 44

def method_missing(name, prev = nil, *args)
  if prev.nil?
    root.send(name, *args)
  else
    root.send(name, prev, *args)
  end
end

Class Attribute Details

.lock_dir ⇒ Object

Returns the value of attribute lock_dir.



12
13
14
 
# File 'lib/rbbt/resource.rb', line 12

def lock_dir
  @lock_dir
end

Instance Attribute Details

#pkgdir ⇒ Object

Returns the value of attribute pkgdir.



33
34
35
 
# File 'lib/rbbt/resource.rb', line 33

def pkgdir
  @pkgdir
end

#rake_dirs ⇒ Object

Returns the value of attribute rake_dirs.



33
34
35
 
# File 'lib/rbbt/resource.rb', line 33

def rake_dirs
  @rake_dirs
end

#remote_server ⇒ Object

Returns the value of attribute remote_server.



33
34
35
 
# File 'lib/rbbt/resource.rb', line 33

def remote_server
  @remote_server
end

#resources ⇒ Object

Returns the value of attribute resources.



33
34
35
 
# File 'lib/rbbt/resource.rb', line 33

def resources
  @resources
end

#search_paths ⇒ Object

Returns the value of attribute search_paths.



33
34
35
 
# File 'lib/rbbt/resource.rb', line 33

def search_paths
  @search_paths
end

#server_missing_resource_cache ⇒ Object

Returns the value of attribute server_missing_resource_cache.



78
79
80
 
# File 'lib/rbbt/resource.rb', line 78

def server_missing_resource_cache
  @server_missing_resource_cache
end

#subdir ⇒ Object

Returns the value of attribute subdir.



33
34
35
 
# File 'lib/rbbt/resource.rb', line 33

def subdir
  @subdir
end

Class Method Details

.extended(base) ⇒ Object 



23
24
25
26
27
28
29
30
31
 
# File 'lib/rbbt/resource.rb', line 23

def self.extended(base)
  base.pkgdir = 'rbbt'
  base.subdir = ''
  base.resources = {}
  base.rake_dirs = {}
  base.search_paths = Path::SEARCH_PATHS.dup
  base.remote_server = Resource.remote_servers[base.to_s] || Resource.remote_servers["*"]
  base
end

.remote_servers ⇒ Object 



19
20
21
 
# File 'lib/rbbt/resource.rb', line 19

def self.remote_servers
  @remote_servers = Rbbt.etc.file_servers.exists? ? Rbbt.etc.file_servers.yaml : {}
end

Instance Method Details

#[](file = nil) ⇒ Object 



52
53
54
55
56
57
58
 
# File 'lib/rbbt/resource.rb', line 52

def [](file = nil)
  if file.nil?
    root
  else
    root.send(file)
  end
end

#claim(path, type, content = nil, &block) ⇒ Object 



60
61
62
63
64
65
66
67
68
69
70
71
72
 
# File 'lib/rbbt/resource.rb', line 60

def claim(path, type, content = nil, &block)
  if type == :rake
    @rake_dirs[path] = content || block
  else
    @resources[path] = [type, content || block]

    if type == :install
      software_dir = path.resource.root.software
      set_software_env(software_dir) unless $set_software_env
      $set_software_env = true
    end
  end
end

#claims ⇒ Object 



74
75
76
 
# File 'lib/rbbt/resource.rb', line 74

def claims
  resources.keys.collect{|path| path.sub(subdir, '').sub(/^\//,'') }
end

#get_from_server(path, final_path, remote_server = nil) ⇒ Object 



79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
 
# File 'lib/rbbt/resource.rb', line 79

def get_from_server(path, final_path, remote_server = nil)
  remote_server ||= self.remote_server
  remote_server = "http://" + remote_server unless remote_server =~ /^[a-z]+:\/\//
  url = File.join(remote_server, '/resource/', self.to_s, 'get_file')
  url << "?" << Misc.hash2GET_params(:file => path, :create => false)

  begin
    @server_missing_resource_cache ||= Set.new
    raise "Resource Not Found" if @server_missing_resource_cache.include? url
    
    #lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})
    
    lock_filename = nil # it seems like this was locked already.

    Log.low "Downloading #{path} from #{url} file server"
    Misc.lock lock_filename do
      begin
        uri = URI(url)

        http = Net::HTTP.new(uri.host, uri.port)

        if uri.scheme == "https"
          http.use_ssl = true
          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
          http.instance_variable_set("@ssl_options", OpenSSL::SSL::OP_NO_SSLv2 + OpenSSL::SSL::OP_NO_SSLv3 + OpenSSL::SSL::OP_NO_COMPRESSION)
        end

        timeout = 60 * 10
        http.read_timeout = timeout
        http.open_timeout = timeout
        request = Net::HTTP::Get.new(uri.request_uri)
        http.request request do |response|
          filename = if response["Content-Disposition"] 
                       response["Content-Disposition"].split(";").select{|f| f.include? "filename"}.collect{|f| f.split("=").last.gsub('"','')}.first
                     else
                       nil
                     end

          if filename && filename =~ /\.b?gz$/ && final_path !~ /\.b?gz$/
            extension = filename.split(".").last
            final_path += '.' + extension
          end
          case response
          when Net::HTTPSuccess, Net::HTTPOK
            Misc.sensiblewrite(final_path) do |file|
              response.read_body do |chunk|
                file.write chunk
              end
            end
          when Net::HTTPRedirection, Net::HTTPFound
            location = response['location']
            if location.include? 'get_directory'
              Log.debug("Feching directory from: #{location}. Into: #{final_path}")
              FileUtils.mkdir_p final_path unless File.exist? final_path
              TmpFile.with_file do |tmp_dir|
                Misc.in_dir tmp_dir do
                  CMD.cmd('tar xvfz -', :in => Open.open(location, :nocache => true))
                end
                FileUtils.mv tmp_dir, final_path
              end
            else
              url = location
              raise TryAgain
              #Open.open(location, :nocache => true) do |s|
              #  Misc.sensiblewrite(final_path, s)
              #end
            end
          when Net::HTTPInternalServerError
            @server_missing_resource_cache << url
            raise "Resource Not Found"
          else
            raise "Response not understood: #{response.inspect}"
          end
        end
      rescue TryAgain
        retry
      end
    end
  rescue
    Log.warn "Could not retrieve (#{self.to_s}) #{ path } from #{ remote_server }"
    Log.error $!.message
    FileUtils.rm_rf final_path if File.exist? final_path
    return false
  end

  final_path
end

#has_rake(path) ⇒ Object 



135
136
137
 
# File 'lib/rbbt/resource/util.rb', line 135

def has_rake(path)
  !! rake_for(path)
end

#identify(path) ⇒ Object 



352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
 
# File 'lib/rbbt/resource.rb', line 352

def identify(path)
  return path unless path.start_with?("/")
  path = File.expand_path(path)
  path += "/" if File.directory?(path)
  resource ||= Rbbt
  locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq
  locations -= [:current, "current"]
  locations << :current
  search_paths = IndiferentHash.setup(resource.search_paths)
  choices = []
  locations.uniq.each do |name|
    pattern = search_paths[name]
    pattern = resource.search_paths[pattern] while Symbol === pattern
    next if pattern.nil?

    pattern = pattern.sub('{PWD}', Dir.pwd)
    if String ===  pattern and pattern.include?('{')
      regexp = "^" + pattern
        .gsub(/{(TOPLEVEL)}/,'(?<\1>[^/]+)')
        .gsub(/{([^}]+)}/,'(?<\1>[^/]+)?') +
      "(?:/(?<REST>.*))?/?$"
      if m = path.match(regexp) 
        if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
          unlocated = ([m["TOPLEVEL"],m["SUBPATH"],m["REST"]] * "/")
          unlocated.gsub!(/\/+/,'/')
          if self.subdir && ! self.subdir.empty? && unlocated.include?(subdir)
            subdir = self.subdir
            subdir += "/" unless subdir.end_with?("/")
            unlocated[subdir] = ""
          end
          choices << self.annotate(unlocated)
        end
      end
    end
  end
  choices.sort_by{|s| s.length }.first
end

#produce(path, force = false) ⇒ Object 



167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
 
# File 'lib/rbbt/resource.rb', line 167

def produce(path, force = false)
  case
  when @resources.include?(path)
    type, content = @resources[path]
  when (Path === path && @resources.include?(path.original))
    type, content = @resources[path.original]
  when has_rake(path)
    type = :rake
    rake_dir, content = rake_for(path)
    rake_dir = Path.setup(rake_dir.dup, self.pkgdir, self)
  else
    if path !~ /\.(gz|bgz)$/
      begin
        produce(path.annotate(path + '.gz'), force)
      rescue ResourceNotFound
        begin
          produce(path.annotate(path + '.bgz'), force)
        rescue ResourceNotFound
          raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
        end
      end
    else
      raise ResourceNotFound, "Resource is missing and does not seem to be claimed: #{ self } -- #{ path } "
    end
  end

  if path.respond_to?(:find) 
    final_path = force ? path.find(:default) : path.find
  else
    final_path = path
  end

  if type and not File.exist?(final_path) or force
    Log.medium "Producing: (#{self.to_s}) #{ final_path }"
    lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})

    Misc.lock lock_filename do
      FileUtils.rm_rf final_path if force and File.exist? final_path

      if ! File.exist?(final_path) || force 

        begin
          case type
          when :string
            Misc.sensiblewrite(final_path, content)
          when :csv
            require 'rbbt/tsv/csv'
            tsv = TSV.csv Open.open(content)
            Misc.sensiblewrite(final_path, tsv.to_s)
          when :url
            options = {}
            options[:noz] = true if Open.gzip?(final_path) || Open.bgzip?(final_path) || Open.zip?(final_path)
            Misc.sensiblewrite(final_path, Open.open(content, options))
          when :proc
            data = case content.arity
                   when 0
                     content.call
                   when 1
                     content.call final_path
                   end
            case data
            when String, IO, StringIO
              Misc.sensiblewrite(final_path, data) 
            when Array
              Misc.sensiblewrite(final_path, data * "\n")
            when TSV
              Misc.sensiblewrite(final_path, data.dumper_stream) 
            when TSV::Dumper
              Misc.sensiblewrite(final_path, data.stream) 
            when nil
            else
              raise "Unkown object produced: #{Misc.fingerprint data}"
            end
          when :rake
            run_rake(path, content, rake_dir)
          when :install
            Log.debug "Installing software: #{path}"

            $set_software_env = false unless File.exist? path
            
            software_dir = path.resource.root.software.find :user
            helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find(:lib, caller_lib_dir(__FILE__)))
            #helper_file = File.expand_path(Rbbt.share.install.software.lib.install_helpers.find)

            preamble = <<-EOF
#!/bin/bash

RBBT_SOFTWARE_DIR="#{software_dir}"

INSTALL_HELPER_FILE="#{helper_file}"
source "$INSTALL_HELPER_FILE"
            EOF

            content = content.call if Proc === content

            content = if content =~ /git:|\.git$/
                        {:git => content}
                      else
                        {:src => content}
                      end if String === content and Open.remote?(content)

            script_text = case content
                          when nil
                            raise "No way to install #{path}"
                          when Path
                            Open.read(content) 
                          when String
                            if Misc.is_filename?(content) and Open.exists?(content)
                              Open.read(content) 
                            else
                              content
                            end
                          when Hash
                            name = content[:name] || File.basename(path)
                            git = content[:git]
                            src = content[:src]
                            url = content[:url]
                            jar = content[:jar]
                            extra = content[:extra]
                            commands = content[:commands]
                            if git
                              <<-EOF

name='#{name}'
url='#{git}'

install_git "$name" "$url" #{extra}

#{commands}
                              EOF
                            elsif src
                              <<-EOF

name='#{name}'
url='#{src}'

install_src "$name" "$url" #{extra}

#{commands}
                              EOF
                            elsif jar
                              <<-EOF

name='#{name}'
url='#{jar}'

install_jar "$name" "$url" #{extra}

#{commands}
                              EOF
                            else
                              <<-EOF

name='#{name}'
url='#{url}'

#{commands}
                              EOF
                            end
                          end

            script = preamble + "\n" + script_text
            Log.debug "Installing software with script:\n" << script
            CMD.cmd_log('bash', :in => script)

            set_software_env(software_dir) unless $set_software_env
            $set_software_env = true
          else
            raise "Could not produce #{ resource }. (#{ type }, #{ content })"
          end
        rescue
          FileUtils.rm_rf final_path if File.exist? final_path
          raise $!
        end unless (remote_server && get_from_server(path, final_path))
      end
    end
  end

  # After producing a file, make sure we recheck all locations, the file
  # might have appeared with '.gz' extension for instance
  path.instance_variable_set("@path", {})

  path
end

#rake_for(path) ⇒ Object 



127
128
129
130
131
132
133
 
# File 'lib/rbbt/resource/util.rb', line 127

def rake_for(path)
  @rake_dirs.select{|dir, content|
    Misc.common_path(dir, path)
  }.sort_by{|dir, content|
    dir.length
  }.last
end

#root ⇒ Object 



40
41
42
 
# File 'lib/rbbt/resource.rb', line 40

def root
  Path.setup @subdir || "", @pkgdir, self, @search_paths
end

#run_rake(path, rakefile, rake_dir) ⇒ Object 



139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
 
# File 'lib/rbbt/resource/util.rb', line 139

def run_rake(path, rakefile, rake_dir)
  task = Misc.path_relative_to rake_dir, path
  rakefile = rakefile.produce if rakefile.respond_to? :produce
  rakefile = rakefile.find if rakefile.respond_to? :find

  rake_dir = rake_dir.find(:user) if rake_dir.respond_to? :find

  begin
    require 'rbbt/resource/rake'
    if Proc === rakefile
      Rake.run(nil, rake_dir, task, &rakefile)
    else
      Rake.run(rakefile, rake_dir, task)
    end
  rescue Rake::TaskNotFound
    raise $! if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
    task = File.join(File.basename(rake_dir), task)
    rake_dir = File.dirname(rake_dir)
    retry
  end
end

#set_libdir(value = nil) ⇒ Object 



35
36
37
38
 
# File 'lib/rbbt/resource.rb', line 35

def set_libdir(value = nil)
  _libdir = value || Path.caller_lib_dir
  search_paths.merge!(:lib => File.join(_libdir, '{TOPLEVEL}', '{SUBPATH}'))
end

#set_software_env(software_dir = self.root.software) ⇒ Object 



48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
 
# File 'lib/rbbt/resource/util.rb', line 48

def set_software_env(software_dir = self.root.software)
  software_dir.opt.find_all.collect{|d| d.annotate(File.dirname(d)) }.reverse.each do |software_dir|
    next unless software_dir.exists?
    Log.medium "Preparing software env at #{software_dir}"

    software_dir = File.expand_path(software_dir)
    opt_dir = File.join(software_dir, 'opt')
    bin_dir = File.join(opt_dir, 'bin')

    Misc.env_add 'PATH', bin_dir

    FileUtils.mkdir_p opt_dir unless File.exist? opt_dir

    %w(.ld-paths .c-paths .pkgconfig-paths .aclocal-paths .java-classpaths).each do |file|
      filename = File.join(opt_dir, file)
      begin
        FileUtils.touch filename unless File.exist? filename
      rescue
        Log.warn("Could not touch #{ filename }")
      end
    end

    Open.read(File.join opt_dir, '.c-paths').split(/\n/).each do |line|
      dir = line.chomp
      dir = File.join(opt_dir, dir) unless dir[0] == "/"
      Misc.env_add('CPLUS_INCLUDE_PATH',dir)
      Misc.env_add('C_INCLUDE_PATH',dir)
    end if File.exist? File.join(opt_dir, '.c-paths')

    Open.read(File.join opt_dir, '.ld-paths').split(/\n/).each do |line|
      dir = line.chomp
      dir = File.join(opt_dir, dir) unless dir[0] == "/"
      Misc.env_add('LIBRARY_PATH',dir)
      Misc.env_add('LD_LIBRARY_PATH',dir)
      Misc.env_add('LD_RUN_PATH',dir)
    end if File.exist? File.join(opt_dir, '.ld-paths')

    Open.read(File.join opt_dir, '.pkgconfig-paths').split(/\n/).each do |line|
      dir = line.chomp
      dir = File.join(opt_dir, dir) unless dir[0] == "/"
      Misc.env_add('PKG_CONFIG_PATH',dir)
    end if File.exist? File.join(opt_dir, '.pkgconfig-paths')

    Open.read(File.join opt_dir, '.aclocal-paths').split(/\n/).each do |line|
      dir = line.chomp
      dir = File.join(opt_dir, dir) unless dir[0] == "/"
      Misc.env_add('ACLOCAL_FLAGS', "-I #{dir}", ' ')
    end if File.exist? File.join(opt_dir, '.aclocal-paths')

    Open.read(File.join opt_dir, '.java-classpaths').split(/\n/).each do |line|
      dir = line.chomp
      dir = File.join(opt_dir, dir) unless dir[0] == "/"
      Misc.env_add('CLASSPATH', "#{dir}")
    end if File.exist? File.join(opt_dir, '.java-classpaths')

    Dir.glob(File.join opt_dir, 'jars', '*.jar').each do |file|
      Misc.env_add('CLASSPATH', "#{file}")
    end

    if File.exist?(File.join(opt_dir, '.post_install')) and File.directory?(File.join(opt_dir, '.post_install'))
      Dir.glob(File.join(opt_dir, '.post_install','*')).each do |file|

        # Load exports
        Open.read(file).split("\n").each do |line|
          next unless line =~ /^\s*export\s+([^=]+)=(.*)/
          var = $1.strip
          value = $2.strip
          value.sub!(/^['"]/,'')
          value.sub!(/['"]$/,'')
          value.gsub!(/\$[a-z_0-9]+/i){|var| ENV[var[1..-1]] }
          Log.debug "Set variable export from .post_install: #{Misc.fingerprint [var,value]*"="}"
          ENV[var] = value
        end
      end
    end
  end
end

#with_key(key) ⇒ Object 



20
21
22
23
24
25
26
27
 
# File 'lib/rbbt/resource/with_key.rb', line 20

def with_key(key)
  klass = self
  o     = Object.new
  o.extend WithKey
  o.klass = self
  o.key   = key
  o
end