Module: Open

Defined in:
lib/rbbt/util/open.rb

Defined Under Namespace

Classes: OpenGzipError, OpenURLError

Constant Summary collapse

REMOTE_CACHEDIR = 
"/tmp/open_cache"
LAST_TIME =

Remote WGET

{}

Class Attribute Summary collapse

Class Method Summary collapse

Class Attribute Details

.repository_dirs ⇒ Object

Returns the value of attribute repository_dirs.



15
16
17
 
# File 'lib/rbbt/util/open.rb', line 15

def repository_dirs
  @repository_dirs
end

Class Method Details

.add_cache(url, data, options = {}) ⇒ Object 



120
121
122
123
 
# File 'lib/rbbt/util/open.rb', line 120

def self.add_cache(url, data, options = {})
  file = File.join(REMOTE_CACHEDIR, digest_url(url, options))
  Misc.sensiblewrite(file, data)
end

.cachedir ⇒ Object 



34
35
36
 
# File 'lib/rbbt/util/open.rb', line 34

def self.cachedir
  REMOTE_CACHEDIR
end

.cachedir=(cachedir) ⇒ Object 



29
30
31
32
 
# File 'lib/rbbt/util/open.rb', line 29

def self.cachedir=(cachedir)
  REMOTE_CACHEDIR.replace cachedir
  FileUtils.mkdir_p REMOTE_CACHEDIR unless File.exist? REMOTE_CACHEDIR
end

.can_open?(file) ⇒ Boolean

Returns:

  • (Boolean) 


357
358
359
 
# File 'lib/rbbt/util/open.rb', line 357

def self.can_open?(file)
  String === file and (File.exists?(file) or remote?(file))
end

.clean_cache(url, options = {}) ⇒ Object

Open Read Write



279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
 
# File 'lib/rbbt/util/open.rb', line 279

def self.clean_cache(url, options = {})
  options = Misc.add_defaults options, :noz => false, :mode => 'r'

  wget_options = options[:wget_options] || {}
  wget_options[:nice] = options.delete(:nice)
  wget_options[:nice_key] = options.delete(:nice_key)
  wget_options[:quiet] = options.delete(:quiet)
  wget_options["--post-data="] = options.delete(:post) if options.include? :post
  wget_options["--post-file"] = options.delete("--post-file") if options.include? "--post-file"
  wget_options["--post-file="] = options.delete("--post-file=") if options.include? "--post-file="
  wget_options[:cookies] = options.delete(:cookies)

  cache_file = in_cache(url, wget_options)
  Misc.lock(cache_file) do
    FileUtils.rm(cache_file)
  end if cache_file
end

.digest_url(url, options = {}) ⇒ Object 



94
95
96
97
 
# File 'lib/rbbt/util/open.rb', line 94

def self.digest_url(url, options = {})
  params = [url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]).split("\n").sort * "\n" : "")]
  digest = Misc.digest(params.inspect)
end

.exists?(file) ⇒ Boolean

Returns:

  • (Boolean) 


226
227
228
229
230
231
232
233
 
# File 'lib/rbbt/util/open.rb', line 226

def self.exists?(file)
  if (dir_sub_path = find_repo_dir(file))
    dir_sub_path.push file
    exists_in_repo(*dir_sub_path)
  else
    File.exists? file
  end
end

.exists_in_repo(dir, sub_path, content) ⇒ Object 



168
169
170
171
 
# File 'lib/rbbt/util/open.rb', line 168

def self.exists_in_repo(dir, sub_path, content)
  repo = get_repo_from_dir(dir)
  repo.include? sub_path
end

.file_open(file, grep, mode = 'r', invert_grep = false) ⇒ Object 



199
200
201
202
203
204
205
206
207
208
209
210
211
 
# File 'lib/rbbt/util/open.rb', line 199

def self.file_open(file, grep, mode = 'r', invert_grep = false)
  if (dir_sub_path = find_repo_dir(file))
    stream =  get_stream_from_repo(*dir_sub_path)
  else
    stream =  File.open(file, mode)
  end

  if grep
    grep(stream, grep, invert_grep)
  else
    stream
  end
end

.file_write(file, content, mode = 'w') ⇒ Object 



213
214
215
216
217
218
219
220
221
222
223
224
 
# File 'lib/rbbt/util/open.rb', line 213

def self.file_write(file, content, mode = 'w')
  if (dir_sub_path = find_repo_dir(file))
    dir_sub_path.push content
    save_content_in_repo(*dir_sub_path)
  else
    File.open(file, mode) do |f|
      f.flock(File::LOCK_EX)
      f.write content 
      f.flock(File::LOCK_UN)
    end
  end
end

.find_repo_dir(file) ⇒ Object 



173
174
175
176
177
178
179
180
181
 
# File 'lib/rbbt/util/open.rb', line 173

def self.find_repo_dir(file)
  self.repository_dirs.each do |dir|
    if file.start_with? dir
      sub_path = file.to_s[dir.length..-1]
      return [dir, sub_path]
    end
  end
  nil
end

.get_repo_from_dir(dir) ⇒ Object 



138
139
140
141
142
143
144
 
# File 'lib/rbbt/util/open.rb', line 138

def self.get_repo_from_dir(dir)
  @repos ||= {}
  @repos[dir] ||= begin
                    repo_path = File.join(dir, '.file_repo')
                    Persist.open_tokyocabinet(repo_path, false, :clean,TokyoCabinet::BDB )
                  end
end

.get_stream_from_repo(dir, sub_path) ⇒ Object 



146
147
148
149
150
 
# File 'lib/rbbt/util/open.rb', line 146

def self.get_stream_from_repo(dir, sub_path)
  repo = get_repo_from_dir(dir)
  repo.read
  StringIO.new repo[sub_path]
end

.grep(stream, grep, invert = false) ⇒ Object

Grep



127
128
129
130
131
132
133
134
135
136
 
# File 'lib/rbbt/util/open.rb', line 127

def self.grep(stream, grep, invert = false)
  case 
  when Array === grep
    TmpFile.with_file(grep * "\n", false) do |f|
      CMD.cmd("grep #{invert ? '-v' : ''}", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
    end
  else
    CMD.cmd("grep #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{stream.force_close if stream.respond_to? :force_close})
  end
end

.gunzip(stream) ⇒ Object

Decompression



248
249
250
251
252
253
254
 
# File 'lib/rbbt/util/open.rb', line 248

def self.gunzip(stream)
  if String === stream
    Zlib::Inflate.inflate(stream)
  else
    CMD.cmd("gunzip", :pipe => true, :in => stream, :post => proc{stream.force_close if stream.respond_to? :force_close})
  end
end

.gzip?(file) ⇒ Boolean

Returns:

  • (Boolean) 


268
269
270
 
# File 'lib/rbbt/util/open.rb', line 268

def self.gzip?(file)
  !! (file =~ /\.gz$/)
end

.in_cache(url, options = {}) ⇒ Object

Cache



100
101
102
103
104
105
106
107
 
# File 'lib/rbbt/util/open.rb', line 100

def self.in_cache(url, options = {})
  filename = File.join(REMOTE_CACHEDIR, digest_url(url, options))
  if File.exists? filename
    return filename 
  else
    nil
  end
end

.lock(file, &block) ⇒ Object 



235
236
237
238
239
240
241
242
243
 
# File 'lib/rbbt/util/open.rb', line 235

def self.lock(file, &block)
  if (dir_sub_path = find_repo_dir(file))
    dir, sub_path = dir_sub_path
    repo = get_repo_from_dir(dir)
    Misc.lock_in_repo(repo, sub_path, &block)
  else
    Misc.lock(file, &block)
  end
end

.open(url, options = {}) ⇒ Object 



297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
 
# File 'lib/rbbt/util/open.rb', line 297

def self.open(url, options = {})
  if IO === url
    if block_given?
      res = yield url 
      url.close
      return res
    else
      return url 
    end
  end
  options = Misc.add_defaults options, :noz => false, :mode => 'r'

  mode = Misc.process_options options, :mode

  wget_options = options[:wget_options] || {}
  wget_options[:nice] = options.delete(:nice)
  wget_options[:nice_key] = options.delete(:nice_key)
  wget_options[:quiet] = options.delete(:quiet)
  wget_options["--post-data="] = options.delete(:post) if options.include? :post
  wget_options["--post-file"] = options.delete("--post-file") if options.include? "--post-file"
  wget_options["--post-file="] = options.delete("--post-file=") if options.include? "--post-file="
  wget_options[:cookies] = options.delete(:cookies)

  io = case
       when (IO === url or StringIO === url)
         url
       when (not remote?(url))
         file_open(url, options[:grep], mode, options[:invert_grep])
       when (options[:nocache] and options[:nocache] != :update)
         # What about grep?
         wget(url, wget_options)
       when (options[:nocache] != :update and in_cache(url, wget_options))
         Misc.lock(in_cache(url, wget_options)) do
           file_open(in_cache(url, wget_options), options[:grep], mode, options[:invert_grep])
         end
       else
         io = wget(url, wget_options)
         add_cache(url, io, wget_options)
         io.close
         file_open(in_cache(url, wget_options), options[:grep], mode, options[:invert_grep])
       end
  io = unzip(io)  if ((String === url and zip?(url))  and not options[:noz]) or options[:zip]
  io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]

  if block_given?
    res = yield(io)
    io.close
    return res
  else
    io
  end

  class << io;
    attr_accessor :filename
  end

  io.filename = url.to_s
  io
end

.read(file, options = {}, &block) ⇒ Object 



361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
 
# File 'lib/rbbt/util/open.rb', line 361

def self.read(file, options = {}, &block)
  f = open(file, options)

  if block_given?
    res = []
    while not f.eof?
      l = f.gets
      l = Misc.fixutf8(l) 
      res << yield(l)
    end
    f.close
    res
  else
    text = Misc.fixutf8(f.read)
    f.close unless f.closed?
    text 
  end
end

.remote?(file) ⇒ Boolean

Questions

Returns:

  • (Boolean) 


264
265
266
 
# File 'lib/rbbt/util/open.rb', line 264

def self.remote?(file)
  !! (file =~ /^(?:https?|ftp):\/\//)
end

.remove_from_cache(url, options = {}) ⇒ Object 



109
110
111
112
113
114
115
116
117
118
 
# File 'lib/rbbt/util/open.rb', line 109

def self.remove_from_cache(url, options = {})
  digest = Misc.digest([url, options.values_at("--post-data", "--post-data="), (options.include?("--post-file")? Open.read(options["--post-file"]) : "")].inspect)

  filename = File.join(REMOTE_CACHEDIR, digest)
  if File.exists? filename
    FileUtils.rm filename 
  else
    nil
  end
end

.remove_from_repo(dir, sub_path, recursive = false) ⇒ Object 



158
159
160
161
162
163
164
165
166
 
# File 'lib/rbbt/util/open.rb', line 158

def self.remove_from_repo(dir, sub_path, recursive = false)
  repo = get_repo_from_dir(dir)
  repo.write
  if recursive
    repo.outlist repo.range sub_path, true, sub_path.sub(/.$/,('\1'.ord + 1).chr), false
  else
    repo.outlist sub_path
  end
end

.rm(file) ⇒ Object 



183
184
185
186
187
188
189
 
# File 'lib/rbbt/util/open.rb', line 183

def self.rm(file)
  if (dir_sub_path = find_repo_dir(file))
    remove_from_repo(*dir_sub_path)
  else
    FileUtils.rm(file)
  end
end

.rm_rf(file) ⇒ Object 



191
192
193
194
195
196
197
 
# File 'lib/rbbt/util/open.rb', line 191

def self.rm_rf(file)
  if (dir_sub_path = find_repo_dir(file))
    remove_from_repo(*dir_sub_path, true)
  else
    FileUtils.rm_rf(file)
  end
end

.save_content_in_repo(dir, sub_path, content) ⇒ Object 



152
153
154
155
156
 
# File 'lib/rbbt/util/open.rb', line 152

def self.save_content_in_repo(dir, sub_path, content)
  repo = get_repo_from_dir(dir)
  repo.write
  repo[sub_path] = content
end

.unzip(stream) ⇒ Object 



256
257
258
259
260
 
# File 'lib/rbbt/util/open.rb', line 256

def self.unzip(stream)
  TmpFile.with_file(stream.read) do |filename|
    StringIO.new(CMD.cmd("unzip '{opt}' #{filename}", "-p" => true, :pipe => true).read)
  end
end

.wait(lag, key = nil) ⇒ Object 



40
41
42
43
44
45
46
47
48
 
# File 'lib/rbbt/util/open.rb', line 40

def self.wait(lag, key = nil)
  time = Time.now   

  if LAST_TIME[key] != nil && (time < LAST_TIME[key] + lag)
    sleep (LAST_TIME[key] + lag) - time
  end

  LAST_TIME[key] = Time.now   
end

.wget(url, options = {}) ⇒ Object 



50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
 
# File 'lib/rbbt/util/open.rb', line 50

def self.wget(url, options = {})
  Log.low "WGET:\n -URL: #{ url }\n -OPTIONS: #{options.inspect}"
  options = Misc.add_defaults options, "--user-agent=" => 'firefox', :pipe => true

  wait(options[:nice], options[:nice_key]) if options[:nice]
  options.delete(:nice)
  options.delete(:nice_key)

  pipe  = options.delete(:pipe)
  quiet = options.delete(:quiet)
  post  = options.delete(:post)
  cookies = options.delete(:cookies)

  options["--quiet"]     = quiet if options["--quiet"].nil?
  options["--post-data="] ||= post if post

  if cookies
    options["--save-cookies"] = cookies
    options["--load-cookies"] = cookies
    options["--keep-session-cookies"] = true
  end


  stderr = case
           when options['stderr']
             options['stderr'] 
           when options['--quiet']
             false
           else
             nil
           end

  begin
    CMD.cmd("wget '#{ url }'", options.merge(
      '-O' => '-', 
      :pipe => pipe, 
      :stderr => stderr
    ))
  rescue
   STDERR.puts $!.backtrace.inspect
   raise OpenURLError, "Error reading remote url: #{ url }.\n#{$!.message}"
  end
end

.write(file, content = nil, options = {}) ⇒ Object 



380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
 
# File 'lib/rbbt/util/open.rb', line 380

def self.write(file, content = nil, options = {})
  options = Misc.add_defaults options, :mode => 'w'

  mode = Misc.process_options options, :mode

  FileUtils.mkdir_p File.dirname(file)
  case
  when content.nil?
    begin
      File.open(file, mode) do |f| 
        yield f
      end
    rescue Exception
      FileUtils.rm file if File.exists? file
      raise $!
    end
  when String === content
    file_write(file, content, mode)
  else
    begin
      File.open(file, mode) do |f| 
        f.flock(File::LOCK_EX)
        while not content.eof?
          f.write content.gets
        end
        f.flock(File::LOCK_UN)
      end
    rescue Exception
      FileUtils.rm_rf file if File.exists? file
      raise $!
    end
    content.close
  end
end

.zip?(file) ⇒ Boolean

Returns:

  • (Boolean) 


272
273
274
 
# File 'lib/rbbt/util/open.rb', line 272

def self.zip?(file)
  !! (file =~ /\.zip/)
end