Class: Distil::RecursiveHTTPFetcher

Inherits:
Object
  • Object
show all
Defined in:
lib/distil/recursive-http-fetcher.rb

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(urls_to_fetch, level = 1, cwd = ".") ⇒ RecursiveHTTPFetcher

Returns a new instance of RecursiveHTTPFetcher.



6
7
8
9
10
11
 
# File 'lib/distil/recursive-http-fetcher.rb', line 6

def initialize(urls_to_fetch, level = 1, cwd = ".")
  @level = level
  @cwd = cwd
  @urls_to_fetch = RUBY_VERSION >= '1.9' ? urls_to_fetch.to_s.lines : urls_to_fetch.to_s.to_a
  @quiet = true
end

Instance Attribute Details

#quiet ⇒ Object

Returns the value of attribute quiet.



4
5
6
 
# File 'lib/distil/recursive-http-fetcher.rb', line 4

def quiet
  @quiet
end

Instance Method Details

#download(link) ⇒ Object 



45
46
47
48
49
50
51
52
 
# File 'lib/distil/recursive-http-fetcher.rb', line 45

def download(link)
  puts "+ #{File.join(@cwd, File.basename(link))}" unless @quiet
  open(link) do |stream|
    File.open(File.join(@cwd, File.basename(link)), "wb") do |file|
      file.write(stream.read)
    end
  end
end

#fetch(links = @urls_to_fetch) ⇒ Object 



54
55
56
57
58
 
# File 'lib/distil/recursive-http-fetcher.rb', line 54

def fetch(links = @urls_to_fetch)
  links.each do |l|
    (l =~ /\/$/ || links == @urls_to_fetch) ? fetch_dir(l) : download(l)
  end
end

#fetch_dir(url) ⇒ Object 



60
61
62
63
64
65
66
67
68
69
 
# File 'lib/distil/recursive-http-fetcher.rb', line 60

def fetch_dir(url)
  @level += 1
  push_d(File.basename(url)) if @level > 0
  open(url) do |stream|
    contents =  stream.read
    fetch(links(url, contents))
  end
  pop_d if @level > 0
  @level -= 1
end

#links(base_url, contents) ⇒ Object 



34
35
36
37
38
39
40
41
42
43
 
# File 'lib/distil/recursive-http-fetcher.rb', line 34

def links(base_url, contents)
  links = []
  contents.scan(/href\s*=\s*\"*[^\">]*/i) do |link|
    link = link.sub(/href="/i, "")
    next if link =~ /svnindex.xsl$/
    next if link =~ /^(\w*:|)\/\// || link =~ /^\./
    links << File.join(base_url, link)
  end
  links
end

#ls ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
 
# File 'lib/distil/recursive-http-fetcher.rb', line 13

def ls
  @urls_to_fetch.collect do |url|
    if url =~ /^svn(\+ssh)?:\/\/.*/ || url =~ /\/svn\//
      `svn ls #{url}`.split("\n").map {|entry| "/#{entry}"} rescue nil
    else
      open(url) do |stream|
        links("", stream.read)
      end rescue nil
    end
  end.flatten
end

#pop_d ⇒ Object 



30
31
32
 
# File 'lib/distil/recursive-http-fetcher.rb', line 30

def pop_d
  @cwd = File.dirname(@cwd)
end

#push_d(dir) ⇒ Object 



25
26
27
28
 
# File 'lib/distil/recursive-http-fetcher.rb', line 25

def push_d(dir)
  @cwd = File.join(@cwd, dir)
  FileUtils.mkdir_p(@cwd)
end