Class: CsvPack::Downloader

Inherits:
Object
  • Object
show all
Defined in:
lib/csvpack/downloader.rb

Constant Summary collapse

SHORTCUTS = 
{
  ## to be done
}

Instance Method Summary collapse

Constructor Details

#initialize(cache_dir = './pack') ⇒ Downloader

Returns a new instance of Downloader.

[View source]

7
8
9
10
 
# File 'lib/csvpack/downloader.rb', line 7

def initialize( cache_dir='./pack' )
  @cache_dir = cache_dir   # todo: check if folder exists now (or on demand)?
  @worker = Fetcher::Worker.new
end

Instance Method Details

#fetch(name_or_shortcut_or_url) ⇒ Object

todo/check: use (re)name to get/update/etc. why? why not??

[View source]

16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/csvpack/downloader.rb', line 16

def fetch( name_or_shortcut_or_url )   ## todo/check: use (re)name to get/update/etc. why? why not??

  name = name_or_shortcut_or_url

  ##
  ## e.g. try
  ##   country-list
  ##

  ## url_base = "http://data.okfn.org/data/core/#{name}"
  ## url_base = "https://datahub.io/core/#{name}"

  ## or use "https://github.com/datasets/#{name}/raw/master"
  url_base = "https://raw.githubusercontent.com/datasets/#{name}/master"


  url = "#{url_base}/datapackage.json"

  dest_dir = "#{@cache_dir}/#{name}"
  FileUtils.mkdir_p( dest_dir )

  pack_path = "#{dest_dir}/datapackage.json"   ## todo/fix: rename to meta_path - why? why not?
  @worker.copy( url, pack_path )

  h = Meta.load_file( pack_path )
  pp h

  ## copy resources (tables)
  h.resources.each do |r|
    puts "== resource:"
    pp r

    res_name          = r['name']
    res_relative_path = r['path']   ## fix/todo: might no contain the url - is now res_url_or_relative_path !!!!!
    if res_relative_path.nil?
      res_relative_path = "#{res_name}.csv"
    end

    res_url       = r['url']   ## check - old package format - url NO longer used!!!!
    if res_url.nil?
       ## build url
       res_url = "#{url_base}/#{res_relative_path}"
    end

    ## todo/fix: rename - use just res_path - why? why not?
    local_res_path = "#{dest_dir}/#{res_relative_path}"
    puts "[debug] local_res_path: >#{local_res_path}<"
    local_res_dir   = File.dirname( local_res_path )
    FileUtils.mkdir_p( local_res_dir )

    @worker.copy( res_url, local_res_path )
  end
end