Method: CsvPack::Downloader#fetch

Defined in:
lib/csvpack/downloader.rb

#fetch(name_or_shortcut_or_url) ⇒ Object

todo/check: use (re)name to get/update/etc. why? why not??



16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
 
# File 'lib/csvpack/downloader.rb', line 16

def fetch( name_or_shortcut_or_url )   ## todo/check: use (re)name to get/update/etc. why? why not??

  name = name_or_shortcut_or_url

  ##
  ## e.g. try
  ##   country-list
  ##

  ## url_base = "http://data.okfn.org/data/core/#{name}"
  ## url_base = "https://datahub.io/core/#{name}"

  ## or use "https://github.com/datasets/#{name}/raw/master"
  url_base = "https://raw.githubusercontent.com/datasets/#{name}/master"


  url = "#{url_base}/datapackage.json"

  dest_dir = "#{@cache_dir}/#{name}"
  FileUtils.mkdir_p( dest_dir )

  pack_path = "#{dest_dir}/datapackage.json"   ## todo/fix: rename to meta_path - why? why not?
  @worker.copy( url, pack_path )

  h = Meta.load_file( pack_path )
  pp h

  ## copy resources (tables)
  h.resources.each do |r|
    puts "== resource:"
    pp r

    res_name          = r['name']
    res_relative_path = r['path']   ## fix/todo: might no contain the url - is now res_url_or_relative_path !!!!!
    if res_relative_path.nil?
      res_relative_path = "#{res_name}.csv"
    end

    res_url       = r['url']   ## check - old package format - url NO longer used!!!!
    if res_url.nil?
       ## build url
       res_url = "#{url_base}/#{res_relative_path}"
    end

    ## todo/fix: rename - use just res_path - why? why not?
    local_res_path = "#{dest_dir}/#{res_relative_path}"
    puts "[debug] local_res_path: >#{local_res_path}<"
    local_res_dir   = File.dirname( local_res_path )
    FileUtils.mkdir_p( local_res_dir )

    @worker.copy( res_url, local_res_path )
  end
end