Class: Geocoder::US::Import

Inherits:
Database show all
Defined in:
lib/geocoder/us/import.rb

Direct Known Subclasses

TIGER

Defined Under Namespace

Classes: TIGER

Constant Summary

Constants inherited from Database

Database::City_Weight, Database::Number_Weight, Database::Parity_Weight, Database::Street_Weight

Class Method Summary collapse

Instance Method Summary collapse

Methods inherited from Database

#add_ranges!, #assign_number!, #best_candidates!, #best_places, #canonicalize_places!, #clean_record!, #distance, #edges, #execute, #execute_statement, #extend_ranges!, #features_by_street, #features_by_street_and_zip, #find_candidates, #flush_statements, #geocode, #geocode_address, #geocode_intersection, #geocode_place, #interpolate, #interpolation_distance, #intersections_by_fid, #merge_edges!, #merge_rows!, #metaphone_placeholders_for, #more_features_by_street_and_zip, #placeholders_for, #places_by_city, #places_by_zip, #prepare, #primary_places, #range_ends, #ranges_by_feature, #rows_to_h, #scale_lon, #score_candidates!, #synchronize, #tune, #unique_values, #unpack_geometry

Constructor Details

#initialize(filename, options) ⇒ Import

Returns a new instance of Import.



19
20
21
22
23
24
 
# File 'lib/geocoder/us/import.rb', line 19

def initialize (filename, options)
  options[:create] = true
  super(filename, options)
  @sqlpath = options[:sql]
  create_tables
end

Class Method Details

.tables ⇒ Object 



11
12
13
 
# File 'lib/geocoder/us/import.rb', line 11

def self.tables
  @tables
end

Instance Method Details

#create_tables ⇒ Object 



165
166
167
168
169
170
171
172
173
174
175
176
177
 
# File 'lib/geocoder/us/import.rb', line 165

def create_tables
  uninit = false
  begin
    @db.execute("SELECT 0 FROM place")
  rescue SQLite3::SQLException
    uninit = true
  end
  if uninit
    log "creating tables\n"
    execute_script "create.sql"
    post_create
  end
end

#execute_batch(*args) ⇒ Object 



37
38
39
 
# File 'lib/geocoder/us/import.rb', line 37

def execute_batch (*args)
  @db.execute_batch(*args) 
end

#execute_script(file) ⇒ Object 



41
42
43
44
45
46
 
# File 'lib/geocoder/us/import.rb', line 41

def execute_script (file)
  if File.expand_path(file) != file
    file = File.join(@sqlpath, file)
  end    
  execute_batch File.open(file).read
end

#import_path(path) ⇒ Object 



138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
 
# File 'lib/geocoder/us/import.rb', line 138

def import_path (path)
  log "\n#{path}: "
  execute_script "setup.sql"
  @db.transaction do
    tables.each do |table, glob|
      file = Dir[File.join(path, glob)][0]
      next unless file
      if file =~ /\.zip$/io
        import_zip file, table
      else
        import_shapefile file, table
      end
    end
  end
  execute_script "convert.sql"
end

#import_tree(root) ⇒ Object 



155
156
157
158
159
160
161
162
163
 
# File 'lib/geocoder/us/import.rb', line 155

def import_tree (root)
  if !Dir[File.join(root, tables.values[0])].empty?
      import_path root
  else
    Dir[File.join(root, "*")].sort.each do |file|
      import_tree file if File.directory? file
    end
  end
end

#import_zip(zipfile, table) ⇒ Object 



120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
# File 'lib/geocoder/us/import.rb', line 120

def import_zip (zipfile, table)
  make_temp_dir do |tmpdir|
    unpack_zip zipfile, tmpdir
    basename = File.join(tmpdir, File.basename(zipfile))[0..-5]
    shpfile = basename + ".shp"
    dbffile = basename + ".dbf"
    if File.exists? shpfile
      log "#{table} "
      insert_shapefile shpfile, table
    elsif File.exists? dbffile
      log "#{table} "
      insert_dbf dbffile, table
    else
      log "\nNOT FOUND: #{shpfile}\n"
    end
  end 
end

#insert_csv(file, table, delimiter = "|") ⇒ Object 



89
90
91
92
93
94
95
 
# File 'lib/geocoder/us/import.rb', line 89

def insert_csv (file, table, delimiter="|")
  st = nil
  File.open(file).readlines.each do |line|
    attrs = line.chomp.split(delimiter)
    insert_data st, table, attrs
  end
end

#insert_data(st, table, attrs) ⇒ Object 



62
63
64
65
66
67
68
 
# File 'lib/geocoder/us/import.rb', line 62

def insert_data (st, table, attrs)
  unless st
   values = placeholders_for attrs
   st = @db.prepare("INSERT INTO #{table} VALUES (#{values});")
  end
  st.execute(attrs)
end

#insert_dbf(file, table) ⇒ Object 



78
79
80
81
82
83
84
85
86
87
 
# File 'lib/geocoder/us/import.rb', line 78

def insert_dbf (file, table)
  st = nil
  GeoRuby::Shp4r::Dbf::Reader.open(file) do |dbf|
    fields  = dbf.fields.map {|f| f.name}
    dbf.rows.each do |record|
      attrs = fields.map {|f| record[f]}
      insert_data st, table, attrs
    end
  end
end

#insert_shapefile(file, table) ⇒ Object 



70
71
72
73
74
75
76
 
# File 'lib/geocoder/us/import.rb', line 70

def insert_shapefile (file, table)
  st = nil
  load_features(file) do |attrs, geom|
    attrs << SQLite3::Blob.new(geom) if geom
    insert_data st, table, attrs
  end
end

#load_features(file) ⇒ Object 



48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/geocoder/us/import.rb', line 48

def load_features (file)
  dataset = GeoRuby::Shp4r::ShpFile.open(file)
  fields  = dataset.fields.map {|f| f.name}
  dataset.each do |record|
    attrs = fields.map {|f| record.data[f]}
    geom = record.geometry
    geom = geom.geometries[0] \
      if geom.kind_of? GeoRuby::SimpleFeatures::GeometryCollection
    points = geom.points.map {|pt| [pt.x, pt.y].map {|i| (i*1000000).to_i}}
    coords = points.flatten.pack("V*")
    yield attrs, coords
  end
end

#log(*args) ⇒ Object 



26
27
28
 
# File 'lib/geocoder/us/import.rb', line 26

def log (*args)
  $stderr.print *args
end

#make_temp_dir(cleanup = true) ⇒ Object 



97
98
99
100
101
102
103
104
105
106
107
108
109
 
# File 'lib/geocoder/us/import.rb', line 97

def make_temp_dir (cleanup=true)
  path = File.join(Dir.tmpdir, "geocoder-#{$$}")
  FileUtils.mkdir_p path
  if block_given?
    begin
      yield path
    ensure
      FileUtils.rm_r(path) if cleanup
    end
  else
    path
  end
end

#post_create ⇒ Object 



179
180
 
# File 'lib/geocoder/us/import.rb', line 179

def post_create
end

#spin ⇒ Object 



30
31
32
33
34
35
 
# File 'lib/geocoder/us/import.rb', line 30

def spin
  @spin ||= 0
  log "|/-\\"[@spin/100..@spin/100]+"\010" if @spin % 100 == 0
  @spin += 1
  @spin %= 400
end

#tables ⇒ Object 



15
16
17
 
# File 'lib/geocoder/us/import.rb', line 15

def tables
  self.class.tables
end

#unpack_zip(file, path) ⇒ Object 



111
112
113
114
115
116
117
118
 
# File 'lib/geocoder/us/import.rb', line 111

def unpack_zip (file, path)
  # log "- unpacking #{file}"
  Zip::ZipFile.open(file).each do |entry|
    target = File.join(path, entry.name)
    # log "  - #{target}"
    entry.extract target
  end
end