Module: NSWTopo

Extended by:
NSWTopo, Log
Included in:
NSWTopo
Defined in:
lib/nswtopo.rb,
lib/nswtopo/os.rb,
lib/nswtopo/log.rb,
lib/nswtopo/map.rb,
lib/nswtopo/svg.rb,
lib/nswtopo/zip.rb,
lib/nswtopo/font.rb,
lib/nswtopo/layer.rb,
lib/nswtopo/chrome.rb,
lib/nswtopo/config.rb,
lib/nswtopo/dither.rb,
lib/nswtopo/safely.rb,
lib/nswtopo/archive.rb,
lib/nswtopo/formats.rb,
lib/nswtopo/gis/dem.rb,
lib/nswtopo/gis/gps.rb,
lib/nswtopo/version.rb,
lib/nswtopo/commands.rb,
lib/nswtopo/layer/grid.rb,
lib/nswtopo/layer/spot.rb,
lib/nswtopo/formats/kmz.rb,
lib/nswtopo/formats/pdf.rb,
lib/nswtopo/formats/svg.rb,
lib/nswtopo/formats/zip.rb,
lib/nswtopo/gis/geojson.rb,
lib/nswtopo/gis/gps/gpx.rb,
lib/nswtopo/gis/gps/kml.rb,
lib/nswtopo/commands/add.rb,
lib/nswtopo/formats/gemf.rb,
lib/nswtopo/formats/svgz.rb,
lib/nswtopo/gis/esri_hdr.rb,
lib/nswtopo/layer/import.rb,
lib/nswtopo/layer/labels.rb,
lib/nswtopo/layer/raster.rb,
lib/nswtopo/layer/relief.rb,
lib/nswtopo/gis/gdal_glob.rb,
lib/nswtopo/gis/shapefile.rb,
lib/nswtopo/layer/contour.rb,
lib/nswtopo/layer/control.rb,
lib/nswtopo/layer/feature.rb,
lib/nswtopo/layer/overlay.rb,
lib/nswtopo/tiled_web_map.rb,
lib/nswtopo/tree_indenter.rb,
lib/nswtopo/gis/projection.rb,
lib/nswtopo/commands/config.rb,
lib/nswtopo/commands/layers.rb,
lib/nswtopo/commands/scrape.rb,
lib/nswtopo/formats/mbtiles.rb,
lib/nswtopo/commands/inspect.rb,
lib/nswtopo/gis/arcgis/layer.rb,
lib/nswtopo/layer/vegetation.rb,
lib/nswtopo/gis/geojson/point.rb,
lib/nswtopo/layer/colour_mask.rb,
lib/nswtopo/layer/declination.rb,
lib/nswtopo/layer/mask_render.rb,
lib/nswtopo/gis/arcgis/service.rb,
lib/nswtopo/layer/labels/label.rb,
lib/nswtopo/gis/geojson/polygon.rb,
lib/nswtopo/layer/arcgis_raster.rb,
lib/nswtopo/layer/raster_import.rb,
lib/nswtopo/layer/raster_render.rb,
lib/nswtopo/layer/vector_render.rb,
lib/nswtopo/gis/arcgis/layer/map.rb,
lib/nswtopo/gis/arcgis/connection.rb,
lib/nswtopo/layer/labels/barriers.rb,
lib/nswtopo/gis/arcgis/layer/query.rb,
lib/nswtopo/gis/geojson/collection.rb,
lib/nswtopo/gis/geojson/line_string.rb,
lib/nswtopo/gis/geojson/multi_point.rb,
lib/nswtopo/layer/labels/convex_hull.rb,
lib/nswtopo/gis/arcgis/layer/renderer.rb,
lib/nswtopo/gis/geojson/multi_polygon.rb,
lib/nswtopo/layer/labels/convex_hulls.rb,
lib/nswtopo/layer/vector_render/cutout.rb,
lib/nswtopo/gis/arcgis/layer/statistics.rb,
lib/nswtopo/layer/vector_render/knockout.rb,
lib/nswtopo/gis/geojson/multi_line_string.rb

Defined Under Namespace

Modules: ArcGIS, ArcGISRaster, ColourMask, Config, Contour, Control, DEM, Declination, Dither, Feature, Font, Formats, GDALGlob, GeoJSON, Grid, Import, Labels, Log, MaskRender, OS, Overlay, Raster, RasterImport, RasterRender, Relief, SVG, Safely, Shapefile, Spot, TiledWebMap, VectorRender, Vegetation, Zip Classes: Archive, Chrome, ESRIHdr, GPS, Layer, Map, Projection, SVGFormatter, TreeIndenter, Version

Constant Summary collapse

PartialFailureError = 
Class.new RuntimeError
VERSION = 
Version["nswtopo 3.1.1"]
MIN_VERSION = 
Version["nswtopo 3.0"]

Constants included from Log

Log::FAILURE, Log::NEUTRAL, Log::SUCCESS, Log::UPDATE

Instance Method Summary collapse

Methods included from Log

log_abort, log_neutral, log_success, log_update, log_warn

Instance Method Details

#add(archive, *layers, after: nil, before: nil, replace: nil, overwrite: false, strict: false, **options) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
 
# File 'lib/nswtopo/commands/add.rb', line 2

def add(archive, *layers, after: nil, before: nil, replace: nil, overwrite: false, strict: false, **options)
  create_options = {
    after: Layer.sanitise(after),
    before: Layer.sanitise(before),
    replace: Layer.sanitise(replace),
    overwrite: overwrite,
    strict: strict
  }
  map = Map.load archive

  Enumerator.new do |yielder|
    while layers.any?
      layer, basedir = layers.shift
      path = Pathname(layer).expand_path(*basedir)
      case layer
      when /^controls\.(gpx|kml)$/i
        yielder << [path.basename(path.extname).to_s, "type" => "Control", "path" => path]
      when /\.(gpx|kml)$/i
        yielder << [path.basename(path.extname).to_s, "type" => "Overlay", "path" => path]
      when /\.(tiff?|png|jpg)$/i
        yielder << [path.basename(path.extname).to_s, "type" => "Import", "path" => path]
      when "contours"
        yielder << [layer, "type" => "Contour"]
      when "spot-heights"
        yielder << [layer, "type" => "Spot"]
      when "relief"
        yielder << [layer, "type" => "Relief"]
      when "grid"
        yielder << [layer, "type" => "Grid"]
      when "declination"
        yielder << [layer, "type" => "Declination"]
      when "controls"
        yielder << [layer, "type" => "Control"]
      when /\.yml$/i
        basedir ||= path.parent
        raise "couldn't find '#{layer}'" unless path.file?
        case contents = YAML.load(path.read)
        when Array
          contents.reverse.map do |item|
            Pathname(item.to_s)
          end.each do |relative_path|
            raise "#{relative_path} is not a relative path" unless relative_path.relative?
            layers.prepend [Pathname(relative_path).expand_path(path.parent).relative_path_from(basedir).to_s, basedir]
          end
        when Hash
          name = path.sub_ext("").relative_path_from(basedir).descend.map(&:basename).join(?.)
          yielder << [name, contents.merge("source" => path)]
        else
          raise "couldn't parse #{path}"
        end
      else
        path = Pathname("#{layer}.yml")
        raise "#{layer} is not a relative path" unless path.relative?
        basedir ||= layer_dirs.find do |root|
          path.expand_path(root).file?
        end
        layers.prepend [path.to_s, basedir]
      end
    end
  rescue YAML::Exception
    raise "couldn't parse #{path}"
  end.map do |name, params|
    params.merge! options.transform_keys(&:to_s)
    params.merge! Config[name] if Config[name]
    Layer.new(name, map, params)
  end.tap do |layers|
    raise OptionParser::MissingArgument, "no layers specified" unless layers.any?
    unless layers.one?
      raise OptionParser::InvalidArgument, "can't specify opacity when adding multiple layers" if options[:opacity]
      raise OptionParser::InvalidArgument, "can't specify data path when adding multiple layers" if options[:path]
    end
    map.add *layers, **create_options
  end
end

#config(layer = nil, **options) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
 
# File 'lib/nswtopo/commands/config.rb', line 2

def config(layer = nil, **options)
  path, resolution = options[:path], options[:resolution]
  layer = Layer.sanitise layer

  case
  when !layer
    raise OptionParser::InvalidArgument, "no layer name specified for path" if path
    raise OptionParser::InvalidArgument, "no layer name specified for resolution" if resolution
  when path || resolution
    Config.store layer, "path", path.to_s if path
    Config.store layer, "resolution", resolution if resolution
  end

  options.each do |key, value|
    case key
    when :chrome
      raise "chrome path is not an executable" unless value.executable? && !value.directory?
      Config.store key.to_s, value.to_s
    when :"layer-dir"
      raise "not a directory: %s" % value unless value.directory?
      Config.store key.to_s, value.to_s
    when *%i[labelling debug gpu versioning zlib-level knockout]
      Config.store key.to_s, value
    when :delete
      Config.delete *layer, value
    end
  end

  if options.empty?
    puts Config.to_str.each_line.drop(1)
    log_neutral "no configuration yet" if Config.empty?
  else
    Config.save
    log_success "configuration updated"
  end
end

#contours(archive, dem_path, **options) ⇒ Object 



77
78
79
 
# File 'lib/nswtopo/commands/add.rb', line 77

def contours(archive, dem_path, **options)
  add archive, "contours", **options, path: Pathname(dem_path)
end

#controls(archive, gps_path, **options) ⇒ Object

Raises:

  • (OptionParser::InvalidArgument) 


97
98
99
100
 
# File 'lib/nswtopo/commands/add.rb', line 97

def controls(archive, gps_path, **options)
  raise OptionParser::InvalidArgument, gps_path unless gps_path =~ /\.(gpx|kml)$/i
  add archive, "controls", **options, path: Pathname(gps_path)
end

#declination(archive, **options) ⇒ Object 



93
94
95
 
# File 'lib/nswtopo/commands/add.rb', line 93

def declination(archive, **options)
  add archive, "declination", **options
end

#delete(archive, *names, **options) ⇒ Object 



17
18
19
20
21
22
23
24
25
26
 
# File 'lib/nswtopo/commands.rb', line 17

def delete(archive, *names, **options)
  map = Map.load archive
  names.map do |name|
    Layer.sanitise name
  end.uniq.map do |name|
    name[?*] ? %r[^#{name.gsub(?., '\.').gsub(?*, '.*')}$] : name
  end.tap do |names|
    map.delete *names
  end
end

#grid(archive, **options) ⇒ Object 



89
90
91
 
# File 'lib/nswtopo/commands/add.rb', line 89

def grid(archive, **options)
  add archive, "grid", **options
end

#info(archive, **options) ⇒ Object

Raises:

  • (OptionParser::InvalidArgument) 


12
13
14
15
 
# File 'lib/nswtopo/commands.rb', line 12

def info(archive, **options)
  raise OptionParser::InvalidArgument, "one output option only" if options.slice(:json, :proj).length > 1
  puts Map.load(archive).info(**options)
end

#init(archive, **options) ⇒ Object 



8
9
10
 
# File 'lib/nswtopo/commands.rb', line 8

def init(archive, **options)
  puts Map.init(archive, **options)
end

#inspect(url_or_path, layer: nil, coords: nil, codes: nil, countwise: nil, **options) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
 
# File 'lib/nswtopo/commands/inspect.rb', line 2

def inspect(url_or_path, layer: nil, coords: nil, codes: nil, countwise: nil, **options)
  options[:geometry] = GeoJSON.multipoint(coords).bbox if coords

  case url_or_path
  when ArcGIS::Service
    source = ArcGIS::Service.new(url_or_path)
  when Shapefile::Source
    raise OptionParser::InvalidOption, "--id only applies to ArcGIS layers" if options[:id]
    raise OptionParser::InvalidOption, "--decode only applies to ArcGIS layers" if options[:decode]
    raise OptionParser::InvalidOption, "--codes only applies to ArcGIS layers" if codes
    source = Shapefile::Source.new(url_or_path)
    layer ||= source.only_layer
  else
    raise OptionParser::InvalidArgument, url_or_path
  end
  layer = source.layer(layer: layer, **options)

  case
  when codes
    TreeIndenter.new(layer.codes) do |level|
      level.map do |key, values|
        case key
        when Array
          code, value = key
          display_value = value.nil? || /[\t\n\r]/ === value ? value.inspect : value
          ["#{code} → #{display_value}", values]
        else
          ["#{key}:", values]
        end
      end
    end.each do |indents, info|
      puts indents.join << info
    end

  when fields = options[:fields]
    template = "%%%is │ %%%is │ %%s"
    TreeIndenter.new(layer.counts) do |counts|
      counts.group_by do |attributes, count|
        attributes.shift
      end.entries.select(&:first).map do |(name, value), counts|
        [[name, counts.sum(&:last), value], counts]
      end.sort do |((name1, count1, value1), counts1), ((name2, count2, value2), counts2)|
        next count2 <=> count1 if countwise
        value1 && value2 ? value1 <=> value2 : value1 ? 1 : value2 ? -1 : 0
      end
    end.map do |indents, (name, count, value)|
      next name, count.to_s, indents.join << (value.nil? || /[\t\n\r]/ === value ? value.inspect : value.to_s)
    end.transpose.tap do |names, counts, lines|
      template %= [names.map(&:size).max, counts.map(&:size).max] if names
    end.transpose.each do |row|
      puts template % row
    end

  else
    TreeIndenter.new(layer.info) do |hash|
      hash.map do |key, value|
        Hash === value ? ["#{key}:", value] : "#{key}: #{value}"
      end
    end.each do |indents, info|
      puts indents.join << info
    end
  end

rescue ArcGIS::Layer::NoLayerError, Shapefile::Layer::NoLayerError => error
  raise OptionParser::MissingArgument, error.message if codes || countwise || options.any?
  puts "layers:"
  TreeIndenter.new(source.layer_info, []).each do |indents, info|
    puts indents.join << info
  end
rescue ArcGIS::Renderer::TooManyFieldsError
  raise OptionParser::InvalidOption, "use less fields with --fields"
end

#layer_dirs ⇒ Object 



54
55
56
 
# File 'lib/nswtopo.rb', line 54

def layer_dirs
  @layer_dirs ||= Array(Config["layer-dir"]).map(&Pathname.method(:new)) << Pathname.pwd
end

#layers(state: nil) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
 
# File 'lib/nswtopo/commands/layers.rb', line 2

def layers(state: nil)
  paths = layer_dirs.grep_v(Pathname.pwd).flat_map do |directory|
    Array(state).inject(directory, &:/).glob("*")
  end.sort
  log_warn "no layers installed" if paths.none?

  TreeIndenter.new(paths) do |paths|
    paths.filter_map do |path|
      case
      when path.glob("**/*.yml").any?
        [path.basename.sub_ext(""), path.children.sort]
      when path.sub_ext("").directory?
      when path.extname == ".yml"
        path.basename.sub_ext("")
      end
    end
  end.each do |indents, name|
    puts [*indents, name].join
  end
end

#move(archive, name, **options) ⇒ Object

Raises:

  • (OptionParser::InvalidArgument) 


28
29
30
31
32
 
# File 'lib/nswtopo/commands.rb', line 28

def move(archive, name, **options)
  raise OptionParser::InvalidArgument, "only one of --before and --after allowed" if options[:after] && options[:before]
  raise OptionParser::MissingArgument, "--before or --after required" unless options[:after] || options[:before]
  Map.load(archive).move(name, **options)
end

#overlay(archive, gps_path, **options) ⇒ Object

Raises:

  • (OptionParser::InvalidArgument) 


102
103
104
105
 
# File 'lib/nswtopo/commands/add.rb', line 102

def overlay(archive, gps_path, **options)
  raise OptionParser::InvalidArgument, gps_path unless gps_path =~ /\.(gpx|kml)$/i
  add archive, gps_path, **options, path: Pathname(gps_path)
end

#relief(archive, dem_path, **options) ⇒ Object 



85
86
87
 
# File 'lib/nswtopo/commands/add.rb', line 85

def relief(archive, dem_path, **options)
  add archive, "relief", **options, path: Pathname(dem_path)
end

#render(archive, basename, *formats, overwrite: false, svg_path: nil, **options) ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
 
# File 'lib/nswtopo/commands.rb', line 34

def render(archive, basename, *formats, overwrite: false, svg_path: nil, **options)
  case
  when formats.any?
  when svg_path
    raise OptionParser::MissingArgument, "no output format specified"
  else
    formats << "svg"
  end

  formats.map do |format|
    Pathname(Formats === format ? "#{basename}.#{format}" : format)
  end.uniq.each do |path|
    format = path.extname.delete_prefix(?.)
    raise "unrecognised format: #{path}" if format.empty?
    raise "unrecognised format: #{format}" unless Formats === format
    raise "already a directory: #{path}" if path.directory?
    raise "file already exists: #{path}" if path.exist? && !overwrite
    raise "no such directory: #{path.parent}" unless path.parent.directory?
  end.tap do |paths|
    map = svg_path ? Map.from_svg(archive, svg_path) : Map.load(archive)
    map.render *paths, **options
  end
end

#scrape(url, path, coords: nil, name: nil, epsg: nil, paginate: nil, concat: nil, **options) ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/nswtopo/commands/scrape.rb', line 2

def scrape(url, path, coords: nil, name: nil, epsg: nil, paginate: nil, concat: nil, **options)
  flags  = %w[-skipfailures]
  flags += %W[-t_srs epsg:#{epsg}] if epsg
  flags += %W[-nln #{name}] if name

  format_flags = case path.to_s
  when Shapefile::Source then %w[-update -overwrite]
  when /\.sqlite3?$/     then %w[-f SQLite -dsco SPATIALITE=YES]
  when /\.db$/           then %w[-f SQLite -dsco SPATIALITE=YES]
  when /\.gpkg$/         then %w[-f GPKG]
  when /\.tab$/          then ["-f", "MapInfo File"]
  else                        ["-f", "ESRI Shapefile", "-lco", "ENCODING=UTF-8"]
  end

  options.merge! case path.to_s
  when /\.sqlite3?$/ then { mixed: concat, launder: true }
  when /\.db$/       then { mixed: concat, launder: true }
  when /\.gpkg$/     then { mixed: concat, launder: true }
  when /\.tab$/      then { }
  else                    { truncate: 10 }
  end

  options[:geometry] = GeoJSON.multipoint(coords).bbox if coords

  log_update "nswtopo: contacting server"
  layer = ArcGIS::Service.new(url).layer(**options)

  queue = Queue.new
  thread = Thread.new do
    while page = queue.pop
      *, status = Open3.capture3 *%W[ogr2ogr #{path} /vsistdin/], *flags, *format_flags, stdin_data: page.to_json
      format_flags = %w[-update -append]
      queue.close unless status.success?
    end
    status
  end

  total_features, percent = "%i feature%s", "%%.%if%%%%"
  Enumerator.new do |yielder|
    hold, ok, count = [], nil, 0
    layer.paged(per_page: paginate).tap do
      total_features %= [layer.count, (?s unless layer.count == 1)]
      percent %= layer.count < 1000 ? 0 : layer.count < 10000 ? 1 : 2
      log_update "nswtopo: retrieving #{total_features}"
    end.each do |page|
      log_update "nswtopo: retrieving #{percent} of #{total_features}" % [100.0 * (count += page.count) / layer.count]
      next hold << page if concat
      next yielder << page if ok
      next hold << page if page.all? do |feature|
        feature.properties.values.any?(&:nil?)
      end
      yielder << page
      ok = true
    end
    next hold.inject(yielder, &:<<) if ok && !concat
    next yielder << hold.inject(&:merge!) if hold.any?
  end.inject(queue) do |queue, page|
    queue << page
  rescue ClosedQueueError
    break queue
  end.close

  log_update "nswtop: saving #{total_features}"
  raise "error while saving features" unless thread.value&.success?
  log_success "saved #{total_features}"

rescue ArcGIS::Layer::NoLayerError
  raise OptionParser::InvalidArgument, "specify an ArcGIS layer in URL or with --layer"
rescue ArcGIS::Map::NoUniqueFieldError
  raise OptionParser::InvalidOption, "--unique required for this layer"
rescue ArcGIS::Renderer::NoGeometryError
  raise OptionParser::InvalidOption, "--coords not available for this layer"
rescue ArcGIS::Query::UniqueFieldError
  raise OptionParser::InvalidOption, "--unique not available for this layer"
rescue ArcGIS::Service::InvalidURLError
  raise OptionParser::InvalidArgument, url
end

#spot_heights(archive, dem_path, **options) ⇒ Object 



81
82
83
 
# File 'lib/nswtopo/commands/add.rb', line 81

def spot_heights(archive, dem_path, **options)
  add archive, "spot-heights", **options, path: Pathname(dem_path)
end