Module: NumRu::GPhys::NetCDF_IO

Defined in:

lib/numru/gphys/gphys_netcdf_io.rb

Class Method Summary

• .__files2varray(files, varname, dim = nil) ⇒ Object
• .__files_dim_matching(files, varname) ⇒ Object

  – re-arrange the array of files and check some inconsistencies.

• .__interpret_files(files, varname) ⇒ Object
• .__to_na_of_netcdf(files) ⇒ Object
• .def_var(file, name, ntype, dimensions, vary = nil) ⇒ Object
• .each_along_dims_write(gphyses, files, *loopdims, &block) ⇒ Object
• .is_a_NetCDF?(filename) ⇒ Boolean
• .open(files, varname) ⇒ Object
• .var_names(files) ⇒ Object
• .var_names_except_coordinates(files) ⇒ Object
• .write(file, gphys, name = nil) ⇒ Object
• .write_grid(file, grid_or_gphys) ⇒ Object

Class Method Details

.__files2varray(files, varname, dim = nil) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 523

def __files2varray( files, varname, dim=nil )
	if files.is_a?(NetCDF)
	  # Single file. Returns a VArrayNetCDF. dim is ignored.
	  file = files
	  var = file.var( varname )
	  raise "variable '#{varname}' not found in #{file.inspect}" if !var
	  VArrayNetCDF.new( var )
	elsif files.is_a?(NArray)
	  # Suppose that files is a NArray of NetCDF. Returns a VArrayCompsite.
	  if dim.is_a?(Integer) && dim>=0 && dim<files.rank
	    files = files[ *([0]*dim+[true]+[0]*(files.rank-dim-1)) ]
	  end
	  varys = NArray.object( *files.shape )
	  for i in 0...files.length
	    var = files[i].var( varname )
	    raise "variable '#{varname}' not found in #{files[i].path}" if !var
	    varys[i] = VArrayNetCDF.new( var )
	  end
	  if files.length != 1
	    VArrayComposite.new( varys )
	  else
	    varys[0]
	  end
	else
	  raise TypeError, "not a NetCDF or NArray"
	end
end

.__files_dim_matching(files, varname) ⇒ Object

–

re-arrange the array of files and check some inconsistencies

e.g.)
   files = NArray[ file0.nc, file1.nc, file2.nc, file3.nc ]
   The files have a variable, u(lat,lev,time),
   which is tiled to four parts as the followings:
              |    lat  |     lev    | time |
     file0.nc | -90...0 | 1000...500 | 0..1 |
     file1.nc |   0..90 | 1000...500 | 0..1 |
     file2.nc | -90...0 |  500..100  | 0..1 |
     file3.nc |   0..90 |  500..100  | 0..1 |

   Then the output is
      NArray[ [ [ file0.nc, file1.nc ],
                [ file2.nc, file3.nc ] ] ].

The order of the elements of the input array is arbitrary.

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 588

def __files_dim_matching( files, varname )
	# files: NArray of NetCDF

	return files if files.length == 1

	# < read the first file and check its rank >

	file0 = files[0]
	ncvar0 = file0.var(varname)
	if files.rank > ncvar0.rank
	  raise ArgumentError, "rank of files > rank of data"
	end
	convention = NetCDF_Conventions.find( ncvar0.file )
	axposnames = convention::coord_var_names(ncvar0)

	#< find correspoding dimensions >

  rank = axposnames.length
  cvals = Array.new(rank){ Array.new }
  sign = Array.new(rank)
  units = Array.new(rank)
  files.each do |file|
    axposnames.each_with_index do |axposname, i|
      nvax = file.var(axposname)
      raise "No coordinate variable: #{axposname}' not found" if !nvax
      axpos = VArrayNetCDF.new( nvax )
      axv0 = axpos.val[0]
      if units[i]
        un = axpos.units
        axv0 = un.convert2(axv0,units[i]) unless un == units[i]
      else # the first file
        units[i] = axpos.units
        sign[i] = (axpos.val[-1] >= axv0 ? 1 : -1)
      end
      cvals[i].push axv0
    end
  end
  # For the above example,
  # cvals = [ [  -90,    0, -90,   0 ],
  #           [ 1000, 1000, 500, 500 ],
  #           [    0,    0,   0,   0 ] ]
  # sign =  [ 1, -1, 1 ]

  axs = Array.new(rank)
  cvals.each_with_index do |cval, i|
    axs[i] = cval.uniq
    axs[i].sort!{|a,b| (a <=> b)*sign[i] }
  end
  # For the above example,
  # axs   = [ [ -90, 0 ], [ 1000, 500 ], [ 0 ] ]
  files_new = NArray.object( *axs.map{|ax| ax.length} )
  # For the above example,
  # shape of the files_new is [2, 2, 1]
  files.length.times do |n|
    file = files[n]
    idx = Array.new(rank)
    rank.times do |i|
      axv0 = cvals[i][n]
      idx[i] = axs[i].index(axv0)
    end
    # For the above example,
    # idx =
    #       [ 0, 0, 0 ] for n==0
    #       [ 1, 0, 0 ] for n==1
    #       [ 0, 1, 0 ] for n==2
    #       [ 1, 1, 0 ] for n==3
    files_new[*idx] = file
  end
  if files_new.eq(nil).count_true > 0
    raise "No dimension correspondence found"
  end
  files = files_new

	files
end

.__interpret_files(files, varname) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 551

def __interpret_files( files, varname )
	case files
	when NetCDF
	  ncvar0 = files.var( varname )
	when String
	  files = NetCDF.open(files)
	  ncvar0 = files.var( varname )
	when NArray, Array, Regexp
	  files = NArray[ *files ] if files.is_a?(Array)
	  files = GPhys::NetCDF_IO.__to_na_of_netcdf( files )
	  files = GPhys::NetCDF_IO.__files_dim_matching( files, varname )
	  ncvar0 = files[0].var( varname )
	else
	  raise TypeError, "argument files: not a NetCDF, String, NArray, Array, or Regexp"
	end
	[files, ncvar0]
end

.__to_na_of_netcdf(files) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 664

def __to_na_of_netcdf( files )
	case files
	when NArray
	  case files[0]
	  when NetCDF
	    files.each{|f|
 raise TypeError, "non-NetCDF obj included" if !f.is_a?(NetCDF)
	    }
	  when String
	    files.each{|f|
 raise TypeError, "non-String obj included" if !f.is_a?(String)
	    }
	    for i in 0...files.length
 files[i] = NetCDF.open(files[i])
	    end
	  end
	when Regexp
	  pat = files
	  if  /^(.*)\\?\/(.*)$/ =~ (pat.source)
	    d=$1
	    f=$2
	    dir = d.gsub(/\\/,'') + '/'
	    pat = Regexp.new(f)
	  else
	    dir = './'
	  end
	  match_data = Array.new
	  fnames = Array.new
	  first_time = true
	  Dir.open(dir).each{|fn| 
	    if pat =~ fn
 fnames.push(fn)
 ary = Regexp.last_match.to_a
 ary.shift  # ==> ary == [$1,$2,...]
        ary.each_with_index{|v,i|
		match_data[i] = Array.new if !match_data[i]
		match_data[i].push(v) if !match_data[i].include?(v)
 }
 if first_time
		body = fn

		first_time = false
 end
	    end
	  }
	  if match_data.length == 0
	    raise ArgumentError, "found no files that matches #{files.source}"
	  end
	  match_data.each{|ary| ary.sort!}
	  shape = match_data.collect{|ary| ary.length}
	  files = NArray.object(*shape)
	  fnames.each{|fn|
	    pat =~ fn
	    ary = Regexp.last_match.to_a
	    ary.shift  # ==> ary == [$1,$2,...]
      index = Array.new        
      ary.each_with_index{|v,i|
 index[i] = match_data[i].index(v)
	    }
	    files[*index] = NetCDF.open(dir+fn)
	  }
	else
	  raise TypeError, "unsupported type #{pat.class}"
	end
	files
end

.def_var(file, name, ntype, dimensions, vary = nil) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 439

def def_var(file, name, ntype, dimensions, vary=nil)
	VArrayNetCDF.def_var(file, name, ntype, dimensions, vary)
end

.each_along_dims_write(gphyses, files, *loopdims, &block) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 457

def each_along_dims_write(gphyses, files, *loopdims, &block)
	files = [files] if !files.is_a?(Array)
	files.each do |fl|
	  NetCDF_Conventions.add_history(fl, "#{File.basename($0)}")
	end
	IO_Common::each_along_dims_write(gphyses, files, loopdims, NetCDF_IO,
&block)
end

.is_a_NetCDF?(filename) ⇒ Boolean

Returns:

• (Boolean)

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 304

def is_a_NetCDF?(filename)
  file = nil
  begin
    file = File.open(filename,"rb")
    str = file.read(3)
  ensure
    file.close if file
  end
  return str=="CDF"
end

.open(files, varname) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 315

def open(files, varname)
	files, ncvar0 = __interpret_files( files, varname )
	data = __files2varray( files, varname )
	convention = NetCDF_Conventions.find( ncvar0.file )
	axposnames = convention::coord_var_names( ncvar0 )
	rank = data.rank
	bare_index = [ false ] * rank # will be true if coord var is not found

  axes = Array.new
	var_names = ncvar0.file.var_names
	for i in 0...rank
	  if var_names.include?(axposnames[i])
	    axpos = __files2varray( files, axposnames[i], i )
	  else
	    bare_index[i]=true
	    na = NArray.float(ncvar0.shape_current[i]).indgen!
	    axpos = VArray.new( na )
      axpos.name = axposnames[i]         # added by S. OTSUKA
	  end
	  cell_center, cell_bounds_name = convention::cell_center?( axpos )
	  cell_bounds, cell_center_name = convention::cell_bounds?( axpos )
	  cell = cell_center || cell_bounds
	  axis = Axis.new(cell,bare_index[i])
	  if !cell
	    axis.set_pos( axpos )
	  else
	    if cell_center
 if cell_bounds_name
		varray_cell_bounds = __files2varray(files, cell_bounds_name, i)
		axis.set_cell(axpos, varray_cell_bounds).set_pos_to_center
 else
		p "cell bounds are guessed"
		axis.set_cell_guess_bounds(axpos).set_pos_to_center
 end
	    else  # then it is cell_bounds
 if cell_center_name
		varray_cell_center = __files2varray(files, cell_center_name, i)
		axis.set_cell(varray_cell_center, axpos).set_pos_to_bounds
 else
		p "cell center is guessed"
		axis.set_cell_guess_center(axpos).set_pos_to_bounds
 end
	    end
	  end
	  
	  aux = convention::aux_var_names( axpos )
	  if aux
	    aux.each{|k,varname| 
 axis.set_aux( k, __files2varray(files,varname,i) )
	    }
	  end
	  
	  axes[i] = axis
	end

	grid = Grid.new( *axes )

  if (nms = convention::assoc_coord_names(data)) && !(nms.include?(varname))
    if files.is_a?(NArray)
      assoc_crds = nms.collect{|nm| 
        index_files_assoc = [0]*(files.rank) 
        tmp_assoc = GPhys::NetCDF_IO.open(files[0],nm)
        tmp_assoc.rank.times{|i_assoc|
          axes.length.times{|i_axes|
            if tmp_assoc.axis(i_assoc).name == axes[i_axes].name
              index_files_assoc[i_axes] = true
            end
          }
        }
        GPhys::NetCDF_IO.open(files[*(index_files_assoc)],nm)
      }
    else
      assoc_crds = nms.collect{|nm| GPhys::NetCDF_IO.open(files,nm)}
    end
    grid.set_assoc_coords(assoc_crds)
  end

	GPhys.new(grid,data)
end

.var_names(files) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 466

def var_names(files)
	case files
	when NetCDF
    file = files
	  opened = true
	when String
	  file = NetCDF.open(files)
	  opened = false
	when NArray, Array, Regexp
	  files = NArray[ *files ] if files.is_a?(Array)
	  files = GPhys::NetCDF_IO.__to_na_of_netcdf( files )
	  files = GPhys::NetCDF_IO.__files_dim_matching( files, varname )
	  file = files[0]
	  files.length.times{|i|
	    files[i].close unless i==0
    }
	  opened = false
	else
	  raise TypeError, "argument files: not a NetCDF, String, NArray, Array, or Regexp"
	end
	var_names = file.var_names
	file.close unless opened
	return var_names
end

.var_names_except_coordinates(files) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 490

def var_names_except_coordinates(files)
	case files
	when NetCDF
    file = files
	  opened = true
	when String
	  file = NetCDF.open(files)
	  opened = false
	when NArray, Array, Regexp
	  files = NArray[ *files ] if files.is_a?(Array)
	  files = GPhys::NetCDF_IO.__to_na_of_netcdf( files )
	  files = GPhys::NetCDF_IO.__files_dim_matching( files, varname )
	  file = files[0]
	  files.length.times{|i|
	    files[i].close unless i==0
    }
	  opened = false
	else
	  raise TypeError, "argument files: not a NetCDF, String, NArray, Array, or Regexp"
	end
  var_names = Array.new
  file.var_names.each{|name|
    f,var = __interpret_files( files, name )
    if var.rank>1 || var.name!=var.dim_names[0]
      var_names.push(name)
    end
  }
	file.close unless opened
  return var_names
end

.write(file, gphys, name = nil) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 443

def write(file, gphys, name=nil)
  write_grid(file, gphys)
	NetCDF_Conventions.add_history(file, "#{File.basename($0)} wrote "+gphys.name)
  data = gphys.data
  if gphys.has_assoccoord?
    if data.class != VArray
      data = data.copy
    end
    data.put_att("coordinates",gphys.assoccoordnames.join(' '))
  end
	VArrayNetCDF.write(file, data, name, gphys.axnames)
	nil
end

.write_grid(file, grid_or_gphys) ⇒ Object

# File 'lib/numru/gphys/gphys_netcdf_io.rb', line 395

def write_grid(file, grid_or_gphys)
	newaxes = Array.new
	(0...(grid_or_gphys.rank)).each{|i|
	  ax = grid_or_gphys.axis(i)
	  dimname = ax.pos.name
	  length = ax.pos.length
	  isfx = 0
	  altdimnames = Hash.new
	  newax = ax.collect{ |va|
	    if va.length == length
 dimnames = [dimname]
	    else
 if (nm=altdimnames[va.length])
		dimnames = [nm]
 else
		dimnames = [ (altdimnames[va.length] = dimname+isfx.to_s) ]
		isfx += 1
 end
	    end
	    if !(already=file.var(va.name))
 newva = VArrayNetCDF.write(file, va, nil, dimnames )
	    else
        #< var with the same name exists --> check it >
        if va.shape_current != already.shape_current
		raise "#{va.name} exisits but the shape is different" 
 #elsif va[0].val != already[0].val #(not allowed in the def mode)
 #	raise "#{va.name} exisits but the first element doesn't agree" 
        else
		newva = VArrayNetCDF.new(already)
 end
	    end
	    newva
	  }
	  newaxes.push( newax )
	}
  if acnms = grid_or_gphys.assoccoordnames
    acnms.each{|nm| 
      acgp = grid_or_gphys.assoc_coord_gphys(nm)
      VArrayNetCDF.write(file, acgp.data, nil, acgp.axnames)
    }
  end
	Grid.new( *newaxes )
end