Module: Hdfs

Includes:

Java

Defined in:

lib/hdfs_jruby.rb,
lib/hdfs_jruby/file.rb,
lib/hdfs_jruby/version.rb

Defined Under Namespace

Classes: Configuration, File, FileSystem, FsPermission, Path

Constant Summary

JAR_PATTERN_0_20 =

"hadoop-core-*.jar"

HADOOP_HOME =

VERSION =

"0.0.7"

Class Method Summary

• ._conv(stat) ⇒ Object
• ._path(path) ⇒ Object
• .connectAsUser(user) ⇒ Object
• .delete(path, r = false) ⇒ Object

  delete.

• .directory?(path) ⇒ Boolean

  True: directory, false: file.

• .exists?(path) ⇒ Boolean
• .file?(path) ⇒ Boolean

  True: file, false: directory.

• .get(remote, local) ⇒ Object

  get file or directory from hdfs.

• .get_home_directory ⇒ Object
• .get_working_directory ⇒ Object
• .list(path, opts = {}) ⇒ Object
• .ls(path) ⇒ Array

  ls.

• .mkdir(path) ⇒ Object

  make directory.

• .move(src, dst) ⇒ Object
• .put(local, remote) ⇒ Object

  put file or directory to hdfs.

• .set_owner(path, owner, group) ⇒ Object
• .set_permission(path, perm) ⇒ Object
• .set_working_directory(path) ⇒ Object
• .size(path) ⇒ Integer

  File size.

Class Method Details

._conv(stat) ⇒ Object

# File 'lib/hdfs_jruby.rb', line 218

def _conv(stat)
  file_info = {}
  file_info['path'] = stat.getPath.to_s
  file_info['length'] = stat.getLen.to_i
  file_info['modificationTime'] = stat.getModificationTime.to_i
  file_info['owner'] = stat.getOwner.to_s
  file_info['group'] = stat.getGroup.to_s
  file_info['permission'] = stat.getPermission.toShort.to_i
  file_info['type'] = !stat.isDir ? 'FILE': 'DIRECTORY'
  return file_info
end

._path(path) ⇒ Object

# File 'lib/hdfs_jruby.rb', line 211

def _path(path)
  if path.nil?
    raise "path is nil"
  end
  Path.new(path)
end

.connectAsUser(user) ⇒ Object

# File 'lib/hdfs_jruby.rb', line 50

def connectAsUser(user)
  uri =  Hdfs::FileSystem.getDefaultUri(@conf)
  @fs.close if ! @fs.nil?
  @fs = Hdfs::FileSystem.get(uri, @conf, user)
end

.delete(path, r = false) ⇒ Object

delete

Parameters:

• path (String)
• r (Boolean) (defaults to: false) —

  recursive false or true (default: false)

# File 'lib/hdfs_jruby.rb', line 128

def delete(path, r=false)
  @fs.delete(_path(path), r)
end

.directory?(path) ⇒ Boolean

Returns true: directory, false: file.

Returns:

• (Boolean) —

  true: directory, false: file

# File 'lib/hdfs_jruby.rb', line 138

def directory?(path)
  @fs.isDirectory(_path(path))
end

.exists?(path) ⇒ Boolean

Parameters:

• path (String)

Returns:

• (Boolean)

# File 'lib/hdfs_jruby.rb', line 114

def exists?(path)
  @fs.exists(_path(path))
end

.file?(path) ⇒ Boolean

Returns true: file, false: directory.

Returns:

• (Boolean) —

  true: file, false: directory

# File 'lib/hdfs_jruby.rb', line 133

def file?(path)
  @fs.isFile(_path(path))
end

.get(remote, local) ⇒ Object

get file or directory from hdfs

Parameters:

• remote (String) —

  surouce (hdfs path)

• local (String) —

  destination (local path)

# File 'lib/hdfs_jruby.rb', line 163

def get(remote, local)
  @fs.copyToLocalFile(Path.new(remote), Path.new(local))
end

.get_home_directory ⇒ Object

# File 'lib/hdfs_jruby.rb', line 167

def get_home_directory()
  @fs.getHomeDirectory()
end

.get_working_directory ⇒ Object

# File 'lib/hdfs_jruby.rb', line 171

def get_working_directory()
  @fs.getWorkingDirectory()
end

.list(path, opts = {}) ⇒ Object

# File 'lib/hdfs_jruby.rb', line 89

def list(path, opts={})
  use_glob = opts[:glob] ? true : false
  p = _path(path)

  list = nil
  if use_glob
    list = @fs.globStatus(p)
  else
    list = @fs.listStatus(p)
  end
    
  if ! block_given?
    ret_list = []
    list.each do | stat |
      ret_list << _conv(stat)
    end
    return ret_list
  else
    list.each do | stat |
      yield _conv(stat)
    end
  end
end

.ls(path) ⇒ Array

ls

Parameters:

• path (String)

Returns:

• (Array) —

  match path list

# File 'lib/hdfs_jruby.rb', line 59

def ls(path)
  p = _path(path)
  list = @fs.globStatus(p)
  return [] if list.nil?

  ret_list = []
  list.each do |stat|
    if stat.isDir
      sub_list = @fs.listStatus(stat.getPath)
      next if sub_list.nil?
      
      sub_list.each do | s |
        if block_given?
          yield _conv(s)
        else
          ret_list << _conv(s)
        end
      end
    else
      if block_given?
        yield _conv(stat)
      else
        ret_list << _conv(stat)
      end
    end
  end
  ret_list if ! block_given?
end

.mkdir(path) ⇒ Object

make directory

Parameters:

• path (String)

# File 'lib/hdfs_jruby.rb', line 149

def mkdir(path)
  @fs.mkdirs(_path(path))
end

.move(src, dst) ⇒ Object

Parameters:

• src (String) —

  hdfs source path

• dst (String) —

  hdfs destination path

# File 'lib/hdfs_jruby.rb', line 120

def move(src, dst)
  @fs.rename(Path.new(src), Path.new(dst))
end

.put(local, remote) ⇒ Object

put file or directory to hdfs

Parameters:

• local (String) —

  surouce (local path)

• remote (String) —

  destination (hdfs path)

# File 'lib/hdfs_jruby.rb', line 156

def put(local, remote)
  @fs.copyFromLocalFile(Path.new(local), Path.new(remote))
end

.set_owner(path, owner, group) ⇒ Object

Parameters:

• path (String)
• owner (String)
• group (String)

# File 'lib/hdfs_jruby.rb', line 188

def set_owner(path, owner, group)
  @fs.setOwner(_path(path), owner, group)
end

.set_permission(path, perm) ⇒ Object

Parameters:

• path (String)
• perm (Integer) —

  permission

# File 'lib/hdfs_jruby.rb', line 181

def set_permission(path, perm)
  @fs.setPermission(_path(path), org.apache.hadoop.fs.permission.FsPermission.new(perm))
end

.set_working_directory(path) ⇒ Object

# File 'lib/hdfs_jruby.rb', line 175

def set_working_directory(path)
  @fs.setWorkingDirectory(_path())
end

.size(path) ⇒ Integer

Returns file size.

Returns:

• (Integer) —

  file size

# File 'lib/hdfs_jruby.rb', line 143

def size(path)
  @fs.getFileStatus(_path(path)).getLen()
end