Module: Swineherd::FileSystem

Defined in:
lib/swineherd-fs.rb

Constant Summary collapse

HDFS_SCHEME_REGEXP = 
/^hdfs:\/\//
S3_SCHEME_REGEXP = 
/^s3n?:\/\//
FILESYSTEMS = 
{
  'file' => Swineherd::LocalFileSystem,
  'hdfs' => Swineherd::HadoopFileSystem,
  's3'   => Swineherd::S3FileSystem,
  's3n'  => Swineherd::S3FileSystem
}

Class Method Summary collapse

Class Method Details

.cp(srcpath, destpath) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
 
# File 'lib/swineherd-fs.rb', line 66

def self.cp(srcpath,destpath)
  src_fs  = scheme_for(srcpath)
  dest_fs = scheme_for(destpath)
  Swineherd.logger.info "#cp - #{src_fs} --> #{dest_fs}"
  if(src_fs.eql?(dest_fs))
    self.get(src_fs).cp(srcpath,destpath)
  elsif src_fs.eql?(:file)
    self.get(dest_fs).copy_from_local(srcpath,destpath)
  elsif dest_fs.eql?(:file)
    self.get(src_fs).copy_to_local(srcpath,destpath)
  else #cp between s3/s3n and hdfs can be handled by Hadoop:FileUtil in HadoopFileSystem
    self.get(:hdfs).cp(srcpath,destpath)
  end
end

.exists?(path) ⇒ Boolean

Returns:

  • (Boolean) 


60
61
62
63
64
 
# File 'lib/swineherd-fs.rb', line 60

def self.exists?(path)
  fs = self.get(scheme_for(path))
  Swineherd.logger.info "#exists? - #{fs.class} for '#{path}'"
  fs.exists?(path)
end

.get(scheme, *args) ⇒ Object

A factory function that returns an instance of the requested class



52
53
54
55
56
57
58
 
# File 'lib/swineherd-fs.rb', line 52

def self.get scheme, *args
  begin
    FILESYSTEMS[scheme.to_s].new *args
  rescue NoMethodError => e
    raise "Filesystem with scheme #{scheme} does not exist.\n #{e.message}"
  end
end