Module: HadoopDsl

Defined in:
lib/core.rb,
lib/util.rb,
lib/mapred_factory.rb,
lib/hadoop_dsl_client.rb

Defined Under Namespace

Modules: DslController, DslElement, HiveLike, LogAnalysis, WordCount Classes: BaseMapRed, BaseMapper, BaseMapperModel, BaseModel, BaseReducer, BaseReducerModel, BaseSetup, Client, MapRedFactory, MapperFactory, ReducerFactory, SetupFactory

Constant Summary collapse

Text =

Hadoop IO types

Text
IntWritable = 
IntWritable
@@file_bodies =

file body cache reading file in map/reduce cause critical issues!

{}

Class Method Summary collapse

Class Method Details

.dsl_init_script ⇒ Object 



8
9
10
 
# File 'lib/hadoop_dsl_client.rb', line 8

def self.dsl_init_script
  File.join(lib_path, "dsl_init.rb")
end

.lib_path ⇒ Object 



4
5
6
 
# File 'lib/hadoop_dsl_client.rb', line 4

def self.lib_path
  File.expand_path(File.dirname(__FILE__))
end

.read_file(file_name) ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
 
# File 'lib/util.rb', line 13

def self.read_file(file_name)
  # use if cached
  body = @@file_bodies[file_name] if @@file_bodies[file_name]

  # read as usual
  body = File.open(file_name).read rescue nil unless body

  # read from loadpath
  unless body
    $:.each do |path|
      body = File.open(File.join(path, file_name)).read rescue next
      break
    end
  end

  raise "cannot find file - #{file_name}" unless body

  # for cache
  @@file_bodies[file_name] = body
  body
end

.reset_dsl_file ⇒ Object 



35
36
37
 
# File 'lib/util.rb', line 35

def self.reset_dsl_file
  @@file_bodies = {}
end

.snake_case(str) ⇒ Object 



9
10
11
 
# File 'lib/util.rb', line 9

def self.snake_case(str)
  str.gsub(/\B[A-Z]/, '_\&').downcase
end