Class: MongoHadoop

Inherits:
Object
  • Object
show all
Defined in:
lib/mongo-hadoop/mapper.rb,
lib/mongo-hadoop/reducer.rb

Class Method Summary collapse

Class Method Details

.kvmap ⇒ Object 



16
17
18
19
20
21
22
23
24
25
26
27
28
 
# File 'lib/mongo-hadoop/mapper.rb', line 16

def self.kvmap
  kvinput = BSONKeyValueInput.new
  kvoutput = BSONKeyValueOutput.new

  kvinput.each do |key, value|
    mapped = yield key, value
    mapped = [mapped] unless profiles.respond_to(:each)

    mapped.each do |mapped|
      kvoutput.write mapped if mapped
    end
  end
end

.kvreduce ⇒ Object 



13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
 
# File 'lib/mongo-hadoop/reducer.rb', line 13

def self.kvreduce
  kvinput = BSONKeyValueInput.new
  kvoutput = BSONKeyValueOutput.new

  grouped = kvinput.inject(Hash.new) do |hash, pair|
    key, value = *pair
    hash[key] ||= []
    hash[key] << value
    hash
  end

  grouped.each do |key, values|
    kvoutput.write yield key, values
  end
end

.map ⇒ Object 



2
3
4
5
6
7
8
9
10
11
12
13
14
 
# File 'lib/mongo-hadoop/mapper.rb', line 2

def self.map
  input = BSONInput.new
  output = BSONOutput.new

  input.each do |doc|
    mapped = yield doc
    mapped = [mapped] unless mapped.respond_to?(:each)

    mapped.each do |mapped|
      output.write mapped if mapped
    end
  end
end

.reduce ⇒ Object 



2
3
4
5
6
7
8
9
10
11
 
# File 'lib/mongo-hadoop/reducer.rb', line 2

def self.reduce
  input = BSONInput.new
  output = BSONOutput.new
  
  grouped = input.group_by { |doc| doc['_id'] }

  grouped.each do |key, values|
    output.write yield key, values
  end
end