Module: Apollo::Helper::Mongo

Defined in:
lib/apollo_crawler/helper/mongo_helper.rb

Class Method Summary collapse

Class Method Details

.connect(conn, opts = {}) ⇒ Object 



29
30
31
32
33
34
35
36
37
38
39
40
41
 
# File 'lib/apollo_crawler/helper/mongo_helper.rb', line 29

def self.connect(conn, opts={})
	if(opts[:verbose])
		puts "MongoDB connecting - '#{conn.inspect}"
	end

	res = ::Mongo::Connection.new(conn['host'])

	if(opts[:verbose])
		puts "MongoDB connected: #{res.inspect}"
	end

	return res
end

.csv_bulk_insert(path, model, bulk_size, validate = false, &block) ⇒ Object 



43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
 
# File 'lib/apollo_crawler/helper/mongo_helper.rb', line 43

def self.csv_bulk_insert(path, model, bulk_size, validate=false, &block)
	batch = []
	
	CSV.foreach(path) do |row|
		res = nil
		if block_given?
			res = yield row
		end

		if res.nil? == false
			if(!validate || model.where(res).length == 0)
				batch << res
			end
		end

		if((batch.length % bulk_size) == 0)
			# puts "Inserting batch '#{batch.inspect}'"
			
			model.collection.insert(batch)
			batch.clear
		end
	end

	if batch.empty? == false
		model.collection.insert(batch)
		batch.clear
	end
end