Class: FReCon::Scraper

Inherits:
Object show all
Defined in:
lib/frecon/scraper.rb

Overview

The default scraper scrapes other FReCon instances. To scrape a different source, a descendant scraper should be used.

Instance Method Summary collapse

Constructor Details

#initialize(base_uri) ⇒ Scraper

Returns a new instance of Scraper.



17
18
19
 
# File 'lib/frecon/scraper.rb', line 17

def initialize(base_uri)
	@base_uri = base_uri
end

Instance Method Details

#get(model = nil, query = {}) ⇒ Object

If no arguments are passed, will import the whole other database. If only one argument is passed, will import all of that model. If two arguments are passed, will import the models that match the query params.



59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
 
# File 'lib/frecon/scraper.rb', line 59

def get(model = nil, query = {})
	# Turns something like 'team' into Team.
	model = ('FReCon::' + model.capitalize).constantize if model.is_a?(String)

	# The route name for the model branch.
	route_name = model.name.gsub(/FReCon::/, '').downcase.pluralize if model

	if !model && query.empty?
		type = :dump
		data = HTTP.get("http://#{@base_uri}/dump")
	elsif model && query.empty?
		type = :index
		data = HTTP.get("http://#{@base_uri}/#{route_name}")
	else
		type = :single
		data = HTTP.get("http://#{@base_uri}/#{route_name}", { query: query })
	end

	read data.body, model: model, type: type
end

#read(data, context = {}) ⇒ Object

Reads and imports a data string. Determines what to do with information in the ‘context` hash.



23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/frecon/scraper.rb', line 23

def read(data, context = {})
	# `data` will be a string, so we need to convert it from JSON.
	data = JSON.parse(data)

	if context[:type] == :single && data.empty?
		return [404, 'Could not find a model with that query.']
	elsif
		puts 'Just a heads up: you are importing an empty array of data.'
	end

	# Here we want `context` to tell us what model we are making.
	if context[:model]
		result = context[:model].controller.create(nil, nil, data)
		result.first == 201 ? result.first : JSON.parse(result.last)
	else
		# Therefore, we must be dealing with a dump.
		statuses = data.map do |key, value|
			begin
				unless value.empty?
					model = ('FReCon::' + key.singularize.capitalize).constantize
					result = model.controller.create(nil, nil, value)
					result.first == 201 ? result.first : JSON.parse(result.last)
				end
			rescue Moped::Errors::OperationFailure => e
				RequestError.new(422, "A model already exists in your database with the key you are trying to import.\nPerhaps you should clear your database?").return_value
			end
		end
		statuses.delete(nil)
		statuses
	end
end