Class: BitReaper

Inherits:
Object
  • Object
show all
Defined in:
lib/bitreaper.rb

Overview

MAIN CLASS

Class Method Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(url, parser, i = 0) ⇒ BitReaper

Returns a new instance of BitReaper.



36
37
38
39
40
41
42
43
44
 
# File 'lib/bitreaper.rb', line 36

def initialize(url,parser,i=0)
	@url = url
	@parser = (parser.is_a? String) ? self.getParser(parser) : parser

	@index = i
	@store = {}

	@noko = self.download(@url)
end

Class Method Details

.getParser(file) ⇒ Object 



46
47
48
49
50
51
52
53
54
55
 
# File 'lib/bitreaper.rb', line 46

def self.getParser(file)
	parserFile = File.read(file)
	parserFile = parserFile.gsub(/([\w]+)\!\s/,'\1=on')
	if true
		puts parserFile.split("\n").map{|l| "   "+l}.join("\n").light_black
		puts ""
	end

	return SDL4R::read(parserFile)
end

Instance Method Details

#download(url, withProgress = true) ⇒ Object 



57
58
59
60
61
 
# File 'lib/bitreaper.rb', line 57

def download(url,withProgress=true)
	printProgress(@url,@index,0) if withProgress

	return Nokogiri::HTML(open(url))
end

#process ⇒ Object 



168
169
170
171
172
173
174
 
# File 'lib/bitreaper.rb', line 168

def process
	printProgress(@url,@index,1)
	processNode(@noko, @parser, @store)

	printProgress(@url,@index,2)
	return @store
end

#processArrayValue(attrb, val, param) ⇒ Object 



87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
 
# File 'lib/bitreaper.rb', line 87

def processArrayValue(attrb,val,param)
	case attrb
		when "join"
			val = val.join(param)
		when "first"
			val = val.first
		when "last"
			val = val.last
		when "index"
			val = val[param.to_i]
		when "select.include"
			if param.start_with? "/"
				val = val.select{|r| r=~Regexp.new(param.tr('/', '')) }
			else
				val = val.select{|r| r.include? param }
			end
		when "select.match"
			if param.start_with? "/"
				val = val.select{|r| r=~Regexp.new("\A#{param.tr('/','')}\Z") }
			else
				val = val.select{|r| r==param }
			end
	end
	return val
end

#processNode(noko, node, store, level = 0) ⇒ Object 



141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
 
# File 'lib/bitreaper.rb', line 141

def processNode(noko,node,store,level=0)
	node.children.each{|child|
		command = child.namespace
		tag = child.name
		pattern = child.values[0]
		attrs = child.attributes

		if child.children.count==0
			# no children, so it's a "get"
			values = noko.search(pattern)

			if values.count>0
				store[tag] = self.processValues(values, attrs)
			end
		else
			# it's a "section"
			store[tag] = {}
			if pattern.nil?
				subnoko = noko
			else
				subnoko = noko.search(pattern)
			end
			processNode(subnoko,child,store[tag],level+1)
		end
	}
end

#processStringValue(attrb, val, param) ⇒ Object 



63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
 
# File 'lib/bitreaper.rb', line 63

def processStringValue(attrb,val,param)
	case attrb
		when "prepend"
			val = param + val
		when "append"
			val = val + param
		when "capitalize"
			val = val.capitalize
		when "uppercase"
			val = val.upcase
		when "lowercase"
			val = val.downcase
		when "trim"
			val = val.strip
		when "replace"
			val = val.gsub(param[0], param[1])
		when "remove"
			val = val.gsub(param,"")
		when "split"
			val = val.split(param)
	end
	return val
end

#processValues(values, attrbs) ⇒ Object 



113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
 
# File 'lib/bitreaper.rb', line 113

def processValues(values,attrbs)
	# check if we have a single value or an array of values
	ret = (values.count==1) ? values[0].content
							: values.map{|v| v.content}

	# no attributes, just return it
	return ret if attrbs.size==0

	attrbs.each{|attrb,arg|
		if arg.is_a? String
			# get params if we have multiple params; or not
			param = (arg.include? "||") ? (arg.split("||").map{|a| Liquid::Template.parse(a).render(@store) }) 
										: Liquid::Template.parse(arg).render(@store)
		end

		if ret.is_a? String
			# if our value is a String, process it accordingly
			ret = self.processStringValue(attrb,ret,param)
		else
			# it's an array of values, so look for array-operating attributes
			ret = self.processArrayValue(attrb,ret,param)
			
		end
	}

	return (ret.nil?) ? "" : ret
end