Class: GamesAndRpgParadise::DSA::ObtainInformationForDsaAdventureBooks
- Inherits:
-
Base
- Object
- Base
- GamesAndRpgParadise::DSA::ObtainInformationForDsaAdventureBooks
show all
- Defined in:
- lib/games_and_rpg_paradise/rpg/dsa/obtain_information_for_dsa_adventure_books.rb
Overview
Constant Summary
collapse
- NAMESPACE =
inspect
- DOWNLOAD_N_PAGES =
100
- REMOTE_URL =
'https://ulisses-spiele.de/produkte/'
- START_POINT =
'802'
- END_POINT =
'700'
- STORE_INTO_THIS_DIRECTORY =
#
STORE_INTO_THIS_DIRECTORY
#
'/home/x/Temp/dsa/'
- SHALL_WE_DOWNLOAD_THE_HTML_FILES =
#
SHALL_WE_DOWNLOAD_THE_HTML_FILES
#
false
- HASH_MONTH_NAMES =
{
'1' => 'Jan',
'2' => 'Feb',
'3' => 'Mar',
'4' => 'Apr',
'5' => 'May',
'6' => 'Jun',
'7' => 'Jul',
'8' => 'Aug',
'9' => 'Sep',
'10' => 'Oct',
'11' => 'Nov',
'12' => 'Dec'
}
Base::Extensions::Colours::ARRAY_AVAILABLE_KONSOLE_COLOURS
CommonExtensions::CONTROL_C_CODE, CommonExtensions::N
Class Method Summary
collapse
Instance Method Summary
collapse
ecomment, #efancy, #eparse, #forestgreen, #gold, #grey, #lightblue, #mediumseagreen, #mediumslateblue, #peru, #rev, sdir, sfancy, #sfile, simp, #teal, #yellow
#commandline_arguments?, #filter_away_commandline_arguments, #first_argument?, #first_non_hyphened_argument?, #set_commandline_arguments
#cat, #cd, #cliner, #copy_file, #delete, #dirname_but_retains_the_trailing_slash, #disable_colours, #ensure_that_the_log_directory_exists, #esystem, #get_user_input, #infer_the_namespace, #is_on_roebe?, #log_dir?, #mkdir, #mkdir_then_cd_into_it, #mv, #namespace?, #opne, #opnn, #project_base_directory?, #project_image_directory?, #project_yaml_directory?, #rds, #register_sigint, #remove_this_directory, #rename_file, #reset_the_internal_hash, #return_pwd, #return_today, #touch_file, #wrap, #write_what_into
Constructor Details
88
89
90
91
92
93
94
95
|
# File 'lib/games_and_rpg_paradise/rpg/dsa/obtain_information_for_dsa_adventure_books.rb', line 88
def initialize(
i = ARGV,
run_already = true
)
reset
set_commandline_arguments(i)
run if run_already
end
|
Class Method Details
.[](i = '') ⇒ Object
248
249
250
|
# File 'lib/games_and_rpg_paradise/rpg/dsa/obtain_information_for_dsa_adventure_books.rb', line 248
def self.[](i = '')
new(i)
end
|
Instance Method Details
#download_the_remote_dataset_into_local_html_files(start_point = START_POINT.to_i, end_point = END_POINT.to_i) ⇒ Object
#
download_the_remote_dataset_into_local_html_files
First, we will batch-download all the remote dataset.
#
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
|
# File 'lib/games_and_rpg_paradise/rpg/dsa/obtain_information_for_dsa_adventure_books.rb', line 160
def download_the_remote_dataset_into_local_html_files(
start_point = START_POINT.to_i,
end_point = END_POINT.to_i
)
begin
require 'cyberweb/web_scraper/scrape_this_url.rb'
rescue LoadError; end
start_point.downto(end_point) {|counter|
full_remote_url = REMOTE_URL+counter.to_s
e royalblue(full_remote_url)
Cyberweb::ScrapeThisUrl.set_start_urls(full_remote_url)
store_here = base_dir?+counter.to_s+'.html'
opnn; e "Will store into the file `#{sfile(store_here)}`."
Cyberweb::ScrapeThisUrl.set_store_into_this_local_file(store_here)
Cyberweb::ScrapeThisUrl.crawl!
}
e 'All downloaded .html files should be available in: '+
sdir(base_dir?)
end
|
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
# File 'lib/games_and_rpg_paradise/rpg/dsa/obtain_information_for_dsa_adventure_books.rb', line 136
def (i)
if i.is_a? Array
i.each {|entry| (entry) }
else
case i
when /^-?-?download$/
@download_the_html_files = true
end
end
end
|
#obtain_the_relevant_dataset_from_the_local_html_files ⇒ Object
#
obtain_the_relevant_dataset_from_the_local_html_files
#
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
|
# File 'lib/games_and_rpg_paradise/rpg/dsa/obtain_information_for_dsa_adventure_books.rb', line 200
def obtain_the_relevant_dataset_from_the_local_html_files
opnn; e 'Next trying to obtain the relevant dataset from '+
sdir(store_into_this_directory)
require 'cyberweb/toplevel_methods/umlaute.rb'
regex_for_title_of_the_book =
/<h2 class="productTitel">(.+)<\/h2>/
regex_for_release_date = /<span class="news-element-release-month cf1" style="font-size:2.2em;font-weight:bold;">([A-Za-z]{1,3})<\/span><span class="news-element-release-year">(\d{4})<\/span>/
work_on_these_files = Dir["#{store_into_this_directory}*.html"].sort.reverse work_on_these_files.each {|this_html_file|
dataset = File.read(this_html_file)
dataset =~ regex_for_title_of_the_book
title = $1.to_s.dup.ljust(34)
title = Cyberweb.convert_back_german_umlauts(title)
title.gsub!(/&/,'&')
dataset =~ regex_for_release_date
month = $1.to_s.dup
month = HASH_MONTH_NAMES[month.to_i.to_s]
year = $2.to_s.dup
release_date = ' '+(month.to_s+' '+year.to_s).ljust(14)
remote_url = REMOTE_URL+File.basename(this_html_file).sub(/\.html$/,'')
result = title+' | '+
' --- | '+
release_date+' | '+
remote_url
e result }
end
|
#reset ⇒ Object
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
# File 'lib/games_and_rpg_paradise/rpg/dsa/obtain_information_for_dsa_adventure_books.rb', line 100
def reset
@namespace = NAMESPACE
@counter = start_point?.to_i
@download_the_html_files = SHALL_WE_DOWNLOAD_THE_HTML_FILES
end
|
#run ⇒ Object
183
184
185
186
187
188
189
190
191
192
193
194
195
|
# File 'lib/games_and_rpg_paradise/rpg/dsa/obtain_information_for_dsa_adventure_books.rb', line 183
def run
mkdir(store_into_this_directory) unless File.directory? store_into_this_directory
(commandline_arguments?)
if @download_the_html_files
download_the_remote_dataset_into_local_html_files
end
cd store_into_this_directory
obtain_the_relevant_dataset_from_the_local_html_files
end
|
#start_point? ⇒ Boolean
121
122
123
|
# File 'lib/games_and_rpg_paradise/rpg/dsa/obtain_information_for_dsa_adventure_books.rb', line 121
def start_point?
START_POINT
end
|
#store_into_this_directory ⇒ Object
Also known as:
base_dir?, base_dir
#
store_into_this_directory
#
128
129
130
|
# File 'lib/games_and_rpg_paradise/rpg/dsa/obtain_information_for_dsa_adventure_books.rb', line 128
def store_into_this_directory
STORE_INTO_THIS_DIRECTORY
end
|