Class: GamesAndRpgParadise::MagicTheGathering::DownloadCardsFromGathererWizardsCom

Inherits:
Base
  • Object
show all
Defined in:
lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb

Overview

GamesAndRpgParadise::MagicTheGathering::DownloadCardsFromGathererWizardsCom

Constant Summary collapse

NAMESPACE =
#

NAMESPACE

#
inspect

Constants inherited from Base

Base::LOG_DIR

Constants included from Base::Extensions::Colours

Base::Extensions::Colours::ARRAY_AVAILABLE_KONSOLE_COLOURS

Constants included from CommonExtensions

CommonExtensions::CONTROL_C_CODE, CommonExtensions::N

Instance Method Summary collapse

Methods inherited from Base

[], #convert, #ensure_that_the_log_directory_exists, #esystem, #is_png_file?, #lightgreen, #log_dir?, #orange, #palegoldenrod, #pngopt, #rev, #slateblue, #springgreen, #turquoise, #write_what_into

Methods included from Base::Extensions::Colours

ecomment, #efancy, #eparse, #forestgreen, #gold, #grey, #lightblue, #mediumseagreen, #mediumslateblue, #peru, #rev, sdir, sfancy, #sfile, simp, #teal, #yellow

Methods included from Base::Extensions::CommandlineArguments

#commandline_arguments?, #filter_away_commandline_arguments, #first_argument?, #first_non_hyphened_argument?, #set_commandline_arguments

Methods included from CommonExtensions

#cat, #cd, #cliner, #copy_file, #delete, #dirname_but_retains_the_trailing_slash, #disable_colours, #ensure_that_the_log_directory_exists, #esystem, #get_user_input, #infer_the_namespace, #is_on_roebe?, #log_dir?, #mkdir, #mkdir_then_cd_into_it, #mv, #namespace?, #opne, #project_base_directory?, #project_image_directory?, #project_yaml_directory?, #rds, #register_sigint, #remove_this_directory, #rename_file, #reset_the_internal_hash, #return_pwd, #return_today, #touch_file, #wrap, #write_what_into

Constructor Details

#initialize(name_of_the_card_set = :default_card_set, run_already = true) ⇒ DownloadCardsFromGathererWizardsCom

#

initialize

#


32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 32

def initialize(
    name_of_the_card_set = :default_card_set,
    run_already          = true
  )
  reset
  # ======================================================================= #
  # === Handle Hash given as input first
  # ======================================================================= #
  if name_of_the_card_set.is_a? Hash
    if name_of_the_card_set.has_key? :n_pages
      n_pages = name_of_the_card_set.delete(:n_pages) # Grab n pages here.
      # =================================================================== #
      # === :name_of_the_set
      # =================================================================== #
      if name_of_the_card_set.has_key? :name_of_the_set
        name_of_the_card_set = name_of_the_card_set.delete(:name_of_the_set).to_s
      end
      name_of_the_card_set = DEFAULT_SET_NAME if name_of_the_card_set.is_a? Hash
      set_name_of_the_card_set(name_of_the_card_set)
      do_pre_run_actions # Must come after set_name_of_the_card_set(). 
      0.upto(n_pages.to_i - 1).each {|link_page|
        download_from_the_official_site(
          link_page
        )
      }
      do_finalizing_actions
    end
  else
    set_name_of_the_card_set(name_of_the_card_set)
    do_pre_run_actions
    run if run_already
  end
end

Instance Method Details

#create_html_page(name_of_the_card_set = name_of_the_card_set?) ) ⇒ Object

#

create_html_page

#


318
319
320
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 318

def create_html_page(name_of_the_card_set = name_of_the_card_set?)
  GamesAndRpgParadise::MagicTheGathering.create_html_page(name_of_the_card_set)
end

#create_subdirectory ⇒ Object

#

create_subdirectory

#


401
402
403
404
405
406
407
408
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 401

def create_subdirectory
  set_sanitized_name_of_the_set
  # ======================================================================= #
  # Since as of Oct 2020 we disallow directories containing ':'; windows
  # may complain otherwise.
  # ======================================================================= #
  mkdir @sanitized_name_of_the_set.delete(':')
end

#do_finalizing_actions ⇒ Object

#

do_finalizing_actions

This is the “post-hook” stage, sort of.

#


334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 334

def do_finalizing_actions
  # ======================================================================= #
  # Decide whether we will create a .zip file or whether we will not.
  # ======================================================================= #
  if @create_a_zip_archive
    # ======================================================================= #
    # And we can next create a .zip file from this.
    # ======================================================================= #
    cd ::GamesAndRpgParadise::MagicTheGathering.log_dir? # Go back to the start directory.
    e 'Next creating a .zip file from this directory.'
    name_of_the_directory = name_of_the_set.tr(' ','_')
    subdirectory = rds(name_of_the_directory+'/**')
    name_of_the_zip_file = name_of_the_directory+'.zip'
    _ = "zip #{name_of_the_zip_file} #{subdirectory}"
    ::GamesAndRpgParadise.esystem(_)
    # ======================================================================= #
    # And now we can upload this file to my FTP site.
    # ======================================================================= #
    FtpParadise.upload_this_binary_file(name_of_the_zip_file)
  end
  # ======================================================================= #
  # Report where the index.html file resides.
  # ======================================================================= #
  opnn; e 'The index.html file can be found at:'
  e
  e "  → #{sfile(sanitized_name_of_the_set?.delete(':')+'index.html')}"
  e
  # ======================================================================= #
  # Next, sort that big fat .md file.
  # ======================================================================= #
  ::GamesAndRpgParadise::MagicTheGathering::SortCards.new(card_set?)
  # ======================================================================= #
  # We also want to generate a cute little .html page - the next code
  # block ensures this. We only need the directory for this.
  # ======================================================================= #
  create_html_page
  # ======================================================================= #
  # Also silently generate the main html file next.
  # ======================================================================= #
  generate_main_html_file
end

#do_merge_all_individual_md_files_into_one_big_yaml_file ⇒ Object

#

do_merge_all_individual_md_files_into_one_big_yaml_file

#


285
286
287
288
289
290
291
292
293
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 285

def do_merge_all_individual_md_files_into_one_big_yaml_file
  # ======================================================================= #
  # And register all these new files too - but this time,
  # we will report to the user.
  # ======================================================================= #
  GamesAndRpgParadise::MagicTheGathering.merge_all_individual_md_files_into_one_big_yaml_file(
    :be_verbose
  )
end

#do_pre_run_actions ⇒ Object

#

do_pre_run_actions

#


298
299
300
301
302
303
304
305
306
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 298

def do_pre_run_actions
  cd LOG_DIR # Start at the base directory.
  # ======================================================================= #
  # Now we are in the base directory. We will first create the necessary
  # subdirectories.
  # ======================================================================= #
  create_subdirectory
  verbose_enter_subdirectory # Now we are in the subdirectory.
end

#download_these_matches(i) ⇒ Object

#

download_these_matches

The input to this method should be an Array that looks like this:

[["417656", "Fretwork Colony"],
 ["417588", "Fumigate"]
#


141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 141

def download_these_matches(i)
  i.each_with_index {|array, index| # This was the original Array with matches.
    the_id, name_of_the_card = *array
    # ======================================================================= #
    # Chop off "(" and ")" entries. Otherwise we may have stupid names
    # such as "Aether Tradewinds (Æther Tradewinds)".
    # ======================================================================= #
    if name_of_the_card.include?('(') and
       name_of_the_card.include?(')')
      name_of_the_card = name_of_the_card[0 .. (name_of_the_card.index('('))-1]
      name_of_the_card.rstrip!
    end
    name_of_the_card = name_of_the_card.encode(USE_THIS_ENCODING)
    index += 1
    # ======================================================================= #
    # First, download the image anyway.
    # ======================================================================= #
    remote_url = GATHERER_WIZARDS_COM+'Handlers/Image.ashx?multiverseid='+the_id+'&type=card'
    if File.basename(name_of_the_card).include? '/' # If the filename itself contains at least one '/'.
      name_of_the_card.gsub!(/\//,'_') # Boom // Bust is a card exception: http://www.mtgprice.com/sets/Planar_Chaos/Boom__Bust
    end
    if name_of_the_card.include? '//' # Must correct these cards.
      name_of_the_card.gsub!(/\/\//, '_')
    end
    modified_card_name = name_of_the_card.tr(' ','_')
    if modified_card_name.include? '__'
      modified_card_name.squeeze!('_')
    end
    name_of_the_image_file = modified_card_name+'.jpg'
    name_of_the_image_file = name_of_the_image_file.encode(USE_THIS_ENCODING)
    # ======================================================================= #
    # Binwrite the dataset here in order to "create" the image file.
    # ======================================================================= #
    unless File.exist? name_of_the_image_file
      File.binwrite(name_of_the_image_file, URI.open(remote_url).read)
      # ======================================================================= #
      # Try to optimize the .jpg file in question, to chop off a few bytes.
      # Oddly enough, the image type may be either .png or .jpg.
      # For now, we consistently convert into .jpg format.
      # ======================================================================= #
      if TRY_TO_USE_JPEGOPTIM_TO_OPTIMIZE_JPG_FILES
        # ======================================================================= #
        # Get rid of '(' thingies.
        # ======================================================================= #
        if name_of_the_image_file.include? '(' # Rename odd symbols here.
          new_name = name_of_the_image_file[0 .. (name_of_the_image_file.index('(') - 2)] # -2 because it has a '_' before.
          mv(name_of_the_image_file, new_name)
          name_of_the_image_file = new_name.dup
        end
        # ======================================================================= #
        # We strip all metadata and preserve at 90% quality only.
        # This should suffice really. We could also use --size=20k
        # but that appears to be a bit ... aggressive.
        # --max=90
        # ======================================================================= #
        # First check whether we have a .png file actually.
        result_of_file_query = `file "#{name_of_the_image_file}"`
        if result_of_file_query.include? 'PNG image data'
          e "Note that this file (#{simp(name_of_the_image_file)}) is a "\
            "#{slateblue('.png')} file, not a #{lightgreen('.jpeg')} file."
          # ============================================================= #
          # Ok, here we assume that it is a .png file so we
          # change the suffix, then run pngopt, then convert
          # back into .jpg.
          # ============================================================= #
          name_of_the_png_file = name_of_the_image_file.sub(/\.jpg$/,'.png')
          mv(name_of_the_image_file, name_of_the_png_file)
          name_of_the_png_file = name_of_the_png_file.encode(USE_THIS_ENCODING)
          pngopt(name_of_the_png_file, :be_verbose)
          convert(name_of_the_png_file, name_of_the_image_file)
          if File.exist?(name_of_the_image_file) and
             File.exist?(name_of_the_png_file)
            delete(name_of_the_png_file) unless !File.exist?(name_of_the_image_file)
          end
        end
        # ======================================================================= #
        # At this point, the file should be a .jpg file, so that
        # we can optimize it.
        # ======================================================================= #
        _ = 'jpegoptim --strip-all "'+name_of_the_image_file+'"'.dup # Must have "" quotes because of the shell being flawed.
        _ << ERROR_LINE # To prevent it being too noisy.
        `#{_}` # Deliberately no output to keep things less verbose.
      end
    end
    # ======================================================================= #
    # Second, obtain the points 1 to 4 from the special multiverseid url.
    # ======================================================================= #
    remote_url = GATHERER_WIZARDS_COM+'Pages/Card/Details.aspx?multiverseid='+the_id
    dataset = URI.open(remote_url).read # Ok, we have the remote page now.
    converted_mana_cost_of_the_card = nil # The converted mana cost of the card in question.
    rarity_of_the_card              = nil # Common, Uncommon, Rare.
    card_number                     = nil # This is the specific number in that set.
    regex_for_converted_mana_cost   = REGEX_FOR_THE_CONVERTED_MANA_COST
    regex_for_rarity_of_the_card    = REGEX_FOR_RARITY_OF_A_CARD
    dataset                         =~ regex_for_converted_mana_cost
    converted_mana_cost_of_the_card = $1.to_i
    regex_for_card_number           = REGEX_FOR_THE_CARD_NUMBER
    dataset                         =~ regex_for_card_number
    card_number                     = $2.to_s # .to_s because it may be "35a"; with .to_i we would lose this information
    dataset                         =~ regex_for_rarity_of_the_card
    rarity_of_the_card              = $1.to_s.dup
    name_of_the_set                 = name_of_the_set?
    # ======================================================================= #
    # First, we need to keep track of whether we have dual-cards or not.
    # ======================================================================= #
    # if card_number.end_with?('a') or card_number.end_with?('b')
    # end
    # ======================================================================= #
    # Notify the user on which card we will be working. Note that a few
    # cards are "dual" cards, that is, they have two-cards-in-one.
    # ======================================================================= #
    e '('+orange(index.to_s.rjust(2))+
      ') Now working on `'+
       turquoise(name_of_the_card)+'`, '\
      'card number '+simp(card_number.to_s)+
      '. (Set: '+
       orange(name_of_the_set)+')'
    e '     Full URL was: '+sfancy(remote_url)
    # ======================================================================= #
    # Also add the information into the main @hash.
    # ======================================================================= #
    @hash[name_of_the_card] = [
      converted_mana_cost_of_the_card, # First, the converted mana cost.
      rarity_of_the_card,              # Second, the rarity of the card.
      card_number,                     # Third, the card number. A String.
      name_of_the_set,                 # Fourth, the name of the set in question.
      remote_url                       # Fifth, the remote URL to the card (multiverseid).
    ]
  }
  # ======================================================================= #
  # @hash should now have all the information. Thus we can store
  # it in our custom format, using a specialized class.
  # ======================================================================= #
  GamesAndRpgParadise::MagicTheGathering::StoreCardInformation.new(@hash, name_of_the_set?)
  # Note that the next step may take too long if you already have many
  # ======================================================================= #
  # cards. Thus, it was disabled for now.
  # ======================================================================= #
  # do_merge_all_individual_md_files_into_one_big_yaml_file
end

#download_this_page(this_page = @which_page_number_to_use) ⇒ Object Also known as: download_from_the_official_site

#

download_this_page

The URL should be the page number to use.

#


432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 432

def download_this_page(this_page = @which_page_number_to_use)
  this_page = this_page.to_s
  # ======================================================================= #
  # Build up the main URL next.
  #
  # That URL will look similar to this:
  #
  #   https://gatherer.wizards.com/Pages/Search/Default.aspx?output=standard&page=0&set=["Kaladesh"]
  #
  # ======================================================================= #
  main_remote_url =
    GATHERER_WIZARDS_COM+'Pages/Search/Default.aspx?output=standard&page='+
    this_page+'&set=["'+name_of_the_set?.to_s+'"]' # This variable stays the same.
  remote_url = main_remote_url.dup # This is initially a copy.
  array_with_matches = []
  e "We will use this URL `#{sfancy(remote_url)}`."
  @dataset = URI.open(remote_url).read.to_s
  # ======================================================================= #
  # The following regex will grab the name of the respective card at hand.
  # ======================================================================= #
  regex_to_determine_how_many_cards_are_in_this_set =
    REGEX_TO_DETERMINE_HOW_MANY_CARDS_ARE_IN_THIS_SET
  match_data = regex_to_determine_how_many_cards_are_in_this_set.match(@dataset)
  n_cards_are_in_this_set = match_data[1]
  # ======================================================================= #
  # Ok, determine the amount of pages. Since as of January 2018, this
  # appears to be no longer necessary though.
  # ======================================================================= #
  # regex = /^\s*<.+>(\d)<\/a>/
  # n_pages = @dataset.scan(regex).flatten.size
  # unless (@dataset =~ regex) # Ok, we found no page here.
  #   n_pages = 1
  # end
  # ======================================================================= #
  # Also notify the user how many cards are available in this set.
  # ======================================================================= #
  e "Total amount of cards in this set: "\
    "#{sfancy(n_cards_are_in_this_set.to_s)}"
  use_this_regex_for_finding_a_card = REGEX_FOR_FINDING_A_CARD
  array_with_matches = @dataset.scan(use_this_regex_for_finding_a_card)
  # ======================================================================= #
  # Some card sets have more than one page. The following code will
  # properly handle this. If we have more than one page, the array
  # with our matches will be larger.
  # ======================================================================= #
  # ======================================================================= #
  # Ok, now we have all matches. We will iterate through them next.
  # We also need to generate a .yml file that will have ALL the cards.
  # But we should only store relevant information. We also have to
  # download the image for local use - the image type will be .jpg for
  # all of these files.
  #
  # What information must we store in that hash?
  #
  #     (1) Name of the card
  #     (2) Colour of the card
  #     (3) Rarity: common, uncommon, rare
  #     (4) Card Number: this one will start at 1
  #     (5) Set type: for instance, "Ice Age" and similar.
  #
  # (2) is not so straightforward - no special entry exists for this.
  #
  # So for now, we will just store the converted mana cost instead there.
  # ======================================================================= #
  download_these_matches(array_with_matches)
end

#generate_main_html_file(report_type = :be_verbose) ⇒ Object

#

generate_main_html_file

#


311
312
313
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 311

def generate_main_html_file(report_type = :be_verbose)
  GamesAndRpgParadise::MagicTheGathering.generate_main_html_file(report_type)
end

#name_of_the_card_set? ⇒ Boolean Also known as: name_of_the_set?, card_set?

#

name_of_the_card_set?

#

Returns:

  • (Boolean) 


117
118
119
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 117

def name_of_the_card_set?
  @name_of_the_card_set
end

#opnn ⇒ Object

#

opnn

#


325
326
327
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 325

def opnn
  super(NAMESPACE)
end

#reset ⇒ Object

#

ensure

#


69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 69

def reset
  ensure_that_the_log_directory_exists
  # ======================================================================= #
  # === @create_a_zip_archive
  #
  # This variable determines whether we will create a .zip archive
  # or whether we will not.
  # ======================================================================= #
  @create_a_zip_archive = false
  # ======================================================================= #
  # === @which_page_number_to_use
  # ======================================================================= #
  @which_page_number_to_use = 2
  # ======================================================================= #
  # === @dataset
  # ======================================================================= #
  @dataset = nil
  @hash = {} # This big, fat Hash will keep all entries.
end

#run ⇒ Object

#

run

#


502
503
504
505
506
507
508
509
510
511
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 502

def run
  if @which_page_number_to_use.is_a? Array
    @which_page_number_to_use.each {|entry|
      download_this_page(entry)
    }
  else
    download_this_page(@which_page_number_to_use)
  end
  do_finalizing_actions
end

#sanitized_name_of_the_set? ⇒ Boolean Also known as: sanitized_name_of_the_set

#

sanitized_name_of_the_set?

#

Returns:

  • (Boolean) 


379
380
381
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 379

def sanitized_name_of_the_set?
  @sanitized_name_of_the_set
end

#set_name_of_the_card_set(i = :default_card_set) ⇒ Object

#

set_name_of_the_card_set

The method reset() will invoke this method as well.

#


94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 94

def set_name_of_the_card_set(i = :default_card_set)
  if i.is_a? Array
    if i.empty?
      i = :default_card_set
    else
      i = i.join(' ').strip
    end
  end
  case i
  when :default_card_set, nil
    i = ::MagicTheGathering.default_set?
  end
  i = i.dup if i.frozen?
  # ======================================================================= #
  # Upcase the first character.
  # ======================================================================= #
  i[0,1] = i[0,1].upcase
  @name_of_the_card_set = i
end

#set_sanitized_name_of_the_set(i = log_dir?+ name_of_the_card_set?. tr(' ','_').downcase) ⇒ Object

#

set_sanitized_name_of_the_set

The sanitized name of the set is simply the one that has the ‘ ’ replaced with ‘_’ characters.

#


389
390
391
392
393
394
395
396
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 389

def set_sanitized_name_of_the_set(
    i = log_dir?+
        name_of_the_card_set?.
        tr(' ','_').downcase
  )
  i << '/' unless i.end_with? '/' # Ought to have a trailing '/' token.
  @sanitized_name_of_the_set = i
end

#set_which_page_number_to_use(i) ⇒ Object Also known as: set_page_number

#

set_which_page_number_to_use

Note that the input can also be an Array, in which case we will download all the respective pages.

#


128
129
130
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 128

def set_which_page_number_to_use(i)
  @which_page_number_to_use = i
end

#verbose_enter_subdirectory ⇒ Object

#

verbose_enter_subdirectory

This method can be used to cd into the target subdirectory.

#


415
416
417
418
419
420
421
422
423
424
425
 
# File 'lib/games_and_rpg_paradise/games/magic_the_gathering/classes/download_cards_from_gatherer_wizards_com.rb', line 415

def verbose_enter_subdirectory
  _ = sanitized_name_of_the_set?.delete(':')
  cd _
  # ======================================================================= #
  # Notify the user into which directory we will log.
  # ======================================================================= #
  e "We will log into the directory `#{sdir(_)}`."
  # ======================================================================= #
  # Now we should be in the proper base-directory.
  # ======================================================================= #
end