Module: OmniScrapper::FileUtils

Defined in:
lib/omni_scrapper/file_utils.rb

Constant Summary collapse

BASE_NAME = 
''
DIRS = 
%w( schemas crawlers scrappers normalizers )

Class Method Summary collapse

Class Method Details

.generate_scrapper(name) ⇒ Object 



14
15
16
17
18
19
20
21
22
23
 
# File 'lib/omni_scrapper/file_utils.rb', line 14

def generate_scrapper(name)
  Dir.mkdir("#{installation_location}/scrappers/#{name}")
  File.open("#{installation_location}/scrappers/#{name}/scrapper.rb", "w+") do |f|
    f.write(scrapper_template(name))
  end

  File.open("#{installation_location}/scrappers/#{name}/scrap_methods.rb", "w+") do |f|
    f.write(scrap_methods_template(name))
  end
end

.hanami? ⇒ Boolean

Returns:

  • (Boolean) 


43
44
45
 
# File 'lib/omni_scrapper/file_utils.rb', line 43

def hanami?
  defined? Hanami
end

.install ⇒ Object 



7
8
9
10
11
12
 
# File 'lib/omni_scrapper/file_utils.rb', line 7

def install
  Dir.mkdir(installation_location)
  DIRS.each do |dir|
    Dir.mkdir("#{installation_location}/#{dir}")
  end
end

.installation_location ⇒ Object 



33
34
35
36
37
38
39
40
41
 
# File 'lib/omni_scrapper/file_utils.rb', line 33

def installation_location
  if hanami?
    'apps/scrappers'
  elsif rails?
    fail OmniScrapper::UnsupportedFrameworkException, 'Rails is not supported yet.'
  else
    fail OmniScrapper::UnknownFrameworkException
  end
end

.installed? ⇒ Boolean

Returns:

  • (Boolean) 


29
30
31
 
# File 'lib/omni_scrapper/file_utils.rb', line 29

def installed?
  Dir.exists?(installation_location)
end

.rails? ⇒ Boolean

Returns:

  • (Boolean) 


47
48
49
 
# File 'lib/omni_scrapper/file_utils.rb', line 47

def rails?
  defined? Rails
end

.scrap_methods_template(name) ⇒ Object 



79
80
81
82
83
84
 
# File 'lib/omni_scrapper/file_utils.rb', line 79

def scrap_methods_template(name)
  <<-TEMPLATE
module ScrapMethods
end
  TEMPLATE
end

.scrapper_template(name) ⇒ Object 



51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
 
# File 'lib/omni_scrapper/file_utils.rb', line 51

def scrapper_template(name)
  <<-TEMPLATE
require_relative '../gallery'
require_relative '../schema'

# Usage example:
# Scrappers::#{name}::Scrapper.run { |data| p data }
module Scrappers
  module #{name}
    class Scrapper < ::Scrappers::Gallery
include OmniScrapper

setup do |config|
  config.schema ::Scrappers::Schema
  config.crawler ::Scrappers::Gallery

  config.entrypoint ''
  config.next_page_link ''

  config.field :name,
    selector: ''
end
    end
  end
end
  TEMPLATE
end

.userspace_files ⇒ Object 



25
26
27
 
# File 'lib/omni_scrapper/file_utils.rb', line 25

def userspace_files
  Dir.glob(File.join(installation_location, "**", "*.rb"))
end