Module: Scry

Extended by:
Helpers
Defined in:
lib/scry/export_failed.rb,
lib/scry.rb,
lib/scry/tasks.rb,
lib/scry/course.rb,
lib/scry/helpers.rb,
lib/scry/scraper.rb,
lib/scry/version.rb,
lib/scry/sidekiq/workers/log_writer.rb,
lib/scry/sidekiq/workers/export_generator.rb,
lib/scry/sidekiq/workers/export_downloader.rb

Overview

Works on generating the export.

Will attempt 5 times before giving up.

Defined Under Namespace

Modules: Helpers Classes: Course, ExportDownloader, ExportFailed, ExportGenerator, LogWriter, Tasks

Constant Summary collapse

VERSION = 
"1.0.0".freeze

Class Method Summary collapse

Methods included from Helpers

click_link, write_log

Class Method Details

.config ⇒ Object 



13
14
15
16
17
18
19
 
# File 'lib/scry.rb', line 13

def self.config
  if File.exists? "scry.yml"
    YAML::load(File.read("scry.yml"))
  else
    {}
  end
end

.courses_downloaded ⇒ Object 



55
56
57
58
59
60
61
 
# File 'lib/scry/scraper.rb', line 55

def self.courses_downloaded
  if File.exists?(Scry.export_download_good)
    File.read(Scry.export_download_good)
  else
    ""
  end
end

.default_dir ⇒ Object 



33
34
35
 
# File 'lib/scry.rb', line 33

def self.default_dir
  Scry.config[:default_dir] || DEFAULT_DIR
end

.export_download_bad ⇒ Object 



49
50
51
 
# File 'lib/scry.rb', line 49

def self.export_download_bad
  Scry.config[:export_download_bad] || EXPORT_DOWNLOAD_BAD
end

.export_download_good ⇒ Object 



45
46
47
 
# File 'lib/scry.rb', line 45

def self.export_download_good
  Scry.config[:export_download_good] || EXPORT_DOWNLOAD_GOOD
end

.export_generation_bad ⇒ Object 



41
42
43
 
# File 'lib/scry.rb', line 41

def self.export_generation_bad
  Scry.config[:export_generation_bad] || EXPORT_GENERATION_BAD
end

.export_generation_good ⇒ Object 



37
38
39
 
# File 'lib/scry.rb', line 37

def self.export_generation_good
  Scry.config[:export_generation_good] || EXPORT_GENERATION_GOOD
end

.export_generation_no_export_button ⇒ Object 



53
54
55
56
 
# File 'lib/scry.rb', line 53

def self.export_generation_no_export_button
  Scry.config[:export_generation_no_export_button] ||
    EXPORT_GENERATION_NO_EXPORT_BUTTON
end

.login ⇒ Object 



25
26
27
 
# File 'lib/scry.rb', line 25

def self.
  Scry.config[:login]
end

.passwd ⇒ Object 



29
30
31
 
# File 'lib/scry.rb', line 29

def self.passwd
  Scry.config[:passwd]
end

.scrape ⇒ Object

Creates sidekiq jobs for each course to generate an export.

Logs in the user and goes over every course and creates a sidekiq to generate an export for it.



17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
 
# File 'lib/scry/scraper.rb', line 17

def self.scrape
  url = Scry.url
   = Scry.
  passwd = Scry.passwd

  agent = Mechanize.new do |secret_agent_man|
    secret_agent_man.follow_meta_refresh = true
  end

  agent.get(url) do |home_page|
    index_page = home_page.form_with(name: "login") do |form|
      form["user_id"] = 
      form["password"] = passwd
    end.submit
    courses_page = click_link(
      agent: agent,
      page: index_page,
      text: /Open Bb Course List/,
    )

    courses_downloaded = Scry.courses_downloaded

    # It is currently unknown if the links on the courses_page are paginated
    # This gets them as if they are not.
    course_links = courses_page.links_with(href: /type=Course/)
    course_links.each do |course_link|
      course_url = course_link.href.strip
      if !courses_downloaded.include? course_url
        cookie_crumbs = agent.cookie_jar.to_yaml
        Scry::ExportGenerator.perform_async(
          cookie_crumbs,
          File.join(url, course_url),
        )
      end
    end
  end
end

.url ⇒ Object 



21
22
23
 
# File 'lib/scry.rb', line 21

def self.url
  Scry.config[:url]
end