Module: TildeScraper

Defined in:
lib/tilde_scraper.rb,
lib/tilde_scraper/api.rb

Defined Under Namespace

Modules: Memorable Classes: CommandLineInterface, Comment, Group, LinkTopic, Page, Scraper, TextTopic, Topic

Constant Summary collapse

@@page_id = 
0

Class Method Summary collapse

Class Method Details

.get_comments(url) ⇒ Object

Scrapes a topic page and returns an array of comment objects



40
41
42
43
 
# File 'lib/tilde_scraper/api.rb', line 40

def self.get_comments(url)
    comment_array = TildeScraper::Scraper.scrape_comments(url)
    TildeScraper::Comment.create_from_array(comment_array)
end

.get_groups ⇒ Object

Scrapes the group page for first level groups and returns an array of group objects



34
35
36
37
 
# File 'lib/tilde_scraper/api.rb', line 34

def self.get_groups
  TildeScraper::Group.all.clear
  TildeScraper::Group.create_from_array(TildeScraper::Scraper.scrape_groups("https://tildes.net/groups"))
end

.get_page(url) ⇒ Object

Scrapes a page at url, creates topic objects for each topic, and returns a page object



4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
 
# File 'lib/tilde_scraper/api.rb', line 4

def self.get_page(url)
    data = TildeScraper::Scraper.scrape_page(url)
    #Set page_id in page data hash
    data[0][:page_id] = @@page_id
    #Create page object
    page = TildeScraper::Page.create(data[0])
    #Set page_id in all topic data hashes
    #Set group in all hashes if applicible
    data[1].each do |topic_hash|
      if page.group
        topic_hash[:group] = page.group
      end
      topic_hash[:page_id] = @@page_id
    end
    @@page_id += 1;
    #Create topic objects
    TildeScraper::Topic.create_from_array(data[1])
    page
end

.get_page_with_comments(url) ⇒ Object

Scrapes a page for topics, and scrapes each topic’s comments, returns a page object



25
26
27
28
29
30
31
 
# File 'lib/tilde_scraper/api.rb', line 25

def self.get_page_with_comments(url)
  page = get_page(url)
  #Create comments for each topic
  page.topics.each do |topic|
    get_comments(topic.comment_link)
  end
end