Class: EchoLink::Scraper

Inherits:
Object
  • Object
show all
Defined in:
lib/echo_link/scraper.rb

Instance Method Summary collapse

Instance Method Details



8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
 
# File 'lib/echo_link/scraper.rb', line 8

def links
  links = []

  first_page_url = "#{EchoLink::URLS[:all_links]}&offset=0"
  html = html(first_page_url)
  links += links_from_table(html.css('table')[5])

  p = html.css('table').first.css('tr')[1].css('td').first.css('p').first
  total_links = p.css('b')[2].text.to_i
  per_page = 100
  page_count = (total_links / per_page.to_f).ceil

  (2..page_count).each do |num|
    offset = per_page * (num - 1)
    page_url = "#{EchoLink::URLS[:all_links]}&offset=#{offset}"

    html = html(page_url)
    links += links_from_table(html.css('table')[5])

    sleep 1
  end
  links
end

#links_from_table(table) ⇒ Object 



32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
 
# File 'lib/echo_link/scraper.rb', line 32

def links_from_table(table)
  links = []
  table.css('tr').each_with_index do |tr, i|
    next if i == 0
    tds = tr.css('td')

    href = tds[0].css('b a').first.attr('href')
    latlong = /^javascript:windowOpener\('googleMap\.jsp.+\|(?<latitude>.+)\|(?<longitude>.+)'\);$/
    (latitude, longitude) = href.match(latlong)[1, 2]

    links << {
      id: tds[2].text.to_i,
      call_sign: tds[0].text.strip,
      description: tds[1].text.chars.select{|i| i.valid_encoding?}.join,
      latitude: latitude.to_f,
      longitude: longitude.to_f,
      grid: tds[4].text,
      frequency: tds[5].text,
      tone_squelch: tds[6].text,
      power: tds[7].text,
      haat: tds[8].text,
      antenna: tds[9].text,
      status: clean_status(tds[10].text),
      comment: tds[11].text,
      status_updated_at: tds[12].text
    }
  end
  links
end