Class: Scraper

Inherits:
UI
  • Object
show all
Defined in:
lib/ruby_doc/data/scraper.rb

Instance Attribute Summary

Attributes inherited from UI

#counter

Class Method Summary collapse

Methods inherited from UI

browse_control, browse_error, browse_list, browse_list_error, browse_menu, clear, cyanH, display_class, display_class_control, display_class_error, display_class_menu, display_method, display_method_control, display_method_error, display_method_menu, favorites_error, favorites_list, favorites_list_control, last_page_menu, learn_more, list_control, list_error, list_menu, loading_animation, loading_message, main_control, main_error, main_menu, method_list, my_gets, nil_error, paginate, prompt, randQ, rdo_prefix, redH, reset_favs_message, search_error, search_list, sepB, sepL, sepR, signature, view_full, wrapped

Class Method Details

.changelog ⇒ Object 



121
122
123
124
125
 
# File 'lib/ruby_doc/data/scraper.rb', line 121

def self.changelog 
  html = Nokogiri::HTML(open("https://github.com/AlphaDaniel/ruby_doc/blob/master/changelog.md"))  
  
  html.search("#readme").text.gsub("\n    ", "").gsub("\n\n\n  ", "") + "\n\n"
end

.class_method_uniq(doc, method) ⇒ Object 



107
108
109
 
# File 'lib/ruby_doc/data/scraper.rb', line 107

def self.class_method_uniq(doc, method)
  doc.methods.none?{|m| m == method }
end

.class_uniq(url) ⇒ Object

HELPERS====================================


95
96
97
 
# File 'lib/ruby_doc/data/scraper.rb', line 95

def self.class_uniq(url) 
  Klass.all.none?{|klass| klass.url == url}
end

.coming_soon ⇒ Object

LEARN MORE===================================


111
112
113
114
115
116
117
118
119
 
# File 'lib/ruby_doc/data/scraper.rb', line 111

def self.coming_soon 
  html = Nokogiri::HTML(open("https://github.com/AlphaDaniel/ruby_doc/blob/master/README.md")) 
  
  list = ""
  html.search("div#readme ul li").each do |li| 
     list << ">> ".cyan + li.text + "\n"
  end
  list
end

.doc_uniq(name) ⇒ Object 



103
104
105
 
# File 'lib/ruby_doc/data/scraper.rb', line 103

def self.doc_uniq(name) 
  $DocDB.none?{|doc| doc.name == name}
end

.load_class_doc(doc) ⇒ Object

LOAD CLASS DOC================================


41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
 
# File 'lib/ruby_doc/data/scraper.rb', line 41

def self.load_class_doc(doc) 
  html = Nokogiri::HTML(open(doc.url))
#--------------------------documentation--------------------------------- 
  container = html.search("#description")
  
  short = container.search("p")[0].text + "\n"
  
  text = "" 
  container.search("p, pre, h2").each {|p| text << p.text + "\n\n"} 
  
  # assign 
  doc.short = short
  doc.documentation = text
#-----------------------------methods------------------------------------ 
  methods = html.search("ul.link-list a")
  
  methods.each do |m| 
    url = doc.url + m["href"] 
    method = Meth.find_by(url) 
    
    doc.methods << method if class_method_uniq(doc, method)
  end
end

.load_classes ⇒ Object

LOAD CLASSES=================================


3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
 
# File 'lib/ruby_doc/data/scraper.rb', line 3

def self.load_classes 
  @counter = 0 #For Loading anim
  loading_message#
  
  html = Nokogiri::HTML(open("https://ruby-doc.org/core-2.4.3/"))
  icontainer = html.search("#class-index .entries")
  
  icontainer.search("p").each do |li| 
    name = li.search("a").text
    url = UI.rdo_prefix + li.search("a")[0]["href"]
    type = li["class"].capitalize
    
    # assigns - Klass :names, :urls
    doc = Klass.new(type, name, url) if class_uniq(url)
    # keeps copy in DocDB
    $DocDB << doc if doc_uniq(url)
  end
end

.load_method_doc(doc) ⇒ Object

LOAD METHOD DOC================================


65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
 
# File 'lib/ruby_doc/data/scraper.rb', line 65

def self.load_method_doc(doc) 
  html = Nokogiri::HTML(open(doc.url))
#--------------------------documentation--------------------------------- 
  selector = "#"+doc.url.gsub(/.+#method.{3}/, "")+"-method"
  
  # if method is alias (linked to another method)
  if html.search("#{selector}").first["class"].include?("method-alias")
    
    conn = html.search("#{selector}").first.search("a")[1]["href"] 
    rebuild = "#"+ conn.gsub(/.+#method.{3}/, "")+"-method" 
    
    container = html.search(rebuild)[0] 
  
    text = "" 
    text << html.search("#{selector} div.aliases").first.text + "\n\n" 
    container.search("p, pre, h2").each {|p| text << p.text + "\n\n" }  
    
    # assign 
    doc.documentation = text
  else
    container = html.search(selector)[0]
  
    text = "" 
    container.search("p, pre, h2").each {|p| text << p.text + "\n\n" } 
    
    # assign 
    doc.documentation = text
  end 
end

.load_methods ⇒ Object

LOAD METHODS=================================


22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
 
# File 'lib/ruby_doc/data/scraper.rb', line 22

def self.load_methods 
  html = Nokogiri::HTML(open("https://ruby-doc.org/core-2.4.3/"))
  icontainer = html.search("#method-index .entries")
  
  icontainer.search("a").each do |li|
    name = li.text
    url = UI.rdo_prefix + li["href"]
    type = "Method"
    
    # assigns - Method :names, :urls
    doc = Meth.new("Method", name, url) if method_uniq(name) 
    # keeps copy in DocDB
    $DocDB << doc if doc_uniq(name)
    
    @counter += 1 #For Loading anim
    loading_animation#
  end
end

.method_uniq(name) ⇒ Object 



99
100
101
 
# File 'lib/ruby_doc/data/scraper.rb', line 99

def self.method_uniq(name)  
  Meth.all.none?{|method| method.name == name}
end